1869 lines
137 KiB
Python
1869 lines
137 KiB
Python
import ast
|
|
import contextlib
|
|
import dataclasses
|
|
import io
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
import unittest.mock
|
|
import uuid
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from routing_figures import archive, state
|
|
from loop_state.waiting.ask import Ask, Asking, OpenInInbox, OpenOnTerminal
|
|
from routing_figures.calls import Call
|
|
from loop_claude.project import checkout, model_choice, records
|
|
from loop_claude.cycle import chunk_items
|
|
from loop_claude.loop import chunk_start, close
|
|
from loop_state.waiting.answers import ClaudeEnded, ClaudeOpening, send, Typed
|
|
from release_plan.release_standing import DefaultPlan, LastCut
|
|
from loop_effects.effects import Effects, POLL_SECONDS, PYTHON_TIMEOUT
|
|
from claude_code import entries, reply, session
|
|
from failure import Failure
|
|
from loop_claude.verdicts import Finished, Interrupt, QuotedAnswer, TakenIn, Unfinished
|
|
from loop_state import hand_over, paths, running_loops, session_lock
|
|
from loop_claude.cycle import interrupt, judge, prompts, retry, run, states, worker
|
|
from model_routing.ladder import LADDER, Rung
|
|
from claude_code.reply import Limited
|
|
from loop_claude.loop import main, reload, routing_check, this_loop, usage
|
|
from loop_state.waiting.talk import make_way
|
|
from machine_load import memory
|
|
from transcript_archive import outcomes, write
|
|
from transcript_archive.location import project_archive
|
|
from model_routing.route import Route, route_from, WorkerLine
|
|
from loop_claude import routing
|
|
from loop_claude.stop import Stop
|
|
from loop_state.running_loops import Taken
|
|
from records.utc_stamp import utc
|
|
from loop_claude.cycle.judge import verdict_from
|
|
from loop_state.waiting.listing import waiting
|
|
|
|
# How the archive records a worker that runs off the ladder.
|
|
OFF_LADDER = '{"model": "worker-model"}'
|
|
# How it records routes by item 179's L of 2 and E + I of 5, on Routing's rungs.
|
|
SONNET_179 = '{"model": "claude-sonnet-5-5", "effort": "high", "item": 179, "likelihood": 2, "escape_and_impact": 5}'
|
|
OPUS_179_AT_3 = '{"model": "claude-opus-5-5", "effort": "medium", "item": 179, "likelihood": 2, "escape_and_impact": 5, "escalated_at": 3}'
|
|
OPUS_179_AT_6 = '{"model": "claude-opus-5-5", "effort": "medium", "item": 179, "likelihood": 2, "escape_and_impact": 5, "escalated_at": 6}'
|
|
|
|
CWD = pathlib.Path("/c")
|
|
NOW = 1790000000
|
|
# How a line said at NOW opens, with FakeEffects' local_time.
|
|
STAMP = f"<{NOW}>"
|
|
# How git answers in a project's checkout it cannot read, such as one that is gone.
|
|
UNREAD = Failure("not a git repository")
|
|
# The id FakeEffects gives each new ask, and its pid for loop-claude's process.
|
|
ASK_ID, PID = "ask-1", 4242
|
|
# The pid of a loop-inbox that opens claude for an ask.
|
|
INBOX = 4343
|
|
|
|
|
|
def assistant(*parts):
|
|
return {"type": "assistant", "message": {"content": list(parts)}}
|
|
|
|
|
|
def kinds(*found):
|
|
return [entries.entry_from(entry) for entry in found]
|
|
|
|
|
|
def human(content):
|
|
return {"type": "user", "message": {"content": content}}
|
|
|
|
|
|
def asks_and_answers(folder):
|
|
"""folder's files but the session's lock file, which stays once the loop lets go of the lock."""
|
|
return [path for path in sorted(folder.iterdir()) if path.suffix != ".lock"]
|
|
|
|
|
|
def a_loop(effects, **fields):
|
|
"""A loop in /r on effects, with fields in place of its defaults."""
|
|
home = pathlib.Path("/h")
|
|
defaults = {"root": "/r", "session": session.session_named("A proj loop-claude", home / ".claude" / "projects"), "listed": running_loops.Listed(home / "waiting", lambda items: None), "paths": paths.paths_from(home), "env": {}, "launcher": pathlib.Path("/b/loop-claude")}
|
|
return this_loop.Loop(**{**defaults, **fields}, effects=effects)
|
|
|
|
|
|
class FakeEffects:
|
|
"""Effects that record each call and answer from a script, one answer per call."""
|
|
|
|
def __init__(self, claude=(), on_terminal_claude=(), git=(), inputs=(), python=(), streams=(), on_terminal=True, free_memory=()):
|
|
self.calls = []
|
|
self.scripts = {"claude": list(claude), "claude_on_terminal": list(on_terminal_claude), "git": list(git), "input": list(inputs), "python": list(python), "stream_claude": list(streams)}
|
|
# Without a script, every read finds all memory available.
|
|
self.free_memory = list(free_memory)
|
|
self.terminal = on_terminal
|
|
self.worker_environments = []
|
|
self.terminal_environments = []
|
|
self.terminal_folders = []
|
|
|
|
def effects(self):
|
|
return Effects(
|
|
alive=lambda pid: True,
|
|
claude=lambda args, cwd, stdin: self.answer("claude", (args, stdin)),
|
|
claude_on_terminal=self.on_terminal_claude,
|
|
discard_typed=lambda: self.calls.append(("discard_typed", None)),
|
|
execv=self.execv,
|
|
git=lambda *args, cwd=None, timeout=None, guarded=False, errors="strict": self.answer("git", list(args)),
|
|
input=lambda prompt: self.answer("input", prompt),
|
|
input_or=self.input_or,
|
|
local_time=lambda seconds: f"<{seconds}>",
|
|
memory=self.memory,
|
|
new_id=lambda: ASK_ID,
|
|
now=lambda: NOW,
|
|
on_terminal=lambda: self.terminal,
|
|
pid=lambda: PID,
|
|
python=lambda args, cwd: self.answer("python", (args, cwd)),
|
|
sleep=lambda seconds: self.calls.append(("sleep", seconds)),
|
|
stream_claude=self.stream,
|
|
)
|
|
|
|
def memory(self):
|
|
self.calls.append(("memory", None))
|
|
return self.free_memory.pop(0) if self.free_memory else memory.Memory(available=16, total=16)
|
|
|
|
def input_or(self, prompt, other):
|
|
"""Answers from the input script as input does; a callable in the script gets other, the source of loop-inbox's answer."""
|
|
self.calls.append(("input", prompt))
|
|
answer = self.scripts["input"].pop(0)
|
|
if isinstance(answer, BaseException):
|
|
raise answer
|
|
return answer(other) if callable(answer) else answer
|
|
|
|
def on_terminal_claude(self, args, cwd, env):
|
|
self.terminal_environments.append(env)
|
|
self.terminal_folders.append(cwd)
|
|
return self.answer("claude_on_terminal", args)
|
|
|
|
def stream(self, args, cwd, env):
|
|
self.worker_environments.append(env)
|
|
return self.answer("stream_claude", args)
|
|
|
|
def answer(self, seam, what):
|
|
self.calls.append((seam, what))
|
|
answer = self.scripts[seam].pop(0)
|
|
if isinstance(answer, BaseException):
|
|
raise answer
|
|
return answer(what) if callable(answer) else answer
|
|
|
|
def execv(self, path, args):
|
|
self.calls.append(("execv", args))
|
|
raise OSError(2, "No such file or directory")
|
|
|
|
def made(self, seam):
|
|
return [what for called, what in self.calls if called == seam]
|
|
|
|
|
|
def ran(stdout="", code=0, stderr=""):
|
|
return subprocess.CompletedProcess([], code, stdout, stderr)
|
|
|
|
|
|
def judged(verdict, evidence="working", answer="", escalation="", release_ready=False, cost=None):
|
|
reply = {"answer": answer, "escalation": escalation, "evidence": evidence, "finished": verdict == "finished", "interrupt": verdict == "interrupt", "question": "Which one?" if verdict in ("interrupt", "quoted") else "", "answered_before": "", "left_open": "", "release_ready": release_ready}
|
|
return ran(json.dumps({"structured_output": reply, **({} if cost is None else {"total_cost_usd": cost})}))
|
|
|
|
|
|
class Worker:
|
|
"""A worker claude: writes a line to the session's transcript, then streams a result."""
|
|
|
|
def __init__(self, home, prompted=False, text="working"):
|
|
self.home = home
|
|
# Whether it records its -p prompt in the transcript first, as claude does.
|
|
self.prompted = prompted
|
|
self.text = text
|
|
|
|
def __call__(self, args):
|
|
flag = "--session-id" if "--session-id" in args else "--resume"
|
|
path = self.home / "projects" / "-fake" / f"{args[args.index(flag) + 1]}.jsonl"
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(path, "a") as handle:
|
|
if flag == "--session-id":
|
|
handle.write(json.dumps({"type": "custom-title"}) + "\n" + json.dumps({"type": "queue-operation", "timestamp": str(uuid.uuid7())}) + "\n")
|
|
if self.prompted:
|
|
handle.write(json.dumps({**human(args[args.index("-p") + 1]), "promptSource": "sdk"}) + "\n")
|
|
handle.write(json.dumps(assistant({"type": "text", "text": self.text})) + "\n")
|
|
return Streamed(0)
|
|
|
|
|
|
class Streamed:
|
|
def __init__(self, code, *events):
|
|
self.code = code
|
|
self.events = events or ({"type": "result"},)
|
|
|
|
def lines(self):
|
|
return [json.dumps(event) + "\n" for event in self.events]
|
|
|
|
def wait(self):
|
|
return self.code
|
|
|
|
|
|
TYPED = states.TypedAnswer("Which one?", "Use B.")
|
|
|
|
|
|
class Transitions(unittest.TestCase):
|
|
def test_each_state_moves_on_its_events_and_only_on_them(self):
|
|
answered = states.Ruled(QuotedAnswer("e", "Which one?", "A."))
|
|
legal = (
|
|
(states.Working(), states.WorkerRan("/t"), states.Judging("/t")),
|
|
(states.Working(QuotedAnswer("e", "Which one?", "A."), 3), states.WorkerRan("/t"), states.Judging("/t", 3)),
|
|
(states.Judging("/t"), states.Ruled(Interrupt("e", "Which one?")), states.Interrupting("/t", Interrupt("e", "Which one?"))),
|
|
(states.Judging("/t", 3), states.Ruled(Interrupt("e", "Which one?")), states.Interrupting("/t", Interrupt("e", "Which one?"), 3)),
|
|
(states.Judging("/t"), answered, states.Working(QuotedAnswer("e", "Which one?", "A."), 1)),
|
|
(states.Judging("/t", 2), answered, states.Working(QuotedAnswer("e", "Which one?", "A."), 3)),
|
|
(states.Judging("/t", 3), answered, states.Interrupting("/t", QuotedAnswer("e", "Which one?", "A."), 3)),
|
|
(states.Judging("/t"), states.Ruled(Finished("e")), states.Closing("/t", Finished("e"))),
|
|
(states.Judging("/t", 3), states.Ruled(Finished("e")), states.Closing("/t", Finished("e"))),
|
|
(states.Judging("/t"), states.Ruled(Unfinished("e")), states.Closing("/t", Unfinished("e"))),
|
|
(states.Judging("/t", 3), states.Ruled(Unfinished("e")), states.Closing("/t", Unfinished("e"))),
|
|
(states.Interrupting("/t", Interrupt("e", "q")), states.HumanTalked(), states.Judging("/t")),
|
|
(states.Interrupting("/t", Interrupt("e", "q"), 3), states.HumanTalked(), states.Judging("/t", 3)),
|
|
(states.Interrupting("/t", Interrupt("e", "q")), states.HumanTyped(TYPED), states.Working(TYPED)),
|
|
(states.Interrupting("/t", Interrupt("e", "q"), 3), states.HumanTyped(TYPED), states.Working(TYPED, 3)),
|
|
(states.Working(TYPED, 3), states.WorkerRan("/t"), states.Judging("/t", 3)),
|
|
(states.Closing("/t", Finished("e")), states.Archived(), states.Ended(Finished("e"))),
|
|
(states.Closing("/t", Unfinished("e")), states.Archived(), states.Ended(Unfinished("e"))),
|
|
)
|
|
for state, event, after in legal:
|
|
with self.subTest(state=state, event=event):
|
|
self.assertEqual(states.transition(state, event), after)
|
|
every_state = (states.Working(), states.Working(QuotedAnswer("e", "Which one?", "A."), 3), states.Working(TYPED, 3), states.Judging("/t"), states.Judging("/t", 3), states.Interrupting("/t", Interrupt("e", "q")), states.Interrupting("/t", Interrupt("e", "q"), 3), states.Closing("/t", Finished("e")), states.Ended(Finished("e")))
|
|
every_event = (states.WorkerRan("/t"), states.Ruled(Interrupt("e", "Which one?")), answered, states.Ruled(Finished("e")), states.Ruled(Unfinished("e")), states.HumanTalked(), states.HumanTyped(TYPED), states.Archived())
|
|
for state in every_state:
|
|
for event in every_event:
|
|
if (state, event) not in [(state, event) for state, event, _ in legal]:
|
|
with self.subTest(state=state, event=event), self.assertRaises(ValueError):
|
|
states.transition(state, event)
|
|
|
|
def test_a_rulings_step_up_reaches_the_worker_it_resumes_through_the_answer_the_human_types_and_no_further(self):
|
|
stepped_to = Route(Rung("claude-opus-5-5", "high"), escalated_at=3)
|
|
interrupt, quoted = Interrupt("e", "q", "round 2: 🔴"), QuotedAnswer("e", "q", "A.", "round 2: 🔴")
|
|
for state, event, after in (
|
|
(states.Judging("/t"), states.Ruled(quoted, stepped_to), states.Working(quoted, 1, True)),
|
|
(states.Judging("/t", 3), states.Ruled(quoted, stepped_to), states.Interrupting("/t", quoted, 3, True)),
|
|
(states.Judging("/t"), states.Ruled(interrupt, stepped_to), states.Interrupting("/t", interrupt, 0, True)),
|
|
(states.Interrupting("/t", interrupt, 0, True), states.HumanTyped(TYPED), states.Working(TYPED, 0, True)),
|
|
(states.Interrupting("/t", interrupt, 0, True), states.HumanTalked(), states.Judging("/t")),
|
|
(states.Working(TYPED, 0, True), states.WorkerRan("/t"), states.Judging("/t")),
|
|
):
|
|
with self.subTest(state=state, event=event):
|
|
self.assertEqual(states.transition(state, event), after)
|
|
|
|
|
|
class QuestionFor(unittest.TestCase):
|
|
def test_puts_an_interrupts_question_as_asked_and_a_quoted_answer_past_the_cap_with_its_quote(self):
|
|
self.assertEqual(run.question_for(Interrupt("e", "Which one?")), "Which one?")
|
|
self.assertEqual(run.question_for(QuotedAnswer("e", "Which one?", "A.")), "Which one?\n\nREADME.md's Goals and Audience, AGENTS.md, docs/decisions.md, ~/.claude/CLAUDE.md, ~/.claude/rules/ or ~/.claude/skills/larv-review/standards/ says:\nA.\n\nThose files have answered 3 questions this cycle, the most allowed, so this one comes to you.")
|
|
|
|
|
|
class Cycles(unittest.TestCase):
|
|
def setUp(self):
|
|
temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temp.cleanup)
|
|
self.home = pathlib.Path(temp.name)
|
|
self.paths = paths.paths_from(self.home)
|
|
self.session = session.session_named("A proj loop-claude", self.paths.claude_code_transcripts)
|
|
self.archive = project_archive(str(self.home / "code" / "proj"), self.home, {})
|
|
|
|
def cycle(self, fake, records="", unfinished_evidence="", facts="- A preflight fact."):
|
|
return run.Cycle(effects=fake.effects(), facts=facts, judge_model="judge-model", premises="## Goals", project=None, project_archive=self.archive, questions=self.paths.waiting, records=records, root=str(self.home), route=Route(Rung("worker-model")), session=self.session, show_taken_in=lambda items: None, task="Do it.", todo="", unfinished_evidence=unfinished_evidence)
|
|
|
|
def archived_outcomes(self):
|
|
return sorted(path.read_text() for path in self.archive.glob("*/*.outcomes.md"))
|
|
|
|
def test_a_finished_chunk_is_archived_and_its_session_deleted(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
|
|
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: finished\n\nEvidence:\n> working\n\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
|
|
|
|
def test_archives_the_cost_the_judges_reply_records_beside_its_verdict(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished", cost=0.25)])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working", judge_cost=0.25))
|
|
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: finished\n\nEvidence:\n> working\n\nJudge: judge-model\nJudge cost: 0.2500\nWorker: {OFF_LADDER}\n"])
|
|
self.assertIsNone(session.transcript_file(self.session))
|
|
|
|
def test_shows_the_items_each_ruling_finds_the_worker_names_on_its_chunk_items_line(self):
|
|
worker = Worker(self.paths.claude_code_transcripts.parent, text="working\nChunk items: 14 (L 2, E + I 3), 3 (L 1, E + I 2)")
|
|
fake = FakeEffects(streams=[worker], claude=[judged("unfinished")])
|
|
shown = []
|
|
run.run_cycle(dataclasses.replace(self.cycle(fake), show_taken_in=shown.append))
|
|
self.assertEqual(shown, [(14, 3)])
|
|
|
|
def test_shows_no_items_for_a_finished_chunk(self):
|
|
worker = Worker(self.paths.claude_code_transcripts.parent, text="working\nChunk items: 3")
|
|
fake = FakeEffects(streams=[worker], claude=[judged("finished")])
|
|
shown = []
|
|
run.run_cycle(dataclasses.replace(self.cycle(fake), show_taken_in=shown.append))
|
|
self.assertEqual(shown, [])
|
|
|
|
def test_a_fresh_worker_reads_the_preflights_facts_and_a_resumed_one_does_not(self):
|
|
worker = Worker(self.paths.claude_code_transcripts.parent)
|
|
fake = FakeEffects(streams=[worker, worker], claude=[judged("unfinished"), judged("finished")])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=3))
|
|
self.assertEqual(run.run_cycle(self.cycle(fake, unfinished_evidence="working")).verdict, Finished("working"))
|
|
fresh, resumed = (args[args.index("-p") + 1] for args in fake.made("stream_claude"))
|
|
self.assertIn("- A preflight fact.", fresh)
|
|
self.assertNotIn("- A preflight fact.", resumed)
|
|
|
|
def test_a_worker_is_handed_the_projects_rung_its_own_its_reviews_e_plus_i_and_its_compaction_window_in_its_environment(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
|
|
cycle = dataclasses.replace(self.cycle(fake), project=Rung("claude-opus-5-5", "medium"), route=Route(Rung("claude-haiku-5-5", "max"), 163, 1, 5))
|
|
self.assertEqual(run.run_cycle(cycle).verdict, Finished("working"))
|
|
self.assertEqual(fake.worker_environments, [{"LOOP_CLAUDE_PROJECT_RUNG": "claude-opus-5-5 at medium effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-haiku-5-5 at max effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "5", "CLAUDE_CODE_AUTO_COMPACT_WINDOW": "200000"}])
|
|
|
|
def test_the_claude_an_interrupt_resumes_is_handed_the_same_rungs_as_the_worker(self):
|
|
def answer(args):
|
|
with open(session.transcript_file(self.session), "a") as handle:
|
|
handle.write(json.dumps(human("the second")) + "\n")
|
|
return 0
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt"), judged("finished")], inputs=["/talk"], on_terminal_claude=[answer])
|
|
cycle = dataclasses.replace(self.cycle(fake), project=Rung("claude-opus-5-5", "medium"), route=Route(Rung("claude-haiku-5-5", "max"), 163, 1, 5))
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(run.run_cycle(cycle).verdict, Finished("working"))
|
|
self.assertEqual(fake.terminal_environments, [{**environment, "CLAUDE_CODE_SANDBOXED": "1"} for environment in fake.worker_environments])
|
|
self.assertEqual(fake.terminal_environments[0]["LOOP_CLAUDE_WORKER_RUNG"], "claude-haiku-5-5 at max effort")
|
|
|
|
def test_an_unfinished_chunks_transcript_is_archived_and_the_next_cycle_resumes_its_session_with_the_judges_evidence(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[judged("unfinished")] * 2)
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=3))
|
|
self.assertTrue(session.transcript_file(self.session))
|
|
self.assertEqual(run.run_cycle(self.cycle(fake, unfinished_evidence="working")).verdict, Unfinished("working", entries_read=4))
|
|
workers = fake.made("stream_claude")
|
|
self.assertIn("--session-id", workers[0])
|
|
self.assertIn("--resume", workers[1])
|
|
self.assertIn("as the work remaining:\n\n> working\n\nThe chunk is finished once", workers[1][workers[1].index("-p") + 1])
|
|
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: unfinished\n\nEvidence:\n> working\n\nJudge: judge-model\nWorker: {OFF_LADDER}\n"] * 2)
|
|
|
|
def test_a_cycle_stopped_by_a_failure_or_ctrl_c_archives_what_its_session_holds(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[ran(code=1)] * 3)
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).code, 1)
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[KeyboardInterrupt()])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)), Stop(130, ""))
|
|
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Stopped: Ctrl-C\n\nWorker: {OFF_LADDER}\n", f"Task: Do it.\n\n2026-09-21T14:13:20Z Stopped: the judge failed three times running: the judge exited 1\n\nWorker: {OFF_LADDER}\n"])
|
|
|
|
def test_each_judge_reply_the_loop_rejects_is_archived_with_why_beside_the_verdict_that_follows(self):
|
|
unfilled = ran(json.dumps({"structured_output": {"interrupt": False}}))
|
|
misquoted = judged("finished", evidence="never said")
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[unfilled, misquoted, judged("finished")])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
|
|
stamp = "2026-09-21T14:13:20Z"
|
|
self.assertEqual(self.archived_outcomes(), [
|
|
"Task: Do it.\n\n"
|
|
f"{stamp} Judge reply rejected: the judge's reply has no answer of type string\n\nReply:\n> {unfilled.stdout}\n\nJudge: judge-model\n\n"
|
|
f"{stamp} Judge reply rejected: the judge quoted 'never said', which is not in the transcript\n\nReply:\n> {misquoted.stdout}\n\nJudge: judge-model\n\n"
|
|
f"{stamp} Verdict: finished\n\nEvidence:\n> working\n\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
|
|
|
|
def test_archives_what_each_judge_run_cost_on_an_interrupt_a_quoted_answer_and_a_rejected_misquote(self):
|
|
for first, then, inputs in (
|
|
(judged("interrupt", cost=0.25), judged("finished"), ["Use B."]),
|
|
(judged("quoted", answer="Use A.", cost=0.25), judged("finished"), []),
|
|
(judged("finished", evidence="never said", cost=0.25), judged("finished"), []),
|
|
):
|
|
with self.subTest(first=first.stdout), tempfile.TemporaryDirectory() as archive:
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[first, then], inputs=inputs)
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
run.run_cycle(dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project_archive=pathlib.Path(archive)))
|
|
kept = "".join(path.read_text() for path in pathlib.Path(archive).glob("*/*.outcomes.md"))
|
|
self.assertEqual(kept.count("Judge cost: 0.2500\n"), 1, kept)
|
|
|
|
def test_stops_with_exit_1_asking_nothing_where_an_interrupt_cannot_be_archived(self):
|
|
self.archive.parent.mkdir(parents=True)
|
|
self.archive.write_text("")
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt")], inputs=["Use B."])
|
|
with contextlib.redirect_stderr(io.StringIO()) as said:
|
|
stop = run.run_cycle(self.cycle(fake))
|
|
self.assertEqual((stop.code, stop.reason.split(":")[0]), (1, "could not archive the transcript"))
|
|
self.assertNotIn("Which one?", said.getvalue())
|
|
self.assertEqual(fake.made("claude_on_terminal"), [])
|
|
|
|
def test_stops_when_a_rejected_reply_cannot_be_archived(self):
|
|
self.archive.parent.mkdir(parents=True)
|
|
self.archive.write_text("")
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[ran(json.dumps({"structured_output": {"interrupt": False}})), judged("finished")])
|
|
stop = run.run_cycle(self.cycle(fake))
|
|
self.assertRegex(stop.reason, f"^could not archive the transcript: {self.archive}/[0-9a-f]{{64}}: Not a directory$")
|
|
self.assertEqual(stop.code, 1)
|
|
self.assertEqual(len(fake.made("claude")), 1)
|
|
|
|
def test_archives_the_judges_rejected_replies_though_it_fails_three_times_running(self):
|
|
unfilled = ran(json.dumps({"structured_output": {"interrupt": False}}))
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[unfilled] * 3)
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).code, 1)
|
|
[archived] = self.archived_outcomes()
|
|
self.assertEqual(archived.count(f"Reply:\n> {unfilled.stdout}\n"), 3)
|
|
self.assertIn("Stopped: the judge failed three times running", archived)
|
|
|
|
def test_archives_each_chunks_session_folder_replacing_a_copy_only_with_what_grew_from_it(self):
|
|
folder = self.paths.claude_code_transcripts / "-fake" / str(self.session.id)
|
|
(folder / "subagents" / "workflows").mkdir(parents=True)
|
|
(folder / "subagents" / "agent-a.jsonl").write_text("a\n")
|
|
(folder / "subagents" / "workflows" / "journal.jsonl").write_text("j\n")
|
|
(folder / "custom-title.json").write_text("first")
|
|
(folder.parent / f"{self.session.id}.orphaned-1.jsonl").write_text("o\n")
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 3, claude=[judged("unfinished"), judged("finished"), judged("unfinished")])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=3))
|
|
[chunk] = self.archive.iterdir()
|
|
kept = chunk / "session" / str(self.session.id)
|
|
self.assertEqual((chunk / "session" / f"{self.session.id}.orphaned-1.jsonl").read_text(), "o\n")
|
|
os.utime(kept / "subagents" / "workflows" / "journal.jsonl", (0, 0))
|
|
with open(folder / "subagents" / "agent-a.jsonl", "a") as handle:
|
|
handle.write("more\n")
|
|
(folder / "custom-title.json").write_text("second")
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
|
|
self.assertFalse(folder.exists())
|
|
self.assertEqual((kept / "subagents" / "agent-a.jsonl").read_text(), "a\nmore\n")
|
|
self.assertEqual((kept / "subagents" / "workflows" / "journal.jsonl").stat().st_mtime, 0)
|
|
self.assertEqual(sorted(path.read_text() for path in kept.glob("custom-title.json*")), ["first", "second"])
|
|
self.assertEqual(stat.S_IMODE((kept / "subagents" / "agent-a.jsonl").stat().st_mode), 0o600)
|
|
folder.mkdir()
|
|
(folder / "custom-title.json").write_text("third")
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=3))
|
|
[next_chunk] = set(self.archive.iterdir()) - {chunk}
|
|
self.assertEqual(sorted(path.read_text() for path in (next_chunk / "session" / str(self.session.id)).glob("custom-title.json*")), ["third"])
|
|
|
|
@unittest.skipIf(os.geteuid() == 0, "root reads an unreadable folder")
|
|
def test_stops_on_a_session_folder_it_cannot_read_whole(self):
|
|
locked = self.paths.claude_code_transcripts / "-fake" / str(self.session.id) / "subagents"
|
|
locked.mkdir(parents=True)
|
|
locked.chmod(0)
|
|
self.addCleanup(locked.chmod, 0o700)
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("unfinished")])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)), Stop(1, f"could not archive the transcript: {locked}: Permission denied"))
|
|
|
|
def test_an_interrupt_talked_over_in_claude_is_archived_answered_and_judged_again(self):
|
|
def answer(args):
|
|
with open(session.transcript_file(self.session), "a") as handle:
|
|
handle.write(json.dumps(human("the second")) + "\n")
|
|
return 0
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt"), judged("unfinished")], inputs=["/talk"], on_terminal_claude=[answer])
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=4))
|
|
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: interrupt\n\nWhich one?\n\nEvidence:\n> working\n\nJudge: judge-model\nWorker: {OFF_LADDER}\n", f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: unfinished\n\nEvidence:\n> working\n\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
|
|
self.assertIn(str(self.session.id), fake.made("claude_on_terminal")[0])
|
|
judge_prompts = [stdin for args, stdin in fake.made("claude")]
|
|
self.assertEqual(len(judge_prompts), 2)
|
|
self.assertNotIn("the second", judge_prompts[0])
|
|
self.assertIn("[human] the second", judge_prompts[1])
|
|
self.assertIn("From README.md:\n## Goals", judge_prompts[0])
|
|
self.assertIn("The task the worker was given:\nDo it.", judge_prompts[0])
|
|
|
|
def test_a_question_the_records_answer_resumes_the_worker_with_the_answer_three_times_a_cycle(self):
|
|
answered = judged("quoted", answer="Use *A*.")
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 4, claude=[answered] * 4, inputs=[""])
|
|
with contextlib.redirect_stderr(io.StringIO()) as said, contextlib.redirect_stdout(io.StringIO()) as printed:
|
|
self.assertEqual(run.run_cycle(self.cycle(fake, records="From AGENTS.md:\nUse A.")), Stop(2, "the human is needed to answer the question above, but nothing was typed"))
|
|
resumed = fake.made("stream_claude")[1]
|
|
self.assertEqual(resumed[resumed.index("-p") + 1], "Your question: Which one?\n\nREADME.md's Goals and Audience, AGENTS.md, docs/decisions.md, ~/.claude/CLAUDE.md, ~/.claude/rules/ or ~/.claude/skills/larv-review/standards/ answers it:\n\nUse *A*.\n\nContinue the task.")
|
|
self.assertEqual(len(fake.made("stream_claude")), 4)
|
|
self.assertEqual(printed.getvalue().count("=== README.md's Goals and Audience, AGENTS.md, docs/decisions.md, ~/.claude/CLAUDE.md, ~/.claude/rules/ or ~/.claude/skills/larv-review/standards/ answered: Which one? ===\nUse *A*."), 3)
|
|
self.assertIn("Which one?\n\nREADME.md's Goals and Audience, AGENTS.md, docs/decisions.md, ~/.claude/CLAUDE.md, ~/.claude/rules/ or ~/.claude/skills/larv-review/standards/ says:\nUse *A*.\n\nThose files have answered 3 questions this cycle", said.getvalue())
|
|
quoted = f"2026-09-21T14:13:20Z Verdict: quoted answer\n\nWhich one?\n\nREADME.md's Goals and Audience, AGENTS.md, docs/decisions.md, ~/.claude/CLAUDE.md, ~/.claude/rules/ or ~/.claude/skills/larv-review/standards/ says:\n> Use *A*.\n\nEvidence:\n> working\n\nJudge: judge-model\nWorker: {OFF_LADDER}\n"
|
|
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: interrupt\n\nWhich one?\n\nREADME.md's Goals and Audience, AGENTS.md, docs/decisions.md, ~/.claude/CLAUDE.md, ~/.claude/rules/ or ~/.claude/skills/larv-review/standards/ says:\nUse *A*.\n\nThose files have answered 3 questions this cycle, the most allowed, so this one comes to you.\n\nEvidence:\n> working\n\nJudge: judge-model\nWorker: {OFF_LADDER}\n\n2026-09-21T14:13:20Z Stopped: the human is needed to answer the question above, but nothing was typed\n\nWorker: {OFF_LADDER}\n", *[f"Task: Do it.\n\n{quoted}"] * 3])
|
|
|
|
def test_a_quoted_answer_that_cites_an_escalation_trigger_steps_the_worker_up_before_it_resumes_and_archives_the_step(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[judged("quoted", answer="Use A.", escalation="working"), judged("finished")])
|
|
cycle = dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project=Routing.OPUS, route=Route(Routing.SONNET, 179, 2, 5))
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
concluded = run.run_cycle(cycle)
|
|
self.assertEqual(concluded.route, Route(Routing.OPUS, 179, 2, 5, escalated_at=3))
|
|
first, resumed = fake.made("stream_claude")
|
|
self.assertEqual([args[args.index("--effort") + 1] for args in (first, resumed)], ["high", "medium"])
|
|
self.assertTrue(resumed[resumed.index("-p") + 1].endswith(f"Continue the task. Your request for a higher rung went through: loop-claude runs you on {Routing.OPUS}. The escalation triggers that fired before this step are spent.{prompts.rungs_held(concluded.route, Routing.OPUS)}"))
|
|
self.assertNotIn("went through", first[first.index("-p") + 1])
|
|
answered = f"Verdict: quoted answer\n\nWhich one?\n\n{prompts.QUOTE_SOURCES} says:\n> Use A.\n\nEvidence:\n> working\n\nEscalation:\n> working\n\nJudge: judge-model\nWorker: {SONNET_179}\n"
|
|
stepped = f"Stepped up\n\nWorker: {OPUS_179_AT_3}\n"
|
|
kept = "".join(self.archived_outcomes())
|
|
self.assertIn(f"Z {answered}\n20", kept)
|
|
self.assertIn(f"Z {stepped}", kept)
|
|
|
|
def test_a_quoted_answer_past_the_cap_archives_the_escalation_it_carried_with_the_interrupt_and_steps_up(self):
|
|
steps = [judged("quoted", answer="Use A.")] * 3 + [judged("quoted", answer="Use A.", escalation="working")]
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 4, claude=steps, inputs=[""])
|
|
cycle = dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project=Routing.OPUS, route=Route(Routing.SONNET, 179, 2, 5))
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(run.run_cycle(cycle), Stop(2, "the human is needed to answer the question above, but nothing was typed"))
|
|
kept = "".join(self.archived_outcomes())
|
|
self.assertIn(f"Evidence:\n> working\n\nEscalation:\n> working\n\nJudge: judge-model\nWorker: {SONNET_179}\n", kept)
|
|
self.assertIn(f"Stepped up\n\nWorker: {OPUS_179_AT_6}\n", kept)
|
|
|
|
def test_a_ruling_steps_the_route_only_on_an_escalation_from_the_rung_its_taken_in_l_picks(self):
|
|
cycle = dataclasses.replace(self.cycle(FakeEffects()), project=Routing.OPUS, route=Route(Routing.OPUS, 3, 3, 4))
|
|
for ruling in (Interrupt("e", "q", entries_read=3), QuotedAnswer("e", "q", "A.", entries_read=3)):
|
|
with self.subTest(ruling=ruling):
|
|
self.assertIsNone(run.ruling_step_up(cycle, ruling))
|
|
self.assertEqual(run.ruling_step_up(cycle, Interrupt("e", "q", "round 2: 🔴", TakenIn(highest_likelihood=4), entries_read=3)), routing.StepUp(cycle.route, Route(Routing.XHIGH, 3, 3, 4, escalated_at=3)))
|
|
|
|
def test_an_escalation_after_an_interrupt_stepped_the_worker_steps_it_again_in_the_same_cycle(self):
|
|
streams = [Worker(self.paths.claude_code_transcripts.parent, text="round 2: 🔴"), Worker(self.paths.claude_code_transcripts.parent), Worker(self.paths.claude_code_transcripts.parent)]
|
|
rulings = [judged("interrupt", evidence="round 2: 🔴", escalation="round 2: 🔴"), judged("quoted", answer="Use A.", escalation="working"), judged("finished")]
|
|
fake = FakeEffects(streams=streams, claude=rulings, inputs=["Use B."])
|
|
cycle = dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project=Routing.OPUS, route=Route(Routing.SONNET, 179, 2, 5))
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
concluded = run.run_cycle(cycle)
|
|
self.assertEqual(concluded.route, Route(Routing.HIGH, 179, 2, 5, escalated_at=4))
|
|
self.assertEqual([args[args.index("--effort") + 1] for args in fake.made("stream_claude")], ["high", "medium", "high"])
|
|
|
|
def test_an_escalation_quoted_from_before_the_workers_last_step_up_steps_nothing_and_a_later_one_reaches_the_verdict(self):
|
|
for escalation, verdict in (("round 2: 🔴", Unfinished("working", entries_read=4)), ("working", Unfinished("working", "working", entries_read=4))):
|
|
with self.subTest(escalation=escalation):
|
|
self.session = session.session_named(f"A proj {escalation} loop-claude", self.paths.claude_code_transcripts)
|
|
streams = [Worker(self.paths.claude_code_transcripts.parent, text="round 2: 🔴"), Worker(self.paths.claude_code_transcripts.parent)]
|
|
rulings = [judged("interrupt", evidence="round 2: 🔴", escalation="round 2: 🔴"), judged("unfinished", escalation=escalation)]
|
|
fake = FakeEffects(streams=streams, claude=rulings, inputs=["Use B."])
|
|
cycle = dataclasses.replace(self.cycle(fake), project=Routing.OPUS, route=Route(Routing.SONNET, 179, 2, 5))
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
concluded = run.run_cycle(cycle)
|
|
self.assertEqual(concluded, run.Concluded(verdict, Route(Routing.OPUS, 179, 2, 5, escalated_at=3)))
|
|
|
|
def test_an_answer_typed_at_the_terminal_resumes_the_worker_headless_and_the_judge_reads_it_as_the_humans(self):
|
|
worker = Worker(self.paths.claude_code_transcripts.parent, prompted=True)
|
|
fake = FakeEffects(streams=[worker, worker], claude=[judged("interrupt"), judged("unfinished")], inputs=["Use B."])
|
|
with contextlib.redirect_stderr(io.StringIO()) as said:
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=6))
|
|
self.assertIn(f"\n{STAMP} Which one?\n", said.getvalue())
|
|
self.assertEqual(fake.made("claude_on_terminal"), [])
|
|
resumed = fake.made("stream_claude")[1]
|
|
self.assertEqual(resumed[resumed.index("--resume") + 1], str(self.session.id))
|
|
typed = resumed[resumed.index("-p") + 1]
|
|
self.assertEqual(prompts.typed_answer(typed).answer, "Use B.")
|
|
self.assertIn("Which one?", typed)
|
|
self.assertIn("Continue the task. The chunk is finished only once every answer the human typed in it", typed)
|
|
judge_prompts = [stdin for args, stdin in fake.made("claude")]
|
|
self.assertRegex(judge_prompts[1], r"(?s)\[loop\] Your question was: Which one\?.*\n\n\[human\] Use B\.\n\nworking\n\nDecide whether")
|
|
self.assertNotIn("The human answered", judge_prompts[1])
|
|
|
|
def test_an_interrupt_that_steps_the_worker_up_names_its_new_rung_in_the_prompt_that_resumes_it_with_the_typed_answer(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[judged("interrupt", escalation="working"), judged("finished")], inputs=["Use B."])
|
|
cycle = dataclasses.replace(self.cycle(fake), project=Routing.OPUS, route=Route(Routing.SONNET, 179, 2, 5))
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
concluded = run.run_cycle(cycle)
|
|
self.assertEqual(concluded.route, Route(Routing.OPUS, 179, 2, 5, escalated_at=3))
|
|
first, resumed = fake.made("stream_claude")
|
|
self.assertEqual([args[args.index("--effort") + 1] for args in (first, resumed)], ["high", "medium"])
|
|
typed = resumed[resumed.index("-p") + 1]
|
|
self.assertIn("Your request for a higher rung went through: loop-claude runs you on claude-opus-5-5 at medium effort.", typed)
|
|
self.assertIn(" Launch each reviewer with the Agent tool set to model opus, effort high, the rung item 179's E + I of 5 picks.", typed)
|
|
self.assertNotIn("went through", first[first.index("-p") + 1])
|
|
|
|
def test_a_typed_answer_past_the_quoted_answers_cap_reaches_the_worker_with_its_own_question(self):
|
|
worker = Worker(self.paths.claude_code_transcripts.parent)
|
|
fake = FakeEffects(streams=[worker] * 5, claude=[judged("quoted", answer="Use A.")] * 4 + [judged("finished")], inputs=["Use B."])
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(run.run_cycle(self.cycle(fake, records="From AGENTS.md:\nUse A.")).verdict, Finished("working"))
|
|
typed = fake.made("stream_claude")[4]
|
|
self.assertIn("Your question was: Which one?\n\nContinue the task.", typed[typed.index("-p") + 1])
|
|
self.assertNotIn("Those files have answered", typed[typed.index("-p") + 1])
|
|
|
|
def test_a_release_ready_to_cut_opens_claudes_session_without_asking_for_an_answer(self):
|
|
def answer(args):
|
|
with open(session.transcript_file(self.session), "a") as handle:
|
|
handle.write(json.dumps(human("cut")) + "\n")
|
|
return 0
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt", release_ready=True), judged("finished")], on_terminal_claude=[answer])
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
|
|
self.assertEqual(fake.made("input"), [])
|
|
self.assertEqual(len(fake.made("claude_on_terminal")), 1)
|
|
|
|
def test_stops_when_an_interrupt_cannot_be_archived(self):
|
|
self.archive.parent.mkdir(parents=True)
|
|
self.archive.write_text("")
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt")], inputs=["Use B."])
|
|
stop = run.run_cycle(self.cycle(fake))
|
|
self.assertRegex(stop.reason, f"^could not archive the transcript: {self.archive}/[0-9a-f]{{64}}: Not a directory$")
|
|
self.assertEqual(fake.made("input"), [])
|
|
|
|
def test_stops_when_a_quoted_answer_cannot_be_archived(self):
|
|
self.archive.parent.mkdir(parents=True)
|
|
self.archive.write_text("")
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("quoted", answer="Use A.")])
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
stop = run.run_cycle(self.cycle(fake, records="From AGENTS.md:\nUse A."))
|
|
self.assertRegex(stop.reason, f"^could not archive the transcript: {self.archive}/[0-9a-f]{{64}}: Not a directory$")
|
|
self.assertEqual(len(fake.made("stream_claude")), 1)
|
|
|
|
def test_an_answer_the_records_do_not_hold_is_dropped_and_the_question_goes_to_the_human(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("quoted", answer="Use ... A.")], inputs=[""])
|
|
with contextlib.redirect_stderr(io.StringIO()) as said:
|
|
self.assertEqual(run.run_cycle(self.cycle(fake, records="From AGENTS.md:\nUse B. Use A.")), Stop(2, "the human is needed to answer the question above, but nothing was typed"))
|
|
self.assertEqual(len(fake.made("stream_claude")), 1)
|
|
self.assertIn(f"\n{STAMP} Which one?\n", said.getvalue())
|
|
self.assertNotIn("Use ... A.", said.getvalue())
|
|
|
|
@unittest.skipIf(os.geteuid() == 0, "root writes into a read-only directory")
|
|
def test_stops_keeping_the_transcript_when_the_finished_chunks_session_cannot_be_deleted(self):
|
|
folder = self.paths.claude_code_transcripts / "-fake" / str(self.session.id)
|
|
(folder / "subagents").mkdir(parents=True)
|
|
folder.chmod(0o500)
|
|
self.addCleanup(folder.chmod, 0o700)
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
|
|
stop = run.run_cycle(self.cycle(fake))
|
|
self.assertEqual(stop, Stop(1, f"could not delete {folder}: Permission denied; delete every file and folder matching {folder}*, then rerun loop-claude; a rerun before that resumes the finished chunk's session"))
|
|
self.assertTrue(session.session_open(self.session))
|
|
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: finished\n\nEvidence:\n> working\n\nJudge: judge-model\nWorker: {OFF_LADDER}\n\n2026-09-21T14:13:20Z Stopped: {stop.reason}\n\nWorker: {OFF_LADDER}\n"])
|
|
|
|
def test_stops_when_the_worker_leaves_no_transcript(self):
|
|
fake = FakeEffects(streams=[Streamed(0)])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)), Stop(1, f"claude left no transcript for session {self.session.id}"))
|
|
|
|
|
|
class Retried(unittest.TestCase):
|
|
def test_retries_a_minute_apart_and_stops_on_the_third_failure(self):
|
|
fake = FakeEffects()
|
|
results = [Failure("one"), "done"]
|
|
self.assertEqual(retry.retried("worker", lambda: results.pop(0), fake.effects()), "done")
|
|
self.assertEqual(fake.calls, [("sleep", 60)])
|
|
fake = FakeEffects()
|
|
self.assertEqual(retry.retried("judge", lambda: Failure("no"), fake.effects()), Stop(1, "the judge failed three times running: no"))
|
|
self.assertEqual(fake.calls, [("sleep", 60), ("sleep", 60)])
|
|
|
|
def test_hands_each_failure_to_the_keeper_and_stops_where_it_stops(self):
|
|
kept = []
|
|
def keep(failure):
|
|
kept.append(failure)
|
|
return Stop(1, "no room") if len(kept) == 2 else None
|
|
fake = FakeEffects()
|
|
self.assertEqual(retry.retried("judge", lambda: Failure("no"), fake.effects(), keep), Stop(1, "no room"))
|
|
self.assertEqual(kept, [Failure("no")] * 2)
|
|
self.assertEqual(fake.calls, [("sleep", 60)])
|
|
|
|
def test_waits_out_a_usage_limit_which_ends_a_run_of_failures(self):
|
|
fake = FakeEffects()
|
|
results = [Failure("one"), Failure("two"), Limited(NOW + 3600, "limit"), Failure("three"), Limited(None, "limit"), Limited(NOW - 600, "limit"), Failure("four"), Failure("five"), Failure("six")]
|
|
self.assertEqual(retry.retried("worker", lambda: results.pop(0), fake.effects()), Stop(1, "the worker failed three times running: six"))
|
|
self.assertEqual(fake.calls, [("sleep", 60), ("sleep", 60), ("sleep", 3720), ("sleep", 60), ("sleep", 600), ("sleep", 600), ("sleep", 60), ("sleep", 60)])
|
|
|
|
def test_says_what_claude_said_and_when_the_next_try_is_due(self):
|
|
results = [Limited(NOW + 3600, "You've hit your session limit"), "done"]
|
|
with contextlib.redirect_stderr(io.StringIO()) as said:
|
|
retry.retried("judge", lambda: results.pop(0), FakeEffects().effects())
|
|
self.assertEqual(said.getvalue(), f"{STAMP} the judge hit a usage limit (You've hit your session limit); retrying at <{NOW + 3720}>\n")
|
|
|
|
|
|
class Limits(unittest.TestCase):
|
|
def test_the_worker_reports_the_latest_reset_its_stream_names(self):
|
|
rejected = [{"type": "rate_limit_event", "rate_limit_info": {"status": "rejected", "resetsAt": resets_at}} for resets_at in (NOW + 60, NOW + 600)]
|
|
allowed = {"type": "rate_limit_event", "rate_limit_info": {"status": "allowed", "resetsAt": NOW + 9000}}
|
|
limited = {"type": "result", "api_error_status": 429, "result": "You've hit your session limit"}
|
|
fake = FakeEffects(streams=[Streamed(1, rejected[1], allowed, rejected[0], limited)])
|
|
self.assertEqual(worker.run_worker("/r", [], {}, fake.effects()), Limited(NOW + 600, "You've hit your session limit"))
|
|
|
|
def test_a_limit_outranks_the_exit_code_and_its_message_fits_one_line(self):
|
|
def limited(**said):
|
|
return {"type": "result", "api_error_status": 429, **said}
|
|
self.assertEqual(worker.worker_outcome(-9, limited(result="limit\n resets 7pm"), []), Limited(None, "limit resets 7pm"))
|
|
self.assertEqual(worker.worker_outcome(1, limited(result=" \n"), [NOW]), Limited(NOW, "claude gave no message"))
|
|
self.assertEqual(worker.worker_outcome(0, limited(), []), Limited(None, "claude gave no message"))
|
|
self.assertEqual(worker.worker_outcome(0, None, [NOW]), Failure("claude exited 0 without a result event"))
|
|
self.assertIsNone(worker.worker_outcome(0, {"type": "result"}, [NOW]))
|
|
|
|
def test_the_judge_reports_a_limit_whatever_its_exit_code(self):
|
|
reply = json.dumps({"type": "result", "api_error_status": 429, "is_error": True, "result": "You've hit your session limit · resets 7:40pm (Europe/Stockholm)"})
|
|
fake = FakeEffects(claude=[ran(reply, code=1, stderr="limited")])
|
|
self.assertEqual(judge.run_judge("/r", [], "", judge.Grounds("", "", ""), fake.effects()), Limited(None, "You've hit your session limit · resets 7:40pm (Europe/Stockholm)"))
|
|
|
|
def test_reads_only_a_rejecting_events_whole_seconds(self):
|
|
for info, want in [({"status": "rejected", "resetsAt": NOW}, NOW), ({"status": "allowed", "resetsAt": NOW}, None), ({"status": "rejected", "resetsAt": True}, None), ({"status": "rejected"}, None), ("rejected", None)]:
|
|
with self.subTest(info=info):
|
|
self.assertEqual(entries.rejected_until({"type": "rate_limit_event", "rate_limit_info": info}), want)
|
|
|
|
|
|
class MemoryWait(unittest.TestCase):
|
|
def test_a_worker_starts_only_once_a_quarter_of_memory_is_available(self):
|
|
fake = FakeEffects(free_memory=[memory.Memory(available=3, total=16), memory.Memory(available=3, total=16), memory.Memory(available=4, total=16)], streams=[Streamed(0)])
|
|
with contextlib.redirect_stderr(io.StringIO()) as said:
|
|
self.assertIsNone(worker.run_worker("/r", [], {}, fake.effects()))
|
|
self.assertEqual([seam for seam, _ in fake.calls], ["memory", "sleep", "memory", "sleep", "memory", "stream_claude"])
|
|
self.assertEqual(fake.made("sleep"), [worker.MEMORY_POLL] * 2)
|
|
self.assertEqual(said.getvalue().count("waiting"), 1, said.getvalue())
|
|
|
|
def test_a_worker_starts_at_once_where_memory_cannot_be_read(self):
|
|
fake = FakeEffects(free_memory=[None], streams=[Streamed(0)])
|
|
self.assertIsNone(worker.run_worker("/r", [], {}, fake.effects()))
|
|
self.assertEqual([seam for seam, _ in fake.calls], ["memory", "stream_claude"])
|
|
|
|
|
|
class Interrupts(unittest.TestCase):
|
|
def setUp(self):
|
|
self.home = pathlib.Path(tempfile.mkdtemp())
|
|
self.addCleanup(shutil.rmtree, self.home)
|
|
self.paths = paths.paths_from(self.home)
|
|
self.transcript = self.home / "t.jsonl"
|
|
self.transcript.write_text(json.dumps(human("a")) + "\n")
|
|
self.session = interrupt.WorkerSession(str(self.home), uuid.uuid7(), Rung("m", "high"), str(self.transcript), {"LOOP_CLAUDE_WORKER_RUNG": "m at high effort"})
|
|
|
|
def put(self, fake, answer_here=True, effects=None, question="Which one?"):
|
|
"""interrupt's answer and what it said, run while the session's lock is held, as a running loop holds it."""
|
|
with contextlib.redirect_stderr(io.StringIO()) as said, session_lock.held(self.paths.waiting, str(self.session.id)):
|
|
answer = interrupt.interrupt(question, answer_here, self.session, self.paths.waiting, effects or fake.effects())
|
|
return answer, said.getvalue()
|
|
|
|
def asked(self):
|
|
return waiting(self.paths.waiting)
|
|
|
|
def ask(self, question="Which one?", state=None):
|
|
"""The ask interrupt makes for question, in state; by default it waits for an answer, and /talk resumes the worker's session."""
|
|
resumed = hand_over.ClaudeCommand(["--model", "m", "--effort", "high", "--permission-mode", "bypassPermissions", "--resume", str(self.session.id)], str(self.home), {"LOOP_CLAUDE_WORKER_RUNG": "m at high effort"})
|
|
return Ask(ASK_ID, str(self.session.id), str(self.home), question, NOW, state or Asking(resumed))
|
|
|
|
def opening(self, other):
|
|
"""loop-inbox's /talk, as the loop's input sees it: loop-inbox claims the one ask listed, and the claim is the first answer."""
|
|
[ask] = self.asked()
|
|
send(self.paths.waiting, ask, ClaudeOpening(INBOX))
|
|
return other()
|
|
|
|
def typed_there(self, args):
|
|
"""The claude opened on the terminal: the human types a line in the worker's session, and it ends."""
|
|
with open(self.transcript, "a") as handle:
|
|
handle.write(json.dumps(human("cut it")) + "\n")
|
|
return 0
|
|
|
|
def test_stops_off_a_terminal_before_asking(self):
|
|
fake = FakeEffects(on_terminal=False)
|
|
self.assertEqual(self.put(fake), (Stop(2, "the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal"), f"\n{STAMP} Which one?\n"))
|
|
self.assertEqual(fake.calls, [])
|
|
|
|
def test_returns_the_line_typed_after_the_question_stripped_dropping_what_was_typed_before(self):
|
|
fake = FakeEffects(inputs=[" Use B. "])
|
|
self.assertEqual(self.put(fake), ("Use B.", f"\n{STAMP} Which one?\n"))
|
|
self.assertEqual(fake.calls, [("discard_typed", None), ("input", "Answer (/talk opens claude, Enter stops the loop): ")])
|
|
|
|
def test_talks_on_talk_or_at_once_where_the_answer_is_not_asked_for_here(self):
|
|
for typed, answer_here in (([" /talk "], True), ([], False)):
|
|
with self.subTest(typed=typed):
|
|
fake = FakeEffects(inputs=typed, on_terminal_claude=[self.typed_there])
|
|
self.assertEqual(self.put(fake, answer_here)[0], interrupt.Talked())
|
|
self.assertEqual(len(fake.made("input")), len(typed))
|
|
self.assertEqual(fake.calls[0], ("discard_typed", None))
|
|
|
|
def test_stops_on_an_empty_answer_or_the_inputs_end(self):
|
|
for typed, why, said in ((" ", "nothing was typed", ""), (EOFError(), "the input ended", "\n")):
|
|
with self.subTest(why=why):
|
|
self.assertEqual(self.put(FakeEffects(inputs=[typed])), (Stop(2, f"the human is needed to answer the question above, but {why}"), f"\n{STAMP} Which one?\n{said}"))
|
|
self.assertEqual(asks_and_answers(self.paths.waiting), [])
|
|
|
|
def test_shows_the_question_in_loop_inbox_while_it_waits_and_only_then(self):
|
|
def typed(_):
|
|
self.assertEqual(self.asked(), [self.ask()])
|
|
return "Use B."
|
|
self.assertEqual(self.put(FakeEffects(inputs=[typed]))[0], "Use B.")
|
|
self.assertEqual(asks_and_answers(self.paths.waiting), [])
|
|
|
|
def test_takes_a_line_typed_in_loop_inbox_for_this_question_and_no_other(self):
|
|
def answered(_, other):
|
|
[ask] = self.asked()
|
|
send(self.paths.waiting, dataclasses.replace(ask, id="an earlier ask"), Typed("Use A."))
|
|
self.assertIsNone(other())
|
|
send(self.paths.waiting, ask, Typed(" Use C. "))
|
|
return other()
|
|
fake = FakeEffects()
|
|
effects = dataclasses.replace(fake.effects(), input_or=answered)
|
|
self.assertEqual(self.put(fake, effects=effects), ("Use C.", f"\n{STAMP} Which one?\n{STAMP} Answered in loop-inbox: Use C. \n"))
|
|
self.assertEqual(asks_and_answers(self.paths.waiting), [])
|
|
|
|
def test_takes_the_answer_typed_on_its_terminal_where_loop_inbox_cannot_list_the_question(self):
|
|
self.paths.waiting.parent.mkdir(parents=True)
|
|
self.paths.waiting.write_text("")
|
|
answer, said = self.put(FakeEffects(inputs=["Use B."]))
|
|
self.assertEqual(answer, "Use B.")
|
|
self.assertIn(f"loop-inbox cannot show this question: cannot make {self.paths.waiting}: File exists", said)
|
|
stop, _ = self.put(FakeEffects(inputs=["/talk"], on_terminal_claude=[0]))
|
|
self.assertEqual(stop, Stop(2, "the human is needed to answer the question above, but the interrupt ended with nothing typed"))
|
|
|
|
def test_makes_way_only_warns_where_loop_inbox_cannot_list_the_open_claude(self):
|
|
self.paths.waiting.parent.mkdir(parents=True)
|
|
self.paths.waiting.write_text("")
|
|
with contextlib.redirect_stderr(io.StringIO()) as said:
|
|
make_way(self.paths.waiting, self.ask(), INBOX, FakeEffects().effects())
|
|
self.assertIn(f"loop-inbox cannot show this question: cannot make {self.paths.waiting}: File exists", said.getvalue())
|
|
|
|
def test_makes_way_for_the_claude_loop_inbox_opens_and_goes_on_once_the_human_typed_there(self):
|
|
def claude_ran(_):
|
|
# loop-inbox opens claude only once the loop lists it as open there.
|
|
self.assertEqual(self.asked(), [self.ask(state=OpenInInbox(INBOX))])
|
|
with open(self.transcript, "a") as handle:
|
|
handle.write(json.dumps(human("cut it")) + "\n")
|
|
send(self.paths.waiting, self.ask(), ClaudeEnded())
|
|
fake = FakeEffects(inputs=[self.opening])
|
|
self.assertEqual(self.put(fake, effects=dataclasses.replace(fake.effects(), sleep=claude_ran)), (interrupt.Talked(), f"\n{STAMP} Which one?\n{STAMP} claude is open for this question in loop-inbox; waiting for it to end\n"))
|
|
self.assertEqual((fake.made("claude_on_terminal"), asks_and_answers(self.paths.waiting)), ([], []))
|
|
|
|
def test_stops_when_loop_inbox_ends_its_claude_with_nothing_typed(self):
|
|
looked, slept = [], []
|
|
def alive(pid):
|
|
looked.append(pid)
|
|
return pid == INBOX
|
|
def ended(seconds):
|
|
slept.append(seconds)
|
|
send(self.paths.waiting, self.ask(), ClaudeEnded())
|
|
fake = FakeEffects(inputs=[self.opening])
|
|
stop, _ = self.put(fake, effects=dataclasses.replace(fake.effects(), alive=alive, sleep=ended))
|
|
self.assertEqual((stop, looked, slept), (Stop(2, "the human is needed to answer the question above, but the interrupt ended with nothing typed"), [INBOX], [POLL_SECONDS]))
|
|
|
|
def test_stops_when_loop_inbox_is_gone_with_nothing_typed(self):
|
|
looked = []
|
|
def alive(pid):
|
|
looked.append(pid)
|
|
# loop-inbox's process ends while the loop waits for its claude.
|
|
return pid == INBOX and len(looked) == 1
|
|
fake = FakeEffects(inputs=[self.opening])
|
|
stop, _ = self.put(fake, effects=dataclasses.replace(fake.effects(), alive=alive))
|
|
self.assertEqual((stop, looked, fake.made("sleep")), (Stop(2, "the human is needed to answer the question above, but the interrupt ended with nothing typed"), [INBOX, INBOX], [POLL_SECONDS]))
|
|
|
|
def test_lists_the_claude_open_on_its_terminal_in_loop_inbox_while_it_runs(self):
|
|
for typed, answer_here in (([" /talk "], True), ([], False)):
|
|
with self.subTest(typed=typed):
|
|
def ran(args):
|
|
self.assertEqual(self.asked(), [self.ask("Cut 0.4.0?", OpenOnTerminal())])
|
|
return 0
|
|
fake = FakeEffects(inputs=typed, on_terminal_claude=[ran])
|
|
self.put(fake, answer_here, question="Cut 0.4.0?")
|
|
self.assertEqual(len(fake.made("claude_on_terminal")), 1)
|
|
self.assertEqual(asks_and_answers(self.paths.waiting), [])
|
|
|
|
def test_stops_when_nothing_was_typed_in_the_session(self):
|
|
fake = FakeEffects(inputs=["/talk"], on_terminal_claude=[0])
|
|
stop, _ = self.put(fake)
|
|
self.assertEqual(fake.made("claude_on_terminal")[0][:4], ["--model", "m", "--effort", "high"])
|
|
self.assertEqual(stop, Stop(2, "the human is needed to answer the question above, but the interrupt ended with nothing typed"))
|
|
|
|
def test_hands_the_claude_that_takes_the_terminal_the_workers_environment(self):
|
|
fake = FakeEffects(inputs=["/talk"], on_terminal_claude=[0])
|
|
self.put(fake)
|
|
self.assertEqual(fake.terminal_environments, [{"LOOP_CLAUDE_WORKER_RUNG": "m at high effort", "CLAUDE_CODE_SANDBOXED": "1"}])
|
|
|
|
|
|
class RoutingCheck(unittest.TestCase):
|
|
def setUp(self):
|
|
self.home = pathlib.Path(tempfile.mkdtemp())
|
|
self.addCleanup(shutil.rmtree, self.home)
|
|
self.root = self.home / "archive"
|
|
self.paths = paths.paths_from(self.home)
|
|
self.session = session.session_named("A log loop-claude", self.paths.claude_code_transcripts)
|
|
|
|
def loop(self, fake):
|
|
"""The loop in /code/log, whose questions loop-inbox lists."""
|
|
return a_loop(fake.effects(), root="/code/log", session=self.session, listed=running_loops.Listed(self.paths.waiting, lambda items: None), paths=self.paths)
|
|
|
|
def chunks(self, count, verdicts, project="log"):
|
|
"""count finished chunks, each closed by verdicts verdicts, each with one Explore subagent whose report outgrew its searches."""
|
|
for number in range(count):
|
|
folder = self.root / project / f"c{number}"
|
|
subagents = folder / "session" / "s" / "subagents"
|
|
subagents.mkdir(parents=True)
|
|
kept = "".join(f"2026-10-14T10:0{verdict}:00Z {outcomes.unfinished('w', 'e', 'j', '')}\n" for verdict in range(verdicts - 1))
|
|
(folder / "copy.outcomes.md").write_text(f"Task: t\n\n{kept}2026-10-14T11:00:00Z {outcomes.finished('w', 'm', 'j')}\n")
|
|
launch = {"type": "assistant", "timestamp": "2026-10-14T09:00:00Z", "message": {"id": "m", "content": [{"type": "tool_use", "id": "a", "name": "Agent", "input": {"subagent_type": "Explore"}}]}}
|
|
report = {"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "a", "content": "long report"}]}}
|
|
(folder / "copy.jsonl").write_text(json.dumps(launch) + "\n" + json.dumps(report) + "\n")
|
|
(subagents / "agent-a.jsonl").write_text("")
|
|
(subagents / "agent-a.meta.json").write_text(json.dumps({"agentType": "Explore", "toolUseId": "a"}))
|
|
|
|
def check(self, fake):
|
|
"""What check_routing returns and says for a loop in /code/log."""
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()) as said:
|
|
self.enterContext(session_lock.held(self.paths.waiting, str(self.session.id)))
|
|
stop = routing_check.check_routing(self.root, Rung("m", "high"), self.loop(fake))
|
|
return stop, said.getvalue()
|
|
|
|
def test_lists_the_calls_in_loop_inbox_while_their_claude_is_open(self):
|
|
self.chunks(5, 4)
|
|
question = f"loop-claude's routing figures call for a routing change:\n- Explore subagents' reports took more of the worker's context than their searches would have.\nThe figures: {self.root / 'routing-figures.md'}"
|
|
def ran(args):
|
|
self.assertEqual(waiting(self.paths.waiting), [Ask(ASK_ID, str(self.session.id), "/code/log", question, NOW, OpenOnTerminal())])
|
|
return 0
|
|
self.check(FakeEffects(on_terminal_claude=[ran], git=[UNREAD]))
|
|
self.assertEqual(asks_and_answers(self.paths.waiting), [])
|
|
|
|
def test_measures_once_20_cycles_have_closed_since_the_last_measurement(self):
|
|
self.chunks(5, 3)
|
|
self.assertEqual(self.check(FakeEffects()), (None, ""))
|
|
self.assertFalse((self.root / "routing-figures.md").exists())
|
|
self.chunks(6, 1, "other")
|
|
self.assertEqual(self.check(FakeEffects(on_terminal=False, git=[UNREAD, UNREAD]))[0], None)
|
|
self.assertIn("at cycle 21 across", (self.root / "routing-figures.md").read_text())
|
|
|
|
def test_reads_each_projects_history_through_the_loops_git_and_names_each_it_cannot_read(self):
|
|
self.chunks(5, 4)
|
|
fake = FakeEffects(on_terminal=False, git=[UNREAD])
|
|
self.check(fake)
|
|
self.assertEqual(fake.made("git"), [["remote"]])
|
|
self.assertIn("## Default branches unread\n\n- log: not a git repository\n", (self.root / "routing-figures.md").read_text())
|
|
|
|
def test_hands_the_terminal_over_once_for_each_new_call(self):
|
|
self.chunks(5, 4)
|
|
fake = FakeEffects(on_terminal_claude=[0], git=[UNREAD])
|
|
stop, said = self.check(fake)
|
|
self.assertIsNone(stop)
|
|
call = "Explore subagents' reports took more of the worker's context than their searches would have."
|
|
self.assertIn(f"- {call}", said)
|
|
self.assertIn(f"- {call}", (self.root / "routing-figures.md").read_text())
|
|
((args),) = fake.made("claude_on_terminal")
|
|
self.assertEqual(args[:6], ["--model", "m", "--effort", "high", "--permission-mode", "bypassPermissions"])
|
|
self.assertIn(f"change nothing until they answer:\n- {call}", args[6])
|
|
self.assertEqual(fake.terminal_environments, [{"CLAUDE_CODE_SANDBOXED": "1"}])
|
|
self.assertEqual(fake.terminal_folders, [str(self.home / "code" / "claude-config")], "the development clone, since ~/.claude holds only a release")
|
|
self.assertIn("Record what they decide in this checkout, the development clone", args[6])
|
|
self.assertIsNone(state.mark_measured(self.root, 0))
|
|
self.assertEqual(self.check(FakeEffects(git=[UNREAD]))[0], None)
|
|
self.assertNotIn(call, self.check(FakeEffects())[1])
|
|
|
|
def test_stamps_the_figures_and_the_calls_raised_when_it_measures(self):
|
|
self.chunks(5, 4)
|
|
(self.root / "routing-calls-raised.md").write_text("# Routing calls raised\n\n- 2026-10-14T10:02:00Z Reset\n")
|
|
self.check(FakeEffects(on_terminal_claude=[0], git=[UNREAD]))
|
|
figures = (self.root / "routing-figures.md").read_text()
|
|
self.assertIn(f"Measured {utc(NOW)}, at cycle 20 across", figures)
|
|
self.assertIn("that finished after the reset of 2026-10-14T10:02:00Z", figures)
|
|
self.assertIn(f"- {utc(NOW)} Explore subagents'", (self.root / "routing-calls-raised.md").read_text())
|
|
|
|
def test_says_a_figure_it_cannot_read(self):
|
|
self.chunks(5, 4)
|
|
(self.root / "routing-state.json").mkdir()
|
|
self.assertIn("routing figures: could not read", self.check(FakeEffects())[1])
|
|
|
|
def test_goes_on_off_a_terminal_and_raises_the_call_again_next_time(self):
|
|
self.chunks(5, 4)
|
|
stop, said = self.check(FakeEffects(on_terminal=False, git=[UNREAD]))
|
|
self.assertIsNone(stop)
|
|
self.assertIn("Not on a terminal: the calls are raised again at the next measurement.", said)
|
|
self.assertIsNone(state.mark_measured(self.root, 0))
|
|
self.assertIn("Not on a terminal", self.check(FakeEffects(on_terminal=False, git=[UNREAD]))[1])
|
|
|
|
def test_leaves_the_measuring_to_a_loop_holding_the_lock(self):
|
|
self.chunks(5, 4)
|
|
with archive.locked(self.root):
|
|
self.assertEqual(self.check(FakeEffects()), (None, ""))
|
|
|
|
def test_stops_when_claude_cannot_start_for_the_calls(self):
|
|
self.chunks(5, 4)
|
|
stop, _ = self.check(FakeEffects(on_terminal_claude=[OSError(2, "No such file or directory")], git=[UNREAD]))
|
|
self.assertEqual(stop, Stop(1, f"could not start claude for the routing calls in {self.home / 'code' / 'claude-config'}, the setup's development clone: No such file or directory"))
|
|
self.assertEqual(asks_and_answers(self.paths.waiting), [])
|
|
|
|
@unittest.skipIf(os.geteuid() == 0, "root writes into a read-only directory")
|
|
def test_says_a_figure_it_cannot_write_and_goes_on(self):
|
|
self.chunks(5, 4)
|
|
(self.root / "routing-figures.lock").touch()
|
|
self.root.chmod(0o555)
|
|
self.addCleanup(self.root.chmod, 0o755)
|
|
stop, said = self.check(FakeEffects(git=[UNREAD]))
|
|
self.assertIsNone(stop)
|
|
self.assertIn("routing figures: could not write", said)
|
|
|
|
def test_puts_every_new_call_to_the_human_and_says_one_it_cannot_keep(self):
|
|
new = [Call("reports", ("Explore",)), Call("rounds", ("x", "y"))]
|
|
fake = FakeEffects(on_terminal_claude=[0])
|
|
with contextlib.redirect_stderr(io.StringIO()) as said, unittest.mock.patch.object(state, "mark_raised", return_value=Failure("full")):
|
|
self.assertIsNone(routing_check.raise_calls(self.root, new, Rung("m", "high"), self.loop(fake)))
|
|
((args),) = fake.made("claude_on_terminal")
|
|
self.assertIn(f"change nothing until they answer:\n- {new[0].sentence()}\n- {new[1].sentence()}\nThe figures", args[6])
|
|
self.assertIn(f"call for a routing change:\n- {new[0].sentence()}\n- {new[1].sentence()}\n", said.getvalue())
|
|
self.assertIn("routing figures: full", said.getvalue())
|
|
|
|
|
|
class ModelPrompt(unittest.TestCase):
|
|
OFFERED = [("claude-opus-5-5", "Opus 5.5"), ("claude-haiku-4-5", "Haiku 4.5")]
|
|
|
|
def test_reads_a_number_as_the_offered_model_and_anything_else_as_a_name(self):
|
|
self.assertEqual(model_choice.model_picked(" 2 ", self.OFFERED), "claude-haiku-4-5")
|
|
self.assertEqual(model_choice.model_picked("opus", self.OFFERED), "opus")
|
|
self.assertEqual(model_choice.model_picked("3", []), "3")
|
|
self.assertEqual(model_choice.model_picked("3", self.OFFERED), Failure("pick 1 to 2"))
|
|
self.assertIsNone(model_choice.model_picked(" ", self.OFFERED))
|
|
|
|
def test_asks_again_until_an_answer_picks_a_model(self):
|
|
with tempfile.TemporaryDirectory() as empty:
|
|
fake = FakeEffects(inputs=["", "my-model"])
|
|
self.assertEqual(model_choice.ask_for_model("proj", pathlib.Path("/m.json"), pathlib.Path(empty), fake.effects()), "my-model")
|
|
self.assertEqual(len(fake.made("input")), 2)
|
|
fake = FakeEffects(inputs=[EOFError()])
|
|
self.assertEqual(model_choice.ask_for_model("proj", pathlib.Path("/m.json"), pathlib.Path(empty), fake.effects()), Stop(2, "the human is needed to pick proj's model, but the input ended"))
|
|
|
|
|
|
# A plan read with no release cut yet.
|
|
NO_CUT = LastCut(None, "", 0, [])
|
|
TODO = """# todo
|
|
|
|
## Items
|
|
|
|
| ID | Release | Exempt | Item | L | E | I | R | S | Moves | Score |
|
|
|---|---|---|---|---|---|---|---|---|---|---|
|
|
| 7 | 0.2.0 | question | **Ask first.** | 4 | 3 | 5 | 10 | 1 | 1 by 9 | 30 |
|
|
| 3 | 0.2.0 | | **Do the next thing.** | 4 | 2 | 3 | 7 | 3 | 1 by 9 | 25 |
|
|
| 1 | 0.3.0 | | **Do a later thing.** | 1 | 1 | 1 | 1 | 1 | 1 by 9 | 30 |
|
|
"""
|
|
|
|
|
|
def unfinished(likelihood=0, escalated_at=None):
|
|
"""An unfinished verdict: the worker took in an item of that L, 0 for none, and the judge found an escalation after reading escalated_at transcript entries, None for none."""
|
|
return Unfinished("e", escalation="round 2: 🔴" if escalated_at is not None else "", taken_in=TakenIn(highest_likelihood=likelihood), entries_read=escalated_at)
|
|
|
|
|
|
def interrupted(likelihood=0, escalated_at=None):
|
|
"""An interrupt, its arguments as unfinished's."""
|
|
return Interrupt("e", "q", escalation="round 2: 🔴" if escalated_at is not None else "", taken_in=TakenIn(highest_likelihood=likelihood), entries_read=escalated_at)
|
|
|
|
|
|
class Routing(unittest.TestCase):
|
|
OPUS = Rung("claude-opus-5-5", "medium")
|
|
HIGH = Rung("claude-opus-5-5", "high")
|
|
XHIGH = Rung("claude-opus-5-5", "xhigh")
|
|
MAX = Rung("claude-opus-5-5", "max")
|
|
SONNET = Rung("claude-sonnet-5-5", "high")
|
|
|
|
def test_routes_on_the_next_releases_top_item_that_waits_on_no_question(self):
|
|
self.assertEqual(routing.routed_item(TODO, NO_CUT).id, 3)
|
|
self.assertIsNone(routing.routed_item(TODO.replace("| | **Do", "| question | **Do"), NO_CUT))
|
|
self.assertIsNone(routing.routed_item("# todo\n", NO_CUT))
|
|
self.assertIsNone(routing.routed_item(TODO[:TODO.index("| 7 |")], NO_CUT))
|
|
old_shape = "## Items\n\n| ID | Release | Exempt | Item | R | S | C | A | G | Goals | Score |\n|---|---|---|---|---|---|---|---|---|---|---|\n| 3 | 0.2.0 | | **Do the next thing.** | 5 | 3 | 6 | 8 | 9 | 1 | 25 |\n"
|
|
self.assertIsNone(routing.routed_item(old_shape, NO_CUT))
|
|
self.assertEqual(routing.routed_item(TODO.replace("| 4 | 2 | 3 |", "| x | 2 | 3 |"), NO_CUT), Failure("item 3's L 'x' is not a whole number from 1 to 4"))
|
|
|
|
def test_routes_on_no_item_where_the_release_after_the_last_cut_has_none(self):
|
|
self.assertEqual(routing.routed_item(TODO, LastCut("v0.1.0", "", 3, [TODO])).id, 3)
|
|
self.assertIsNone(routing.routed_item(TODO, LastCut("v0.2.0", "", 3, [TODO])))
|
|
later_only = TODO[:TODO.index("| 7 |")] + TODO[TODO.index("| 1 | 0.3.0 |"):]
|
|
self.assertIsNone(routing.routed_item(later_only, LastCut("v0.1.0", "", 3, [TODO])))
|
|
self.assertIsNone(routing.routed_item(later_only, LastCut("v0.1.0", "", 0, [TODO])))
|
|
self.assertEqual(routing.routed_item(later_only, LastCut("v0.2.0", "", 3, [later_only])).id, 1)
|
|
|
|
def test_routes_past_an_item_a_waiting_item_blocks(self):
|
|
todo = "## Items\n\n| ID | Release | Blocked by | Exempt | Item | L | E | I | R | S | Moves | Score |\n|---|---|---|---|---|---|---|---|---|---|---|---|\n| 7 | 0.2.0 | | question | **Ask first.** | 4 | 3 | 5 | 10 | 1 | 1 by 9 | 30 |\n| 8 | 0.2.0 | 7 | | **Use the answer.** | 1 | 1 | 1 | 1 | 1 | 1 by 9 | 30 |\n| 9 | 0.2.0 | 8 | | **Build on it.** | 1 | 1 | 1 | 1 | 1 | 1 by 9 | 30 |\n| 3 | 0.2.0 | | | **Do the next thing.** | 4 | 2 | 3 | 7 | 3 | 1 by 9 | 25 |\n"
|
|
self.assertEqual(routing.routed_item(todo, NO_CUT).id, 3)
|
|
self.assertEqual(routing.routed_item(todo.replace("| question |", "| |"), NO_CUT).id, 7)
|
|
first = "| 7 | 0.2.0 | | question | **Ask first.** | 4 | 3 | 5 | 10 | 1 | 1 by 9 | 30 |\n"
|
|
self.assertEqual(routing.routed_item(todo.replace(first, "") + first, NO_CUT).id, 3)
|
|
|
|
def test_starts_a_chunk_on_the_rung_its_items_l_picks_below_the_projects_too(self):
|
|
self.assertEqual(routing.first_route(self.OPUS, routing.routed_item(TODO, NO_CUT)), Route(self.HIGH, 3, 4, 5))
|
|
cheap = routing.routed_item(TODO.replace("| 4 | 2 | 3 |", "| 1 | 2 | 3 |"), NO_CUT)
|
|
self.assertEqual(routing.first_route(self.OPUS, cheap), Route(Rung("claude-sonnet-5-5", "high"), 3, 1, 5))
|
|
self.assertEqual(routing.first_route(self.OPUS, None), Route(self.OPUS))
|
|
|
|
def test_steps_up_a_rung_after_every_second_unfinished_verdict(self):
|
|
once = routing.next_route(self.OPUS, Route(self.OPUS, 3, 3, 4), unfinished())
|
|
self.assertEqual(once, Route(self.OPUS, 3, 3, 4, unfinished=1))
|
|
self.assertEqual(routing.next_route(self.OPUS, once, unfinished()), Route(self.HIGH, 3, 3, 4))
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(LADDER[-1], unfinished=1), unfinished()).rung, LADDER[-1])
|
|
|
|
def test_steps_up_to_the_rung_a_taken_in_items_l_picks_and_never_down(self):
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.OPUS), unfinished(4)).rung, self.HIGH)
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.XHIGH), unfinished(4)).rung, self.XHIGH)
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.OPUS, unfinished=1), unfinished(4)).rung, self.XHIGH)
|
|
|
|
def test_steps_up_from_a_rung_below_the_projects_one_rung_at_a_time(self):
|
|
low = Route(self.SONNET, 3, 1, 3, unfinished=1)
|
|
self.assertEqual(routing.next_route(self.OPUS, low, unfinished()).rung, self.OPUS)
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(low.rung, 3, 1, 3), unfinished()).rung, low.rung)
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(low.rung, 3, 1, 3), unfinished(3)).rung, self.OPUS)
|
|
|
|
def test_steps_up_a_rung_on_an_escalation_the_judge_found_up_to_two_above_the_items_rung_marking_the_transcript_entries_it_read(self):
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.HIGH, 3, 4, 5), unfinished(escalated_at=7)), Route(self.XHIGH, 3, 4, 5, escalated_at=7))
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.XHIGH, 3, 4, 5, escalated_at=7), unfinished(escalated_at=9)), Route(self.MAX, 3, 4, 5, escalated_at=9))
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.MAX, 3, 4, 5, escalated_at=9), unfinished(escalated_at=12)), Route(self.MAX, 3, 4, 5, unfinished=1, escalated_at=9))
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.HIGH, 3, 4, 5, unfinished=1), unfinished(escalated_at=7)), Route(self.XHIGH, 3, 4, 5, escalated_at=7))
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.XHIGH), unfinished(escalated_at=7)), Route(self.XHIGH, unfinished=1))
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.SONNET, 3, 2, 4), unfinished(4, escalated_at=7)), Route(self.HIGH, 3, 2, 4, unfinished=1))
|
|
|
|
def test_keeps_the_mark_through_a_step_no_escalation_took(self):
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.OPUS, 3, 3, 4, unfinished=1, escalated_at=5), unfinished()), Route(self.HIGH, 3, 3, 4, escalated_at=5))
|
|
|
|
def test_steps_up_a_rung_as_the_judge_interrupts_up_to_two_above_the_items_rung_without_counting_an_unfinished_verdict(self):
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.HIGH, 3, 4, 5, unfinished=1), interrupted(escalated_at=7)), Route(self.XHIGH, 3, 4, 5, escalated_at=7))
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.XHIGH, 3, 4, 5, escalated_at=7), interrupted(escalated_at=9)), Route(self.MAX, 3, 4, 5, escalated_at=9))
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.MAX, 3, 4, 5, unfinished=1, escalated_at=9), interrupted(escalated_at=12)), Route(self.MAX, 3, 4, 5, unfinished=1, escalated_at=9))
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.SONNET, 3, 2, 4), interrupted(escalated_at=7)), Route(self.OPUS, 3, 2, 4, escalated_at=7))
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.HIGH, 3, 4, 5, unfinished=1, escalated_at=2), interrupted(4)), Route(self.HIGH, 3, 4, 5, unfinished=1, escalated_at=2))
|
|
|
|
def test_raises_the_worker_to_a_taken_in_items_rung_above_the_escalation_cap_without_a_step_as_the_judge_interrupts(self):
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.OPUS, 3, 1, 3, escalated_at=2), interrupted(4, escalated_at=7)), Route(self.HIGH, 3, 1, 3, escalated_at=2))
|
|
|
|
def test_steps_up_from_the_rung_a_taken_in_item_picks_as_the_judge_interrupts(self):
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.OPUS, 3, 3, 4), interrupted(4, escalated_at=7)), Route(self.XHIGH, 3, 3, 4, escalated_at=7))
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.XHIGH, 3, 4, 5), interrupted(4, escalated_at=7)), Route(self.MAX, 3, 4, 5, escalated_at=7))
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(self.MAX, 3, 4, 5, unfinished=1), interrupted(4, escalated_at=7)), Route(self.MAX, 3, 4, 5, unfinished=1))
|
|
|
|
def test_keeps_an_e_plus_i_taken_in_only_where_it_picks_a_higher_review_rung(self):
|
|
taken = Route(self.OPUS, 3, 3, 4).taking_in(6)
|
|
self.assertEqual(taken, Route(self.OPUS, 3, 3, 4, escape_and_impact_taken_in=6))
|
|
self.assertEqual(taken.taking_in(7), Route(self.OPUS, 3, 3, 4, escape_and_impact_taken_in=7))
|
|
self.assertEqual(taken.taking_in(5), taken)
|
|
self.assertEqual(taken.taking_in(6), taken)
|
|
self.assertEqual(Route(self.OPUS, 3, 3, 4).taking_in(4), Route(self.OPUS, 3, 3, 4))
|
|
self.assertEqual(Route(self.OPUS, 3, 2, 2).taking_in(4), Route(self.OPUS, 3, 2, 2))
|
|
self.assertEqual(Route(self.OPUS).taking_in(5), Route(self.OPUS, escape_and_impact_taken_in=5))
|
|
self.assertEqual(Route(self.OPUS).taking_in(3), Route(self.OPUS))
|
|
self.assertEqual(Route(self.OPUS).taking_in(0), Route(self.OPUS))
|
|
|
|
def test_a_rung_an_edit_took_off_the_ladder_counts_as_the_projects(self):
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(Rung("claude-opus-5-5", "low")), unfinished()).rung, self.OPUS)
|
|
self.assertEqual(routing.next_route(self.OPUS, Route(Rung("claude-opus-5-5", "low")), unfinished(2)).rung, self.OPUS)
|
|
|
|
def test_runs_the_judge_on_the_rung_its_chunks_items_e_plus_i_picks_never_below_the_projects(self):
|
|
self.assertEqual(routing.judge_rung("claude-opus-5-5", Route(self.OPUS, 3, 3, 5)), self.HIGH)
|
|
self.assertEqual(routing.judge_rung("claude-opus-5-5", Route(self.XHIGH, 3, 4, 5)), self.HIGH)
|
|
self.assertEqual(routing.judge_rung("claude-opus-5-5", Route(self.SONNET, 3, 2, 7)), self.MAX)
|
|
self.assertEqual(routing.judge_rung("claude-opus-5-5", Route(Rung("claude-haiku-5-5", "max"), 3, 1, 3)), self.OPUS)
|
|
self.assertEqual(routing.judge_rung("claude-opus-5-5", Route(self.HIGH)), self.OPUS)
|
|
self.assertEqual(routing.judge_rung("judge-model", Route(self.HIGH, 3, 4, 5)), Rung("judge-model"))
|
|
self.assertEqual(routing.judge_rung("claude-sonnet-5-5", Route(self.HIGH, 3, 4, 5)), Rung("claude-sonnet-5-5"))
|
|
|
|
def test_runs_the_judge_on_the_rung_the_highest_e_plus_i_taken_in_picks(self):
|
|
self.assertEqual(routing.judge_rung("claude-opus-5-5", Route(self.OPUS, 3, 3, 4, escape_and_impact_taken_in=6)), self.XHIGH)
|
|
self.assertEqual(routing.judge_rung("claude-opus-5-5", Route(self.OPUS, escape_and_impact_taken_in=5)), self.HIGH)
|
|
|
|
def test_refuses_a_route_whose_item_has_levels_off_their_scales_or_levels_without_an_item(self):
|
|
for item, likelihood, escape_and_impact in ((3, 0, 5), (3, 5, 5), (3, 4, 1), (3, 4, 9), (0, 4, 5), (0, 4, 0), (0, 0, 5)):
|
|
with self.subTest(item=item, likelihood=likelihood, escape_and_impact=escape_and_impact):
|
|
with self.assertRaises(ValueError):
|
|
Route(self.HIGH, item, likelihood, escape_and_impact)
|
|
|
|
def test_refuses_a_route_taking_in_an_e_plus_i_off_its_scale_or_not_above_its_items(self):
|
|
for taken_in in (1, 9, -1, 5, 4):
|
|
with self.subTest(taken_in=taken_in):
|
|
with self.assertRaises(ValueError):
|
|
Route(self.HIGH, 3, 4, 5, escape_and_impact_taken_in=taken_in)
|
|
|
|
def test_takes_a_route_whose_item_holds_each_levels_lowest_value(self):
|
|
self.assertEqual(Route(self.HIGH, 3, 1, 2).escape_and_impact, 2)
|
|
|
|
def test_routes_nothing_on_an_item_without_e_or_i(self):
|
|
without_i = TODO.replace("| E | I | R |", "| E | R |").replace("|---|---|---|---|---|---|---|---|---|---|---|", "|---|---|---|---|---|---|---|---|---|---|").replace("| 4 | 3 | 5 | 10 |", "| 4 | 3 | 10 |").replace("| 4 | 2 | 3 | 7 |", "| 4 | 2 | 7 |").replace("| 1 | 1 | 1 | 1 |", "| 1 | 1 | 1 |")
|
|
self.assertIsNone(routing.routed_item(without_i, NO_CUT))
|
|
|
|
def test_archives_the_route_as_a_json_worker_line_it_reads_back_field_by_field(self):
|
|
route = Route(self.HIGH, 3, 4, 5)
|
|
self.assertEqual(route.worker_line(), '{"model": "claude-opus-5-5", "effort": "high", "item": 3, "likelihood": 4, "escape_and_impact": 5}')
|
|
self.assertEqual(route_from(route.worker_line()), route)
|
|
self.assertEqual(Route(Rung("worker-model")).worker_line(), '{"model": "worker-model"}')
|
|
self.assertEqual(route_from('{"model": "worker-model"}'), Route(Rung("worker-model")))
|
|
marked = Route(self.HIGH, 3, 4, 5, escape_and_impact_taken_in=7, escalated_at=12)
|
|
self.assertEqual(marked.worker_line(), '{"model": "claude-opus-5-5", "effort": "high", "item": 3, "likelihood": 4, "escape_and_impact": 5, "escape_and_impact_taken_in": 7, "escalated_at": 12}')
|
|
self.assertEqual(route_from(marked.worker_line()), marked)
|
|
self.assertEqual(route_from(Route(self.HIGH, escalated_at=3).worker_line()), Route(self.HIGH, escalated_at=3))
|
|
|
|
def test_leaves_the_count_of_unfinished_verdicts_out_of_the_worker_line_so_a_rerun_restarts_it(self):
|
|
self.assertEqual(route_from(Route(self.HIGH, 3, 4, 5, unfinished=1).worker_line()), Route(self.HIGH, 3, 4, 5))
|
|
|
|
def test_reads_no_route_from_a_worker_line_that_route_worker_line_could_not_have_written(self):
|
|
for line in ('{"effort": "high"}', '{"model": "claude-opus-5-5", "effort": "high", "unfinished": 1}', '{"model": "claude-opus-5-5", "item": "3"}', '{"model": "claude-opus-5-5", "effort": 3}', '{"model": 5}', '{"model": "claude-opus-5-5", "effort": "high", "escalated_at": -3}', '{"model": "claude-opus-5-5", "item": true, "likelihood": 4, "escape_and_impact": 5}', '{"model": "claude-opus-5-5", "effort": "high", "item": 3, "likelihood": 5, "escape_and_impact": 5}', '{"model": "claude-opus-5-5", "effort": "high", "escape_and_impact_taken_in": 9}', '["claude-opus-5-5"]', '{"model": ', "worker-model", "claude-opus-5-5 at high effort, routed by item 3's L of 4 and E + I of 5"):
|
|
with self.subTest(line=line):
|
|
self.assertIsNone(route_from(line))
|
|
|
|
def test_keeps_a_chunks_route_or_its_archived_one_at_or_above_the_rung_its_item_picks_and_else_routes_afresh(self):
|
|
item = routing.routed_item(TODO, NO_CUT)
|
|
archived = Route(self.XHIGH, 3, 4, 5)
|
|
def never():
|
|
raise AssertionError("not asked")
|
|
self.assertEqual(chunk_start.routed(archived, None, "worker-model", never, never), Route(Rung("worker-model")))
|
|
self.assertEqual(chunk_start.routed(archived, self.OPUS, "claude-opus-5-5", never, never), archived)
|
|
self.assertEqual(chunk_start.routed(None, self.OPUS, "claude-opus-5-5", lambda: archived, never), archived)
|
|
self.assertEqual(chunk_start.routed(None, self.OPUS, "claude-opus-5-5", lambda: None, lambda: item), Route(self.HIGH, 3, 4, 5))
|
|
self.assertEqual(chunk_start.routed(Route(self.SONNET), self.OPUS, "claude-opus-5-5", never, lambda: None), Route(self.OPUS))
|
|
cheap = Route(self.SONNET, 4, 2, 3)
|
|
self.assertEqual(chunk_start.routed(cheap, self.OPUS, "claude-opus-5-5", never, never), cheap)
|
|
self.assertEqual(chunk_start.routed(Route(Rung("claude-haiku-5-5", "max"), 4, 2, 3), self.OPUS, "claude-opus-5-5", never, lambda: None), Route(self.OPUS))
|
|
|
|
|
|
class RouteSources(unittest.TestCase):
|
|
def test_routes_on_the_plan_item_and_says_why_it_routes_on_the_projects_rung(self):
|
|
self.assertIsNone(chunk_start.plan_item(DefaultPlan("", NO_CUT), FakeEffects().effects()))
|
|
self.assertEqual(chunk_start.plan_item(DefaultPlan(TODO, LastCut("v0.1.0", "", 3, [TODO])), FakeEffects().effects()).id, 3)
|
|
self.assertIsNone(chunk_start.plan_item(DefaultPlan(TODO, LastCut("v0.2.0", "", 3, [TODO])), FakeEffects().effects()))
|
|
unreadable = Failure("git show origin/main:todo.md printed text that is not UTF-8")
|
|
for plan, said in ((DefaultPlan(TODO.replace("| 4 | 2 | 3 |", "| x | 2 | 3 |"), NO_CUT), "routing on the project's rung: item 3's L 'x' is not a whole number from 1 to 4\n"), (DefaultPlan(unreadable, NO_CUT), f"routing on the project's rung: {unreadable.reason}\n"), (Failure("no origin remote"), "routing on the project's rung: no origin remote\n")):
|
|
with self.subTest(said=said):
|
|
with contextlib.redirect_stderr(io.StringIO()) as stderr:
|
|
self.assertIsNone(chunk_start.plan_item(plan, FakeEffects().effects()))
|
|
self.assertTrue(stderr.getvalue().startswith(f"{STAMP} {said}"), stderr.getvalue())
|
|
|
|
def test_resumes_on_the_archived_route_and_says_why_it_routes_afresh(self):
|
|
with tempfile.TemporaryDirectory() as home:
|
|
opened = session.session_named("A proj loop-claude", pathlib.Path(home, "projects"))
|
|
archive = pathlib.Path(home, "archive")
|
|
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
|
|
route = Route(Rung("claude-opus-5-5", "high"), 3, 4, 5)
|
|
with contextlib.redirect_stderr(io.StringIO()) as stderr:
|
|
self.assertIsNone(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)))
|
|
transcript.parent.mkdir(parents=True)
|
|
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
|
self.assertIsNone(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)))
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(route.worker_line(), "working", "j", ""), NOW))
|
|
with contextlib.redirect_stderr(stderr):
|
|
self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), route)
|
|
self.assertEqual(stderr.getvalue(), "")
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.stepped_up(WorkerLine('{"model": "claude-opus-5-5", "rung": 3}')), NOW + 1))
|
|
with contextlib.redirect_stderr(io.StringIO()) as stderr:
|
|
self.assertIsNone(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)))
|
|
self.assertEqual(stderr.getvalue(), f'{STAMP} routing the chunk afresh: the archive\'s latest Worker: line names no route loop-claude can read: {{"model": "claude-opus-5-5", "rung": 3}}\n')
|
|
(chunk,) = archive.iterdir()
|
|
chunk.chmod(0)
|
|
try:
|
|
with contextlib.redirect_stderr(io.StringIO()) as stderr:
|
|
self.assertIsNone(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)))
|
|
finally:
|
|
chunk.chmod(0o700)
|
|
self.assertTrue(stderr.getvalue().startswith(f"{STAMP} routing the chunk afresh: could not read the archive"), stderr.getvalue())
|
|
|
|
|
|
class Steps(unittest.TestCase):
|
|
OPUS = Rung("claude-opus-5-5", "medium")
|
|
|
|
def test_a_finished_verdict_clears_the_route_and_the_evidence(self):
|
|
run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="old", route=Route(self.OPUS))
|
|
self.assertEqual(close.after_verdict(run_with, self.OPUS, run.Concluded(Finished("x"), run_with.route)), dataclasses.replace(run_with, route=None, unfinished_evidence=""))
|
|
self.assertEqual(close.after_verdict(dataclasses.replace(run_with, unfinished_run=3), self.OPUS, run.Concluded(Finished("x"), run_with.route)).unfinished_run, 0)
|
|
self.assertEqual(close.after_verdict(run_with, self.OPUS, run.Concluded(Unfinished("new", taken_in=TakenIn(highest_likelihood=4)), run_with.route)), dataclasses.replace(run_with, route=Route(Rung("claude-opus-5-5", "high"), unfinished=1), unfinished_evidence="new", unfinished_run=1))
|
|
|
|
def test_an_unfinished_chunk_keeps_the_items_its_verdict_found_taken_in_and_a_finished_one_drops_them(self):
|
|
run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(self.OPUS), items_taken_in=(3,))
|
|
self.assertEqual(close.after_verdict(run_with, self.OPUS, run.Concluded(Unfinished("new", taken_in=TakenIn((3, 12))), run_with.route)).items_taken_in, (3, 12))
|
|
self.assertEqual(close.after_verdict(run_with, self.OPUS, run.Concluded(Finished("x"), run_with.route)).items_taken_in, ())
|
|
|
|
def test_steps_from_the_route_its_cycle_ended_on_which_a_ruling_may_have_raised(self):
|
|
run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(self.OPUS, 3, 3, 4))
|
|
raised = Route(Rung("claude-opus-5-5", "high"), 3, 3, 4, escalated_at=5)
|
|
self.assertEqual(close.after_verdict(run_with, self.OPUS, run.Concluded(Unfinished("new"), raised)).route, dataclasses.replace(raised, unfinished=1))
|
|
|
|
def test_keeps_the_route_its_cycle_ended_on_off_the_ladder(self):
|
|
off_ladder = Route(Rung("worker-model"))
|
|
run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="")
|
|
self.assertEqual(close.after_verdict(run_with, None, run.Concluded(Unfinished("new", "round 2: 🔴", TakenIn(highest_likelihood=4), entries_read=9), off_ladder)).route, off_ladder)
|
|
|
|
def test_an_unfinished_verdicts_escalation_marks_the_transcript_entries_the_judge_read(self):
|
|
run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(self.OPUS, 3, 3, 4, escalated_at=2))
|
|
self.assertEqual(close.after_verdict(run_with, self.OPUS, run.Concluded(Unfinished("new", "round 2: 🔴", entries_read=9), run_with.route)).route, Route(Rung("claude-opus-5-5", "high"), 3, 3, 4, escalated_at=9))
|
|
self.assertEqual(close.after_verdict(run_with, self.OPUS, run.Concluded(Unfinished("new", entries_read=9), run_with.route)).route, Route(self.OPUS, 3, 3, 4, unfinished=1, escalated_at=2))
|
|
|
|
def test_archives_a_step_up_as_it_happens_so_a_rerun_resumes_on_it(self):
|
|
with tempfile.TemporaryDirectory() as home:
|
|
opened = session.session_named("A proj loop-claude", pathlib.Path(home, "projects"))
|
|
archive = pathlib.Path(home, "archive")
|
|
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
|
|
transcript.parent.mkdir(parents=True)
|
|
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
|
before = Route(self.OPUS, 3, 3, 4)
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW))
|
|
after = close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), self.OPUS, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), archive, a_loop(FakeEffects().effects(), session=opened))
|
|
self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(Rung("claude-opus-5-5", "high"), 3, 3, 4))
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
close.closed_cycle(after, self.OPUS, run.Concluded(Finished("done"), after.route), archive, a_loop(FakeEffects().effects(), session=opened))
|
|
self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(Rung("claude-opus-5-5", "high"), 3, 3, 4))
|
|
blocked = pathlib.Path(home, "blocked")
|
|
blocked.write_text("")
|
|
stop = close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), self.OPUS, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), blocked, a_loop(FakeEffects().effects(), session=opened))
|
|
self.assertEqual((stop.code, stop.reason.split(":")[0]), (1, "could not archive the transcript"))
|
|
|
|
def test_names_no_step_up_where_only_the_count_of_unfinished_verdicts_changed(self):
|
|
route = Route(self.OPUS, 3, 3, 4)
|
|
self.assertIsNone(routing.step_up(route, dataclasses.replace(route, unfinished=1)))
|
|
self.assertEqual(routing.step_up(route, route.taking_in(6)), routing.StepUp(route, route.taking_in(6)))
|
|
|
|
def test_archives_a_higher_e_plus_i_taken_in_so_a_rerun_keeps_it(self):
|
|
with tempfile.TemporaryDirectory() as home:
|
|
opened = session.session_named("A proj loop-claude", pathlib.Path(home, "projects"))
|
|
archive = pathlib.Path(home, "archive")
|
|
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
|
|
transcript.parent.mkdir(parents=True)
|
|
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
|
before = Route(self.OPUS, 3, 3, 4)
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW))
|
|
close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), self.OPUS, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_escape_and_impact=6)), before), archive, a_loop(FakeEffects().effects(), session=opened))
|
|
self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.OPUS, 3, 3, 4, escape_and_impact_taken_in=6))
|
|
|
|
|
|
class ChunkFacts(unittest.TestCase):
|
|
def test_names_the_routed_item_then_each_taken_in_once_by_its_tagline_on_the_default_branch(self):
|
|
self.assertEqual(running_loops.taken_items(TODO, 3, (99, 3, 99)), (Taken(3, "Do the next thing."), Taken(99, "")))
|
|
self.assertEqual(running_loops.taken_items(TODO, 0, (3,)), (Taken(3, "Do the next thing."),))
|
|
self.assertEqual(running_loops.taken_items(Failure("no todo.md"), 3, ()), (Taken(3, ""),))
|
|
self.assertEqual(running_loops.taken_items(TODO, 0, ()), ())
|
|
|
|
# The launcher beside which chunk_start finds /b/chunk-preflight.
|
|
LAUNCHER = pathlib.Path("/b/loop-claude")
|
|
|
|
def test_runs_the_preflight_at_the_checkout_root_and_quotes_what_it_printed(self):
|
|
fake = FakeEffects(python=[ran("- A fact.\n")])
|
|
said = chunk_start.chunk_facts(a_loop(fake.effects(), launcher=self.LAUNCHER))
|
|
self.assertEqual(fake.made("python"), [(["/b/chunk-preflight", "--worktree"], "/r")])
|
|
self.assertIn("\n\n- A fact.", said)
|
|
self.assertNotIn("yourself", said)
|
|
|
|
def test_has_the_worker_find_the_facts_itself_where_the_preflight_failed(self):
|
|
for failed, why in ((ran(code=1, stderr="git could not start\n"), "git could not start"), (ran(code=1), "exited 1")):
|
|
with self.subTest(why=why):
|
|
said = chunk_start.chunk_facts(a_loop(FakeEffects(python=[failed]).effects(), launcher=self.LAUNCHER))
|
|
self.assertIn("yourself", said)
|
|
self.assertIn(why, said)
|
|
|
|
def test_runs_the_preflight_only_where_the_session_is_not_open(self):
|
|
with tempfile.TemporaryDirectory() as home:
|
|
opened = session.session_named("A proj loop-claude", pathlib.Path(home, "projects"))
|
|
fresh = FakeEffects(python=[ran("- A fact.\n")])
|
|
facts, _ = chunk_start.chunk_start(a_loop(fresh.effects(), session=opened, launcher=self.LAUNCHER))
|
|
self.assertEqual((fresh.made("python"), "- A fact." in facts), ([(["/b/chunk-preflight", "--worktree"], "/r")], True))
|
|
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
|
|
transcript.parent.mkdir(parents=True)
|
|
transcript.write_text("")
|
|
resumed = FakeEffects()
|
|
facts, _ = chunk_start.chunk_start(a_loop(resumed.effects(), session=opened, launcher=self.LAUNCHER))
|
|
self.assertEqual((resumed.made("python"), facts), ([], ""))
|
|
|
|
|
|
class TypedAnswers(unittest.TestCase):
|
|
def test_splits_the_prompt_handing_over_the_answer_into_the_answer_on_the_line_after_its_marker_and_the_rest_and_no_other_prompt(self):
|
|
route = Route(Rung("worker-model"))
|
|
marked = "The human answered your question at loop-claude's terminal:\nB"
|
|
typed = prompts.typed_answer(prompts.typed("Which one?", "Use B.", route, None, False))
|
|
self.assertEqual(typed.answer, "Use B.")
|
|
self.assertTrue(typed.loop_words.startswith("Your question was: Which one?\n\nContinue the task."))
|
|
self.assertEqual(prompts.typed_answer(prompts.typed(f"Which one?\n{marked}", "Use B.", route, None, False)).answer, "Use B.")
|
|
self.assertEqual(prompts.typed_answer(prompts.typed("Which one?", "Use B.", Route(Routing.SONNET, 179, 2, 5), Routing.OPUS, True)).answer, "Use B.")
|
|
self.assertIsNone(prompts.typed_answer(prompts.answered("Which one?", marked, route, None, False)))
|
|
self.assertIsNone(prompts.typed_answer(prompts.resumed(marked, route, None)))
|
|
|
|
|
|
class WorkerArgs(unittest.TestCase):
|
|
SCAN = " Run a full scan only from such a subagent, since the scan's agents run on the model that starts them."
|
|
SEATS = " Launch each comprehension seat and the security pass's subagent with the Agent tool set to model opus, effort medium, the project's rung."
|
|
MARKDOWN = " Where your branch changes only Markdown, none of it `CLAUDE.md`, `AGENTS.md`, `workflow-bench/*-prompt.md` or under `agents/`, `principles/`, `rules/` or `skills/`, launch each reviewer on the project's rung instead, with the Agent tool set to model opus, effort medium."
|
|
# What the preflight found, as the fresh prompt quotes it.
|
|
FACTS = "- A preflight fact."
|
|
|
|
def test_names_the_route_in_a_fresh_prompt_only_where_an_item_routed_it(self):
|
|
routed = prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), None, self.FACTS)
|
|
self.assertIn(f"Continue any open draft PR or worktree for the chunk before creating a new one. {prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\nloop-claude runs you on m at high effort, routed by item 3's L of 4.", routed)
|
|
unrouted = prompts.fresh("Do it.", "## Goals", Route(Rung("m")), None, self.FACTS)
|
|
self.assertIn(f"before creating a new one. {prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\n{self.FACTS}\n\nThe chunk's", unrouted)
|
|
self.assertNotIn("loop-claude runs you on", unrouted)
|
|
|
|
def test_has_the_routed_worker_count_its_items_size_again_and_puts_the_facts_before_the_scope(self):
|
|
routed = prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), None, self.FACTS)
|
|
self.assertIn("item 3's size again", routed)
|
|
self.assertIn("escalation trigger 1", routed)
|
|
self.assertIn(f"\n\n{self.FACTS}\n\nThe chunk's", routed)
|
|
self.assertNotIn("size again", prompts.fresh("Do it.", "## Goals", Route(Rung("m")), None, self.FACTS))
|
|
|
|
def test_gives_each_fresh_worker_the_reading_rule_once(self):
|
|
rule = "never read more than 80 lines of a file at once, nor take a tool output over 3,000 characters into your context"
|
|
for route in (Route(Rung("m", "high"), 3, 4, 5), Route(Rung("m"))):
|
|
with self.subTest(route=route):
|
|
self.assertEqual(prompts.fresh("Do it.", "## Goals", route, None, self.FACTS).count(rule), 1)
|
|
|
|
def test_quotes_what_the_preflight_printed_or_has_the_worker_find_the_facts_itself(self):
|
|
self.assertIn("\n\n- A fact.", prompts.preflight_facts("- A fact."))
|
|
failed = prompts.preflight_failed("git could not start")
|
|
self.assertIn("yourself", failed)
|
|
self.assertIn("git could not start", failed)
|
|
|
|
def test_has_each_fresh_worker_continue_an_open_draft_pr_or_worktree_once_whatever_the_preflight_found(self):
|
|
for facts in (prompts.preflight_facts("- A fact."), prompts.preflight_failed("git could not start")):
|
|
with self.subTest(facts=facts):
|
|
self.assertEqual(prompts.fresh("Do it.", "## Goals", Route(Rung("m")), None, facts).count("Continue any open draft PR or worktree"), 1)
|
|
|
|
def test_keeps_every_review_on_the_projects_rung_while_the_worker_runs_below_it(self):
|
|
opus = Rung("claude-opus-5-5", "medium")
|
|
below = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 4), opus, self.FACTS)
|
|
self.assertIn(f"item 3's L of 2. Launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool set to model opus, effort medium, the project's rung.{self.SCAN} Your rung is below the project's: launch each subagent that files or scores todo.md items on the rung `~/.claude/bin/subagent-rung --plan claude-opus-5-5 <L>` prints. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model opus, effort high` from 5, `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8. When an escalation trigger", below)
|
|
self.assertNotIn("Launch each", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS))
|
|
|
|
def test_launches_reviewers_on_the_rung_the_items_e_plus_i_picks_above_the_workers(self):
|
|
opus = Rung("claude-opus-5-5", "medium")
|
|
level = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 6), opus, self.FACTS)
|
|
self.assertIn(f"item 3's L of 3. Launch each reviewer with the Agent tool set to model opus, effort xhigh, the rung item 3's E + I of 6 picks. Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort max` from 7 and `model fable, effort max` from 8.{self.MARKDOWN} When an escalation trigger", level)
|
|
below = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 5), opus, self.FACTS)
|
|
self.assertIn(f"item 3's L of 2. Launch each reviewer with the Agent tool set to model opus, effort high, the rung item 3's E + I of 5 picks.{self.SEATS}{self.SCAN} Your rung is below the project's:", below)
|
|
self.assertNotIn("Launch each reviewer", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 4, 6), opus))
|
|
|
|
def test_names_the_reviewers_rung_each_higher_e_plus_i_an_item_taken_in_picks(self):
|
|
opus = Rung("claude-opus-5-5", "medium")
|
|
taking_in = " Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model opus, effort high` from 5, `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8."
|
|
self.assertIn(f"item 3's L of 3.{taking_in} When an escalation trigger", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS))
|
|
self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort max` from 7 and `model fable, effort max` from 8.", prompts.resumed("", Route(opus, 3, 3, 6), opus))
|
|
self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort max` from 7 and `model fable, effort max` from 8.", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 4, 4), opus))
|
|
self.assertNotIn("Where an item you take into", prompts.resumed("", Route(opus, 3, 3, 8), opus))
|
|
self.assertNotIn("Where an item you take into", prompts.resumed("", Route(Rung("m")), None))
|
|
|
|
def test_launches_reviewers_on_the_rung_the_highest_e_plus_i_taken_in_picks(self):
|
|
opus = Rung("claude-opus-5-5", "medium")
|
|
self.assertIn(f"loop-claude runs you on claude-opus-5-5 at medium effort. Launch each reviewer with the Agent tool set to model opus, effort max, the rung picked by an item of E + I 7 taken into the chunk. Where an item you take into the chunk has an E + I of 8 or more, launch each reviewer with the Agent tool set to `model fable, effort max` from 8.{self.MARKDOWN} When an escalation trigger", prompts.resumed("", Route(opus, 3, 3, 5, escape_and_impact_taken_in=7), opus))
|
|
self.assertIn(" Launch each reviewer with the Agent tool set to model opus, effort high, the rung picked by an item of E + I 5 taken into the chunk.", prompts.resumed("", Route(opus, escape_and_impact_taken_in=5), opus))
|
|
|
|
def test_keeps_seats_and_the_security_pass_on_the_projects_rung_while_the_worker_runs_above_it(self):
|
|
opus = Rung("claude-opus-5-5", "medium")
|
|
above = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-opus-5-5", "high"), 3, 4, 4), opus, self.FACTS)
|
|
stepped_up = prompts.resumed("", Route(Rung("claude-opus-5-5", "high"), 3, 3, 4), opus)
|
|
self.assertIn(f"{prompts.ANSWERS_RECORDED}. loop-claude runs you on claude-opus-5-5 at high effort.{self.SEATS}{self.SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8.{self.MARKDOWN} When an escalation trigger", stepped_up)
|
|
self.assertNotIn("the project's", prompts.resumed("", Route(opus, 3, 3, 4), opus))
|
|
self.assertNotIn("runs you on", prompts.resumed("", Route(Rung("m")), None))
|
|
self.assertIn(f"item 3's L of 4.{self.SEATS}{self.SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8.{self.MARKDOWN} When an escalation trigger", above)
|
|
|
|
def test_names_the_rungs_an_escalation_trigger_lets_the_worker_hand_a_task_to_at_most_two_above_its_items(self):
|
|
opus = Rung("claude-opus-5-5", "medium")
|
|
fresh = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS)
|
|
self.assertIn("item 3's L of 3. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model opus, effort high` from 5, `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort high`, one rung above yours, and on each further trigger one rung higher, up to `model opus, effort xhigh`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", fresh)
|
|
self.assertIn(" When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort xhigh`, one rung above yours; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", prompts.resumed("", Route(Rung("claude-opus-5-5", "high"), 3, 3, 4), opus))
|
|
self.assertIn(" An escalation trigger in `~/.claude/CLAUDE.md` lets you hand the hard part of a task to a subagent with the Agent tool set to `model opus, effort max`, one rung above yours, only once the human agrees; loop-claude will not resume you higher for a trigger either, so ask the human for that yes. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 3, 4), opus))
|
|
self.assertNotIn("escalation", prompts.resumed("", Route(Rung("claude-fable-5-1", "max"), 3, 4, 8), opus))
|
|
self.assertIn("only once the human agrees", prompts.resumed("", Route(Rung("claude-opus-5-5", "max"), 3, 4, 5), opus))
|
|
self.assertIn("hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort max`, one rung above yours;", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 4, 5), opus))
|
|
self.assertNotIn("escalation", prompts.resumed("", Route(Rung("m")), None))
|
|
self.assertNotIn("escalation", prompts.resumed("", Route(Rung("claude-opus-5-5", "low"), 3, 3, 4), opus))
|
|
self.assertIn(f"{prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\nWhere an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model opus, effort high` from 5, `model opus, effort xhigh` from 6, `model opus, effort max` from 7 and `model fable, effort max` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort high`, one rung above yours, and on each further trigger one rung higher, up to `model opus, effort xhigh`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number. Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `model opus, effort xhigh`. A reviewer", prompts.fresh("Do it.", "## Goals", Route(opus), opus, self.FACTS))
|
|
|
|
def test_offers_the_worker_a_higher_rung_for_the_rest_of_the_chunk_up_to_the_escalation_cap(self):
|
|
opus = Rung("claude-opus-5-5", "medium")
|
|
offer = " Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `model opus, effort xhigh`."
|
|
self.assertIn(f"`Escalation trigger <n>:`, n the trigger's number.{offer}", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS))
|
|
self.assertIn(f"`Escalation trigger <n>:`, n the trigger's number.{offer}", prompts.resumed("", Route(Rung("claude-opus-5-5", "high"), 3, 3, 4), opus))
|
|
self.assertNotIn("Where the hard part", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 3, 4), opus))
|
|
|
|
def test_lets_a_markdown_only_branch_review_on_the_projects_rung_where_the_reviewers_run_above_it(self):
|
|
opus = Rung("claude-opus-5-5", "medium")
|
|
markdown = " Where your branch changes only Markdown, none of it `CLAUDE.md`, `AGENTS.md`, `workflow-bench/*-prompt.md` or under `agents/`, `principles/`, `rules/` or `skills/`, launch each reviewer on the project's rung instead, with the Agent tool set to model opus, effort medium."
|
|
self.assertIn(markdown, prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 7), opus, self.FACTS))
|
|
self.assertIn(markdown, prompts.resumed("", Route(Rung("claude-opus-5-5", "high"), 3, 4, 4), opus))
|
|
self.assertNotIn("only Markdown", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS))
|
|
|
|
def test_names_the_rung_a_reviewer_or_seat_reruns_on_one_above_its_own(self):
|
|
opus = Rung("claude-opus-5-5", "medium")
|
|
both = " A reviewer or comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to `model opus, effort high`."
|
|
self.assertTrue(prompts.resumed("", Route(opus, 3, 3, 4), opus).endswith(both))
|
|
self.assertTrue(prompts.resumed("", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 4), opus).endswith(both))
|
|
self.assertTrue(prompts.fresh("Do it.", "## Goals", Route(opus), opus, self.FACTS).split(f"\n\n{self.FACTS}")[0].endswith(both))
|
|
self.assertTrue(prompts.resumed("", Route(Rung("claude-opus-5-5", "high"), 3, 4, 4), opus).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model opus, effort xhigh`, and a comprehension seat with `model opus, effort high`."))
|
|
self.assertTrue(prompts.resumed("", Route(opus, 3, 3, 7), opus).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort max`, and a comprehension seat with `model opus, effort high`."))
|
|
self.assertTrue(prompts.resumed("", Route(Rung("claude-fable-5-1", "max"), 3, 4, 8), opus).endswith(" A comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to `model opus, effort high`."))
|
|
self.assertNotIn("rerun", prompts.resumed("", Route(Rung("m")), None))
|
|
self.assertNotIn("rerun", prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), None, self.FACTS))
|
|
self.assertIn(f"{both} After your first read", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS))
|
|
fable = Rung("claude-fable-5-1", "max")
|
|
self.assertNotIn("rerun", prompts.resumed("", Route(fable), fable))
|
|
|
|
def test_opens_a_fresh_session_by_name_and_resumes_an_open_one(self):
|
|
opened = session.session_named("A proj loop-claude", pathlib.Path("/h/.claude/projects"))
|
|
fresh = worker.worker_args(opened, Rung("m"), prompts.fresh("Do it.", "## Goals\n1. g\n\n## Audience\nme", Route(Rung("m")), None, self.FACTS), resume=False)
|
|
self.assertEqual(fresh[fresh.index("--session-id") + 1], str(opened.id))
|
|
self.assertEqual(fresh[fresh.index("--name") + 1], "A proj loop-claude")
|
|
resumed = worker.worker_args(opened, Rung("m"), prompts.resumed("", Route(Rung("m")), None), resume=True)
|
|
self.assertEqual(resumed[resumed.index("--resume") + 1], str(opened.id))
|
|
self.assertNotIn("--session-id", resumed)
|
|
self.assertTrue(resumed[resumed.index("-p") + 1].startswith("Continue the task; the previous run ended before the chunk was finished. The chunk is finished once"))
|
|
|
|
def test_disallows_a_wakeup_so_a_worker_that_ends_its_turn_exits(self):
|
|
opened = session.session_named("A proj loop-claude", pathlib.Path("/h/.claude/projects"))
|
|
for resume in (False, True):
|
|
with self.subTest(resume=resume):
|
|
args = worker.worker_args(opened, Rung("m"), "Do it.", resume=resume)
|
|
self.assertEqual(args[args.index("--disallowedTools") + 1], "ScheduleWakeup")
|
|
|
|
|
|
class Records(unittest.TestCase):
|
|
def test_heads_each_record_by_its_name_the_humans_rules_last(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
claude_md = pathlib.Path(root, "home", ".claude", "CLAUDE.md")
|
|
self.assertEqual(records.read_records(root, claude_md.parent), "")
|
|
claude_md.parent.mkdir(parents=True)
|
|
claude_md.write_text("# Global\n")
|
|
self.assertEqual(records.read_records(root, claude_md.parent), "From ~/.claude/CLAUDE.md:\n# Global")
|
|
pathlib.Path(root, "docs").mkdir()
|
|
pathlib.Path(root, "docs", "decisions.md").write_text("# Decisions\n")
|
|
pathlib.Path(root, "AGENTS.md").write_text("\n# Rules\n")
|
|
self.assertEqual(records.read_records(root, claude_md.parent), "From AGENTS.md:\n# Rules\n\nFrom docs/decisions.md:\n# Decisions\n\nFrom ~/.claude/CLAUDE.md:\n# Global")
|
|
pathlib.Path(claude_md.parent, "rules").mkdir()
|
|
pathlib.Path(claude_md.parent, "rules", "todo-items.md").write_text("# Todo\n")
|
|
pathlib.Path(claude_md.parent, "rules", "goals-and-audience.md").write_text("# Goals\n")
|
|
pathlib.Path(claude_md.parent, "rules", "notes.txt").write_text("not a rule\n")
|
|
pathlib.Path(claude_md.parent, "skills", "larv-review", "standards").mkdir(parents=True)
|
|
pathlib.Path(claude_md.parent, "skills", "larv-review", "standards", "comments.md").write_text("# Comments\n")
|
|
self.assertTrue(records.read_records(root, claude_md.parent).endswith("From ~/.claude/CLAUDE.md:\n# Global\n\nFrom ~/.claude/rules/goals-and-audience.md:\n# Goals\n\nFrom ~/.claude/rules/todo-items.md:\n# Todo\n\nFrom ~/.claude/skills/larv-review/standards/comments.md:\n# Comments"))
|
|
|
|
def test_stops_on_a_record_it_cannot_read(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
claude_md = pathlib.Path(root, "home", ".claude", "CLAUDE.md")
|
|
claude_md.mkdir(parents=True)
|
|
self.assertEqual(records.read_records(root, claude_md.parent), Failure(f"CLAUDE.md in {claude_md.parent}: Is a directory - the judge cannot read the records."))
|
|
pathlib.Path(root, "docs", "decisions.md").mkdir(parents=True)
|
|
self.assertEqual(records.read_records(root, claude_md.parent), Failure(f"docs/decisions.md in {root}: Is a directory - the judge cannot read the records."))
|
|
pathlib.Path(root, "AGENTS.md").write_bytes(b"# Rules\n\xff\n")
|
|
self.assertEqual(records.read_records(root, claude_md.parent), Failure(f"AGENTS.md in {root} is not UTF-8 on line 2 - the judge cannot read the records."))
|
|
|
|
|
|
def test_stops_on_a_rule_it_cannot_read(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
claude_md = pathlib.Path(root, "home", ".claude", "CLAUDE.md")
|
|
rules = pathlib.Path(claude_md.parent, "rules")
|
|
rules.mkdir(parents=True)
|
|
pathlib.Path(rules, "todo-items.md").write_bytes(b"\xff\n")
|
|
self.assertEqual(records.read_records(root, claude_md.parent), Failure(f"todo-items.md in {rules} is not UTF-8 on line 1 - the judge cannot read the records."))
|
|
|
|
class JudgePrompt(unittest.TestCase):
|
|
def test_shows_the_transcripts_tail_and_returns_what_it_showed(self):
|
|
entries = [assistant({"type": "text", "text": "old " + "x" * judge.JUDGE_PROMPT_CHARS}), assistant({"type": "text", "text": "new"})]
|
|
prompt, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", kinds(*entries), 0))
|
|
self.assertTrue(grounds.shown.endswith("\n\nnew"))
|
|
self.assertNotIn("old", grounds.shown)
|
|
self.assertIn(grounds.shown, prompt)
|
|
self.assertLessEqual(len(prompt), judge.JUDGE_PROMPT_CHARS)
|
|
self.assertEqual(grounds.after_mark, "old " + "x" * judge.JUDGE_PROMPT_CHARS + "\n\nnew")
|
|
self.assertEqual(grounds.quotable, "## Goals\n\n")
|
|
|
|
def test_the_judge_prompt_and_schema_call_the_worker_the_worker(self):
|
|
prompt, _ = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", kinds(assistant({"type": "text", "text": "done"})), 1))
|
|
self.assertNotRegex(prompt + json.dumps(judge.JUDGE_SCHEMA), r"\b[Aa]gent")
|
|
|
|
def test_the_judge_prompt_names_every_rule_the_judge_reads_from_the_human(self):
|
|
self.assertEqual([name for name in records.HUMAN_RULE_NAMES if name not in judge.JUDGE_RULES], [])
|
|
|
|
def test_shows_the_judge_where_the_worker_last_stepped_up_and_checks_an_escalation_only_after_it(self):
|
|
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "round 3: 🔴"})]
|
|
prompt, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", kinds(*entries), 1))
|
|
self.assertRegex(prompt, "round 2: 🔴\n\n\\[step\\] [^\n]+\n\nround 3: 🔴\n\n")
|
|
self.assertEqual(grounds.after_mark, "round 3: 🔴")
|
|
prompt, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", kinds(*entries), 0))
|
|
self.assertNotIn("stepped the worker up", prompt)
|
|
self.assertEqual(grounds.shown, "round 2: 🔴\n\nround 3: 🔴")
|
|
self.assertEqual(grounds.after_mark, "round 2: 🔴\n\nround 3: 🔴")
|
|
|
|
def test_drops_an_escalation_quote_from_after_the_step_up_that_the_tail_it_shows_cuts_off(self):
|
|
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "old " + "x" * judge.JUDGE_PROMPT_CHARS}), assistant({"type": "text", "text": "new"})]
|
|
_, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", kinds(*entries), 1))
|
|
self.assertEqual(judge.quotes_checked(Unfinished("e", "old x", entries_read=3), grounds), Unfinished("e", entries_read=3))
|
|
self.assertEqual(judge.quotes_checked(Unfinished("e", "new", entries_read=3), grounds), Unfinished("e", "new", entries_read=3))
|
|
|
|
def test_shows_a_typed_answer_before_the_step_up_as_the_humans(self):
|
|
typed = prompts.typed("Which one?", "Use B.", Route(Rung("worker-model")), None, False)
|
|
entries = [{**human(typed), "promptSource": "sdk"}, assistant({"type": "text", "text": "round 3: 🔴"})]
|
|
_, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", kinds(*entries), 1))
|
|
self.assertIn("\n\n[human] Use B.\n\n[step] ", grounds.shown)
|
|
|
|
|
|
class Grounded(unittest.TestCase):
|
|
def test_every_verdict_must_quote_the_transcript(self):
|
|
self.assertTrue(judge.grounded("Needs you: x", "so.\n\nNeeds you: x\n"))
|
|
self.assertFalse(judge.grounded("invented", "the transcript"))
|
|
self.assertFalse(judge.grounded("", "the transcript"))
|
|
self.assertFalse(judge.grounded(" ", "the transcript"))
|
|
transcript = "done.\n\n**Your step:**\n```\n./run.sh\n```\nRun it."
|
|
self.assertTrue(judge.grounded("Your step:\n```\n./run.sh\n```\nRun it.", transcript))
|
|
self.assertTrue(judge.grounded("Your step: ``` ./run.sh ``` Run it.", transcript))
|
|
self.assertFalse(judge.grounded("**", transcript))
|
|
|
|
def test_an_elided_quote_holds_when_its_fragments_appear_in_order(self):
|
|
transcript = "**Decision for you:** security, or not.\n\nI recommend one more round of each.\n\nI recommend skipping it: it costs."
|
|
self.assertTrue(judge.grounded("**Decision for you:** security, … ... I recommend one more round of each ... I recommend skipping it: …", transcript))
|
|
self.assertFalse(judge.grounded("I recommend skipping it ... Decision for you", transcript))
|
|
self.assertFalse(judge.grounded("Decision for you ... invented", transcript))
|
|
self.assertFalse(judge.grounded("... …", transcript))
|
|
self.assertTrue(judge.grounded("a ... b ... c", "x a b a c"))
|
|
|
|
|
|
# Item 297 scores the higher L, item 298 the higher E + I.
|
|
SCORED = "## Items\n\n| ID | Release | Exempt | Item | L | E | I | R | S | Moves | Score |\n|---|---|---|---|---|---|---|---|---|---|---|\n| 297 | 0.8.0 | | **Do it.** | 4 | 2 | 3 | 7 | 2 | 1 by 9 | 25 |\n| 298 | 0.8.0 | | **Do more.** | 3 | 3 | 4 | 8 | 2 | 1 by 9 | 20 |\n"
|
|
|
|
|
|
def said(*texts):
|
|
return entries.AssistantTurn(parts=tuple(entries.Text(text) for text in texts))
|
|
|
|
|
|
class ChunkItems(unittest.TestCase):
|
|
def test_names_the_items_on_the_workers_latest_chunk_items_line(self):
|
|
earlier = said("Chunk items: 297 (L 3, E + I 5)")
|
|
latest = said("Starting.\n**Chunk items:** 297 (L 4, E + I 5), item 298 (L 3, E + I 7), #12, 298\nOn to the tests.")
|
|
self.assertEqual(chunk_items.named([earlier, latest]), (297, 298, 12))
|
|
self.assertEqual(chunk_items.named([latest, earlier]), (297,))
|
|
self.assertEqual(chunk_items.named([earlier, said("Chunk items: none")]), ())
|
|
|
|
def test_reads_only_a_line_the_worker_opens_with_the_label_in_its_own_text(self):
|
|
prompt = entries.LoopPrompt(text="Chunk items: 297 (L 3, E + I 5)")
|
|
written = entries.AssistantTurn(parts=(entries.ToolCall("t", "Write", {"content": "Chunk items: 9"}),))
|
|
mentioned = said("Once I know them, I write `Chunk items: 5`.")
|
|
self.assertEqual(chunk_items.named([prompt, written, mentioned]), ())
|
|
|
|
def test_takes_the_highest_l_and_e_plus_i_todo_md_scores_among_the_named_items(self):
|
|
self.assertEqual(chunk_items.taken_in([said("Chunk items: 297 (L 1, E + I 2), 298, 400")], SCORED), TakenIn((297, 298, 400), 4, 7))
|
|
self.assertEqual(chunk_items.taken_in([said("Chunk items: 400")], SCORED), TakenIn((400,)))
|
|
self.assertEqual(chunk_items.taken_in([], SCORED), TakenIn())
|
|
|
|
def test_names_the_items_at_no_level_where_todo_md_cannot_be_read(self):
|
|
self.assertEqual(chunk_items.taken_in([said("Chunk items: 297")], Failure("no todo.md")), TakenIn((297,)))
|
|
self.assertEqual(chunk_items.taken_in([said("Chunk items: 297")], "no items table"), TakenIn((297,)))
|
|
|
|
|
|
class Verdict(unittest.TestCase):
|
|
def test_needs_every_field_the_schema_requires(self):
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": False, "evidence": "e", "finished": True, "question": "", "answered_before": "", "left_open": "", "release_ready": False}), Finished("e"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": False, "evidence": "e", "finished": False, "question": "ignored", "answered_before": "", "left_open": "", "release_ready": False}), Unfinished("e"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": True, "evidence": "e", "finished": True, "question": "Which one?", "answered_before": "", "left_open": "", "release_ready": False}), Interrupt("e", "Which one?"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": True, "evidence": "e", "finished": False, "question": "Cut 0.2.0?", "answered_before": "", "left_open": "", "release_ready": True}), Interrupt("e", "Cut 0.2.0?", release_ready=True))
|
|
self.assertEqual(verdict_from({"escalation": "", "answered_before": "", "left_open": "", "answer": "", "interrupt": True, "evidence": "e", "finished": False, "question": "Cut 0.2.0?"}), Failure("the judge's reply has no release_ready of type boolean"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "A.", "interrupt": False, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "release_ready": False}), QuotedAnswer("e", "Which one?", "A."))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "A.", "interrupt": True, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "release_ready": False}), Interrupt("e", "Which one?"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "A.", "interrupt": False, "evidence": "e", "finished": True, "question": "", "answered_before": "", "left_open": "", "release_ready": False}), Failure("the judge quoted an answer without a question"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": "no", "evidence": "", "finished": True, "question": "", "answered_before": "", "left_open": "", "release_ready": False}), Failure("the judge's reply has no interrupt of type boolean"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": False, "evidence": None, "finished": True, "question": "", "answered_before": "", "left_open": "", "release_ready": False}), Failure("the judge's reply has no evidence of type string"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": True, "evidence": "x", "finished": False, "question": " ", "answered_before": "", "left_open": "", "release_ready": False}), Failure("the judge interrupted without a question"))
|
|
self.assertEqual(verdict_from(None), Failure("the judge returned no reply"))
|
|
|
|
def test_carries_the_earlier_answer_an_interrupt_names(self):
|
|
reply = {"answer": "", "escalation": "", "interrupt": True, "evidence": "e", "finished": False, "release_ready": False, "answered_before": "Grant one more round.", "left_open": "one more", "question": "Which one?"}
|
|
self.assertEqual(verdict_from(reply), Interrupt("e", "Which one?", answered_before="Grant one more round.", left_open="one more"))
|
|
self.assertEqual(verdict_from({**reply, "interrupt": False, "finished": True}), Finished("e"))
|
|
|
|
def test_takes_an_escalation_quote_on_an_unfinished_verdict(self):
|
|
reply = {"answer": "", "escalation": "round 2: 🔴", "interrupt": False, "evidence": "e", "finished": False, "question": "", "answered_before": "", "left_open": "", "release_ready": False}
|
|
self.assertEqual(verdict_from(reply), Unfinished("e", "round 2: 🔴"))
|
|
self.assertEqual(verdict_from({**reply, "finished": True}), Finished("e"))
|
|
self.assertEqual(verdict_from({**reply, "escalation": None}), Failure("the judge's reply has no escalation of type string"))
|
|
|
|
def test_takes_an_escalation_quote_on_an_interrupt(self):
|
|
reply = {"answer": "", "escalation": "round 2: 🔴", "interrupt": True, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "release_ready": False}
|
|
self.assertEqual(verdict_from(reply), Interrupt("e", "Which one?", "round 2: 🔴"))
|
|
|
|
def test_drops_an_escalation_quote_the_transcript_does_not_hold_word_for_word_from_an_interrupt_or_an_unfinished_verdict(self):
|
|
shown = "The agent: round 2: 🔴. Which one?"
|
|
self.assertEqual(judge.quotes_checked(Interrupt("e", "q", "round 2: 🔴"), judge.Grounds(shown, shown, "")), Interrupt("e", "q", "round 2: 🔴"))
|
|
self.assertEqual(judge.quotes_checked(Interrupt("e", "q", "round 2: … 🔴"), judge.Grounds(shown, shown, "")), Interrupt("e", "q"))
|
|
self.assertEqual(judge.quotes_checked(Unfinished("e", "round 3: 🔴"), judge.Grounds(shown, shown, "")), Unfinished("e"))
|
|
|
|
def test_turns_an_interrupt_that_re_asks_a_human_answer_into_that_answer_quoted_back(self):
|
|
answer = "Grant one more round; ask again if a 🔴 survives it."
|
|
shown = f"[loop] Asked.\n\n[human] {answer}\n\n[step] Round 2: a 🔴 survived."
|
|
grounds = judge.Grounds(shown, shown, "")
|
|
asking = "Grant a third round? Your answer left open: a 🔴 survives it"
|
|
kept = Interrupt("[step] Round 2", asking, answered_before=answer, left_open="a 🔴 survives it")
|
|
self.assertEqual(judge.quotes_checked(kept, grounds), kept)
|
|
re_asked = Interrupt("[step] Round 2", "Grant a third round?", "Round 2: a 🔴", answered_before=answer, left_open="a 🔴 survives it")
|
|
self.assertEqual(judge.quotes_checked(re_asked, grounds), Unfinished(answer, "Round 2: a 🔴"))
|
|
self.assertEqual(judge.quotes_checked(dataclasses.replace(kept, left_open=""), grounds), Unfinished(answer))
|
|
self.assertEqual(judge.quotes_checked(dataclasses.replace(kept, left_open="a 🔴 survived"), grounds), Unfinished(answer))
|
|
invented = dataclasses.replace(re_asked, answered_before="Grant two more rounds.")
|
|
self.assertEqual(judge.quotes_checked(invented, grounds), invented)
|
|
unnamed = Interrupt("[step] Round 2", "Grant a third round?")
|
|
self.assertEqual(judge.quotes_checked(unnamed, grounds), unnamed)
|
|
|
|
def test_takes_an_escalation_quote_on_a_quoted_answer(self):
|
|
reply = {"answer": "A.", "escalation": "round 2: 🔴", "interrupt": False, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "release_ready": False}
|
|
self.assertEqual(verdict_from(reply), QuotedAnswer("e", "Which one?", "A.", "round 2: 🔴"))
|
|
|
|
def test_drops_an_escalation_quote_the_transcript_does_not_hold_word_for_word_from_a_quoted_answer(self):
|
|
shown = "The agent: round 2: 🔴. Which one?"
|
|
self.assertEqual(judge.quotes_checked(QuotedAnswer("e", "q", "A.", "round 2: 🔴"), judge.Grounds(shown, shown, "A.")), QuotedAnswer("e", "q", "A.", "round 2: 🔴"))
|
|
self.assertEqual(judge.quotes_checked(QuotedAnswer("e", "q", "A.", "round 2: … 🔴"), judge.Grounds(shown, shown, "A.")), QuotedAnswer("e", "q", "A."))
|
|
|
|
def test_puts_a_quoted_answer_the_records_do_not_hold_to_the_human_with_the_escalation_it_carried(self):
|
|
self.assertEqual(judge.quotes_checked(QuotedAnswer("e", "q", "not there", "round 2: 🔴"), judge.Grounds("The agent: round 2: 🔴.", "The agent: round 2: 🔴.", "A.")), Interrupt("e", "q", "round 2: 🔴"))
|
|
|
|
def test_puts_a_quoted_answer_the_records_do_not_hold_word_for_word_to_the_human(self):
|
|
self.assertEqual(judge.quotes_checked(QuotedAnswer("s", "q", "not there"), judge.Grounds("shown", "shown", "the records")), Interrupt("s", "q"))
|
|
self.assertEqual(judge.quotes_checked(QuotedAnswer("s", "q", "the records"), judge.Grounds("shown", "shown", "From the records: the records")), QuotedAnswer("s", "q", "the records"))
|
|
|
|
def test_drops_an_escalation_quote_found_only_in_the_transcript_entries_before_the_workers_last_step_up(self):
|
|
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "round 3: 🔴"}), assistant({"type": "text", "text": "working"})]
|
|
def judged_on(mark, escalation):
|
|
return judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", kinds(*entries), mark), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation=escalation)]).effects())
|
|
self.assertEqual(judged_on(0, "round 2: 🔴"), Unfinished("working", "round 2: 🔴", entries_read=3))
|
|
self.assertEqual(judged_on(1, "round 2: 🔴"), Unfinished("working", entries_read=3))
|
|
self.assertEqual(judged_on(1, "round 3: 🔴"), Unfinished("working", "round 3: 🔴", entries_read=3))
|
|
self.assertEqual(judged_on(1, "round 2: 🔴\n\nround 3: 🔴"), Unfinished("working", entries_read=3))
|
|
self.assertEqual(judged_on(3, "round 3: 🔴"), Unfinished("working", entries_read=3))
|
|
|
|
def test_drops_an_escalation_quote_that_only_loop_claudes_prompts_or_the_humans_lines_repeat_after_the_step_up(self):
|
|
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), {**human("as the work remaining: round 2: 🔴"), "promptSource": "sdk"}, human("round 2: 🔴 again"), assistant({"type": "text", "text": "working"})]
|
|
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", kinds(*entries), 1), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation="round 2: 🔴")]).effects())
|
|
self.assertEqual(verdict, Unfinished("working", entries_read=4))
|
|
|
|
def test_takes_evidence_from_before_the_workers_last_step_up(self):
|
|
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "working"})]
|
|
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", kinds(*entries), 1), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", evidence="round 2: 🔴")]).effects())
|
|
self.assertEqual(verdict, Unfinished("round 2: 🔴", entries_read=2))
|
|
|
|
|
|
class LoopRuns(unittest.TestCase):
|
|
def test_stops_naming_the_reload_when_it_hands_over_anything_but_the_state_with_a_task(self):
|
|
for handed in (["3", "2", "/r", "do it"], ["3"], ["[1]"], [json.dumps({"cycle_number": 3, "root": "/r", "unfinished_evidence": ""})], [json.dumps({"cycle_number": 3, "root": "/r", "task": "", "unfinished_evidence": ""})], [json.dumps({"cycle_number": "3", "root": "/r", "task": "do it", "unfinished_evidence": ""})], [json.dumps({"cycle_number": 3, "root": "", "task": "do it", "unfinished_evidence": ""})], [json.dumps({"cycle_number": 3, "root": "/r", "task": "do it", "turns": 1, "unfinished_evidence": ""})], [json.dumps({"cycle_number": 3, "failures": 0, "root": "/r", "task": "do it", "unfinished_evidence": ""})]):
|
|
with self.subTest(handed=handed):
|
|
self.assertEqual(main.loop_run_from([main.RELOAD_FLAG, *handed], CWD, FakeEffects().effects()), Stop(1, f"the edited loop-claude cannot read the state the running one handed it, likely because the edit changed that handover; rerun loop-claude <task>. Handed: {' '.join(handed)}"))
|
|
|
|
def test_decodes_every_field_of_the_state_and_its_route(self):
|
|
state = main.LoopRun(cycle_number=2, root="/r", task="do it", unfinished_evidence="old", route=Route(Rung("claude-opus-5-5", "high"), 3, 4, 5, 1, 7, 9), unfinished_run=2, items_taken_in=(12, 14))
|
|
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(state))), state)
|
|
self.assertEqual(set(reload.LOOP_RUN_FIELDS), {field.name for field in dataclasses.fields(main.LoopRun)})
|
|
|
|
def test_rejects_a_field_of_another_type_or_an_unknown_field(self):
|
|
state = dataclasses.asdict(main.LoopRun(cycle_number=2, root="/r", task="do it", unfinished_evidence="", route=Route(Rung("claude-opus-5-5", "high"), 3, 4, 5, 1, 7, 9), unfinished_run=2))
|
|
for path, wrong in ((("unfinished_run",), "2"), (("unfinished_run",), True), (("route", "unfinished"), 1.0), (("route", "escalated_at"), "9"), (("route", "item"), True), (("route", "rung"), "claude-opus-5-5"), (("route", "rung", "model"), None), (("route",), "r"), (("route", "extra"), 1), (("route", "model"), "claude-opus-5-5"), (("route", "effort"), "high")):
|
|
with self.subTest(path=path, wrong=wrong):
|
|
handed = json.loads(json.dumps(state))
|
|
target = handed
|
|
for key in path[:-1]:
|
|
target = target[key]
|
|
target[path[-1]] = wrong
|
|
self.assertIsNone(reload.decode_handed(json.dumps(handed)))
|
|
|
|
def test_rejects_a_state_or_rung_missing_a_required_field(self):
|
|
state = dataclasses.asdict(main.LoopRun(cycle_number=2, root="/r", task="do it", unfinished_evidence="", route=Route(Rung("claude-opus-5-5", "high"))))
|
|
for path in (("cycle_number",), ("root",), ("task",), ("unfinished_evidence",), ("route", "rung"), ("route", "rung", "model")):
|
|
with self.subTest(path=path):
|
|
handed = json.loads(json.dumps(state))
|
|
target = handed
|
|
for key in path[:-1]:
|
|
target = target[key]
|
|
del target[path[-1]]
|
|
self.assertIsNone(reload.decode_handed(json.dumps(handed)))
|
|
|
|
def test_decodes_a_rung_without_an_effort(self):
|
|
state = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(Rung("some-model")))
|
|
self.assertEqual(reload.decode_handed(json.dumps({**dataclasses.asdict(state), "route": {"rung": {"model": "some-model"}}})), state)
|
|
|
|
def test_rejects_text_that_is_not_json(self):
|
|
self.assertIsNone(reload.decode_handed("not json"))
|
|
|
|
def test_decodes_a_route_inside_the_state_or_none(self):
|
|
state = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(Rung("claude-opus-5-5", "high"), 3, 4, 5, 1))
|
|
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(state))), state)
|
|
unrouted = dataclasses.replace(state, route=None)
|
|
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(unrouted))), unrouted)
|
|
handed = dataclasses.asdict(state)
|
|
handed["route"]["rung"]["effort"] = 1
|
|
self.assertIsNone(reload.decode_handed(json.dumps(handed)))
|
|
del handed["route"]["rung"]
|
|
self.assertIsNone(reload.decode_handed(json.dumps(handed)))
|
|
off_scale = dataclasses.asdict(state)
|
|
off_scale["route"]["likelihood"] = 9
|
|
self.assertIsNone(reload.decode_handed(json.dumps(off_scale)))
|
|
older = dataclasses.asdict(state)
|
|
del older["route"]["escape_and_impact_taken_in"]
|
|
self.assertEqual(reload.decode_handed(json.dumps(older)), state)
|
|
taken = dataclasses.replace(state, route=dataclasses.replace(state.route, escape_and_impact_taken_in=7))
|
|
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(taken))), taken)
|
|
for taken_in in (9, 5):
|
|
with self.subTest(taken_in=taken_in):
|
|
handed = dataclasses.asdict(state)
|
|
handed["route"]["escape_and_impact_taken_in"] = taken_in
|
|
self.assertIsNone(reload.decode_handed(json.dumps(handed)))
|
|
|
|
def test_decodes_a_state_from_before_a_field_with_a_default_joined_it(self):
|
|
self.assertEqual(reload.decode_handed(json.dumps({"cycle_number": 3, "root": "/r", "task": "do it", "unfinished_evidence": ""})), main.LoopRun(3, "/r", "do it", ""))
|
|
|
|
def test_decodes_the_items_taken_in_as_a_tuple_of_ids(self):
|
|
state = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", items_taken_in=(12, 14))
|
|
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(state))), state)
|
|
for taken_in in ([12, "14"], [True], "12", {"12": 1}, "", {}):
|
|
with self.subTest(taken_in=taken_in):
|
|
self.assertIsNone(reload.decode_handed(json.dumps(dataclasses.asdict(state) | {"items_taken_in": taken_in})))
|
|
|
|
def test_stops_when_the_reload_cannot_start_python(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
launcher = pathlib.Path(root, "loop-claude")
|
|
launcher.write_text("")
|
|
fake = FakeEffects(python=[ran()])
|
|
self.assertEqual(reload.exec_if_edited(a_loop(fake.effects(), launcher=launcher), "stale", main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="")), Stop(1, "could not reload loop-claude: No such file or directory; rerun loop-claude <task>"))
|
|
|
|
|
|
class HelpLimits(unittest.TestCase):
|
|
def test_help_wraps_at_100_columns(self):
|
|
self.assertEqual([line for line in usage.HELP.splitlines() if len(line) > 100], [])
|
|
|
|
def test_help_states_the_limits_the_code_holds(self):
|
|
said = " ".join(usage.HELP.split())
|
|
self.assertIn(f"The {['second', 'third', 'fourth', 'fifth'][states.QUOTED_ANSWERS_PER_CYCLE - 1]} question they answer in one cycle", said)
|
|
# The retry messages say these limits in words too.
|
|
self.assertEqual((retry.FAILURES_RUNNING, retry.RETRY_DELAY), (3, 60))
|
|
self.assertIn("fails three times running, retried a minute apart", said)
|
|
self.assertIn(f"retried {retry.PAST_RESET // 60} minutes after the limit resets, or every {retry.UNKNOWN_RESET_POLL // 60} minutes", said)
|
|
self.assertIn(f"still running after {PYTHON_TIMEOUT} s", said)
|
|
self.assertIn(f"Once {state.CYCLES_PER_CHECK} cycles have closed across all projects", said)
|
|
self.assertIn(f"run with a {worker.COMPACT_WINDOW // 1000}k auto-compact window", said)
|
|
self.assertIn(f"unfinished {close.UNFINISHED_LIMIT} cycles running", said)
|
|
self.assertIn(f"rerun to give it {close.UNFINISHED_LIMIT} more", said)
|
|
|
|
|
|
class ExitContract(unittest.TestCase):
|
|
def test_every_exit_1_stop_has_its_clause_in_help(self):
|
|
read, git, premises_ = "a transcript cannot be read or archived", "git cannot start or finds no checkout", "README.md holds no premises"
|
|
models_ = "the models file cannot be read or written"
|
|
names = (*records.RECORD_FILES, records.HUMAN_RULE_NAMES[0], *(f"{folder}*.md" for folder in records.HUMAN_RULE_NAMES[1:]))
|
|
unreadable = f"README.md, or {', '.join(names[:-1])} or {names[-1]} where they exist, cannot be read or is not UTF-8"
|
|
# Every Stop whose code is not a literal 0, 2 or 130, in source order: path, function, code, start of reason, its --help clause.
|
|
clauses = [
|
|
("cycle/interrupt.py", "claude_open", "1", "f'could not start claude for the interrupt:", "claude cannot start for an interrupt"),
|
|
("cycle/interrupt.py", "human_typed", "1", "entries.reason", read),
|
|
("cycle/interrupt.py", "human_typed", "1", "f'could not tell whether you typed anything:", read),
|
|
("cycle/retry.py", "retried", "1", "f'the {who} failed three times running:", "the worker or the judge fails three times running"),
|
|
("cycle/run.py", "archived_stop", "1", "f'{failure.reason}", read),
|
|
("cycle/run.py", "work", "1", "f'claude left no transcript for session", "claude leaves no transcript"),
|
|
("cycle/run.py", "rule", "1", "entries.reason", read),
|
|
("cycle/run.py", "rule", "1", "failure.reason", read),
|
|
("cycle/run.py", "keep_rejection", "1", "unkept.reason", read),
|
|
("cycle/run.py", "close", "1", "failure.reason", read),
|
|
("cycle/run.py", "close", "1", "failure.reason", "a finished chunk's session cannot be deleted"),
|
|
("loop/close.py", "closed_cycle", "1", "failure.reason", read),
|
|
("loop/main.py", "next_cycle", "1", "premises.reason", premises_),
|
|
("loop/main.py", "next_cycle", "1", "records.reason", unreadable),
|
|
("loop/main.py", "loop_run_from", "0 if task else 1", "HELP", "no task is given"),
|
|
("loop/main.py", "loop_run_from", "1", 'f"the edited loop-claude cannot read the state', "a reload cannot read the state the running script handed it"),
|
|
("loop/reload.py", "exec_if_edited", "1", "f'could not reload loop-claude:", "python cannot start for a reload"),
|
|
("loop/routing_check.py", "raise_calls", "1", "f\"could not start claude for the routing calls in {loop.paths.config_clone}", "claude cannot start for an interrupt"),
|
|
("project/checkout.py", "checkout_root", "1", "found.reason", git),
|
|
("project/model_choice.py", "get_or_ask_model", "1", "models.reason", models_),
|
|
("project/model_choice.py", "get_or_ask_model", "1", "models.reason", models_),
|
|
("project/model_choice.py", "get_or_ask_model", "1", "f'could not save the model in", models_),
|
|
]
|
|
root = pathlib.Path(main.__file__).parents[1]
|
|
found = []
|
|
for path in sorted(path for path in root.rglob("*.py") if not path.name.startswith("test_")):
|
|
functions = [node for node in ast.walk(ast.parse(path.read_text())) if isinstance(node, ast.FunctionDef)]
|
|
stops = [(call.lineno, function.name, ast.unparse(call.args[0]), ast.unparse(call.args[1])) for function in functions for call in ast.walk(function) if isinstance(call, ast.Call) and getattr(call.func, "id", None) == "Stop" and not (isinstance(call.args[0], ast.Constant) and call.args[0].value in (0, 2, 130))]
|
|
found += [(str(path.relative_to(root)), function, code, reason) for _, function, code, reason in sorted(stops)]
|
|
self.assertEqual(len(found), len(clauses), found)
|
|
for (path, function, code, reason), (want_path, want_function, want_code, reason_start, _) in zip(found, clauses):
|
|
with self.subTest(path=path, function=function):
|
|
self.assertEqual((path, function, code), (want_path, want_function, want_code))
|
|
self.assertTrue(reason.startswith(reason_start), reason)
|
|
help_exit_1 = " ".join(usage.HELP[usage.HELP.index("Exit 1"):usage.HELP.index("Exit 2")].split())
|
|
self.assertEqual([clause for *_, clause in clauses if f"- {clause}" not in help_exit_1], [])
|
|
|
|
|
|
class JudgeFailures(unittest.TestCase):
|
|
def test_every_failed_judge_run_has_its_clause_in_help(self):
|
|
started, exited, unparsed = "claude cannot start", "exits non-zero", "prints no json reply"
|
|
field, unasked = "its reply misses or mistypes a field", "interrupts or quotes an answer without a question"
|
|
# Every Failure the judge's run returns, in source order of judge.py, then of the claude run it calls: function, start of reason, its --help clause.
|
|
clauses = [
|
|
("verdict_from", "'the judge returned no reply'", field),
|
|
("verdict_from", 'f"the judge\'s reply has no {name}', field),
|
|
("verdict_from", "'the judge interrupted without a question'", unasked),
|
|
("verdict_from", "'the judge quoted an answer without a question'", unasked),
|
|
("run_judge", "verdict.reason", field),
|
|
("run_judge", "checked.reason", "quotes a passage the transcript does not hold"),
|
|
("misquoted_evidence", "f'the judge quoted {verdict.evidence!r}, which is not in the transcript", "quotes a passage the transcript does not hold"),
|
|
("claude_reply", "f'could not start claude:", started),
|
|
("claude_reply", "proc.stderr.strip() or f'{who} exited", exited),
|
|
("claude_reply", 'f"{who} printed no json reply', unparsed),
|
|
]
|
|
found = []
|
|
for module in (judge, reply):
|
|
functions = [node for node in ast.walk(ast.parse(pathlib.Path(module.__file__).read_text())) if isinstance(node, ast.FunctionDef)]
|
|
found += sorted((call.lineno, function.name, ast.unparse(call.args[0])) for function in functions for call in ast.walk(function) if isinstance(call, ast.Call) and getattr(call.func, "id", None) in ("Failure", "Rejected"))
|
|
self.assertEqual(len(found), len(clauses), found)
|
|
for (_, function, reason), (want_function, reason_start, _) in zip(found, clauses):
|
|
with self.subTest(function=function):
|
|
self.assertEqual(function, want_function)
|
|
self.assertTrue(reason.startswith(reason_start), reason)
|
|
judge_clause = " ".join(usage.HELP[usage.HELP.index("a judge when"):usage.HELP.index("A usage limit")].split())
|
|
self.assertEqual([clause for *_, clause in clauses if clause not in judge_clause], [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|