Files
claude-config/bin/loop_claude/test_loop_claude.py
T
lilleman e55b8826fb Merge remote-tracking branch 'origin/main' into item-310
# Conflicts:
#	bin/agent_cycle/interrupt.py
#	bin/agent_cycle/run.py
#	bin/agent_cycle/states.py
#	docs/reading-cost.md
#	todo.md
2026-11-19 04:47:53 +01:00

2733 lines
199 KiB
Python

import ast
import contextlib
import dataclasses
import io
import json
import os
import pathlib
import re
import shutil
import stat
import subprocess
import sys
import tempfile
import types
import unittest
import unittest.mock
import uuid
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from routing_figures import archive, state
from test_env import assert_not_root
from claude_code.output_limits import BASH_CEILING_CHARACTERS
from routing_figures.calls import Call
from loop_claude.project import checkout, model_choice, records
from loop_claude.cycle import chunk_items
from loop_claude.loop import chunk_start, chunk_windows, close
from project_config.test_project_config import configured
from release_plan.release_standing import DefaultPlan, LastCut, UNSEEN_RELEASES
from loop_effects.effects import Effects, PYTHON_TIMEOUT
from claude_code import entries, reply, session
from failure import Failure
from agent_cycle.verdicts import Finished, Interrupt, Judged, QuotedAnswer, TakenIn, Unfinished
from git_cli.git_repo import git
from agent_cycle import interrupt, judge, retry, run, spend, states, worker, worker_prompts
from claude_code.reply import Limited
import loop_claude.routed_item
from loop_claude.loop import main, reload, routing_check, this_loop, usage
from machine_load import memory
from release_plan import next_chunk
from transcript_archive import outcomes, write
from loop_state import paths
from plan_store.todo_md import plan_of
from transcript_archive.location import project_archive
from model_routing.ladder import ProjectRung, Rung
from loop_claude.cycle import prompts
from model_routing.ladder_file import read_ladder, SHIPPED
from model_routing.route import Route, route_from, WorkerLine
from agent_cycle import routing
from agent_cycle.stop import Stop
from agent_cycle import tmux
from claude_code.tokens import Tokens, tokens_of
from touch_predictions.test_touch_predictions import plan as touch_plan
from text_files.utc_stamp import utc
from agent_cycle.judge import verdict_from
# How the archive records a worker that runs off the ladder.
LADDER = read_ladder(SHIPPED)
OPUS_PROJECT = ProjectRung(LADDER, Rung("claude-opus-5-5", "medium"))
OFF_LADDER = '{"model": "worker-model"}'
# The claude-config release and commit each cycle the tests run is under.
CONFIG = "v0.8.0 (abc123)"
# How it records routes by item 179's L of 2 and E + I of 5, on Routing's rungs.
SONNET_179 = '{"model": "claude-sonnet-5-5", "effort": "high", "item": 179, "likelihood": 2, "escape_and_impact": 5}'
OPUS_179_AT_3 = '{"model": "claude-opus-5-5", "effort": "medium", "item": 179, "likelihood": 2, "escape_and_impact": 5, "escalated_at": 3}'
OPUS_179_AT_6 = '{"model": "claude-opus-5-5", "effort": "medium", "item": 179, "likelihood": 2, "escape_and_impact": 5, "escalated_at": 6}'
CWD = pathlib.Path("/c")
NOW = 1790000000
# How a line said at NOW opens, with FakeEffects' local_time.
STAMP = f"<{NOW}>"
# How git answers in a project's checkout it cannot read, such as one that is gone.
UNREAD = Failure("not a git repository")
def assistant(*parts):
return {"type": "assistant", "message": {"content": list(parts)}}
def kinds(*found):
return [entries.entry_from(entry) for entry in found]
def human(content):
return {"type": "user", "message": {"content": content}}
# The window the loops under test run in.
WINDOW = tmux.Window("loop-proj", "item-7", "%4")
def a_loop(effects, **fields):
"""A loop in /r on effects, with fields in place of its defaults."""
home = pathlib.Path("/h")
defaults = {"root": "/r", "session": session.session_named("A proj loop-claude", home / ".claude" / "projects"), "paths": paths.paths_from(home), "env": {}, "launcher": pathlib.Path("/b/loop-claude"), "window": WINDOW}
return this_loop.Loop(**{**defaults, **fields}, effects=effects)
class FakeEffects:
"""Effects that record each call and answer from a script, one answer per call."""
def __init__(self, claude=(), on_terminal_claude=(), git=(), inputs=(), python=(), streams=(), on_terminal=True, free_memory=(), dm_failures=(), tmux_version="tmux 3.2", windows=None, owner="/r"):
self.calls = []
# Each tmux command run. They stay out of self.calls, as the DMs do. tmux prints `windows` for list-windows, or without them finds no session.
self.tmux_calls = []
self.tmux_version = tmux_version
self.windows = windows
# The checkout the session's @loop-root option names.
self.owner = owner
# Each direct message sent. They stay out of self.calls, so a test that lists every call doesn't list them. Without a script, each is sent.
self.dms = []
# The directory each DM named, in the order of self.dms; None where it named none.
self.dm_directories = []
self.dm_failures = list(dm_failures)
self.scripts = {"claude": list(claude), "claude_on_terminal": list(on_terminal_claude), "git": list(git), "input": list(inputs), "python": list(python), "stream_claude": list(streams)}
# Without a script, every read finds all memory available.
self.free_memory = list(free_memory)
self.terminal = on_terminal
self.worker_environments = []
self.worker_folders = []
self.terminal_environments = []
self.terminal_folders = []
# Each timer started, as its delay and action, which a test runs where the timer fires.
self.timers = []
def effects(self):
return Effects(
claude=lambda args, cwd, stdin: self.answer("claude", (args, stdin)),
claude_on_terminal=self.on_terminal_claude,
discard_typed=lambda: self.calls.append(("discard_typed", None)),
dm=self.dm,
execv=self.execv,
git=lambda *args, cwd=None, timeout=None, guarded=False, errors="strict": self.answer("git", list(args)),
input=lambda prompt: self.answer("input", prompt),
local_time=lambda seconds: f"<{seconds}>",
memory=self.memory,
now=lambda: NOW,
on_terminal=lambda: self.terminal,
python=lambda args, cwd: self.answer("python", (args, cwd)),
sleep=lambda seconds: self.calls.append(("sleep", seconds)),
stream_claude=self.stream,
timer=self.timer,
tmux=self.tmux,
tmux_on_terminal=lambda args: self.tmux(args).returncode,
)
def timer(self, delay, action):
self.timers.append((delay, action))
return lambda: self.calls.append(("stop_timer", delay))
def tmux(self, args):
self.tmux_calls.append(args)
if args[0] == "list-windows":
return subprocess.CompletedProcess(args, 1 if self.windows is None else 0, self.windows or "", "no server")
return subprocess.CompletedProcess(args, 0, {"-V": self.tmux_version + "\n", "show-options": self.owner + "\n"}.get(args[0], "@9\n"), "")
def dm(self, text, directory):
self.dms.append(text)
self.dm_directories.append(directory)
return self.dm_failures.pop(0) if self.dm_failures else None
def memory(self):
self.calls.append(("memory", None))
return self.free_memory.pop(0) if self.free_memory else memory.Memory(available=16, total=16)
def on_terminal_claude(self, args, cwd, env):
self.terminal_environments.append(env)
self.terminal_folders.append(cwd)
return self.answer("claude_on_terminal", args)
def stream(self, args, cwd, env):
self.worker_environments.append(env)
self.worker_folders.append(cwd)
return self.answer("stream_claude", args)
def answer(self, seam, what):
self.calls.append((seam, what))
answer = self.scripts[seam].pop(0)
if isinstance(answer, BaseException):
raise answer
return answer(what) if callable(answer) else answer
def execv(self, path, args):
self.calls.append(("execv", args))
raise OSError(2, "No such file or directory")
def made(self, seam):
return [what for called, what in self.calls if called == seam]
def ran(stdout="", code=0, stderr=""):
return subprocess.CompletedProcess([], code, stdout, stderr)
JUDGE_RUN_LINE = re.compile(r"^Judge run: judge/[0-9a-f]{64}\.json\n", re.MULTILINE)
def judged(verdict, evidence="working", answer="", escalation="", release_ready=False, cost=None, reversed=""):
reply = {"answer": answer, "escalation": escalation, "evidence": evidence, "finished": verdict == "finished", "interrupt": verdict == "interrupt", "question": "Which one?" if verdict in ("interrupt", "quoted") else "", "answered_before": "", "left_open": "", "release_ready": release_ready, "reversed": reversed}
return ran(json.dumps({"structured_output": reply, **({} if cost is None else {"total_cost_usd": cost})}))
class Worker:
"""A worker claude: writes a line to the session's transcript, then streams a result."""
def __init__(self, home, prompted=False, text="working", usage=None, sent=None):
self.home = home
# Whether it records its -p prompt in the transcript first, as claude does.
self.prompted = prompted
self.text = text
# The usage its message records, twice as Claude Code writes a message of two blocks; none where None.
self.usage = usage
# The time its message records, as utc() writes it; none where None.
self.sent = sent
def __call__(self, args):
flag = "--session-id" if "--session-id" in args else "--resume"
path = self.home / "projects" / "-fake" / f"{args[args.index(flag) + 1]}.jsonl"
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "a") as handle:
if flag == "--session-id":
handle.write(json.dumps({"type": "custom-title"}) + "\n" + json.dumps({"type": "queue-operation", "timestamp": str(uuid.uuid7())}) + "\n")
if self.prompted:
handle.write(json.dumps({**human(args[args.index("-p") + 1]), "promptSource": "sdk"}) + "\n")
said = assistant({"type": "text", "text": self.text})
if self.usage is not None:
said["message"] |= {"id": "msg-1", "model": "claude-opus-5-5", "usage": self.usage}
if self.sent is not None:
said["timestamp"] = self.sent
handle.write((json.dumps(said) + "\n") * (1 if self.usage is None else 2))
return Streamed(0)
class Streamed:
def __init__(self, code, *events):
self.code = code
self.events = events or ({"type": "result"},)
def lines(self):
return [json.dumps(event) + "\n" for event in self.events]
def wait(self):
return self.code
TYPED = states.TypedAnswer("Which one?", "Use B.")
class Transitions(unittest.TestCase):
def test_each_state_moves_on_its_events_and_only_on_them(self):
answered = states.Ruled(QuotedAnswer("e", "Which one?", "A."))
legal = (
(states.Working(), states.WorkerRan("/t"), states.Judging("/t")),
(states.Working(QuotedAnswer("e", "Which one?", "A."), 3), states.WorkerRan("/t"), states.Judging("/t", 3)),
(states.Judging("/t"), states.Ruled(Interrupt("e", "Which one?")), states.Interrupting("/t", Interrupt("e", "Which one?"))),
(states.Judging("/t", 3), states.Ruled(Interrupt("e", "Which one?")), states.Interrupting("/t", Interrupt("e", "Which one?"), 3)),
(states.Judging("/t"), answered, states.Working(QuotedAnswer("e", "Which one?", "A."), 1)),
(states.Judging("/t", 2), answered, states.Working(QuotedAnswer("e", "Which one?", "A."), 3)),
(states.Judging("/t", 3), answered, states.Interrupting("/t", QuotedAnswer("e", "Which one?", "A."), 3)),
(states.Judging("/t"), states.Ruled(Finished("e")), states.Closing("/t", Finished("e"))),
(states.Judging("/t", 3), states.Ruled(Finished("e")), states.Closing("/t", Finished("e"))),
(states.Judging("/t"), states.Ruled(Unfinished("e")), states.Closing("/t", Unfinished("e"))),
(states.Judging("/t", 3), states.Ruled(Unfinished("e")), states.Closing("/t", Unfinished("e"))),
(states.Interrupting("/t", Interrupt("e", "q")), states.HumanTalked(), states.Judging("/t")),
(states.Interrupting("/t", Interrupt("e", "q"), 3), states.HumanTalked(), states.Judging("/t", 3)),
(states.Interrupting("/t", Interrupt("e", "q")), states.HumanTyped(TYPED), states.Working(TYPED)),
(states.Interrupting("/t", Interrupt("e", "q"), 3), states.HumanTyped(TYPED), states.Working(TYPED, 3)),
(states.Working(TYPED, 3), states.WorkerRan("/t"), states.Judging("/t", 3)),
(states.Closing("/t", Finished("e")), states.Archived(), states.Ended(Finished("e"))),
(states.Closing("/t", Unfinished("e")), states.Archived(), states.Ended(Unfinished("e"))),
)
for state, event, after in legal:
with self.subTest(state=state, event=event):
self.assertEqual(states.transition(state, event), after)
every_state = (states.Working(), states.Working(QuotedAnswer("e", "Which one?", "A."), 3), states.Working(TYPED, 3), states.Judging("/t"), states.Judging("/t", 3), states.Interrupting("/t", Interrupt("e", "q")), states.Interrupting("/t", Interrupt("e", "q"), 3), states.Closing("/t", Finished("e")), states.Ended(Finished("e")))
every_event = (states.WorkerRan("/t"), states.Ruled(Interrupt("e", "Which one?")), answered, states.Ruled(Finished("e")), states.Ruled(Unfinished("e")), states.HumanTalked(), states.HumanTyped(TYPED), states.Archived())
for state in every_state:
for event in every_event:
if (state, event) not in [(state, event) for state, event, _ in legal]:
with self.subTest(state=state, event=event), self.assertRaises(ValueError):
states.transition(state, event)
def test_a_rulings_step_up_reaches_the_worker_it_resumes_through_the_answer_the_human_types_and_no_further(self):
ran_on = Rung("claude-opus-5-5", "xhigh")
stepped = routing.StepUp(Route(ran_on), Route(Rung("claude-fable-5-2", "low"), escalated_at=3))
interrupt, quoted = Interrupt("e", "q", "round 2: 🔴"), QuotedAnswer("e", "q", "A.", "round 2: 🔴")
for state, event, after in (
(states.Judging("/t"), states.Ruled(quoted, stepped), states.Working(quoted, 1, True)),
(states.Judging("/t", 3), states.Ruled(quoted, stepped), states.Interrupting("/t", quoted, 3, ran_on)),
(states.Judging("/t"), states.Ruled(interrupt, stepped), states.Interrupting("/t", interrupt, 0, ran_on)),
(states.Interrupting("/t", interrupt, 0, ran_on), states.HumanTyped(TYPED), states.Working(TYPED, 0, True)),
(states.Interrupting("/t", interrupt, 0, ran_on), states.HumanTalked(), states.Judging("/t")),
(states.Working(TYPED, 0, True), states.WorkerRan("/t"), states.Judging("/t")),
):
with self.subTest(state=state, event=event):
self.assertEqual(states.transition(state, event), after)
class QuestionFor(unittest.TestCase):
def test_puts_an_interrupts_question_as_asked_and_a_quoted_answer_past_the_cap_with_its_quote(self):
cycle = types.SimpleNamespace(prompts=prompts.WORKER)
self.assertEqual(run.question_for(cycle, Interrupt("e", "Which one?")), "Which one?")
self.assertEqual(run.question_for(cycle, QuotedAnswer("e", "Which one?", "A.")), "Which one?\n\nthe goals file's Goals and Audience, AGENTS.md, docs/decisions.md, ~/.claude/CLAUDE.md, ~/.claude/rules/ or the project's review standards says:\nA.\n\nThose files have answered 3 questions this cycle, the most allowed, so this one comes to you.")
class Cycles(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
self.home = pathlib.Path(temp.name)
self.paths = paths.paths_from(self.home)
self.session = session.session_named("A proj loop-claude", self.paths.claude_code_transcripts)
self.archive = project_archive(str(self.home / "code" / "proj"), self.home, {})
def cycle(self, fake, records="", unfinished_evidence="", facts="- A preflight fact."):
return run.Cycle(config=CONFIG, window=WINDOW, effects=fake.effects(), facts=facts, judge=Rung("judge-model"), premises="## Goals", project=None, project_archive=self.archive, records=records, root=str(self.home), route=Route(Rung("worker-model")), session=self.session, task="Do it.", prompts=prompts.WORKER, taken_in=lambda entries: chunk_items.taken_in(entries, plan_of("")), unfinished_evidence=unfinished_evidence)
def archived_outcomes(self):
"""Each outcomes file's text without its `Judge run:` lines, which name a file by the hash of a prompt and its reply: the tests of the judge's runs check them."""
return sorted(JUDGE_RUN_LINE.sub("", path.read_text()) for path in self.archive.glob("*/*.outcomes.md"))
def test_a_finished_chunk_is_archived_and_its_session_deleted(self):
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: finished\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
def test_archives_the_cost_the_judges_reply_records_beside_its_verdict(self):
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished", cost=0.25)])
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working", judged=Judged(cost=0.25)))
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: finished\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nJudge cost: 0.2500\nWorker: {OFF_LADDER}\n"])
self.assertIsNone(session.transcript_file(self.session))
def test_archives_what_the_worker_and_the_judge_spent_beside_the_verdict(self):
worker = Worker(self.paths.claude_code_transcripts.parent, usage={"input_tokens": 1, "cache_creation": {"ephemeral_5m_input_tokens": 2, "ephemeral_1h_input_tokens": 3}, "cache_read_input_tokens": 4, "output_tokens": 5})
answer = json.loads(judged("finished").stdout)
fake = FakeEffects(streams=[worker], claude=[ran(json.dumps({**answer, "usage": {"input_tokens": 6, "output_tokens": 7}}))])
run.run_cycle(self.cycle(fake))
self.assertEqual(self.archived_outcomes(), [f"""Task: Do it.
2026-09-21T14:13:20Z Verdict: finished
Evidence:
> working
Spent since the chunk began, in tokens of input, cache write 5m, cache write 1h, cache read, output:
- worker, claude-opus-5-5: 1, 2, 3, 4, 5
- judge, judge-model: 6, 0, 0, 0, 7
List prices in dollars per million tokens of input, cache write 5m, cache write 1h, cache read, output:
- claude-opus-5-5: 4, 5, 8, 0.2, 20
Config: {CONFIG}
Judge: judge-model
Worker: {OFF_LADDER}
"""])
def test_a_fresh_worker_reads_the_preflights_facts_and_a_resumed_one_does_not(self):
worker = Worker(self.paths.claude_code_transcripts.parent)
fake = FakeEffects(streams=[worker, worker], claude=[judged("unfinished"), judged("finished")])
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=3))
self.assertEqual(run.run_cycle(self.cycle(fake, unfinished_evidence="working")).verdict, Finished("working"))
fresh, resumed = (args[args.index("-p") + 1] for args in fake.made("stream_claude"))
self.assertIn("- A preflight fact.", fresh)
self.assertNotIn("- A preflight fact.", resumed)
def test_a_worker_is_handed_the_projects_rung_its_own_its_reviews_e_plus_i_and_its_compaction_window_in_its_environment(self):
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
cycle = dataclasses.replace(self.cycle(fake), project=OPUS_PROJECT, route=Route(Rung("claude-haiku-5-5", "max"), 163, 1, 5))
self.assertEqual(run.run_cycle(cycle).verdict, Finished("working"))
self.assertEqual(fake.worker_environments, [{"LOOP_CLAUDE_PROJECT_RUNG": "claude-opus-5-5 at medium effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-haiku-5-5 at max effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "5", "CLAUDE_CODE_AUTO_COMPACT_WINDOW": "200000", "BASH_MAX_OUTPUT_LENGTH": "6000"}])
def test_the_claude_an_interrupt_resumes_is_handed_the_same_rungs_as_the_worker(self):
def answer(args):
with open(session.transcript_file(self.session), "a") as handle:
handle.write(json.dumps(human("the second")) + "\n")
return 0
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt"), judged("finished")], inputs=["/talk"], on_terminal_claude=[answer])
cycle = dataclasses.replace(self.cycle(fake), project=OPUS_PROJECT, route=Route(Rung("claude-haiku-5-5", "max"), 163, 1, 5))
with contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(run.run_cycle(cycle).verdict, Finished("working"))
self.assertEqual(fake.terminal_environments, [{**environment, "CLAUDE_CODE_SANDBOXED": "1"} for environment in fake.worker_environments])
self.assertEqual(fake.terminal_environments[0]["LOOP_CLAUDE_WORKER_RUNG"], "claude-haiku-5-5 at max effort")
def test_an_unfinished_chunks_transcript_is_archived_and_the_next_cycle_resumes_its_session_with_the_judges_evidence(self):
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[judged("unfinished")] * 2)
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=3))
self.assertTrue(session.transcript_file(self.session))
self.assertEqual(run.run_cycle(self.cycle(fake, unfinished_evidence="working")).verdict, Unfinished("working", entries_read=4))
workers = fake.made("stream_claude")
self.assertIn("--session-id", workers[0])
self.assertIn("--resume", workers[1])
self.assertIn("as the work remaining:\n\n> working\n\nThe chunk is finished once", workers[1][workers[1].index("-p") + 1])
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: unfinished\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n"] * 2)
def test_a_cycle_stopped_by_a_failure_or_ctrl_c_archives_what_its_session_holds(self):
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[ran(code=1)] * 3)
self.assertEqual(run.run_cycle(self.cycle(fake)).code, 1)
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[KeyboardInterrupt()])
self.assertEqual(run.run_cycle(self.cycle(fake)), Stop(130, ""))
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Stopped: Ctrl-C\n\nWorker: {OFF_LADDER}\n", f"Task: Do it.\n\n2026-09-21T14:13:20Z Stopped: the judge failed three times running: the judge exited 1\n\nWorker: {OFF_LADDER}\n"])
def test_each_judge_reply_the_loop_rejects_is_archived_with_why_beside_the_verdict_that_follows(self):
unfilled = ran(json.dumps({"structured_output": {"interrupt": False}}))
misquoted = judged("finished", evidence="never said")
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[unfilled, misquoted, judged("finished")])
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
stamp = "2026-09-21T14:13:20Z"
self.assertEqual(self.archived_outcomes(), [
"Task: Do it.\n\n"
f"{stamp} Judge reply rejected: the judge's reply has no answer of type string\n\nReply:\n> {unfilled.stdout}\n\nJudge: judge-model\n\n"
f"{stamp} Judge reply rejected: the judge quoted 'never said', which is not in the transcript\n\nReply:\n> {misquoted.stdout}\n\nJudge: judge-model\n\n"
f"{stamp} Verdict: finished\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
def test_archives_each_judge_runs_prompt_and_reply_beside_the_rejection_interrupt_and_verdict_it_gave(self):
rejected, interrupted, finished = ran(json.dumps({"structured_output": {"interrupt": False}})), judged("interrupt", cost=0.25), judged("finished")
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[rejected, interrupted, finished], inputs=["Use B."])
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
prompts = [stdin for args, stdin in fake.made("claude")[:3]]
[chunk] = self.archive.iterdir()
kept = "\n".join(path.read_text() for path in chunk.glob("*.outcomes.md"))
named = re.findall(r"^Judge run: (judge/[0-9a-f]{64}\.json)$", kept, re.MULTILINE)
self.assertEqual(len(named), 3, kept)
runs = [json.loads((chunk / name).read_text()) for name in named]
replies = [json.loads(reply.stdout) for reply in (rejected, interrupted, finished)]
self.assertEqual(sorted((found["prompt"], json.dumps(found["reply"])) for found in runs), sorted((prompt, json.dumps(reply)) for prompt, reply in zip(prompts, replies)))
self.assertEqual(sorted(path.name for path in (chunk / "judge").iterdir()), sorted(name.removeprefix("judge/") for name in named))
def test_stops_when_a_judge_run_cannot_be_archived(self):
self.archive.mkdir(parents=True)
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
def replace_file(path, data):
return Failure("disk full") if "/judge/" in str(path) else real_replace_file(path, data)
real_replace_file = write.replace_file
with unittest.mock.patch("transcript_archive.write.replace_file", replace_file):
stop = run.run_cycle(self.cycle(fake))
self.assertEqual(stop.code, 1)
self.assertRegex(stop.reason, r"^could not archive the transcript: .*/judge/[0-9a-f]{64}\.json: disk full$")
def test_archives_what_each_judge_run_cost_on_an_interrupt_a_quoted_answer_and_a_rejected_misquote(self):
for first, then, inputs in (
(judged("interrupt", cost=0.25), judged("finished"), ["Use B."]),
(judged("quoted", answer="Use A.", cost=0.25), judged("finished"), []),
(judged("unfinished", cost=0.25), judged("finished"), []),
(judged("finished", evidence="never said", cost=0.25), judged("finished"), []),
):
with self.subTest(first=first.stdout), tempfile.TemporaryDirectory() as archive:
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[first, then], inputs=inputs)
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
run.run_cycle(dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project_archive=pathlib.Path(archive)))
kept = "".join(path.read_text() for path in pathlib.Path(archive).glob("*/*.outcomes.md"))
self.assertEqual(kept.count("Judge cost: 0.2500\n"), 1, kept)
self.assertIn("Judge cost: 0.2500\nJudge run: judge/", kept)
def test_stops_with_exit_1_where_a_subagents_transcript_cannot_be_read_for_the_spend(self):
for verdict in ("interrupt", "finished"):
with self.subTest(verdict=verdict), tempfile.TemporaryDirectory() as archive:
unreadable = self.paths.claude_code_transcripts / "-fake" / str(self.session.id) / "subagents" / "agent-x.jsonl"
unreadable.mkdir(parents=True)
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged(verdict)], inputs=["Use B."])
with contextlib.redirect_stderr(io.StringIO()):
stop = run.run_cycle(dataclasses.replace(self.cycle(fake), project_archive=pathlib.Path(archive)))
self.assertEqual((stop.code, stop.reason), (1, f"could not read the transcript {unreadable}: Is a directory"))
shutil.rmtree(unreadable.parent.parent)
(self.paths.claude_code_transcripts / "-fake" / f"{self.session.id}.jsonl").unlink()
def test_counts_the_spend_since_the_last_outcome_that_recorded_one_and_archives_the_reversal(self):
transcript = self.paths.claude_code_transcripts / "-fake" / f"{self.session.id}.jsonl"
transcript.parent.mkdir(parents=True)
old = {"type": "assistant", "timestamp": "2026-09-21T14:13:19Z", "message": {"id": "old", "model": "claude-opus-5-5", "content": [], "usage": {"output_tokens": 100}}}
new = {"type": "assistant", "timestamp": "2026-09-21T14:13:21Z", "message": {"id": "new", "model": "claude-opus-5-5", "content": [], "usage": {"output_tokens": 5}}}
transcript.write_text("\n".join(json.dumps(entry) for entry in [{"type": "queue-operation", "timestamp": "2026-09-21T14:00:00Z"}, old, human("No, keep the old name."), new]) + "\n")
with contextlib.redirect_stderr(io.StringIO()):
self.assertIsNone(write.archive(self.archive, str(transcript), "Do it.", outcomes.unfinished("w", "e", "j", "", annotation=outcomes.Annotation("c", "Spent since the chunk began, in tokens of x:\n- worker, m: 1")), NOW))
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("unfinished", reversed="No, keep the old name.")])
run.run_cycle(self.cycle(fake))
latest = max(self.archived_outcomes(), key=len)
self.assertIn(f"""Reversed:
> No, keep the old name.
Spent since 2026-09-21T14:13:20Z, in tokens of input, cache write 5m, cache write 1h, cache read, output:
- worker, claude-opus-5-5: 0, 0, 0, 0, 5
""", latest)
def test_stops_with_exit_1_asking_nothing_where_an_interrupt_cannot_be_archived(self):
self.archive.parent.mkdir(parents=True)
self.archive.write_text("")
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt")], inputs=["Use B."])
with contextlib.redirect_stderr(io.StringIO()) as said:
stop = run.run_cycle(self.cycle(fake))
self.assertEqual((stop.code, stop.reason.split(":")[0]), (1, "could not archive the transcript"))
self.assertNotIn("Which one?", said.getvalue())
self.assertEqual(fake.made("claude_on_terminal"), [])
def test_stops_when_a_rejected_reply_cannot_be_archived(self):
self.archive.parent.mkdir(parents=True)
self.archive.write_text("")
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[ran(json.dumps({"structured_output": {"interrupt": False}})), judged("finished")])
stop = run.run_cycle(self.cycle(fake))
self.assertRegex(stop.reason, f"^could not archive the transcript: {self.archive}/[0-9a-f]{{64}}: Not a directory$")
self.assertEqual(stop.code, 1)
self.assertEqual(len(fake.made("claude")), 1)
def test_archives_the_judges_rejected_replies_though_it_fails_three_times_running(self):
unfilled = ran(json.dumps({"structured_output": {"interrupt": False}}))
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[unfilled] * 3)
self.assertEqual(run.run_cycle(self.cycle(fake)).code, 1)
[archived] = self.archived_outcomes()
self.assertEqual(archived.count(f"Reply:\n> {unfilled.stdout}\n"), 3)
self.assertIn("Stopped: the judge failed three times running", archived)
def test_archives_each_chunks_session_folder_replacing_a_copy_only_with_what_grew_from_it(self):
folder = self.paths.claude_code_transcripts / "-fake" / str(self.session.id)
(folder / "subagents" / "workflows").mkdir(parents=True)
(folder / "subagents" / "agent-a.jsonl").write_text("a\n")
(folder / "subagents" / "workflows" / "journal.jsonl").write_text("j\n")
(folder / "custom-title.json").write_text("first")
(folder.parent / f"{self.session.id}.orphaned-1.jsonl").write_text("o\n")
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 3, claude=[judged("unfinished"), judged("finished"), judged("unfinished")])
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=3))
[chunk] = self.archive.iterdir()
kept = chunk / "session" / str(self.session.id)
self.assertEqual((chunk / "session" / f"{self.session.id}.orphaned-1.jsonl").read_text(), "o\n")
os.utime(kept / "subagents" / "workflows" / "journal.jsonl", (0, 0))
with open(folder / "subagents" / "agent-a.jsonl", "a") as handle:
handle.write("more\n")
(folder / "custom-title.json").write_text("second")
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
self.assertFalse(folder.exists())
self.assertEqual((kept / "subagents" / "agent-a.jsonl").read_text(), "a\nmore\n")
self.assertEqual((kept / "subagents" / "workflows" / "journal.jsonl").stat().st_mtime, 0)
self.assertEqual(sorted(path.read_text() for path in kept.glob("custom-title.json*")), ["first", "second"])
self.assertEqual(stat.S_IMODE((kept / "subagents" / "agent-a.jsonl").stat().st_mode), 0o600)
folder.mkdir()
(folder / "custom-title.json").write_text("third")
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=3))
[next_chunk] = set(self.archive.iterdir()) - {chunk}
self.assertEqual(sorted(path.read_text() for path in (next_chunk / "session" / str(self.session.id)).glob("custom-title.json*")), ["third"])
def test_stops_on_a_session_folder_it_cannot_read_whole(self):
assert_not_root(self)
locked = self.paths.claude_code_transcripts / "-fake" / str(self.session.id) / "subagents"
locked.mkdir(parents=True)
locked.chmod(0)
self.addCleanup(locked.chmod, 0o700)
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("unfinished")])
self.assertEqual(run.run_cycle(self.cycle(fake)), Stop(1, f"could not archive the transcript: {locked}: Permission denied"))
def test_an_interrupt_talked_over_in_claude_is_archived_answered_and_judged_again(self):
def answer(args):
with open(session.transcript_file(self.session), "a") as handle:
handle.write(json.dumps(human("the second")) + "\n")
return 0
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt"), judged("unfinished")], inputs=["/talk"], on_terminal_claude=[answer])
with contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=4))
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: interrupt\n\nWhich one?\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n", f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: unfinished\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
self.assertIn(str(self.session.id), fake.made("claude_on_terminal")[0])
judge_prompts = [stdin for args, stdin in fake.made("claude")]
self.assertEqual(len(judge_prompts), 2)
self.assertNotIn("the second", judge_prompts[0])
self.assertIn("[human] the second", judge_prompts[1])
self.assertIn("From the goals file, ## Goals", judge_prompts[0])
self.assertIn(f"writing it into {prompts.ANSWER_DESTINATIONS},", judge_prompts[0])
self.assertIn("The task the worker was given:\nDo it.", judge_prompts[0])
def test_a_question_the_records_answer_resumes_the_worker_with_the_answer_three_times_a_cycle(self):
answered = judged("quoted", answer="Use *A*.")
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 4, claude=[answered] * 4, inputs=[""])
with contextlib.redirect_stderr(io.StringIO()) as said, contextlib.redirect_stdout(io.StringIO()) as printed:
self.assertEqual(run.run_cycle(self.cycle(fake, records="From AGENTS.md:\nUse A.")), Stop(2, "the human is needed to answer the question above, but nothing was typed"))
resumed = fake.made("stream_claude")[1]
self.assertEqual(resumed[resumed.index("-p") + 1], "Your question: Which one?\n\nthe goals file's Goals and Audience, AGENTS.md, docs/decisions.md, ~/.claude/CLAUDE.md, ~/.claude/rules/ or the project's review standards answers it:\n\nUse *A*.\n\nContinue the task.")
self.assertEqual(len(fake.made("stream_claude")), 4)
self.assertEqual(printed.getvalue().count("=== the goals file's Goals and Audience, AGENTS.md, docs/decisions.md, ~/.claude/CLAUDE.md, ~/.claude/rules/ or the project's review standards answered: Which one? ===\nUse *A*."), 3)
self.assertIn("Which one?\n\nthe goals file's Goals and Audience, AGENTS.md, docs/decisions.md, ~/.claude/CLAUDE.md, ~/.claude/rules/ or the project's review standards says:\nUse *A*.\n\nThose files have answered 3 questions this cycle", said.getvalue())
quoted = f"2026-09-21T14:13:20Z Verdict: quoted answer\n\nWhich one?\n\nthe goals file's Goals and Audience, AGENTS.md, docs/decisions.md, ~/.claude/CLAUDE.md, ~/.claude/rules/ or the project's review standards says:\n> Use *A*.\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n"
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: interrupt\n\nWhich one?\n\nthe goals file's Goals and Audience, AGENTS.md, docs/decisions.md, ~/.claude/CLAUDE.md, ~/.claude/rules/ or the project's review standards says:\nUse *A*.\n\nThose files have answered 3 questions this cycle, the most allowed, so this one comes to you.\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n\n2026-09-21T14:13:20Z Stopped: the human is needed to answer the question above, but nothing was typed\n\nWorker: {OFF_LADDER}\n", *[f"Task: Do it.\n\n{quoted}"] * 3])
def test_a_quoted_answer_that_cites_an_escalation_trigger_steps_the_worker_up_before_it_resumes_and_archives_the_step(self):
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[judged("quoted", answer="Use A.", escalation="working"), judged("finished")])
cycle = dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project=OPUS_PROJECT, route=Route(Routing.SONNET, 179, 2, 5))
with contextlib.redirect_stdout(io.StringIO()):
concluded = run.run_cycle(cycle)
self.assertEqual(concluded.route, Route(Routing.OPUS, 179, 2, 5, escalated_at=3))
first, resumed = fake.made("stream_claude")
self.assertEqual([args[args.index("--effort") + 1] for args in (first, resumed)], ["high", "medium"])
self.assertTrue(resumed[resumed.index("-p") + 1].endswith(f"Continue the task. Your request for a higher rung went through: loop-claude runs you on {Routing.OPUS}. The escalation triggers that fired before this step are spent.{prompts.WORKER.rungs_held(concluded.route, OPUS_PROJECT)}"))
self.assertNotIn("went through", first[first.index("-p") + 1])
answered = f"Verdict: quoted answer\n\nWhich one?\n\n{prompts.QUOTE_SOURCES} says:\n> Use A.\n\nEvidence:\n> working\n\nEscalation:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {SONNET_179}\n"
stepped = f"Stepped up\n\nWorker: {OPUS_179_AT_3}\n"
kept = "".join(self.archived_outcomes())
self.assertIn(f"Z {answered}\n20", kept)
self.assertIn(f"Z {stepped}", kept)
def test_a_quoted_answer_past_the_cap_archives_the_escalation_it_carried_with_the_interrupt_and_steps_up(self):
steps = [judged("quoted", answer="Use A.")] * 3 + [judged("quoted", answer="Use A.", escalation="working")]
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 4, claude=steps, inputs=[""])
cycle = dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project=OPUS_PROJECT, route=Route(Routing.SONNET, 179, 2, 5))
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(run.run_cycle(cycle), Stop(2, "the human is needed to answer the question above, but nothing was typed"))
kept = "".join(self.archived_outcomes())
self.assertIn(f"Evidence:\n> working\n\nEscalation:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {SONNET_179}\n", kept)
self.assertIn(f"Stepped up\n\nWorker: {OPUS_179_AT_6}\n", kept)
def test_a_ruling_steps_the_route_only_on_an_escalation_from_the_rung_its_taken_in_l_picks(self):
cycle = dataclasses.replace(self.cycle(FakeEffects()), project=OPUS_PROJECT, route=Route(Routing.ONE_UP, 3, 3, 4))
for ruling in (Interrupt("e", "q", entries_read=3), QuotedAnswer("e", "q", "A.", entries_read=3)):
with self.subTest(ruling=ruling):
self.assertIsNone(run.ruling_step_up(cycle, ruling))
self.assertEqual(run.ruling_step_up(cycle, Interrupt("e", "q", "round 2: 🔴", TakenIn(highest_likelihood=4), entries_read=3)), routing.StepUp(cycle.route, Route(Routing.THREE_UP, 3, 3, 4, escalated_at=3)))
def test_an_escalation_after_an_interrupt_stepped_the_worker_steps_it_again_in_the_same_cycle(self):
streams = [Worker(self.paths.claude_code_transcripts.parent, text="round 2: 🔴"), Worker(self.paths.claude_code_transcripts.parent), Worker(self.paths.claude_code_transcripts.parent)]
rulings = [judged("interrupt", evidence="round 2: 🔴", escalation="round 2: 🔴"), judged("quoted", answer="Use A.", escalation="working"), judged("finished")]
fake = FakeEffects(streams=streams, claude=rulings, inputs=["Use B."])
cycle = dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project=OPUS_PROJECT, route=Route(Routing.SONNET, 179, 2, 5))
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
concluded = run.run_cycle(cycle)
self.assertEqual(concluded.route, Route(Routing.ONE_UP, 179, 2, 5, escalated_at=4))
self.assertEqual([args[args.index("--effort") + 1] for args in fake.made("stream_claude")], ["high", "medium", "low"])
def test_an_escalation_quoted_from_before_the_workers_last_step_up_steps_nothing_and_a_later_one_reaches_the_verdict(self):
for escalation, verdict in (("round 2: 🔴", Unfinished("working", entries_read=4)), ("working", Unfinished("working", "working", entries_read=4))):
with self.subTest(escalation=escalation):
self.session = session.session_named(f"A proj {escalation} loop-claude", self.paths.claude_code_transcripts)
streams = [Worker(self.paths.claude_code_transcripts.parent, text="round 2: 🔴"), Worker(self.paths.claude_code_transcripts.parent)]
rulings = [judged("interrupt", evidence="round 2: 🔴", escalation="round 2: 🔴"), judged("unfinished", escalation=escalation)]
fake = FakeEffects(streams=streams, claude=rulings, inputs=["Use B."])
cycle = dataclasses.replace(self.cycle(fake), project=OPUS_PROJECT, route=Route(Routing.SONNET, 179, 2, 5))
with contextlib.redirect_stderr(io.StringIO()):
concluded = run.run_cycle(cycle)
self.assertEqual(concluded, run.Concluded(verdict, Route(Routing.OPUS, 179, 2, 5, escalated_at=3)))
def test_an_answer_typed_at_the_terminal_resumes_the_worker_headless_and_the_judge_reads_it_as_the_humans(self):
worker = Worker(self.paths.claude_code_transcripts.parent, prompted=True)
fake = FakeEffects(streams=[worker, worker], claude=[judged("interrupt"), judged("unfinished")], inputs=["Use B."])
with contextlib.redirect_stderr(io.StringIO()) as said:
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=6))
self.assertIn(f"\n{STAMP} Which one?\a\n", said.getvalue())
self.assertEqual(fake.made("claude_on_terminal"), [])
resumed = fake.made("stream_claude")[1]
self.assertEqual(resumed[resumed.index("--resume") + 1], str(self.session.id))
typed = resumed[resumed.index("-p") + 1]
self.assertEqual(worker_prompts.typed_answer(typed).answer, "Use B.")
self.assertIn("Which one?", typed)
self.assertIn("Continue the task. The chunk is finished only once every answer the human typed in it", typed)
judge_prompts = [stdin for args, stdin in fake.made("claude")]
self.assertRegex(judge_prompts[1], r"(?s)\[loop\] Your question was: Which one\?.*\n\n\[human\] Use B\.\n\nworking\n\nDecide whether")
self.assertNotIn("The human answered", judge_prompts[1])
def test_an_interrupt_that_steps_the_worker_up_names_its_new_rung_in_the_prompt_that_resumes_it_with_the_typed_answer(self):
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[judged("interrupt", escalation="working"), judged("finished")], inputs=["Use B."])
cycle = dataclasses.replace(self.cycle(fake), project=OPUS_PROJECT, route=Route(Routing.SONNET, 179, 2, 5))
with contextlib.redirect_stderr(io.StringIO()):
concluded = run.run_cycle(cycle)
self.assertEqual(concluded.route, Route(Routing.OPUS, 179, 2, 5, escalated_at=3))
first, resumed = fake.made("stream_claude")
self.assertEqual([args[args.index("--effort") + 1] for args in (first, resumed)], ["high", "medium"])
typed = resumed[resumed.index("-p") + 1]
self.assertIn("Your request for a higher rung went through: loop-claude runs you on claude-opus-5-5 at medium effort.", typed)
self.assertIn(" Launch each reviewer with the Agent tool set to model fable, effort low, the rung item 179's E + I of 5 picks.", typed)
self.assertNotIn("went through", first[first.index("-p") + 1])
def test_an_interrupt_that_steps_the_worker_up_compacts_its_session_on_the_rung_it_ran_on(self):
usage = {"input_tokens": 2, "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 5000}, "cache_read_input_tokens": 70000, "output_tokens": 100}
worker = Worker(self.paths.claude_code_transcripts.parent, usage=usage, sent=utc(NOW - 600))
fake = FakeEffects(streams=[worker, Streamed(0), worker], claude=[judged("interrupt", escalation="working"), judged("finished")])
def typed(prompt):
for _, action in fake.timers:
action()
return "Use B."
fake.scripts["input"] = [typed]
cycle = dataclasses.replace(self.cycle(fake), project=OPUS_PROJECT, route=Route(Routing.SONNET, 179, 2, 5))
with contextlib.redirect_stderr(io.StringIO()):
run.run_cycle(cycle)
_, compaction, resumed = fake.made("stream_claude")
self.assertEqual(compaction[compaction.index("-p") + 1], "/compact")
self.assertEqual([(args[args.index("--model") + 1], args[args.index("--effort") + 1]) for args in (compaction, resumed)], [("claude-sonnet-5-5", "high"), ("claude-opus-5-5", "medium")])
def test_a_typed_answer_past_the_quoted_answers_cap_reaches_the_worker_with_its_own_question(self):
worker = Worker(self.paths.claude_code_transcripts.parent)
fake = FakeEffects(streams=[worker] * 5, claude=[judged("quoted", answer="Use A.")] * 4 + [judged("finished")], inputs=["Use B."])
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(run.run_cycle(self.cycle(fake, records="From AGENTS.md:\nUse A.")).verdict, Finished("working"))
typed = fake.made("stream_claude")[4]
self.assertIn("Your question was: Which one?\n\nContinue the task.", typed[typed.index("-p") + 1])
self.assertNotIn("Those files have answered", typed[typed.index("-p") + 1])
def test_a_release_ready_to_cut_opens_claudes_session_without_asking_for_an_answer(self):
def answer(args):
with open(session.transcript_file(self.session), "a") as handle:
handle.write(json.dumps(human("cut")) + "\n")
return 0
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt", release_ready=True), judged("finished")], on_terminal_claude=[answer])
with contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
self.assertEqual(fake.made("input"), [])
self.assertEqual(len(fake.made("claude_on_terminal")), 1)
def test_stops_when_an_interrupt_cannot_be_archived(self):
self.archive.parent.mkdir(parents=True)
self.archive.write_text("")
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt")], inputs=["Use B."])
stop = run.run_cycle(self.cycle(fake))
self.assertRegex(stop.reason, f"^could not archive the transcript: {self.archive}/[0-9a-f]{{64}}: Not a directory$")
self.assertEqual(fake.made("input"), [])
def test_stops_when_a_quoted_answer_cannot_be_archived(self):
self.archive.parent.mkdir(parents=True)
self.archive.write_text("")
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("quoted", answer="Use A.")])
with contextlib.redirect_stdout(io.StringIO()):
stop = run.run_cycle(self.cycle(fake, records="From AGENTS.md:\nUse A."))
self.assertRegex(stop.reason, f"^could not archive the transcript: {self.archive}/[0-9a-f]{{64}}: Not a directory$")
self.assertEqual(len(fake.made("stream_claude")), 1)
def test_an_answer_the_records_do_not_hold_is_dropped_and_the_question_goes_to_the_human(self):
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("quoted", answer="Use ... A.")], inputs=[""])
with contextlib.redirect_stderr(io.StringIO()) as said:
self.assertEqual(run.run_cycle(self.cycle(fake, records="From AGENTS.md:\nUse B. Use A.")), Stop(2, "the human is needed to answer the question above, but nothing was typed"))
self.assertEqual(len(fake.made("stream_claude")), 1)
self.assertIn(f"\n{STAMP} Which one?\a\n", said.getvalue())
self.assertNotIn("Use ... A.", said.getvalue())
def test_stops_keeping_the_transcript_when_the_finished_chunks_session_cannot_be_deleted(self):
assert_not_root(self)
folder = self.paths.claude_code_transcripts / "-fake" / str(self.session.id)
(folder / "subagents").mkdir(parents=True)
folder.chmod(0o500)
self.addCleanup(folder.chmod, 0o700)
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
stop = run.run_cycle(self.cycle(fake))
self.assertEqual(stop, Stop(1, f"could not delete {folder}: Permission denied; delete every file and folder matching {folder}*, then run it again; a run before that resumes the finished chunk's session"))
self.assertTrue(session.session_open(self.session))
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: finished\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n\n2026-09-21T14:13:20Z Stopped: {stop.reason}\n\nWorker: {OFF_LADDER}\n"])
def test_stops_when_the_worker_leaves_no_transcript(self):
fake = FakeEffects(streams=[Streamed(0)])
self.assertEqual(run.run_cycle(self.cycle(fake)), Stop(1, f"claude left no transcript for session {self.session.id}"))
class Retried(unittest.TestCase):
def test_retries_a_minute_apart_and_stops_on_the_third_failure(self):
fake = FakeEffects()
results = [Failure("one"), "done"]
self.assertEqual(retry.retried("worker", lambda: results.pop(0), fake.effects()), "done")
self.assertEqual(fake.calls, [("sleep", 60)])
fake = FakeEffects()
self.assertEqual(retry.retried("judge", lambda: Failure("no"), fake.effects()), Stop(1, "the judge failed three times running: no"))
self.assertEqual(fake.calls, [("sleep", 60), ("sleep", 60)])
def test_hands_each_failure_to_the_keeper_and_stops_where_it_stops(self):
kept = []
def keep(failure):
kept.append(failure)
return Stop(1, "no room") if len(kept) == 2 else None
fake = FakeEffects()
self.assertEqual(retry.retried("judge", lambda: Failure("no"), fake.effects(), keep), Stop(1, "no room"))
self.assertEqual(kept, [Failure("no")] * 2)
self.assertEqual(fake.calls, [("sleep", 60)])
def test_waits_out_a_usage_limit_which_ends_a_run_of_failures(self):
fake = FakeEffects()
results = [Failure("one"), Failure("two"), Limited(NOW + 3600, "limit"), Failure("three"), Limited(None, "limit"), Limited(NOW - 600, "limit"), Failure("four"), Failure("five"), Failure("six")]
self.assertEqual(retry.retried("worker", lambda: results.pop(0), fake.effects()), Stop(1, "the worker failed three times running: six"))
self.assertEqual(fake.calls, [("sleep", 60), ("sleep", 60), ("sleep", 3720), ("sleep", 60), ("sleep", 600), ("sleep", 600), ("sleep", 60), ("sleep", 60)])
def test_says_what_claude_said_and_when_the_next_try_is_due(self):
results = [Limited(NOW + 3600, "You've hit your session limit"), "done"]
with contextlib.redirect_stderr(io.StringIO()) as said:
retry.retried("judge", lambda: results.pop(0), FakeEffects().effects())
self.assertEqual(said.getvalue(), f"{STAMP} the judge hit a usage limit (You've hit your session limit); retrying at <{NOW + 3720}>\n")
class Limits(unittest.TestCase):
def test_the_worker_reports_the_latest_reset_its_stream_names(self):
rejected = [{"type": "rate_limit_event", "rate_limit_info": {"status": "rejected", "resetsAt": resets_at}} for resets_at in (NOW + 60, NOW + 600)]
allowed = {"type": "rate_limit_event", "rate_limit_info": {"status": "allowed", "resetsAt": NOW + 9000}}
limited = {"type": "result", "api_error_status": 429, "result": "You've hit your session limit"}
fake = FakeEffects(streams=[Streamed(1, rejected[1], allowed, rejected[0], limited)])
self.assertEqual(worker.run_worker("/r", [], {}, fake.effects()), Limited(NOW + 600, "You've hit your session limit"))
def test_a_limit_outranks_the_exit_code_and_its_message_fits_one_line(self):
def limited(**said):
return {"type": "result", "api_error_status": 429, **said}
self.assertEqual(worker.worker_outcome(-9, limited(result="limit\n resets 7pm"), []), Limited(None, "limit resets 7pm"))
self.assertEqual(worker.worker_outcome(1, limited(result=" \n"), [NOW]), Limited(NOW, "claude gave no message"))
self.assertEqual(worker.worker_outcome(0, limited(), []), Limited(None, "claude gave no message"))
self.assertEqual(worker.worker_outcome(0, None, [NOW]), Failure("claude exited 0 without a result event"))
self.assertIsNone(worker.worker_outcome(0, {"type": "result"}, [NOW]))
def test_the_judge_reports_a_limit_whatever_its_exit_code(self):
reply = json.dumps({"type": "result", "api_error_status": 429, "is_error": True, "result": "You've hit your session limit · resets 7:40pm (Europe/Stockholm)"})
fake = FakeEffects(claude=[ran(reply, code=1, stderr="limited")])
self.assertEqual(judge.run_judge("/r", [], "", judge.Grounds("", "", ""), fake.effects()), Limited(None, "You've hit your session limit · resets 7:40pm (Europe/Stockholm)"))
def test_reads_only_a_rejecting_events_whole_seconds(self):
for info, want in [({"status": "rejected", "resetsAt": NOW}, NOW), ({"status": "allowed", "resetsAt": NOW}, None), ({"status": "rejected", "resetsAt": True}, None), ({"status": "rejected"}, None), ("rejected", None)]:
with self.subTest(info=info):
self.assertEqual(entries.rejected_until({"type": "rate_limit_event", "rate_limit_info": info}), want)
class MemoryWait(unittest.TestCase):
def test_a_worker_starts_only_once_a_quarter_of_memory_is_available(self):
fake = FakeEffects(free_memory=[memory.Memory(available=3, total=16), memory.Memory(available=3, total=16), memory.Memory(available=4, total=16)], streams=[Streamed(0)])
with contextlib.redirect_stderr(io.StringIO()) as said:
self.assertIsNone(worker.run_worker("/r", [], {}, fake.effects()))
self.assertEqual([seam for seam, _ in fake.calls], ["memory", "sleep", "memory", "sleep", "memory", "stream_claude"])
self.assertEqual(fake.made("sleep"), [worker.MEMORY_POLL] * 2)
self.assertEqual(said.getvalue().count("waiting"), 1, said.getvalue())
def test_a_worker_starts_at_once_where_memory_cannot_be_read(self):
fake = FakeEffects(free_memory=[None], streams=[Streamed(0)])
self.assertIsNone(worker.run_worker("/r", [], {}, fake.effects()))
self.assertEqual([seam for seam, _ in fake.calls], ["memory", "stream_claude"])
class Interrupts(unittest.TestCase):
def setUp(self):
self.home = pathlib.Path(tempfile.mkdtemp())
self.addCleanup(shutil.rmtree, self.home)
self.paths = paths.paths_from(self.home)
self.transcript = self.home / "t.jsonl"
self.transcript.write_text(json.dumps(human("a")) + "\n")
self.session = interrupt.WorkerSession(str(self.home), uuid.uuid7(), Rung("m", "high"), Rung("m", "high"), str(self.transcript), {"LOOP_CLAUDE_WORKER_RUNG": "m at high effort"})
def put(self, fake, answer_here=True, effects=None, question="Which one?"):
"""interrupt's answer and what it said, without the bell it rang, which self.bells counts."""
with contextlib.redirect_stderr(io.StringIO()) as said:
answer = interrupt.interrupt(question, answer_here, self.session, WINDOW, prompts.DRIVER, effects or fake.effects())
self.bells = said.getvalue().count(tmux.BELL)
return answer, said.getvalue().replace(tmux.BELL, "")
def typed_there(self, args):
"""The claude opened on the terminal: the human types a line in the worker's session, and it ends."""
with open(self.transcript, "a") as handle:
handle.write(json.dumps(human("cut it")) + "\n")
return 0
def worked(self, minutes_ago, written_1h=5000, read=70000, written_5m=0):
"""Appends the worker's last message, sent minutes_ago before NOW, whose usage wrote written_1h tokens to the hour-long cache, written_5m to the five-minute one, and read read tokens from the cache."""
said = assistant({"type": "text", "text": "Which one?"})
said["timestamp"] = utc(NOW - minutes_ago * 60)
said["message"] |= {"id": "msg-1", "model": "m", "usage": {"input_tokens": 2, "cache_creation": {"ephemeral_5m_input_tokens": written_5m, "ephemeral_1h_input_tokens": written_1h}, "cache_read_input_tokens": read, "output_tokens": 100}}
with open(self.transcript, "a") as handle:
handle.write(json.dumps(said) + "\n")
def fired(self, fake, answer):
"""An input that first fires every timer started, then returns answer as the line the human typed."""
def typed(prompt):
for _, action in fake.timers:
action()
return answer
return typed
def test_compacts_the_workers_session_ten_minutes_before_its_hour_long_prompt_cache_expires_and_keeps_waiting(self):
self.worked(minutes_ago=10)
fake = FakeEffects(streams=[Streamed(0)])
fake.scripts["input"] = [self.fired(fake, "Use B.")]
answer, said = self.put(fake)
self.assertEqual(answer, "Use B.")
self.assertEqual([delay for delay, _ in fake.timers], [40 * 60])
self.assertEqual([what for seam, what in fake.calls if seam == "stream_claude"], [["--model", "m", "--effort", "high", "--permission-mode", "bypassPermissions", "-p", "/compact", "--resume", str(self.session.id), "--output-format", "stream-json", "--verbose"]])
self.assertEqual(fake.worker_environments, [self.session.env])
self.assertEqual(fake.worker_folders, [self.session.root])
self.assertIn(f"\n{STAMP} compacting the worker's claude session before its prompt cache expires; an answer typed meanwhile is read once that ends\n\n{STAMP} compacted the worker's claude session\n", said)
self.assertNotIn("still waiting", said)
def test_compacts_on_the_rung_the_worker_ran_on_where_a_ruling_stepped_it_up_and_talk_opens_claude_on_the_new_rung(self):
self.worked(minutes_ago=10)
self.session = dataclasses.replace(self.session, rung=Rung("n", "medium"))
fake = FakeEffects(streams=[Streamed(0)], on_terminal_claude=[self.typed_there])
fake.scripts["input"] = [self.fired(fake, "/talk")]
self.assertEqual(self.put(fake)[0], interrupt.Talked())
(compaction,), (opened,) = fake.made("stream_claude"), fake.made("claude_on_terminal")
self.assertEqual(compaction[:4], ["--model", "m", "--effort", "high"])
self.assertEqual(opened[:4], ["--model", "n", "--effort", "medium"])
def test_compacts_until_the_prompt_cache_expires_and_not_from_then_on(self):
for name, late, compactions in (("a second before", 50 * 60 - 1, 1), ("as it expires", 50 * 60, 0)):
with self.subTest(name):
self.transcript.write_text(json.dumps(human("a")) + "\n")
self.worked(minutes_ago=10)
fake = FakeEffects(streams=[Streamed(0)])
clock = [NOW]
def typed(prompt):
clock[0] = NOW + late
for _, action in fake.timers:
action()
return "Use B."
fake.scripts["input"] = [typed]
answer, said = self.put(fake, effects=dataclasses.replace(fake.effects(), now=lambda: clock[0]))
self.assertEqual(answer, "Use B.")
self.assertEqual(len(fake.made("stream_claude")), compactions)
self.assertEqual(said.count("compact"), 2 * compactions, said)
def test_a_compaction_doesnt_wait_for_memory(self):
self.worked(minutes_ago=10)
fake = FakeEffects(streams=[Streamed(0)], free_memory=[memory.Memory(available=3, total=16)])
fake.scripts["input"] = [self.fired(fake, "Use B.")]
said = self.put(fake)[1]
self.assertEqual([seam for seam, _ in fake.calls if seam in ("memory", "sleep", "stream_claude")], ["stream_claude"])
self.assertNotIn("waiting to start the worker", said)
def test_an_answer_before_the_timer_fires_stops_it_and_doesnt_compact_anything(self):
self.worked(minutes_ago=10)
fake = FakeEffects(inputs=["Use B."])
self.assertEqual(self.put(fake)[0], "Use B.")
self.assertEqual([call for call in fake.calls if call[0] in ("stop_timer", "stream_claude")], [("stop_timer", 40 * 60)])
def test_stops_the_timer_before_talk_opens_the_workers_session(self):
self.worked(minutes_ago=10)
fake = FakeEffects(inputs=["/talk"], on_terminal_claude=[self.typed_there])
self.assertEqual(self.put(fake)[0], interrupt.Talked())
self.assertEqual([call[0] for call in fake.calls if call[0] in ("stop_timer", "claude_on_terminal")], ["stop_timer", "claude_on_terminal"])
def test_starts_no_timer_under_60k_tokens_of_context_past_the_margin_on_a_five_minute_cache_or_where_claude_opens_at_once(self):
for name, minutes_ago, written_1h, written_5m, read, answer_here in (("small", 10, 5000, 0, 54997, True), ("margin begun", 50, 5000, 0, 70000, True), ("five minutes", 1, 0, 5000, 70000, True), ("nothing written", 1, 0, 0, 70000, True), ("opens at once", 10, 5000, 0, 70000, False)):
with self.subTest(name):
self.transcript.write_text(json.dumps(human("a")) + "\n")
self.worked(minutes_ago, written_1h, read, written_5m)
fake = FakeEffects(inputs=["Use B."], on_terminal_claude=[self.typed_there])
self.put(fake, answer_here=answer_here)
self.assertEqual(fake.timers, [])
def test_starts_the_timer_at_60k_tokens_of_context(self):
self.worked(minutes_ago=10, read=54998)
fake = FakeEffects(inputs=["Use B."])
self.put(fake)
self.assertEqual([delay for delay, _ in fake.timers], [40 * 60])
def test_starts_no_timer_where_the_transcript_cannot_be_read_and_still_asks(self):
self.transcript.unlink()
fake = FakeEffects(inputs=["Use B."])
self.assertEqual(self.put(fake)[0], "Use B.")
self.assertEqual(fake.timers, [])
def test_a_compaction_that_hits_a_usage_limit_says_the_limits_message_and_keeps_waiting(self):
self.worked(minutes_ago=10)
fake = FakeEffects(streams=[Streamed(0, {"type": "result", "api_error_status": 429, "result": "You've hit your session limit"})])
fake.scripts["input"] = [self.fired(fake, "Use B.")]
answer, said = self.put(fake)
self.assertEqual(answer, "Use B.")
self.assertIn(f"\n{STAMP} could not compact the worker's claude session: You've hit your session limit\n", said)
def test_a_compaction_that_fails_says_why_and_keeps_waiting(self):
self.worked(minutes_ago=10)
fake = FakeEffects(streams=[Streamed(1)])
fake.scripts["input"] = [self.fired(fake, "Use B.")]
answer, said = self.put(fake)
self.assertEqual(answer, "Use B.")
self.assertIn(f"\n{STAMP} could not compact the worker's claude session: claude exited 1\n", said)
def test_stops_off_a_terminal_before_asking(self):
fake = FakeEffects(on_terminal=False)
self.assertEqual(self.put(fake), (Stop(2, "the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal"), f"\n{STAMP} Which one?\n"))
self.assertEqual(fake.calls, [])
def test_dms_the_question_naming_the_project_on_a_terminal_or_off_one(self):
for on_terminal in (True, False):
with self.subTest(on_terminal=on_terminal):
fake = FakeEffects(inputs=["Use B."], on_terminal=on_terminal)
self.put(fake)
self.assertEqual(fake.dms, [f"loop-claude in {self.home} asks, in the tmux window loop-proj:item-7:\n\nWhich one?"])
def test_marks_its_window_and_rings_the_bell_while_it_waits_and_clears_the_mark_once_answered(self):
fake = FakeEffects(inputs=["Use B."])
answer, said = self.put(fake)
self.assertEqual(answer, "Use B.")
self.assertEqual(self.bells, 1)
self.assertEqual(fake.tmux_calls, [["rename-window", "-t", "%4", "item-7 ?"], ["rename-window", "-t", "%4", "item-7"]])
def test_a_loop_that_stops_unanswered_leaves_its_window_marked(self):
fake = FakeEffects(on_terminal=False)
self.put(fake)
self.assertEqual(fake.tmux_calls, [["rename-window", "-t", "%4", "item-7 ?"]])
def test_drops_the_keys_typed_before_the_question_ahead_of_the_dm_so_an_answer_typed_while_it_sends_counts(self):
fake = FakeEffects(inputs=["Use B."])
seen = []
effects = dataclasses.replace(fake.effects(), dm=lambda text, directory: seen.append(list(fake.calls)))
self.assertEqual(self.put(fake, effects=effects)[0], "Use B.")
self.assertEqual(seen, [[("discard_typed", None)]])
def test_says_why_a_dm_failed_and_still_takes_the_answer(self):
fake = FakeEffects(inputs=["Use B."], dm_failures=[Failure("cannot reach https://chat.example: timed out")])
self.assertEqual(self.put(fake), ("Use B.", f"\n{STAMP} Which one?\n{STAMP} could not DM you: cannot reach https://chat.example: timed out\n"))
def test_returns_the_line_typed_after_the_question_stripped_dropping_what_was_typed_before(self):
fake = FakeEffects(inputs=[" Use B. "])
self.assertEqual(self.put(fake), ("Use B.", f"\n{STAMP} Which one?\n"))
self.assertEqual(fake.calls, [("discard_typed", None), ("input", "Answer (/talk opens claude, Enter stops the loop): ")])
def test_talks_on_talk_or_at_once_where_the_answer_is_not_asked_for_here(self):
for typed, answer_here in (([" /talk "], True), ([], False)):
with self.subTest(typed=typed):
fake = FakeEffects(inputs=typed, on_terminal_claude=[self.typed_there])
self.assertEqual(self.put(fake, answer_here)[0], interrupt.Talked())
self.assertEqual(len(fake.made("input")), len(typed))
self.assertEqual(fake.calls[0], ("discard_typed", None))
def test_stops_on_an_empty_answer_or_the_inputs_end(self):
for typed, why, said in ((" ", "nothing was typed", ""), (EOFError(), "the input ended", "\n")):
with self.subTest(why=why):
self.assertEqual(self.put(FakeEffects(inputs=[typed])), (Stop(2, f"the human is needed to answer the question above, but {why}"), f"\n{STAMP} Which one?\n{said}"))
def test_stops_when_nothing_was_typed_in_the_session(self):
fake = FakeEffects(inputs=["/talk"], on_terminal_claude=[0])
stop, _ = self.put(fake)
self.assertEqual(fake.made("claude_on_terminal")[0][:4], ["--model", "m", "--effort", "high"])
self.assertEqual(stop, Stop(2, "the human is needed to answer the question above, but the interrupt ended without anything typed"))
def test_hands_the_claude_that_takes_the_terminal_the_workers_environment(self):
fake = FakeEffects(inputs=["/talk"], on_terminal_claude=[0])
self.put(fake)
self.assertEqual(fake.terminal_environments, [{"LOOP_CLAUDE_WORKER_RUNG": "m at high effort", "CLAUDE_CODE_SANDBOXED": "1"}])
class RoutingCheck(unittest.TestCase):
def setUp(self):
self.home = pathlib.Path(tempfile.mkdtemp())
self.addCleanup(shutil.rmtree, self.home)
self.root = self.home / "archive"
self.paths = paths.paths_from(self.home)
self.session = session.session_named("A log loop-claude", self.paths.claude_code_transcripts)
def loop(self, fake):
"""The loop in /code/log."""
return a_loop(fake.effects(), root="/code/log", session=self.session, paths=self.paths)
def chunks(self, count, verdicts, project="log"):
"""count finished chunks, each closed by verdicts verdicts, each with one Explore subagent whose report outgrew its searches."""
for number in range(count):
folder = self.root / project / f"c{number}"
subagents = folder / "session" / "s" / "subagents"
subagents.mkdir(parents=True)
kept = "".join(f"2026-10-14T10:0{verdict}:00Z {outcomes.unfinished('w', 'e', 'j', '')}\n" for verdict in range(verdicts - 1))
(folder / "copy.outcomes.md").write_text(f"Task: t\n\n{kept}2026-10-14T11:00:00Z {outcomes.finished('w', 'm', 'j')}\n")
launch = {"type": "assistant", "timestamp": "2026-10-14T09:00:00Z", "message": {"id": "m", "content": [{"type": "tool_use", "id": "a", "name": "Agent", "input": {"subagent_type": "Explore"}}]}}
report = {"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "a", "content": "long report"}]}}
(folder / "copy.jsonl").write_text(json.dumps(launch) + "\n" + json.dumps(report) + "\n")
(subagents / "agent-a.jsonl").write_text("")
(subagents / "agent-a.meta.json").write_text(json.dumps({"agentType": "Explore", "toolUseId": "a"}))
def check(self, fake):
"""What check_routing returns and says for a loop in /code/log."""
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()) as said:
stop = routing_check.check_routing(self.root, Rung("m", "high"), self.loop(fake))
return stop, said.getvalue()
def test_measures_once_20_cycles_have_closed_since_the_last_measurement(self):
self.chunks(5, 3)
self.assertEqual(self.check(FakeEffects()), (None, ""))
self.assertFalse((self.root / "routing-figures.md").exists())
self.chunks(6, 1, "other")
self.assertEqual(self.check(FakeEffects(on_terminal=False, git=[UNREAD] * 6))[0], None)
self.assertIn("at cycle 21 across", (self.root / "routing-figures.md").read_text())
def test_reads_each_projects_history_through_the_loops_git_and_names_each_it_cannot_read(self):
self.chunks(5, 4)
fake = FakeEffects(on_terminal=False, git=[UNREAD] * 3)
self.check(fake)
located = ["rev-parse", "--path-format=absolute", "--show-toplevel", "--git-common-dir"]
self.assertEqual(fake.made("git"), [located, located, ["remote"]], "the config is located for its title templates, then for where it keeps the plan")
self.assertIn("## Default branches unread\n\n- log: not a git repository\n", (self.root / "routing-figures.md").read_text())
def test_hands_the_terminal_over_once_for_each_new_call(self):
self.chunks(5, 4)
fake = FakeEffects(on_terminal_claude=[0], git=[UNREAD] * 3)
stop, said = self.check(fake)
self.assertIsNone(stop)
call = "Explore subagents' reports took more of the worker's context than their searches would have."
self.assertIn(f"- {call}", said)
self.assertIn(f"- {call}", (self.root / "routing-figures.md").read_text())
((args),) = fake.made("claude_on_terminal")
self.assertEqual(args[:6], ["--model", "m", "--effort", "high", "--permission-mode", "bypassPermissions"])
self.assertIn(f"don't change anything until they answer:\n- {call}", args[6])
self.assertEqual(fake.terminal_environments, [{"CLAUDE_CODE_SANDBOXED": "1"}])
self.assertEqual(fake.terminal_folders, [str(self.home / "code" / "claude-config")], "the development clone, since ~/.claude holds only a release")
self.assertIn("Record what they decide in this checkout, the development clone", args[6])
self.assertIsNone(state.mark_measured(self.root, 0))
self.assertEqual(self.check(FakeEffects(git=[UNREAD] * 3))[0], None)
self.assertNotIn(call, self.check(FakeEffects())[1])
def test_stamps_the_figures_and_the_calls_raised_when_it_measures(self):
self.chunks(5, 4)
(self.root / "routing-calls-raised.md").write_text("# Routing calls raised\n\n- 2026-10-14T10:02:00Z Reset\n")
self.check(FakeEffects(on_terminal_claude=[0], git=[UNREAD] * 3))
figures = (self.root / "routing-figures.md").read_text()
self.assertIn(f"Measured {utc(NOW)}, at cycle 20 across", figures)
self.assertIn("that finished after the reset of 2026-10-14T10:02:00Z", figures)
self.assertIn(f"- {utc(NOW)} Explore subagents'", (self.root / "routing-calls-raised.md").read_text())
def test_says_a_figure_it_cannot_read(self):
self.chunks(5, 4)
(self.root / "routing-state.json").mkdir()
self.assertIn("routing figures: could not read", self.check(FakeEffects())[1])
def test_goes_on_off_a_terminal_and_raises_the_call_again_next_time(self):
self.chunks(5, 4)
stop, said = self.check(FakeEffects(on_terminal=False, git=[UNREAD] * 3))
self.assertIsNone(stop)
self.assertIn("Not on a terminal: the calls are raised again at the next measurement.", said)
self.assertIsNone(state.mark_measured(self.root, 0))
self.assertIn("Not on a terminal", self.check(FakeEffects(on_terminal=False, git=[UNREAD] * 3))[1])
def test_leaves_the_measuring_to_a_loop_holding_the_lock(self):
self.chunks(5, 4)
with archive.locked(self.root):
self.assertEqual(self.check(FakeEffects()), (None, ""))
def test_stops_when_claude_cannot_start_for_the_calls(self):
self.chunks(5, 4)
stop, _ = self.check(FakeEffects(on_terminal_claude=[OSError(2, "No such file or directory")], git=[UNREAD] * 3))
self.assertEqual(stop, Stop(1, f"could not start claude for the routing calls in {self.home / 'code' / 'claude-config'}, the setup's development clone: No such file or directory"))
def test_says_a_figure_it_cannot_write_and_goes_on(self):
assert_not_root(self)
self.chunks(5, 4)
(self.root / "routing-figures.lock").touch()
self.root.chmod(0o555)
self.addCleanup(self.root.chmod, 0o755)
stop, said = self.check(FakeEffects(git=[UNREAD] * 3))
self.assertIsNone(stop)
self.assertIn("routing figures: could not write", said)
def test_puts_every_new_call_to_the_human_and_says_one_it_cannot_keep(self):
new = [Call("reports", ("Explore",)), Call("rounds", ("x", "y"))]
fake = FakeEffects(on_terminal_claude=[0])
with contextlib.redirect_stderr(io.StringIO()) as said, unittest.mock.patch.object(state, "mark_raised", return_value=Failure("full")):
self.assertIsNone(routing_check.raise_calls(self.root, new, Rung("m", "high"), self.loop(fake)))
((args),) = fake.made("claude_on_terminal")
self.assertIn(f"don't change anything until they answer:\n- {new[0].sentence()}\n- {new[1].sentence()}\nThe figures", args[6])
self.assertIn(f"call for a routing change:\n- {new[0].sentence()}\n- {new[1].sentence()}\n", said.getvalue())
self.assertIn("routing figures: full", said.getvalue())
[sent] = fake.dms
self.assertTrue(sent.startswith(f"loop-claude in {self.loop(fake).root} asks:\n\nloop-claude's routing figures call for a routing change:\n- {new[0].sentence()}\n"), sent)
class ModelPrompt(unittest.TestCase):
OFFERED = [("claude-opus-5-5", "Opus 5.5"), ("claude-haiku-4-5", "Haiku 4.5")]
def test_reads_a_number_as_the_offered_model_and_anything_else_as_a_name(self):
self.assertEqual(model_choice.model_picked(" 2 ", self.OFFERED), "claude-haiku-4-5")
self.assertEqual(model_choice.model_picked("opus", self.OFFERED), "opus")
self.assertEqual(model_choice.model_picked("3", []), "3")
self.assertEqual(model_choice.model_picked("3", self.OFFERED), Failure("pick 1 to 2"))
self.assertIsNone(model_choice.model_picked(" ", self.OFFERED))
def test_asks_again_until_an_answer_picks_a_model(self):
with tempfile.TemporaryDirectory() as empty:
fake = FakeEffects(inputs=["", "my-model"])
self.assertEqual(model_choice.ask_for_model("proj", pathlib.Path("/m.json"), pathlib.Path(empty), prompts.DRIVER, fake.effects()), "my-model")
self.assertEqual(len(fake.made("input")), 2)
fake = FakeEffects(inputs=[EOFError()])
self.assertEqual(model_choice.ask_for_model("proj", pathlib.Path("/m.json"), pathlib.Path(empty), prompts.DRIVER, fake.effects()), Stop(2, "the human is needed to pick proj's model, but the input ended"))
# A plan read with no release cut yet.
NO_CUT = LastCut(None, "", 0, [])
def routed_item(todo, cut):
"""loop_claude.routed_item.routed_item for the plan a todo.md's text holds."""
return loop_claude.routed_item.routed_item(plan_of(todo), cut)
TODO = """# todo
## Items
| ID | Release | Exempt | Item | L | E | I | R | S | Moves | Score |
|---|---|---|---|---|---|---|---|---|---|---|
| 7 | 0.2.0 | question | **Ask first.** | 4 | 3 | 5 | 10 | 1 | 1 by 9 | 30 |
| 3 | 0.2.0 | | **Do the next thing.** | 4 | 2 | 3 | 7 | 3 | 1 by 9 | 25 |
| 1 | 0.3.0 | | **Do a later thing.** | 1 | 1 | 1 | 1 | 1 | 1 by 9 | 30 |
"""
def unfinished(likelihood=0, escalated_at=None):
"""An unfinished verdict: the worker took in an item of that L, 0 for none, and the judge found an escalation after reading escalated_at transcript entries, None for none."""
return Unfinished("e", escalation="round 2: 🔴" if escalated_at is not None else "", taken_in=TakenIn(highest_likelihood=likelihood), entries_read=escalated_at)
def interrupted(likelihood=0, escalated_at=None):
"""An interrupt, its arguments as unfinished's."""
return Interrupt("e", "q", escalation="round 2: 🔴" if escalated_at is not None else "", taken_in=TakenIn(highest_likelihood=likelihood), entries_read=escalated_at)
class Routing(unittest.TestCase):
OPUS = Rung("claude-opus-5-5", "medium")
ONE_UP = Rung("claude-fable-5-2", "low")
TWO_UP = Rung("claude-fable-5-2", "medium")
THREE_UP = Rung("claude-opus-5-5", "xhigh")
FOUR_UP = Rung("claude-fable-5-2", "high")
SONNET = Rung("claude-sonnet-5-5", "high")
def test_routes_on_the_next_releases_top_item_that_waits_on_no_question(self):
self.assertEqual(routed_item(TODO, NO_CUT).id, 3)
self.assertIsNone(routed_item(TODO.replace("| | **Do", "| question | **Do"), NO_CUT))
self.assertIsNone(routed_item("# todo\n", NO_CUT))
self.assertIsNone(routed_item(TODO[:TODO.index("| 7 |")], NO_CUT))
old_shape = "## Items\n\n| ID | Release | Exempt | Item | R | S | C | A | G | Goals | Score |\n|---|---|---|---|---|---|---|---|---|---|---|\n| 3 | 0.2.0 | | **Do the next thing.** | 5 | 3 | 6 | 8 | 9 | 1 | 25 |\n"
self.assertIsNone(routed_item(old_shape, NO_CUT))
self.assertEqual(routed_item(TODO.replace("| 4 | 2 | 3 |", "| x | 2 | 3 |"), NO_CUT), Failure("item 3's L 'x' is not a whole number from 1 to 4"))
def test_routes_on_no_item_where_the_release_after_the_last_cut_has_none(self):
self.assertEqual(routed_item(TODO, LastCut("v0.1.0", "", 3, [TODO])).id, 3)
# Someone cut 0.2.0 while it held items: they count as 0.3.0's, the next release, and where no later release is known, no item routes.
self.assertEqual(routed_item(TODO, LastCut("v0.2.0", "", 3, [TODO])).id, 3)
self.assertIsNone(routed_item(re.sub(r"(?m)^\| \d+ \| 0\.3\.0 \|.*\n", "", TODO), LastCut("v0.2.0", "", 3, [])))
later_only = TODO[:TODO.index("| 7 |")] + TODO[TODO.index("| 1 | 0.3.0 |"):]
self.assertIsNone(routed_item(later_only, LastCut("v0.1.0", "", 3, [TODO])))
self.assertIsNone(routed_item(later_only, LastCut("v0.1.0", "", 0, [TODO])))
self.assertEqual(routed_item(later_only, LastCut("v0.2.0", "", 3, [later_only])).id, 1)
def test_routes_past_an_item_a_waiting_item_blocks(self):
todo = "## Items\n\n| ID | Release | Blocked by | Exempt | Item | L | E | I | R | S | Moves | Score |\n|---|---|---|---|---|---|---|---|---|---|---|---|\n| 7 | 0.2.0 | | question | **Ask first.** | 4 | 3 | 5 | 10 | 1 | 1 by 9 | 30 |\n| 8 | 0.2.0 | 7 | | **Use the answer.** | 1 | 1 | 1 | 1 | 1 | 1 by 9 | 30 |\n| 9 | 0.2.0 | 8 | | **Build on it.** | 1 | 1 | 1 | 1 | 1 | 1 by 9 | 30 |\n| 3 | 0.2.0 | | | **Do the next thing.** | 4 | 2 | 3 | 7 | 3 | 1 by 9 | 25 |\n"
self.assertEqual(routed_item(todo, NO_CUT).id, 3)
self.assertEqual(routed_item(todo.replace("| question |", "| |"), NO_CUT).id, 7)
first = "| 7 | 0.2.0 | | question | **Ask first.** | 4 | 3 | 5 | 10 | 1 | 1 by 9 | 30 |\n"
self.assertEqual(routed_item(todo.replace(first, "") + first, NO_CUT).id, 3)
def test_starts_a_chunk_on_the_rung_its_items_l_picks_below_the_projects_too(self):
self.assertEqual(routing.first_route(OPUS_PROJECT, routed_item(TODO, NO_CUT)), Route(self.TWO_UP, 3, 4, 5))
cheap = routed_item(TODO.replace("| 4 | 2 | 3 |", "| 1 | 2 | 3 |"), NO_CUT)
self.assertEqual(routing.first_route(OPUS_PROJECT, cheap), Route(Rung("claude-sonnet-5-5", "high"), 3, 1, 5))
self.assertEqual(routing.first_route(OPUS_PROJECT, None), Route(self.OPUS))
def test_steps_up_a_rung_after_every_second_unfinished_verdict(self):
once = routing.next_route(OPUS_PROJECT, Route(self.ONE_UP, 3, 3, 4), unfinished())
self.assertEqual(once, Route(self.ONE_UP, 3, 3, 4, unfinished=1))
self.assertEqual(routing.next_route(OPUS_PROJECT, once, unfinished()), Route(self.TWO_UP, 3, 3, 4))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(LADDER.rungs[-1], unfinished=1), unfinished()).rung, LADDER.rungs[-1])
def test_steps_up_to_the_rung_a_taken_in_items_l_picks_and_never_down(self):
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.OPUS), unfinished(4)).rung, self.TWO_UP)
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.THREE_UP), unfinished(4)).rung, self.THREE_UP)
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.OPUS, unfinished=1), unfinished(4)).rung, self.THREE_UP)
def test_steps_up_from_a_rung_below_the_projects_one_rung_at_a_time(self):
low = Route(self.SONNET, 3, 1, 3, unfinished=1)
self.assertEqual(routing.next_route(OPUS_PROJECT, low, unfinished()).rung, self.OPUS)
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(low.rung, 3, 1, 3), unfinished()).rung, low.rung)
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(low.rung, 3, 1, 3), unfinished(3)).rung, self.ONE_UP)
def test_steps_up_a_rung_on_an_escalation_the_judge_found_up_to_two_above_the_items_rung_marking_the_transcript_entries_it_read(self):
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.TWO_UP, 3, 4, 5), unfinished(escalated_at=7)), Route(self.THREE_UP, 3, 4, 5, escalated_at=7))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.THREE_UP, 3, 4, 5, escalated_at=7), unfinished(escalated_at=9)), Route(self.FOUR_UP, 3, 4, 5, escalated_at=9))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.FOUR_UP, 3, 4, 5, escalated_at=9), unfinished(escalated_at=12)), Route(self.FOUR_UP, 3, 4, 5, unfinished=1, escalated_at=9))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.TWO_UP, 3, 4, 5, unfinished=1), unfinished(escalated_at=7)), Route(self.THREE_UP, 3, 4, 5, escalated_at=7))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.TWO_UP), unfinished(escalated_at=7)), Route(self.TWO_UP, unfinished=1))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.SONNET, 3, 2, 4), unfinished(4, escalated_at=7)), Route(self.TWO_UP, 3, 2, 4, unfinished=1))
def test_keeps_the_mark_through_a_step_no_escalation_took(self):
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.ONE_UP, 3, 3, 4, unfinished=1, escalated_at=5), unfinished()), Route(self.TWO_UP, 3, 3, 4, escalated_at=5))
def test_steps_up_a_rung_as_the_judge_interrupts_up_to_two_above_the_items_rung_without_counting_an_unfinished_verdict(self):
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.TWO_UP, 3, 4, 5, unfinished=1), interrupted(escalated_at=7)), Route(self.THREE_UP, 3, 4, 5, escalated_at=7))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.THREE_UP, 3, 4, 5, escalated_at=7), interrupted(escalated_at=9)), Route(self.FOUR_UP, 3, 4, 5, escalated_at=9))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.FOUR_UP, 3, 4, 5, unfinished=1, escalated_at=9), interrupted(escalated_at=12)), Route(self.FOUR_UP, 3, 4, 5, unfinished=1, escalated_at=9))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.SONNET, 3, 2, 4), interrupted(escalated_at=7)), Route(self.OPUS, 3, 2, 4, escalated_at=7))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.TWO_UP, 3, 4, 5, unfinished=1, escalated_at=2), interrupted(4)), Route(self.TWO_UP, 3, 4, 5, unfinished=1, escalated_at=2))
def test_raises_the_worker_to_a_taken_in_items_rung_above_the_escalation_cap_without_a_step_as_the_judge_interrupts(self):
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.OPUS, 3, 1, 3, escalated_at=2), interrupted(4, escalated_at=7)), Route(self.TWO_UP, 3, 1, 3, escalated_at=2))
def test_steps_up_from_the_rung_a_taken_in_item_picks_as_the_judge_interrupts(self):
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.ONE_UP, 3, 3, 4), interrupted(4, escalated_at=7)), Route(self.THREE_UP, 3, 3, 4, escalated_at=7))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.TWO_UP, 3, 4, 5), interrupted(4, escalated_at=7)), Route(self.THREE_UP, 3, 4, 5, escalated_at=7))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.FOUR_UP, 3, 4, 5, unfinished=1), interrupted(4, escalated_at=7)), Route(self.FOUR_UP, 3, 4, 5, unfinished=1))
def test_keeps_an_e_plus_i_taken_in_only_where_it_picks_a_higher_review_rung(self):
taken = Route(self.OPUS, 3, 3, 4).taking_in(6)
self.assertEqual(taken, Route(self.OPUS, 3, 3, 4, escape_and_impact_taken_in=6))
self.assertEqual(taken.taking_in(7), Route(self.OPUS, 3, 3, 4, escape_and_impact_taken_in=7))
self.assertEqual(taken.taking_in(5), taken)
self.assertEqual(taken.taking_in(6), taken)
self.assertEqual(Route(self.OPUS, 3, 3, 4).taking_in(4), Route(self.OPUS, 3, 3, 4))
self.assertEqual(Route(self.OPUS, 3, 2, 2).taking_in(4), Route(self.OPUS, 3, 2, 2))
self.assertEqual(Route(self.OPUS).taking_in(5), Route(self.OPUS, escape_and_impact_taken_in=5))
self.assertEqual(Route(self.OPUS).taking_in(3), Route(self.OPUS))
self.assertEqual(Route(self.OPUS).taking_in(0), Route(self.OPUS))
def test_a_rung_an_edit_took_off_the_ladder_counts_as_the_projects(self):
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(Rung("claude-opus-5-5", "low")), unfinished()).rung, self.OPUS)
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(Rung("claude-opus-5-5", "low")), unfinished(2)).rung, self.OPUS)
def test_runs_the_judge_of_a_replayed_model_on_its_models_highest_rung(self):
self.assertEqual(routing.judge_rung(LADDER, "claude-opus-5-5"), self.THREE_UP)
# A model off the ladder, or not replayed, gets no effort flag.
self.assertEqual(routing.judge_rung(LADDER, "judge-model"), Rung("judge-model"))
self.assertEqual(routing.judge_rung(LADDER, "claude-sonnet-5-5"), Rung("claude-sonnet-5-5"))
def test_refuses_a_route_whose_item_has_levels_off_their_scales_or_levels_without_an_item(self):
for item, likelihood, escape_and_impact in ((3, 0, 5), (3, 5, 5), (3, 4, 1), (3, 4, 9), (0, 4, 5), (0, 4, 0), (0, 0, 5)):
with self.subTest(item=item, likelihood=likelihood, escape_and_impact=escape_and_impact):
with self.assertRaises(ValueError):
Route(self.ONE_UP, item, likelihood, escape_and_impact)
def test_refuses_a_route_taking_in_an_e_plus_i_off_its_scale_or_not_above_its_items(self):
for taken_in in (1, 9, -1, 5, 4):
with self.subTest(taken_in=taken_in):
with self.assertRaises(ValueError):
Route(self.ONE_UP, 3, 4, 5, escape_and_impact_taken_in=taken_in)
def test_takes_a_route_whose_item_holds_each_levels_lowest_value(self):
self.assertEqual(Route(self.ONE_UP, 3, 1, 2).escape_and_impact, 2)
def test_routes_nothing_on_an_item_without_e_or_i(self):
without_i = TODO.replace("| E | I | R |", "| E | R |").replace("|---|---|---|---|---|---|---|---|---|---|---|", "|---|---|---|---|---|---|---|---|---|---|").replace("| 4 | 3 | 5 | 10 |", "| 4 | 3 | 10 |").replace("| 4 | 2 | 3 | 7 |", "| 4 | 2 | 7 |").replace("| 1 | 1 | 1 | 1 |", "| 1 | 1 | 1 |")
self.assertIsNone(routed_item(without_i, NO_CUT))
def test_archives_the_route_as_a_json_worker_line_it_reads_back_field_by_field(self):
route = Route(self.ONE_UP, 3, 4, 5)
self.assertEqual(route.worker_line(), '{"model": "claude-fable-5-2", "effort": "low", "item": 3, "likelihood": 4, "escape_and_impact": 5}')
self.assertEqual(route_from(route.worker_line()), route)
self.assertEqual(Route(Rung("worker-model")).worker_line(), '{"model": "worker-model"}')
self.assertEqual(route_from('{"model": "worker-model"}'), Route(Rung("worker-model")))
marked = Route(self.ONE_UP, 3, 4, 5, escape_and_impact_taken_in=7, escalated_at=12)
self.assertEqual(marked.worker_line(), '{"model": "claude-fable-5-2", "effort": "low", "item": 3, "likelihood": 4, "escape_and_impact": 5, "escape_and_impact_taken_in": 7, "escalated_at": 12}')
self.assertEqual(route_from(marked.worker_line()), marked)
self.assertEqual(route_from(Route(self.ONE_UP, escalated_at=3).worker_line()), Route(self.ONE_UP, escalated_at=3))
def test_leaves_the_count_of_unfinished_verdicts_out_of_the_worker_line_so_a_rerun_restarts_it(self):
self.assertEqual(route_from(Route(self.ONE_UP, 3, 4, 5, unfinished=1).worker_line()), Route(self.ONE_UP, 3, 4, 5))
def test_reads_no_route_from_a_worker_line_that_route_worker_line_could_not_have_written(self):
for line in ('{"effort": "high"}', '{"model": "claude-opus-5-5", "effort": "high", "unfinished": 1}', '{"model": "claude-opus-5-5", "item": "3"}', '{"model": "claude-opus-5-5", "effort": 3}', '{"model": 5}', '{"model": "claude-opus-5-5", "effort": "high", "escalated_at": -3}', '{"model": "claude-opus-5-5", "item": true, "likelihood": 4, "escape_and_impact": 5}', '{"model": "claude-opus-5-5", "effort": "high", "item": 3, "likelihood": 5, "escape_and_impact": 5}', '{"model": "claude-opus-5-5", "effort": "high", "escape_and_impact_taken_in": 9}', '["claude-opus-5-5"]', '{"model": ', "worker-model", "claude-fable-5-2 at low effort, routed by item 3's L of 4 and E + I of 5"):
with self.subTest(line=line):
self.assertIsNone(route_from(line))
def test_keeps_a_chunks_route_or_its_archived_one_at_or_above_the_rung_its_item_picks_and_else_routes_afresh(self):
item = routed_item(TODO, NO_CUT)
archived = Route(self.THREE_UP, 3, 4, 5)
def never():
raise AssertionError("not asked")
self.assertEqual(chunk_start.routed(archived, None, "worker-model", never, never), Route(Rung("worker-model")))
self.assertEqual(chunk_start.routed(archived, OPUS_PROJECT, "claude-opus-5-5", never, never), archived)
self.assertEqual(chunk_start.routed(None, OPUS_PROJECT, "claude-opus-5-5", lambda: archived, never), archived)
self.assertEqual(chunk_start.routed(None, OPUS_PROJECT, "claude-opus-5-5", lambda: None, lambda: item), Route(self.TWO_UP, 3, 4, 5))
self.assertEqual(chunk_start.routed(Route(self.SONNET), OPUS_PROJECT, "claude-opus-5-5", never, lambda: None), Route(self.OPUS))
cheap = Route(self.SONNET, 4, 2, 3)
self.assertEqual(chunk_start.routed(cheap, OPUS_PROJECT, "claude-opus-5-5", never, never), cheap)
self.assertEqual(chunk_start.routed(Route(Rung("claude-haiku-5-5", "max"), 4, 2, 3), OPUS_PROJECT, "claude-opus-5-5", never, lambda: None), Route(self.OPUS))
class RouteSources(unittest.TestCase):
def test_routes_on_the_plan_item_and_says_why_it_routes_on_the_projects_rung(self):
self.assertIsNone(chunk_start.plan_item(DefaultPlan(plan_of(""), NO_CUT), None, FakeEffects().effects()))
self.assertEqual(chunk_start.plan_item(DefaultPlan(plan_of(TODO), LastCut("v0.1.0", "", 3, [TODO])), None, FakeEffects().effects()).id, 3)
self.assertIsNone(chunk_start.plan_item(DefaultPlan(plan_of(TODO), LastCut("v0.3.0", "", 3, [TODO])), None, FakeEffects().effects()))
def test_routes_a_window_named_for_an_item_on_that_item_whatever_the_plan_takes_next(self):
plan = DefaultPlan(plan_of(TODO), LastCut("v0.1.0", "", 3, [TODO]))
later = next(item.id for item in plan.plan.items if item.id != 3)
self.assertEqual(chunk_start.plan_item(plan, later, FakeEffects().effects()).id, later)
# An item that left the plan, as once it shipped.
self.assertIsNone(chunk_start.plan_item(plan, 999, FakeEffects().effects()))
# A chunk start reads the default branch once, for routing and for scoring the items the worker names.
with unittest.mock.patch.object(chunk_start, "session_open", return_value=True), unittest.mock.patch.object(chunk_start, "fetched_plan", return_value=plan) as fetched:
loop = a_loop(FakeEffects().effects())
self.assertEqual(chunk_start.chunk_start(loop), ("", plan))
# An open session is no fresh chunk, so the human isn't told again of releases the plan cannot show.
fetched.assert_called_once_with(loop, False)
unreadable = Failure("git show origin/main:todo.md printed text that is not UTF-8")
for plan, said in ((DefaultPlan(plan_of(TODO.replace("| 4 | 2 | 3 |", "| x | 2 | 3 |")), NO_CUT), "routing on the project's rung: item 3's L 'x' is not a whole number from 1 to 4\n"), (DefaultPlan(unreadable, NO_CUT), f"routing on the project's rung: {unreadable.reason}\n"), (Failure("no origin remote"), "routing on the project's rung: no origin remote\n")):
with self.subTest(said=said):
with contextlib.redirect_stderr(io.StringIO()) as stderr:
self.assertIsNone(chunk_start.plan_item(plan, None, FakeEffects().effects()))
self.assertTrue(stderr.getvalue().startswith(f"{STAMP} {said}"), stderr.getvalue())
def test_says_why_the_items_the_worker_names_lift_no_rung(self):
unscored = plan_of(TODO.replace("| 4 | 2 | 3 |", "| x | 2 | 3 |"))
for default, said in ((DefaultPlan(unscored, NO_CUT), "the items the worker names lift no rung: item 3's L 'x' is not a whole number from 1 to 4\n"), (Failure("no git"), "the items the worker names lift no rung: no git\n"), (DefaultPlan(plan_of(TODO), NO_CUT), "")):
with self.subTest(said=said):
with contextlib.redirect_stderr(io.StringIO()) as stderr:
scored = chunk_start.scored_plan(default, FakeEffects().effects())
self.assertEqual(stderr.getvalue(), f"{STAMP} {said}" if said else "")
self.assertEqual(scored, default.plan if isinstance(default, DefaultPlan) else default)
def test_reads_the_changelog_the_default_branch_holds_through_the_loops_git(self):
def git(*args, **options):
answers = {"remote": "origin\n", "symbolic-ref": "origin/main\n", "show": f"what {args[-1]} holds\n"}
return answers.get(args[0], "1\n")
with tempfile.TemporaryDirectory() as root:
folder = configured(self, pathlib.Path(root))
effects = dataclasses.replace(FakeEffects().effects(), git=git)
held = chunk_start.fetched_plan(a_loop(effects, root=root, paths=paths.paths_from(folder.parents[1])))
self.assertEqual(held.cut.changelog, "what origin/main:CHANGELOG.md holds\n")
def test_reads_a_plan_outside_the_checkout_as_it_stands_and_stops_without_a_config(self):
with tempfile.TemporaryDirectory() as root:
folder = configured(self, pathlib.Path(root))
loop = a_loop(FakeEffects(git=["origin\n", "origin/main\n"] * 2).effects(), root=root, paths=paths.paths_from(folder.parents[1]))
config = folder / "config.toml"
tracked = a_loop(FakeEffects(git=["origin\n", "origin/main\n", *[Failure("no git")] * 4]).effects(), root=root, paths=paths.paths_from(folder.parents[1]))
with contextlib.redirect_stderr(io.StringIO()) as stderr:
chunk_start.fetched_plan(tracked)
self.assertEqual(stderr.getvalue(), "", "a plan git tracks hides no release")
config.write_text(config.read_text().replace('"checkout:todo.md"', '"folder:todo.md"'))
(folder / "todo.md").write_text(TODO)
unread = Failure("no git")
standing = a_loop(FakeEffects(git=["origin\n", "origin/main\n", unread, unread, unread, unread]).effects(), root=root, paths=paths.paths_from(folder.parents[1]))
with contextlib.redirect_stderr(io.StringIO()) as stderr:
held = chunk_start.fetched_plan(standing)
self.assertEqual((held.plan, held.cut.todos_since, held.cut.changelog), (plan_of(TODO), [], ""), "the plan as it stands, whatever git answers")
self.assertEqual(stderr.getvalue(), f"{STAMP} {UNSEEN_RELEASES}\n", "a release review loop-claude cannot call due is never skipped silently")
quiet = a_loop(FakeEffects(git=["origin\n", "origin/main\n", unread, unread, unread, unread]).effects(), root=root, paths=paths.paths_from(folder.parents[1]))
with contextlib.redirect_stderr(io.StringIO()) as stderr:
chunk_start.fetched_plan(quiet, announce=False)
self.assertEqual(stderr.getvalue(), "", "said once per chunk, as it starts")
config.unlink()
with contextlib.redirect_stderr(io.StringIO()) as stderr:
self.assertIn("no config names the checkout", chunk_start.scored_plan(chunk_start.fetched_plan(loop), loop.effects).reason)
self.assertIn("the items the worker names lift no rung: no config names the checkout", stderr.getvalue())
self.assertIn("no config names the checkout", chunk_start.fetched_plan(loop).reason)
def test_resumes_on_the_archived_route_and_says_why_it_routes_afresh(self):
with tempfile.TemporaryDirectory() as home:
opened = session.session_named("A proj loop-claude", pathlib.Path(home, "projects"))
archive = pathlib.Path(home, "archive")
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
route = Route(Rung("claude-fable-5-2", "low"), 3, 4, 5)
with contextlib.redirect_stderr(io.StringIO()) as stderr:
self.assertIsNone(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)))
transcript.parent.mkdir(parents=True)
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
self.assertIsNone(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)))
with contextlib.redirect_stderr(io.StringIO()):
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(route.worker_line(), "working", "j", ""), NOW))
with contextlib.redirect_stderr(stderr):
self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), route)
self.assertEqual(stderr.getvalue(), "")
with contextlib.redirect_stderr(io.StringIO()):
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.stepped_up(WorkerLine('{"model": "claude-opus-5-5", "rung": 3}')), NOW + 1))
with contextlib.redirect_stderr(io.StringIO()) as stderr:
self.assertIsNone(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)))
self.assertEqual(stderr.getvalue(), f'{STAMP} routing the chunk afresh: the archive\'s latest Worker: line names no route that can be read: {{"model": "claude-opus-5-5", "rung": 3}}\n')
(chunk,) = archive.iterdir()
chunk.chmod(0)
try:
with contextlib.redirect_stderr(io.StringIO()) as stderr:
self.assertIsNone(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)))
finally:
chunk.chmod(0o700)
self.assertTrue(stderr.getvalue().startswith(f"{STAMP} routing the chunk afresh: could not read the archive"), stderr.getvalue())
class TmuxCommands(unittest.TestCase):
def test_refuses_a_tmux_older_than_needed_naming_both_versions(self):
for found, reason in (("tmux 3.2", None), ("tmux 3.8", None), ("tmux next-3.4", None), ("tmux 10.0", None),
("tmux 3.1c", "loop-claude runs each chunk in a tmux window, and needs tmux 3.2 or later; it found tmux 3.1c"),
("tmux 2.9a", "loop-claude runs each chunk in a tmux window, and needs tmux 3.2 or later; it found tmux 2.9a"),
("", "loop-claude runs each chunk in a tmux window, and needs tmux 3.2 or later; it found no tmux")):
with self.subTest(found=found):
fake = FakeEffects(tmux_version=found)
self.assertEqual(tmux.unusable(fake.tmux, prompts.DRIVER), reason)
self.assertEqual(fake.tmux_calls, [["-V"]])
def test_names_the_session_for_the_project(self):
self.assertEqual(tmux.session_name(tmux.LOOP, "claude-config"), "loop-claude-config")
# tmux reads `.` and `:` in a target as separators.
self.assertEqual(tmux.session_name(tmux.LOOP, "my.site:2 x"), "loop-my-site-2-x")
self.assertEqual(tmux.session_name(tmux.LOOP, "Acme/Billing"), "loop-Acme-Billing")
def test_lists_the_windows_without_the_waiting_mark(self):
fake = FakeEffects(windows="item-372 ?\t0\t@1\nitem-380\t1\t@2\nno-item\t0\t@5\n")
self.assertEqual(tmux.listed("loop-proj", fake.tmux), [tmux.Listed("item-372", "@1", False, asks=True), tmux.Listed("item-380", "@2", True), tmux.Listed("no-item", "@5", False)])
self.assertEqual(fake.tmux_calls, [["list-windows", "-t", "=loop-proj", "-F", "#{window_name}\t#{pane_dead}\t#{window_id}"]])
self.assertEqual(tmux.listed("loop-proj", FakeEffects().tmux), [])
failed = lambda args: subprocess.CompletedProcess(args, 1, "item-1\t0\t@1\n", "no server")
self.assertEqual(tmux.listed("loop-proj", failed), [])
def test_the_first_window_makes_the_session_and_each_later_one_joins_it(self):
fake = FakeEffects()
window = tmux.Window("loop-proj", "item-7", "")
self.assertIsNone(tmux.start_window(window, "/r", {"LOOP_CLAUDE_X": "1"}, ["/b/loop-claude", "--chunk", "item-7", "ship it;"], True, True, fake.tmux))
self.assertIsNone(tmux.start_window(window, "/r", {}, ["/b/loop-claude"], False, True, fake.tmux))
self.assertEqual(fake.tmux_calls, [
["new-session", "-d", "-s", "loop-proj", "-n", "item-7", "-c", "/r", "-e", "LOOP_CLAUDE_X=1", "-P", "-F", "#{window_id}", "--", "/b/loop-claude", "--chunk", "item-7", "ship it\;"],
["set-option", "-t", "=loop-proj:", "@loop-root", "/r"],
["set-option", "-w", "-t", "@9", "@chunk-root", "/r"],
["set-option", "-w", "-t", "@9", "remain-on-exit", "failed"],
["new-window", "-d", "-t", "=loop-proj:", "-n", "item-7", "-c", "/r", "-P", "-F", "#{window_id}", "--", "/b/loop-claude"],
["set-option", "-w", "-t", "@9", "@chunk-root", "/r"],
["set-option", "-w", "-t", "@9", "remain-on-exit", "failed"],
])
def test_says_why_tmux_started_no_window(self):
failed = lambda args: subprocess.CompletedProcess(args, 1, "", "duplicate session: loop-proj\n")
silent = lambda args: subprocess.CompletedProcess(args, 2, "", "")
window = tmux.Window("loop-proj", "item-7", "")
self.assertEqual(tmux.start_window(window, "/r", {}, ["x"], True, True, failed), Failure("tmux could not start the window loop-proj:item-7: duplicate session: loop-proj"))
self.assertEqual(tmux.start_window(window, "/r", {}, ["x"], True, True, silent), Failure("tmux could not start the window loop-proj:item-7: it exited 2"))
def test_fails_where_it_cannot_mark_a_session_it_made_as_the_checkouts_but_not_where_remain_on_exit_fails(self):
def answering(failing: str):
def run(args):
fails = args[:1] == ["set-option"] and failing in args
return subprocess.CompletedProcess(args, 1 if fails else 0, "" if fails else "@9\n", "no such session\n" if fails else "")
return run
window = tmux.Window("loop-proj", "item-7", "")
self.assertEqual(tmux.start_window(window, "/r", {}, ["x"], True, True, answering(tmux.OWNER)), Failure("tmux started the session loop-proj but could not mark it as this checkout's: no such session"))
self.assertIsNone(tmux.start_window(window, "/r", {}, ["x"], True, True, answering("remain-on-exit")))
self.assertIsNone(tmux.start_window(window, "/r", {}, ["x"], False, True, answering(tmux.OWNER)))
def test_reads_the_checkout_the_session_names(self):
fake = FakeEffects(owner="/c/proj")
self.assertEqual(tmux.owner("loop-proj", fake.tmux), "/c/proj")
self.assertEqual(fake.tmux_calls, [["show-options", "-qv", "-t", "=loop-proj:", "@loop-root"]])
def test_marks_a_waiting_window_by_its_pane_and_clears_the_mark(self):
fake = FakeEffects()
tmux.mark(WINDOW, True, fake.tmux)
tmux.mark(WINDOW, False, fake.tmux)
tmux.close(tmux.Listed("item-7", "@3", True), fake.tmux)
self.assertEqual(fake.tmux_calls, [["rename-window", "-t", "%4", "item-7 ?"], ["rename-window", "-t", "%4", "item-7"], ["kill-window", "-t", "@3"]])
self.assertEqual(WINDOW.target, "loop-proj:item-7")
def test_shows_the_session_by_attaching_or_inside_tmux_by_switching(self):
self.assertEqual(tmux.show("loop-proj", False), ["attach-session", "-t", "=loop-proj"])
self.assertEqual(tmux.show("loop-proj", True), ["switch-client", "-t", "=loop-proj"])
class ChunkWindows(unittest.TestCase):
def plan(self, *touched):
"""A plan of items 1, 2 and on, each predicted to touch the path given for it."""
items = touch_plan(*((number, "1.0.0", "", f"- `{path}`") for number, path in enumerate(touched, 1)))
return next_chunk.NextChunk(items=items, first=items[0], text="", branch_template="")
def test_starts_the_next_item_and_those_picked_beside_it_up_to_the_limit(self):
# Item 2 overlaps item 1, so it waits for it; items 3 and 4 run beside it.
plan = self.plan("a.py", "a.py", "c.py", "d.py")
self.assertEqual(chunk_windows.may_start(plan, [], set(), 1), ["item-1"])
self.assertEqual(chunk_windows.may_start(plan, [], set(), 2), ["item-1", "item-3"])
self.assertEqual(chunk_windows.may_start(plan, [], set(), 9), ["item-1", "item-3", "item-4"])
def test_leaves_the_items_running_chunks_hold_and_starts_only_as_many_as_the_room_left(self):
plan = self.plan("a.py", "a.py", "c.py", "d.py")
self.assertEqual(chunk_windows.may_start(plan, ["item-1"], {1, 3}, 1), ["item-4"])
self.assertEqual(chunk_windows.may_start(plan, ["item-1"], {1}, 2), ["item-3", "item-4"])
self.assertEqual(chunk_windows.may_start(plan, ["item-1"], {1}, 0), [])
def test_counts_the_running_chunks_against_the_limit_and_restarts_stopped_ones_first(self):
windows = [tmux.Listed("item-1", "", False), tmux.Listed("item-3", "", True), tmux.Listed("item-4", "", True)]
self.assertEqual(chunk_windows.room_of(windows, "item-1", False, 3), chunk_windows.Room(["item-1"], ["item-3", "item-4"], ["item-3", "item-4"], 0))
self.assertEqual(chunk_windows.room_of(windows, "item-1", False, 2), chunk_windows.Room(["item-1"], ["item-3", "item-4"], ["item-3"], 0))
self.assertEqual(chunk_windows.room_of(windows, "item-1", False, 5), chunk_windows.Room(["item-1"], ["item-3", "item-4"], ["item-3", "item-4"], 2))
def test_a_loop_whose_chunk_finished_leaves_its_room_and_restarts_no_stopped_chunk(self):
windows = [tmux.Listed("item-1", "", False), tmux.Listed("item-3", "", True)]
self.assertEqual(chunk_windows.room_of(windows, "item-1", True, 2), chunk_windows.Room([], ["item-3"], [], 2))
def test_leaves_no_room_where_more_chunks_run_than_the_limit(self):
windows = [tmux.Listed(f"item-{n}", "", False) for n in (1, 2, 3)]
self.assertEqual(chunk_windows.room_of(windows, "item-1", False, 2).left, 0)
def test_starts_no_item_that_overlaps_a_running_chunks_items(self):
# Item 1 merged, so 2 is the plan's next item; it overlaps item 3, which still runs, at c.py.
items = touch_plan((2, "1.0.0", "", "- `a.py`\n- `c.py`"), (3, "1.0.0", "", "- `c.py`"), (4, "1.0.0", "", "- `d.py`"))
plan = next_chunk.NextChunk(items=items, first=items[0], text="", branch_template="")
self.assertEqual(chunk_windows.may_start(plan, ["item-3"], {3}, 2), ["item-4"])
# A running chunk's item without a prediction leaves no other item free to start beside it.
unpredicted = touch_plan((2, "1.0.0", "", "- `a.py`"), (3, "1.0.0", "", None))
self.assertEqual(chunk_windows.may_start(next_chunk.NextChunk(items=unpredicted, first=unpredicted[0], text="", branch_template=""), ["item-3"], {3}, 2), [])
# A held item that left the plan, as once it merged, overlaps nothing.
self.assertEqual(chunk_windows.may_start(plan, ["item-9"], {9}, 2), ["item-2", "item-4"])
def test_a_plan_without_an_item_starts_one_chunk_and_only_alone(self):
none = next_chunk.NoItem("release 1.0.0 has no items")
self.assertEqual(chunk_windows.may_start(none, [], set(), 3), ["no-item"])
self.assertEqual(chunk_windows.may_start(none, ["item-1"], {1}, 3), [])
def test_names_a_window_for_its_item_and_reads_the_item_back(self):
self.assertEqual(chunk_windows.window_name(372), "item-372")
self.assertEqual([chunk_windows.item_of(chunk) for chunk in ("item-372", "no-item", "item-", "item-3x", "xitem-3")], [372, None, None, None, None])
# Only ASCII digits name an item, as window_name writes them.
self.assertIsNone(chunk_windows.item_of("item-\u0663"))
def test_starts_only_the_next_item_where_the_touch_predictions_cannot_be_read(self):
with unittest.mock.patch.object(chunk_windows, "chunks_beside", return_value="item 1's Touches: list cannot be read"):
self.assertEqual(chunk_windows.may_start(self.plan("a.py", "c.py"), [], set(), 3), ["item-1"])
def test_a_chunk_whose_transcript_cannot_be_read_holds_the_item_it_started_from(self):
with tempfile.TemporaryDirectory() as temp:
transcripts = pathlib.Path(temp)
folder = transcripts / "-r"
folder.mkdir()
# A folder in place of the transcript file, so reading it fails.
(folder / f"{chunk_windows.chunk_session('proj', 'item-7', transcripts).id}.jsonl").mkdir()
self.assertEqual(chunk_windows.held_items(["item-7"], "proj", transcripts), {7})
def test_tells_a_chunk_the_items_the_projects_other_live_chunks_hold(self):
with tempfile.TemporaryDirectory() as temp:
home = pathlib.Path(temp)
held = paths.paths_from(home)
folder = held.claude_code_transcripts / "-r"
folder.mkdir(parents=True)
lines = [assistant({"type": "text", "text": "Chunk items: 8 (L 1, E + I 2), 11 (L 2, E + I 3)"})]
(folder / f"{chunk_windows.chunk_session('r', 'item-8', held.claude_code_transcripts).id}.jsonl").write_text("".join(json.dumps(line) + "\n" for line in lines))
# Its own window, item-7, holds nothing for it; item-9's stopped chunk holds its item.
fake = FakeEffects(windows="item-7\t0\t@1\nitem-8 ?\t0\t@2\nitem-9\t1\t@3\nitem-10\t0\t@4\n")
self.assertEqual(chunk_windows.held_beside("item-7", "/r", held, fake.effects()), [8, 9, 10, 11])
self.assertEqual(fake.tmux_calls, [["list-windows", "-t", "=loop-r", "-F", "#{window_name}\t#{pane_dead}\t#{window_id}"]])
class StartChunks(unittest.TestCase):
"""start_chunks: the windows a loop opens for the chunks that may start now."""
LAUNCHER = pathlib.Path("/b/loop-claude")
def setUp(self):
made = tempfile.TemporaryDirectory()
self.addCleanup(made.cleanup)
self.root = pathlib.Path(made.name, "r")
self.home = configured(self, self.root).parents[1]
self.session = "loop-" + str(self.root).lstrip("/").replace("/", "-").replace(".", "-")
# Items 1 and 2 overlap, so 2 waits; 3 runs beside 1.
self.plan = ChunkWindows.plan(self, "a.py", "a.py", "c.py")
def limit(self, chunks):
config = self.home / ".claude-projects" / self.root.name / "config.toml"
config.write_text(config.read_text() + f"parallel-chunks = {chunks}\n")
def start(self, fake, chunk=None, leaving=False, plan=None, env=None, owner=None):
"""start_chunks' answer, for a session whose @loop-root names owner, by default this checkout."""
fake.owner = str(self.root) if owner is None else owner
run = main.LoopRun(0, str(self.root), "ship it", "", chunk=chunk)
with unittest.mock.patch.object(chunk_windows, "next_plan", return_value=plan or self.plan) as planned, contextlib.redirect_stderr(io.StringIO()) as said:
effects = fake.effects()
started = chunk_windows.start_chunks(run, leaving, paths.paths_from(self.home), env or {}, self.LAUNCHER, effects)
self.said = said.getvalue()
self.planned = planned.call_count
if planned.call_count:
planned.assert_called_once_with(str(self.root), self.home, effects)
return started
def starts(self, fake):
"""The window starts of fake's tmux calls, each as (command, window name, what it runs)."""
return [(call[0], call[call.index("-n") + 1], call[call.index("--") + 1:]) for call in fake.tmux_calls if call[0] in ("new-session", "new-window")]
def test_opens_a_window_for_each_chunk_that_may_start_the_first_making_the_session(self):
self.limit(2)
fake = FakeEffects()
self.assertEqual(self.start(fake, env={"LOOP_CLAUDE_WORKER_MODEL": "m", "PATH": "/bin"}), ["item-1", "item-3"])
self.assertEqual(self.starts(fake), [("new-session", "item-1", [str(self.LAUNCHER), "--chunk", "item-1", "ship it"]), ("new-window", "item-3", [str(self.LAUNCHER), "--chunk", "item-3", "ship it"])])
first = next(call for call in fake.tmux_calls if call[0] == "new-session")
self.assertEqual((first[first.index("-s") + 1], first[first.index("-c") + 1]), (self.session, str(self.root)))
# Only loop-claude's own variables are handed on, since a window gets the tmux server's environment.
self.assertEqual([word for word in first if "=" in word], ["LOOP_CLAUDE_WORKER_MODEL=m"])
self.assertEqual(fake.tmux_calls[0], ["list-windows", "-t", f"={self.session}", "-F", "#{window_name}\t#{pane_dead}\t#{window_id}"])
# The session names the checkout whose chunks it runs.
self.assertIn(["set-option", "-t", f"={self.session}:", "@loop-root", str(self.root)], fake.tmux_calls)
def test_refuses_a_session_that_runs_another_checkouts_chunks(self):
for owner, named in (("/c/acme.api", "/c/acme.api"), ("", "another checkout")):
with self.subTest(owner=owner):
fake = FakeEffects(windows="item-5\t0\t@1\n")
self.assertEqual(self.start(fake, owner=owner), Failure(f"the tmux session {self.session} runs the chunks of {named}, whose key maps to the same session name; rename one of the two checkouts' folders"))
self.assertEqual(self.starts(fake), [])
self.assertIn(["show-options", "-qv", "-t", f"={self.session}:", "@loop-root"], fake.tmux_calls)
def test_refuses_a_checkout_do_epic_runs_chunks_in(self):
fake = FakeEffects(windows=f"epic-ABC-12\tABC-34 ?\t0\t@4\t{self.root}\nepic-ABC-12\tABC-35\t0\t@5\t/c/other\nepic-ABC-12\tABC-36\t1\t@6\t{self.root}\nloop-x\titem-1\t0\t@7\t{self.root}\n")
self.assertEqual(self.start(fake), Failure(f"do-epic runs chunks in {self.root}: ABC-34 of epic-ABC-12; loop-claude doesn't start a chunk there until those chunks finish"))
self.assertEqual(self.starts(fake), [])
def test_leaves_out_the_items_a_running_chunk_names_on_its_chunk_items_line(self):
self.limit(2)
transcripts = paths.paths_from(self.home).claude_code_transcripts
folder = transcripts / "-r"
folder.mkdir(parents=True)
opened = chunk_windows.chunk_session(str(self.root).lstrip("/"), "item-5", transcripts)
(folder / f"{opened.id}.jsonl").write_text(json.dumps(assistant({"type": "text", "text": "Chunk items: 5 (L 1, E + I 2), 1 (L 1, E + I 2)"})) + "\n")
self.assertEqual(self.start(FakeEffects(windows="item-5\t0\t@1\n")), ["item-3"])
def test_starts_one_chunk_without_parallel_chunks(self):
fake = FakeEffects()
self.assertEqual(self.start(fake), ["item-1"])
def test_starts_none_and_reads_no_plan_while_the_running_chunks_fill_the_limit(self):
self.limit(2)
fake = FakeEffects(windows="item-5\t0\t@1\nitem-6 ?\t0\t@2\n")
self.assertEqual((self.start(fake), self.planned, self.starts(fake)), ([], 0, []))
# More chunks run than the limit allows, as after it was lowered; a stopped chunk still waits.
fake = FakeEffects(windows="item-5\t0\t@1\nitem-6\t0\t@2\nitem-7\t0\t@3\nitem-8\t1\t@4\nitem-9\t1\t@5\n")
self.assertEqual((self.start(fake), self.planned), ([], 0))
def test_a_finishing_chunks_own_window_counts_as_no_longer_running(self):
fake = FakeEffects(windows="item-5\t0\t@1\n")
self.assertEqual(self.start(fake, chunk="item-5"), [])
fake = FakeEffects(windows="item-5\t0\t@1\n")
self.assertEqual(self.start(fake, chunk="item-5", leaving=True), ["item-1"])
# The session exists, since the finishing loop's window is still in it.
self.assertEqual(self.starts(fake)[0][0], "new-window")
def test_the_humans_loop_claude_starts_a_stopped_chunk_again_first_closing_its_dead_window(self):
fake = FakeEffects(windows="item-5\t1\t@3\n")
self.assertEqual((self.start(fake), self.planned), (["item-5"], 0))
self.assertIn(["kill-window", "-t", "@3"], fake.tmux_calls)
# Its dead window was the session's last, so the session is made again.
self.assertEqual(self.starts(fake)[0][0], "new-session")
self.limit(2)
fake = FakeEffects(windows="item-5\t1\t@3\n")
self.assertEqual(self.start(fake), ["item-5", "item-1"])
def test_a_plan_with_no_item_starts_nothing_beside_a_restarted_chunk(self):
self.limit(2)
fake = FakeEffects(windows="item-5\t1\t@3\n")
self.assertEqual(self.start(fake, plan=next_chunk.NoItem("release 1.0.0 has no items")), ["item-5"])
def test_a_finishing_chunk_leaves_a_stopped_chunk_and_its_items_to_the_human(self):
self.limit(2)
transcripts = paths.paths_from(self.home).claude_code_transcripts
folder = transcripts / "-r"
folder.mkdir(parents=True)
opened = chunk_windows.chunk_session(str(self.root).lstrip("/"), "item-5", transcripts)
(folder / f"{opened.id}.jsonl").write_text(json.dumps(assistant({"type": "text", "text": "Chunk items: 5 (L 1, E + I 2), 1 (L 1, E + I 2)"})) + "\n")
fake = FakeEffects(windows="item-7\t0\t@1\nitem-5\t1\t@3\n")
# Item 1 is the stopped chunk's, so item 3 starts; the stopped one doesn't count against parallel-chunks.
self.assertEqual(self.start(fake, chunk="item-7", leaving=True), ["item-3"])
self.assertEqual([call for call in fake.tmux_calls if call[0] == "kill-window"], [])
fake = FakeEffects(windows="item-7\t0\t@1\nno-item\t1\t@3\n")
self.assertEqual(self.start(fake, chunk="item-7", leaving=True, plan=next_chunk.NoItem("release 1.0.0 has no items")), [])
def test_starts_the_chunk_without_a_plan_item_where_the_plan_cannot_be_read(self):
fake = FakeEffects()
self.assertEqual(self.start(fake, plan=Failure("no origin/main")), ["no-item"])
self.assertIn("starting a chunk without a plan item, since the plan cannot be read: no origin/main", self.said)
def test_says_why_a_window_could_not_start(self):
with unittest.mock.patch.object(tmux, "start_window", return_value=Failure("tmux could not start the window")):
self.assertEqual(self.start(FakeEffects()), Failure("tmux could not start the window"))
def test_says_why_it_cannot_take_the_lock_on_starting_windows(self):
(self.home / ".local" / "state").mkdir(parents=True)
(self.home / ".local" / "state" / "loop-claude").write_text("")
started = self.start(FakeEffects())
self.assertIsInstance(started, Failure)
self.assertTrue(started.reason.startswith(f"cannot lock {self.home}/.local/state/loop-claude/"), started.reason)
def test_a_chunk_holds_its_own_item_and_those_on_its_latest_chunk_items_line(self):
with tempfile.TemporaryDirectory() as temp:
transcripts = pathlib.Path(temp)
opened = chunk_windows.chunk_session("proj", "item-7", transcripts)
self.assertEqual(opened.name, "A proj loop-claude item-7")
folder = transcripts / "-r"
folder.mkdir()
lines = [assistant({"type": "text", "text": "Chunk items: 7 (L 1, E + I 2)"}), assistant({"type": "text", "text": "Chunk items: 7 (L 1, E + I 2), 9 (L 2, E + I 3)"})]
(folder / f"{opened.id}.jsonl").write_text("".join(json.dumps(line) + "\n" for line in lines))
self.assertEqual(chunk_windows.held_items(["item-7", "item-8", "no-item"], "proj", transcripts), {7, 8, 9})
class Steps(unittest.TestCase):
OPUS = Rung("claude-opus-5-5", "medium")
# The rungs an L 3 and an L 4 pick from OPUS.
FABLE_LOW = Rung("claude-fable-5-2", "low")
FABLE_MEDIUM = Rung("claude-fable-5-2", "medium")
def test_a_finished_verdict_clears_the_route_and_the_evidence(self):
run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="old", route=Route(self.OPUS))
self.assertEqual(close.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Finished("x"), run_with.route)), dataclasses.replace(run_with, route=None, unfinished_evidence=""))
self.assertEqual(close.after_verdict(dataclasses.replace(run_with, unfinished_run=3), OPUS_PROJECT, run.Concluded(Finished("x"), run_with.route)).unfinished_run, 0)
self.assertEqual(close.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", taken_in=TakenIn(highest_likelihood=4)), run_with.route)), dataclasses.replace(run_with, route=Route(self.FABLE_MEDIUM, unfinished=1), unfinished_evidence="new", unfinished_run=1))
def test_steps_from_the_route_its_cycle_ended_on_which_a_ruling_may_have_raised(self):
run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(self.OPUS, 3, 3, 4))
raised = Route(self.FABLE_MEDIUM, 3, 3, 4, escalated_at=5)
self.assertEqual(close.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new"), raised)).route, dataclasses.replace(raised, unfinished=1))
def test_keeps_the_route_its_cycle_ended_on_off_the_ladder(self):
off_ladder = Route(Rung("worker-model"))
run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="")
self.assertEqual(close.after_verdict(run_with, None, run.Concluded(Unfinished("new", "round 2: 🔴", TakenIn(highest_likelihood=4), entries_read=9), off_ladder)).route, off_ladder)
def test_an_unfinished_verdicts_escalation_marks_the_transcript_entries_the_judge_read(self):
run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(self.FABLE_LOW, 3, 3, 4, escalated_at=2))
self.assertEqual(close.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", "round 2: 🔴", entries_read=9), run_with.route)).route, Route(self.FABLE_MEDIUM, 3, 3, 4, escalated_at=9))
self.assertEqual(close.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", entries_read=9), run_with.route)).route, Route(self.FABLE_LOW, 3, 3, 4, unfinished=1, escalated_at=2))
def test_archives_a_step_up_as_it_happens_so_a_rerun_resumes_on_it(self):
with tempfile.TemporaryDirectory() as home:
opened = session.session_named("A proj loop-claude", pathlib.Path(home, "projects"))
archive = pathlib.Path(home, "archive")
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
transcript.parent.mkdir(parents=True)
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
before = Route(self.FABLE_LOW, 3, 3, 4)
with contextlib.redirect_stderr(io.StringIO()):
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW))
after = close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), archive, prompts.DRIVER, a_loop(FakeEffects().effects(), session=opened))
self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_MEDIUM, 3, 3, 4))
with contextlib.redirect_stderr(io.StringIO()):
close.closed_cycle(after, OPUS_PROJECT, run.Concluded(Finished("done"), after.route), archive, prompts.DRIVER, a_loop(FakeEffects().effects(), session=opened))
self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_MEDIUM, 3, 3, 4))
blocked = pathlib.Path(home, "blocked")
blocked.write_text("")
stop = close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), blocked, prompts.DRIVER, a_loop(FakeEffects().effects(), session=opened))
self.assertEqual((stop.code, stop.reason.split(":")[0]), (1, "could not archive the transcript"))
def test_names_no_step_up_where_only_the_count_of_unfinished_verdicts_changed(self):
route = Route(self.OPUS, 3, 3, 4)
self.assertIsNone(routing.step_up(route, dataclasses.replace(route, unfinished=1)))
self.assertEqual(routing.step_up(route, route.taking_in(6)), routing.StepUp(route, route.taking_in(6)))
def test_archives_a_higher_e_plus_i_taken_in_so_a_rerun_keeps_it(self):
with tempfile.TemporaryDirectory() as home:
opened = session.session_named("A proj loop-claude", pathlib.Path(home, "projects"))
archive = pathlib.Path(home, "archive")
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
transcript.parent.mkdir(parents=True)
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
before = Route(self.FABLE_LOW, 3, 3, 4)
with contextlib.redirect_stderr(io.StringIO()):
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW))
close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_escape_and_impact=6)), before), archive, prompts.DRIVER, a_loop(FakeEffects().effects(), session=opened))
self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_LOW, 3, 3, 4, escape_and_impact_taken_in=6))
class ChunkFacts(unittest.TestCase):
# The launcher beside which chunk_start finds /b/chunk-preflight.
LAUNCHER = pathlib.Path("/b/loop-claude")
def test_runs_the_preflight_at_the_checkout_root_and_quotes_what_it_printed(self):
fake = FakeEffects(python=[ran("- A fact.\n")])
said = chunk_start.chunk_facts(a_loop(fake.effects(), launcher=self.LAUNCHER))
self.assertEqual(fake.made("python"), [(["/b/chunk-preflight", "--worktree", "--item", "7"], "/r")])
self.assertTrue(said.endswith("\n\n- A fact."), said)
self.assertNotIn("yourself", said)
def test_a_chunk_without_an_item_runs_the_preflight_without_naming_one(self):
fake = FakeEffects(python=[ran("- A fact.\n")])
chunk_start.chunk_facts(a_loop(fake.effects(), launcher=self.LAUNCHER, window=tmux.Window("loop-proj", "no-item", "%4")))
self.assertEqual(fake.made("python"), [(["/b/chunk-preflight", "--worktree"], "/r")])
def test_the_facts_end_with_the_items_the_projects_other_running_chunks_hold(self):
# The loop's own window, a stopped one and two that run, one of them waiting for the human.
fake = FakeEffects(python=[ran("- A fact.\n")], windows="item-7\t0\t@1\nitem-5\t1\t@2\nitem-12 ?\t0\t@3\nitem-9\t0\t@4\n")
said = chunk_start.chunk_facts(a_loop(fake.effects(), launcher=self.LAUNCHER))
self.assertTrue(said.endswith("- A fact.\n\nOther chunks of this project, running or stopped, hold items 5, 9, 12. Leave those items out of this chunk, even where the facts above list them as items that may join it."), said)
self.assertEqual(prompts.held_by_others([9]), "\n\nOther chunks of this project, running or stopped, hold item 9. Leave those items out of this chunk, even where the facts above list them as items that may join it.")
self.assertTrue(prompts.held_by_others([4, 9]).startswith("\n\nOther chunks of this project, running or stopped, hold items 4, 9."))
# A worker whose preflight failed is told them too.
fake = FakeEffects(python=[ran(code=1, stderr="git could not start\n")], windows="item-9\t0\t@4\n")
said = chunk_start.chunk_facts(a_loop(fake.effects(), launcher=self.LAUNCHER))
self.assertTrue(said.endswith("Other chunks of this project, running or stopped, hold item 9. Leave those items out of this chunk, even where the facts above list them as items that may join it."), said)
def test_has_the_worker_find_the_facts_itself_where_the_preflight_failed(self):
for failed, why in ((ran(code=1, stderr="git could not start\n"), "git could not start"), (ran(code=1), "exited 1")):
with self.subTest(why=why):
said = chunk_start.chunk_facts(a_loop(FakeEffects(python=[failed]).effects(), launcher=self.LAUNCHER))
self.assertIn("yourself", said)
self.assertIn(why, said)
def test_runs_the_preflight_only_where_the_session_is_not_open(self):
with tempfile.TemporaryDirectory() as home:
opened = session.session_named("A proj loop-claude", pathlib.Path(home, "projects"))
fresh = FakeEffects(python=[ran("- A fact.\n")])
facts, _ = chunk_start.chunk_start(a_loop(fresh.effects(), session=opened, launcher=self.LAUNCHER))
self.assertEqual((fresh.made("python"), "- A fact." in facts), ([(["/b/chunk-preflight", "--worktree", "--item", "7"], "/r")], True))
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
transcript.parent.mkdir(parents=True)
transcript.write_text("")
resumed = FakeEffects()
facts, _ = chunk_start.chunk_start(a_loop(resumed.effects(), session=opened, launcher=self.LAUNCHER))
self.assertEqual((resumed.made("python"), facts), ([], ""))
class TypedAnswers(unittest.TestCase):
def test_splits_the_prompt_handing_over_the_answer_into_the_answer_on_the_line_after_its_marker_and_the_rest_and_no_other_prompt(self):
route = Route(Rung("worker-model"))
marked = "The human answered your question at loop-claude's terminal:\nB"
typed = worker_prompts.typed_answer(prompts.WORKER.typed("Which one?", "Use B.", route, None, False))
self.assertEqual(typed.answer, "Use B.")
self.assertTrue(typed.loop_words.startswith("Your question was: Which one?\n\nContinue the task."))
self.assertEqual(worker_prompts.typed_answer(prompts.WORKER.typed(f"Which one?\n{marked}", "Use B.", route, None, False)).answer, "Use B.")
self.assertEqual(worker_prompts.typed_answer(prompts.WORKER.typed("Which one?", "Use B.", Route(Routing.SONNET, 179, 2, 5), OPUS_PROJECT, True)).answer, "Use B.")
self.assertIsNone(worker_prompts.typed_answer(prompts.WORKER.answered("Which one?", marked, route, None, False)))
self.assertIsNone(worker_prompts.typed_answer(prompts.WORKER.resumed(marked, route, None)))
class WorkerArgs(unittest.TestCase):
SCAN = " Run a full scan only from such a subagent, since the scan's agents run on the model that starts them."
SEATS = " Launch each comprehension seat and the security pass's subagent with the Agent tool set to model opus, effort medium, the project's rung."
MARKDOWN = " Where your branch changes only Markdown, none of it `CLAUDE.md`, `AGENTS.md`, `workflow-bench/*-prompt.md` or under `agents/`, `principles/`, `rules/` or `skills/`, launch each reviewer on the project's rung instead, with the Agent tool set to model opus, effort medium."
# What the preflight found, as the fresh prompt quotes it.
FACTS = "- A preflight fact."
def test_names_the_route_in_a_fresh_prompt_only_where_an_item_routed_it(self):
routed = prompts.WORKER.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), None, self.FACTS)
self.assertIn(f"Continue any open draft PR or worktree for the chunk before creating a new one. {prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\nloop-claude runs you on m at high effort, routed by item 3's L of 4.", routed)
unrouted = prompts.WORKER.fresh("Do it.", "## Goals", Route(Rung("m")), None, self.FACTS)
self.assertIn(f"before creating a new one. {prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\n{self.FACTS}\n\nThe chunk's", unrouted)
self.assertNotIn("loop-claude runs you on", unrouted)
def test_has_the_routed_worker_count_its_items_size_again_and_puts_the_facts_before_the_scope(self):
routed = prompts.WORKER.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), None, self.FACTS)
self.assertIn("item 3's size again", routed)
self.assertIn("escalation trigger 1", routed)
self.assertIn(f"\n\n{self.FACTS}\n\nThe chunk's", routed)
self.assertNotIn("size again", prompts.WORKER.fresh("Do it.", "## Goals", Route(Rung("m")), None, self.FACTS))
def test_gives_each_fresh_worker_the_reading_rule_once(self):
rule = "never read more than 80 lines of a file at once, nor take a tool output over 3,000 characters into your context"
for route in (Route(Rung("m", "high"), 3, 4, 5), Route(Rung("m"))):
with self.subTest(route=route):
self.assertEqual(prompts.WORKER.fresh("Do it.", "## Goals", route, None, self.FACTS).count(rule), 1)
def test_quotes_what_the_preflight_printed_or_has_the_worker_find_the_facts_itself(self):
self.assertIn("\n\n- A fact.", prompts.preflight_facts("- A fact."))
failed = prompts.preflight_failed("git could not start")
self.assertIn("yourself", failed)
self.assertIn("git could not start", failed)
def test_has_each_fresh_worker_continue_an_open_draft_pr_or_worktree_once_whatever_the_preflight_found(self):
for facts in (prompts.preflight_facts("- A fact."), prompts.preflight_failed("git could not start")):
with self.subTest(facts=facts):
self.assertEqual(prompts.WORKER.fresh("Do it.", "## Goals", Route(Rung("m")), None, facts).count("Continue any open draft PR or worktree"), 1)
def test_keeps_every_review_on_the_projects_rung_while_the_worker_runs_below_it(self):
opus = Rung("claude-opus-5-5", "medium")
below = prompts.WORKER.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 4), OPUS_PROJECT, self.FACTS)
self.assertIn(f"item 3's L of 2. Launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool set to model opus, effort medium, the project's rung.{self.SCAN} Your rung is below the project's: launch each subagent that files or scores plan items on the rung `~/.claude/bin/subagent-rung --plan claude-opus-5-5 <L>` prints. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger", below)
self.assertNotIn("Launch each", prompts.WORKER.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS))
def test_launches_reviewers_on_the_rung_the_items_e_plus_i_picks_above_the_workers(self):
opus = Rung("claude-opus-5-5", "medium")
level = prompts.WORKER.fresh("Do it.", "## Goals", Route(opus, 3, 2, 6), OPUS_PROJECT, self.FACTS)
self.assertIn(f"item 3's L of 2. Launch each reviewer with the Agent tool set to model fable, effort medium, the rung item 3's E + I of 6 picks. Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", level)
below = prompts.WORKER.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 5), OPUS_PROJECT, self.FACTS)
self.assertIn(f"item 3's L of 2. Launch each reviewer with the Agent tool set to model fable, effort low, the rung item 3's E + I of 5 picks.{self.SEATS}{self.SCAN} Your rung is below the project's:", below)
self.assertNotIn("Launch each reviewer", prompts.WORKER.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 4, 6), OPUS_PROJECT))
def test_names_the_reviewers_rung_each_higher_e_plus_i_an_item_taken_in_picks(self):
opus = Rung("claude-opus-5-5", "medium")
taking_in = " Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8."
self.assertIn(f"item 3's L of 2.{taking_in} When an escalation trigger", prompts.WORKER.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS))
self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.", prompts.WORKER.resumed("", Route(opus, 3, 2, 6), OPUS_PROJECT))
self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.", prompts.WORKER.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 4, 4), OPUS_PROJECT))
self.assertNotIn("Where an item you take into", prompts.WORKER.resumed("", Route(opus, 3, 2, 8), OPUS_PROJECT))
self.assertNotIn("Where an item you take into", prompts.WORKER.resumed("", Route(Rung("m")), None))
def test_launches_reviewers_on_the_rung_the_highest_e_plus_i_taken_in_picks(self):
opus = Rung("claude-opus-5-5", "medium")
self.assertIn(f"loop-claude runs you on claude-opus-5-5 at medium effort. Launch each reviewer with the Agent tool set to model opus, effort xhigh, the rung picked by an item of E + I 7 taken into the chunk. Where an item you take into the chunk has an E + I of 8 or more, launch each reviewer with the Agent tool set to `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", prompts.WORKER.resumed("", Route(opus, 3, 2, 5, escape_and_impact_taken_in=7), OPUS_PROJECT))
self.assertIn(" Launch each reviewer with the Agent tool set to model fable, effort low, the rung picked by an item of E + I 5 taken into the chunk.", prompts.WORKER.resumed("", Route(opus, escape_and_impact_taken_in=5), OPUS_PROJECT))
def test_keeps_seats_and_the_security_pass_on_the_projects_rung_while_the_worker_runs_above_it(self):
opus = Rung("claude-opus-5-5", "medium")
above = prompts.WORKER.fresh("Do it.", "## Goals", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT, self.FACTS)
stepped_up = prompts.WORKER.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT)
self.assertIn(f"{prompts.ANSWERS_RECORDED}. {prompts.NAME_THE_ITEMS} loop-claude runs you on claude-fable-5-2 at low effort.{self.SEATS}{self.SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", stepped_up)
self.assertNotIn("the project's", prompts.WORKER.resumed("", Route(opus, 3, 2, 4), OPUS_PROJECT))
self.assertNotIn("runs you on", prompts.WORKER.resumed("", Route(Rung("m")), None))
self.assertIn(f"item 3's L of 3.{self.SEATS}{self.SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", above)
def test_names_the_rungs_an_escalation_trigger_lets_the_worker_hand_a_task_to_at_most_two_above_its_items(self):
opus = Rung("claude-opus-5-5", "medium")
fresh = prompts.WORKER.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS)
self.assertIn("item 3's L of 2. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort low`, one rung above yours; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", fresh)
self.assertIn(" When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort medium`, one rung above yours, and on each further trigger one rung higher, up to `model opus, effort xhigh`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", prompts.WORKER.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT))
self.assertIn(" When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort xhigh`, one rung above yours; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", prompts.WORKER.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 3, 4), OPUS_PROJECT))
self.assertNotIn("escalation", prompts.WORKER.resumed("", Route(Rung("claude-fable-5-2", "max"), 3, 4, 8), OPUS_PROJECT))
self.assertIn(" An escalation trigger in `~/.claude/CLAUDE.md` lets you hand the hard part of a task to a subagent with the Agent tool set to `model fable, effort high`, one rung above yours, only once the human agrees;", prompts.WORKER.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 3, 4), OPUS_PROJECT))
self.assertIn("set to `model fable, effort max`, one rung above yours, only once the human agrees;", prompts.WORKER.resumed("", Route(Rung("claude-fable-5-2", "high"), 3, 4, 5), OPUS_PROJECT), "one rung below the top still names the rung above")
self.assertNotIn("rerun", prompts.WORKER.resumed("", Route(Rung("claude-opus-5-5", "low"), 3, 3, 4), OPUS_PROJECT), "a worker off the ladder is told no rerun rung")
self.assertIn("hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort high`, one rung above yours;", prompts.WORKER.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 4, 5), OPUS_PROJECT))
self.assertNotIn("escalation", prompts.WORKER.resumed("", Route(Rung("m")), None))
self.assertNotIn("escalation", prompts.WORKER.resumed("", Route(Rung("claude-opus-5-5", "low"), 3, 3, 4), OPUS_PROJECT))
self.assertIn(f"{prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\nWhere an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort low`, one rung above yours, and on each further trigger one rung higher, up to `model fable, effort medium`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number. Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `model fable, effort medium`. A reviewer", prompts.WORKER.fresh("Do it.", "## Goals", Route(opus), OPUS_PROJECT, self.FACTS))
def test_offers_the_worker_a_higher_rung_for_the_rest_of_the_chunk_up_to_the_escalation_cap(self):
opus = Rung("claude-opus-5-5", "medium")
offer = lambda cap: f" Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `{cap}`."
self.assertIn(f"`Escalation trigger <n>:`, n the trigger's number.{offer('model fable, effort low')}", prompts.WORKER.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS))
self.assertIn(f"`Escalation trigger <n>:`, n the trigger's number.{offer('model opus, effort xhigh')}", prompts.WORKER.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT))
self.assertNotIn("Where the hard part", prompts.WORKER.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 3, 4), OPUS_PROJECT))
def test_lets_a_markdown_only_branch_review_on_the_projects_rung_where_the_reviewers_run_above_it(self):
opus = Rung("claude-opus-5-5", "medium")
markdown = " Where your branch changes only Markdown, none of it `CLAUDE.md`, `AGENTS.md`, `workflow-bench/*-prompt.md` or under `agents/`, `principles/`, `rules/` or `skills/`, launch each reviewer on the project's rung instead, with the Agent tool set to model opus, effort medium."
self.assertIn(markdown, prompts.WORKER.fresh("Do it.", "## Goals", Route(opus, 3, 2, 7), OPUS_PROJECT, self.FACTS))
self.assertIn(markdown, prompts.WORKER.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT))
self.assertNotIn("only Markdown", prompts.WORKER.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS))
def test_names_the_rung_a_reviewer_or_seat_reruns_on_one_above_its_own(self):
opus = Rung("claude-opus-5-5", "medium")
both = " A reviewer or comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort low`."
self.assertTrue(prompts.WORKER.resumed("", Route(opus, 3, 2, 4), OPUS_PROJECT).endswith(both))
self.assertTrue(prompts.WORKER.resumed("", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 4), OPUS_PROJECT).endswith(both))
self.assertTrue(prompts.WORKER.fresh("Do it.", "## Goals", Route(opus), OPUS_PROJECT, self.FACTS).split(f"\n\n{self.FACTS}")[0].endswith(both))
self.assertTrue(prompts.WORKER.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort medium`, and a comprehension seat with `model fable, effort low`."))
self.assertTrue(prompts.WORKER.resumed("", Route(opus, 3, 2, 7), OPUS_PROJECT).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort high`, and a comprehension seat with `model fable, effort low`."))
self.assertTrue(prompts.WORKER.resumed("", Route(Rung("claude-fable-5-2", "max"), 3, 4, 8), OPUS_PROJECT).endswith(" A comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort low`."))
self.assertNotIn("rerun", prompts.WORKER.resumed("", Route(Rung("m")), None))
self.assertNotIn("rerun", prompts.WORKER.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), None, self.FACTS))
self.assertIn(f"{both} After your first read", prompts.WORKER.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS))
fable = Rung("claude-fable-5-2", "max")
self.assertNotIn("rerun", prompts.WORKER.resumed("", Route(fable), ProjectRung(LADDER, fable)))
def test_opens_a_fresh_session_by_name_and_resumes_an_open_one(self):
opened = session.session_named("A proj loop-claude", pathlib.Path("/h/.claude/projects"))
fresh = worker.worker_args(opened, Rung("m"), prompts.WORKER.fresh("Do it.", "## Goals\n1. g\n\n## Audience\nme", Route(Rung("m")), None, self.FACTS), resume=False)
self.assertEqual(fresh[fresh.index("--session-id") + 1], str(opened.id))
self.assertEqual(fresh[fresh.index("--name") + 1], "A proj loop-claude")
resumed = worker.worker_args(opened, Rung("m"), prompts.WORKER.resumed("", Route(Rung("m")), None), resume=True)
self.assertEqual(resumed[resumed.index("--resume") + 1], str(opened.id))
self.assertNotIn("--session-id", resumed)
self.assertTrue(resumed[resumed.index("-p") + 1].startswith("Continue the task; the previous run ended before the chunk was finished. The chunk is finished once"))
def test_disallows_a_wakeup_so_a_worker_that_ends_its_turn_exits(self):
opened = session.session_named("A proj loop-claude", pathlib.Path("/h/.claude/projects"))
for resume in (False, True):
with self.subTest(resume=resume):
args = worker.worker_args(opened, Rung("m"), "Do it.", resume=resume)
self.assertEqual(args[args.index("--disallowedTools") + 1], "ScheduleWakeup")
class Records(unittest.TestCase):
def test_heads_each_record_by_its_name_the_humans_rules_last(self):
with tempfile.TemporaryDirectory() as root:
claude_md = pathlib.Path(root, "home", ".claude", "CLAUDE.md")
config = configured(self, pathlib.Path(root), home=claude_md.parents[1]) / "config.toml"
self.assertEqual(records.read_records(root, claude_md.parents[1], claude_md.parent), "")
claude_md.parent.mkdir(parents=True)
claude_md.write_text("# Global\n")
self.assertEqual(records.read_records(root, claude_md.parents[1], claude_md.parent), "From ~/.claude/CLAUDE.md:\n# Global")
pathlib.Path(root, "docs").mkdir()
pathlib.Path(root, "docs", "decisions.md").write_text("# Decisions\n")
pathlib.Path(root, "AGENTS.md").write_text("\n# Rules\n")
self.assertEqual(records.read_records(root, claude_md.parents[1], claude_md.parent), "From AGENTS.md:\n# Rules\n\nFrom docs/decisions.md:\n# Decisions\n\nFrom ~/.claude/CLAUDE.md:\n# Global")
pathlib.Path(claude_md.parent, "rules").mkdir()
pathlib.Path(claude_md.parent, "rules", "todo-items.md").write_text("# Todo\n")
pathlib.Path(claude_md.parent, "rules", "goals-and-audience.md").write_text("# Goals\n")
pathlib.Path(claude_md.parent, "rules", "notes.txt").write_text("not a rule\n")
pathlib.Path(claude_md.parent, "skills", "larv-review", "standards").mkdir(parents=True)
pathlib.Path(claude_md.parent, "skills", "larv-review", "standards", "comments.md").write_text("# Comments\n")
self.assertTrue(records.read_records(root, claude_md.parents[1], claude_md.parent).endswith("From ~/.claude/CLAUDE.md:\n# Global\n\nFrom ~/.claude/rules/goals-and-audience.md:\n# Goals\n\nFrom ~/.claude/rules/todo-items.md:\n# Todo\n\nFrom ~/.claude/skills/larv-review/standards/comments.md:\n# Comments"))
config.write_text(config.read_text().replace('"~/.claude/skills/larv-review/standards"', '"~/review-standards"'))
pathlib.Path(claude_md.parents[1], "review-standards").mkdir()
pathlib.Path(claude_md.parents[1], "review-standards", "quality.md").write_text("# Ours\n")
self.assertTrue(records.read_records(root, claude_md.parents[1], claude_md.parent).endswith("From ~/.claude/rules/todo-items.md:\n# Todo\n\nFrom ~/review-standards/quality.md:\n# Ours"), "the config's standards folder replaces the default")
config.write_text(config.read_text().replace('"~/review-standards"', f'"{root}/standards"'))
pathlib.Path(root, "standards").mkdir()
pathlib.Path(root, "standards", "quality.md").write_text("# Elsewhere\n")
self.assertTrue(records.read_records(root, claude_md.parents[1], claude_md.parent).endswith(f"From {root}/standards/quality.md:\n# Elsewhere"), "a folder outside home is named in full")
config.write_text(config.read_text().replace('"checkout:AGENTS.md"', '"folder:AGENTS.md"').replace('"checkout:docs/decisions.md"', '"folder:log/decisions.md"'))
self.assertTrue(records.read_records(root, claude_md.parents[1], claude_md.parent).startswith("From ~/.claude/CLAUDE.md:\n# Global"), "the checkout's AGENTS.md and docs/decisions.md are its owners'")
pathlib.Path(config.parent, "log").mkdir()
pathlib.Path(config.parent, "log", "decisions.md").write_text("# Ours\n")
pathlib.Path(config.parent, "AGENTS.md").write_text("# Our rules\n")
self.assertTrue(records.read_records(root, claude_md.parents[1], claude_md.parent).startswith("From AGENTS.md:\n# Our rules\n\nFrom docs/decisions.md:\n# Ours\n\nFrom ~/.claude/CLAUDE.md:\n# Global"))
config.unlink()
self.assertEqual(records.read_records(root, claude_md.parents[1], claude_md.parent), Failure(f"no config names the checkout {root}: write ~/.claude-projects/<key>/config.toml, as `project-config --help` shows - the judge cannot read the records."))
def test_puts_the_records_handed_in_first_and_reads_only_the_humans_rules_without_a_checkout(self):
with tempfile.TemporaryDirectory() as root:
claude_md = pathlib.Path(root, "home", ".claude", "CLAUDE.md")
configured(self, pathlib.Path(root), home=claude_md.parents[1])
claude_md.parent.mkdir(parents=True)
claude_md.write_text("# Global\n")
pathlib.Path(claude_md.parent, "rules").mkdir()
pathlib.Path(claude_md.parent, "rules", "todo-items.md").write_text("# Todo\n")
pathlib.Path(claude_md.parent, "skills", "larv-review", "standards").mkdir(parents=True)
pathlib.Path(claude_md.parent, "skills", "larv-review", "standards", "comments.md").write_text("# Comments\n")
pathlib.Path(root, "AGENTS.md").write_text("# Rules\n")
first = (("the epic ABC-1's description, ## Decisions", "Squash.\n"), ("the epic ABC-1's description, ## Scoring", " \n"))
self.assertEqual(records.read_records(root, claude_md.parents[1], claude_md.parent, first), "From the epic ABC-1's description, ## Decisions:\nSquash.\n\nFrom AGENTS.md:\n# Rules\n\nFrom ~/.claude/CLAUDE.md:\n# Global\n\nFrom ~/.claude/rules/todo-items.md:\n# Todo\n\nFrom ~/.claude/skills/larv-review/standards/comments.md:\n# Comments")
self.assertEqual(records.read_records(None, claude_md.parents[1], claude_md.parent, first), "From the epic ABC-1's description, ## Decisions:\nSquash.\n\nFrom ~/.claude/CLAUDE.md:\n# Global\n\nFrom ~/.claude/rules/todo-items.md:\n# Todo")
def test_stops_on_a_record_it_cannot_read(self):
with tempfile.TemporaryDirectory() as root:
claude_md = pathlib.Path(root, "home", ".claude", "CLAUDE.md")
configured(self, pathlib.Path(root), home=claude_md.parents[1])
claude_md.mkdir(parents=True)
self.assertEqual(records.read_records(root, claude_md.parents[1], claude_md.parent), Failure(f"CLAUDE.md in {claude_md.parent}: Is a directory - the judge cannot read the records."))
pathlib.Path(root, "docs", "decisions.md").mkdir(parents=True)
self.assertEqual(records.read_records(root, claude_md.parents[1], claude_md.parent), Failure(f"cannot read {root}/docs/decisions.md: Is a directory - the judge cannot read the records."))
pathlib.Path(root, "AGENTS.md").write_bytes(b"# Rules\n\xff\n")
self.assertEqual(records.read_records(root, claude_md.parents[1], claude_md.parent), Failure(f"AGENTS.md in {root} is not UTF-8 on line 2 - the judge cannot read the records."))
def test_stops_on_a_rule_it_cannot_read(self):
with tempfile.TemporaryDirectory() as root:
claude_md = pathlib.Path(root, "home", ".claude", "CLAUDE.md")
configured(self, pathlib.Path(root), home=claude_md.parents[1])
rules = pathlib.Path(claude_md.parent, "rules")
rules.mkdir(parents=True)
pathlib.Path(rules, "todo-items.md").write_bytes(b"\xff\n")
self.assertEqual(records.read_records(root, claude_md.parents[1], claude_md.parent), Failure(f"todo-items.md in {rules} is not UTF-8 on line 1 - the judge cannot read the records."))
class JudgePrompt(unittest.TestCase):
def test_shows_the_transcripts_tail_and_returns_what_it_showed(self):
entries = [assistant({"type": "text", "text": "old " + "x" * judge.JUDGE_PROMPT_CHARS}), assistant({"type": "text", "text": "new"})]
prompt, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", records.RECORDS_NAMED, prompts.ANSWER_DESTINATIONS, kinds(*entries), 0, prompts.DRIVER))
self.assertTrue(grounds.shown.endswith("\n\nnew"))
self.assertNotIn("old", grounds.shown)
self.assertIn(grounds.shown, prompt)
self.assertLessEqual(len(prompt), judge.JUDGE_PROMPT_CHARS)
self.assertEqual(grounds.after_mark, "old " + "x" * judge.JUDGE_PROMPT_CHARS + "\n\nnew")
self.assertEqual(grounds.quotable, "## Goals\n\n")
def test_the_judge_prompt_and_schema_call_the_worker_the_worker(self):
prompt, _ = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", records.RECORDS_NAMED, prompts.ANSWER_DESTINATIONS, kinds(assistant({"type": "text", "text": "done"})), 1, prompts.DRIVER))
self.assertNotRegex(prompt + json.dumps(judge.judge_schema(prompts.DRIVER)), r"\b[Aa]gent")
def test_the_judge_prompt_names_the_entry_points_answer_destinations_in_both_rules_on_a_recorded_answer(self):
prompt, _ = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", records.RECORDS_NAMED, "the ledger", kinds(assistant({"type": "text", "text": "done"})), 0, prompts.DRIVER))
self.assertIn("is recorded in the ledger at the one place it governs", prompt)
self.assertIn("the worker writing it into the ledger, but not into one of the human's rules", prompt)
def test_the_typed_answers_prompt_names_the_entry_points_answer_destinations(self):
ledger = dataclasses.replace(prompts.WORKER, answer_destinations="the ledger")
self.assertIn("is recorded in the ledger at the one place it governs.", ledger.typed("Which one?", "Use B.", Route(Rung("worker-model")), None, False))
def test_the_judge_prompt_names_every_rule_the_judge_reads_from_the_human(self):
self.assertEqual([name for name in records.HUMAN_RULE_NAMES if name not in judge.judge_rules(prompts.ANSWER_DESTINATIONS, prompts.DRIVER)], [])
def test_shows_the_judge_where_the_worker_last_stepped_up_and_checks_an_escalation_only_after_it(self):
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "round 3: 🔴"})]
prompt, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", records.RECORDS_NAMED, prompts.ANSWER_DESTINATIONS, kinds(*entries), 1, prompts.DRIVER))
self.assertRegex(prompt, "round 2: 🔴\n\n\\[step\\] [^\n]+\n\nround 3: 🔴\n\n")
self.assertEqual(grounds.after_mark, "round 3: 🔴")
prompt, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", records.RECORDS_NAMED, prompts.ANSWER_DESTINATIONS, kinds(*entries), 0, prompts.DRIVER))
self.assertNotIn("stepped the worker up", prompt)
self.assertEqual(grounds.shown, "round 2: 🔴\n\nround 3: 🔴")
self.assertEqual(grounds.after_mark, "round 2: 🔴\n\nround 3: 🔴")
def test_drops_an_escalation_quote_from_after_the_step_up_that_the_tail_it_shows_cuts_off(self):
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "old " + "x" * judge.JUDGE_PROMPT_CHARS}), assistant({"type": "text", "text": "new"})]
_, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", records.RECORDS_NAMED, prompts.ANSWER_DESTINATIONS, kinds(*entries), 1, prompts.DRIVER))
self.assertEqual(judge.quotes_checked(Unfinished("e", "old x", entries_read=3), grounds), Unfinished("e", entries_read=3))
self.assertEqual(judge.quotes_checked(Unfinished("e", "new", entries_read=3), grounds), Unfinished("e", "new", entries_read=3))
def test_shows_a_typed_answer_before_the_step_up_as_the_humans(self):
typed = prompts.WORKER.typed("Which one?", "Use B.", Route(Rung("worker-model")), None, False)
entries = [{**human(typed), "promptSource": "sdk"}, assistant({"type": "text", "text": "round 3: 🔴"})]
_, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", records.RECORDS_NAMED, prompts.ANSWER_DESTINATIONS, kinds(*entries), 1, prompts.DRIVER))
self.assertIn("\n\n[human] Use B.\n\n[step] ", grounds.shown)
def turn(message_id, model, at, **usage):
return entries.AssistantTurn(message_id=message_id, model=model, timestamp=at, usage=usage)
class Spend(unittest.TestCase):
def test_reads_the_cache_writes_by_lifetime_and_counts_one_without_the_split_as_five_minutes(self):
self.assertEqual(tokens_of({"input_tokens": 1, "cache_creation_input_tokens": 9, "cache_creation": {"ephemeral_5m_input_tokens": 2, "ephemeral_1h_input_tokens": 3}, "cache_read_input_tokens": 4, "output_tokens": 5}), Tokens(1, 2, 3, 4, 5))
self.assertEqual(tokens_of({"input_tokens": 1, "cache_creation_input_tokens": 9}), Tokens(1, 9, 0, 0, 0))
self.assertEqual(tokens_of(None), Tokens())
def test_counts_a_messages_context_as_every_prompt_token_and_no_output(self):
self.assertEqual(Tokens(1, 2, 3, 4, 5).context(), 10)
def test_counts_a_message_written_in_the_stamped_second_or_without_a_readable_time_as_before_since(self):
thread = [turn("m1", "a", "2026-11-01T10:00:01Z", output_tokens=7), turn("m2", "a", "not a time", output_tokens=3), turn("m3", "a", "2026-11-01T10:00:02Z", output_tokens=1)]
self.assertEqual(spend.spent("2026-11-01T10:00:01Z", thread, [], "j", Tokens()), {("worker", "a"): Tokens(output=1)})
def test_sums_two_subagents_of_one_name_on_one_model(self):
helpers = [spend.Subagent("Explore", [turn("s1", "b", "2026-11-01T10:00:00Z", input_tokens=3)]), spend.Subagent("Explore", [turn("s2", "b", "2026-11-01T10:00:00Z", input_tokens=4)])]
self.assertEqual(spend.spent("", [], helpers, "j", Tokens()), {("Explore", "b"): Tokens(input=7)})
def test_counts_each_message_once_by_its_id_and_only_after_since(self):
thread = [
turn("m1", "a", "2026-11-01T10:00:00Z", output_tokens=7),
turn("m2", "a", "2026-11-01T10:00:02Z", input_tokens=1, output_tokens=2),
turn("m2", "a", "2026-11-01T10:00:02Z", input_tokens=1, output_tokens=2),
turn("m3", "b", "2026-11-01T10:00:03Z", cache_read_input_tokens=5),
]
self.assertEqual(spend.thread_spend(thread, "2026-11-01T10:00:01Z"), {"a": Tokens(1, 0, 0, 0, 2), "b": Tokens(0, 0, 0, 5, 0)})
self.assertEqual(spend.thread_spend(thread, ""), {"a": Tokens(1, 0, 0, 0, 9), "b": Tokens(0, 0, 0, 5, 0)})
def test_names_each_spender_and_leaves_out_one_that_spent_nothing(self):
worker = [turn("m1", "a", "2026-11-01T10:00:00Z", output_tokens=7)]
helper = spend.Subagent('Explore "Search"', [turn("s1", "b", "2026-11-01T10:00:00Z", input_tokens=3)])
self.assertEqual(spend.spent("", worker, [helper], "j", Tokens()), {("worker", "a"): Tokens(output=7), ('Explore "Search"', "b"): Tokens(input=3)})
self.assertEqual(spend.spent("", [], [], "j", Tokens(input=1)), {("judge", "j"): Tokens(input=1)})
def test_names_a_subagent_by_its_agent_type_and_description(self):
with tempfile.TemporaryDirectory() as root:
folder = pathlib.Path(root, "s", "subagents")
folder.mkdir(parents=True)
for name, meta in [("agent-a", {"agentType": "Explore", "description": "Search the hooks"}), ("agent-b", None), ("agent-c", ["Explore"]), ("agent-d", {"description": "Läs"}), ("agent-e", {"agentType": "Ex\nConfig: x"})]:
(folder / f"{name}.jsonl").write_text(json.dumps({"type": "assistant", "message": {"id": name, "model": "m", "content": [], "usage": {"output_tokens": 1}}, "timestamp": "2026-11-01T10:00:00Z"}) + "\n")
if meta:
(folder / f"{name}.meta.json").write_text(json.dumps(meta))
found = spend.subagents(str(pathlib.Path(root, "s.jsonl")))
self.assertEqual([subagent.name for subagent in found], ['Explore "Search the hooks"', "subagent", "subagent", 'subagent "Läs"', "Ex Config: x"])
self.assertEqual(spend.spent("", [], found[:2], "j", Tokens()), {('Explore "Search the hooks"', "m"): Tokens(output=1), ("subagent", "m"): Tokens(output=1)})
self.assertEqual(spend.subagents(str(pathlib.Path(root, "none.jsonl"))), [])
def test_renders_the_spend_then_the_list_prices_of_each_model_it_knows(self):
found = {("worker", "claude-opus-5-5"): Tokens(1, 2, 3, 4, 5), ("judge", "unpriced"): Tokens(input=6)}
self.assertEqual(outcomes.spent_block("2026-11-01T10:00:00Z", found), """Spent since 2026-11-01T10:00:00Z, in tokens of input, cache write 5m, cache write 1h, cache read, output:
- worker, claude-opus-5-5: 1, 2, 3, 4, 5
- judge, unpriced: 6, 0, 0, 0, 0
List prices in dollars per million tokens of input, cache write 5m, cache write 1h, cache read, output:
- claude-opus-5-5: 4, 5, 8, 0.2, 20""")
self.assertEqual(outcomes.spent_block("", {("judge", "unpriced"): Tokens(input=6)}), "Spent since the chunk began, in tokens of input, cache write 5m, cache write 1h, cache read, output:\n- judge, unpriced: 6, 0, 0, 0, 0")
self.assertEqual(outcomes.spent_block("", {}), "")
def test_keeps_the_whole_typed_answer_that_holds_the_judges_quote_and_its_tokens(self):
shown = "[human] Use A.\n[human] No, keep the old name. Use X.\nworking: keep the new name"
typed = ("Use A.", "No, keep the old name. Use X.")
for quoted, kept in [("No, keep the old name. Use X.", "No, keep the old name. Use X."), ("keep the old name", "No, keep the old name. Use X."), ("Use", "No, keep the old name. Use X."), ("keep the new name", ""), ("No, keep the name.", "")]:
with self.subTest(quoted=quoted):
answer = json.loads(judged("unfinished", reversed=quoted).stdout)
fake = FakeEffects(claude=[ran(json.dumps({**answer, "usage": {"input_tokens": 2, "output_tokens": 3}}))])
self.assertEqual(judge.run_judge("/r", [], "", judge.Grounds(shown, shown, "", typed), fake.effects()), Unfinished("working", judged=Judged(tokens=Tokens(input=2, output=3), reversed=kept)))
class SetupVersion(unittest.TestCase):
def test_names_the_release_and_the_commit_or_why_git_cannot_tell(self):
with tempfile.TemporaryDirectory() as root:
isolated = {"GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1", "GIT_CEILING_DIRECTORIES": root}
with unittest.mock.patch.dict(os.environ, isolated):
gone = main.setup_version(pathlib.Path(root), git)
self.assertRegex(gone, r"^unknown: [^\n]*not a git repository")
for step in (["init", "-q"], ["-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", "one"]):
self.assertIsInstance(git(*step, cwd=root), str)
head = git("rev-parse", "HEAD", cwd=root).strip()
self.assertEqual(main.setup_version(pathlib.Path(root), git), f"{head[:7]} ({head})")
git("tag", "v0.1.0", cwd=root)
self.assertEqual(main.setup_version(pathlib.Path(root), git), f"v0.1.0 ({head})")
git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", "two", cwd=root)
head = git("rev-parse", "HEAD", cwd=root).strip()
self.assertEqual(main.setup_version(pathlib.Path(root), git), f"v0.1.0-1-g{head[:7]} ({head})")
def test_names_why_git_cannot_name_the_commit_on_one_line(self):
def fake_git(*args, cwd=None):
return Failure("fatal: bad HEAD\nhint: repair it") if args[0] == "rev-parse" else "v0.1.0\n"
self.assertEqual(main.setup_version(pathlib.Path("/r"), fake_git), "unknown: fatal: bad HEAD hint: repair it")
class Grounded(unittest.TestCase):
def test_every_verdict_must_quote_the_transcript(self):
self.assertTrue(judge.grounded("Needs you: x", "so.\n\nNeeds you: x\n"))
self.assertFalse(judge.grounded("invented", "the transcript"))
self.assertFalse(judge.grounded("", "the transcript"))
self.assertFalse(judge.grounded(" ", "the transcript"))
transcript = "done.\n\n**Your step:**\n```\n./run.sh\n```\nRun it."
self.assertTrue(judge.grounded("Your step:\n```\n./run.sh\n```\nRun it.", transcript))
self.assertTrue(judge.grounded("Your step: ``` ./run.sh ``` Run it.", transcript))
self.assertFalse(judge.grounded("**", transcript))
def test_an_elided_quote_holds_when_its_fragments_appear_in_order(self):
transcript = "**Decision for you:** security, or not.\n\nI recommend one more round of each.\n\nI recommend skipping it: it costs."
self.assertTrue(judge.grounded("**Decision for you:** security, … ... I recommend one more round of each ... I recommend skipping it: …", transcript))
self.assertFalse(judge.grounded("I recommend skipping it ... Decision for you", transcript))
self.assertFalse(judge.grounded("Decision for you ... invented", transcript))
self.assertFalse(judge.grounded("... …", transcript))
self.assertTrue(judge.grounded("a ... b ... c", "x a b a c"))
# Item 297 scores the higher L, item 298 the higher E + I.
SCORED = "## Items\n\n| ID | Release | Exempt | Item | L | E | I | R | S | Moves | Score |\n|---|---|---|---|---|---|---|---|---|---|---|\n| 297 | 0.8.0 | | **Do it.** | 4 | 2 | 3 | 7 | 2 | 1 by 9 | 25 |\n| 298 | 0.8.0 | | **Do more.** | 3 | 3 | 4 | 8 | 2 | 1 by 9 | 20 |\n"
def said(*texts):
return entries.AssistantTurn(parts=tuple(entries.Text(text) for text in texts))
class ChunkItems(unittest.TestCase):
def test_names_the_items_on_the_workers_latest_chunk_items_line(self):
earlier = said("Chunk items: 297 (L 3, E + I 5)")
latest = said("Starting.\n**Chunk items:** 297 (L 4, E + I 5), item 298 (L 3, E + I 7), #12, 298\nOn to the tests.")
self.assertEqual(chunk_items.named([earlier, latest]), (297, 298, 12))
self.assertEqual(chunk_items.named([latest, earlier]), (297,))
self.assertEqual(chunk_items.named([earlier, said("Chunk items: none")]), ())
self.assertEqual(chunk_items.named([said("Chunk items: 297 and 298 (L 3, E + I 7)")]), (297, 298))
def test_reads_the_example_line_the_prompt_gives_the_worker(self):
example = re.search(r"`(Chunk items:[^`]*)`", prompts.NAME_THE_ITEMS)[1]
self.assertEqual(chunk_items.named([said(example)]), (297, 298))
def test_every_prompt_that_starts_or_resumes_a_chunk_asks_for_the_line(self):
self.assertIn("on one line, as in `Chunk items: ", prompts.WORKER.resumed("", Route(Rung("m")), None))
def test_reads_only_a_line_the_worker_opens_with_the_label_in_its_own_text(self):
prompt = entries.LoopPrompt(text="Chunk items: 297 (L 3, E + I 5)")
written = entries.AssistantTurn(parts=(entries.ToolCall("t", "Write", {"content": "Chunk items: 9"}),))
mentioned = said("Once I know them, I write `Chunk items: 5`.")
self.assertEqual(chunk_items.named([prompt, written, mentioned]), ())
def test_takes_the_highest_l_and_e_plus_i_todo_md_scores_among_the_named_items(self):
self.assertEqual(chunk_items.taken_in([said("Chunk items: 297 (L 1, E + I 2), 298, 400")], plan_of(SCORED)), TakenIn(4, 7))
self.assertEqual(chunk_items.taken_in([said("Chunk items: 400")], plan_of(SCORED)), TakenIn())
self.assertEqual(chunk_items.taken_in([], plan_of(SCORED)), TakenIn())
def test_names_the_items_at_no_level_where_todo_md_cannot_be_read(self):
self.assertEqual(chunk_items.taken_in([said("Chunk items: 297")], Failure("no todo.md")), TakenIn())
self.assertEqual(chunk_items.taken_in([said("Chunk items: 297")], plan_of("no items table")), TakenIn())
class Verdict(unittest.TestCase):
def test_needs_every_field_the_schema_requires(self):
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": False, "evidence": "e", "finished": True, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Finished("e"))
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": False, "evidence": "e", "finished": False, "question": "ignored", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Unfinished("e"))
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": True, "evidence": "e", "finished": True, "question": "Which one?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Interrupt("e", "Which one?"))
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": True, "evidence": "e", "finished": False, "question": "Cut 0.2.0?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": True}), Interrupt("e", "Cut 0.2.0?", release_ready=True))
self.assertEqual(verdict_from({"escalation": "", "answered_before": "", "left_open": "", "reversed": "", "answer": "", "interrupt": True, "evidence": "e", "finished": False, "question": "Cut 0.2.0?"}), Failure("the judge's reply has no release_ready of type boolean"))
self.assertEqual(verdict_from({"escalation": "", "answer": "A.", "interrupt": False, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), QuotedAnswer("e", "Which one?", "A."))
self.assertEqual(verdict_from({"escalation": "", "answer": "A.", "interrupt": True, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Interrupt("e", "Which one?"))
self.assertEqual(verdict_from({"escalation": "", "answer": "A.", "interrupt": False, "evidence": "e", "finished": True, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Failure("the judge quoted an answer without a question"))
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": "no", "evidence": "", "finished": True, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Failure("the judge's reply has no interrupt of type boolean"))
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": False, "evidence": None, "finished": True, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Failure("the judge's reply has no evidence of type string"))
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": True, "evidence": "x", "finished": False, "question": " ", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Failure("the judge interrupted without a question"))
self.assertEqual(verdict_from(None), Failure("the judge returned no reply"))
def test_carries_the_earlier_answer_an_interrupt_names(self):
reply = {"answer": "", "escalation": "", "interrupt": True, "evidence": "e", "finished": False, "release_ready": False, "answered_before": "Grant one more round.", "left_open": "one more", "question": "Which one?", "reversed": ""}
self.assertEqual(verdict_from(reply), Interrupt("e", "Which one?", answered_before="Grant one more round.", left_open="one more"))
self.assertEqual(verdict_from({**reply, "interrupt": False, "finished": True}), Finished("e"))
def test_takes_an_escalation_quote_on_an_unfinished_verdict(self):
reply = {"answer": "", "escalation": "round 2: 🔴", "interrupt": False, "evidence": "e", "finished": False, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}
self.assertEqual(verdict_from(reply), Unfinished("e", "round 2: 🔴"))
self.assertEqual(verdict_from({**reply, "finished": True}), Finished("e"))
self.assertEqual(verdict_from({**reply, "escalation": None}), Failure("the judge's reply has no escalation of type string"))
def test_takes_an_escalation_quote_on_an_interrupt(self):
reply = {"answer": "", "escalation": "round 2: 🔴", "interrupt": True, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}
self.assertEqual(verdict_from(reply), Interrupt("e", "Which one?", "round 2: 🔴"))
def test_drops_an_escalation_quote_the_transcript_does_not_hold_word_for_word_from_an_interrupt_or_an_unfinished_verdict(self):
shown = "The agent: round 2: 🔴. Which one?"
self.assertEqual(judge.quotes_checked(Interrupt("e", "q", "round 2: 🔴"), judge.Grounds(shown, shown, "")), Interrupt("e", "q", "round 2: 🔴"))
self.assertEqual(judge.quotes_checked(Interrupt("e", "q", "round 2: … 🔴"), judge.Grounds(shown, shown, "")), Interrupt("e", "q"))
self.assertEqual(judge.quotes_checked(Unfinished("e", "round 3: 🔴"), judge.Grounds(shown, shown, "")), Unfinished("e"))
def test_turns_an_interrupt_that_re_asks_a_human_answer_into_that_answer_quoted_back(self):
answer = "Grant one more round; ask again if a 🔴 survives it."
shown = f"[loop] Asked.\n\n[human] {answer}\n\n[step] Round 2: a 🔴 survived."
grounds = judge.Grounds(shown, shown, "")
asking = "Grant a third round? Your answer left open: a 🔴 survives it"
kept = Interrupt("[step] Round 2", asking, answered_before=answer, left_open="a 🔴 survives it")
self.assertEqual(judge.quotes_checked(kept, grounds), kept)
re_asked = Interrupt("[step] Round 2", "Grant a third round?", "Round 2: a 🔴", answered_before=answer, left_open="a 🔴 survives it")
self.assertEqual(judge.quotes_checked(re_asked, grounds), Unfinished(answer, "Round 2: a 🔴"))
self.assertEqual(judge.quotes_checked(dataclasses.replace(kept, left_open=""), grounds), Unfinished(answer))
self.assertEqual(judge.quotes_checked(dataclasses.replace(kept, left_open="a 🔴 survived"), grounds), Unfinished(answer))
invented = dataclasses.replace(re_asked, answered_before="Grant two more rounds.")
self.assertEqual(judge.quotes_checked(invented, grounds), invented)
unnamed = Interrupt("[step] Round 2", "Grant a third round?")
self.assertEqual(judge.quotes_checked(unnamed, grounds), unnamed)
def test_takes_an_escalation_quote_on_a_quoted_answer(self):
reply = {"answer": "A.", "escalation": "round 2: 🔴", "interrupt": False, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}
self.assertEqual(verdict_from(reply), QuotedAnswer("e", "Which one?", "A.", "round 2: 🔴"))
def test_drops_an_escalation_quote_the_transcript_does_not_hold_word_for_word_from_a_quoted_answer(self):
shown = "The agent: round 2: 🔴. Which one?"
self.assertEqual(judge.quotes_checked(QuotedAnswer("e", "q", "A.", "round 2: 🔴"), judge.Grounds(shown, shown, "A.")), QuotedAnswer("e", "q", "A.", "round 2: 🔴"))
self.assertEqual(judge.quotes_checked(QuotedAnswer("e", "q", "A.", "round 2: … 🔴"), judge.Grounds(shown, shown, "A.")), QuotedAnswer("e", "q", "A."))
def test_puts_a_quoted_answer_the_records_do_not_hold_to_the_human_with_the_escalation_it_carried(self):
self.assertEqual(judge.quotes_checked(QuotedAnswer("e", "q", "not there", "round 2: 🔴"), judge.Grounds("The agent: round 2: 🔴.", "The agent: round 2: 🔴.", "A.")), Interrupt("e", "q", "round 2: 🔴"))
def test_puts_a_quoted_answer_the_records_do_not_hold_word_for_word_to_the_human(self):
self.assertEqual(judge.quotes_checked(QuotedAnswer("s", "q", "not there"), judge.Grounds("shown", "shown", "the records")), Interrupt("s", "q"))
self.assertEqual(judge.quotes_checked(QuotedAnswer("s", "q", "the records"), judge.Grounds("shown", "shown", "From the records: the records")), QuotedAnswer("s", "q", "the records"))
def test_drops_an_escalation_quote_found_only_in_the_transcript_entries_before_the_workers_last_step_up(self):
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "round 3: 🔴"}), assistant({"type": "text", "text": "working"})]
def judged_on(mark, escalation):
return judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", records.RECORDS_NAMED, prompts.ANSWER_DESTINATIONS, kinds(*entries), mark, prompts.DRIVER), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation=escalation)]).effects())
self.assertEqual(judged_on(0, "round 2: 🔴"), Unfinished("working", "round 2: 🔴", entries_read=3))
self.assertEqual(judged_on(1, "round 2: 🔴"), Unfinished("working", entries_read=3))
self.assertEqual(judged_on(1, "round 3: 🔴"), Unfinished("working", "round 3: 🔴", entries_read=3))
self.assertEqual(judged_on(1, "round 2: 🔴\n\nround 3: 🔴"), Unfinished("working", entries_read=3))
self.assertEqual(judged_on(3, "round 3: 🔴"), Unfinished("working", entries_read=3))
def test_drops_an_escalation_quote_that_only_loop_claudes_prompts_or_the_humans_lines_repeat_after_the_step_up(self):
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), {**human("as the work remaining: round 2: 🔴"), "promptSource": "sdk"}, human("round 2: 🔴 again"), assistant({"type": "text", "text": "working"})]
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", records.RECORDS_NAMED, prompts.ANSWER_DESTINATIONS, kinds(*entries), 1, prompts.DRIVER), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation="round 2: 🔴")]).effects())
self.assertEqual(verdict, Unfinished("working", entries_read=4))
def test_takes_evidence_from_before_the_workers_last_step_up(self):
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "working"})]
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", records.RECORDS_NAMED, prompts.ANSWER_DESTINATIONS, kinds(*entries), 1, prompts.DRIVER), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", evidence="round 2: 🔴")]).effects())
self.assertEqual(verdict, Unfinished("round 2: 🔴", entries_read=2))
class LoopRuns(unittest.TestCase):
def test_stops_naming_the_reload_when_it_hands_over_anything_but_the_state_with_a_task(self):
for handed in (["3", "2", "/r", "do it"], ["3"], ["[1]"], [json.dumps({"cycle_number": 3, "root": "/r", "unfinished_evidence": ""})], [json.dumps({"cycle_number": 3, "root": "/r", "task": "", "unfinished_evidence": ""})], [json.dumps({"cycle_number": "3", "root": "/r", "task": "do it", "unfinished_evidence": ""})], [json.dumps({"cycle_number": 3, "root": "", "task": "do it", "unfinished_evidence": ""})], [json.dumps({"cycle_number": 3, "root": "/r", "task": "do it", "turns": 1, "unfinished_evidence": ""})], [json.dumps({"cycle_number": 3, "failures": 0, "root": "/r", "task": "do it", "unfinished_evidence": ""})]):
with self.subTest(handed=handed):
self.assertEqual(main.loop_run_from([main.RELOAD_FLAG, *handed], CWD, FakeEffects().effects()), Stop(1, f"the edited loop-claude cannot read the state the running one handed it, likely because the edit changed that handover; rerun loop-claude <task>. Handed: {' '.join(handed)}"))
def test_decodes_every_field_of_the_state_and_its_route(self):
state = main.LoopRun(cycle_number=2, root="/r", task="do it", unfinished_evidence="old", route=Route(Rung("claude-fable-5-2", "low"), 3, 4, 5, 1, 7, 9), unfinished_run=2)
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(state))), state)
self.assertEqual(set(reload.LOOP_RUN_FIELDS), {field.name for field in dataclasses.fields(main.LoopRun)})
def test_rejects_a_field_of_another_type_or_an_unknown_field(self):
state = dataclasses.asdict(main.LoopRun(cycle_number=2, root="/r", task="do it", unfinished_evidence="", route=Route(Rung("claude-fable-5-2", "low"), 3, 4, 5, 1, 7, 9), unfinished_run=2))
for path, wrong in ((("unfinished_run",), "2"), (("unfinished_run",), True), (("route", "unfinished"), 1.0), (("route", "escalated_at"), "9"), (("route", "item"), True), (("route", "rung"), "claude-opus-5-5"), (("route", "rung", "model"), None), (("route",), "r"), (("route", "extra"), 1), (("route", "model"), "claude-opus-5-5"), (("route", "effort"), "high")):
with self.subTest(path=path, wrong=wrong):
handed = json.loads(json.dumps(state))
target = handed
for key in path[:-1]:
target = target[key]
target[path[-1]] = wrong
self.assertIsNone(reload.decode_handed(json.dumps(handed)))
def test_rejects_a_state_or_rung_missing_a_required_field(self):
state = dataclasses.asdict(main.LoopRun(cycle_number=2, root="/r", task="do it", unfinished_evidence="", route=Route(Rung("claude-fable-5-2", "low"))))
for path in (("cycle_number",), ("root",), ("task",), ("unfinished_evidence",), ("route", "rung"), ("route", "rung", "model")):
with self.subTest(path=path):
handed = json.loads(json.dumps(state))
target = handed
for key in path[:-1]:
target = target[key]
del target[path[-1]]
self.assertIsNone(reload.decode_handed(json.dumps(handed)))
def test_decodes_a_rung_without_an_effort(self):
state = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(Rung("some-model")))
self.assertEqual(reload.decode_handed(json.dumps({**dataclasses.asdict(state), "route": {"rung": {"model": "some-model"}}})), state)
def test_rejects_text_that_is_not_json(self):
self.assertIsNone(reload.decode_handed("not json"))
def test_decodes_a_route_inside_the_state_or_none(self):
state = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(Rung("claude-fable-5-2", "low"), 3, 4, 5, 1))
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(state))), state)
unrouted = dataclasses.replace(state, route=None)
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(unrouted))), unrouted)
handed = dataclasses.asdict(state)
handed["route"]["rung"]["effort"] = 1
self.assertIsNone(reload.decode_handed(json.dumps(handed)))
del handed["route"]["rung"]
self.assertIsNone(reload.decode_handed(json.dumps(handed)))
off_scale = dataclasses.asdict(state)
off_scale["route"]["likelihood"] = 9
self.assertIsNone(reload.decode_handed(json.dumps(off_scale)))
older = dataclasses.asdict(state)
del older["route"]["escape_and_impact_taken_in"]
self.assertEqual(reload.decode_handed(json.dumps(older)), state)
taken = dataclasses.replace(state, route=dataclasses.replace(state.route, escape_and_impact_taken_in=7))
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(taken))), taken)
for taken_in in (9, 5):
with self.subTest(taken_in=taken_in):
handed = dataclasses.asdict(state)
handed["route"]["escape_and_impact_taken_in"] = taken_in
self.assertIsNone(reload.decode_handed(json.dumps(handed)))
def test_decodes_a_state_from_before_a_field_with_a_default_joined_it(self):
self.assertEqual(reload.decode_handed(json.dumps({"cycle_number": 3, "root": "/r", "task": "do it", "unfinished_evidence": ""})), main.LoopRun(3, "/r", "do it", ""))
def test_drops_the_items_taken_in_that_a_loop_started_before_0_11_0_hands_on(self):
state = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="")
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(state) | {"items_taken_in": [12, 14]})), state)
def test_drops_the_slot_that_a_loop_started_before_chunks_ran_in_windows_hands_on(self):
state = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="")
self.assertEqual(reload.decode_handed(json.dumps({key: value for key, value in dataclasses.asdict(state).items() if key != "chunk"} | {"slot": 1})), state)
def test_stops_when_the_reload_cannot_start_python(self):
with tempfile.TemporaryDirectory() as root:
launcher = pathlib.Path(root, "loop-claude")
launcher.write_text("")
fake = FakeEffects(python=[ran()])
self.assertEqual(reload.exec_if_edited(a_loop(fake.effects(), launcher=launcher), "stale", main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="")), Stop(1, "could not reload loop-claude: No such file or directory; rerun loop-claude <task>"))
class WindowStarts(unittest.TestCase):
"""The loop-claude the human runs, which starts the windows, and the stop of a window's loop whose chunk finished."""
HOME = pathlib.Path("/h")
def run_loop(self, argv, fake, env=None, started=("item-3", "item-4")):
"""run_loop's stop and what it said on stdout, with start_chunks answering `started`; self.asked holds what start_chunks was asked."""
with unittest.mock.patch.object(main, "start_chunks", return_value=list(started) if isinstance(started, tuple) else started) as start, contextlib.redirect_stdout(io.StringIO()) as said:
stop = main.run_loop(argv, env or {}, paths.paths_from(self.HOME), pathlib.Path("/b/loop-claude"), CWD, fake.effects())
self.asked = [call.args[:2] for call in start.call_args_list]
return stop, said.getvalue()
def test_refuses_to_start_anything_on_a_tmux_older_than_needed(self):
fake = FakeEffects(git=["/r\n"], tmux_version="tmux 3.1c")
self.assertEqual(self.run_loop(["do it"], fake), (Stop(1, "loop-claude runs each chunk in a tmux window, and needs tmux 3.2 or later; it found tmux 3.1c"), ""))
self.assertEqual((fake.tmux_calls, self.asked), ([["-V"]], []))
def test_prints_the_help_without_tmux(self):
fake = FakeEffects(tmux_version="")
self.assertEqual(self.run_loop(["--help"], fake)[0], Stop(0, usage.HELP))
self.assertEqual(fake.tmux_calls, [])
def test_starts_the_chunks_then_attaches_to_the_session_or_inside_tmux_switches_to_it(self):
for env, shown in (({}, ["attach-session", "-t", "=loop-r"]), ({"TMUX": "/tmp/tmux-1000/default,1,0"}, ["switch-client", "-t", "=loop-r"])):
with self.subTest(shown=shown):
fake = FakeEffects(git=["/r\n"])
stop, said = self.run_loop(["do", "it"], fake, env)
self.assertEqual((stop, said), (Stop(0, ""), f"{STAMP} started item-3, item-4 in the tmux session loop-r\n"))
self.assertEqual(fake.tmux_calls, [["-V"], shown])
self.assertEqual(self.asked, [(main.LoopRun(0, "/r", "do it", ""), False)])
def test_hands_the_window_starts_its_paths_environment_launcher_and_effects(self):
fake = FakeEffects(git=["/r\n"], on_terminal=False)
effects, env, launcher = fake.effects(), {"LOOP_CLAUDE_X": "1"}, pathlib.Path("/b/loop-claude")
with unittest.mock.patch.object(main, "start_chunks", return_value=["item-3"]) as start, contextlib.redirect_stdout(io.StringIO()):
main.run_loop(["do it"], env, paths.paths_from(self.HOME), launcher, CWD, effects)
self.assertEqual(start.call_args.args, (main.LoopRun(0, "/r", "do it", ""), False, paths.paths_from(self.HOME), env, launcher, effects))
def test_a_window_loop_runs_its_chunk_in_its_own_session_and_window_and_hands_on_once_it_finished(self):
fake = FakeEffects(git=["/r\n"])
effects, env = fake.effects(), {"TMUX_PANE": "%4"}
launcher = pathlib.Path("/b/loop-claude")
cycle = unittest.mock.Mock(project=OPUS_PROJECT)
unfinished = unittest.mock.Mock(verdict=unittest.mock.Mock(spec=Unfinished))
finished = unittest.mock.Mock(verdict=unittest.mock.Mock(spec=Finished))
closed = [main.LoopRun(1, "/r", "do it", "", chunk="item-3"), main.LoopRun(2, "/r", "do it", "", chunk="item-3")]
with unittest.mock.patch.object(main, "fingerprint", return_value=""), unittest.mock.patch.object(main, "exec_if_edited", return_value=None), unittest.mock.patch.object(main, "next_cycle", return_value=cycle) as next_cycle, \
unittest.mock.patch.object(main, "run_cycle", side_effect=[unfinished, finished]), unittest.mock.patch.object(main, "closed_cycle", side_effect=closed), \
unittest.mock.patch.object(main, "check_routing", return_value=None), unittest.mock.patch.object(main, "handed_on", return_value=Stop(0, "")) as handed_on:
stop = main.run_loop(["--chunk", "item-3", "do it"], env, paths.paths_from(self.HOME), launcher, CWD, effects)
loop = this_loop.Loop(root="/r", session=session.session_named("A r loop-claude item-3", self.HOME / ".claude" / "projects"), window=tmux.Window("loop-r", "item-3", "%4"), paths=paths.paths_from(self.HOME), env=env, launcher=launcher, effects=effects)
self.assertEqual(stop, Stop(0, ""))
self.assertEqual([call.args for call in next_cycle.call_args_list], [(main.LoopRun(1, "/r", "do it", "", chunk="item-3"), loop), (main.LoopRun(2, "/r", "do it", "", chunk="item-3"), loop)])
handed_on.assert_called_once_with(closed[1], loop)
def test_a_window_loop_given_no_task_prints_the_help(self):
self.assertEqual(main.loop_run_from(["--chunk", "item-3"], CWD, FakeEffects(git=["/r\n"]).effects()), Stop(1, usage.HELP))
def test_a_window_loop_outside_a_checkout_stops(self):
self.assertIsInstance(main.loop_run_from(["--chunk", "item-3", "do it"], CWD, FakeEffects(git=[Failure("git cannot start")]).effects()), Stop)
def test_says_that_nothing_started_where_the_session_already_runs_every_chunk_that_may(self):
self.assertEqual(self.run_loop(["do it"], FakeEffects(git=["/r\n"]), started=())[1], f"{STAMP} the tmux session loop-r runs every chunk that may run now, so `do it` started in no new window\n")
def test_off_a_terminal_it_prints_how_to_attach(self):
fake = FakeEffects(git=["/r\n"], on_terminal=False)
stop, said = self.run_loop(["do it"], fake)
self.assertEqual(stop, Stop(0, ""))
self.assertTrue(said.endswith(f"{STAMP} not on a terminal; see the chunks with `tmux attach -t =loop-r`\n"), said)
self.assertEqual(fake.tmux_calls, [["-V"]])
def test_stops_where_no_chunk_could_start(self):
self.assertEqual(self.run_loop(["do it"], FakeEffects(git=["/r\n"]), started=Failure("no plan"))[0], Stop(1, "no plan"))
def test_stops_where_tmux_cannot_show_the_session(self):
fake = FakeEffects(git=["/r\n"])
effects = dataclasses.replace(fake.effects(), tmux_on_terminal=lambda args: 1)
with unittest.mock.patch.object(main, "start_chunks", return_value=["item-3"]), contextlib.redirect_stdout(io.StringIO()):
stop = main.run_loop(["do it"], {}, paths.paths_from(self.HOME), pathlib.Path("/b/loop-claude"), CWD, effects)
self.assertEqual(stop, Stop(1, "`tmux attach-session -t =loop-r` failed; the chunks go on in the tmux session"))
def test_a_window_loop_outside_tmux_stops(self):
fake = FakeEffects(git=["/r\n"])
self.assertEqual(self.run_loop(["--chunk", "item-3", "do it"], fake)[0], Stop(1, "--chunk is how loop-claude starts itself in a tmux window; run loop-claude <task>"))
self.assertEqual(self.asked, [])
def test_reads_the_chunk_a_window_loop_is_started_with(self):
self.assertEqual(main.loop_run_from(["--chunk", "item-3", "do", "it"], CWD, FakeEffects(git=["/r\n"]).effects()), main.LoopRun(0, "/r", "do it", "", chunk="item-3"))
self.assertEqual(main.loop_run_from(["do", "it"], CWD, FakeEffects(git=["/r\n"]).effects()), main.LoopRun(0, "/r", "do it", ""))
def test_a_finished_chunk_starts_each_chunk_that_may_start_and_stops_without_a_reason(self):
loop_run = main.LoopRun(4, "/r", "do it", "", chunk="item-7")
for started, said, stop in ((["item-9", "item-12"], "the chunk finished; started item-9, item-12", Stop(0, "")), ([], "the chunk finished; started no other, since the chunks running fill what may run now", Stop(0, "")),
(Failure("no plan"), "", Stop(1, "the chunk finished, but the next could not start: no plan"))):
with self.subTest(started=started), unittest.mock.patch.object(main, "start_chunks", return_value=started) as start, contextlib.redirect_stdout(io.StringIO()) as printed:
loop = a_loop(FakeEffects().effects(), env={"LOOP_CLAUDE_X": "1"})
self.assertEqual(main.handed_on(loop_run, loop), stop)
self.assertEqual(printed.getvalue(), f"{STAMP} {said}\n" if said else "")
self.assertEqual(start.call_args.args, (loop_run, True, loop.paths, loop.env, loop.launcher, loop.effects))
class StopDms(unittest.TestCase):
def run_main(self, argv, fake):
with tempfile.TemporaryDirectory() as home, contextlib.redirect_stderr(io.StringIO()) as said, contextlib.redirect_stdout(io.StringIO()):
code = main.main(argv, {}, paths.paths_from(pathlib.Path(home)), pathlib.Path(home, "loop-claude"), CWD, fake.effects())
return code, said.getvalue()
def test_dms_the_reason_of_a_stop_with_exit_2_such_as_a_chunk_left_unfinished_too_often(self):
fake = FakeEffects()
with unittest.mock.patch.object(main, "run_loop", return_value=Stop(2, "the human is needed to look at the chunk, but it stayed unfinished")):
self.assertEqual(self.run_main(["do it"], fake)[0], 2)
self.assertEqual(fake.dms, [f"loop-claude in {CWD} stopped: the human is needed to look at the chunk, but it stayed unfinished"])
def test_skips_the_dm_of_a_stop_without_a_reason_as_once_the_windows_started(self):
fake = FakeEffects()
with unittest.mock.patch.object(main, "run_loop", return_value=Stop(0, "")):
self.assertEqual(self.run_main(["do it"], fake), (0, ""))
self.assertEqual(fake.dms, [])
def test_skips_the_dm_where_ctrl_c_stopped_the_loop(self):
fake = FakeEffects()
with unittest.mock.patch.object(main, "run_loop", side_effect=KeyboardInterrupt):
self.assertEqual(self.run_main(["do it"], fake)[0], 130)
self.assertEqual(fake.dms, [])
def test_dms_a_crash_before_it_reaches_the_terminal(self):
fake = FakeEffects()
with unittest.mock.patch.object(main, "run_loop", side_effect=RuntimeError("boom")), self.assertRaises(RuntimeError):
self.run_main(["do it"], fake)
self.assertEqual(fake.dms, [f"loop-claude in {CWD} crashed: RuntimeError('boom')"])
def test_dms_the_reason_where_a_loop_given_a_task_stops_with_exit_1(self):
fake = FakeEffects()
code, said = self.run_main([main.RELOAD_FLAG, "3"], fake)
self.assertEqual(code, 1)
self.assertEqual(fake.dms, [f"loop-claude in {CWD} stopped: the edited loop-claude cannot read the state the running one handed it, likely because the edit changed that handover; rerun loop-claude <task>. Handed: 3"])
def test_says_why_the_dm_of_a_stop_failed_after_the_reason(self):
fake = FakeEffects(dm_failures=[Failure("cannot reach https://chat.example: timed out")])
self.assertTrue(self.run_main([main.RELOAD_FLAG, "3"], fake)[1].endswith(f"Handed: 3\n{STAMP} could not DM you: cannot reach https://chat.example: timed out\n"))
def test_skips_the_dm_where_it_prints_its_help(self):
for argv, code in (([], 1), (["--help"], 0)):
with self.subTest(argv=argv):
fake = FakeEffects()
self.assertEqual(self.run_main(argv, fake)[0], code)
self.assertEqual(fake.dms, [])
class HelpLimits(unittest.TestCase):
def test_help_wraps_at_100_columns(self):
self.assertEqual([line for line in usage.HELP.splitlines() if len(line) > 100], [])
def test_help_states_the_limits_the_code_holds(self):
said = " ".join(usage.HELP.split())
self.assertIn(f"The {['second', 'third', 'fourth', 'fifth'][states.QUOTED_ANSWERS_PER_CYCLE - 1]} question they answer in one cycle", said)
# The retry messages say these limits in words too.
self.assertEqual((retry.FAILURES_RUNNING, retry.RETRY_DELAY), (3, 60))
self.assertIn("fails three times running, retried a minute apart", said)
self.assertIn(f"retried {retry.PAST_RESET // 60} minutes after the limit resets, or every {retry.UNKNOWN_RESET_POLL // 60} minutes", said)
self.assertIn(f"still running after {PYTHON_TIMEOUT} s", said)
self.assertIn(f"Once {state.CYCLES_PER_CHECK} cycles have closed across all projects", said)
self.assertIn(f"run with a {worker.COMPACT_WINDOW // 1000}k auto-compact window", said)
self.assertIn(f"{BASH_CEILING_CHARACTERS:,} characters", said)
self.assertIn(f"unfinished {close.UNFINISHED_LIMIT} cycles running", said)
self.assertIn(f"rerun to give it {close.UNFINISHED_LIMIT} more", said)
class OwnLadder(unittest.TestCase):
def test_a_cycle_on_a_projects_own_ladder_names_its_file_and_says_where_the_workers_model_is_off_it(self):
file = pathlib.Path("/h/.claude-projects/acme/routing-ladder.json")
self.assertEqual(main.own_ladder(file, "claude-fable-5-2", False), f"routing on the project's own ladder, {file}.")
self.assertEqual(main.own_ladder(file, "claude-opus-5-5", True), f"routing on the project's own ladder, {file}. claude-opus-5-5 is not on it, so the worker runs on claude-opus-5-5 unrouted; LOOP_CLAUDE_WORKER_MODEL and LOOP_CLAUDE_JUDGE_MODEL name other models for a run.")
class ExitContract(unittest.TestCase):
def test_every_exit_1_stop_has_its_clause_in_help(self):
read, git, premises_ = "a transcript cannot be read or archived", "git cannot start or finds no checkout", "no config names the checkout, its config cannot be read, or the goals file holds no premises"
models_ = "the models file cannot be read or written"
names = (*records.RECORD_FILES, records.HUMAN_RULE_NAMES[0], f"{records.HUMAN_RULE_NAMES[1]}*.md", records.HUMAN_RULE_NAMES[2])
unreadable = f"the goals file, or {', '.join(names[:-1])} or {names[-1]} where they exist, cannot be read or is not UTF-8"
# Every Stop whose code is not a literal 0, 2 or 130, in source order: path, function, code, start of reason, its --help clause.
clauses = [
("agent_cycle/interrupt.py", "claude_open", "1", "f'could not start claude for the interrupt:", "claude cannot start for an interrupt"),
("agent_cycle/interrupt.py", "human_typed", "1", "entries.reason", read),
("agent_cycle/interrupt.py", "human_typed", "1", "f'could not tell whether you typed anything:", read),
("agent_cycle/retry.py", "retried", "1", "f'the {who} failed three times running:", "the worker or the judge fails three times running"),
("agent_cycle/run.py", "archived_stop", "1", "f'{failure.reason}", read),
("agent_cycle/run.py", "work", "1", "f'claude left no transcript for session", "claude leaves no transcript"),
("agent_cycle/run.py", "rule", "1", "entries.reason", read),
("agent_cycle/run.py", "rule", "1", "annotation.reason", read),
("agent_cycle/run.py", "rule", "1", "failure.reason", read),
("agent_cycle/run.py", "keep_rejection", "1", "unkept.reason", read),
("agent_cycle/run.py", "close", "1", "annotation.reason", read),
("agent_cycle/run.py", "close", "1", "failure.reason", read),
("agent_cycle/run.py", "close", "1", "failure.reason", "a finished chunk's session cannot be deleted"),
("loop_claude/loop/close.py", "closed_cycle", "1", "failure.reason", read),
("loop_claude/loop/main.py", "run_loop", "1", "reason", "tmux is missing or older than 3.2"),
("loop_claude/loop/main.py", "run_loop", "1", "f'{CHUNK_FLAG} is how loop-claude starts itself", "--chunk is given outside a tmux window"),
("loop_claude/loop/main.py", "windows_opened", "1", "started.reason", "a chunk's window cannot be started"),
("loop_claude/loop/main.py", "windows_opened", "1", "f\"`tmux {' '.join(shown)}` failed", "the tmux session cannot be put on the terminal"),
("loop_claude/loop/main.py", "handed_on", "1", "f'the chunk finished, but the next could not start", "a chunk's window cannot be started"),
("loop_claude/loop/main.py", "next_cycle", "1", "premises.reason", premises_),
("loop_claude/loop/main.py", "next_cycle", "1", "records.reason", unreadable),
("loop_claude/loop/main.py", "next_cycle", "1", "ladder_file.reason", premises_),
("loop_claude/loop/main.py", "next_cycle", "1", "ladder.reason", "the project's routing ladder cannot be read or holds a fault"),
("loop_claude/loop/main.py", "loop_run_from", "0 if task else 1", "HELP", "no task is given"),
("loop_claude/loop/main.py", "loop_run_from", "1", 'f"the edited loop-claude cannot read the state', "a reload cannot read the state the running script handed it"),
("loop_claude/loop/reload.py", "exec_if_edited", "1", "f'could not reload loop-claude:", "python cannot start for a reload"),
("loop_claude/loop/routing_check.py", "raise_calls", "1", "f\"could not start claude for the routing calls in {loop.paths.config_clone}", "claude cannot start for an interrupt"),
("loop_claude/project/checkout.py", "checkout_root", "1", "found.reason", git),
("loop_claude/project/model_choice.py", "get_or_ask_model", "1", "models.reason", models_),
("loop_claude/project/model_choice.py", "get_or_ask_model", "1", "models.reason", models_),
("loop_claude/project/model_choice.py", "get_or_ask_model", "1", "f'could not save the model in", models_),
]
root = pathlib.Path(main.__file__).parents[2]
found = []
for path in sorted(path for package in ("agent_cycle", "loop_claude") for path in (root / package).rglob("*.py") if not path.name.startswith("test_")):
functions = [node for node in ast.walk(ast.parse(path.read_text())) if isinstance(node, ast.FunctionDef)]
stops = [(call.lineno, function.name, ast.unparse(call.args[0]), ast.unparse(call.args[1])) for function in functions for call in ast.walk(function) if isinstance(call, ast.Call) and getattr(call.func, "id", None) == "Stop" and not (isinstance(call.args[0], ast.Constant) and call.args[0].value in (0, 2, 130))]
found += [(str(path.relative_to(root)), function, code, reason) for _, function, code, reason in sorted(stops)]
self.assertEqual(len(found), len(clauses), found)
for (path, function, code, reason), (want_path, want_function, want_code, reason_start, _) in zip(found, clauses):
with self.subTest(path=path, function=function):
self.assertEqual((path, function, code), (want_path, want_function, want_code))
self.assertTrue(reason.startswith(reason_start), reason)
help_exit_1 = " ".join(usage.HELP[usage.HELP.index("Exit 1"):usage.HELP.index("Exit 2")].split())
self.assertEqual([clause for *_, clause in clauses if f"- {clause}" not in help_exit_1], [])
class JudgeFailures(unittest.TestCase):
def test_every_failed_judge_run_has_its_clause_in_help(self):
started, exited, unparsed = "claude cannot start", "exits non-zero", "prints no json reply"
field, unasked = "its reply misses or mistypes a field", "interrupts or quotes an answer without a question"
# Every Failure the judge's run returns, in source order of judge.py, then of the claude run it calls: function, start of reason, its --help clause.
clauses = [
("verdict_from", "'the judge returned no reply'", field),
("verdict_from", 'f"the judge\'s reply has no {name}', field),
("verdict_from", "'the judge interrupted without a question'", unasked),
("verdict_from", "'the judge quoted an answer without a question'", unasked),
("run_judge", "verdict.reason", field),
("run_judge", "checked.reason", "quotes a passage the transcript does not hold"),
("misquoted_evidence", "f'the judge quoted {verdict.evidence!r}, which is not in the transcript", "quotes a passage the transcript does not hold"),
("claude_reply", "f'could not start claude:", started),
("claude_reply", "proc.stderr.strip() or f'{who} exited", exited),
("claude_reply", 'f"{who} printed no json reply', unparsed),
]
found = []
for module in (judge, reply):
functions = [node for node in ast.walk(ast.parse(pathlib.Path(module.__file__).read_text())) if isinstance(node, ast.FunctionDef)]
found += sorted((call.lineno, function.name, ast.unparse(call.args[0])) for function in functions for call in ast.walk(function) if isinstance(call, ast.Call) and getattr(call.func, "id", None) in ("Failure", "Rejected"))
self.assertEqual(len(found), len(clauses), found)
for (_, function, reason), (want_function, reason_start, _) in zip(found, clauses):
with self.subTest(function=function):
self.assertEqual(function, want_function)
self.assertTrue(reason.startswith(reason_start), reason)
judge_clause = " ".join(usage.HELP[usage.HELP.index("a judge when"):usage.HELP.index("A usage limit")].split())
self.assertEqual([clause for *_, clause in clauses if clause not in judge_clause], [])
if __name__ == "__main__":
unittest.main()