Files
claude-config/bin/test_loop_claude.py
T

658 lines
40 KiB
Python

import ast
import contextlib
import dataclasses
import hashlib
import io
import json
import os
import pathlib
import stat
import subprocess
import sys
import tempfile
import time
import unittest
import uuid
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
from loop_claude.cycle import archive, interrupt, judge, retry, run, states, worker
from loop_claude.project import checkout, model_choice, records
from loop_claude.effects import Effects
from loop_claude.claude_code import entries, session, transcript, trust
from failure import Failure
from loop_claude.cycle.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished, verdict_from
from loop_claude.cycle.retry import Limited
from loop_claude.loop import main, reload
from loop_claude import paths
from loop_claude.stop import Stop
CWD = pathlib.Path("/c")
NOW = 1790000000
def assistant(*parts):
return {"type": "assistant", "message": {"content": list(parts)}}
def human(content):
return {"type": "user", "message": {"content": content}}
def queued(prompt, kind):
return {"type": "attachment", "attachment": {"type": "queued_command", "commandMode": "prompt", "origin": {"kind": kind}, "prompt": prompt}}
class FakeEffects:
"""Effects that record each call and answer from a script, one answer per call."""
def __init__(self, claude=(), on_terminal_claude=(), git=(), inputs=(), python=(), streams=(), on_terminal=True):
self.calls = []
self.scripts = {"claude": list(claude), "claude_on_terminal": list(on_terminal_claude), "git": list(git), "input": list(inputs), "python": list(python), "stream_claude": list(streams)}
self.terminal = on_terminal
def effects(self):
return Effects(
claude=lambda args, cwd, stdin: self.answer("claude", (args, stdin)),
claude_on_terminal=lambda args, cwd: self.answer("claude_on_terminal", args),
execv=self.execv,
git=lambda args, cwd: self.answer("git", args),
input=lambda prompt: self.answer("input", prompt),
local_time=lambda seconds: f"<{seconds}>",
now=lambda: NOW,
on_terminal=lambda: self.terminal,
python=lambda args: self.answer("python", args),
sleep=lambda seconds: self.calls.append(("sleep", seconds)),
stream_claude=lambda args, cwd: self.answer("stream_claude", args),
)
def answer(self, seam, what):
self.calls.append((seam, what))
answer = self.scripts[seam].pop(0)
if isinstance(answer, BaseException):
raise answer
return answer(what) if callable(answer) else answer
def execv(self, path, args):
self.calls.append(("execv", args))
raise OSError(2, "No such file or directory")
def made(self, seam):
return [what for called, what in self.calls if called == seam]
def uuid7():
# Python 3.11, the floor, has no uuid.uuid7.
random = int.from_bytes(os.urandom(10))
return uuid.UUID(int=int(time.time() * 1000) << 80 | 7 << 76 | (random >> 62 & 0xFFF) << 64 | 2 << 62 | random & (1 << 62) - 1)
def ran(stdout="", code=0, stderr=""):
return subprocess.CompletedProcess([], code, stdout, stderr)
def judged(verdict, evidence="working", answer=""):
reply = {"answer": answer, "evidence": evidence, "finished": verdict == "finished", "interrupt": verdict == "interrupt", "question": "Which one?" if verdict in ("interrupt", "quoted") else ""}
return ran(json.dumps({"structured_output": reply}))
class Worker:
"""A worker claude: writes a line to the session's transcript, then streams a result."""
def __init__(self, home):
self.home = home
def __call__(self, args):
flag = "--session-id" if "--session-id" in args else "--resume"
path = self.home / "projects" / "-fake" / f"{args[args.index(flag) + 1]}.jsonl"
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "a") as handle:
if flag == "--session-id":
handle.write(json.dumps({"type": "custom-title"}) + "\n" + json.dumps({"type": "queue-operation", "timestamp": str(uuid7())}) + "\n")
handle.write(json.dumps(assistant({"type": "text", "text": "working"})) + "\n")
return Streamed(0)
class Streamed:
def __init__(self, code, *events):
self.code = code
self.events = events or ({"type": "result"},)
def lines(self):
return [json.dumps(event) + "\n" for event in self.events]
def wait(self):
return self.code
class Transitions(unittest.TestCase):
def test_each_state_moves_on_its_events_and_only_on_them(self):
answered = states.Ruled(QuotedAnswer("e", "Which one?", "A."))
legal = (
(states.Working(), states.WorkerRan("/t"), states.Judging("/t")),
(states.Working(QuotedAnswer("e", "Which one?", "A."), 3), states.WorkerRan("/t"), states.Judging("/t", 3)),
(states.Judging("/t"), states.Ruled(Interrupt("e", "Which one?")), states.Interrupting("/t", Interrupt("e", "Which one?"))),
(states.Judging("/t", 3), states.Ruled(Interrupt("e", "Which one?")), states.Interrupting("/t", Interrupt("e", "Which one?"), 3)),
(states.Judging("/t"), answered, states.Working(QuotedAnswer("e", "Which one?", "A."), 1)),
(states.Judging("/t", 2), answered, states.Working(QuotedAnswer("e", "Which one?", "A."), 3)),
(states.Judging("/t", 3), answered, states.Interrupting("/t", QuotedAnswer("e", "Which one?", "A."), 3)),
(states.Judging("/t"), states.Ruled(Finished("e")), states.Closing("/t", Finished("e"))),
(states.Judging("/t", 3), states.Ruled(Finished("e")), states.Closing("/t", Finished("e"))),
(states.Judging("/t"), states.Ruled(Unfinished("e")), states.Closing("/t", Unfinished("e"))),
(states.Judging("/t", 3), states.Ruled(Unfinished("e")), states.Closing("/t", Unfinished("e"))),
(states.Interrupting("/t", Interrupt("e", "q")), states.HumanAnswered(), states.Judging("/t")),
(states.Interrupting("/t", Interrupt("e", "q"), 3), states.HumanAnswered(), states.Judging("/t", 3)),
(states.Closing("/t", Finished("e")), states.Archived(), states.Ended()),
)
for state, event, after in legal:
with self.subTest(state=state, event=event):
self.assertEqual(states.transition(state, event), after)
every_state = (states.Working(), states.Working(QuotedAnswer("e", "Which one?", "A."), 3), states.Judging("/t"), states.Judging("/t", 3), states.Interrupting("/t", Interrupt("e", "q")), states.Interrupting("/t", Interrupt("e", "q"), 3), states.Closing("/t", Finished("e")), states.Ended())
every_event = (states.WorkerRan("/t"), states.Ruled(Interrupt("e", "Which one?")), answered, states.Ruled(Finished("e")), states.Ruled(Unfinished("e")), states.HumanAnswered(), states.Archived())
for state in every_state:
for event in every_event:
if (state, event) not in [(state, event) for state, event, _ in legal]:
with self.subTest(state=state, event=event), self.assertRaises(ValueError):
states.transition(state, event)
class QuestionFor(unittest.TestCase):
def test_puts_an_interrupts_question_as_asked_and_a_quoted_answer_past_the_cap_with_its_quote(self):
self.assertEqual(run.question_for(Interrupt("e", "Which one?")), "Which one?")
self.assertEqual(run.question_for(QuotedAnswer("e", "Which one?", "A.")), "Which one?\n\nREADME.md's Goals and Audience, AGENTS.md or docs/decisions.md says:\nA.\n\nThose files have answered 3 questions this cycle, the most allowed, so this one comes to you.")
class Cycles(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
self.home = pathlib.Path(temp.name)
self.paths = paths.paths_from(self.home, {})
self.session = session.session_for("proj", self.paths)
def cycle(self, fake, records=""):
return run.Cycle(effects=fake.effects(), models=model_choice.Models("judge-model", "worker-model"), paths=self.paths, premises="## Goals", project_key="proj", records=records, root=str(self.home), session=self.session, task="Do it.")
def archived_verdicts(self):
return sorted(path.read_text() for path in (self.paths.transcript_archive / "proj").glob("*/*.outcomes.md"))
def test_a_finished_chunk_is_archived_and_its_session_deleted(self):
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
self.assertIsNone(run.run_cycle(self.cycle(fake)))
self.assertEqual(self.archived_verdicts(), ["Task: Do it.\n\nVerdict: finished\n"])
self.assertIsNone(session.transcript_file(self.session))
def test_an_unfinished_chunks_transcript_is_archived_and_the_next_cycle_resumes_its_session(self):
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[judged("unfinished")] * 2)
self.assertIsNone(run.run_cycle(self.cycle(fake)))
self.assertTrue(session.transcript_file(self.session))
self.assertIsNone(run.run_cycle(self.cycle(fake)))
workers = fake.made("stream_claude")
self.assertIn("--session-id", workers[0])
self.assertIn("--resume", workers[1])
self.assertEqual(self.archived_verdicts(), ["Task: Do it.\n\nVerdict: unfinished\n"] * 2)
def test_a_cycle_stopped_by_a_failure_or_ctrl_c_archives_what_its_session_holds(self):
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[ran(code=1)] * 3)
self.assertEqual(run.run_cycle(self.cycle(fake)).code, 1)
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[KeyboardInterrupt()])
self.assertEqual(run.run_cycle(self.cycle(fake)), Stop(130, ""))
self.assertEqual(self.archived_verdicts(), ["Task: Do it.\n\nStopped: Ctrl-C\n", "Task: Do it.\n\nStopped: the judge failed three times running: the judge exited 1\n"])
def test_archives_each_chunks_session_folder_replacing_a_copy_only_with_what_grew_from_it(self):
folder = self.paths.claude_code_transcripts / "-fake" / str(self.session.id)
(folder / "subagents" / "workflows").mkdir(parents=True)
(folder / "subagents" / "agent-a.jsonl").write_text("a\n")
(folder / "subagents" / "workflows" / "journal.jsonl").write_text("j\n")
(folder / "custom-title.json").write_text("first")
(folder.parent / f"{self.session.id}.orphaned-1.jsonl").write_text("o\n")
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 3, claude=[judged("unfinished"), judged("finished"), judged("unfinished")])
self.assertIsNone(run.run_cycle(self.cycle(fake)))
[chunk] = (self.paths.transcript_archive / "proj").iterdir()
kept = chunk / "session" / str(self.session.id)
self.assertEqual((chunk / "session" / f"{self.session.id}.orphaned-1.jsonl").read_text(), "o\n")
os.utime(kept / "subagents" / "workflows" / "journal.jsonl", (0, 0))
with open(folder / "subagents" / "agent-a.jsonl", "a") as handle:
handle.write("more\n")
(folder / "custom-title.json").write_text("second")
self.assertIsNone(run.run_cycle(self.cycle(fake)))
self.assertFalse(folder.exists())
self.assertEqual((kept / "subagents" / "agent-a.jsonl").read_text(), "a\nmore\n")
self.assertEqual((kept / "subagents" / "workflows" / "journal.jsonl").stat().st_mtime, 0)
self.assertEqual(sorted(path.read_text() for path in kept.glob("custom-title.json*")), ["first", "second"])
self.assertEqual(stat.S_IMODE((kept / "subagents" / "agent-a.jsonl").stat().st_mode), 0o600)
folder.mkdir()
(folder / "custom-title.json").write_text("third")
self.assertIsNone(run.run_cycle(self.cycle(fake)))
[next_chunk] = set((self.paths.transcript_archive / "proj").iterdir()) - {chunk}
self.assertEqual(sorted(path.read_text() for path in (next_chunk / "session" / str(self.session.id)).glob("custom-title.json*")), ["third"])
@unittest.skipIf(os.geteuid() == 0, "root reads an unreadable folder")
def test_stops_on_a_session_folder_it_cannot_read_whole(self):
locked = self.paths.claude_code_transcripts / "-fake" / str(self.session.id) / "subagents"
locked.mkdir(parents=True)
locked.chmod(0)
self.addCleanup(locked.chmod, 0o700)
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("unfinished")])
self.assertEqual(run.run_cycle(self.cycle(fake)), Stop(1, f"could not archive the transcript: {locked}: Permission denied"))
def test_an_interrupt_is_archived_answered_and_judged_again(self):
def answer(args):
with open(session.transcript_file(self.session), "a") as handle:
handle.write(json.dumps(human("the second")) + "\n")
return 0
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt"), judged("unfinished")], on_terminal_claude=[answer], git=[ran()])
self.assertIsNone(run.run_cycle(self.cycle(fake)))
self.assertEqual(self.archived_verdicts(), ["Task: Do it.\n\nVerdict: interrupt\n\nWhich one?\n", "Task: Do it.\n\nVerdict: unfinished\n"])
self.assertIn(str(self.session.id), fake.made("claude_on_terminal")[0])
prompts = [stdin for args, stdin in fake.made("claude")]
self.assertEqual(len(prompts), 2)
self.assertNotIn("the second", prompts[0])
self.assertIn("[human] the second", prompts[1])
def test_a_question_the_records_answer_resumes_the_worker_with_the_answer_three_times_a_cycle(self):
answered = judged("quoted", answer="Use *A*.")
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 4, claude=[answered] * 4, on_terminal_claude=[0], git=[ran()])
with contextlib.redirect_stderr(io.StringIO()) as said, contextlib.redirect_stdout(io.StringIO()) as printed:
self.assertEqual(run.run_cycle(self.cycle(fake, records="From AGENTS.md:\nUse A.")), Stop(2, "the human is needed to answer the question above, but the interrupt ended with nothing typed"))
resumed = fake.made("stream_claude")[1]
self.assertEqual(resumed[resumed.index("-p") + 1], "Your question: Which one?\n\nREADME.md's Goals and Audience, AGENTS.md or docs/decisions.md answers it:\n\nUse *A*.\n\nContinue the task.")
self.assertEqual(len(fake.made("stream_claude")), 4)
self.assertEqual(printed.getvalue().count("=== README.md's Goals and Audience, AGENTS.md or docs/decisions.md answered: Which one? ===\nUse *A*."), 3)
self.assertIn("Which one?\n\nREADME.md's Goals and Audience, AGENTS.md or docs/decisions.md says:\nUse *A*.\n\nThose files have answered 3 questions this cycle", said.getvalue())
def test_an_answer_the_records_do_not_hold_is_dropped_and_the_question_goes_to_the_human(self):
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("quoted", answer="Use ... A.")], on_terminal_claude=[0], git=[ran()])
with contextlib.redirect_stderr(io.StringIO()) as said:
self.assertEqual(run.run_cycle(self.cycle(fake, records="From AGENTS.md:\nUse B. Use A.")), Stop(2, "the human is needed to answer the question above, but the interrupt ended with nothing typed"))
self.assertEqual(len(fake.made("stream_claude")), 1)
self.assertIn("\nWhich one?\n", said.getvalue())
self.assertNotIn("Use ... A.", said.getvalue())
@unittest.skipIf(os.geteuid() == 0, "root writes into a read-only directory")
def test_stops_keeping_the_transcript_when_the_finished_chunks_session_cannot_be_deleted(self):
folder = self.paths.claude_code_transcripts / "-fake" / str(self.session.id)
(folder / "subagents").mkdir(parents=True)
folder.chmod(0o500)
self.addCleanup(folder.chmod, 0o700)
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
stop = run.run_cycle(self.cycle(fake))
self.assertEqual(stop, Stop(1, f"could not delete {folder}: Permission denied; delete every file and folder matching {folder}*, then rerun loop-claude; a rerun before that resumes the finished chunk's session"))
self.assertTrue(session.session_open(self.session))
self.assertEqual(self.archived_verdicts(), [f"Task: Do it.\n\nVerdict: finished\n\nStopped: {stop.reason}\n"])
def test_stops_when_the_worker_leaves_no_transcript(self):
fake = FakeEffects(streams=[Streamed(0)])
self.assertEqual(run.run_cycle(self.cycle(fake)), Stop(1, f"claude left no transcript for session {self.session.id}"))
class Retried(unittest.TestCase):
def test_retries_a_minute_apart_and_stops_on_the_third_failure(self):
fake = FakeEffects()
results = [Failure("one"), "done"]
self.assertEqual(retry.retried("worker", lambda: results.pop(0), fake.effects()), "done")
self.assertEqual(fake.calls, [("sleep", 60)])
fake = FakeEffects()
self.assertEqual(retry.retried("judge", lambda: Failure("no"), fake.effects()), Stop(1, "the judge failed three times running: no"))
self.assertEqual(fake.calls, [("sleep", 60), ("sleep", 60)])
def test_waits_out_a_usage_limit_which_ends_a_run_of_failures(self):
fake = FakeEffects()
results = [Failure("one"), Failure("two"), Limited(NOW + 3600, "limit"), Failure("three"), Limited(None, "limit"), Limited(NOW - 600, "limit"), Failure("four"), Failure("five"), Failure("six")]
self.assertEqual(retry.retried("worker", lambda: results.pop(0), fake.effects()), Stop(1, "the worker failed three times running: six"))
self.assertEqual(fake.calls, [("sleep", 60), ("sleep", 60), ("sleep", 3720), ("sleep", 60), ("sleep", 600), ("sleep", 600), ("sleep", 60), ("sleep", 60)])
def test_says_what_claude_said_and_when_the_next_try_is_due(self):
results = [Limited(NOW + 3600, "You've hit your session limit"), "done"]
with contextlib.redirect_stderr(io.StringIO()) as said:
retry.retried("judge", lambda: results.pop(0), FakeEffects().effects())
self.assertEqual(said.getvalue(), f"the judge hit a usage limit (You've hit your session limit); retrying at <{NOW + 3720}>\n")
class Limits(unittest.TestCase):
def test_the_worker_reports_the_latest_reset_its_stream_names(self):
rejected = [{"type": "rate_limit_event", "rate_limit_info": {"status": "rejected", "resetsAt": resets_at}} for resets_at in (NOW + 60, NOW + 600)]
allowed = {"type": "rate_limit_event", "rate_limit_info": {"status": "allowed", "resetsAt": NOW + 9000}}
limited = {"type": "result", "api_error_status": 429, "result": "You've hit your session limit"}
fake = FakeEffects(streams=[Streamed(1, rejected[1], allowed, rejected[0], limited)])
self.assertEqual(worker.run_worker("/r", [], fake.effects()), Limited(NOW + 600, "You've hit your session limit"))
def test_a_limit_outranks_the_exit_code_and_its_message_fits_one_line(self):
def limited(**said):
return {"type": "result", "api_error_status": 429, **said}
self.assertEqual(worker.worker_outcome(-9, limited(result="limit\n resets 7pm"), []), Limited(None, "limit resets 7pm"))
self.assertEqual(worker.worker_outcome(1, limited(result=" \n"), [NOW]), Limited(NOW, "claude gave no message"))
self.assertEqual(worker.worker_outcome(0, limited(), []), Limited(None, "claude gave no message"))
self.assertEqual(worker.worker_outcome(0, None, [NOW]), Failure("claude exited 0 without a result event"))
self.assertIsNone(worker.worker_outcome(0, {"type": "result"}, [NOW]))
def test_the_judge_reports_a_limit_whatever_its_exit_code(self):
reply = json.dumps({"type": "result", "api_error_status": 429, "is_error": True, "result": "You've hit your session limit · resets 7:40pm (Europe/Stockholm)"})
fake = FakeEffects(claude=[ran(reply, code=1, stderr="limited")])
self.assertEqual(judge.run_judge("/r", [], "", "", "", fake.effects()), Limited(None, "You've hit your session limit · resets 7:40pm (Europe/Stockholm)"))
def test_reads_only_a_rejecting_events_whole_seconds(self):
for info, want in [({"status": "rejected", "resetsAt": NOW}, NOW), ({"status": "allowed", "resetsAt": NOW}, None), ({"status": "rejected", "resetsAt": True}, None), ({"status": "rejected"}, None), ("rejected", None)]:
self.assertEqual(entries.rejected_until({"type": "rate_limit_event", "rate_limit_info": info}), want, info)
class Interrupts(unittest.TestCase):
def test_stops_off_a_terminal_before_starting_claude(self):
fake = FakeEffects(on_terminal=False)
stop = interrupt.interrupt("/r", uuid7(), "m", "/t", "Which one?", paths.paths_from(pathlib.Path("/h"), {}), fake.effects())
self.assertEqual(stop, Stop(2, "the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal"))
self.assertEqual(fake.calls, [])
def test_stops_when_nothing_was_typed_in_the_handover(self):
with tempfile.TemporaryDirectory() as home:
path = pathlib.Path(home, "t.jsonl")
path.write_text(json.dumps(human("a")) + "\n")
fake = FakeEffects(on_terminal_claude=[0], git=[ran()])
stop = interrupt.interrupt(home, uuid7(), "m", str(path), "Which one?", paths.paths_from(pathlib.Path(home), {}), fake.effects())
self.assertEqual(stop, Stop(2, "the human is needed to answer the question above, but the interrupt ended with nothing typed"))
class ModelPrompt(unittest.TestCase):
OFFERED = [("claude-opus-5-5", "Opus 5.5"), ("claude-haiku-4-5", "Haiku 4.5")]
def test_reads_a_number_as_the_offered_model_and_anything_else_as_a_name(self):
self.assertEqual(model_choice.model_picked(" 2 ", self.OFFERED), "claude-haiku-4-5")
self.assertEqual(model_choice.model_picked("opus", self.OFFERED), "opus")
self.assertEqual(model_choice.model_picked("3", []), "3")
self.assertEqual(model_choice.model_picked("3", self.OFFERED), Failure("pick 1 to 2"))
self.assertIsNone(model_choice.model_picked(" ", self.OFFERED))
def test_asks_again_until_an_answer_picks_a_model(self):
with tempfile.TemporaryDirectory() as empty:
fake = FakeEffects(inputs=["", "my-model"])
self.assertEqual(model_choice.ask_for_model("proj", pathlib.Path("/m.json"), pathlib.Path(empty), fake.effects()), "my-model")
self.assertEqual(len(fake.made("input")), 2)
fake = FakeEffects(inputs=[EOFError()])
self.assertEqual(model_choice.ask_for_model("proj", pathlib.Path("/m.json"), pathlib.Path(empty), fake.effects()), Stop(2, "the human is needed to pick proj's model, but the input ended"))
class WorkerArgs(unittest.TestCase):
def test_opens_a_fresh_session_by_name_and_resumes_an_open_one(self):
opened = session.session_for("proj", paths.paths_from(pathlib.Path("/h"), {}))
fresh = worker.worker_args("Do it.", opened, "m", False)
self.assertEqual(fresh[fresh.index("--session-id") + 1], str(opened.id))
self.assertEqual(fresh[fresh.index("--name") + 1], "A proj loop-claude")
resumed = worker.worker_args("Do it.", opened, "m", True)
self.assertEqual(resumed[resumed.index("--resume") + 1], str(opened.id))
self.assertNotIn("Do it.", " ".join(resumed))
class Records(unittest.TestCase):
def test_heads_each_record_the_project_has_by_its_path(self):
with tempfile.TemporaryDirectory() as root:
self.assertEqual(records.read_records(root), "")
pathlib.Path(root, "docs").mkdir()
pathlib.Path(root, "docs", "decisions.md").write_text("# Decisions\n")
self.assertEqual(records.read_records(root), "From docs/decisions.md:\n# Decisions")
pathlib.Path(root, "AGENTS.md").write_text("\n# Rules\n")
self.assertEqual(records.read_records(root), "From AGENTS.md:\n# Rules\n\nFrom docs/decisions.md:\n# Decisions")
def test_stops_on_a_record_it_cannot_read(self):
with tempfile.TemporaryDirectory() as root:
pathlib.Path(root, "docs", "decisions.md").mkdir(parents=True)
self.assertEqual(records.read_records(root), Stop(1, f"docs/decisions.md in {root}: Is a directory - the judge cannot read the project's records."))
pathlib.Path(root, "AGENTS.md").write_bytes(b"# Rules\n\xff\n")
self.assertEqual(records.read_records(root), Stop(1, f"AGENTS.md in {root} is not UTF-8 on line 2 - the judge cannot read the project's records."))
class Render(unittest.TestCase):
def test_joins_assistant_text_tool_calls_human_turns_and_loop_prompts_and_skips_the_rest(self):
said = transcript.render([
human("do it"),
assistant({"type": "text", "text": "Starting."}, {"type": "tool_use", "name": "Bash", "input": {"command": "ls"}}),
assistant({"type": "text", "text": " "}),
human([{"type": "tool_result", "content": "not typed"}]),
human("<command-name>/exit</command-name>"),
human([{"type": "text", "text": "yes, "}, {"type": "text", "text": "merge it"}]),
queued("B, while you work", "human"),
queued("origin/main moved", "coordinator"),
{**human("Continue the task; the previous run stopped before it was finished."), "promptSource": "sdk"},
])
self.assertEqual(said, '[human] do it\n\nStarting.\n\n[tool Bash] {"command": "ls"}\n\n[human] yes, merge it\n\n[human] B, while you work\n\n[loop] Continue the task; the previous run stopped before it was finished.')
class Archive(unittest.TestCase):
def test_writes_afresh_a_copy_whose_outcomes_file_is_missing(self):
with tempfile.TemporaryDirectory() as root:
kept = paths.paths_from(pathlib.Path(root), {})
transcript = pathlib.Path(root, "t.jsonl")
transcript.write_text('{"timestamp": "2026-10-03T00:00:00Z"}\n')
with contextlib.redirect_stderr(io.StringIO()):
self.assertIsNone(archive.archive("proj", str(transcript), "Do it.", "Verdict: unfinished", kept))
(outcomes,) = kept.transcript_archive.glob("proj/*/*.outcomes.md")
outcomes.unlink()
self.assertIsNone(archive.archive("proj", str(transcript), "Do it.", "Stopped: Ctrl-C", kept))
self.assertEqual(outcomes.read_text(), "Task: Do it.\n\nStopped: Ctrl-C\n")
class CopyFor(unittest.TestCase):
def test_skips_a_version_already_kept_beside_its_copy(self):
with tempfile.TemporaryDirectory() as temp:
copy = pathlib.Path(temp, "custom-title.json")
copy.write_bytes(b"first")
beside = archive.copy_for(copy, b"second")
self.assertEqual(beside, copy.with_name(f"custom-title.json.{hashlib.sha256(b'second').hexdigest()}"))
beside.write_bytes(b"second")
self.assertIsNone(archive.copy_for(copy, b"second"))
self.assertEqual(archive.copy_for(copy, b"first, grown"), copy)
class FirstTimestampedLine(unittest.TestCase):
def test_skips_the_lines_every_chunk_shares_and_else_takes_the_first(self):
self.assertEqual(transcript.first_timestamped_line(b'{"type": "custom-title"}\nnot JSON\n{"timestamp": "t"}\n{"timestamp": "u"}\n'), b'{"timestamp": "t"}')
self.assertEqual(transcript.first_timestamped_line(b'{"type": "custom-title"}\n[1]\n'), b'{"type": "custom-title"}')
self.assertEqual(transcript.first_timestamped_line(b""), b"")
class HumanTurns(unittest.TestCase):
def test_counts_typed_turns_and_not_tool_results(self):
self.assertEqual(transcript.human_turns([
human("typed"),
human([{"type": "text", "text": "typed as a block"}]),
human([{"type": "tool_result", "content": "not typed"}]),
human("<command-name>/model</command-name>"),
{**human("<3 merge it"), "promptSource": "typed"},
{**human("<task-notification>done</task-notification>"), "promptSource": "system"},
{**human([{"type": "text", "text": "<task-notification>done</task-notification>"}]), "promptSource": "system"},
{**human("peer message"), "isMeta": True},
{**human("the loop's own prompt"), "promptSource": "sdk"},
human([{"type": "text", "text": "[Request interrupted by user]"}]),
human([]),
queued("typed while claude worked", "human"),
queued("<task-notification>done</task-notification>", "task-notification"),
{"type": "attachment", "attachment": {"type": "queued_command", "commandMode": "task-notification", "prompt": "<task-notification>done</task-notification>"}},
assistant({"type": "text", "text": "said"}),
]), 4)
class PartLine(unittest.TestCase):
def test_renders_text_and_tool_calls_and_nothing_else(self):
self.assertEqual(transcript.part_line({"type": "text", "text": "hi"}, 20), "hi")
self.assertEqual(transcript.part_line({"type": "text", "text": " \n"}, 20), "")
self.assertEqual(transcript.part_line({"type": "tool_use", "name": "Bash", "input": {"command": "x" * 500}}, 20), '[tool Bash] {"command": "xxxxxxx')
self.assertEqual(transcript.part_line("not a part", 20), "")
self.assertEqual(transcript.part_line({"type": "tool_use", "name": "Write", "input": {"content": "Hälsa på"}}, 40), '[tool Write] {"content": "Hälsa på"}')
class Parse(unittest.TestCase):
def test_yields_only_json_objects(self):
self.assertEqual(entries.parse_entry('{"type": "user"}'), {"type": "user"})
self.assertIsNone(entries.parse_entry("not json"))
self.assertIsNone(entries.parse_entry("[1, 2]"))
class MarkTrusted(unittest.TestCase):
def test_returns_the_updated_config_and_leaves_the_one_it_was_handed(self):
config = {"other": 1, "projects": {"/a": {"hasTrustDialogAccepted": True}, "/b": {"kept": True}}}
handed = json.dumps(config)
self.assertEqual(trust.mark_trusted(config, ["/a", "/b", "/c"]), ({"other": 1, "projects": {"/a": {"hasTrustDialogAccepted": True}, "/b": {"hasTrustDialogAccepted": True, "kept": True}, "/c": {"hasTrustDialogAccepted": True}}}, ["/b", "/c"]))
self.assertEqual(json.dumps(config), handed)
self.assertEqual(trust.mark_trusted({}, ["/a"]), ({"projects": {"/a": {"hasTrustDialogAccepted": True}}}, ["/a"]))
class JudgePrompt(unittest.TestCase):
def test_shows_the_transcripts_tail_and_returns_what_it_showed(self):
prompt, shown = judge.judge_prompt("Do it.", "## Goals", "", "old " + "x" * judge.JUDGE_PROMPT_CHARS + " new")
self.assertTrue(shown.endswith(" new"))
self.assertNotIn("old", shown)
self.assertIn(shown, prompt)
self.assertLessEqual(len(prompt), judge.JUDGE_PROMPT_CHARS)
class Grounded(unittest.TestCase):
def test_every_verdict_must_quote_the_transcript(self):
self.assertTrue(judge.grounded("Needs you: x", "so.\n\nNeeds you: x\n"))
self.assertFalse(judge.grounded("invented", "the transcript"))
self.assertFalse(judge.grounded("", "the transcript"))
self.assertFalse(judge.grounded(" ", "the transcript"))
transcript = "done.\n\n**Your step:**\n```\n./run.sh\n```\nRun it."
self.assertTrue(judge.grounded("Your step:\n```\n./run.sh\n```\nRun it.", transcript))
self.assertTrue(judge.grounded("Your step: ``` ./run.sh ``` Run it.", transcript))
self.assertFalse(judge.grounded("**", transcript))
def test_an_elided_quote_holds_when_its_fragments_appear_in_order(self):
transcript = "**Decision for you:** security, or not.\n\nI recommend one more round of each.\n\nI recommend skipping it: it costs."
self.assertTrue(judge.grounded("**Decision for you:** security, … ... I recommend one more round of each ... I recommend skipping it: …", transcript))
self.assertFalse(judge.grounded("I recommend skipping it ... Decision for you", transcript))
self.assertFalse(judge.grounded("Decision for you ... invented", transcript))
self.assertFalse(judge.grounded("... …", transcript))
def test_an_answer_holds_only_word_for_word(self):
self.assertTrue(judge.verbatim("Merge **ff-only**.", "Merge ff-only.\nNever squash."))
self.assertFalse(judge.verbatim("Merge ... squash.", "Merge ff-only.\nNever squash."))
self.assertFalse(judge.verbatim("", "Merge ff-only."))
class Verdict(unittest.TestCase):
def test_needs_every_field_the_schema_requires(self):
self.assertEqual(verdict_from({"answer": "", "interrupt": False, "evidence": "e", "finished": True, "question": ""}), Finished("e"))
self.assertEqual(verdict_from({"answer": "", "interrupt": False, "evidence": "e", "finished": False, "question": "ignored"}), Unfinished("e"))
self.assertEqual(verdict_from({"answer": "", "interrupt": True, "evidence": "e", "finished": True, "question": "Which one?"}), Interrupt("e", "Which one?"))
self.assertEqual(verdict_from({"answer": "A.", "interrupt": False, "evidence": "e", "finished": False, "question": "Which one?"}), QuotedAnswer("e", "Which one?", "A."))
self.assertEqual(verdict_from({"answer": "A.", "interrupt": True, "evidence": "e", "finished": False, "question": "Which one?"}), Interrupt("e", "Which one?"))
self.assertEqual(verdict_from({"answer": "A.", "interrupt": False, "evidence": "e", "finished": True, "question": ""}), Failure("the judge quoted an answer without a question"))
self.assertEqual(verdict_from({"answer": "", "interrupt": "no", "evidence": "", "finished": True, "question": ""}), Failure("the judge's reply has no interrupt of type boolean"))
self.assertEqual(verdict_from({"answer": "", "interrupt": False, "evidence": None, "finished": True, "question": ""}), Failure("the judge's reply has no evidence of type string"))
self.assertEqual(verdict_from({"answer": "", "interrupt": True, "evidence": "x", "finished": False, "question": " "}), Failure("the judge interrupted without a question"))
self.assertEqual(verdict_from(None), Failure("the judge returned no reply"))
class LoopRuns(unittest.TestCase):
def test_stops_naming_the_reload_when_it_hands_over_anything_but_the_state_with_a_task(self):
for handed in (["3", "2", "/r", "do it"], ["3"], ["[1]"], [json.dumps({"cycle_number": 3, "root": "/r"})], [json.dumps({"cycle_number": 3, "root": "/r", "task": ""})], [json.dumps({"cycle_number": "3", "root": "/r", "task": "do it"})], [json.dumps({"cycle_number": 3, "root": "", "task": "do it"})], [json.dumps({"cycle_number": 3, "root": "/r", "task": "do it", "turns": 1})], [json.dumps({"cycle_number": 3, "failures": 0, "root": "/r", "task": "do it"})]):
with self.subTest(handed=handed):
self.assertEqual(main.loop_run_from([main.RELOAD_FLAG, *handed], CWD, FakeEffects().effects()), Stop(1, f"the edited loop-claude cannot read the cycle number, checkout root and task the running one handed it, likely because the edit changed that handover; rerun loop-claude <task>. Handed: {' '.join(handed)}"))
def test_decodes_a_state_whose_annotations_are_strings(self):
@dataclasses.dataclass(frozen=True)
class Stringly:
count: "int"
name: "str"
self.assertEqual(reload.decode_handed('{"count": 1, "name": "n"}', Stringly), Stringly(1, "n"))
self.assertIsNone(reload.decode_handed('{"count": "1", "name": "n"}', Stringly))
def test_stops_when_the_reload_cannot_start_python(self):
with tempfile.TemporaryDirectory() as root:
launcher = pathlib.Path(root, "loop-claude")
launcher.write_text("")
fake = FakeEffects(python=[ran()])
self.assertEqual(reload.exec_if_edited(launcher, "stale", main.LoopRun(1, "/r", "do it"), fake.effects()), Stop(1, "could not reload loop-claude: No such file or directory; rerun loop-claude <task>"))
class HelpLimits(unittest.TestCase):
def test_help_states_the_limits_the_code_holds(self):
said = " ".join(main.__doc__.split())
self.assertIn(f"The {['second', 'third', 'fourth', 'fifth'][states.QUOTED_ANSWERS_PER_CYCLE - 1]} question they answer in one cycle", said)
# The retry messages say these limits in words too.
self.assertEqual((retry.FAILURES_RUNNING, retry.RETRY_DELAY), (3, 60))
self.assertIn("fails three times running, retried a minute apart", said)
self.assertIn(f"retried {retry.PAST_RESET // 60} minutes after the limit resets, or every {retry.UNKNOWN_RESET_POLL // 60} minutes", said)
class ExitContract(unittest.TestCase):
def test_every_exit_1_stop_has_its_clause_in_help(self):
read, git, premises_ = "a transcript cannot be read or archived", "git cannot start or finds no checkout", "README.md holds no premises"
models_ = "the models file cannot be read or written"
unreadable = f"README.md, or {' or '.join(records.RECORD_FILES)} where they exist, cannot be read or is not UTF-8"
# Every Stop whose code is not a literal 0, 2 or 130, in source order: path, function, code, start of reason, its --help clause.
clauses = [
("cycle/interrupt.py", "interrupt", "1", "entries.reason", read),
("cycle/interrupt.py", "interrupt", "1", "f'could not start claude for the interrupt:", "claude cannot start for an interrupt"),
("cycle/interrupt.py", "interrupt", "1", "f'could not tell whether you typed anything:", read),
("cycle/retry.py", "retried", "1", "f'the {who} failed three times running:", "the worker or the judge fails three times running"),
("cycle/run.py", "stopped", "1", "f'{failure.reason}", read),
("cycle/run.py", "work", "1", "f'claude left no transcript for session", "claude leaves no transcript"),
("cycle/run.py", "rule", "1", "entries.reason", read),
("cycle/run.py", "ask", "1", "failure.reason", read),
("cycle/run.py", "close", "1", "failure.reason", read),
("cycle/run.py", "close", "1", "failure.reason", "a finished chunk's session cannot be deleted"),
("loop/main.py", "loop_run_from", "0 if task else 1", "__doc__.strip()", "no task is given"),
("loop/main.py", "loop_run_from", "1", 'f"the edited loop-claude cannot read the cycle number,', "a reload cannot read the cycle number, checkout root and task"),
("loop/reload.py", "exec_if_edited", "1", "f'could not reload loop-claude:", "python cannot start for a reload"),
("project/checkout.py", "checkout_root", "1", "f'could not start git:", git),
("project/checkout.py", "checkout_root", "1", "found.stderr.strip()", git),
("project/checkout.py", "read_checkout_file", "1", "f'{name} in {root}:", unreadable),
("project/checkout.py", "read_checkout_file", "1", "f'{name} in {root} is not UTF-8", unreadable),
("project/model_choice.py", "get_or_ask_model", "1", "f'could not save the model in", models_),
("project/model_choice.py", "read_models_file", "1", 'f"could not read {saved}: {getattr(', models_),
("project/model_choice.py", "read_models_file", "1", "f'could not read {saved}: not an object", models_),
("project/premises.py", "read_premises", "1", "f'README.md has no ##", premises_),
("project/premises.py", "read_premises", "1", "f'README.md has ## {name} more than once", premises_),
("project/premises.py", "read_premises", "1", "f'README.md has nothing under", premises_),
]
root = pathlib.Path(main.__file__).parents[1]
found = []
for path in sorted(root.rglob("*.py")):
functions = [node for node in ast.walk(ast.parse(path.read_text())) if isinstance(node, ast.FunctionDef)]
stops = [(call.lineno, function.name, ast.unparse(call.args[0]), ast.unparse(call.args[1])) for function in functions for call in ast.walk(function) if isinstance(call, ast.Call) and getattr(call.func, "id", None) == "Stop" and not (isinstance(call.args[0], ast.Constant) and call.args[0].value in (0, 2, 130))]
found += [(str(path.relative_to(root)), function, code, reason) for _, function, code, reason in sorted(stops)]
self.assertEqual(len(found), len(clauses), found)
for (path, function, code, reason), (want_path, want_function, want_code, reason_start, _) in zip(found, clauses):
self.assertEqual((path, function, code), (want_path, want_function, want_code))
self.assertTrue(reason.startswith(reason_start), f"{path} {function}: {reason}")
help_exit_1 = main.__doc__[main.__doc__.index("Exit 1"):main.__doc__.index("Exit 2")]
self.assertEqual([clause for *_, clause in clauses if f"- {clause}" not in help_exit_1], [])
class JudgeFailures(unittest.TestCase):
def test_every_failed_judge_run_has_its_clause_in_help(self):
started, field, unasked = "claude cannot start or exits non-zero", "its reply misses or mistypes a field", "interrupts or quotes an answer without a question"
# Every Failure the judge's run returns, in source order: path, function, start of reason, its --help clause.
clauses = [
("cycle/judge.py", "run_judge", "f'could not start claude:", started),
("cycle/judge.py", "run_judge", "proc.stderr.strip() or f'the judge exited", started),
("cycle/judge.py", "run_judge", 'f"{verdict.reason}; it replied:', field),
("cycle/judge.py", "run_judge", "f'the judge quoted {verdict.evidence!r}, which is not in the transcript", "quotes a passage the transcript does not hold"),
("cycle/verdicts.py", "verdict_from", "'the judge returned no reply'", field),
("cycle/verdicts.py", "verdict_from", 'f"the judge\'s reply has no {name}', field),
("cycle/verdicts.py", "verdict_from", "'the judge interrupted without a question'", unasked),
("cycle/verdicts.py", "verdict_from", "'the judge quoted an answer without a question'", unasked),
]
root = pathlib.Path(main.__file__).parents[1]
found = []
for path in ("cycle/judge.py", "cycle/verdicts.py"):
functions = [node for node in ast.walk(ast.parse((root / path).read_text())) if isinstance(node, ast.FunctionDef)]
failures = sorted((call.lineno, function.name, ast.unparse(call.args[0])) for function in functions for call in ast.walk(function) if isinstance(call, ast.Call) and getattr(call.func, "id", None) == "Failure")
found += [(path, function, reason) for _, function, reason in failures]
self.assertEqual(len(found), len(clauses), found)
for (path, function, reason), (want_path, want_function, reason_start, _) in zip(found, clauses):
self.assertEqual((path, function), (want_path, want_function))
self.assertTrue(reason.startswith(reason_start), f"{path} {function}: {reason}")
judge_clause = " ".join(main.__doc__[main.__doc__.index("a judge when"):main.__doc__.index("A usage limit")].split())
self.assertEqual([clause for *_, clause in clauses if clause not in judge_clause], [])
class Worktrees(unittest.TestCase):
def test_lists_the_root_alone_when_git_cannot_start(self):
fake = FakeEffects(git=[FileNotFoundError(2, "No such file or directory")])
self.assertEqual(checkout.worktrees("/r", fake.effects()), ["/r"])
if __name__ == "__main__":
unittest.main()