512 lines
37 KiB
Python
512 lines
37 KiB
Python
import contextlib
|
|
import dataclasses
|
|
import io
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import sys
|
|
import tempfile
|
|
import types
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from test_env import assert_not_root
|
|
from failure import Failure
|
|
from loop_effects.doubles import FakeEffects, human, judged, NOW, ran, STAMP, Streamed, Worker
|
|
from agent_cycle.verdicts import Finished, Interrupt, Judged, QuotedAnswer, TakenIn, Unfinished
|
|
from transcript_archive import outcomes, write
|
|
from loop_state import paths
|
|
from transcript_archive.location import project_archive
|
|
from model_routing.ladder import ProjectRung, Rung
|
|
from agent_cycle.doubles import PROMPTS, WINDOW
|
|
from model_routing.ladder_file import read_ladder, SHIPPED
|
|
from text_files.file_writes import replace_file
|
|
from model_routing.route import Route
|
|
from agent_cycle import routing
|
|
from agent_cycle import run, stuck_workers, worker_prompts
|
|
from claude_code import session
|
|
from loop_effects.child_shells import Shell
|
|
from agent_cycle.stop import Stop
|
|
from text_files.utc_stamp import utc
|
|
|
|
|
|
LADDER = read_ladder(SHIPPED)
|
|
OPUS_PROJECT = ProjectRung(LADDER, Rung("claude-opus-5-5", "medium"))
|
|
# The ladder's rungs the cycles run on, and those one and three steps up from OPUS.
|
|
SONNET = Rung("claude-sonnet-5-5", "high")
|
|
OPUS = Rung("claude-opus-5-5", "medium")
|
|
ONE_UP = Rung("claude-fable-5-2", "low")
|
|
THREE_UP = Rung("claude-opus-5-5", "xhigh")
|
|
# How the archive records a worker that runs off the ladder.
|
|
OFF_LADDER = '{"model": "worker-model"}'
|
|
# The claude-config release and commit each cycle the tests run is under.
|
|
CONFIG = "v0.8.0 (abc123)"
|
|
# How it records routes by item 179's L of 2 and E + I of 5.
|
|
SONNET_179 = '{"model": "claude-sonnet-5-5", "effort": "high", "item": 179, "likelihood": 2, "escape_and_impact": 5}'
|
|
OPUS_179_AT_3 = '{"model": "claude-opus-5-5", "effort": "medium", "item": 179, "likelihood": 2, "escape_and_impact": 5, "escalated_at": 3}'
|
|
OPUS_179_AT_6 = '{"model": "claude-opus-5-5", "effort": "medium", "item": 179, "likelihood": 2, "escape_and_impact": 5, "escalated_at": 6}'
|
|
JUDGE_RUN_LINE = re.compile(r"^Judge run: judge/[0-9a-f]{64}\.json\n", re.MULTILINE)
|
|
|
|
|
|
class QuestionFor(unittest.TestCase):
|
|
def test_puts_an_interrupts_question_as_asked_and_a_quoted_answer_past_the_cap_with_its_quote(self):
|
|
cycle = types.SimpleNamespace(prompts=PROMPTS)
|
|
self.assertEqual(run.question_for(cycle, Interrupt("e", "Which one?")), "Which one?")
|
|
self.assertEqual(run.question_for(cycle, QuotedAnswer("e", "Which one?", "A.")), "Which one?\n\nthe project's records says:\nA.\n\nThose files have answered 3 questions this cycle, the most allowed, so this one comes to you.")
|
|
|
|
|
|
class Cycles(unittest.TestCase):
|
|
def setUp(self):
|
|
temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temp.cleanup)
|
|
self.home = pathlib.Path(temp.name)
|
|
self.paths = paths.paths_from(self.home)
|
|
self.session = session.session_named("A proj a-driver", self.paths.claude_code_transcripts)
|
|
self.archive = project_archive(str(self.home / "code" / "proj"), self.home, {})
|
|
|
|
def cycle(self, fake, records="", unfinished_evidence="", facts="- A preflight fact."):
|
|
return run.Cycle(config=CONFIG, window=WINDOW, effects=fake.effects(), facts=facts, judge=Rung("judge-model"), premises="## Goals", project=None, project_archive=self.archive, records=records, root=str(self.home), route=Route(Rung("worker-model")), session=self.session, task="Do it.", prompts=PROMPTS, taken_in=lambda entries: TakenIn(), unfinished_evidence=unfinished_evidence)
|
|
|
|
def archived_outcomes(self):
|
|
"""Each outcomes file's text without its `Judge run:` lines, which name a file by the hash of a prompt and its reply: the tests of the judge's runs check them."""
|
|
return sorted(JUDGE_RUN_LINE.sub("", path.read_text()) for path in self.archive.glob("*/*.outcomes.md"))
|
|
|
|
def test_a_finished_chunk_is_archived_and_its_session_deleted(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
|
|
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: finished\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
|
|
|
|
def test_archives_the_cost_the_judges_reply_records_beside_its_verdict(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished", cost=0.25)])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working", judged=Judged(cost=0.25)))
|
|
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: finished\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nJudge cost: 0.2500\nWorker: {OFF_LADDER}\n"])
|
|
self.assertIsNone(session.transcript_file(self.session))
|
|
|
|
def test_archives_what_the_worker_and_the_judge_spent_beside_the_verdict(self):
|
|
worker = Worker(self.paths.claude_code_transcripts.parent, usage={"input_tokens": 1, "cache_creation": {"ephemeral_5m_input_tokens": 2, "ephemeral_1h_input_tokens": 3}, "cache_read_input_tokens": 4, "output_tokens": 5})
|
|
answer = json.loads(judged("finished").stdout)
|
|
fake = FakeEffects(streams=[worker], claude=[ran(json.dumps({**answer, "usage": {"input_tokens": 6, "output_tokens": 7}}))])
|
|
run.run_cycle(self.cycle(fake))
|
|
self.assertEqual(self.archived_outcomes(), [f"""Task: Do it.
|
|
|
|
2026-09-21T14:13:20Z Verdict: finished
|
|
|
|
Evidence:
|
|
> working
|
|
|
|
Spent since the chunk began, in tokens of input, cache write 5m, cache write 1h, cache read, output:
|
|
- worker, claude-opus-5-5: 1, 2, 3, 4, 5
|
|
- judge, judge-model: 6, 0, 0, 0, 7
|
|
|
|
List prices in dollars per million tokens of input, cache write 5m, cache write 1h, cache read, output:
|
|
- claude-opus-5-5: 4, 5, 8, 0.2, 20
|
|
|
|
Config: {CONFIG}
|
|
Judge: judge-model
|
|
Worker: {OFF_LADDER}
|
|
"""])
|
|
|
|
def test_a_fresh_worker_reads_the_preflights_facts_and_a_resumed_one_does_not(self):
|
|
worker = Worker(self.paths.claude_code_transcripts.parent)
|
|
fake = FakeEffects(streams=[worker, worker], claude=[judged("unfinished"), judged("finished")])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=3))
|
|
self.assertEqual(run.run_cycle(self.cycle(fake, unfinished_evidence="working")).verdict, Finished("working"))
|
|
fresh, resumed = (args[args.index("-p") + 1] for args in fake.made("stream_claude"))
|
|
self.assertIn("- A preflight fact.", fresh)
|
|
self.assertNotIn("- A preflight fact.", resumed)
|
|
|
|
def test_a_worker_is_handed_the_projects_rung_its_own_its_reviews_e_plus_i_and_its_compaction_window_in_its_environment(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
|
|
cycle = dataclasses.replace(self.cycle(fake), project=OPUS_PROJECT, route=Route(Rung("claude-haiku-5-5", "max"), 163, 1, 5))
|
|
self.assertEqual(run.run_cycle(cycle).verdict, Finished("working"))
|
|
self.assertEqual(fake.worker_environments, [{"LOOP_CLAUDE_PROJECT_RUNG": "claude-opus-5-5 at medium effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-haiku-5-5 at max effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "5", "CLAUDE_CODE_AUTO_COMPACT_WINDOW": "200000", "BASH_MAX_OUTPUT_LENGTH": "6000"}])
|
|
|
|
def test_a_stuck_check_of_the_worker_runs_on_the_projects_rung_and_off_the_ladder_on_the_workers_own(self):
|
|
for project, model in ((OPUS_PROJECT, "claude-opus-5-5"), (None, "claude-haiku-5-5")):
|
|
with self.subTest(project=project):
|
|
clock = [NOW]
|
|
quiet = Shell(pid=7, started=NOW, command="bin/run-tests", written=NOW, runs_busy_tool=False)
|
|
events = ({"type": "assistant", "message": {"content": []}}, None, None, {"type": "result"})
|
|
stream = lambda args: (Worker(self.paths.claude_code_transcripts.parent)(args), Streamed(0, *events, clock=clock, held=[quiet]))[1]
|
|
verdict = ran(json.dumps({"type": "result", "structured_output": {"verdict": "stuck", "process": 7, "why": "it waits"}}))
|
|
fake = FakeEffects(streams=[stream], claude=[verdict, judged("finished")])
|
|
cycle = dataclasses.replace(self.cycle(fake), project=project, route=Route(Rung("claude-haiku-5-5", "max")))
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
run.run_cycle(dataclasses.replace(cycle, effects=dataclasses.replace(cycle.effects, now=lambda: clock[0])))
|
|
[investigator, _] = [args for args, _ in fake.made("claude")]
|
|
self.assertEqual(investigator[investigator.index("--model") + 1], model)
|
|
self.assertEqual(fake.worker_folders, [str(self.home)])
|
|
self.assertIn("judged the worker stuck", (self.archive / stuck_workers.RECORD).read_text())
|
|
|
|
def test_the_claude_an_interrupt_resumes_is_handed_the_same_rungs_as_the_worker(self):
|
|
def answer(args):
|
|
with open(session.transcript_file(self.session), "a") as handle:
|
|
handle.write(json.dumps(human("the second")) + "\n")
|
|
return 0
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt"), judged("finished")], inputs=["/talk"], on_terminal_claude=[answer])
|
|
cycle = dataclasses.replace(self.cycle(fake), project=OPUS_PROJECT, route=Route(Rung("claude-haiku-5-5", "max"), 163, 1, 5))
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(run.run_cycle(cycle).verdict, Finished("working"))
|
|
self.assertEqual(fake.terminal_environments, [{**environment, "CLAUDE_CODE_SANDBOXED": "1"} for environment in fake.worker_environments])
|
|
self.assertEqual(fake.terminal_environments[0]["LOOP_CLAUDE_WORKER_RUNG"], "claude-haiku-5-5 at max effort")
|
|
|
|
def test_an_unfinished_chunks_transcript_is_archived_and_the_next_cycle_resumes_its_session_with_the_judges_evidence(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[judged("unfinished")] * 2)
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=3))
|
|
self.assertTrue(session.transcript_file(self.session))
|
|
self.assertEqual(run.run_cycle(self.cycle(fake, unfinished_evidence="working")).verdict, Unfinished("working", entries_read=4))
|
|
workers = fake.made("stream_claude")
|
|
self.assertIn("--session-id", workers[0])
|
|
self.assertIn("--resume", workers[1])
|
|
self.assertEqual(workers[1][workers[1].index("-p") + 1], "Resumed: working")
|
|
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: unfinished\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n"] * 2)
|
|
|
|
def test_a_cycle_stopped_by_a_failure_or_ctrl_c_archives_what_its_session_holds(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[ran(code=1)] * 3)
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).code, 1)
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[KeyboardInterrupt()])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)), Stop(130, ""))
|
|
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Stopped: Ctrl-C\n\nWorker: {OFF_LADDER}\n", f"Task: Do it.\n\n2026-09-21T14:13:20Z Stopped: the judge failed three times running: the judge exited 1\n\nWorker: {OFF_LADDER}\n"])
|
|
|
|
def test_each_judge_reply_the_loop_rejects_is_archived_with_why_beside_the_verdict_that_follows(self):
|
|
unfilled = ran(json.dumps({"structured_output": {"interrupt": False}}))
|
|
misquoted = judged("finished", evidence="never said")
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[unfilled, misquoted, judged("finished")])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
|
|
stamp = "2026-09-21T14:13:20Z"
|
|
self.assertEqual(self.archived_outcomes(), [
|
|
"Task: Do it.\n\n"
|
|
f"{stamp} Judge reply rejected: the judge's reply has no answer of type string\n\nReply:\n> {unfilled.stdout}\n\nJudge: judge-model\n\n"
|
|
f"{stamp} Judge reply rejected: the judge quoted 'never said', which is not in the transcript\n\nReply:\n> {misquoted.stdout}\n\nJudge: judge-model\n\n"
|
|
f"{stamp} Verdict: finished\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
|
|
|
|
def test_archives_each_judge_runs_prompt_and_reply_beside_the_rejection_interrupt_and_verdict_it_gave(self):
|
|
rejected, interrupted, finished = ran(json.dumps({"structured_output": {"interrupt": False}})), judged("interrupt", cost=0.25), judged("finished")
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[rejected, interrupted, finished], inputs=["Use B."])
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
|
|
prompts = [stdin for args, stdin in fake.made("claude")[:3]]
|
|
[chunk] = self.archive.iterdir()
|
|
kept = "\n".join(path.read_text() for path in chunk.glob("*.outcomes.md"))
|
|
named = re.findall(r"^Judge run: (judge/[0-9a-f]{64}\.json)$", kept, re.MULTILINE)
|
|
self.assertEqual(len(named), 3, kept)
|
|
runs = [json.loads((chunk / name).read_text()) for name in named]
|
|
replies = [json.loads(reply.stdout) for reply in (rejected, interrupted, finished)]
|
|
self.assertEqual(sorted((found["prompt"], json.dumps(found["reply"])) for found in runs), sorted((prompt, json.dumps(reply)) for prompt, reply in zip(prompts, replies)))
|
|
self.assertEqual(sorted(path.name for path in (chunk / "judge").iterdir()), sorted(name.removeprefix("judge/") for name in named))
|
|
|
|
def test_stops_when_a_judge_run_cannot_be_archived(self):
|
|
self.archive.mkdir(parents=True)
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
|
|
fake.replace_file = lambda path, data: Failure("disk full") if "/judge/" in str(path) else replace_file(path, data)
|
|
stop = run.run_cycle(self.cycle(fake))
|
|
self.assertEqual(stop.code, 1)
|
|
self.assertRegex(stop.reason, r"^could not archive the transcript: .*/judge/[0-9a-f]{64}\.json: disk full$")
|
|
|
|
def test_archives_what_each_judge_run_cost_on_an_interrupt_a_quoted_answer_and_a_rejected_misquote(self):
|
|
for first, then, inputs in (
|
|
(judged("interrupt", cost=0.25), judged("finished"), ["Use B."]),
|
|
(judged("quoted", answer="Use A.", cost=0.25), judged("finished"), []),
|
|
(judged("unfinished", cost=0.25), judged("finished"), []),
|
|
(judged("finished", evidence="never said", cost=0.25), judged("finished"), []),
|
|
):
|
|
with self.subTest(first=first.stdout), tempfile.TemporaryDirectory() as archive:
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[first, then], inputs=inputs)
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
run.run_cycle(dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project_archive=pathlib.Path(archive)))
|
|
kept = "".join(path.read_text() for path in pathlib.Path(archive).glob("*/*.outcomes.md"))
|
|
self.assertEqual(kept.count("Judge cost: 0.2500\n"), 1, kept)
|
|
self.assertIn("Judge cost: 0.2500\nJudge run: judge/", kept)
|
|
|
|
def test_stops_with_exit_1_where_a_subagents_transcript_cannot_be_read_for_the_spend(self):
|
|
for verdict in ("interrupt", "finished"):
|
|
with self.subTest(verdict=verdict), tempfile.TemporaryDirectory() as archive:
|
|
unreadable = self.paths.claude_code_transcripts / "-fake" / str(self.session.id) / "subagents" / "agent-x.jsonl"
|
|
unreadable.mkdir(parents=True)
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged(verdict)], inputs=["Use B."])
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
stop = run.run_cycle(dataclasses.replace(self.cycle(fake), project_archive=pathlib.Path(archive)))
|
|
self.assertEqual((stop.code, stop.reason), (1, f"could not read the transcript {unreadable}: Is a directory"))
|
|
shutil.rmtree(unreadable.parent.parent)
|
|
(self.paths.claude_code_transcripts / "-fake" / f"{self.session.id}.jsonl").unlink()
|
|
|
|
def test_counts_the_spend_since_the_last_outcome_that_recorded_one_and_archives_the_reversal(self):
|
|
transcript = self.paths.claude_code_transcripts / "-fake" / f"{self.session.id}.jsonl"
|
|
transcript.parent.mkdir(parents=True)
|
|
old = {"type": "assistant", "timestamp": "2026-09-21T14:13:19Z", "message": {"id": "old", "model": "claude-opus-5-5", "content": [], "usage": {"output_tokens": 100}}}
|
|
new = {"type": "assistant", "timestamp": "2026-09-21T14:13:21Z", "message": {"id": "new", "model": "claude-opus-5-5", "content": [], "usage": {"output_tokens": 5}}}
|
|
transcript.write_text("\n".join(json.dumps(entry) for entry in [{"type": "queue-operation", "timestamp": "2026-09-21T14:00:00Z"}, old, human("No, keep the old name."), new]) + "\n")
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertIsNone(write.archive(self.archive, str(transcript), "Do it.", outcomes.unfinished("w", "e", "j", "", annotation=outcomes.Annotation("c", "Spent since the chunk began, in tokens of x:\n- worker, m: 1")), NOW, replace=replace_file))
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("unfinished", reversed="No, keep the old name.")])
|
|
run.run_cycle(self.cycle(fake))
|
|
latest = max(self.archived_outcomes(), key=len)
|
|
self.assertIn(f"""Reversed:
|
|
> No, keep the old name.
|
|
|
|
Spent since 2026-09-21T14:13:20Z, in tokens of input, cache write 5m, cache write 1h, cache read, output:
|
|
- worker, claude-opus-5-5: 0, 0, 0, 0, 5
|
|
""", latest)
|
|
|
|
def test_stops_with_exit_1_asking_nothing_where_an_interrupt_cannot_be_archived(self):
|
|
self.archive.parent.mkdir(parents=True)
|
|
self.archive.write_text("")
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt")], inputs=["Use B."])
|
|
with contextlib.redirect_stderr(io.StringIO()) as said:
|
|
stop = run.run_cycle(self.cycle(fake))
|
|
self.assertEqual((stop.code, stop.reason.split(":")[0]), (1, "could not archive the transcript"))
|
|
self.assertNotIn("Which one?", said.getvalue())
|
|
self.assertEqual(fake.made("claude_on_terminal"), [])
|
|
|
|
def test_stops_when_a_rejected_reply_cannot_be_archived(self):
|
|
self.archive.parent.mkdir(parents=True)
|
|
self.archive.write_text("")
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[ran(json.dumps({"structured_output": {"interrupt": False}})), judged("finished")])
|
|
stop = run.run_cycle(self.cycle(fake))
|
|
self.assertRegex(stop.reason, f"^could not archive the transcript: {self.archive}/[0-9a-f]{{64}}: Not a directory$")
|
|
self.assertEqual(stop.code, 1)
|
|
self.assertEqual(len(fake.made("claude")), 1)
|
|
|
|
def test_archives_the_judges_rejected_replies_though_it_fails_three_times_running(self):
|
|
unfilled = ran(json.dumps({"structured_output": {"interrupt": False}}))
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[unfilled] * 3)
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).code, 1)
|
|
[archived] = self.archived_outcomes()
|
|
self.assertEqual(archived.count(f"Reply:\n> {unfilled.stdout}\n"), 3)
|
|
self.assertIn("Stopped: the judge failed three times running", archived)
|
|
|
|
def test_archives_each_chunks_session_folder_replacing_a_copy_only_with_what_grew_from_it(self):
|
|
folder = self.paths.claude_code_transcripts / "-fake" / str(self.session.id)
|
|
(folder / "subagents" / "workflows").mkdir(parents=True)
|
|
(folder / "subagents" / "agent-a.jsonl").write_text("a\n")
|
|
(folder / "subagents" / "workflows" / "journal.jsonl").write_text("j\n")
|
|
(folder / "custom-title.json").write_text("first")
|
|
(folder.parent / f"{self.session.id}.orphaned-1.jsonl").write_text("o\n")
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 3, claude=[judged("unfinished"), judged("finished"), judged("unfinished")])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=3))
|
|
[chunk] = self.archive.iterdir()
|
|
kept = chunk / "session" / str(self.session.id)
|
|
self.assertEqual((chunk / "session" / f"{self.session.id}.orphaned-1.jsonl").read_text(), "o\n")
|
|
os.utime(kept / "subagents" / "workflows" / "journal.jsonl", (0, 0))
|
|
with open(folder / "subagents" / "agent-a.jsonl", "a") as handle:
|
|
handle.write("more\n")
|
|
(folder / "custom-title.json").write_text("second")
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
|
|
self.assertFalse(folder.exists())
|
|
self.assertEqual((kept / "subagents" / "agent-a.jsonl").read_text(), "a\nmore\n")
|
|
self.assertEqual((kept / "subagents" / "workflows" / "journal.jsonl").stat().st_mtime, 0)
|
|
self.assertEqual(sorted(path.read_text() for path in kept.glob("custom-title.json*")), ["first", "second"])
|
|
self.assertEqual(stat.S_IMODE((kept / "subagents" / "agent-a.jsonl").stat().st_mode), 0o600)
|
|
folder.mkdir()
|
|
(folder / "custom-title.json").write_text("third")
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=3))
|
|
[next_chunk] = set(self.archive.iterdir()) - {chunk}
|
|
self.assertEqual(sorted(path.read_text() for path in (next_chunk / "session" / str(self.session.id)).glob("custom-title.json*")), ["third"])
|
|
|
|
def test_stops_on_a_session_folder_it_cannot_read_whole(self):
|
|
assert_not_root(self)
|
|
locked = self.paths.claude_code_transcripts / "-fake" / str(self.session.id) / "subagents"
|
|
locked.mkdir(parents=True)
|
|
locked.chmod(0)
|
|
self.addCleanup(locked.chmod, 0o700)
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("unfinished")])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)), Stop(1, f"could not archive the transcript: {locked}: Permission denied"))
|
|
|
|
def test_an_interrupt_talked_over_in_claude_is_archived_answered_and_judged_again(self):
|
|
def answer(args):
|
|
with open(session.transcript_file(self.session), "a") as handle:
|
|
handle.write(json.dumps(human("the second")) + "\n")
|
|
return 0
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt"), judged("unfinished")], inputs=["/talk"], on_terminal_claude=[answer])
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=4))
|
|
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: interrupt\n\nWhich one?\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n", f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: unfinished\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
|
|
self.assertIn(str(self.session.id), fake.made("claude_on_terminal")[0])
|
|
judge_prompts = [stdin for args, stdin in fake.made("claude")]
|
|
self.assertEqual(len(judge_prompts), 2)
|
|
self.assertNotIn("the second", judge_prompts[0])
|
|
self.assertIn("[human] the second", judge_prompts[1])
|
|
self.assertIn("From the goals file, ## Goals", judge_prompts[0])
|
|
self.assertIn("writing it into the plan or a comment,", judge_prompts[0])
|
|
self.assertIn("The task the worker was given:\nDo it.", judge_prompts[0])
|
|
|
|
def test_a_question_the_records_answer_resumes_the_worker_with_the_answer_three_times_a_cycle(self):
|
|
answered = judged("quoted", answer="Use *A*.")
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 4, claude=[answered] * 4, inputs=[""])
|
|
with contextlib.redirect_stderr(io.StringIO()) as said, contextlib.redirect_stdout(io.StringIO()) as printed:
|
|
self.assertEqual(run.run_cycle(self.cycle(fake, records="From AGENTS.md:\nUse A.")), Stop(2, "the human is needed to answer the question above, but nothing was typed"))
|
|
resumed = fake.made("stream_claude")[1]
|
|
self.assertEqual(resumed[resumed.index("-p") + 1], "Your question: Which one?\n\nthe project's records answers it:\n\nUse *A*.\n\nContinue the task.")
|
|
self.assertEqual(len(fake.made("stream_claude")), 4)
|
|
self.assertEqual(printed.getvalue().count("=== the project's records answered: Which one? ===\nUse *A*."), 3)
|
|
self.assertIn("Which one?\n\nthe project's records says:\nUse *A*.\n\nThose files have answered 3 questions this cycle", said.getvalue())
|
|
quoted = f"2026-09-21T14:13:20Z Verdict: quoted answer\n\nWhich one?\n\nthe project's records says:\n> Use *A*.\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n"
|
|
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: interrupt\n\nWhich one?\n\nthe project's records says:\nUse *A*.\n\nThose files have answered 3 questions this cycle, the most allowed, so this one comes to you.\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n\n2026-09-21T14:13:20Z Stopped: the human is needed to answer the question above, but nothing was typed\n\nWorker: {OFF_LADDER}\n", *[f"Task: Do it.\n\n{quoted}"] * 3])
|
|
|
|
def test_a_quoted_answer_that_cites_an_escalation_trigger_steps_the_worker_up_before_it_resumes_and_archives_the_step(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[judged("quoted", answer="Use A.", escalation="working"), judged("finished")])
|
|
cycle = dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project=OPUS_PROJECT, route=Route(SONNET, 179, 2, 5))
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
concluded = run.run_cycle(cycle)
|
|
self.assertEqual(concluded.route, Route(OPUS, 179, 2, 5, escalated_at=3))
|
|
first, resumed = fake.made("stream_claude")
|
|
self.assertEqual([args[args.index("--effort") + 1] for args in (first, resumed)], ["high", "medium"])
|
|
self.assertTrue(resumed[resumed.index("-p") + 1].endswith("Continue the task. Your request for a higher rung went through: a-driver runs you on claude-opus-5-5 at medium effort. The escalation triggers that fired before this step are spent. You hold claude-opus-5-5 at medium effort."))
|
|
self.assertNotIn("went through", first[first.index("-p") + 1])
|
|
answered = f"Verdict: quoted answer\n\nWhich one?\n\nthe project's records says:\n> Use A.\n\nEvidence:\n> working\n\nEscalation:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {SONNET_179}\n"
|
|
stepped = f"Stepped up\n\nWorker: {OPUS_179_AT_3}\n"
|
|
kept = "".join(self.archived_outcomes())
|
|
self.assertIn(f"Z {answered}\n20", kept)
|
|
self.assertIn(f"Z {stepped}", kept)
|
|
|
|
def test_a_quoted_answer_past_the_cap_archives_the_escalation_it_carried_with_the_interrupt_and_steps_up(self):
|
|
steps = [judged("quoted", answer="Use A.")] * 3 + [judged("quoted", answer="Use A.", escalation="working")]
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 4, claude=steps, inputs=[""])
|
|
cycle = dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project=OPUS_PROJECT, route=Route(SONNET, 179, 2, 5))
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(run.run_cycle(cycle), Stop(2, "the human is needed to answer the question above, but nothing was typed"))
|
|
kept = "".join(self.archived_outcomes())
|
|
self.assertIn(f"Evidence:\n> working\n\nEscalation:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {SONNET_179}\n", kept)
|
|
self.assertIn(f"Stepped up\n\nWorker: {OPUS_179_AT_6}\n", kept)
|
|
|
|
def test_a_ruling_steps_the_route_only_on_an_escalation_from_the_rung_its_taken_in_l_picks(self):
|
|
cycle = dataclasses.replace(self.cycle(FakeEffects()), project=OPUS_PROJECT, route=Route(ONE_UP, 3, 3, 4))
|
|
for ruling in (Interrupt("e", "q", entries_read=3), QuotedAnswer("e", "q", "A.", entries_read=3)):
|
|
with self.subTest(ruling=ruling):
|
|
self.assertIsNone(run.ruling_step_up(cycle, ruling))
|
|
self.assertEqual(run.ruling_step_up(cycle, Interrupt("e", "q", "round 2: 🔴", TakenIn(highest_likelihood=4), entries_read=3)), routing.StepUp(cycle.route, Route(THREE_UP, 3, 3, 4, escalated_at=3)))
|
|
|
|
def test_an_escalation_after_an_interrupt_stepped_the_worker_steps_it_again_in_the_same_cycle(self):
|
|
streams = [Worker(self.paths.claude_code_transcripts.parent, text="round 2: 🔴"), Worker(self.paths.claude_code_transcripts.parent), Worker(self.paths.claude_code_transcripts.parent)]
|
|
rulings = [judged("interrupt", evidence="round 2: 🔴", escalation="round 2: 🔴"), judged("quoted", answer="Use A.", escalation="working"), judged("finished")]
|
|
fake = FakeEffects(streams=streams, claude=rulings, inputs=["Use B."])
|
|
cycle = dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project=OPUS_PROJECT, route=Route(SONNET, 179, 2, 5))
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
concluded = run.run_cycle(cycle)
|
|
self.assertEqual(concluded.route, Route(ONE_UP, 179, 2, 5, escalated_at=4))
|
|
self.assertEqual([args[args.index("--effort") + 1] for args in fake.made("stream_claude")], ["high", "medium", "low"])
|
|
|
|
def test_an_escalation_quoted_from_before_the_workers_last_step_up_steps_nothing_and_a_later_one_reaches_the_verdict(self):
|
|
for escalation, verdict in (("round 2: 🔴", Unfinished("working", entries_read=4)), ("working", Unfinished("working", "working", entries_read=4))):
|
|
with self.subTest(escalation=escalation):
|
|
self.session = session.session_named(f"A proj {escalation} a-driver", self.paths.claude_code_transcripts)
|
|
streams = [Worker(self.paths.claude_code_transcripts.parent, text="round 2: 🔴"), Worker(self.paths.claude_code_transcripts.parent)]
|
|
rulings = [judged("interrupt", evidence="round 2: 🔴", escalation="round 2: 🔴"), judged("unfinished", escalation=escalation)]
|
|
fake = FakeEffects(streams=streams, claude=rulings, inputs=["Use B."])
|
|
cycle = dataclasses.replace(self.cycle(fake), project=OPUS_PROJECT, route=Route(SONNET, 179, 2, 5))
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
concluded = run.run_cycle(cycle)
|
|
self.assertEqual(concluded, run.Concluded(verdict, Route(OPUS, 179, 2, 5, escalated_at=3)))
|
|
|
|
def test_an_answer_typed_at_the_terminal_resumes_the_worker_headless_and_the_judge_reads_it_as_the_humans(self):
|
|
worker = Worker(self.paths.claude_code_transcripts.parent, prompted=True)
|
|
fake = FakeEffects(streams=[worker, worker], claude=[judged("interrupt"), judged("unfinished")], inputs=["Use B."])
|
|
with contextlib.redirect_stderr(io.StringIO()) as said:
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Unfinished("working", entries_read=6))
|
|
self.assertIn(f"\n{STAMP} Which one?\a\n", said.getvalue())
|
|
self.assertEqual(fake.made("claude_on_terminal"), [])
|
|
resumed = fake.made("stream_claude")[1]
|
|
self.assertEqual(resumed[resumed.index("--resume") + 1], str(self.session.id))
|
|
typed = resumed[resumed.index("-p") + 1]
|
|
self.assertEqual(worker_prompts.typed_answer(typed).answer, "Use B.")
|
|
self.assertIn("Which one?", typed)
|
|
self.assertIn("Continue the task. The chunk is finished only once every answer the human typed in it", typed)
|
|
judge_prompts = [stdin for args, stdin in fake.made("claude")]
|
|
self.assertRegex(judge_prompts[1], r"(?s)\[loop\] Your question was: Which one\?.*\n\n\[human\] Use B\.\n\nworking\n\nDecide whether")
|
|
self.assertNotIn("The human answered", judge_prompts[1])
|
|
|
|
def test_an_interrupt_that_steps_the_worker_up_names_its_new_rung_in_the_prompt_that_resumes_it_with_the_typed_answer(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[judged("interrupt", escalation="working"), judged("finished")], inputs=["Use B."])
|
|
cycle = dataclasses.replace(self.cycle(fake), project=OPUS_PROJECT, route=Route(SONNET, 179, 2, 5))
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
concluded = run.run_cycle(cycle)
|
|
self.assertEqual(concluded.route, Route(OPUS, 179, 2, 5, escalated_at=3))
|
|
first, resumed = fake.made("stream_claude")
|
|
self.assertEqual([args[args.index("--effort") + 1] for args in (first, resumed)], ["high", "medium"])
|
|
typed = resumed[resumed.index("-p") + 1]
|
|
self.assertIn("Your request for a higher rung went through: a-driver runs you on claude-opus-5-5 at medium effort.", typed)
|
|
self.assertIn(" You hold claude-opus-5-5 at medium effort.", typed)
|
|
self.assertNotIn("went through", first[first.index("-p") + 1])
|
|
|
|
def test_an_interrupt_that_steps_the_worker_up_compacts_its_session_on_the_rung_it_ran_on(self):
|
|
usage = {"input_tokens": 2, "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 5000}, "cache_read_input_tokens": 70000, "output_tokens": 100}
|
|
worker = Worker(self.paths.claude_code_transcripts.parent, usage=usage, sent=utc(NOW - 600))
|
|
fake = FakeEffects(streams=[worker, Streamed(0), worker], claude=[judged("interrupt", escalation="working"), judged("finished")])
|
|
|
|
def typed(prompt):
|
|
for _, action in fake.timers:
|
|
action()
|
|
return "Use B."
|
|
fake.scripts["input"] = [typed]
|
|
cycle = dataclasses.replace(self.cycle(fake), project=OPUS_PROJECT, route=Route(SONNET, 179, 2, 5))
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
run.run_cycle(cycle)
|
|
_, compaction, resumed = fake.made("stream_claude")
|
|
self.assertEqual(compaction[compaction.index("-p") + 1], "/compact")
|
|
self.assertEqual([(args[args.index("--model") + 1], args[args.index("--effort") + 1]) for args in (compaction, resumed)], [("claude-sonnet-5-5", "high"), ("claude-opus-5-5", "medium")])
|
|
|
|
def test_a_typed_answer_past_the_quoted_answers_cap_reaches_the_worker_with_its_own_question(self):
|
|
worker = Worker(self.paths.claude_code_transcripts.parent)
|
|
fake = FakeEffects(streams=[worker] * 5, claude=[judged("quoted", answer="Use A.")] * 4 + [judged("finished")], inputs=["Use B."])
|
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(run.run_cycle(self.cycle(fake, records="From AGENTS.md:\nUse A.")).verdict, Finished("working"))
|
|
typed = fake.made("stream_claude")[4]
|
|
self.assertIn("Your question was: Which one?\n\nContinue the task.", typed[typed.index("-p") + 1])
|
|
self.assertNotIn("Those files have answered", typed[typed.index("-p") + 1])
|
|
|
|
def test_a_release_ready_to_cut_opens_claudes_session_without_asking_for_an_answer(self):
|
|
def answer(args):
|
|
with open(session.transcript_file(self.session), "a") as handle:
|
|
handle.write(json.dumps(human("cut")) + "\n")
|
|
return 0
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt", release_ready=True), judged("finished")], on_terminal_claude=[answer])
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
|
|
self.assertEqual(fake.made("input"), [])
|
|
self.assertEqual(len(fake.made("claude_on_terminal")), 1)
|
|
|
|
def test_stops_when_an_interrupt_cannot_be_archived(self):
|
|
self.archive.parent.mkdir(parents=True)
|
|
self.archive.write_text("")
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt")], inputs=["Use B."])
|
|
stop = run.run_cycle(self.cycle(fake))
|
|
self.assertRegex(stop.reason, f"^could not archive the transcript: {self.archive}/[0-9a-f]{{64}}: Not a directory$")
|
|
self.assertEqual(fake.made("input"), [])
|
|
|
|
def test_stops_when_a_quoted_answer_cannot_be_archived(self):
|
|
self.archive.parent.mkdir(parents=True)
|
|
self.archive.write_text("")
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("quoted", answer="Use A.")])
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
stop = run.run_cycle(self.cycle(fake, records="From AGENTS.md:\nUse A."))
|
|
self.assertRegex(stop.reason, f"^could not archive the transcript: {self.archive}/[0-9a-f]{{64}}: Not a directory$")
|
|
self.assertEqual(len(fake.made("stream_claude")), 1)
|
|
|
|
def test_an_answer_the_records_do_not_hold_is_dropped_and_the_question_goes_to_the_human(self):
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("quoted", answer="Use ... A.")], inputs=[""])
|
|
with contextlib.redirect_stderr(io.StringIO()) as said:
|
|
self.assertEqual(run.run_cycle(self.cycle(fake, records="From AGENTS.md:\nUse B. Use A.")), Stop(2, "the human is needed to answer the question above, but nothing was typed"))
|
|
self.assertEqual(len(fake.made("stream_claude")), 1)
|
|
self.assertIn(f"\n{STAMP} Which one?\a\n", said.getvalue())
|
|
self.assertNotIn("Use ... A.", said.getvalue())
|
|
|
|
def test_stops_keeping_the_transcript_when_the_finished_chunks_session_cannot_be_deleted(self):
|
|
assert_not_root(self)
|
|
folder = self.paths.claude_code_transcripts / "-fake" / str(self.session.id)
|
|
(folder / "subagents").mkdir(parents=True)
|
|
folder.chmod(0o500)
|
|
self.addCleanup(folder.chmod, 0o700)
|
|
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
|
|
stop = run.run_cycle(self.cycle(fake))
|
|
self.assertEqual(stop, Stop(1, f"could not delete {folder}: Permission denied; delete every file and folder matching {folder}*, then run it again; a run before that resumes the finished chunk's session"))
|
|
self.assertTrue(session.session_open(self.session))
|
|
self.assertEqual(self.archived_outcomes(), [f"Task: Do it.\n\n2026-09-21T14:13:20Z Verdict: finished\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n\n2026-09-21T14:13:20Z Stopped: {stop.reason}\n\nWorker: {OFF_LADDER}\n"])
|
|
|
|
def test_stops_when_the_worker_leaves_no_transcript(self):
|
|
fake = FakeEffects(streams=[Streamed(0)])
|
|
self.assertEqual(run.run_cycle(self.cycle(fake)), Stop(1, f"claude left no transcript for session {self.session.id}"))
|