Files
claude-config/bin/loop_claude/test_black_box.py
T

964 lines
54 KiB
Python

import dataclasses
import datetime
import hashlib
import json
import os
import pathlib
import pty
import re
import select
import shutil
import stat
import subprocess
import sys
import tempfile
import time
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from test_env import isolated_env
from loop_state.waiting.answers import send, Typed
from loop_state import session_lock
from loop_state.waiting.listing import waiting
from project_config.test_project_config import write_config
BIN = pathlib.Path(__file__).resolve().parents[1]
TASK = "Do the next chunk."
# How the archive records a worker that runs off the ladder.
OFF_LADDER = '{"model": "worker-model"}'
# How a reload hands that worker's route.
OFF_LADDER_ROUTE = {"rung": {"model": "worker-model", "effort": ""}, "item": 0, "likelihood": 0, "escape_and_impact": 0, "unfinished": 0, "escape_and_impact_taken_in": 0, "escalated_at": 0}
# Plays claude from FAKE_CLAUDE_SCENARIO, one step per call, logging each call; a step for another role,
# or none left, is Ctrl-C to loop-claude, which otherwise loops forever.
FAKE_CLAUDE = """#!{python}
import json, os, pathlib, signal, sys, time
argv = sys.argv[1:]
role = "judge" if "--json-schema" in argv else "worker" if "-p" in argv else "interrupt"
scenario = pathlib.Path(os.environ["FAKE_CLAUDE_SCENARIO"])
steps = json.loads(scenario.read_text())
call = {{"argv": argv, "cwd": os.getcwd(), "role": role, "ctrl_c_ignored": signal.getsignal(signal.SIGINT) == signal.SIG_IGN, "sandboxed": os.environ.get("CLAUDE_CODE_SANDBOXED")}}
# The items each loop lists in loop-inbox as this claude starts.
call["running"] = [json.loads(path.read_text())["items"] for path in sorted((pathlib.Path.home() / ".local" / "state" / "loop-claude").glob("*.running"))]
if role == "judge":
call["stdin"] = sys.stdin.read()
with open(os.environ["FAKE_CLAUDE_LOG"], "a") as log:
log.write(json.dumps(call) + "\\n")
if not steps or steps[0]["role"] != role:
parent = os.getppid()
os.kill(parent, signal.SIGINT)
while os.getppid() == parent:
time.sleep(0.05)
sys.exit(0)
step = steps.pop(0)
scenario.write_text(json.dumps(steps))
def session():
flag = "--session-id" if "--session-id" in argv else "--resume"
return argv[argv.index(flag) + 1]
def transcript():
return pathlib.Path.home() / ".claude" / "projects" / "-fake" / f"{{session()}}.jsonl"
def append(*entries):
transcript().parent.mkdir(parents=True, exist_ok=True)
with open(transcript(), "a") as handle:
handle.writelines(json.dumps(entry) + "\\n" for entry in entries)
if role == "worker":
if "--session-id" in argv and step.get("transcript", True):
append({{"type": "custom-title"}}, {{"type": "queue-operation", "timestamp": str(time.time_ns())}})
said = {{"type": "assistant", "message": {{"content": [{{"type": "text", "text": step.get("say", "working")}}]}}}}
if step.get("transcript", True):
append({{"type": "user", "promptSource": "sdk", "message": {{"content": argv[argv.index("-p") + 1]}}}})
with open(transcript(), "ab") as handle:
handle.write(step.get("raw", "").encode("latin-1"))
append(said)
if step.get("siblings"):
for sibling in (f"{{session()}}/subagents", f"{{session()}}.superseded-1/tool-results"):
(transcript().parent / sibling).mkdir(parents=True, exist_ok=True)
(transcript().parent / f"{{session()}}.orphaned-1-ab.jsonl").write_text("{{}}\\n")
if "append" in step:
edited = pathlib.Path(os.environ["FAKE_CLAUDE_EDIT"], step["append"][0])
edited.parent.mkdir(parents=True, exist_ok=True)
with open(edited, "a") as script:
script.write(step["append"][1])
if "replace" in step:
script = pathlib.Path(os.environ["FAKE_CLAUDE_EDIT"], step["replace"][0])
script.write_text(script.read_text().replace(*step["replace"][1:]))
sys.stdout.flush()
sys.stdout.buffer.write(step.get("stream", "").encode("latin-1"))
print(json.dumps(said))
if step.get("result", True):
print(json.dumps({{"type": "result"}}))
if "signal" in step:
os.kill(os.getpid(), step["signal"])
elif role == "judge":
print(step["stdout"] if "stdout" in step else json.dumps({{"structured_output": step["verdict"]}}))
print(step.get("stderr", ""), file=sys.stderr)
else:
if step.get("ctrl_c"):
# A Ctrl-C on the terminal reaches every process in its foreground, loop-claude too. kill returns once the signal is pending in loop-claude, before this process exits and loop-claude stops waiting for it.
os.kill(os.getppid(), signal.SIGINT)
if "type" in step:
append({{"type": "user", "message": {{"content": step["type"]}}}})
if step.get("unreadable"):
transcript().chmod(0)
if step.get("vanish"):
os.remove(__file__)
sys.exit(step.get("exit", 0))
"""
# Records every start of loop-claude and turns its retry waits into log lines.
SITECUSTOMIZE = """import json, os, sys, time
if os.path.basename(sys.argv[0]) == "loop-claude":
def record(event):
with open(os.environ["LOOP_TEST_LOG"], "a") as log:
log.write(json.dumps(event) + "\\n")
record({"argv": sys.argv[1:]})
pause = time.sleep
time.sleep = lambda seconds: record({"sleep": seconds}) if seconds >= 1 else pause(seconds)
# The host's own memory doesn't matter: every read finds all of it available.
sys.path.insert(0, os.path.dirname(os.path.realpath(sys.argv[0])))
import pathlib
from machine_load import memory
memory.MEMINFO = pathlib.Path(os.environ["FAKE_MEMINFO"])
"""
PREMISES = "# Project\n\n## Goals\n\n1. Work.\n\n## Audience\n\nMe.\n"
# Item 4's L of 4 steps one rung up from the project's.
TODO = "## Items\n\n| ID | Release | Exempt | Item | L | E | I | R | S | Moves | Score |\n|---|---|---|---|---|---|---|---|---|---|---|\n| 4 | 0.1.0 | | **Do it.** | 4 | 2 | 3 | 7 | 2 | 1 by 9 | 25 |\n"
# Item 4's L of 2 steps one rung down.
CHEAP = TODO.replace("| 4 | 2 | 3 | 7 |", "| 2 | 1 | 2 | 3 |")
# Item 9 follows CHEAP's item 4, and a worker routed by item 4 may take it in: at L 4 it picks a higher rung for the worker, and at E + I 6 for the reviewers. At E + I 3, or at L 2, it picks the rung item 4 picks.
TAKEN_IN_AT_L_4 = CHEAP + "| 9 | 0.1.0 | | **Do more.** | 4 | 1 | 2 | 5 | 2 | 1 by 9 | 20 |\n"
TAKEN_IN_AT_E_PLUS_I_6 = CHEAP + "| 9 | 0.1.0 | | **Do more.** | 2 | 3 | 3 | 6 | 2 | 1 by 9 | 20 |\n"
# Waits for the interrupt's prompt before typing the answer, since loop-claude drops what was typed before it.
ANSWERING = ("Answer (", lambda: None)
def worker(**step):
return {"role": "worker", **step}
def judge(interrupt=False, finished=False, evidence="working", question="", escalation="", release_ready=False, **step):
verdict = {"answer": "", "escalation": escalation, "evidence": evidence, "finished": finished, "interrupt": interrupt, "question": question, "answered_before": "", "left_open": "", "release_ready": release_ready, "reversed": ""}
return {"role": "judge", "verdict": verdict, **step}
def interrupt(**step):
return {"role": "interrupt", **step}
def lines(path):
return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else []
@dataclasses.dataclass
class Run:
calls: list
code: int
sleeps: list
starts: list
# stderr and stdout are without the local time that opens each message loop-claude prints; raw_stderr and raw_stdout keep it.
stderr: str
stdout: str
raw_stderr: str
raw_stdout: str
def roles(self):
return [call["role"] for call in self.calls]
class Checkout:
"""A temporary HOME with a git checkout at ~/code/proj, where loop-claude runs against the fake claude."""
def __init__(self, test):
self.test = test
temp = tempfile.TemporaryDirectory()
test.addCleanup(temp.cleanup)
self.temp = pathlib.Path(os.path.realpath(temp.name))
self.home = self.temp / "home"
self.root = self.home / "code" / "proj"
self.root.mkdir(parents=True)
(self.root / "README.md").write_text(PREMISES)
write_config(self.home, self.root)
self.script = BIN / "loop-claude"
self.env = isolated_env()
fake = self.temp / "fake-bin"
fake.mkdir()
(fake / "claude").write_text(FAKE_CLAUDE.format(python=sys.executable))
(fake / "claude").chmod(0o755)
(fake / "git").symlink_to(shutil.which("git"))
site = self.temp / "site"
site.mkdir()
(site / "sitecustomize.py").write_text(SITECUSTOMIZE)
(self.temp / "meminfo").write_text("MemTotal: 16 kB\nMemAvailable: 16 kB\n")
self.env.update({
"FAKE_CLAUDE_LOG": str(self.temp / "calls.jsonl"),
"FAKE_MEMINFO": str(self.temp / "meminfo"),
"FAKE_CLAUDE_SCENARIO": str(self.temp / "scenario.json"),
"GIT_CONFIG_GLOBAL": os.devnull,
"GIT_CONFIG_NOSYSTEM": "1",
"HOME": str(self.home),
"LOOP_CLAUDE_JUDGE_MODEL": "judge-model",
"LOOP_CLAUDE_WORKER_MODEL": "worker-model",
"LOOP_TEST_LOG": str(self.temp / "starts.jsonl"),
"PATH": str(fake),
"PYTHONPATH": str(site),
})
self.git("init", "--quiet")
def git(self, *args):
subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@example.com", *args], capture_output=True, check=True, cwd=self.root, env=self.env)
def commit_plan(self, todo):
"""Commits todo as todo.md on the checkout's branch, the default branch routing reads it from."""
(self.root / "todo.md").write_text(todo)
self.git("add", "todo.md")
self.git("commit", "--quiet", "-m", "plan")
def run(self, steps, argv=(TASK,), cwd=None, terminal_input=None, stdout_on_terminal=True, before_typing=None, typed_early=""):
for name in ("FAKE_CLAUDE_LOG", "LOOP_TEST_LOG"):
pathlib.Path(self.env[name]).unlink(missing_ok=True)
pathlib.Path(self.env["FAKE_CLAUDE_SCENARIO"]).write_text(json.dumps(steps))
command = [sys.executable, str(self.script), *argv]
cwd = cwd or self.root
if terminal_input is None:
done = subprocess.run(command, capture_output=True, cwd=cwd, env=self.env, stdin=subprocess.DEVNULL, text=True, timeout=60)
code, stdout, stderr = done.returncode, done.stdout, done.stderr
else:
code, stdout, stderr = self.run_on_terminal(command, cwd, terminal_input, stdout_on_terminal, before_typing, typed_early)
started = lines(pathlib.Path(self.env["LOOP_TEST_LOG"]))
return Run(
calls=lines(pathlib.Path(self.env["FAKE_CLAUDE_LOG"])),
code=code,
sleeps=[event["sleep"] for event in started if "sleep" in event],
starts=[event["argv"] for event in started if "argv" in event],
stderr=unstamped(stderr),
stdout=unstamped(stdout),
raw_stderr=stderr,
raw_stdout=stdout,
)
def run_on_terminal(self, command, cwd, typed, stdout_on_terminal, before_typing, typed_early):
"""Types typed_early at once, then the input at once, or once stderr shows before_typing's text and its callback has run."""
controller, terminal = pty.openpty()
with tempfile.TemporaryFile() as stderr, tempfile.TemporaryFile() as stdout:
try:
proc = subprocess.Popen(command, cwd=cwd, env=self.env, stderr=stderr, stdin=terminal, stdout=terminal if stdout_on_terminal else stdout)
finally:
os.close(terminal)
try:
os.write(controller, typed_early.encode())
prompt, callback = before_typing or (None, None)
if prompt is None:
os.write(controller, typed.encode())
output, deadline = b"", time.monotonic() + 60
while time.monotonic() < deadline:
if select.select([controller], [], [], 0.05)[0]:
try:
chunk = os.read(controller, 4096)
except OSError:
break
if not chunk:
break
output += chunk
if prompt is not None and prompt.encode() in os.pread(stderr.fileno(), 1 << 20, 0):
callback()
os.write(controller, typed.encode())
prompt = None
code = proc.wait(timeout=10)
finally:
os.close(controller)
proc.kill()
proc.wait()
stderr.seek(0)
stdout.seek(0)
output += stdout.read()
return code, output.decode(errors="replace").replace("\r\n", "\n"), stderr.read().decode()
def claude_json(self):
return self.home / ".claude.json"
def models_file(self):
return self.home / ".claude" / "loop-claude-models.json"
def transcripts(self):
return sorted((self.home / ".claude" / "projects").glob("*/*.jsonl"))
def archive(self):
return pathlib.Path(self.env.get("LOOP_CLAUDE_TRANSCRIPTS", self.home / ".claude" / "loop-claude-transcripts"))
def archived(self):
"""Each archived transcript's outcomes, with every UTC stamp replaced by <UTC> and every setup version by <config>, after checking the copy is named by its content hash."""
outcomes = {}
for folder in (self.archive(), self.archive() / "proj", *(self.archive() / "proj").iterdir()):
self.test.assertEqual(stat.S_IMODE(folder.stat().st_mode), 0o700, folder)
for copy in sorted((self.archive() / "proj").glob("*/*.jsonl")):
self.test.assertEqual(copy.stem, hashlib.sha256(copy.read_bytes()).hexdigest())
kept = copy.with_suffix(".outcomes.md")
for archived in (copy, kept):
self.test.assertEqual(stat.S_IMODE(archived.stat().st_mode), 0o600)
stamped = re.sub(r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ ", "<UTC> ", kept.read_text(), flags=re.MULTILINE)
outcomes[copy.read_text()] = re.sub(r"^Config: .*$", "Config: <config>", stamped, flags=re.MULTILINE)
return outcomes
STAMP = r"\d{4}-\d\d-\d\d \d\d:\d\d:\d\d"
def unstamped(text):
return re.sub(rf"(?m)^{STAMP} ", "", text)
def argument(call, flag):
return call["argv"][call["argv"].index(flag) + 1] if flag in call["argv"] else None
class Premises(unittest.TestCase):
def test_refuses_to_start_without_a_task_a_checkout_or_premises(self):
checkout = Checkout(self)
for argv, code, said in (([], 1, "usage: loop-claude <task>"), (["--help"], 0, "usage: loop-claude <task>"), (["-h"], 0, "usage: loop-claude <task>")):
with self.subTest(argv=argv):
run = checkout.run([], argv=argv)
self.assertEqual(run.code, code)
self.assertIn(said, run.stdout if code == 0 else run.stderr)
outside = checkout.temp / "outside"
outside.mkdir()
run = checkout.run([], cwd=outside)
self.assertEqual((run.code, run.calls), (1, []))
self.assertRegex(run.raw_stderr, rf"(?m)^{STAMP} .*not a git repository")
readmes = {
None: "README.md in {root}: No such file or directory - no premises.",
b"# P\n## Goals\n1. g\xff\n## Audience\nme\n": "README.md in {root} is not UTF-8 on line 3 - no premises.",
b"# P\n## Audience\nme\n": "{root}/README.md has no ## Goals heading - no premises.",
b"# P\n## Goals\n1. g\n": "{root}/README.md has no ## Audience heading - no premises.",
b"# P\n## Goals\n\n## Audience\nme\n": "{root}/README.md has no text under ## Goals - no premises.",
b"# P\n## Goals\n1. g\n## Audience\nme\n## Audience\nyou\n": "{root}/README.md has ## Audience more than once - no premises.",
}
(checkout.root / "sub").mkdir()
for text, said in readmes.items():
with self.subTest(said=said):
readme = checkout.root / "README.md"
readme.unlink(missing_ok=True)
if text is not None:
readme.write_bytes(text)
run = checkout.run([worker()], cwd=checkout.root / "sub")
self.assertEqual((run.code, run.calls), (1, []))
self.assertEqual(run.stderr.strip(), said.format(root=checkout.root))
(checkout.root / "README.md").unlink()
(checkout.root / "README.md").mkdir()
run = checkout.run([worker()])
self.assertEqual((run.code, run.calls), (1, []))
self.assertEqual(run.stderr.strip(), f"README.md in {checkout.root}: Is a directory - no premises.")
(checkout.home / ".claude-projects" / "proj" / "config.toml").unlink()
run = checkout.run([worker()])
self.assertEqual((run.code, run.calls), (1, []))
self.assertEqual(run.stderr.strip(), f"no config names the checkout {checkout.root}: write ~/.claude-projects/<key>/config.toml, as `project-config --help` shows - no premises.")
(checkout.temp / "fake-bin" / "git").unlink()
run = checkout.run([worker()])
self.assertEqual((run.code, run.calls), (1, []))
self.assertTrue(run.stderr.startswith("git rev-parse --show-toplevel could not start: "), run.stderr)
def test_reads_the_premises_from_the_file_the_config_names(self):
checkout = Checkout(self)
checkout.commit_plan(TODO)
(checkout.root / "README.md").write_text("# Project\n")
folder = write_config(checkout.home, checkout.root, goals="folder:goals.md")
(folder / "goals.md").write_text("## Goals\n1. Kept apart.\n\n## Audience\nThe client.\n")
run = checkout.run([worker()])
self.assertTrue(run.calls, run.stderr)
self.assertIn("## Goals\n1. Kept apart.\n\n## Audience\nThe client.", argument(run.calls[0], "-p"))
def test_makes_no_worktree_for_a_readme_without_premises(self):
checkout = Checkout(self)
checkout.commit_plan(TODO)
(checkout.root / "README.md").write_text("# Project\n")
run = checkout.run([worker()])
self.assertEqual((run.code, run.calls), (1, []))
self.assertFalse((checkout.home / "code" / "proj-item-4").exists())
def test_names_the_session_by_the_checkout_relative_to_code_or_else_to_the_root(self):
checkout = Checkout(self)
for root in (checkout.temp / "elsewhere", checkout.home / "code"):
with self.subTest(root=root):
root.mkdir(exist_ok=True)
(root / "README.md").write_text(PREMISES)
write_config(checkout.home, root)
checkout.git("init", "--quiet", str(root))
run = checkout.run([worker()], cwd=root)
self.assertEqual(argument(run.calls[0], "--name"), f"A {str(root).lstrip('/')} loop-claude")
class Cycles(unittest.TestCase):
def test_resumes_the_session_while_unfinished_and_starts_it_fresh_after_finishing(self):
checkout = Checkout(self)
(checkout.root / "sub").mkdir()
(checkout.root / "docs").mkdir()
(checkout.root / "docs" / "decisions.md").write_text("# Decisions\n")
(checkout.home / ".claude").mkdir()
(checkout.home / ".claude" / "CLAUDE.md").write_text("# Global instructions\n")
run = checkout.run([
worker(say="started"), judge(evidence="started"),
worker(say="landed-it", siblings=True), judge(finished=True, evidence="landed-it"),
worker(say="next chunk"),
], cwd=checkout.root / "sub")
self.assertEqual(run.code, 130)
self.assertEqual(run.roles(), ["worker", "judge", "worker", "judge", "worker", "judge"])
workers = [call for call in run.calls if call["role"] == "worker"]
session = argument(workers[0], "--session-id")
self.assertEqual(argument(workers[0], "--name"), "A proj loop-claude")
self.assertEqual(argument(workers[1], "--resume"), session)
self.assertIsNone(argument(workers[1], "--session-id"))
self.assertIsNone(argument(workers[1], "--name"))
self.assertEqual(argument(workers[2], "--session-id"), session)
self.assertTrue(argument(workers[0], "-p").startswith(f"{TASK}\n\nContinue any open draft PR"))
self.assertTrue(argument(workers[0], "-p").endswith("The chunk's todo.md items, or else the task above, set its scope. The goals below, " 'which rank after code health (~/.claude/CLAUDE.md → "Shared goal") unless the goals file voids it' ", choose between ways to deliver that scope, and never widen it.\n\nFrom the goals file, " f"{checkout.root}/README.md:\n## Goals\n1. Work.\n\n## Audience\nMe."))
self.assertIn("main or master", argument(workers[0], "-p"))
self.assertNotIn(TASK, argument(workers[1], "-p"))
self.assertIn("Even then, it is finished only once every answer the human typed in it", argument(workers[1], "-p"))
self.assertIn("as the work remaining:\n\n> started\n\n", argument(workers[1], "-p"))
self.assertEqual({call["cwd"] for call in run.calls}, {str(checkout.root)})
self.assertIn("1. Work.", run.calls[1]["stdin"])
self.assertIn("From docs/decisions.md:\n# Decisions\n\nFrom ~/.claude/CLAUDE.md:\n# Global instructions", run.calls[1]["stdin"])
self.assertEqual({argument(call, "--model") for call in workers}, {"worker-model"})
self.assertEqual({argument(call, "--model") for call in run.calls if call["role"] == "judge"}, {"judge-model"})
self.assertIn("landed-it", run.calls[3]["stdin"])
self.assertNotIn("landed-it", run.calls[5]["stdin"])
self.assertEqual([path.name for path in checkout.transcripts()[0].parent.iterdir()], [checkout.transcripts()[0].name])
self.assertIn("=== cycle 3, worker on worker-model ===", run.stdout)
self.assertEqual(len(re.findall(rf"(?m)^{STAMP} === cycle 3, worker on worker-model ===$", run.raw_stdout)), 1)
copies = {verdict.removeprefix(f"Task: {TASK}\n\n"): copy for copy, verdict in checkout.archived().items()}
self.assertEqual(sorted(copies), [f"<UTC> Stopped: Ctrl-C\n\nWorker: {OFF_LADDER}\n", f"<UTC> Verdict: finished\n\nEvidence:\n> landed-it\n\nConfig: <config>\nJudge: judge-model\nWorker: {OFF_LADDER}\n", f"<UTC> Verdict: unfinished\n\nEvidence:\n> started\n\nConfig: <config>\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
self.assertIn('"text": "landed-it"', copies[f"<UTC> Verdict: finished\n\nEvidence:\n> landed-it\n\nConfig: <config>\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
self.assertIn('"text": "next chunk"', copies[f"<UTC> Stopped: Ctrl-C\n\nWorker: {OFF_LADDER}\n"])
self.assertEqual(len(list(checkout.archive().glob(f"proj/*/session/{session}.orphaned-1-ab.jsonl"))), 1)
def test_routes_the_worker_by_its_items_l_and_steps_it_up_within_the_chunk(self):
checkout = Checkout(self)
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
checkout.commit_plan(TAKEN_IN_AT_L_4)
run = checkout.run([
worker(say="one"), judge(evidence="one"),
worker(say="two"), judge(evidence="two"),
worker(say="three\nChunk items: 4 (L 2, E + I 3), 9 (L 4, E + I 5)"), judge(evidence="three"),
worker(say="four"), judge(finished=True, evidence="four"),
worker(say="next chunk"),
])
self.assertEqual(run.code, 130)
workers = [call for call in run.calls if call["role"] == "worker"]
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in workers], [
("claude-sonnet-5-5", "high"),
("claude-sonnet-5-5", "high"),
("claude-opus-5-5", "medium"),
("claude-fable-5-2", "low"),
("claude-sonnet-5-5", "high"),
])
self.assertEqual({(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "judge"}, {("judge-model", None)})
self.assertIn("loop-claude runs you on claude-sonnet-5-5 at high effort, routed by item 4's L of 2.", argument(workers[0], "-p"))
self.assertIn("on one line, as in `Chunk items: ", argument(workers[0], "-p"))
outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values())
self.assertIn('<UTC> Verdict: unfinished\n\nEvidence:\n> three\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-opus-5-5", "effort": "medium", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n\n<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n', outcomes)
self.assertIn('<UTC> Verdict: finished\n\nEvidence:\n> four\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n', outcomes)
def test_routes_on_and_states_the_same_item_from_the_fetched_default_branch_whatever_the_checkout_holds(self):
checkout = Checkout(self)
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
origin = checkout.temp / "origin.git"
checkout.git("init", "--quiet", "--bare", "--initial-branch=main", str(origin))
checkout.git("symbolic-ref", "HEAD", "refs/heads/main")
checkout.commit_plan(TODO)
checkout.git("remote", "add", "origin", str(origin))
checkout.git("push", "--quiet", "--set-upstream", "origin", "main")
# Item 4 ships elsewhere, and item 6 is next; the checkout is behind origin, and holds a change of its own.
elsewhere = checkout.temp / "elsewhere"
checkout.git("clone", "--quiet", str(origin), str(elsewhere))
(elsewhere / "todo.md").write_text(TODO.replace("| 4 | 0.1.0 |", "| 6 | 0.1.0 |"))
checkout.git("-C", str(elsewhere), "commit", "--quiet", "--all", "-m", "item 4 shipped, plan item 6")
checkout.git("-C", str(elsewhere), "push", "--quiet", "origin", "main")
(checkout.root / "todo.md").write_text(TODO + "\nA change of the checkout's own.\n")
(checkout.root / "sub").mkdir()
run = checkout.run([worker()], cwd=checkout.root / "sub")
self.assertEqual(run.code, 130)
prompt = argument(run.calls[0], "-p")
self.assertIn("routed by item 6's L of 4", prompt)
self.assertIn("item 6, L 4, E + I 5", prompt)
self.assertTrue((checkout.home / "code" / "proj-item-6").is_dir())
def test_lists_itself_in_loop_inbox_with_its_routed_item_then_those_the_worker_names_on_its_chunk_items_line_until_the_chunk_finishes(self):
checkout = Checkout(self)
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
checkout.commit_plan(TODO)
run = checkout.run([worker(say="working\nChunk items: 7, 4"), judge(), worker(), judge(finished=True), worker()])
self.assertEqual(run.code, 130)
routed, taken_in = {"id": 4, "tagline": "Do it."}, {"id": 7, "tagline": ""}
self.assertEqual([call["running"] for call in run.calls], [[[routed]], [[routed]], [[routed, taken_in]], [[routed, taken_in]], [[routed]], [[routed]]])
self.assertEqual(list((checkout.home / ".local" / "state" / "loop-claude").glob("*.running")), [])
def test_steps_the_worker_up_a_rung_when_the_judge_finds_in_the_transcript_the_escalation_trigger_it_cites(self):
checkout = Checkout(self)
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
checkout.commit_plan(TODO)
run = checkout.run([
worker(say="Stability round 2: a 🔴. Trigger 3 fired; I ask for a higher rung."), judge(evidence="I ask for a higher rung.", escalation="Stability round 2: a 🔴."),
worker(say="Trigger 2 fired; I ask for a higher rung."), judge(evidence="I ask for a higher rung.", escalation="Trigger 2 fired; … a higher rung."),
worker(say="merged"), judge(finished=True, evidence="merged"),
worker(say="next chunk"),
])
self.assertEqual(run.code, 130)
workers = [call for call in run.calls if call["role"] == "worker"]
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in workers], [
("claude-fable-5-2", "low"),
("claude-fable-5-2", "medium"),
("claude-fable-5-2", "medium"),
("claude-fable-5-2", "low"),
])
outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values())
self.assertIn('<UTC> Verdict: unfinished\n\nEvidence:\n> I ask for a higher rung.\n\nEscalation:\n> Stability round 2: a 🔴.\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 4, "escape_and_impact": 5}\n\n<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', outcomes)
self.assertIn('<UTC> Verdict: unfinished\n\nEvidence:\n> I ask for a higher rung.\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', outcomes)
def test_steps_the_worker_up_once_as_the_judge_interrupts_when_the_run_also_cites_an_escalation_trigger(self):
checkout = Checkout(self)
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
checkout.commit_plan(TODO)
cited = "Stability round 2: a 🔴."
run = checkout.run([
worker(say=f"{cited} Trigger 3 fired; I ask for a higher rung. Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?", escalation=cited),
interrupt(type="B"), judge(evidence="B", escalation="Trigger 3 fired; I ask for a higher rung."),
worker(say="carrying on"),
], terminal_input="/talk\n", before_typing=ANSWERING)
self.assertEqual(run.code, 130)
self.assertEqual(run.roles(), ["worker", "judge", "interrupt", "judge", "worker", "judge"])
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] != "judge"], [("claude-fable-5-2", "low"), ("claude-fable-5-2", "medium"), ("claude-fable-5-2", "medium")])
outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values())
self.assertIn('<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', "".join(outcomes))
self.assertNotIn("at max effort", "".join(outcomes))
self.assertIn('Verdict: interrupt\n\nA or B?\n\nEvidence:\n> Shall I use A or B?\n\nEscalation:\n> Stability round 2: a 🔴.\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 4, "escape_and_impact": 5}\n\n<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', "".join(outcomes))
def test_runs_a_cheap_items_worker_below_the_projects_rung_with_its_reviewers_on_it_and_its_judge_on_the_models_highest(self):
checkout = Checkout(self)
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "claude-opus-5-5"
checkout.commit_plan(CHEAP)
run = checkout.run([worker(say="one"), judge(finished=True, evidence="one"), worker(say="next chunk")])
self.assertEqual(run.code, 130)
first = next(call for call in run.calls if call["role"] == "worker")
self.assertEqual((argument(first, "--model"), argument(first, "--effort")), ("claude-sonnet-5-5", "high"))
ruling = next(call for call in run.calls if call["role"] == "judge")
self.assertEqual((argument(ruling, "--model"), argument(ruling, "--effort")), ("claude-opus-5-5", "xhigh"))
self.assertIn('Judge: claude-opus-5-5 at xhigh effort\nWorker: {"model": "claude-sonnet-5-5", "effort": "high", ', "".join(checkout.archived().values()))
self.assertIn("Launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool set to model opus, effort medium, the project's rung.", argument(first, "-p"))
def test_routes_the_reviewers_on_the_highest_e_plus_i_the_worker_takes_in_and_keeps_the_judge_on_the_models_highest(self):
checkout = Checkout(self)
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "claude-opus-5-5"
checkout.commit_plan(TAKEN_IN_AT_E_PLUS_I_6)
run = checkout.run([
worker(say="one\nChunk items: 4, 9"), judge(evidence="one"),
worker(say="two"), judge(finished=True, evidence="two"),
worker(say="next chunk"),
])
self.assertEqual(run.code, 130)
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "judge"], [("claude-opus-5-5", "xhigh")] * 3)
workers = [call for call in run.calls if call["role"] == "worker"]
self.assertEqual({argument(call, "--effort") for call in workers}, {"high"})
self.assertIn("Launch each reviewer with the Agent tool set to model fable, effort medium, the rung picked by an item of E + I 6 taken into the chunk.", argument(workers[1], "-p"))
outcomes = "".join(checkout.archived().values())
self.assertIn('<UTC> Stepped up\n\nWorker: {"model": "claude-sonnet-5-5", "effort": "high", "item": 4, "likelihood": 2, "escape_and_impact": 3, "escape_and_impact_taken_in": 6}\n', outcomes)
self.assertIn('Judge: claude-opus-5-5 at xhigh effort\nWorker: {"model": "claude-sonnet-5-5", "effort": "high", "item": 4, "likelihood": 2, "escape_and_impact": 3, "escape_and_impact_taken_in": 6}\n', outcomes)
def test_a_rerun_resumes_an_open_chunk_on_the_rung_it_last_ran(self):
checkout = Checkout(self)
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
checkout.commit_plan(TODO)
run = checkout.run([worker(), judge(), worker(), judge()])
self.assertEqual(run.code, 130)
self.assertEqual((argument(run.calls[-1], "--model"), argument(run.calls[-1], "--effort")), ("claude-fable-5-2", "medium"))
checkout.git("rm", "--quiet", "todo.md")
checkout.git("commit", "--quiet", "-m", "drop the plan")
run = checkout.run([worker()])
self.assertIsNotNone(argument(run.calls[0], "--resume"))
self.assertEqual((argument(run.calls[0], "--model"), argument(run.calls[0], "--effort")), ("claude-fable-5-2", "medium"))
def test_exits_1_when_the_transcript_cannot_be_archived_and_resumes_on_the_next_run(self):
checkout = Checkout(self)
checkout.archive().parent.mkdir(parents=True)
checkout.archive().write_text("in the way")
run = checkout.run([worker(say="landed-it"), judge(finished=True, evidence="landed-it")])
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge"]))
self.assertRegex(run.stderr.strip().splitlines()[-1], r"^could not archive the transcript: .*/loop-claude-transcripts/proj: Not a directory$")
self.assertEqual(run.stderr.count("could not archive the transcript"), 1)
session = argument(run.calls[0], "--session-id")
checkout.archive().unlink()
run = checkout.run([worker(), judge(finished=True)])
self.assertEqual(argument(run.calls[0], "--resume"), session)
self.assertEqual(len(checkout.archived()), 1)
def test_stops_after_three_failures_running_retrying_a_minute_apart(self):
checkout = Checkout(self)
run = checkout.run([worker(exit=1), worker(result=False), worker(signal=9)])
self.assertEqual(run.code, 1)
self.assertEqual(run.roles(), ["worker"] * 3)
self.assertEqual(run.sleeps, [60, 60])
self.assertIn("retrying the worker in a minute: claude exited 1", run.stderr)
self.assertIn("claude exited 0 without a result event", run.stderr)
self.assertEqual(run.stderr.strip().splitlines()[-1], "the worker failed three times running: claude killed by signal 9")
self.assertNotIn("=== cycle 2,", run.stdout)
self.assertEqual(argument(run.calls[1], "--resume"), argument(run.calls[0], "--session-id"))
run = checkout.run([worker(exit=1), worker(exit=1), worker(), judge(), worker(exit=1), worker(exit=1)])
self.assertEqual(run.code, 130)
self.assertEqual(run.sleeps, [60, 60, 60, 60])
run = checkout.run([worker(), judge(exit=1, stderr="judge down"), judge(stdout=""), judge(verdict={"interrupt": False})])
self.assertEqual(run.code, 1)
self.assertEqual(run.roles(), ["worker", "judge", "judge", "judge"])
self.assertEqual(run.sleeps, [60, 60])
self.assertIn("retrying the judge in a minute: judge down", run.stderr)
self.assertIn("retrying the judge in a minute: the judge printed no json reply; it printed: nothing\n", run.stderr)
self.assertEqual(run.stderr.strip().splitlines()[-1], 'the judge failed three times running: the judge\'s reply has no answer of type string')
def test_waits_out_a_usage_limit_which_ends_a_run_of_failures(self):
checkout = Checkout(self)
checkout.env["TZ"] = "UTC"
resets_at = int(time.time()) + 3600
rejected = json.dumps({"type": "rate_limit_event", "rate_limit_info": {"status": "rejected", "resetsAt": resets_at}})
limited = json.dumps({"type": "result", "api_error_status": 429, "is_error": True, "result": "You've hit your session limit"})
run = checkout.run([worker(exit=1), worker(exit=1), worker(stream=f"{rejected}\n{limited}\n", result=False, exit=1), worker(exit=1), worker(), judge(stdout=limited, exit=1), judge()])
self.assertEqual(run.code, 130)
self.assertEqual(run.roles(), ["worker"] * 5 + ["judge"] * 2 + ["worker"])
self.assertEqual([round(seconds, -1) for seconds in run.sleeps], [60, 60, 3720, 60, 600])
self.assertIn(f"the worker hit a usage limit (You've hit your session limit); retrying at {datetime.datetime.fromtimestamp(resets_at + 120, datetime.timezone.utc):%Y-%m-%d %H:%M:%S}\n", run.stderr)
self.assertIn("the judge hit a usage limit (You've hit your session limit); retrying at ", run.stderr)
def test_retries_the_judge_alone_when_its_verdict_quotes_nothing_in_the_transcript(self):
checkout = Checkout(self)
run = checkout.run([
worker(say="Shall I use A or B?"),
judge(interrupt=True, evidence="never said", question="A or B?"),
judge(finished=True, evidence=""),
judge(evidence="A or B"),
worker(),
])
self.assertEqual(run.code, 130)
self.assertEqual(run.roles(), ["worker", "judge", "judge", "judge", "worker", "judge"])
self.assertEqual(run.sleeps, [60, 60])
self.assertIn("the judge quoted 'never said', which is not in the transcript, for the question: A or B?", run.stderr)
self.assertIn("retrying the judge in a minute: the judge quoted '', which is not in the transcript\n", run.stderr)
def test_retries_a_judge_claude_cannot_start(self):
checkout = Checkout(self)
run = checkout.run([worker(vanish=True)])
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
self.assertEqual(run.sleeps, [60, 60])
self.assertEqual(run.stderr.strip().splitlines()[-1], "the judge failed three times running: could not start claude: No such file or directory")
def test_hands_the_judge_the_tail_of_a_long_transcript(self):
checkout = Checkout(self)
run = checkout.run([worker(say="early " * 80000 + "LATE")])
self.assertIn("LATE", run.calls[1]["stdin"])
self.assertEqual(len(run.calls[1]["stdin"]), 400000)
self.assertLess(run.calls[1]["stdin"].count("early"), 80000)
self.assertTrue(run.calls[1]["stdin"].rstrip().endswith("CI not yet reported, an answer not yet recorded."))
def test_skips_a_malformed_transcript_or_stream_line(self):
checkout = Checkout(self)
raw = "\xff not UTF-8\nnot JSON\n[1]\n" + json.dumps({"type": "assistant", "message": "not an object"}) + "\n" + json.dumps({"type": "user", "message": None}) + "\n"
run = checkout.run([worker(raw=raw, say="past the noise", stream="\xff not UTF-8\n"), judge(evidence="past the noise")])
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge", "worker"]))
self.assertIn("past the noise", run.stdout)
self.assertIn("past the noise", run.calls[1]["stdin"])
@unittest.skipIf(os.geteuid() == 0, "root reads an unreadable file")
def test_exits_1_when_the_transcript_cannot_be_read(self):
checkout = Checkout(self)
run = checkout.run([worker(unreadable=True)])
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
self.assertRegex(run.stderr.strip().splitlines()[-2], r"^could not archive the transcript: .*\.jsonl: Permission denied$")
self.assertRegex(run.stderr.strip().splitlines()[-1], r"^could not read the transcript .*\.jsonl: Permission denied$")
def test_retries_a_worker_claude_cannot_start(self):
checkout = Checkout(self)
(checkout.temp / "fake-bin" / "claude").unlink()
run = checkout.run([])
self.assertEqual((run.code, run.calls), (1, []))
self.assertEqual(run.sleeps, [60, 60])
self.assertEqual(run.stderr.strip().splitlines()[-1], "the worker failed three times running: could not start claude: No such file or directory")
def test_exits_1_when_claude_leaves_no_transcript(self):
checkout = Checkout(self)
run = checkout.run([worker(transcript=False)])
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
self.assertIn("claude left no transcript for session", run.stderr)
def test_exits_2_after_six_unfinished_verdicts_running_and_a_rerun_gets_six_more(self):
checkout = Checkout(self)
six = [step for _ in range(6) for step in (worker(), judge())]
run = checkout.run(six + [worker()])
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge"] * 6))
self.assertIn("the judge ruled it unfinished 6 cycles running", run.stderr.strip().splitlines()[-1])
self.assertIn("rerun loop-claude to give it 6 more cycles", run.stderr.strip().splitlines()[-1])
stops = [outcome for outcome in checkout.archived().values() if "Stopped: " in outcome]
self.assertEqual(len(stops), 1)
self.assertIn(f"\n\nWorker: {OFF_LADDER}\n", stops[0])
run = checkout.run(six[:-2] + [worker()])
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge"] * 6))
def test_archives_the_stop_on_the_route_the_sixth_verdict_stepped_to_and_a_rerun_resumes_there(self):
checkout = Checkout(self)
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
checkout.commit_plan(CHEAP)
run = checkout.run([step for _ in range(6) for step in (worker(), judge())] + [worker()])
self.assertEqual(run.code, 2)
stepped = '<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n\n<UTC> Stopped: '
self.assertEqual(len([outcome for outcome in checkout.archived().values() if stepped in outcome]), 1)
run = checkout.run([worker()])
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "worker"], [("claude-fable-5-2", "medium")])
def test_five_unfinished_verdicts_and_a_finished_one_do_not_add_up_to_six(self):
checkout = Checkout(self)
five = [step for _ in range(5) for step in (worker(), judge())]
run = checkout.run(five + [worker(), judge(finished=True)] + five + [worker()])
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge"] * 12))
class Interrupt(unittest.TestCase):
def test_exits_2_off_a_terminal_and_resumes_on_the_next_run(self):
checkout = Checkout(self)
checkout.env["LOOP_CLAUDE_TRANSCRIPTS"] = str(checkout.temp / "deep" / "archive")
steps = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
run = checkout.run(steps)
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge"]))
self.assertEqual(run.stderr.strip().splitlines()[-2:], ["A or B?", "the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal"])
[(copy, verdict)] = checkout.archived().items()
self.assertEqual(copy, (checkout.transcripts()[0]).read_text())
self.assertEqual(verdict, f"Task: {TASK}\n\n<UTC> Verdict: interrupt\n\nA or B?\n\nEvidence:\n> Shall I use A or B?\n\nConfig: <config>\nJudge: judge-model\nWorker: {OFF_LADDER}\n\n<UTC> Stopped: the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal\n\nWorker: {OFF_LADDER}\n")
self.assertEqual(stat.S_IMODE((checkout.temp / "deep").stat().st_mode), 0o700)
self.assertIn(f"transcript: {checkout.archive() / 'proj'}/", run.stderr)
self.assertRegex(run.raw_stderr, rf"(?m)^{STAMP} transcript: ")
session = argument(run.calls[0], "--session-id")
run = checkout.run(steps, terminal_input="", stdout_on_terminal=False)
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge"]))
self.assertEqual(argument(run.calls[0], "--resume"), session)
self.assertEqual(len(checkout.archived()), 2)
self.assertEqual(run.stderr.strip().splitlines()[-1], "the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal")
run = checkout.run([worker(transcript=False), steps[1]])
self.assertEqual(run.code, 2)
self.assertEqual(len(checkout.archived()), 2)
def test_resumes_the_worker_headless_with_the_answer_typed_at_its_terminal_not_with_keys_typed_before_it(self):
checkout = Checkout(self)
asked = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
run = checkout.run([*asked, worker(say="Using B."), judge(evidence="Using B.")], terminal_input="B\n", before_typing=ANSWERING, typed_early="stray\n")
self.assertEqual(run.code, 130)
self.assertEqual(run.roles(), ["worker", "judge", "worker", "judge", "worker"])
self.assertIn("A or B?\nAnswer (/talk opens claude, Enter stops the loop): ", run.stderr)
self.assertEqual(argument(run.calls[2], "--resume"), argument(run.calls[0], "--session-id"))
self.assertIn("[human] B\n\nUsing B.", run.calls[3]["stdin"])
self.assertNotIn("stray", run.calls[3]["stdin"])
def test_resumes_the_worker_headless_with_the_answer_typed_in_loop_inbox(self):
checkout = Checkout(self)
folder = checkout.home / ".local" / "state" / "loop-claude"
def answered_in_inbox():
[ask] = waiting(folder)
send(folder, ask, Typed("B"))
asked = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
run = checkout.run([*asked, worker(say="Using B."), judge(evidence="Using B.")], terminal_input="", before_typing=("Answer (", answered_in_inbox))
self.assertEqual(run.code, 130)
self.assertEqual(run.roles(), ["worker", "judge", "worker", "judge", "worker"])
self.assertIn("Answer (/talk opens claude, Enter stops the loop): \nAnswered in loop-inbox: B\n", run.stderr)
self.assertIn("[human] B\n\nUsing B.", run.calls[3]["stdin"])
self.assertEqual([path for path in folder.iterdir() if path.suffix != ".lock"], [])
def test_a_second_loop_on_the_session_runs_unlisted_and_keeps_its_question_out_of_the_first_loops_folder(self):
checkout = Checkout(self)
folder = checkout.home / ".local" / "state" / "loop-claude"
asked = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
session = argument(checkout.run(asked).calls[0], "--session-id")
# The first loop on the session, still running.
self.enterContext(session_lock.held(folder, session))
asks = []
run = checkout.run(asked, terminal_input="\n", before_typing=("Answer (", lambda: asks.extend(folder.glob("*.json"))))
self.assertEqual(run.code, 2)
self.assertIn(f"loop-inbox cannot list this loop or its questions: another process holds {session_lock.lock_file(folder, session)}", run.stderr)
self.assertEqual((asks, run.calls[0]["running"]), ([], []))
self.assertFalse((folder / "unlisted").exists() and any((folder / "unlisted").iterdir()))
def test_hands_the_terminal_over_and_exits_2_when_nothing_is_typed(self):
checkout = Checkout(self)
checkout.git("commit", "--allow-empty", "--quiet", "-m", "start")
claude_json = json.dumps({"projects": {str(checkout.root): {"kept": True}}})
checkout.claude_json().write_text(claude_json)
asked = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
(checkout.root / "sub").mkdir()
run = checkout.run([*asked, interrupt(type="B"), judge(evidence="B"), worker()], cwd=checkout.root / "sub", terminal_input="/talk\n", before_typing=ANSWERING)
self.assertEqual(run.code, 130)
self.assertEqual(run.roles(), ["worker", "judge", "interrupt", "judge", "worker", "judge"])
handed = run.calls[2]["argv"]
self.assertEqual(run.calls[2]["cwd"], str(checkout.root))
self.assertEqual(argument(run.calls[2], "--resume"), argument(run.calls[0], "--session-id"))
self.assertEqual(argument(run.calls[2], "--model"), "worker-model")
self.assertNotIn("-p", handed)
self.assertIn("[human] B", run.calls[3]["stdin"])
self.assertEqual(run.calls[2]["sandboxed"], "1")
self.assertEqual(checkout.claude_json().read_text(), claude_json)
for transcript in checkout.transcripts():
transcript.unlink()
run = checkout.run([*asked, interrupt()], terminal_input="/talk\n", before_typing=ANSWERING)
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge", "interrupt"]))
self.assertTrue(run.stderr.strip().endswith("the human is needed to answer the question above, but the interrupt ended without anything typed"))
run = checkout.run([worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?", vanish=True)], terminal_input="/talk\n", before_typing=ANSWERING)
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge"]))
self.assertTrue(run.stderr.strip().endswith("could not start claude for the interrupt: No such file or directory"))
def test_leaves_ctrl_c_to_the_claude_open_for_talk(self):
checkout = Checkout(self)
asked = [worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?")]
run = checkout.run([*asked, interrupt(type="B", ctrl_c=True), judge(evidence="B")], terminal_input="/talk\n", before_typing=ANSWERING)
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge", "interrupt", "judge", "worker"]))
self.assertFalse(run.calls[2]["ctrl_c_ignored"])
self.assertIn("[human] B", run.calls[3]["stdin"])
@unittest.skipIf(os.geteuid() == 0, "root reads an unreadable file")
def test_exits_1_when_the_transcript_cannot_be_read_after_the_interrupt(self):
checkout = Checkout(self)
run = checkout.run([worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?"), interrupt(type="B", unreadable=True)], terminal_input="/talk\n", before_typing=ANSWERING)
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge", "interrupt"]))
self.assertRegex(run.stderr.strip().splitlines()[-1], r"^could not tell whether you typed anything: could not read the transcript .*\.jsonl: Permission denied$")
class Models(unittest.TestCase):
def catalog(self, checkout):
folder = checkout.home / ".claude" / "cache" / "model-catalog"
folder.mkdir(parents=True)
(folder / "old.json").write_text(json.dumps({"catalog": {"config": {"models": [{"id": "old-model"}]}}, "fetchedAt": 1}))
(folder / "new.json").write_text(json.dumps({"catalog": {"config": {"models": [{"id": "model-a", "name": "A"}, {"id": "model-b", "name": "B"}]}}, "fetchedAt": 2}))
def test_asks_a_project_without_a_model_on_the_terminal_and_saves_the_answer(self):
checkout = Checkout(self)
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
self.catalog(checkout)
run = checkout.run([worker()])
self.assertEqual((run.code, run.calls), (2, []))
self.assertEqual(run.stderr.strip(), f'the human is needed to pick proj\'s model, or merge {{"proj": "<model>"}} into {checkout.models_file()}, but stdin or stdout is not a terminal; rerun loop-claude on a terminal')
run = checkout.run([worker()], terminal_input="\x04")
self.assertEqual((run.code, run.calls), (2, []))
self.assertEqual(run.stderr.strip().splitlines()[-1], "the human is needed to pick proj's model, but the input ended")
checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "judge-model"
run = checkout.run([worker()])
self.assertEqual((run.code, run.calls), (2, []))
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"]
run = checkout.run([worker(), judge()], terminal_input="\n9\n2\n")
self.assertEqual(run.code, 130)
self.assertIn("proj runs on model-b", run.stderr)
self.assertIn("1. A (model-a)", run.stderr)
self.assertNotIn("old-model", run.stderr)
self.assertIn("pick 1 to 2", run.stderr)
self.assertEqual({argument(call, "--model") for call in run.calls}, {"model-b"})
self.assertEqual(json.loads(checkout.models_file().read_text()), {"proj": "model-b"})
self.assertEqual(stat.S_IMODE(checkout.models_file().stat().st_mode), 0o600)
checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "judge-model"
run = checkout.run([worker(), judge()])
self.assertEqual([argument(call, "--model") for call in run.calls], ["model-b", "judge-model", "model-b"])
def test_keeps_the_models_file_mode_and_other_projects_when_saving(self):
checkout = Checkout(self)
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
checkout.models_file().parent.mkdir(parents=True)
checkout.models_file().write_text(json.dumps({"other": "model-x"}))
checkout.models_file().chmod(0o640)
def saved_elsewhere():
checkout.models_file().write_text(json.dumps({"late": "model-y", "other": "model-x"}))
run = checkout.run([worker()], terminal_input="custom-model\n", before_typing=("no model catalog cached", saved_elsewhere))
self.assertEqual(run.code, 130)
self.assertIn("no model catalog cached", run.stderr)
self.assertIn("A name claude --model takes: ", run.stderr)
self.assertEqual(json.loads(checkout.models_file().read_text()), {"late": "model-y", "other": "model-x", "proj": "custom-model"})
self.assertEqual(stat.S_IMODE(checkout.models_file().stat().st_mode), 0o640)
for broken in ("{", "[]", '{"proj": 1}'):
with self.subTest(broken=broken):
checkout.models_file().write_text(broken)
run = checkout.run([worker()])
self.assertEqual((run.code, run.calls), (1, []))
self.assertTrue(run.stderr.startswith(f"could not read {checkout.models_file()}: "))
self.assertEqual(checkout.models_file().read_text(), broken)
def test_offers_no_catalog_whose_models_are_not_a_list(self):
checkout = Checkout(self)
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
self.catalog(checkout)
catalog = checkout.home / ".claude" / "cache" / "model-catalog" / "new.json"
for shape in ({"catalog": {"config": {"models": []}}}, {"catalog": None}, {"catalog": {"config": []}}, {"catalog": {"config": {"models": "x"}}}):
with self.subTest(shape=shape):
catalog.write_text(json.dumps({**shape, "fetchedAt": 3}))
checkout.models_file().unlink(missing_ok=True)
run = checkout.run([worker()], terminal_input="1\n")
self.assertIn("no model catalog cached", run.stderr)
self.assertEqual(argument(run.calls[0], "--model"), "1")
@unittest.skipIf(os.geteuid() == 0, "root writes into a read-only directory")
def test_exits_1_when_the_models_file_cannot_be_written(self):
checkout = Checkout(self)
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
checkout.models_file().parent.mkdir(parents=True, mode=0o500)
self.addCleanup(checkout.models_file().parent.chmod, 0o700)
run = checkout.run([worker()], terminal_input="custom-model\n")
self.assertEqual((run.code, run.calls), (1, []))
self.assertIn(f"could not save the model in {checkout.models_file()}: Permission denied", run.stderr)
class Reload(unittest.TestCase):
def test_an_edit_takes_effect_at_the_next_cycle_boundary_once_it_answers_help(self):
checkout = Checkout(self)
copy = checkout.temp / "bin"
shutil.copytree(BIN, copy, ignore=shutil.ignore_patterns("__pycache__", "test_*"))
checkout.script = copy / "loop-claude"
checkout.env["FAKE_CLAUDE_EDIT"] = str(copy)
run = checkout.run([
worker(append=["loop-claude", "\ndef\n"]), judge(),
worker(replace=["loop-claude", "\ndef\n", "\n# edited\n"]), judge(),
worker(append=["loop_claude/nested/edited.py", "# edited\n"]), judge(),
worker(append=["records/file_writes.py", "# edited\n"]), judge(),
worker(),
])
self.assertEqual(run.code, 130)
self.assertEqual(run.roles(), ["worker", "judge"] * 5)
self.assertIn("keeping the running script", run.stderr)
reloads = [json.loads(argv[1]) for argv in run.starts if argv[:1] == ["--reloaded"]]
self.assertEqual(reloads, [{"cycle_number": cycle, "root": str(checkout.root), "task": TASK, "unfinished_evidence": "working", "route": OFF_LADDER_ROUTE, "unfinished_run": cycle, "items_taken_in": []} for cycle in (2, 3, 4)])
self.assertIn("=== cycle 5,", run.stdout)
outside = checkout.temp / "outside"
outside.mkdir()
state = {"cycle_number": 3, "root": str(checkout.root), "task": TASK, "unfinished_evidence": "", "route": None}
run = checkout.run([worker()], argv=["--reloaded", json.dumps(state)], cwd=outside)
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge"]))
self.assertEqual(run.calls[0]["cwd"], str(checkout.root))
self.assertIn("=== cycle 4,", run.stdout)
run = checkout.run([], argv=["--reloaded", "{}"])
self.assertEqual((run.code, run.calls), (1, []))
self.assertIn("rerun loop-claude <task>", run.stderr)
if __name__ == "__main__":
unittest.main()