Files
claude-config/bin/test_loop_claude_black_box.py
T

698 lines
34 KiB
Python

import dataclasses
import datetime
import hashlib
import json
import os
import pathlib
import pty
import select
import shutil
import stat
import subprocess
import sys
import tempfile
import time
import unittest
BIN = pathlib.Path(__file__).resolve().parent
TASK = "Do the next chunk."
# Plays claude from FAKE_CLAUDE_SCENARIO, one step per call, logging each call; a step for another role,
# or none left, is Ctrl-C to loop-claude, which otherwise loops forever.
FAKE_CLAUDE = """#!{python}
import json, os, pathlib, signal, sys, time
argv = sys.argv[1:]
role = "judge" if "--json-schema" in argv else "worker" if "-p" in argv else "interrupt"
scenario = pathlib.Path(os.environ["FAKE_CLAUDE_SCENARIO"])
steps = json.loads(scenario.read_text())
call = {{"argv": argv, "cwd": os.getcwd(), "role": role}}
if role == "judge":
call["stdin"] = sys.stdin.read()
with open(os.environ["FAKE_CLAUDE_LOG"], "a") as log:
log.write(json.dumps(call) + "\\n")
if not steps or steps[0]["role"] != role:
parent = os.getppid()
os.kill(parent, signal.SIGINT)
while os.getppid() == parent:
time.sleep(0.05)
sys.exit(0)
step = steps.pop(0)
scenario.write_text(json.dumps(steps))
def session():
flag = "--session-id" if "--session-id" in argv else "--resume"
return argv[argv.index(flag) + 1]
def transcript():
return pathlib.Path.home() / ".claude" / "projects" / "-fake" / f"{{session()}}.jsonl"
def append(*entries):
transcript().parent.mkdir(parents=True, exist_ok=True)
with open(transcript(), "a") as handle:
handle.writelines(json.dumps(entry) + "\\n" for entry in entries)
if role == "worker":
if "--session-id" in argv and step.get("transcript", True):
append({{"type": "custom-title"}}, {{"type": "queue-operation", "timestamp": str(time.time_ns())}})
said = {{"type": "assistant", "message": {{"content": [{{"type": "text", "text": step.get("say", "working")}}]}}}}
if step.get("transcript", True):
append({{"type": "user", "promptSource": "sdk", "message": {{"content": argv[argv.index("-p") + 1]}}}})
with open(transcript(), "ab") as handle:
handle.write(step.get("raw", "").encode("latin-1"))
append(said)
if step.get("siblings"):
for sibling in (f"{{session()}}/subagents", f"{{session()}}.superseded-1/tool-results"):
(transcript().parent / sibling).mkdir(parents=True, exist_ok=True)
(transcript().parent / f"{{session()}}.orphaned-1-ab.jsonl").write_text("{{}}\\n")
if "append" in step:
edited = pathlib.Path(os.environ["FAKE_CLAUDE_EDIT"], step["append"][0])
edited.parent.mkdir(parents=True, exist_ok=True)
with open(edited, "a") as script:
script.write(step["append"][1])
if "replace" in step:
script = pathlib.Path(os.environ["FAKE_CLAUDE_EDIT"], step["replace"][0])
script.write_text(script.read_text().replace(*step["replace"][1:]))
sys.stdout.flush()
sys.stdout.buffer.write(step.get("stream", "").encode("latin-1"))
print(json.dumps(said))
if step.get("result", True):
print(json.dumps({{"type": "result"}}))
if "signal" in step:
os.kill(os.getpid(), step["signal"])
elif role == "judge":
print(step["stdout"] if "stdout" in step else json.dumps({{"structured_output": step["verdict"]}}))
print(step.get("stderr", ""), file=sys.stderr)
elif "type" in step:
append({{"type": "user", "message": {{"content": step["type"]}}}})
if step.get("unreadable"):
transcript().chmod(0)
if step.get("vanish"):
os.remove(__file__)
sys.exit(step.get("exit", 0))
"""
# Records every start of loop-claude and turns its retry waits into log lines.
SITECUSTOMIZE = """import json, os, sys, time
if os.path.basename(sys.argv[0]) == "loop-claude":
def record(event):
with open(os.environ["LOOP_TEST_LOG"], "a") as log:
log.write(json.dumps(event) + "\\n")
record({"argv": sys.argv[1:]})
pause = time.sleep
time.sleep = lambda seconds: record({"sleep": seconds}) if seconds >= 1 else pause(seconds)
"""
PREMISES = "# Project\n\n## Goals\n\n1. Work.\n\n## Audience\n\nMe.\n"
def worker(**step):
return {"role": "worker", **step}
def judge(interrupt=False, finished=False, evidence="working", question="", **step):
verdict = {"answer": "", "evidence": evidence, "finished": finished, "interrupt": interrupt, "question": question}
return {"role": "judge", "verdict": verdict, **step}
def interrupt(**step):
return {"role": "interrupt", **step}
def lines(path):
return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else []
@dataclasses.dataclass
class Run:
calls: list
code: int
sleeps: list
starts: list
stderr: str
stdout: str
def roles(self):
return [call["role"] for call in self.calls]
class Checkout:
"""A temporary HOME with a git checkout at ~/code/proj, where loop-claude runs against the fake claude."""
def __init__(self, test):
self.test = test
temp = tempfile.TemporaryDirectory()
test.addCleanup(temp.cleanup)
self.temp = pathlib.Path(os.path.realpath(temp.name))
self.home = self.temp / "home"
self.root = self.home / "code" / "proj"
self.root.mkdir(parents=True)
(self.root / "README.md").write_text(PREMISES)
self.script = BIN / "loop-claude"
self.env = {name: value for name, value in os.environ.items() if not name.startswith(("GIT_", "LOOP_CLAUDE_", "PYTHON"))}
fake = self.temp / "fake-bin"
fake.mkdir()
(fake / "claude").write_text(FAKE_CLAUDE.format(python=sys.executable))
(fake / "claude").chmod(0o755)
(fake / "git").symlink_to(shutil.which("git"))
site = self.temp / "site"
site.mkdir()
(site / "sitecustomize.py").write_text(SITECUSTOMIZE)
self.env.update({
"FAKE_CLAUDE_LOG": str(self.temp / "calls.jsonl"),
"FAKE_CLAUDE_SCENARIO": str(self.temp / "scenario.json"),
"GIT_CONFIG_GLOBAL": os.devnull,
"GIT_CONFIG_NOSYSTEM": "1",
"HOME": str(self.home),
"LOOP_CLAUDE_JUDGE_MODEL": "judge-model",
"LOOP_CLAUDE_WORKER_MODEL": "worker-model",
"LOOP_TEST_LOG": str(self.temp / "starts.jsonl"),
"PATH": str(fake),
"PYTHONPATH": str(site),
})
self.git("init", "--quiet")
def git(self, *args):
subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@example.com", *args], capture_output=True, check=True, cwd=self.root, env=self.env)
def run(self, steps, argv=(TASK,), cwd=None, terminal_input=None, stdout_on_terminal=True, before_typing=None):
for name in ("FAKE_CLAUDE_LOG", "LOOP_TEST_LOG"):
pathlib.Path(self.env[name]).unlink(missing_ok=True)
pathlib.Path(self.env["FAKE_CLAUDE_SCENARIO"]).write_text(json.dumps(steps))
command = [sys.executable, str(self.script), *argv]
cwd = cwd or self.root
if terminal_input is None:
done = subprocess.run(command, capture_output=True, cwd=cwd, env=self.env, stdin=subprocess.DEVNULL, text=True, timeout=60)
code, stdout, stderr = done.returncode, done.stdout, done.stderr
else:
code, stdout, stderr = self.run_on_terminal(command, cwd, terminal_input, stdout_on_terminal, before_typing)
started = lines(pathlib.Path(self.env["LOOP_TEST_LOG"]))
return Run(
calls=lines(pathlib.Path(self.env["FAKE_CLAUDE_LOG"])),
code=code,
sleeps=[event["sleep"] for event in started if "sleep" in event],
starts=[event["argv"] for event in started if "argv" in event],
stderr=stderr,
stdout=stdout,
)
def run_on_terminal(self, command, cwd, typed, stdout_on_terminal, before_typing):
"""Types the input at once, or once stderr shows before_typing's text and its callback has run."""
controller, terminal = pty.openpty()
with tempfile.TemporaryFile() as stderr, tempfile.TemporaryFile() as stdout:
try:
proc = subprocess.Popen(command, cwd=cwd, env=self.env, stderr=stderr, stdin=terminal, stdout=terminal if stdout_on_terminal else stdout)
finally:
os.close(terminal)
try:
prompt, callback = before_typing or (None, None)
if prompt is None:
os.write(controller, typed.encode())
output, deadline = b"", time.monotonic() + 60
while time.monotonic() < deadline:
if select.select([controller], [], [], 1)[0]:
try:
chunk = os.read(controller, 4096)
except OSError:
break
if not chunk:
break
output += chunk
if prompt is not None and prompt.encode() in os.pread(stderr.fileno(), 1 << 20, 0):
callback()
os.write(controller, typed.encode())
prompt = None
code = proc.wait(timeout=10)
finally:
os.close(controller)
proc.kill()
proc.wait()
stderr.seek(0)
stdout.seek(0)
output += stdout.read()
return code, output.decode(errors="replace").replace("\r\n", "\n"), stderr.read().decode()
def claude_json(self):
return self.home / ".claude.json"
def models_file(self):
return self.home / ".claude" / "loop-claude-models.json"
def transcripts(self):
return sorted((self.home / ".claude" / "projects").glob("*/*.jsonl"))
def archive(self):
return pathlib.Path(self.env.get("LOOP_CLAUDE_TRANSCRIPTS", self.home / ".claude" / "loop-claude-transcripts"))
def archived(self):
"""Each archived transcript's outcomes, after checking the copy is named by its content hash."""
outcomes = {}
for folder in (self.archive(), self.archive() / "proj", *(self.archive() / "proj").iterdir()):
self.test.assertEqual(stat.S_IMODE(folder.stat().st_mode), 0o700, folder)
for copy in sorted((self.archive() / "proj").glob("*/*.jsonl")):
self.test.assertEqual(copy.stem, hashlib.sha256(copy.read_bytes()).hexdigest())
kept = copy.with_suffix(".outcomes.md")
for archived in (copy, kept):
self.test.assertEqual(stat.S_IMODE(archived.stat().st_mode), 0o600)
outcomes[copy.read_text()] = kept.read_text()
return outcomes
def argument(call, flag):
return call["argv"][call["argv"].index(flag) + 1] if flag in call["argv"] else None
class Premises(unittest.TestCase):
def test_refuses_to_start_without_a_task_a_checkout_or_premises(self):
checkout = Checkout(self)
for argv, code, said in (([], 1, "usage: loop-claude <task>"), (["--help"], 0, "usage: loop-claude <task>"), (["-h"], 0, "usage: loop-claude <task>")):
with self.subTest(argv=argv):
run = checkout.run([], argv=argv)
self.assertEqual(run.code, code)
self.assertIn(said, run.stdout if code == 0 else run.stderr)
outside = checkout.temp / "outside"
outside.mkdir()
run = checkout.run([], cwd=outside)
self.assertEqual((run.code, run.calls), (1, []))
self.assertIn("not a git repository", run.stderr)
readmes = {
None: "README.md in {root}: No such file or directory - no premises.",
b"# P\n## Goals\n1. g\xff\n## Audience\nme\n": "README.md in {root} is not UTF-8 on line 3 - no premises.",
b"# P\n## Audience\nme\n": "README.md has no ## Goals heading - no premises.",
b"# P\n## Goals\n1. g\n": "README.md has no ## Audience heading - no premises.",
b"# P\n## Goals\n\n## Audience\nme\n": "README.md has nothing under ## Goals - no premises.",
b"# P\n## Goals\n1. g\n## Audience\nme\n## Audience\nyou\n": "README.md has ## Audience more than once - no premises.",
}
(checkout.root / "sub").mkdir()
for text, said in readmes.items():
with self.subTest(said=said):
readme = checkout.root / "README.md"
readme.unlink(missing_ok=True)
if text is not None:
readme.write_bytes(text)
run = checkout.run([worker()], cwd=checkout.root / "sub")
self.assertEqual((run.code, run.calls), (1, []))
self.assertEqual(run.stderr.strip(), said.format(root=checkout.root))
(checkout.root / "README.md").unlink()
(checkout.root / "README.md").mkdir()
run = checkout.run([worker()])
self.assertEqual((run.code, run.calls), (1, []))
self.assertEqual(run.stderr.strip(), f"README.md in {checkout.root}: Is a directory - no premises.")
(checkout.temp / "fake-bin" / "git").unlink()
run = checkout.run([worker()])
self.assertEqual((run.code, run.calls), (1, []))
self.assertEqual(run.stderr.strip(), "could not start git: No such file or directory")
def test_names_the_session_by_the_checkout_relative_to_code_or_else_to_the_root(self):
checkout = Checkout(self)
for root in (checkout.temp / "elsewhere", checkout.home / "code"):
with self.subTest(root=root):
root.mkdir(exist_ok=True)
(root / "README.md").write_text(PREMISES)
checkout.git("init", "--quiet", str(root))
run = checkout.run([worker()], cwd=root)
self.assertEqual(argument(run.calls[0], "--name"), f"A {str(root).lstrip('/')} loop-claude")
class Cycles(unittest.TestCase):
def test_resumes_the_session_while_unfinished_and_starts_it_fresh_after_finishing(self):
checkout = Checkout(self)
(checkout.root / "sub").mkdir()
(checkout.root / "docs").mkdir()
(checkout.root / "docs" / "decisions.md").write_text("# Decisions\n")
run = checkout.run([
worker(say="started"), judge(evidence="started"),
worker(say="landed-it", siblings=True), judge(finished=True, evidence="landed-it"),
worker(say="next chunk"),
], cwd=checkout.root / "sub")
self.assertEqual(run.code, 130)
self.assertEqual(run.roles(), ["worker", "judge", "worker", "judge", "worker", "judge"])
workers = [call for call in run.calls if call["role"] == "worker"]
session = argument(workers[0], "--session-id")
self.assertEqual(argument(workers[0], "--name"), "A proj loop-claude")
self.assertEqual(argument(workers[1], "--resume"), session)
self.assertIsNone(argument(workers[1], "--session-id"))
self.assertIsNone(argument(workers[1], "--name"))
self.assertEqual(argument(workers[2], "--session-id"), session)
self.assertTrue(argument(workers[0], "-p").startswith(f"{TASK}\n\nContinue any open draft PR"))
self.assertNotIn(TASK, argument(workers[1], "-p"))
self.assertEqual({call["cwd"] for call in run.calls}, {str(checkout.root)})
self.assertIn("1. Work.", run.calls[1]["stdin"])
self.assertIn("From docs/decisions.md:\n# Decisions", run.calls[1]["stdin"])
self.assertEqual({argument(call, "--model") for call in workers}, {"worker-model"})
self.assertEqual({argument(call, "--model") for call in run.calls if call["role"] == "judge"}, {"judge-model"})
self.assertIn("landed-it", run.calls[3]["stdin"])
self.assertNotIn("landed-it", run.calls[5]["stdin"])
self.assertEqual([path.name for path in checkout.transcripts()[0].parent.iterdir()], [checkout.transcripts()[0].name])
self.assertIn("=== cycle 3 ===", run.stdout)
copies = {verdict.removeprefix(f"Task: {TASK}\n\n"): copy for copy, verdict in checkout.archived().items()}
self.assertEqual(sorted(copies), ["Stopped: Ctrl-C\n", "Verdict: finished\n", "Verdict: unfinished\n"])
self.assertIn('"text": "landed-it"', copies["Verdict: finished\n"])
self.assertIn('"text": "next chunk"', copies["Stopped: Ctrl-C\n"])
self.assertEqual(len(list(checkout.archive().glob(f"proj/*/session/{session}.orphaned-1-ab.jsonl"))), 1)
def test_exits_1_when_the_transcript_cannot_be_archived_and_resumes_on_the_next_run(self):
checkout = Checkout(self)
checkout.archive().parent.mkdir(parents=True)
checkout.archive().write_text("in the way")
run = checkout.run([worker(say="landed-it"), judge(finished=True, evidence="landed-it")])
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge"]))
self.assertRegex(run.stderr.strip().splitlines()[-1], r"^could not archive the transcript: .*/loop-claude-transcripts/proj: Not a directory$")
self.assertEqual(run.stderr.count("could not archive the transcript"), 1)
session = argument(run.calls[0], "--session-id")
checkout.archive().unlink()
run = checkout.run([worker(), judge(finished=True)])
self.assertEqual(argument(run.calls[0], "--resume"), session)
self.assertEqual(len(checkout.archived()), 1)
def test_stops_after_three_failures_running_retrying_a_minute_apart(self):
checkout = Checkout(self)
run = checkout.run([worker(exit=1), worker(result=False), worker(signal=9)])
self.assertEqual(run.code, 1)
self.assertEqual(run.roles(), ["worker"] * 3)
self.assertEqual(run.sleeps, [60, 60])
self.assertIn("retrying the worker in a minute: claude exited 1", run.stderr)
self.assertIn("claude exited 0 without a result event", run.stderr)
self.assertEqual(run.stderr.strip().splitlines()[-1], "the worker failed three times running: claude killed by signal 9")
self.assertNotIn("=== cycle 2 ===", run.stdout)
self.assertEqual(argument(run.calls[1], "--resume"), argument(run.calls[0], "--session-id"))
run = checkout.run([worker(exit=1), worker(exit=1), worker(), judge(), worker(exit=1), worker(exit=1)])
self.assertEqual(run.code, 130)
self.assertEqual(run.sleeps, [60, 60, 60, 60])
run = checkout.run([worker(), judge(exit=1, stderr="judge down"), judge(stdout=""), judge(verdict={"interrupt": False})])
self.assertEqual(run.code, 1)
self.assertEqual(run.roles(), ["worker", "judge", "judge", "judge"])
self.assertEqual(run.sleeps, [60, 60])
self.assertIn("retrying the judge in a minute: judge down", run.stderr)
self.assertIn("retrying the judge in a minute: the judge returned no reply; it replied: nothing\n", run.stderr)
self.assertEqual(run.stderr.strip().splitlines()[-1], 'the judge failed three times running: the judge\'s reply has no answer of type string; it replied: {"structured_output": {"interrupt": false}}')
def test_waits_out_a_usage_limit_which_ends_a_run_of_failures(self):
checkout = Checkout(self)
checkout.env["TZ"] = "UTC"
resets_at = int(time.time()) + 3600
rejected = json.dumps({"type": "rate_limit_event", "rate_limit_info": {"status": "rejected", "resetsAt": resets_at}})
limited = json.dumps({"type": "result", "api_error_status": 429, "is_error": True, "result": "You've hit your session limit"})
run = checkout.run([worker(exit=1), worker(exit=1), worker(stream=f"{rejected}\n{limited}\n", result=False, exit=1), worker(exit=1), worker(), judge(stdout=limited, exit=1), judge()])
self.assertEqual(run.code, 130)
self.assertEqual(run.roles(), ["worker"] * 5 + ["judge"] * 2 + ["worker"])
self.assertEqual([round(seconds, -1) for seconds in run.sleeps], [60, 60, 3720, 60, 600])
self.assertIn(f"the worker hit a usage limit (You've hit your session limit); retrying at {datetime.datetime.fromtimestamp(resets_at + 120, datetime.timezone.utc):%Y-%m-%d %H:%M} UTC\n", run.stderr)
self.assertIn("the judge hit a usage limit (You've hit your session limit); retrying at ", run.stderr)
def test_retries_the_judge_alone_when_its_verdict_quotes_nothing_in_the_transcript(self):
checkout = Checkout(self)
run = checkout.run([
worker(say="Shall I use A or B?"),
judge(interrupt=True, evidence="never said", question="A or B?"),
judge(finished=True, evidence=""),
judge(evidence="A or B"),
worker(),
])
self.assertEqual(run.code, 130)
self.assertEqual(run.roles(), ["worker", "judge", "judge", "judge", "worker", "judge"])
self.assertEqual(run.sleeps, [60, 60])
self.assertIn("the judge quoted 'never said', which is not in the transcript, for the question: A or B?", run.stderr)
self.assertIn("retrying the judge in a minute: the judge quoted '', which is not in the transcript\n", run.stderr)
def test_retries_a_judge_claude_cannot_start(self):
checkout = Checkout(self)
run = checkout.run([worker(vanish=True)])
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
self.assertEqual(run.sleeps, [60, 60])
self.assertEqual(run.stderr.strip().splitlines()[-1], "the judge failed three times running: could not start claude: No such file or directory")
def test_hands_the_judge_the_tail_of_a_long_transcript(self):
checkout = Checkout(self)
run = checkout.run([worker(say="early " * 80000 + "LATE")])
self.assertIn("LATE", run.calls[1]["stdin"])
self.assertEqual(len(run.calls[1]["stdin"]), 400000)
self.assertLess(run.calls[1]["stdin"].count("early"), 80000)
self.assertTrue(run.calls[1]["stdin"].rstrip().endswith("an open PR, a review not yet run, CI not yet reported."))
def test_skips_a_malformed_transcript_or_stream_line(self):
checkout = Checkout(self)
raw = "\xff not UTF-8\nnot JSON\n[1]\n" + json.dumps({"type": "assistant", "message": "not an object"}) + "\n" + json.dumps({"type": "user", "message": None}) + "\n"
run = checkout.run([worker(raw=raw, say="past the noise", stream="\xff not UTF-8\n"), judge(evidence="past the noise")])
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge", "worker"]))
self.assertIn("past the noise", run.stdout)
self.assertIn("past the noise", run.calls[1]["stdin"])
@unittest.skipIf(os.geteuid() == 0, "root reads an unreadable file")
def test_exits_1_when_the_transcript_cannot_be_read(self):
checkout = Checkout(self)
run = checkout.run([worker(unreadable=True)])
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
self.assertRegex(run.stderr.strip().splitlines()[-2], r"^could not archive the transcript: .*\.jsonl: Permission denied$")
self.assertRegex(run.stderr.strip().splitlines()[-1], r"^could not read the transcript .*\.jsonl: Permission denied$")
def test_retries_a_worker_claude_cannot_start(self):
checkout = Checkout(self)
(checkout.temp / "fake-bin" / "claude").unlink()
run = checkout.run([])
self.assertEqual((run.code, run.calls), (1, []))
self.assertEqual(run.sleeps, [60, 60])
self.assertEqual(run.stderr.strip().splitlines()[-1], "the worker failed three times running: could not start claude: No such file or directory")
def test_exits_1_when_claude_leaves_no_transcript(self):
checkout = Checkout(self)
run = checkout.run([worker(transcript=False)])
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
self.assertIn("claude left no transcript for session", run.stderr)
class Interrupt(unittest.TestCase):
def test_exits_2_off_a_terminal_and_resumes_on_the_next_run(self):
checkout = Checkout(self)
checkout.env["LOOP_CLAUDE_TRANSCRIPTS"] = str(checkout.temp / "deep" / "archive")
steps = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
run = checkout.run(steps)
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge"]))
self.assertEqual(run.stderr.strip().splitlines()[-2:], ["A or B?", "the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal"])
[(copy, verdict)] = checkout.archived().items()
self.assertEqual(copy, (checkout.transcripts()[0]).read_text())
self.assertEqual(verdict, f"Task: {TASK}\n\nVerdict: interrupt\n\nA or B?\n\nStopped: the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal\n")
self.assertEqual(stat.S_IMODE((checkout.temp / "deep").stat().st_mode), 0o700)
self.assertIn(f"transcript: {checkout.archive() / 'proj'}/", run.stderr)
session = argument(run.calls[0], "--session-id")
run = checkout.run(steps, terminal_input="", stdout_on_terminal=False)
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge"]))
self.assertEqual(argument(run.calls[0], "--resume"), session)
self.assertEqual(len(checkout.archived()), 2)
self.assertEqual(run.stderr.strip().splitlines()[-1], "the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal")
run = checkout.run([worker(transcript=False), steps[1]])
self.assertEqual(run.code, 2)
self.assertEqual(len(checkout.archived()), 2)
def test_hands_the_terminal_over_and_exits_2_when_nothing_is_typed(self):
checkout = Checkout(self)
checkout.git("commit", "--allow-empty", "--quiet", "-m", "start")
checkout.git("worktree", "add", "--quiet", str(checkout.home / "code" / "proj-wt"))
checkout.claude_json().write_text(json.dumps({"other": 1, "projects": {str(checkout.root): {"kept": True}}}))
checkout.claude_json().chmod(0o644)
asked = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
(checkout.root / "sub").mkdir()
run = checkout.run([*asked, interrupt(type="B"), judge(evidence="B"), worker()], cwd=checkout.root / "sub", terminal_input="")
self.assertEqual(run.code, 130)
self.assertEqual(run.roles(), ["worker", "judge", "interrupt", "judge", "worker", "judge"])
handed = run.calls[2]["argv"]
self.assertEqual(run.calls[2]["cwd"], str(checkout.root))
self.assertEqual(argument(run.calls[2], "--resume"), argument(run.calls[0], "--session-id"))
self.assertEqual(argument(run.calls[2], "--model"), "worker-model")
self.assertNotIn("-p", handed)
self.assertIn("[human] B", run.calls[3]["stdin"])
trusted = json.loads(checkout.claude_json().read_text())
self.assertEqual(trusted["other"], 1)
self.assertEqual(trusted["projects"][str(checkout.root)], {"hasTrustDialogAccepted": True, "kept": True})
self.assertEqual(trusted["projects"][str(checkout.home / "code" / "proj-wt")], {"hasTrustDialogAccepted": True})
self.assertEqual(stat.S_IMODE(checkout.claude_json().stat().st_mode), 0o644)
self.assertIn(f"trusted for the interrupt: {checkout.root}, {checkout.home / 'code' / 'proj-wt'}\n", run.stderr)
for transcript in checkout.transcripts():
transcript.unlink()
run = checkout.run([*asked, interrupt()], terminal_input="")
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge", "interrupt"]))
self.assertEqual(run.stderr.strip().splitlines()[-1], "the human is needed to answer the question above, but the interrupt ended with nothing typed")
run = checkout.run([worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?", vanish=True)], terminal_input="")
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge"]))
self.assertEqual(run.stderr.strip().splitlines()[-1], "could not start claude for the interrupt: No such file or directory")
def test_says_why_the_trust_file_cannot_be_read_and_leaves_it_as_it_was(self):
checkout = Checkout(self)
asked = [worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?"), interrupt()]
root = json.dumps(str(checkout.root))
shapes = {
None: "No such file or directory",
"{": "Expecting property name enclosed in double quotes: line 1 column 2 (char 1)",
"[]": 'not an object whose "projects" is an object',
'{"projects": []}': 'not an object whose "projects" is an object',
f'{{"projects": {{{root}: 1}}}}': f"projects[{root}] is not an object",
}
for text, said in shapes.items():
with self.subTest(said=said):
checkout.claude_json().unlink(missing_ok=True)
if text is not None:
checkout.claude_json().write_text(text)
run = checkout.run(asked, terminal_input="")
self.assertEqual(run.code, 2)
self.assertIn(f"could not read {checkout.claude_json()}: {said}\n", run.stderr)
self.assertEqual(checkout.claude_json().read_text() if checkout.claude_json().exists() else None, text)
trusted = json.dumps({"projects": {str(checkout.root): {"hasTrustDialogAccepted": True}}})
checkout.claude_json().write_text(trusted)
run = checkout.run(asked, terminal_input="")
self.assertNotIn(str(checkout.claude_json()), run.stderr)
self.assertNotIn("trusted for the interrupt", run.stderr)
self.assertEqual(checkout.claude_json().read_text(), trusted)
@unittest.skipIf(os.geteuid() == 0, "root writes into a read-only directory")
def test_says_why_the_trust_file_cannot_be_written(self):
checkout = Checkout(self)
checkout.claude_json().write_text("{}")
asked = [worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?"), interrupt()]
checkout.run(asked[:2])
checkout.home.chmod(0o500)
self.addCleanup(checkout.home.chmod, 0o700)
run = checkout.run([worker(), *asked[1:]], terminal_input="")
self.assertEqual(run.code, 2)
self.assertIn(f"could not write {checkout.claude_json()}: Permission denied\n", run.stderr)
self.assertEqual(checkout.claude_json().read_text(), "{}")
@unittest.skipIf(os.geteuid() == 0, "root reads an unreadable file")
def test_exits_1_when_the_transcript_cannot_be_read_after_the_interrupt(self):
checkout = Checkout(self)
run = checkout.run([worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?"), interrupt(type="B", unreadable=True)], terminal_input="")
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge", "interrupt"]))
self.assertRegex(run.stderr.strip().splitlines()[-1], r"^could not tell whether you typed anything: could not read the transcript .*\.jsonl: Permission denied$")
class Models(unittest.TestCase):
def catalog(self, checkout):
folder = checkout.home / ".claude" / "cache" / "model-catalog"
folder.mkdir(parents=True)
(folder / "old.json").write_text(json.dumps({"catalog": {"config": {"models": [{"id": "old-model"}]}}, "fetchedAt": 1}))
(folder / "new.json").write_text(json.dumps({"catalog": {"config": {"models": [{"id": "model-a", "name": "A"}, {"id": "model-b", "name": "B"}]}}, "fetchedAt": 2}))
def test_asks_a_project_without_a_model_on_the_terminal_and_saves_the_answer(self):
checkout = Checkout(self)
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
self.catalog(checkout)
run = checkout.run([worker()])
self.assertEqual((run.code, run.calls), (2, []))
self.assertEqual(run.stderr.strip(), f'the human is needed to pick proj\'s model, or merge {{"proj": "<model>"}} into {checkout.models_file()}, but stdin or stdout is not a terminal; rerun loop-claude on a terminal')
run = checkout.run([worker()], terminal_input="\x04")
self.assertEqual((run.code, run.calls), (2, []))
self.assertEqual(run.stderr.strip().splitlines()[-1], "the human is needed to pick proj's model, but the input ended")
checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "judge-model"
run = checkout.run([worker()])
self.assertEqual((run.code, run.calls), (2, []))
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"]
run = checkout.run([worker(), judge()], terminal_input="\n9\n2\n")
self.assertEqual(run.code, 130)
self.assertIn("proj runs on model-b", run.stderr)
self.assertIn("1. A (model-a)", run.stderr)
self.assertNotIn("old-model", run.stderr)
self.assertIn("pick 1 to 2", run.stderr)
self.assertEqual({argument(call, "--model") for call in run.calls}, {"model-b"})
self.assertEqual(json.loads(checkout.models_file().read_text()), {"proj": "model-b"})
self.assertEqual(stat.S_IMODE(checkout.models_file().stat().st_mode), 0o600)
checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "judge-model"
run = checkout.run([worker(), judge()])
self.assertEqual([argument(call, "--model") for call in run.calls], ["model-b", "judge-model", "model-b"])
def test_keeps_the_models_file_mode_and_other_projects_when_saving(self):
checkout = Checkout(self)
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
checkout.models_file().parent.mkdir(parents=True)
checkout.models_file().write_text(json.dumps({"other": "model-x"}))
checkout.models_file().chmod(0o640)
def saved_elsewhere():
checkout.models_file().write_text(json.dumps({"late": "model-y", "other": "model-x"}))
run = checkout.run([worker()], terminal_input="custom-model\n", before_typing=("no model catalog cached", saved_elsewhere))
self.assertEqual(run.code, 130)
self.assertIn("no model catalog cached", run.stderr)
self.assertIn("A name claude --model takes: ", run.stderr)
self.assertEqual(json.loads(checkout.models_file().read_text()), {"late": "model-y", "other": "model-x", "proj": "custom-model"})
self.assertEqual(stat.S_IMODE(checkout.models_file().stat().st_mode), 0o640)
for broken in ("{", "[]", '{"proj": 1}'):
with self.subTest(broken=broken):
checkout.models_file().write_text(broken)
run = checkout.run([worker()])
self.assertEqual((run.code, run.calls), (1, []))
self.assertTrue(run.stderr.startswith(f"could not read {checkout.models_file()}: "))
self.assertEqual(checkout.models_file().read_text(), broken)
def test_offers_no_catalog_whose_models_are_not_a_list(self):
checkout = Checkout(self)
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
self.catalog(checkout)
catalog = checkout.home / ".claude" / "cache" / "model-catalog" / "new.json"
for shape in ({"catalog": {"config": {"models": []}}}, {"catalog": None}, {"catalog": {"config": []}}, {"catalog": {"config": {"models": "x"}}}):
with self.subTest(shape=shape):
catalog.write_text(json.dumps({**shape, "fetchedAt": 3}))
checkout.models_file().unlink(missing_ok=True)
run = checkout.run([worker()], terminal_input="1\n")
self.assertIn("no model catalog cached", run.stderr)
self.assertEqual(argument(run.calls[0], "--model"), "1")
@unittest.skipIf(os.geteuid() == 0, "root writes into a read-only directory")
def test_exits_1_when_the_models_file_cannot_be_written(self):
checkout = Checkout(self)
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
checkout.models_file().parent.mkdir(parents=True, mode=0o500)
self.addCleanup(checkout.models_file().parent.chmod, 0o700)
run = checkout.run([worker()], terminal_input="custom-model\n")
self.assertEqual((run.code, run.calls), (1, []))
self.assertIn(f"could not save the model in {checkout.models_file()}: Permission denied", run.stderr)
class Reload(unittest.TestCase):
def test_an_edit_takes_effect_at_the_next_cycle_boundary_once_it_answers_help(self):
checkout = Checkout(self)
copy = checkout.temp / "bin"
shutil.copytree(BIN, copy, ignore=shutil.ignore_patterns("__pycache__", "test_*"))
checkout.script = copy / "loop-claude"
checkout.env["FAKE_CLAUDE_EDIT"] = str(copy)
run = checkout.run([
worker(append=["loop-claude", "\ndef\n"]), judge(),
worker(replace=["loop-claude", "\ndef\n", "\n# edited\n"]), judge(),
worker(append=["loop_claude/nested/edited.py", "# edited\n"]), judge(),
worker(append=["files/writes.py", "# edited\n"]), judge(),
worker(),
])
self.assertEqual(run.code, 130)
self.assertEqual(run.roles(), ["worker", "judge"] * 5)
self.assertIn("keeping the running script", run.stderr)
reloads = [json.loads(argv[1]) for argv in run.starts if argv[:1] == ["--reloaded"]]
self.assertEqual(reloads, [{"cycle_number": cycle, "root": str(checkout.root), "task": TASK} for cycle in (2, 3, 4)])
self.assertIn("=== cycle 5 ===", run.stdout)
outside = checkout.temp / "outside"
outside.mkdir()
state = {"cycle_number": 3, "root": str(checkout.root), "task": TASK}
run = checkout.run([worker()], argv=["--reloaded", json.dumps(state)], cwd=outside)
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge"]))
self.assertEqual(run.calls[0]["cwd"], str(checkout.root))
self.assertIn("=== cycle 4 ===", run.stdout)
run = checkout.run([], argv=["--reloaded", "{}"])
self.assertEqual((run.code, run.calls), (1, []))
self.assertIn("rerun loop-claude <task>", run.stderr)
if __name__ == "__main__":
unittest.main()