844 lines
44 KiB
Python
844 lines
44 KiB
Python
import dataclasses
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import pty
|
|
import re
|
|
import select
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from test_env import isolated_env
|
|
|
|
BIN = pathlib.Path(__file__).resolve().parents[1]
|
|
TASK = "Do the next chunk."
|
|
|
|
|
|
# Plays claude from FAKE_CLAUDE_SCENARIO, one step per call, logging each call; a step for another role,
|
|
# or none left, is Ctrl-C to loop-claude, which otherwise loops forever.
|
|
FAKE_CLAUDE = """#!{python}
|
|
import json, os, pathlib, signal, sys, time
|
|
|
|
argv = sys.argv[1:]
|
|
role = "judge" if "--json-schema" in argv else "worker" if "-p" in argv else "interrupt"
|
|
scenario = pathlib.Path(os.environ["FAKE_CLAUDE_SCENARIO"])
|
|
steps = json.loads(scenario.read_text())
|
|
call = {{"argv": argv, "cwd": os.getcwd(), "role": role}}
|
|
if role == "judge":
|
|
call["stdin"] = sys.stdin.read()
|
|
with open(os.environ["FAKE_CLAUDE_LOG"], "a") as log:
|
|
log.write(json.dumps(call) + "\\n")
|
|
if not steps or steps[0]["role"] != role:
|
|
parent = os.getppid()
|
|
os.kill(parent, signal.SIGINT)
|
|
while os.getppid() == parent:
|
|
time.sleep(0.05)
|
|
sys.exit(0)
|
|
step = steps.pop(0)
|
|
scenario.write_text(json.dumps(steps))
|
|
|
|
|
|
def session():
|
|
flag = "--session-id" if "--session-id" in argv else "--resume"
|
|
return argv[argv.index(flag) + 1]
|
|
|
|
|
|
def transcript():
|
|
return pathlib.Path.home() / ".claude" / "projects" / "-fake" / f"{{session()}}.jsonl"
|
|
|
|
|
|
def append(*entries):
|
|
transcript().parent.mkdir(parents=True, exist_ok=True)
|
|
with open(transcript(), "a") as handle:
|
|
handle.writelines(json.dumps(entry) + "\\n" for entry in entries)
|
|
|
|
|
|
if role == "worker":
|
|
if "--session-id" in argv and step.get("transcript", True):
|
|
append({{"type": "custom-title"}}, {{"type": "queue-operation", "timestamp": str(time.time_ns())}})
|
|
said = {{"type": "assistant", "message": {{"content": [{{"type": "text", "text": step.get("say", "working")}}]}}}}
|
|
if step.get("transcript", True):
|
|
append({{"type": "user", "promptSource": "sdk", "message": {{"content": argv[argv.index("-p") + 1]}}}})
|
|
with open(transcript(), "ab") as handle:
|
|
handle.write(step.get("raw", "").encode("latin-1"))
|
|
append(said)
|
|
if step.get("siblings"):
|
|
for sibling in (f"{{session()}}/subagents", f"{{session()}}.superseded-1/tool-results"):
|
|
(transcript().parent / sibling).mkdir(parents=True, exist_ok=True)
|
|
(transcript().parent / f"{{session()}}.orphaned-1-ab.jsonl").write_text("{{}}\\n")
|
|
if "append" in step:
|
|
edited = pathlib.Path(os.environ["FAKE_CLAUDE_EDIT"], step["append"][0])
|
|
edited.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(edited, "a") as script:
|
|
script.write(step["append"][1])
|
|
if "replace" in step:
|
|
script = pathlib.Path(os.environ["FAKE_CLAUDE_EDIT"], step["replace"][0])
|
|
script.write_text(script.read_text().replace(*step["replace"][1:]))
|
|
sys.stdout.flush()
|
|
sys.stdout.buffer.write(step.get("stream", "").encode("latin-1"))
|
|
print(json.dumps(said))
|
|
if step.get("result", True):
|
|
print(json.dumps({{"type": "result"}}))
|
|
if "signal" in step:
|
|
os.kill(os.getpid(), step["signal"])
|
|
elif role == "judge":
|
|
print(step["stdout"] if "stdout" in step else json.dumps({{"structured_output": step["verdict"]}}))
|
|
print(step.get("stderr", ""), file=sys.stderr)
|
|
elif "type" in step:
|
|
append({{"type": "user", "message": {{"content": step["type"]}}}})
|
|
if step.get("unreadable"):
|
|
transcript().chmod(0)
|
|
if step.get("vanish"):
|
|
os.remove(__file__)
|
|
sys.exit(step.get("exit", 0))
|
|
"""
|
|
|
|
# Records every start of loop-claude and turns its retry waits into log lines.
|
|
SITECUSTOMIZE = """import json, os, sys, time
|
|
|
|
if os.path.basename(sys.argv[0]) == "loop-claude":
|
|
def record(event):
|
|
with open(os.environ["LOOP_TEST_LOG"], "a") as log:
|
|
log.write(json.dumps(event) + "\\n")
|
|
record({"argv": sys.argv[1:]})
|
|
pause = time.sleep
|
|
time.sleep = lambda seconds: record({"sleep": seconds}) if seconds >= 1 else pause(seconds)
|
|
"""
|
|
|
|
PREMISES = "# Project\n\n## Goals\n\n1. Work.\n\n## Audience\n\nMe.\n"
|
|
# Item 4's L of 4 steps one rung up from the project's.
|
|
TODO = "## Items\n\n| ID | Release | Exempt | Item | L | E | I | R | S | Moves | Score |\n|---|---|---|---|---|---|---|---|---|---|---|\n| 4 | 0.1.0 | | **Do it.** | 4 | 2 | 3 | 7 | 2 | 1 by 9 | 25 |\n"
|
|
# Item 4's L of 2 steps one rung down.
|
|
CHEAP = TODO.replace("| 4 | 2 | 3 | 7 |", "| 2 | 1 | 2 | 3 |")
|
|
|
|
|
|
def worker(**step):
|
|
return {"role": "worker", **step}
|
|
|
|
|
|
def judge(interrupt=False, finished=False, evidence="working", question="", bundled=0, bundled_escape_and_impact=0, escalation="", **step):
|
|
verdict = {"answer": "", "bundled": bundled, "bundled_escape_and_impact": bundled_escape_and_impact, "escalation": escalation, "evidence": evidence, "finished": finished, "interrupt": interrupt, "question": question}
|
|
return {"role": "judge", "verdict": verdict, **step}
|
|
|
|
|
|
def interrupt(**step):
|
|
return {"role": "interrupt", **step}
|
|
|
|
|
|
def lines(path):
|
|
return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else []
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class Run:
|
|
calls: list
|
|
code: int
|
|
sleeps: list
|
|
starts: list
|
|
stderr: str
|
|
stdout: str
|
|
|
|
def roles(self):
|
|
return [call["role"] for call in self.calls]
|
|
|
|
|
|
class Checkout:
|
|
"""A temporary HOME with a git checkout at ~/code/proj, where loop-claude runs against the fake claude."""
|
|
|
|
def __init__(self, test):
|
|
self.test = test
|
|
temp = tempfile.TemporaryDirectory()
|
|
test.addCleanup(temp.cleanup)
|
|
self.temp = pathlib.Path(os.path.realpath(temp.name))
|
|
self.home = self.temp / "home"
|
|
self.root = self.home / "code" / "proj"
|
|
self.root.mkdir(parents=True)
|
|
(self.root / "README.md").write_text(PREMISES)
|
|
self.script = BIN / "loop-claude"
|
|
self.env = isolated_env()
|
|
fake = self.temp / "fake-bin"
|
|
fake.mkdir()
|
|
(fake / "claude").write_text(FAKE_CLAUDE.format(python=sys.executable))
|
|
(fake / "claude").chmod(0o755)
|
|
(fake / "git").symlink_to(shutil.which("git"))
|
|
site = self.temp / "site"
|
|
site.mkdir()
|
|
(site / "sitecustomize.py").write_text(SITECUSTOMIZE)
|
|
self.env.update({
|
|
"FAKE_CLAUDE_LOG": str(self.temp / "calls.jsonl"),
|
|
"FAKE_CLAUDE_SCENARIO": str(self.temp / "scenario.json"),
|
|
"GIT_CONFIG_GLOBAL": os.devnull,
|
|
"GIT_CONFIG_NOSYSTEM": "1",
|
|
"HOME": str(self.home),
|
|
"LOOP_CLAUDE_JUDGE_MODEL": "judge-model",
|
|
"LOOP_CLAUDE_WORKER_MODEL": "worker-model",
|
|
"LOOP_TEST_LOG": str(self.temp / "starts.jsonl"),
|
|
"PATH": str(fake),
|
|
"PYTHONPATH": str(site),
|
|
})
|
|
self.git("init", "--quiet")
|
|
|
|
def git(self, *args):
|
|
subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@example.com", *args], capture_output=True, check=True, cwd=self.root, env=self.env)
|
|
|
|
def commit_plan(self, todo):
|
|
"""Commits todo as todo.md on the checkout's branch, the default branch routing reads it from."""
|
|
(self.root / "todo.md").write_text(todo)
|
|
self.git("add", "todo.md")
|
|
self.git("commit", "--quiet", "-m", "plan")
|
|
|
|
def run(self, steps, argv=(TASK,), cwd=None, terminal_input=None, stdout_on_terminal=True, before_typing=None):
|
|
for name in ("FAKE_CLAUDE_LOG", "LOOP_TEST_LOG"):
|
|
pathlib.Path(self.env[name]).unlink(missing_ok=True)
|
|
pathlib.Path(self.env["FAKE_CLAUDE_SCENARIO"]).write_text(json.dumps(steps))
|
|
command = [sys.executable, str(self.script), *argv]
|
|
cwd = cwd or self.root
|
|
if terminal_input is None:
|
|
done = subprocess.run(command, capture_output=True, cwd=cwd, env=self.env, stdin=subprocess.DEVNULL, text=True, timeout=60)
|
|
code, stdout, stderr = done.returncode, done.stdout, done.stderr
|
|
else:
|
|
code, stdout, stderr = self.run_on_terminal(command, cwd, terminal_input, stdout_on_terminal, before_typing)
|
|
started = lines(pathlib.Path(self.env["LOOP_TEST_LOG"]))
|
|
return Run(
|
|
calls=lines(pathlib.Path(self.env["FAKE_CLAUDE_LOG"])),
|
|
code=code,
|
|
sleeps=[event["sleep"] for event in started if "sleep" in event],
|
|
starts=[event["argv"] for event in started if "argv" in event],
|
|
stderr=stderr,
|
|
stdout=stdout,
|
|
)
|
|
|
|
def run_on_terminal(self, command, cwd, typed, stdout_on_terminal, before_typing):
|
|
"""Types the input at once, or once stderr shows before_typing's text and its callback has run."""
|
|
controller, terminal = pty.openpty()
|
|
with tempfile.TemporaryFile() as stderr, tempfile.TemporaryFile() as stdout:
|
|
try:
|
|
proc = subprocess.Popen(command, cwd=cwd, env=self.env, stderr=stderr, stdin=terminal, stdout=terminal if stdout_on_terminal else stdout)
|
|
finally:
|
|
os.close(terminal)
|
|
try:
|
|
prompt, callback = before_typing or (None, None)
|
|
if prompt is None:
|
|
os.write(controller, typed.encode())
|
|
output, deadline = b"", time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
if select.select([controller], [], [], 1)[0]:
|
|
try:
|
|
chunk = os.read(controller, 4096)
|
|
except OSError:
|
|
break
|
|
if not chunk:
|
|
break
|
|
output += chunk
|
|
if prompt is not None and prompt.encode() in os.pread(stderr.fileno(), 1 << 20, 0):
|
|
callback()
|
|
os.write(controller, typed.encode())
|
|
prompt = None
|
|
code = proc.wait(timeout=10)
|
|
finally:
|
|
os.close(controller)
|
|
proc.kill()
|
|
proc.wait()
|
|
stderr.seek(0)
|
|
stdout.seek(0)
|
|
output += stdout.read()
|
|
return code, output.decode(errors="replace").replace("\r\n", "\n"), stderr.read().decode()
|
|
|
|
def claude_json(self):
|
|
return self.home / ".claude.json"
|
|
|
|
def models_file(self):
|
|
return self.home / ".claude" / "loop-claude-models.json"
|
|
|
|
def transcripts(self):
|
|
return sorted((self.home / ".claude" / "projects").glob("*/*.jsonl"))
|
|
|
|
def archive(self):
|
|
return pathlib.Path(self.env.get("LOOP_CLAUDE_TRANSCRIPTS", self.home / ".claude" / "loop-claude-transcripts"))
|
|
|
|
def archived(self):
|
|
"""Each archived transcript's outcomes, with every UTC stamp replaced by <UTC>, after checking the copy is named by its content hash."""
|
|
outcomes = {}
|
|
for folder in (self.archive(), self.archive() / "proj", *(self.archive() / "proj").iterdir()):
|
|
self.test.assertEqual(stat.S_IMODE(folder.stat().st_mode), 0o700, folder)
|
|
for copy in sorted((self.archive() / "proj").glob("*/*.jsonl")):
|
|
self.test.assertEqual(copy.stem, hashlib.sha256(copy.read_bytes()).hexdigest())
|
|
kept = copy.with_suffix(".outcomes.md")
|
|
for archived in (copy, kept):
|
|
self.test.assertEqual(stat.S_IMODE(archived.stat().st_mode), 0o600)
|
|
outcomes[copy.read_text()] = re.sub(r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ ", "<UTC> ", kept.read_text(), flags=re.MULTILINE)
|
|
return outcomes
|
|
|
|
|
|
def argument(call, flag):
|
|
return call["argv"][call["argv"].index(flag) + 1] if flag in call["argv"] else None
|
|
|
|
|
|
class Premises(unittest.TestCase):
|
|
def test_refuses_to_start_without_a_task_a_checkout_or_premises(self):
|
|
checkout = Checkout(self)
|
|
for argv, code, said in (([], 1, "usage: loop-claude <task>"), (["--help"], 0, "usage: loop-claude <task>"), (["-h"], 0, "usage: loop-claude <task>")):
|
|
with self.subTest(argv=argv):
|
|
run = checkout.run([], argv=argv)
|
|
self.assertEqual(run.code, code)
|
|
self.assertIn(said, run.stdout if code == 0 else run.stderr)
|
|
outside = checkout.temp / "outside"
|
|
outside.mkdir()
|
|
run = checkout.run([], cwd=outside)
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertIn("not a git repository", run.stderr)
|
|
readmes = {
|
|
None: "README.md in {root}: No such file or directory - no premises.",
|
|
b"# P\n## Goals\n1. g\xff\n## Audience\nme\n": "README.md in {root} is not UTF-8 on line 3 - no premises.",
|
|
b"# P\n## Audience\nme\n": "README.md has no ## Goals heading - no premises.",
|
|
b"# P\n## Goals\n1. g\n": "README.md has no ## Audience heading - no premises.",
|
|
b"# P\n## Goals\n\n## Audience\nme\n": "README.md has nothing under ## Goals - no premises.",
|
|
b"# P\n## Goals\n1. g\n## Audience\nme\n## Audience\nyou\n": "README.md has ## Audience more than once - no premises.",
|
|
}
|
|
(checkout.root / "sub").mkdir()
|
|
for text, said in readmes.items():
|
|
with self.subTest(said=said):
|
|
readme = checkout.root / "README.md"
|
|
readme.unlink(missing_ok=True)
|
|
if text is not None:
|
|
readme.write_bytes(text)
|
|
run = checkout.run([worker()], cwd=checkout.root / "sub")
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertEqual(run.stderr.strip(), said.format(root=checkout.root))
|
|
(checkout.root / "README.md").unlink()
|
|
(checkout.root / "README.md").mkdir()
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertEqual(run.stderr.strip(), f"README.md in {checkout.root}: Is a directory - no premises.")
|
|
(checkout.temp / "fake-bin" / "git").unlink()
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertEqual(run.stderr.strip(), "could not start git: No such file or directory")
|
|
|
|
def test_makes_no_worktree_for_a_readme_without_premises(self):
|
|
checkout = Checkout(self)
|
|
checkout.commit_plan(TODO)
|
|
(checkout.root / "README.md").write_text("# Project\n")
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertFalse((checkout.home / "code" / "proj-item-4").exists())
|
|
|
|
def test_names_the_session_by_the_checkout_relative_to_code_or_else_to_the_root(self):
|
|
checkout = Checkout(self)
|
|
for root in (checkout.temp / "elsewhere", checkout.home / "code"):
|
|
with self.subTest(root=root):
|
|
root.mkdir(exist_ok=True)
|
|
(root / "README.md").write_text(PREMISES)
|
|
checkout.git("init", "--quiet", str(root))
|
|
run = checkout.run([worker()], cwd=root)
|
|
self.assertEqual(argument(run.calls[0], "--name"), f"A {str(root).lstrip('/')} loop-claude")
|
|
|
|
|
|
class Cycles(unittest.TestCase):
|
|
def test_resumes_the_session_while_unfinished_and_starts_it_fresh_after_finishing(self):
|
|
checkout = Checkout(self)
|
|
(checkout.root / "sub").mkdir()
|
|
(checkout.root / "docs").mkdir()
|
|
(checkout.root / "docs" / "decisions.md").write_text("# Decisions\n")
|
|
(checkout.home / ".claude").mkdir()
|
|
(checkout.home / ".claude" / "CLAUDE.md").write_text("# Global instructions\n")
|
|
run = checkout.run([
|
|
worker(say="started"), judge(evidence="started"),
|
|
worker(say="landed-it", siblings=True), judge(finished=True, evidence="landed-it"),
|
|
worker(say="next chunk"),
|
|
], cwd=checkout.root / "sub")
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "worker", "judge", "worker", "judge"])
|
|
workers = [call for call in run.calls if call["role"] == "worker"]
|
|
session = argument(workers[0], "--session-id")
|
|
self.assertEqual(argument(workers[0], "--name"), "A proj loop-claude")
|
|
self.assertEqual(argument(workers[1], "--resume"), session)
|
|
self.assertIsNone(argument(workers[1], "--session-id"))
|
|
self.assertIsNone(argument(workers[1], "--name"))
|
|
self.assertEqual(argument(workers[2], "--session-id"), session)
|
|
self.assertTrue(argument(workers[0], "-p").startswith(f"{TASK}\n\nContinue any open draft PR"))
|
|
self.assertTrue(argument(workers[0], "-p").endswith("The chunk's todo.md items, or else the task above, set its scope. The goals below choose between ways to deliver that scope, and never widen it.\n\nFrom README.md:\n## Goals\n1. Work.\n\n## Audience\nMe."))
|
|
self.assertIn("main or master", argument(workers[0], "-p"))
|
|
self.assertNotIn(TASK, argument(workers[1], "-p"))
|
|
self.assertIn("Even then, it is finished only once every answer the human typed in it", argument(workers[1], "-p"))
|
|
self.assertIn("as the work remaining:\n\n> started\n\n", argument(workers[1], "-p"))
|
|
self.assertEqual({call["cwd"] for call in run.calls}, {str(checkout.root)})
|
|
self.assertIn("1. Work.", run.calls[1]["stdin"])
|
|
self.assertIn("From docs/decisions.md:\n# Decisions\n\nFrom ~/.claude/CLAUDE.md:\n# Global instructions", run.calls[1]["stdin"])
|
|
self.assertEqual({argument(call, "--model") for call in workers}, {"worker-model"})
|
|
self.assertEqual({argument(call, "--model") for call in run.calls if call["role"] == "judge"}, {"judge-model"})
|
|
self.assertIn("landed-it", run.calls[3]["stdin"])
|
|
self.assertNotIn("landed-it", run.calls[5]["stdin"])
|
|
self.assertEqual([path.name for path in checkout.transcripts()[0].parent.iterdir()], [checkout.transcripts()[0].name])
|
|
self.assertIn("=== cycle 3, worker on worker-model ===", run.stdout)
|
|
copies = {verdict.removeprefix(f"Task: {TASK}\n\n"): copy for copy, verdict in checkout.archived().items()}
|
|
self.assertEqual(sorted(copies), ["<UTC> Stopped: Ctrl-C\n\nWorker: worker-model\n", "<UTC> Verdict: finished\n\nEvidence:\n> landed-it\n\nJudge: judge-model\nWorker: worker-model\n", "<UTC> Verdict: unfinished\n\nEvidence:\n> started\n\nJudge: judge-model\nWorker: worker-model\n"])
|
|
self.assertIn('"text": "landed-it"', copies["<UTC> Verdict: finished\n\nEvidence:\n> landed-it\n\nJudge: judge-model\nWorker: worker-model\n"])
|
|
self.assertIn('"text": "next chunk"', copies["<UTC> Stopped: Ctrl-C\n\nWorker: worker-model\n"])
|
|
self.assertEqual(len(list(checkout.archive().glob(f"proj/*/session/{session}.orphaned-1-ab.jsonl"))), 1)
|
|
|
|
def test_routes_the_worker_by_its_items_l_and_steps_it_up_within_the_chunk(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(CHEAP)
|
|
run = checkout.run([
|
|
worker(say="one"), judge(evidence="one"),
|
|
worker(say="two"), judge(evidence="two"),
|
|
worker(say="three"), judge(evidence="three", bundled=4),
|
|
worker(say="four"), judge(finished=True, evidence="four"),
|
|
worker(say="next chunk"),
|
|
])
|
|
self.assertEqual(run.code, 130)
|
|
workers = [call for call in run.calls if call["role"] == "worker"]
|
|
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in workers], [
|
|
("claude-sonnet-5-5", "high"),
|
|
("claude-sonnet-5-5", "high"),
|
|
("claude-opus-5-5", "medium"),
|
|
("claude-opus-5-5", "high"),
|
|
("claude-sonnet-5-5", "high"),
|
|
])
|
|
self.assertEqual({(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "judge"}, {("judge-model", None)})
|
|
self.assertIn("loop-claude runs you on claude-sonnet-5-5 at high effort, routed by item 4's L of 2. Name each todo.md item this chunk takes, with its L and E + I, whether or not it is item 4.", argument(workers[0], "-p"))
|
|
outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values())
|
|
self.assertIn("<UTC> Verdict: unfinished\n\nEvidence:\n> three\n\nJudge: judge-model\nWorker: claude-opus-5-5 at medium effort, routed by item 4's L of 2 and E + I of 3\n\n<UTC> Stepped up\n\nWorker: claude-opus-5-5 at high effort, routed by item 4's L of 2 and E + I of 3\n", outcomes)
|
|
self.assertIn("<UTC> Verdict: finished\n\nEvidence:\n> four\n\nJudge: judge-model\nWorker: claude-opus-5-5 at high effort, routed by item 4's L of 2 and E + I of 3\n", outcomes)
|
|
|
|
def test_routes_on_and_states_the_same_item_from_the_fetched_default_branch_whatever_the_checkout_holds(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
origin = checkout.temp / "origin.git"
|
|
checkout.git("init", "--quiet", "--bare", "--initial-branch=main", str(origin))
|
|
checkout.git("symbolic-ref", "HEAD", "refs/heads/main")
|
|
checkout.commit_plan(TODO)
|
|
checkout.git("remote", "add", "origin", str(origin))
|
|
checkout.git("push", "--quiet", "--set-upstream", "origin", "main")
|
|
# Item 4 ships elsewhere, and item 6 is next; the checkout is behind origin, and holds a change of its own.
|
|
elsewhere = checkout.temp / "elsewhere"
|
|
checkout.git("clone", "--quiet", str(origin), str(elsewhere))
|
|
(elsewhere / "todo.md").write_text(TODO.replace("| 4 | 0.1.0 |", "| 6 | 0.1.0 |"))
|
|
checkout.git("-C", str(elsewhere), "commit", "--quiet", "--all", "-m", "item 4 shipped, plan item 6")
|
|
checkout.git("-C", str(elsewhere), "push", "--quiet", "origin", "main")
|
|
(checkout.root / "todo.md").write_text(TODO + "\nA change of the checkout's own.\n")
|
|
(checkout.root / "sub").mkdir()
|
|
run = checkout.run([worker()], cwd=checkout.root / "sub")
|
|
self.assertEqual(run.code, 130)
|
|
prompt = argument(run.calls[0], "-p")
|
|
self.assertIn("routed by item 6's L of 4", prompt)
|
|
self.assertIn("item 6, L 4, E + I 5", prompt)
|
|
self.assertTrue((checkout.home / "code" / "proj-item-6").is_dir())
|
|
|
|
def test_steps_the_worker_up_a_rung_when_the_judge_finds_in_the_transcript_the_escalation_trigger_it_cites(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(TODO)
|
|
run = checkout.run([
|
|
worker(say="Stability round 2: a 🔴. Trigger 3 fired; I ask for a higher rung."), judge(evidence="I ask for a higher rung.", escalation="Stability round 2: a 🔴."),
|
|
worker(say="Trigger 2 fired; I ask for a higher rung."), judge(evidence="I ask for a higher rung.", escalation="Trigger 2 fired; … a higher rung."),
|
|
worker(say="merged"), judge(finished=True, evidence="merged"),
|
|
worker(say="next chunk"),
|
|
])
|
|
self.assertEqual(run.code, 130)
|
|
workers = [call for call in run.calls if call["role"] == "worker"]
|
|
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in workers], [
|
|
("claude-opus-5-5", "high"),
|
|
("claude-opus-5-5", "xhigh"),
|
|
("claude-opus-5-5", "xhigh"),
|
|
("claude-opus-5-5", "high"),
|
|
])
|
|
outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values())
|
|
self.assertIn("<UTC> Verdict: unfinished\n\nEvidence:\n> I ask for a higher rung.\n\nEscalation:\n> Stability round 2: a 🔴.\n\nJudge: judge-model\nWorker: claude-opus-5-5 at high effort, routed by item 4's L of 4 and E + I of 5\n\n<UTC> Stepped up\n\nWorker: claude-opus-5-5 at xhigh effort, routed by item 4's L of 4 and E + I of 5\n", outcomes)
|
|
self.assertIn("<UTC> Verdict: unfinished\n\nEvidence:\n> I ask for a higher rung.\n\nJudge: judge-model\nWorker: claude-opus-5-5 at xhigh effort, routed by item 4's L of 4 and E + I of 5\n", outcomes)
|
|
|
|
def test_runs_a_cheap_items_worker_below_the_projects_rung_with_its_reviewers_and_judge_on_it(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(CHEAP)
|
|
run = checkout.run([worker(say="one"), judge(finished=True, evidence="one"), worker(say="next chunk")])
|
|
self.assertEqual(run.code, 130)
|
|
first = next(call for call in run.calls if call["role"] == "worker")
|
|
self.assertEqual((argument(first, "--model"), argument(first, "--effort")), ("claude-sonnet-5-5", "high"))
|
|
ruling = next(call for call in run.calls if call["role"] == "judge")
|
|
self.assertEqual((argument(ruling, "--model"), argument(ruling, "--effort")), ("claude-opus-5-5", "medium"))
|
|
self.assertIn("Judge: claude-opus-5-5 at medium effort\nWorker: claude-sonnet-5-5 at high effort", "".join(checkout.archived().values()))
|
|
self.assertIn("Launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool set to model opus, effort medium, the project's rung.", argument(first, "-p"))
|
|
|
|
def test_routes_the_judge_and_the_reviewers_on_the_highest_e_plus_i_the_worker_takes_in(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(CHEAP)
|
|
run = checkout.run([
|
|
worker(say="one"), judge(evidence="one", bundled_escape_and_impact=6),
|
|
worker(say="two"), judge(finished=True, evidence="two"),
|
|
worker(say="next chunk"),
|
|
])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual([argument(call, "--effort") for call in run.calls if call["role"] == "judge"], ["medium", "xhigh", "medium"])
|
|
workers = [call for call in run.calls if call["role"] == "worker"]
|
|
self.assertEqual({argument(call, "--effort") for call in workers}, {"high"})
|
|
self.assertIn("Launch each reviewer with the Agent tool set to model opus, effort xhigh, the rung picked by an item of E + I 6 taken into the chunk.", argument(workers[1], "-p"))
|
|
outcomes = "".join(checkout.archived().values())
|
|
self.assertIn("<UTC> Stepped up\n\nWorker: claude-sonnet-5-5 at high effort, routed by item 4's L of 2 and E + I of 3, taking in an E + I of 6\n", outcomes)
|
|
self.assertIn("Judge: claude-opus-5-5 at xhigh effort\nWorker: claude-sonnet-5-5 at high effort, routed by item 4's L of 2 and E + I of 3, taking in an E + I of 6\n", outcomes)
|
|
|
|
def test_a_rerun_resumes_an_open_chunk_on_the_rung_it_last_ran(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(TODO)
|
|
run = checkout.run([worker(), judge(), worker(), judge()])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(argument(run.calls[-1], "--effort"), "xhigh")
|
|
checkout.git("rm", "--quiet", "todo.md")
|
|
checkout.git("commit", "--quiet", "-m", "drop the plan")
|
|
run = checkout.run([worker()])
|
|
self.assertIsNotNone(argument(run.calls[0], "--resume"))
|
|
self.assertEqual((argument(run.calls[0], "--model"), argument(run.calls[0], "--effort")), ("claude-opus-5-5", "xhigh"))
|
|
|
|
def test_exits_1_when_the_transcript_cannot_be_archived_and_resumes_on_the_next_run(self):
|
|
checkout = Checkout(self)
|
|
checkout.archive().parent.mkdir(parents=True)
|
|
checkout.archive().write_text("in the way")
|
|
run = checkout.run([worker(say="landed-it"), judge(finished=True, evidence="landed-it")])
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge"]))
|
|
self.assertRegex(run.stderr.strip().splitlines()[-1], r"^could not archive the transcript: .*/loop-claude-transcripts/proj: Not a directory$")
|
|
self.assertEqual(run.stderr.count("could not archive the transcript"), 1)
|
|
session = argument(run.calls[0], "--session-id")
|
|
checkout.archive().unlink()
|
|
run = checkout.run([worker(), judge(finished=True)])
|
|
self.assertEqual(argument(run.calls[0], "--resume"), session)
|
|
self.assertEqual(len(checkout.archived()), 1)
|
|
|
|
def test_stops_after_three_failures_running_retrying_a_minute_apart(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(exit=1), worker(result=False), worker(signal=9)])
|
|
self.assertEqual(run.code, 1)
|
|
self.assertEqual(run.roles(), ["worker"] * 3)
|
|
self.assertEqual(run.sleeps, [60, 60])
|
|
self.assertIn("retrying the worker in a minute: claude exited 1", run.stderr)
|
|
self.assertIn("claude exited 0 without a result event", run.stderr)
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the worker failed three times running: claude killed by signal 9")
|
|
self.assertNotIn("=== cycle 2,", run.stdout)
|
|
self.assertEqual(argument(run.calls[1], "--resume"), argument(run.calls[0], "--session-id"))
|
|
|
|
run = checkout.run([worker(exit=1), worker(exit=1), worker(), judge(), worker(exit=1), worker(exit=1)])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.sleeps, [60, 60, 60, 60])
|
|
|
|
run = checkout.run([worker(), judge(exit=1, stderr="judge down"), judge(stdout=""), judge(verdict={"interrupt": False})])
|
|
self.assertEqual(run.code, 1)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "judge", "judge"])
|
|
self.assertEqual(run.sleeps, [60, 60])
|
|
self.assertIn("retrying the judge in a minute: judge down", run.stderr)
|
|
self.assertIn("retrying the judge in a minute: the judge printed no json reply; it printed: nothing\n", run.stderr)
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], 'the judge failed three times running: the judge\'s reply has no answer of type string')
|
|
|
|
def test_waits_out_a_usage_limit_which_ends_a_run_of_failures(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["TZ"] = "UTC"
|
|
resets_at = int(time.time()) + 3600
|
|
rejected = json.dumps({"type": "rate_limit_event", "rate_limit_info": {"status": "rejected", "resetsAt": resets_at}})
|
|
limited = json.dumps({"type": "result", "api_error_status": 429, "is_error": True, "result": "You've hit your session limit"})
|
|
run = checkout.run([worker(exit=1), worker(exit=1), worker(stream=f"{rejected}\n{limited}\n", result=False, exit=1), worker(exit=1), worker(), judge(stdout=limited, exit=1), judge()])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker"] * 5 + ["judge"] * 2 + ["worker"])
|
|
self.assertEqual([round(seconds, -1) for seconds in run.sleeps], [60, 60, 3720, 60, 600])
|
|
self.assertIn(f"the worker hit a usage limit (You've hit your session limit); retrying at {datetime.datetime.fromtimestamp(resets_at + 120, datetime.timezone.utc):%Y-%m-%d %H:%M} UTC\n", run.stderr)
|
|
self.assertIn("the judge hit a usage limit (You've hit your session limit); retrying at ", run.stderr)
|
|
|
|
def test_retries_the_judge_alone_when_its_verdict_quotes_nothing_in_the_transcript(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([
|
|
worker(say="Shall I use A or B?"),
|
|
judge(interrupt=True, evidence="never said", question="A or B?"),
|
|
judge(finished=True, evidence=""),
|
|
judge(evidence="A or B"),
|
|
worker(),
|
|
])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "judge", "judge", "worker", "judge"])
|
|
self.assertEqual(run.sleeps, [60, 60])
|
|
self.assertIn("the judge quoted 'never said', which is not in the transcript, for the question: A or B?", run.stderr)
|
|
self.assertIn("retrying the judge in a minute: the judge quoted '', which is not in the transcript\n", run.stderr)
|
|
|
|
def test_retries_a_judge_claude_cannot_start(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(vanish=True)])
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
|
|
self.assertEqual(run.sleeps, [60, 60])
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the judge failed three times running: could not start claude: No such file or directory")
|
|
|
|
def test_hands_the_judge_the_tail_of_a_long_transcript(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(say="early " * 80000 + "LATE")])
|
|
self.assertIn("LATE", run.calls[1]["stdin"])
|
|
self.assertEqual(len(run.calls[1]["stdin"]), 400000)
|
|
self.assertLess(run.calls[1]["stdin"].count("early"), 80000)
|
|
self.assertTrue(run.calls[1]["stdin"].rstrip().endswith("CI not yet reported, an answer not yet recorded."))
|
|
|
|
def test_skips_a_malformed_transcript_or_stream_line(self):
|
|
checkout = Checkout(self)
|
|
raw = "\xff not UTF-8\nnot JSON\n[1]\n" + json.dumps({"type": "assistant", "message": "not an object"}) + "\n" + json.dumps({"type": "user", "message": None}) + "\n"
|
|
run = checkout.run([worker(raw=raw, say="past the noise", stream="\xff not UTF-8\n"), judge(evidence="past the noise")])
|
|
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge", "worker"]))
|
|
self.assertIn("past the noise", run.stdout)
|
|
self.assertIn("past the noise", run.calls[1]["stdin"])
|
|
|
|
@unittest.skipIf(os.geteuid() == 0, "root reads an unreadable file")
|
|
def test_exits_1_when_the_transcript_cannot_be_read(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(unreadable=True)])
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
|
|
self.assertRegex(run.stderr.strip().splitlines()[-2], r"^could not archive the transcript: .*\.jsonl: Permission denied$")
|
|
self.assertRegex(run.stderr.strip().splitlines()[-1], r"^could not read the transcript .*\.jsonl: Permission denied$")
|
|
|
|
def test_retries_a_worker_claude_cannot_start(self):
|
|
checkout = Checkout(self)
|
|
(checkout.temp / "fake-bin" / "claude").unlink()
|
|
run = checkout.run([])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertEqual(run.sleeps, [60, 60])
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the worker failed three times running: could not start claude: No such file or directory")
|
|
|
|
def test_exits_1_when_claude_leaves_no_transcript(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(transcript=False)])
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
|
|
self.assertIn("claude left no transcript for session", run.stderr)
|
|
|
|
|
|
class Interrupt(unittest.TestCase):
|
|
def test_exits_2_off_a_terminal_and_resumes_on_the_next_run(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_TRANSCRIPTS"] = str(checkout.temp / "deep" / "archive")
|
|
steps = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
|
|
run = checkout.run(steps)
|
|
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge"]))
|
|
self.assertEqual(run.stderr.strip().splitlines()[-2:], ["A or B?", "the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal"])
|
|
[(copy, verdict)] = checkout.archived().items()
|
|
self.assertEqual(copy, (checkout.transcripts()[0]).read_text())
|
|
self.assertEqual(verdict, f"Task: {TASK}\n\n<UTC> Verdict: interrupt\n\nA or B?\n\nEvidence:\n> Shall I use A or B?\n\nJudge: judge-model\n\n<UTC> Stopped: the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal\n\nWorker: worker-model\n")
|
|
self.assertEqual(stat.S_IMODE((checkout.temp / "deep").stat().st_mode), 0o700)
|
|
self.assertIn(f"transcript: {checkout.archive() / 'proj'}/", run.stderr)
|
|
session = argument(run.calls[0], "--session-id")
|
|
run = checkout.run(steps, terminal_input="", stdout_on_terminal=False)
|
|
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge"]))
|
|
self.assertEqual(argument(run.calls[0], "--resume"), session)
|
|
self.assertEqual(len(checkout.archived()), 2)
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal")
|
|
run = checkout.run([worker(transcript=False), steps[1]])
|
|
self.assertEqual(run.code, 2)
|
|
self.assertEqual(len(checkout.archived()), 2)
|
|
|
|
def test_hands_the_terminal_over_and_exits_2_when_nothing_is_typed(self):
|
|
checkout = Checkout(self)
|
|
checkout.git("commit", "--allow-empty", "--quiet", "-m", "start")
|
|
checkout.git("worktree", "add", "--quiet", str(checkout.home / "code" / "proj-wt"))
|
|
checkout.claude_json().write_text(json.dumps({"other": 1, "projects": {str(checkout.root): {"kept": True}}}))
|
|
checkout.claude_json().chmod(0o644)
|
|
asked = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
|
|
(checkout.root / "sub").mkdir()
|
|
run = checkout.run([*asked, interrupt(type="B"), judge(evidence="B"), worker()], cwd=checkout.root / "sub", terminal_input="")
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "interrupt", "judge", "worker", "judge"])
|
|
handed = run.calls[2]["argv"]
|
|
self.assertEqual(run.calls[2]["cwd"], str(checkout.root))
|
|
self.assertEqual(argument(run.calls[2], "--resume"), argument(run.calls[0], "--session-id"))
|
|
self.assertEqual(argument(run.calls[2], "--model"), "worker-model")
|
|
self.assertNotIn("-p", handed)
|
|
self.assertIn("[human] B", run.calls[3]["stdin"])
|
|
trusted = json.loads(checkout.claude_json().read_text())
|
|
self.assertEqual(trusted["other"], 1)
|
|
self.assertEqual(trusted["projects"][str(checkout.root)], {"hasTrustDialogAccepted": True, "kept": True})
|
|
self.assertEqual(trusted["projects"][str(checkout.home / "code" / "proj-wt")], {"hasTrustDialogAccepted": True})
|
|
self.assertEqual(stat.S_IMODE(checkout.claude_json().stat().st_mode), 0o644)
|
|
self.assertIn(f"trusted for the interrupt: {checkout.root}, {checkout.home / 'code' / 'proj-wt'}\n", run.stderr)
|
|
|
|
for transcript in checkout.transcripts():
|
|
transcript.unlink()
|
|
run = checkout.run([*asked, interrupt()], terminal_input="")
|
|
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge", "interrupt"]))
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the human is needed to answer the question above, but the interrupt ended with nothing typed")
|
|
|
|
run = checkout.run([worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?", vanish=True)], terminal_input="")
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge"]))
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "could not start claude for the interrupt: No such file or directory")
|
|
|
|
def test_says_why_the_trust_file_cannot_be_read_and_leaves_it_as_it_was(self):
|
|
checkout = Checkout(self)
|
|
asked = [worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?"), interrupt()]
|
|
root = json.dumps(str(checkout.root))
|
|
shapes = {
|
|
None: "No such file or directory",
|
|
"{": "Expecting property name enclosed in double quotes: line 1 column 2 (char 1)",
|
|
"[]": 'not an object whose "projects" is an object',
|
|
'{"projects": []}': 'not an object whose "projects" is an object',
|
|
f'{{"projects": {{{root}: 1}}}}': f"projects[{root}] is not an object",
|
|
}
|
|
for text, said in shapes.items():
|
|
with self.subTest(said=said):
|
|
checkout.claude_json().unlink(missing_ok=True)
|
|
if text is not None:
|
|
checkout.claude_json().write_text(text)
|
|
run = checkout.run(asked, terminal_input="")
|
|
self.assertEqual(run.code, 2)
|
|
self.assertIn(f"could not read {checkout.claude_json()}: {said}\n", run.stderr)
|
|
self.assertEqual(checkout.claude_json().read_text() if checkout.claude_json().exists() else None, text)
|
|
|
|
trusted = json.dumps({"projects": {str(checkout.root): {"hasTrustDialogAccepted": True}}})
|
|
checkout.claude_json().write_text(trusted)
|
|
run = checkout.run(asked, terminal_input="")
|
|
self.assertNotIn(str(checkout.claude_json()), run.stderr)
|
|
self.assertNotIn("trusted for the interrupt", run.stderr)
|
|
self.assertEqual(checkout.claude_json().read_text(), trusted)
|
|
|
|
@unittest.skipIf(os.geteuid() == 0, "root writes into a read-only directory")
|
|
def test_says_why_the_trust_file_cannot_be_written(self):
|
|
checkout = Checkout(self)
|
|
checkout.claude_json().write_text("{}")
|
|
asked = [worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?"), interrupt()]
|
|
checkout.run(asked[:2])
|
|
checkout.home.chmod(0o500)
|
|
self.addCleanup(checkout.home.chmod, 0o700)
|
|
run = checkout.run([worker(), *asked[1:]], terminal_input="")
|
|
self.assertEqual(run.code, 2)
|
|
self.assertIn(f"could not write {checkout.claude_json()}: Permission denied\n", run.stderr)
|
|
self.assertEqual(checkout.claude_json().read_text(), "{}")
|
|
|
|
@unittest.skipIf(os.geteuid() == 0, "root reads an unreadable file")
|
|
def test_exits_1_when_the_transcript_cannot_be_read_after_the_interrupt(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?"), interrupt(type="B", unreadable=True)], terminal_input="")
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge", "interrupt"]))
|
|
self.assertRegex(run.stderr.strip().splitlines()[-1], r"^could not tell whether you typed anything: could not read the transcript .*\.jsonl: Permission denied$")
|
|
|
|
|
|
class Models(unittest.TestCase):
|
|
def catalog(self, checkout):
|
|
folder = checkout.home / ".claude" / "cache" / "model-catalog"
|
|
folder.mkdir(parents=True)
|
|
(folder / "old.json").write_text(json.dumps({"catalog": {"config": {"models": [{"id": "old-model"}]}}, "fetchedAt": 1}))
|
|
(folder / "new.json").write_text(json.dumps({"catalog": {"config": {"models": [{"id": "model-a", "name": "A"}, {"id": "model-b", "name": "B"}]}}, "fetchedAt": 2}))
|
|
|
|
def test_asks_a_project_without_a_model_on_the_terminal_and_saves_the_answer(self):
|
|
checkout = Checkout(self)
|
|
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
|
|
self.catalog(checkout)
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (2, []))
|
|
self.assertEqual(run.stderr.strip(), f'the human is needed to pick proj\'s model, or merge {{"proj": "<model>"}} into {checkout.models_file()}, but stdin or stdout is not a terminal; rerun loop-claude on a terminal')
|
|
|
|
run = checkout.run([worker()], terminal_input="\x04")
|
|
self.assertEqual((run.code, run.calls), (2, []))
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the human is needed to pick proj's model, but the input ended")
|
|
|
|
checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "judge-model"
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (2, []))
|
|
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"]
|
|
run = checkout.run([worker(), judge()], terminal_input="\n9\n2\n")
|
|
self.assertEqual(run.code, 130)
|
|
self.assertIn("proj runs on model-b", run.stderr)
|
|
self.assertIn("1. A (model-a)", run.stderr)
|
|
self.assertNotIn("old-model", run.stderr)
|
|
self.assertIn("pick 1 to 2", run.stderr)
|
|
self.assertEqual({argument(call, "--model") for call in run.calls}, {"model-b"})
|
|
self.assertEqual(json.loads(checkout.models_file().read_text()), {"proj": "model-b"})
|
|
self.assertEqual(stat.S_IMODE(checkout.models_file().stat().st_mode), 0o600)
|
|
|
|
checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "judge-model"
|
|
run = checkout.run([worker(), judge()])
|
|
self.assertEqual([argument(call, "--model") for call in run.calls], ["model-b", "judge-model", "model-b"])
|
|
|
|
def test_keeps_the_models_file_mode_and_other_projects_when_saving(self):
|
|
checkout = Checkout(self)
|
|
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
|
|
checkout.models_file().parent.mkdir(parents=True)
|
|
checkout.models_file().write_text(json.dumps({"other": "model-x"}))
|
|
checkout.models_file().chmod(0o640)
|
|
def saved_elsewhere():
|
|
checkout.models_file().write_text(json.dumps({"late": "model-y", "other": "model-x"}))
|
|
|
|
run = checkout.run([worker()], terminal_input="custom-model\n", before_typing=("no model catalog cached", saved_elsewhere))
|
|
self.assertEqual(run.code, 130)
|
|
self.assertIn("no model catalog cached", run.stderr)
|
|
self.assertIn("A name claude --model takes: ", run.stderr)
|
|
self.assertEqual(json.loads(checkout.models_file().read_text()), {"late": "model-y", "other": "model-x", "proj": "custom-model"})
|
|
self.assertEqual(stat.S_IMODE(checkout.models_file().stat().st_mode), 0o640)
|
|
|
|
for broken in ("{", "[]", '{"proj": 1}'):
|
|
with self.subTest(broken=broken):
|
|
checkout.models_file().write_text(broken)
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertTrue(run.stderr.startswith(f"could not read {checkout.models_file()}: "))
|
|
self.assertEqual(checkout.models_file().read_text(), broken)
|
|
|
|
def test_offers_no_catalog_whose_models_are_not_a_list(self):
|
|
checkout = Checkout(self)
|
|
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
|
|
self.catalog(checkout)
|
|
catalog = checkout.home / ".claude" / "cache" / "model-catalog" / "new.json"
|
|
for shape in ({"catalog": {"config": {"models": []}}}, {"catalog": None}, {"catalog": {"config": []}}, {"catalog": {"config": {"models": "x"}}}):
|
|
with self.subTest(shape=shape):
|
|
catalog.write_text(json.dumps({**shape, "fetchedAt": 3}))
|
|
checkout.models_file().unlink(missing_ok=True)
|
|
run = checkout.run([worker()], terminal_input="1\n")
|
|
self.assertIn("no model catalog cached", run.stderr)
|
|
self.assertEqual(argument(run.calls[0], "--model"), "1")
|
|
|
|
@unittest.skipIf(os.geteuid() == 0, "root writes into a read-only directory")
|
|
def test_exits_1_when_the_models_file_cannot_be_written(self):
|
|
checkout = Checkout(self)
|
|
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
|
|
checkout.models_file().parent.mkdir(parents=True, mode=0o500)
|
|
self.addCleanup(checkout.models_file().parent.chmod, 0o700)
|
|
run = checkout.run([worker()], terminal_input="custom-model\n")
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertIn(f"could not save the model in {checkout.models_file()}: Permission denied", run.stderr)
|
|
|
|
|
|
class Reload(unittest.TestCase):
|
|
def test_an_edit_takes_effect_at_the_next_cycle_boundary_once_it_answers_help(self):
|
|
checkout = Checkout(self)
|
|
copy = checkout.temp / "bin"
|
|
shutil.copytree(BIN, copy, ignore=shutil.ignore_patterns("__pycache__", "test_*"))
|
|
checkout.script = copy / "loop-claude"
|
|
checkout.env["FAKE_CLAUDE_EDIT"] = str(copy)
|
|
run = checkout.run([
|
|
worker(append=["loop-claude", "\ndef\n"]), judge(),
|
|
worker(replace=["loop-claude", "\ndef\n", "\n# edited\n"]), judge(),
|
|
worker(append=["loop_claude/nested/edited.py", "# edited\n"]), judge(),
|
|
worker(append=["file_writes.py", "# edited\n"]), judge(),
|
|
worker(),
|
|
])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge"] * 5)
|
|
self.assertIn("keeping the running script", run.stderr)
|
|
reloads = [json.loads(argv[1]) for argv in run.starts if argv[:1] == ["--reloaded"]]
|
|
self.assertEqual(reloads, [{"cycle_number": cycle, "root": str(checkout.root), "task": TASK, "unfinished_evidence": "working", "route": None} for cycle in (2, 3, 4)])
|
|
self.assertIn("=== cycle 5,", run.stdout)
|
|
|
|
outside = checkout.temp / "outside"
|
|
outside.mkdir()
|
|
state = {"cycle_number": 3, "root": str(checkout.root), "task": TASK, "unfinished_evidence": "", "route": None}
|
|
run = checkout.run([worker()], argv=["--reloaded", json.dumps(state)], cwd=outside)
|
|
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge"]))
|
|
self.assertEqual(run.calls[0]["cwd"], str(checkout.root))
|
|
self.assertIn("=== cycle 4,", run.stdout)
|
|
|
|
run = checkout.run([], argv=["--reloaded", "{}"])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertIn("rerun loop-claude <task>", run.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|