988 lines
56 KiB
Python
988 lines
56 KiB
Python
import dataclasses
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import pty
|
|
import re
|
|
import select
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from test_env import isolated_env
|
|
from loop_state.waiting.answers import send, Typed
|
|
from loop_state import session_lock
|
|
from loop_state.waiting.listing import waiting
|
|
from project_config.test_project_config import write_config
|
|
|
|
BIN = pathlib.Path(__file__).resolve().parents[1]
|
|
TASK = "Do the next chunk."
|
|
# How the archive records a worker that runs off the ladder.
|
|
OFF_LADDER = '{"model": "worker-model"}'
|
|
# How a reload hands that worker's route.
|
|
OFF_LADDER_ROUTE = {"rung": {"model": "worker-model", "effort": ""}, "item": 0, "likelihood": 0, "escape_and_impact": 0, "unfinished": 0, "escape_and_impact_taken_in": 0, "escalated_at": 0}
|
|
|
|
|
|
# Plays claude from FAKE_CLAUDE_SCENARIO, one step per call, logging each call; a step for another role,
|
|
# or none left, is Ctrl-C to loop-claude, which otherwise loops forever.
|
|
FAKE_CLAUDE = """#!{python}
|
|
import json, os, pathlib, signal, sys, time
|
|
|
|
argv = sys.argv[1:]
|
|
role = "judge" if "--json-schema" in argv else "worker" if "-p" in argv else "interrupt"
|
|
scenario = pathlib.Path(os.environ["FAKE_CLAUDE_SCENARIO"])
|
|
steps = json.loads(scenario.read_text())
|
|
call = {{"argv": argv, "cwd": os.getcwd(), "role": role, "ctrl_c_ignored": signal.getsignal(signal.SIGINT) == signal.SIG_IGN, "sandboxed": os.environ.get("CLAUDE_CODE_SANDBOXED")}}
|
|
# The items each loop lists in loop-inbox as this claude starts.
|
|
call["running"] = [json.loads(path.read_text())["items"] for path in sorted((pathlib.Path.home() / ".local" / "state" / "loop-claude").glob("*.running"))]
|
|
if role == "judge":
|
|
call["stdin"] = sys.stdin.read()
|
|
with open(os.environ["FAKE_CLAUDE_LOG"], "a") as log:
|
|
log.write(json.dumps(call) + "\\n")
|
|
if not steps or steps[0]["role"] != role:
|
|
parent = os.getppid()
|
|
os.kill(parent, signal.SIGINT)
|
|
while os.getppid() == parent:
|
|
time.sleep(0.05)
|
|
sys.exit(0)
|
|
step = steps.pop(0)
|
|
scenario.write_text(json.dumps(steps))
|
|
|
|
|
|
def session():
|
|
flag = "--session-id" if "--session-id" in argv else "--resume"
|
|
return argv[argv.index(flag) + 1]
|
|
|
|
|
|
def transcript():
|
|
return pathlib.Path.home() / ".claude" / "projects" / "-fake" / f"{{session()}}.jsonl"
|
|
|
|
|
|
def append(*entries):
|
|
transcript().parent.mkdir(parents=True, exist_ok=True)
|
|
with open(transcript(), "a") as handle:
|
|
handle.writelines(json.dumps(entry) + "\\n" for entry in entries)
|
|
|
|
|
|
if role == "worker":
|
|
if "--session-id" in argv and step.get("transcript", True):
|
|
append({{"type": "custom-title"}}, {{"type": "queue-operation", "timestamp": str(time.time_ns())}})
|
|
said = {{"type": "assistant", "message": {{"content": [{{"type": "text", "text": step.get("say", "working")}}]}}}}
|
|
if step.get("transcript", True):
|
|
append({{"type": "user", "promptSource": "sdk", "message": {{"content": argv[argv.index("-p") + 1]}}}})
|
|
with open(transcript(), "ab") as handle:
|
|
handle.write(step.get("raw", "").encode("latin-1"))
|
|
append(said)
|
|
if step.get("siblings"):
|
|
for sibling in (f"{{session()}}/subagents", f"{{session()}}.superseded-1/tool-results"):
|
|
(transcript().parent / sibling).mkdir(parents=True, exist_ok=True)
|
|
(transcript().parent / f"{{session()}}.orphaned-1-ab.jsonl").write_text("{{}}\\n")
|
|
if "append" in step:
|
|
edited = pathlib.Path(os.environ["FAKE_CLAUDE_EDIT"], step["append"][0])
|
|
edited.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(edited, "a") as script:
|
|
script.write(step["append"][1])
|
|
if "replace" in step:
|
|
script = pathlib.Path(os.environ["FAKE_CLAUDE_EDIT"], step["replace"][0])
|
|
script.write_text(script.read_text().replace(*step["replace"][1:]))
|
|
sys.stdout.flush()
|
|
sys.stdout.buffer.write(step.get("stream", "").encode("latin-1"))
|
|
print(json.dumps(said))
|
|
if step.get("result", True):
|
|
print(json.dumps({{"type": "result"}}))
|
|
if "signal" in step:
|
|
os.kill(os.getpid(), step["signal"])
|
|
elif role == "judge":
|
|
print(step["stdout"] if "stdout" in step else json.dumps({{"structured_output": step["verdict"]}}))
|
|
print(step.get("stderr", ""), file=sys.stderr)
|
|
else:
|
|
if step.get("ctrl_c"):
|
|
# A Ctrl-C on the terminal reaches every process in its foreground, loop-claude too. kill returns once the signal is pending in loop-claude, before this process exits and loop-claude stops waiting for it.
|
|
os.kill(os.getppid(), signal.SIGINT)
|
|
if "type" in step:
|
|
append({{"type": "user", "message": {{"content": step["type"]}}}})
|
|
if step.get("unreadable"):
|
|
transcript().chmod(0)
|
|
if step.get("vanish"):
|
|
os.remove(__file__)
|
|
sys.exit(step.get("exit", 0))
|
|
"""
|
|
|
|
# Records every start of loop-claude and turns its retry waits into log lines.
|
|
SITECUSTOMIZE = """import json, os, sys, time
|
|
|
|
if os.path.basename(sys.argv[0]) == "loop-claude":
|
|
def record(event):
|
|
with open(os.environ["LOOP_TEST_LOG"], "a") as log:
|
|
log.write(json.dumps(event) + "\\n")
|
|
record({"argv": sys.argv[1:]})
|
|
pause = time.sleep
|
|
time.sleep = lambda seconds: record({"sleep": seconds}) if seconds >= 1 else pause(seconds)
|
|
# The host's own memory doesn't matter: every read finds all of it available.
|
|
sys.path.insert(0, os.path.dirname(os.path.realpath(sys.argv[0])))
|
|
import pathlib
|
|
from machine_load import memory
|
|
memory.MEMINFO = pathlib.Path(os.environ["FAKE_MEMINFO"])
|
|
"""
|
|
|
|
PREMISES = "# Project\n\n## Goals\n\n1. Work.\n\n## Audience\n\nMe.\n"
|
|
# Item 4's L of 4 steps one rung up from the project's.
|
|
TODO = "## Items\n\n| ID | Release | Exempt | Item | L | E | I | R | S | Moves | Score |\n|---|---|---|---|---|---|---|---|---|---|---|\n| 4 | 0.1.0 | | **Do it.** | 4 | 2 | 3 | 7 | 2 | 1 by 9 | 25 |\n"
|
|
# Item 4's L of 2 steps one rung down.
|
|
CHEAP = TODO.replace("| 4 | 2 | 3 | 7 |", "| 2 | 1 | 2 | 3 |")
|
|
# Item 9 follows CHEAP's item 4, and a worker routed by item 4 may take it in: at L 4 it picks a higher rung for the worker, and at E + I 6 for the reviewers. At E + I 3, or at L 2, it picks the rung item 4 picks.
|
|
TAKEN_IN_AT_L_4 = CHEAP + "| 9 | 0.1.0 | | **Do more.** | 4 | 1 | 2 | 5 | 2 | 1 by 9 | 20 |\n"
|
|
TAKEN_IN_AT_E_PLUS_I_6 = CHEAP + "| 9 | 0.1.0 | | **Do more.** | 2 | 3 | 3 | 6 | 2 | 1 by 9 | 20 |\n"
|
|
|
|
|
|
# Waits for the interrupt's prompt before typing the answer, since loop-claude drops what was typed before it.
|
|
ANSWERING = ("Answer (", lambda: None)
|
|
|
|
|
|
def worker(**step):
|
|
return {"role": "worker", **step}
|
|
|
|
|
|
def judge(interrupt=False, finished=False, evidence="working", question="", escalation="", release_ready=False, **step):
|
|
verdict = {"answer": "", "escalation": escalation, "evidence": evidence, "finished": finished, "interrupt": interrupt, "question": question, "answered_before": "", "left_open": "", "release_ready": release_ready, "reversed": ""}
|
|
return {"role": "judge", "verdict": verdict, **step}
|
|
|
|
|
|
def interrupt(**step):
|
|
return {"role": "interrupt", **step}
|
|
|
|
|
|
def lines(path):
|
|
return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else []
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class Run:
|
|
calls: list
|
|
code: int
|
|
sleeps: list
|
|
starts: list
|
|
# stderr and stdout are without the local time that opens each message loop-claude prints; raw_stderr and raw_stdout keep it.
|
|
stderr: str
|
|
stdout: str
|
|
raw_stderr: str
|
|
raw_stdout: str
|
|
|
|
def roles(self):
|
|
return [call["role"] for call in self.calls]
|
|
|
|
|
|
class Checkout:
|
|
"""A temporary HOME with a git checkout at ~/code/proj, where loop-claude runs against the fake claude."""
|
|
|
|
def __init__(self, test):
|
|
self.test = test
|
|
temp = tempfile.TemporaryDirectory()
|
|
test.addCleanup(temp.cleanup)
|
|
self.temp = pathlib.Path(os.path.realpath(temp.name))
|
|
self.home = self.temp / "home"
|
|
self.root = self.home / "code" / "proj"
|
|
self.root.mkdir(parents=True)
|
|
(self.root / "README.md").write_text(PREMISES)
|
|
write_config(self.home, self.root)
|
|
self.script = BIN / "loop-claude"
|
|
self.env = isolated_env()
|
|
fake = self.temp / "fake-bin"
|
|
fake.mkdir()
|
|
(fake / "claude").write_text(FAKE_CLAUDE.format(python=sys.executable))
|
|
(fake / "claude").chmod(0o755)
|
|
(fake / "git").symlink_to(shutil.which("git"))
|
|
site = self.temp / "site"
|
|
site.mkdir()
|
|
(site / "sitecustomize.py").write_text(SITECUSTOMIZE)
|
|
(self.temp / "meminfo").write_text("MemTotal: 16 kB\nMemAvailable: 16 kB\n")
|
|
self.env.update({
|
|
"FAKE_CLAUDE_LOG": str(self.temp / "calls.jsonl"),
|
|
"FAKE_MEMINFO": str(self.temp / "meminfo"),
|
|
"FAKE_CLAUDE_SCENARIO": str(self.temp / "scenario.json"),
|
|
"GIT_CONFIG_GLOBAL": os.devnull,
|
|
"GIT_CONFIG_NOSYSTEM": "1",
|
|
"HOME": str(self.home),
|
|
"LOOP_CLAUDE_JUDGE_MODEL": "judge-model",
|
|
"LOOP_CLAUDE_WORKER_MODEL": "worker-model",
|
|
"LOOP_TEST_LOG": str(self.temp / "starts.jsonl"),
|
|
"PATH": str(fake),
|
|
"PYTHONPATH": str(site),
|
|
})
|
|
self.git("init", "--quiet")
|
|
|
|
def git(self, *args):
|
|
subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@example.com", *args], capture_output=True, check=True, cwd=self.root, env=self.env)
|
|
|
|
def commit_plan(self, todo):
|
|
"""Commits todo as todo.md on the checkout's branch, the default branch routing reads it from."""
|
|
(self.root / "todo.md").write_text(todo)
|
|
self.git("add", "todo.md")
|
|
self.git("commit", "--quiet", "-m", "plan")
|
|
|
|
def run(self, steps, argv=(TASK,), cwd=None, terminal_input=None, stdout_on_terminal=True, before_typing=None, typed_early=""):
|
|
for name in ("FAKE_CLAUDE_LOG", "LOOP_TEST_LOG"):
|
|
pathlib.Path(self.env[name]).unlink(missing_ok=True)
|
|
pathlib.Path(self.env["FAKE_CLAUDE_SCENARIO"]).write_text(json.dumps(steps))
|
|
command = [sys.executable, str(self.script), *argv]
|
|
cwd = cwd or self.root
|
|
if terminal_input is None:
|
|
done = subprocess.run(command, capture_output=True, cwd=cwd, env=self.env, stdin=subprocess.DEVNULL, text=True, timeout=60)
|
|
code, stdout, stderr = done.returncode, done.stdout, done.stderr
|
|
else:
|
|
code, stdout, stderr = self.run_on_terminal(command, cwd, terminal_input, stdout_on_terminal, before_typing, typed_early)
|
|
started = lines(pathlib.Path(self.env["LOOP_TEST_LOG"]))
|
|
return Run(
|
|
calls=lines(pathlib.Path(self.env["FAKE_CLAUDE_LOG"])),
|
|
code=code,
|
|
sleeps=[event["sleep"] for event in started if "sleep" in event],
|
|
starts=[event["argv"] for event in started if "argv" in event],
|
|
stderr=unstamped(stderr),
|
|
stdout=unstamped(stdout),
|
|
raw_stderr=stderr,
|
|
raw_stdout=stdout,
|
|
)
|
|
|
|
def run_on_terminal(self, command, cwd, typed, stdout_on_terminal, before_typing, typed_early):
|
|
"""Types typed_early at once, then the input at once, or once stderr shows before_typing's text and its callback has run."""
|
|
controller, terminal = pty.openpty()
|
|
with tempfile.TemporaryFile() as stderr, tempfile.TemporaryFile() as stdout:
|
|
try:
|
|
proc = subprocess.Popen(command, cwd=cwd, env=self.env, stderr=stderr, stdin=terminal, stdout=terminal if stdout_on_terminal else stdout)
|
|
finally:
|
|
os.close(terminal)
|
|
try:
|
|
os.write(controller, typed_early.encode())
|
|
prompt, callback = before_typing or (None, None)
|
|
if prompt is None:
|
|
os.write(controller, typed.encode())
|
|
output, deadline = b"", time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
if select.select([controller], [], [], 0.05)[0]:
|
|
try:
|
|
chunk = os.read(controller, 4096)
|
|
except OSError:
|
|
break
|
|
if not chunk:
|
|
break
|
|
output += chunk
|
|
if prompt is not None and prompt.encode() in os.pread(stderr.fileno(), 1 << 20, 0):
|
|
callback()
|
|
os.write(controller, typed.encode())
|
|
prompt = None
|
|
code = proc.wait(timeout=10)
|
|
finally:
|
|
os.close(controller)
|
|
proc.kill()
|
|
proc.wait()
|
|
stderr.seek(0)
|
|
stdout.seek(0)
|
|
output += stdout.read()
|
|
return code, output.decode(errors="replace").replace("\r\n", "\n"), stderr.read().decode()
|
|
|
|
def claude_json(self):
|
|
return self.home / ".claude.json"
|
|
|
|
def models_file(self):
|
|
return self.home / ".claude" / "loop-claude-models.json"
|
|
|
|
def transcripts(self):
|
|
return sorted((self.home / ".claude" / "projects").glob("*/*.jsonl"))
|
|
|
|
def archive(self):
|
|
return pathlib.Path(self.env.get("LOOP_CLAUDE_TRANSCRIPTS", self.home / ".claude" / "loop-claude-transcripts"))
|
|
|
|
def archived(self):
|
|
"""Each archived transcript's outcomes, with every UTC stamp replaced by <UTC> and every setup version by <config>, after checking the copy is named by its content hash."""
|
|
outcomes = {}
|
|
for folder in (self.archive(), self.archive() / "proj", *(self.archive() / "proj").iterdir()):
|
|
self.test.assertEqual(stat.S_IMODE(folder.stat().st_mode), 0o700, folder)
|
|
for copy in sorted((self.archive() / "proj").glob("*/*.jsonl")):
|
|
self.test.assertEqual(copy.stem, hashlib.sha256(copy.read_bytes()).hexdigest())
|
|
kept = copy.with_suffix(".outcomes.md")
|
|
for archived in (copy, kept):
|
|
self.test.assertEqual(stat.S_IMODE(archived.stat().st_mode), 0o600)
|
|
stamped = re.sub(r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ ", "<UTC> ", kept.read_text(), flags=re.MULTILINE)
|
|
outcomes[copy.read_text()] = re.sub(r"^Config: .*$", "Config: <config>", stamped, flags=re.MULTILINE)
|
|
return outcomes
|
|
|
|
|
|
STAMP = r"\d{4}-\d\d-\d\d \d\d:\d\d:\d\d"
|
|
|
|
|
|
def unstamped(text):
|
|
return re.sub(rf"(?m)^{STAMP} ", "", text)
|
|
|
|
|
|
def argument(call, flag):
|
|
return call["argv"][call["argv"].index(flag) + 1] if flag in call["argv"] else None
|
|
|
|
|
|
class Premises(unittest.TestCase):
|
|
def test_refuses_to_start_without_a_task_a_checkout_or_premises(self):
|
|
checkout = Checkout(self)
|
|
for argv, code, said in (([], 1, "usage: loop-claude <task>"), (["--help"], 0, "usage: loop-claude <task>"), (["-h"], 0, "usage: loop-claude <task>")):
|
|
with self.subTest(argv=argv):
|
|
run = checkout.run([], argv=argv)
|
|
self.assertEqual(run.code, code)
|
|
self.assertIn(said, run.stdout if code == 0 else run.stderr)
|
|
outside = checkout.temp / "outside"
|
|
outside.mkdir()
|
|
run = checkout.run([], cwd=outside)
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertRegex(run.raw_stderr, rf"(?m)^{STAMP} .*not a git repository")
|
|
readmes = {
|
|
None: "README.md in {root}: No such file or directory - no premises.",
|
|
b"# P\n## Goals\n1. g\xff\n## Audience\nme\n": "README.md in {root} is not UTF-8 on line 3 - no premises.",
|
|
b"# P\n## Audience\nme\n": "{root}/README.md has no ## Goals heading - no premises.",
|
|
b"# P\n## Goals\n1. g\n": "{root}/README.md has no ## Audience heading - no premises.",
|
|
b"# P\n## Goals\n\n## Audience\nme\n": "{root}/README.md has no text under ## Goals - no premises.",
|
|
b"# P\n## Goals\n1. g\n## Audience\nme\n## Audience\nyou\n": "{root}/README.md has ## Audience more than once - no premises.",
|
|
}
|
|
(checkout.root / "sub").mkdir()
|
|
for text, said in readmes.items():
|
|
with self.subTest(said=said):
|
|
readme = checkout.root / "README.md"
|
|
readme.unlink(missing_ok=True)
|
|
if text is not None:
|
|
readme.write_bytes(text)
|
|
run = checkout.run([worker()], cwd=checkout.root / "sub")
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertEqual(run.stderr.strip(), said.format(root=checkout.root))
|
|
(checkout.root / "README.md").unlink()
|
|
(checkout.root / "README.md").mkdir()
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertEqual(run.stderr.strip(), f"README.md in {checkout.root}: Is a directory - no premises.")
|
|
(checkout.home / ".claude-projects" / "proj" / "config.toml").unlink()
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertEqual(run.stderr.strip(), f"no config names the checkout {checkout.root}: write ~/.claude-projects/<key>/config.toml, as `project-config --help` shows - no premises.")
|
|
(checkout.temp / "fake-bin" / "git").unlink()
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertTrue(run.stderr.startswith("git rev-parse --show-toplevel could not start: "), run.stderr)
|
|
|
|
def test_reads_the_premises_from_the_file_the_config_names(self):
|
|
checkout = Checkout(self)
|
|
checkout.commit_plan(TODO)
|
|
(checkout.root / "README.md").write_text("# Project\n")
|
|
folder = write_config(checkout.home, checkout.root, goals="folder:goals.md")
|
|
(folder / "goals.md").write_text("## Goals\n1. Kept apart.\n\n## Audience\nThe client.\n")
|
|
run = checkout.run([worker()])
|
|
self.assertTrue(run.calls, run.stderr)
|
|
self.assertIn("## Goals\n1. Kept apart.\n\n## Audience\nThe client.", argument(run.calls[0], "-p"))
|
|
|
|
def test_makes_no_worktree_for_a_readme_without_premises(self):
|
|
checkout = Checkout(self)
|
|
checkout.commit_plan(TODO)
|
|
(checkout.root / "README.md").write_text("# Project\n")
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertFalse((checkout.home / "code" / "proj-item-4").exists())
|
|
|
|
def test_names_the_session_by_the_checkout_relative_to_code_or_else_to_the_root(self):
|
|
checkout = Checkout(self)
|
|
for root in (checkout.temp / "elsewhere", checkout.home / "code"):
|
|
with self.subTest(root=root):
|
|
root.mkdir(exist_ok=True)
|
|
(root / "README.md").write_text(PREMISES)
|
|
write_config(checkout.home, root)
|
|
checkout.git("init", "--quiet", str(root))
|
|
run = checkout.run([worker()], cwd=root)
|
|
self.assertEqual(argument(run.calls[0], "--name"), f"A {str(root).lstrip('/')} loop-claude")
|
|
|
|
|
|
class Cycles(unittest.TestCase):
|
|
def test_resumes_the_session_while_unfinished_and_starts_it_fresh_after_finishing(self):
|
|
checkout = Checkout(self)
|
|
(checkout.root / "sub").mkdir()
|
|
(checkout.root / "docs").mkdir()
|
|
(checkout.root / "docs" / "decisions.md").write_text("# Decisions\n")
|
|
(checkout.home / ".claude").mkdir()
|
|
(checkout.home / ".claude" / "CLAUDE.md").write_text("# Global instructions\n")
|
|
run = checkout.run([
|
|
worker(say="started"), judge(evidence="started"),
|
|
worker(say="landed-it", siblings=True), judge(finished=True, evidence="landed-it"),
|
|
worker(say="next chunk"),
|
|
], cwd=checkout.root / "sub")
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "worker", "judge", "worker", "judge"])
|
|
workers = [call for call in run.calls if call["role"] == "worker"]
|
|
session = argument(workers[0], "--session-id")
|
|
self.assertEqual(argument(workers[0], "--name"), "A proj loop-claude")
|
|
self.assertEqual(argument(workers[1], "--resume"), session)
|
|
self.assertIsNone(argument(workers[1], "--session-id"))
|
|
self.assertIsNone(argument(workers[1], "--name"))
|
|
self.assertEqual(argument(workers[2], "--session-id"), session)
|
|
self.assertTrue(argument(workers[0], "-p").startswith(f"{TASK}\n\nContinue any open draft PR"))
|
|
self.assertTrue(argument(workers[0], "-p").endswith("The chunk's todo.md items, or else the task above, set its scope. The goals below, " 'which rank after code health (~/.claude/CLAUDE.md → "Shared goal") unless the goals file voids it' ", choose between ways to deliver that scope, and never widen it.\n\nFrom the goals file, " f"{checkout.root}/README.md:\n## Goals\n1. Work.\n\n## Audience\nMe."))
|
|
self.assertIn("main or master", argument(workers[0], "-p"))
|
|
self.assertNotIn(TASK, argument(workers[1], "-p"))
|
|
self.assertIn("Even then, it is finished only once every answer the human typed in it", argument(workers[1], "-p"))
|
|
self.assertIn("as the work remaining:\n\n> started\n\n", argument(workers[1], "-p"))
|
|
self.assertEqual({call["cwd"] for call in run.calls}, {str(checkout.root)})
|
|
self.assertIn("1. Work.", run.calls[1]["stdin"])
|
|
self.assertIn("From docs/decisions.md:\n# Decisions\n\nFrom ~/.claude/CLAUDE.md:\n# Global instructions", run.calls[1]["stdin"])
|
|
self.assertEqual({argument(call, "--model") for call in workers}, {"worker-model"})
|
|
self.assertEqual({argument(call, "--model") for call in run.calls if call["role"] == "judge"}, {"judge-model"})
|
|
self.assertIn("landed-it", run.calls[3]["stdin"])
|
|
self.assertNotIn("landed-it", run.calls[5]["stdin"])
|
|
self.assertEqual([path.name for path in checkout.transcripts()[0].parent.iterdir()], [checkout.transcripts()[0].name])
|
|
self.assertIn("=== cycle 3, worker on worker-model ===", run.stdout)
|
|
self.assertEqual(len(re.findall(rf"(?m)^{STAMP} === cycle 3, worker on worker-model ===$", run.raw_stdout)), 1)
|
|
copies = {verdict.removeprefix(f"Task: {TASK}\n\n"): copy for copy, verdict in checkout.archived().items()}
|
|
self.assertEqual(sorted(copies), [f"<UTC> Stopped: Ctrl-C\n\nWorker: {OFF_LADDER}\n", f"<UTC> Verdict: finished\n\nEvidence:\n> landed-it\n\nConfig: <config>\nJudge: judge-model\nWorker: {OFF_LADDER}\n", f"<UTC> Verdict: unfinished\n\nEvidence:\n> started\n\nConfig: <config>\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
|
|
self.assertIn('"text": "landed-it"', copies[f"<UTC> Verdict: finished\n\nEvidence:\n> landed-it\n\nConfig: <config>\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
|
|
self.assertIn('"text": "next chunk"', copies[f"<UTC> Stopped: Ctrl-C\n\nWorker: {OFF_LADDER}\n"])
|
|
self.assertEqual(len(list(checkout.archive().glob(f"proj/*/session/{session}.orphaned-1-ab.jsonl"))), 1)
|
|
|
|
def test_routes_the_worker_by_its_items_l_and_steps_it_up_within_the_chunk(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(TAKEN_IN_AT_L_4)
|
|
run = checkout.run([
|
|
worker(say="one"), judge(evidence="one"),
|
|
worker(say="two"), judge(evidence="two"),
|
|
worker(say="three\nChunk items: 4 (L 2, E + I 3), 9 (L 4, E + I 5)"), judge(evidence="three"),
|
|
worker(say="four"), judge(finished=True, evidence="four"),
|
|
worker(say="next chunk"),
|
|
])
|
|
self.assertEqual(run.code, 130)
|
|
workers = [call for call in run.calls if call["role"] == "worker"]
|
|
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in workers], [
|
|
("claude-sonnet-5-5", "high"),
|
|
("claude-sonnet-5-5", "high"),
|
|
("claude-opus-5-5", "medium"),
|
|
("claude-fable-5-2", "low"),
|
|
("claude-sonnet-5-5", "high"),
|
|
])
|
|
self.assertEqual({(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "judge"}, {("judge-model", None)})
|
|
self.assertIn("loop-claude runs you on claude-sonnet-5-5 at high effort, routed by item 4's L of 2.", argument(workers[0], "-p"))
|
|
self.assertIn("on one line, as in `Chunk items: ", argument(workers[0], "-p"))
|
|
outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values())
|
|
self.assertIn('<UTC> Verdict: unfinished\n\nEvidence:\n> three\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-opus-5-5", "effort": "medium", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n\n<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n', outcomes)
|
|
self.assertIn('<UTC> Verdict: finished\n\nEvidence:\n> four\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n', outcomes)
|
|
|
|
def test_routes_on_the_ladder_the_projects_config_folder_holds_in_place_of_the_shipped_one(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-fable-5-2"
|
|
ladder = checkout.home / ".claude-projects" / "proj" / "routing-ladder.json"
|
|
ladder.write_text('[{"model": "claude-fable-5-2", "effort": "low"}, {"model": "claude-fable-5-2", "effort": "max"}]')
|
|
checkout.commit_plan(CHEAP)
|
|
run = checkout.run([worker(), judge(), worker(), judge(), worker()])
|
|
self.assertEqual(run.code, 130)
|
|
# Item 4's L of 2 steps one rung down, to Opus 5.5 on the shipped ladder; the project's holds no rung below its own, and one above it, which the second unfinished verdict steps to.
|
|
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "worker"], [("claude-fable-5-2", "low"), ("claude-fable-5-2", "low"), ("claude-fable-5-2", "max")])
|
|
|
|
def test_reads_the_projects_ladder_afresh_each_cycle_and_exits_1_once_it_holds_a_fault(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-fable-5-2"
|
|
ladder = checkout.home / ".claude-projects" / "proj" / "routing-ladder.json"
|
|
ladder.write_text('[{"model": "claude-fable-5-2", "effort": "high"}]')
|
|
checkout.env["FAKE_CLAUDE_EDIT"] = str(ladder.parent)
|
|
run = checkout.run([worker(replace=["routing-ladder.json", '"high"}', '"high"},']), judge()])
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge"]))
|
|
self.assertEqual((argument(run.calls[0], "--model"), argument(run.calls[0], "--effort")), ("claude-fable-5-2", "high"))
|
|
self.assertIn(f"cannot read the routing ladder {ladder}: ", run.stderr)
|
|
|
|
def test_routes_on_and_states_the_same_item_from_the_fetched_default_branch_whatever_the_checkout_holds(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
origin = checkout.temp / "origin.git"
|
|
checkout.git("init", "--quiet", "--bare", "--initial-branch=main", str(origin))
|
|
checkout.git("symbolic-ref", "HEAD", "refs/heads/main")
|
|
checkout.commit_plan(TODO)
|
|
checkout.git("remote", "add", "origin", str(origin))
|
|
checkout.git("push", "--quiet", "--set-upstream", "origin", "main")
|
|
# Item 4 ships elsewhere, and item 6 is next; the checkout is behind origin, and holds a change of its own.
|
|
elsewhere = checkout.temp / "elsewhere"
|
|
checkout.git("clone", "--quiet", str(origin), str(elsewhere))
|
|
(elsewhere / "todo.md").write_text(TODO.replace("| 4 | 0.1.0 |", "| 6 | 0.1.0 |"))
|
|
checkout.git("-C", str(elsewhere), "commit", "--quiet", "--all", "-m", "item 4 shipped, plan item 6")
|
|
checkout.git("-C", str(elsewhere), "push", "--quiet", "origin", "main")
|
|
(checkout.root / "todo.md").write_text(TODO + "\nA change of the checkout's own.\n")
|
|
(checkout.root / "sub").mkdir()
|
|
run = checkout.run([worker()], cwd=checkout.root / "sub")
|
|
self.assertEqual(run.code, 130)
|
|
prompt = argument(run.calls[0], "-p")
|
|
self.assertIn("routed by item 6's L of 4", prompt)
|
|
self.assertIn("item 6, L 4, E + I 5", prompt)
|
|
self.assertTrue((checkout.home / "code" / "proj-item-6").is_dir())
|
|
|
|
def test_lists_itself_in_loop_inbox_with_its_routed_item_then_those_the_worker_names_on_its_chunk_items_line_until_the_chunk_finishes(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(TODO)
|
|
run = checkout.run([worker(say="working\nChunk items: 7, 4"), judge(), worker(), judge(finished=True), worker()])
|
|
self.assertEqual(run.code, 130)
|
|
routed, taken_in = {"id": 4, "tagline": "Do it."}, {"id": 7, "tagline": ""}
|
|
self.assertEqual([call["running"] for call in run.calls], [[[routed]], [[routed]], [[routed, taken_in]], [[routed, taken_in]], [[routed]], [[routed]]])
|
|
self.assertEqual(list((checkout.home / ".local" / "state" / "loop-claude").glob("*.running")), [])
|
|
|
|
def test_steps_the_worker_up_a_rung_when_the_judge_finds_in_the_transcript_the_escalation_trigger_it_cites(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(TODO)
|
|
run = checkout.run([
|
|
worker(say="Stability round 2: a 🔴. Trigger 3 fired; I ask for a higher rung."), judge(evidence="I ask for a higher rung.", escalation="Stability round 2: a 🔴."),
|
|
worker(say="Trigger 2 fired; I ask for a higher rung."), judge(evidence="I ask for a higher rung.", escalation="Trigger 2 fired; … a higher rung."),
|
|
worker(say="merged"), judge(finished=True, evidence="merged"),
|
|
worker(say="next chunk"),
|
|
])
|
|
self.assertEqual(run.code, 130)
|
|
workers = [call for call in run.calls if call["role"] == "worker"]
|
|
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in workers], [
|
|
("claude-fable-5-2", "low"),
|
|
("claude-fable-5-2", "medium"),
|
|
("claude-fable-5-2", "medium"),
|
|
("claude-fable-5-2", "low"),
|
|
])
|
|
outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values())
|
|
self.assertIn('<UTC> Verdict: unfinished\n\nEvidence:\n> I ask for a higher rung.\n\nEscalation:\n> Stability round 2: a 🔴.\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 4, "escape_and_impact": 5}\n\n<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', outcomes)
|
|
self.assertIn('<UTC> Verdict: unfinished\n\nEvidence:\n> I ask for a higher rung.\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', outcomes)
|
|
|
|
def test_steps_the_worker_up_once_as_the_judge_interrupts_when_the_run_also_cites_an_escalation_trigger(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(TODO)
|
|
cited = "Stability round 2: a 🔴."
|
|
run = checkout.run([
|
|
worker(say=f"{cited} Trigger 3 fired; I ask for a higher rung. Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?", escalation=cited),
|
|
interrupt(type="B"), judge(evidence="B", escalation="Trigger 3 fired; I ask for a higher rung."),
|
|
worker(say="carrying on"),
|
|
], terminal_input="/talk\n", before_typing=ANSWERING)
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "interrupt", "judge", "worker", "judge"])
|
|
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] != "judge"], [("claude-fable-5-2", "low"), ("claude-fable-5-2", "medium"), ("claude-fable-5-2", "medium")])
|
|
outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values())
|
|
self.assertIn('<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', "".join(outcomes))
|
|
self.assertNotIn("at max effort", "".join(outcomes))
|
|
self.assertIn('Verdict: interrupt\n\nA or B?\n\nEvidence:\n> Shall I use A or B?\n\nEscalation:\n> Stability round 2: a 🔴.\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 4, "escape_and_impact": 5}\n\n<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', "".join(outcomes))
|
|
|
|
def test_runs_a_cheap_items_worker_below_the_projects_rung_with_its_reviewers_on_it_and_its_judge_on_the_models_highest(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(CHEAP)
|
|
run = checkout.run([worker(say="one"), judge(finished=True, evidence="one"), worker(say="next chunk")])
|
|
self.assertEqual(run.code, 130)
|
|
first = next(call for call in run.calls if call["role"] == "worker")
|
|
self.assertEqual((argument(first, "--model"), argument(first, "--effort")), ("claude-sonnet-5-5", "high"))
|
|
ruling = next(call for call in run.calls if call["role"] == "judge")
|
|
self.assertEqual((argument(ruling, "--model"), argument(ruling, "--effort")), ("claude-opus-5-5", "xhigh"))
|
|
self.assertIn('Judge: claude-opus-5-5 at xhigh effort\nWorker: {"model": "claude-sonnet-5-5", "effort": "high", ', "".join(checkout.archived().values()))
|
|
self.assertIn("Launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool set to model opus, effort medium, the project's rung.", argument(first, "-p"))
|
|
|
|
def test_routes_the_reviewers_on_the_highest_e_plus_i_the_worker_takes_in_and_keeps_the_judge_on_the_models_highest(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(TAKEN_IN_AT_E_PLUS_I_6)
|
|
run = checkout.run([
|
|
worker(say="one\nChunk items: 4, 9"), judge(evidence="one"),
|
|
worker(say="two"), judge(finished=True, evidence="two"),
|
|
worker(say="next chunk"),
|
|
])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "judge"], [("claude-opus-5-5", "xhigh")] * 3)
|
|
workers = [call for call in run.calls if call["role"] == "worker"]
|
|
self.assertEqual({argument(call, "--effort") for call in workers}, {"high"})
|
|
self.assertIn("Launch each reviewer with the Agent tool set to model fable, effort medium, the rung picked by an item of E + I 6 taken into the chunk.", argument(workers[1], "-p"))
|
|
outcomes = "".join(checkout.archived().values())
|
|
self.assertIn('<UTC> Stepped up\n\nWorker: {"model": "claude-sonnet-5-5", "effort": "high", "item": 4, "likelihood": 2, "escape_and_impact": 3, "escape_and_impact_taken_in": 6}\n', outcomes)
|
|
self.assertIn('Judge: claude-opus-5-5 at xhigh effort\nWorker: {"model": "claude-sonnet-5-5", "effort": "high", "item": 4, "likelihood": 2, "escape_and_impact": 3, "escape_and_impact_taken_in": 6}\n', outcomes)
|
|
|
|
def test_a_rerun_resumes_an_open_chunk_on_the_rung_it_last_ran(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(TODO)
|
|
run = checkout.run([worker(), judge(), worker(), judge()])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual((argument(run.calls[-1], "--model"), argument(run.calls[-1], "--effort")), ("claude-fable-5-2", "medium"))
|
|
checkout.git("rm", "--quiet", "todo.md")
|
|
checkout.git("commit", "--quiet", "-m", "drop the plan")
|
|
run = checkout.run([worker()])
|
|
self.assertIsNotNone(argument(run.calls[0], "--resume"))
|
|
self.assertEqual((argument(run.calls[0], "--model"), argument(run.calls[0], "--effort")), ("claude-fable-5-2", "medium"))
|
|
|
|
def test_exits_1_when_the_transcript_cannot_be_archived_and_resumes_on_the_next_run(self):
|
|
checkout = Checkout(self)
|
|
checkout.archive().parent.mkdir(parents=True)
|
|
checkout.archive().write_text("in the way")
|
|
run = checkout.run([worker(say="landed-it"), judge(finished=True, evidence="landed-it")])
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge"]))
|
|
self.assertRegex(run.stderr.strip().splitlines()[-1], r"^could not archive the transcript: .*/loop-claude-transcripts/proj: Not a directory$")
|
|
self.assertEqual(run.stderr.count("could not archive the transcript"), 1)
|
|
session = argument(run.calls[0], "--session-id")
|
|
checkout.archive().unlink()
|
|
run = checkout.run([worker(), judge(finished=True)])
|
|
self.assertEqual(argument(run.calls[0], "--resume"), session)
|
|
self.assertEqual(len(checkout.archived()), 1)
|
|
|
|
def test_stops_after_three_failures_running_retrying_a_minute_apart(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(exit=1), worker(result=False), worker(signal=9)])
|
|
self.assertEqual(run.code, 1)
|
|
self.assertEqual(run.roles(), ["worker"] * 3)
|
|
self.assertEqual(run.sleeps, [60, 60])
|
|
self.assertIn("retrying the worker in a minute: claude exited 1", run.stderr)
|
|
self.assertIn("claude exited 0 without a result event", run.stderr)
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the worker failed three times running: claude killed by signal 9")
|
|
self.assertNotIn("=== cycle 2,", run.stdout)
|
|
self.assertEqual(argument(run.calls[1], "--resume"), argument(run.calls[0], "--session-id"))
|
|
|
|
run = checkout.run([worker(exit=1), worker(exit=1), worker(), judge(), worker(exit=1), worker(exit=1)])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.sleeps, [60, 60, 60, 60])
|
|
|
|
run = checkout.run([worker(), judge(exit=1, stderr="judge down"), judge(stdout=""), judge(verdict={"interrupt": False})])
|
|
self.assertEqual(run.code, 1)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "judge", "judge"])
|
|
self.assertEqual(run.sleeps, [60, 60])
|
|
self.assertIn("retrying the judge in a minute: judge down", run.stderr)
|
|
self.assertIn("retrying the judge in a minute: the judge printed no json reply; it printed: nothing\n", run.stderr)
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], 'the judge failed three times running: the judge\'s reply has no answer of type string')
|
|
|
|
def test_waits_out_a_usage_limit_which_ends_a_run_of_failures(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["TZ"] = "UTC"
|
|
resets_at = int(time.time()) + 3600
|
|
rejected = json.dumps({"type": "rate_limit_event", "rate_limit_info": {"status": "rejected", "resetsAt": resets_at}})
|
|
limited = json.dumps({"type": "result", "api_error_status": 429, "is_error": True, "result": "You've hit your session limit"})
|
|
run = checkout.run([worker(exit=1), worker(exit=1), worker(stream=f"{rejected}\n{limited}\n", result=False, exit=1), worker(exit=1), worker(), judge(stdout=limited, exit=1), judge()])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker"] * 5 + ["judge"] * 2 + ["worker"])
|
|
self.assertEqual([round(seconds, -1) for seconds in run.sleeps], [60, 60, 3720, 60, 600])
|
|
self.assertIn(f"the worker hit a usage limit (You've hit your session limit); retrying at {datetime.datetime.fromtimestamp(resets_at + 120, datetime.timezone.utc):%Y-%m-%d %H:%M:%S}\n", run.stderr)
|
|
self.assertIn("the judge hit a usage limit (You've hit your session limit); retrying at ", run.stderr)
|
|
|
|
def test_retries_the_judge_alone_when_its_verdict_quotes_nothing_in_the_transcript(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([
|
|
worker(say="Shall I use A or B?"),
|
|
judge(interrupt=True, evidence="never said", question="A or B?"),
|
|
judge(finished=True, evidence=""),
|
|
judge(evidence="A or B"),
|
|
worker(),
|
|
])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "judge", "judge", "worker", "judge"])
|
|
self.assertEqual(run.sleeps, [60, 60])
|
|
self.assertIn("the judge quoted 'never said', which is not in the transcript, for the question: A or B?", run.stderr)
|
|
self.assertIn("retrying the judge in a minute: the judge quoted '', which is not in the transcript\n", run.stderr)
|
|
|
|
def test_retries_a_judge_claude_cannot_start(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(vanish=True)])
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
|
|
self.assertEqual(run.sleeps, [60, 60])
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the judge failed three times running: could not start claude: No such file or directory")
|
|
|
|
def test_hands_the_judge_the_tail_of_a_long_transcript(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(say="early " * 80000 + "LATE")])
|
|
self.assertIn("LATE", run.calls[1]["stdin"])
|
|
self.assertEqual(len(run.calls[1]["stdin"]), 400000)
|
|
self.assertLess(run.calls[1]["stdin"].count("early"), 80000)
|
|
self.assertTrue(run.calls[1]["stdin"].rstrip().endswith("CI not yet reported, an answer not yet recorded."))
|
|
|
|
def test_skips_a_malformed_transcript_or_stream_line(self):
|
|
checkout = Checkout(self)
|
|
raw = "\xff not UTF-8\nnot JSON\n[1]\n" + json.dumps({"type": "assistant", "message": "not an object"}) + "\n" + json.dumps({"type": "user", "message": None}) + "\n"
|
|
run = checkout.run([worker(raw=raw, say="past the noise", stream="\xff not UTF-8\n"), judge(evidence="past the noise")])
|
|
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge", "worker"]))
|
|
self.assertIn("past the noise", run.stdout)
|
|
self.assertIn("past the noise", run.calls[1]["stdin"])
|
|
|
|
@unittest.skipIf(os.geteuid() == 0, "root reads an unreadable file")
|
|
def test_exits_1_when_the_transcript_cannot_be_read(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(unreadable=True)])
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
|
|
self.assertRegex(run.stderr.strip().splitlines()[-2], r"^could not archive the transcript: .*\.jsonl: Permission denied$")
|
|
self.assertRegex(run.stderr.strip().splitlines()[-1], r"^could not read the transcript .*\.jsonl: Permission denied$")
|
|
|
|
def test_retries_a_worker_claude_cannot_start(self):
|
|
checkout = Checkout(self)
|
|
(checkout.temp / "fake-bin" / "claude").unlink()
|
|
run = checkout.run([])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertEqual(run.sleeps, [60, 60])
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the worker failed three times running: could not start claude: No such file or directory")
|
|
|
|
def test_exits_1_when_claude_leaves_no_transcript(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(transcript=False)])
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
|
|
self.assertIn("claude left no transcript for session", run.stderr)
|
|
|
|
def test_exits_2_after_six_unfinished_verdicts_running_and_a_rerun_gets_six_more(self):
|
|
checkout = Checkout(self)
|
|
six = [step for _ in range(6) for step in (worker(), judge())]
|
|
run = checkout.run(six + [worker()])
|
|
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge"] * 6))
|
|
self.assertIn("the judge ruled it unfinished 6 cycles running", run.stderr.strip().splitlines()[-1])
|
|
self.assertIn("rerun loop-claude to give it 6 more cycles", run.stderr.strip().splitlines()[-1])
|
|
stops = [outcome for outcome in checkout.archived().values() if "Stopped: " in outcome]
|
|
self.assertEqual(len(stops), 1)
|
|
self.assertIn(f"\n\nWorker: {OFF_LADDER}\n", stops[0])
|
|
run = checkout.run(six[:-2] + [worker()])
|
|
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge"] * 6))
|
|
|
|
def test_archives_the_stop_on_the_route_the_sixth_verdict_stepped_to_and_a_rerun_resumes_there(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(CHEAP)
|
|
run = checkout.run([step for _ in range(6) for step in (worker(), judge())] + [worker()])
|
|
self.assertEqual(run.code, 2)
|
|
stepped = '<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n\n<UTC> Stopped: '
|
|
self.assertEqual(len([outcome for outcome in checkout.archived().values() if stepped in outcome]), 1)
|
|
run = checkout.run([worker()])
|
|
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "worker"], [("claude-fable-5-2", "medium")])
|
|
|
|
def test_five_unfinished_verdicts_and_a_finished_one_do_not_add_up_to_six(self):
|
|
checkout = Checkout(self)
|
|
five = [step for _ in range(5) for step in (worker(), judge())]
|
|
run = checkout.run(five + [worker(), judge(finished=True)] + five + [worker()])
|
|
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge"] * 12))
|
|
|
|
|
|
class Interrupt(unittest.TestCase):
|
|
def test_exits_2_off_a_terminal_and_resumes_on_the_next_run(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_TRANSCRIPTS"] = str(checkout.temp / "deep" / "archive")
|
|
steps = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
|
|
run = checkout.run(steps)
|
|
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge"]))
|
|
self.assertEqual(run.stderr.strip().splitlines()[-2:], ["A or B?", "the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal"])
|
|
[(copy, verdict)] = checkout.archived().items()
|
|
self.assertEqual(copy, (checkout.transcripts()[0]).read_text())
|
|
self.assertEqual(verdict, f"Task: {TASK}\n\n<UTC> Verdict: interrupt\n\nA or B?\n\nEvidence:\n> Shall I use A or B?\n\nConfig: <config>\nJudge: judge-model\nWorker: {OFF_LADDER}\n\n<UTC> Stopped: the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal\n\nWorker: {OFF_LADDER}\n")
|
|
self.assertEqual(stat.S_IMODE((checkout.temp / "deep").stat().st_mode), 0o700)
|
|
self.assertIn(f"transcript: {checkout.archive() / 'proj'}/", run.stderr)
|
|
self.assertRegex(run.raw_stderr, rf"(?m)^{STAMP} transcript: ")
|
|
session = argument(run.calls[0], "--session-id")
|
|
run = checkout.run(steps, terminal_input="", stdout_on_terminal=False)
|
|
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge"]))
|
|
self.assertEqual(argument(run.calls[0], "--resume"), session)
|
|
self.assertEqual(len(checkout.archived()), 2)
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal")
|
|
run = checkout.run([worker(transcript=False), steps[1]])
|
|
self.assertEqual(run.code, 2)
|
|
self.assertEqual(len(checkout.archived()), 2)
|
|
|
|
def test_resumes_the_worker_headless_with_the_answer_typed_at_its_terminal_not_with_keys_typed_before_it(self):
|
|
checkout = Checkout(self)
|
|
asked = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
|
|
run = checkout.run([*asked, worker(say="Using B."), judge(evidence="Using B.")], terminal_input="B\n", before_typing=ANSWERING, typed_early="stray\n")
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "worker", "judge", "worker"])
|
|
self.assertIn("A or B?\nAnswer (/talk opens claude, Enter stops the loop): ", run.stderr)
|
|
self.assertEqual(argument(run.calls[2], "--resume"), argument(run.calls[0], "--session-id"))
|
|
self.assertIn("[human] B\n\nUsing B.", run.calls[3]["stdin"])
|
|
self.assertNotIn("stray", run.calls[3]["stdin"])
|
|
|
|
def test_resumes_the_worker_headless_with_the_answer_typed_in_loop_inbox(self):
|
|
checkout = Checkout(self)
|
|
folder = checkout.home / ".local" / "state" / "loop-claude"
|
|
def answered_in_inbox():
|
|
[ask] = waiting(folder)
|
|
send(folder, ask, Typed("B"))
|
|
asked = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
|
|
run = checkout.run([*asked, worker(say="Using B."), judge(evidence="Using B.")], terminal_input="", before_typing=("Answer (", answered_in_inbox))
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "worker", "judge", "worker"])
|
|
self.assertIn("Answer (/talk opens claude, Enter stops the loop): \nAnswered in loop-inbox: B\n", run.stderr)
|
|
self.assertIn("[human] B\n\nUsing B.", run.calls[3]["stdin"])
|
|
self.assertEqual([path for path in folder.iterdir() if path.suffix != ".lock"], [])
|
|
|
|
def test_a_second_loop_on_the_session_runs_unlisted_and_keeps_its_question_out_of_the_first_loops_folder(self):
|
|
checkout = Checkout(self)
|
|
folder = checkout.home / ".local" / "state" / "loop-claude"
|
|
asked = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
|
|
session = argument(checkout.run(asked).calls[0], "--session-id")
|
|
# The first loop on the session, still running.
|
|
self.enterContext(session_lock.held(folder, session))
|
|
asks = []
|
|
run = checkout.run(asked, terminal_input="\n", before_typing=("Answer (", lambda: asks.extend(folder.glob("*.json"))))
|
|
self.assertEqual(run.code, 2)
|
|
self.assertIn(f"loop-inbox cannot list this loop or its questions: another process holds {session_lock.lock_file(folder, session)}", run.stderr)
|
|
self.assertEqual((asks, run.calls[0]["running"]), ([], []))
|
|
self.assertFalse((folder / "unlisted").exists() and any((folder / "unlisted").iterdir()))
|
|
|
|
def test_hands_the_terminal_over_and_exits_2_when_nothing_is_typed(self):
|
|
checkout = Checkout(self)
|
|
checkout.git("commit", "--allow-empty", "--quiet", "-m", "start")
|
|
claude_json = json.dumps({"projects": {str(checkout.root): {"kept": True}}})
|
|
checkout.claude_json().write_text(claude_json)
|
|
asked = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
|
|
(checkout.root / "sub").mkdir()
|
|
run = checkout.run([*asked, interrupt(type="B"), judge(evidence="B"), worker()], cwd=checkout.root / "sub", terminal_input="/talk\n", before_typing=ANSWERING)
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "interrupt", "judge", "worker", "judge"])
|
|
handed = run.calls[2]["argv"]
|
|
self.assertEqual(run.calls[2]["cwd"], str(checkout.root))
|
|
self.assertEqual(argument(run.calls[2], "--resume"), argument(run.calls[0], "--session-id"))
|
|
self.assertEqual(argument(run.calls[2], "--model"), "worker-model")
|
|
self.assertNotIn("-p", handed)
|
|
self.assertIn("[human] B", run.calls[3]["stdin"])
|
|
self.assertEqual(run.calls[2]["sandboxed"], "1")
|
|
self.assertEqual(checkout.claude_json().read_text(), claude_json)
|
|
|
|
for transcript in checkout.transcripts():
|
|
transcript.unlink()
|
|
run = checkout.run([*asked, interrupt()], terminal_input="/talk\n", before_typing=ANSWERING)
|
|
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge", "interrupt"]))
|
|
self.assertTrue(run.stderr.strip().endswith("the human is needed to answer the question above, but the interrupt ended without anything typed"))
|
|
|
|
run = checkout.run([worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?", vanish=True)], terminal_input="/talk\n", before_typing=ANSWERING)
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge"]))
|
|
self.assertTrue(run.stderr.strip().endswith("could not start claude for the interrupt: No such file or directory"))
|
|
|
|
def test_leaves_ctrl_c_to_the_claude_open_for_talk(self):
|
|
checkout = Checkout(self)
|
|
asked = [worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?")]
|
|
run = checkout.run([*asked, interrupt(type="B", ctrl_c=True), judge(evidence="B")], terminal_input="/talk\n", before_typing=ANSWERING)
|
|
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge", "interrupt", "judge", "worker"]))
|
|
self.assertFalse(run.calls[2]["ctrl_c_ignored"])
|
|
self.assertIn("[human] B", run.calls[3]["stdin"])
|
|
|
|
@unittest.skipIf(os.geteuid() == 0, "root reads an unreadable file")
|
|
def test_exits_1_when_the_transcript_cannot_be_read_after_the_interrupt(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?"), interrupt(type="B", unreadable=True)], terminal_input="/talk\n", before_typing=ANSWERING)
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge", "interrupt"]))
|
|
self.assertRegex(run.stderr.strip().splitlines()[-1], r"^could not tell whether you typed anything: could not read the transcript .*\.jsonl: Permission denied$")
|
|
|
|
|
|
class Models(unittest.TestCase):
|
|
def catalog(self, checkout):
|
|
folder = checkout.home / ".claude" / "cache" / "model-catalog"
|
|
folder.mkdir(parents=True)
|
|
(folder / "old.json").write_text(json.dumps({"catalog": {"config": {"models": [{"id": "old-model"}]}}, "fetchedAt": 1}))
|
|
(folder / "new.json").write_text(json.dumps({"catalog": {"config": {"models": [{"id": "model-a", "name": "A"}, {"id": "model-b", "name": "B"}]}}, "fetchedAt": 2}))
|
|
|
|
def test_asks_a_project_without_a_model_on_the_terminal_and_saves_the_answer(self):
|
|
checkout = Checkout(self)
|
|
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
|
|
self.catalog(checkout)
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (2, []))
|
|
self.assertEqual(run.stderr.strip(), f'the human is needed to pick proj\'s model, or merge {{"proj": "<model>"}} into {checkout.models_file()}, but stdin or stdout is not a terminal; rerun loop-claude on a terminal')
|
|
|
|
run = checkout.run([worker()], terminal_input="\x04")
|
|
self.assertEqual((run.code, run.calls), (2, []))
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the human is needed to pick proj's model, but the input ended")
|
|
|
|
checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "judge-model"
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (2, []))
|
|
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"]
|
|
run = checkout.run([worker(), judge()], terminal_input="\n9\n2\n")
|
|
self.assertEqual(run.code, 130)
|
|
self.assertIn("proj runs on model-b", run.stderr)
|
|
self.assertIn("1. A (model-a)", run.stderr)
|
|
self.assertNotIn("old-model", run.stderr)
|
|
self.assertIn("pick 1 to 2", run.stderr)
|
|
self.assertEqual({argument(call, "--model") for call in run.calls}, {"model-b"})
|
|
self.assertEqual(json.loads(checkout.models_file().read_text()), {"proj": "model-b"})
|
|
self.assertEqual(stat.S_IMODE(checkout.models_file().stat().st_mode), 0o600)
|
|
|
|
checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "judge-model"
|
|
run = checkout.run([worker(), judge()])
|
|
self.assertEqual([argument(call, "--model") for call in run.calls], ["model-b", "judge-model", "model-b"])
|
|
|
|
def test_keeps_the_models_file_mode_and_other_projects_when_saving(self):
|
|
checkout = Checkout(self)
|
|
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
|
|
checkout.models_file().parent.mkdir(parents=True)
|
|
checkout.models_file().write_text(json.dumps({"other": "model-x"}))
|
|
checkout.models_file().chmod(0o640)
|
|
def saved_elsewhere():
|
|
checkout.models_file().write_text(json.dumps({"late": "model-y", "other": "model-x"}))
|
|
|
|
run = checkout.run([worker()], terminal_input="custom-model\n", before_typing=("no model catalog cached", saved_elsewhere))
|
|
self.assertEqual(run.code, 130)
|
|
self.assertIn("no model catalog cached", run.stderr)
|
|
self.assertIn("A name claude --model takes: ", run.stderr)
|
|
self.assertEqual(json.loads(checkout.models_file().read_text()), {"late": "model-y", "other": "model-x", "proj": "custom-model"})
|
|
self.assertEqual(stat.S_IMODE(checkout.models_file().stat().st_mode), 0o640)
|
|
|
|
for broken in ("{", "[]", '{"proj": 1}'):
|
|
with self.subTest(broken=broken):
|
|
checkout.models_file().write_text(broken)
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertTrue(run.stderr.startswith(f"could not read {checkout.models_file()}: "))
|
|
self.assertEqual(checkout.models_file().read_text(), broken)
|
|
|
|
def test_offers_no_catalog_whose_models_are_not_a_list(self):
|
|
checkout = Checkout(self)
|
|
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
|
|
self.catalog(checkout)
|
|
catalog = checkout.home / ".claude" / "cache" / "model-catalog" / "new.json"
|
|
for shape in ({"catalog": {"config": {"models": []}}}, {"catalog": None}, {"catalog": {"config": []}}, {"catalog": {"config": {"models": "x"}}}):
|
|
with self.subTest(shape=shape):
|
|
catalog.write_text(json.dumps({**shape, "fetchedAt": 3}))
|
|
checkout.models_file().unlink(missing_ok=True)
|
|
run = checkout.run([worker()], terminal_input="1\n")
|
|
self.assertIn("no model catalog cached", run.stderr)
|
|
self.assertEqual(argument(run.calls[0], "--model"), "1")
|
|
|
|
@unittest.skipIf(os.geteuid() == 0, "root writes into a read-only directory")
|
|
def test_exits_1_when_the_models_file_cannot_be_written(self):
|
|
checkout = Checkout(self)
|
|
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
|
|
checkout.models_file().parent.mkdir(parents=True, mode=0o500)
|
|
self.addCleanup(checkout.models_file().parent.chmod, 0o700)
|
|
run = checkout.run([worker()], terminal_input="custom-model\n")
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertIn(f"could not save the model in {checkout.models_file()}: Permission denied", run.stderr)
|
|
|
|
|
|
class Reload(unittest.TestCase):
|
|
def test_an_edit_takes_effect_at_the_next_cycle_boundary_once_it_answers_help(self):
|
|
checkout = Checkout(self)
|
|
copy = checkout.temp / "bin"
|
|
shutil.copytree(BIN, copy, ignore=shutil.ignore_patterns("__pycache__", "test_*"))
|
|
# The copied tools read the ladder their setup ships from the folder above their bin/.
|
|
shutil.copy(BIN.parent / "routing-ladder.json", copy.parent)
|
|
checkout.script = copy / "loop-claude"
|
|
checkout.env["FAKE_CLAUDE_EDIT"] = str(copy)
|
|
run = checkout.run([
|
|
worker(append=["loop-claude", "\ndef\n"]), judge(),
|
|
worker(replace=["loop-claude", "\ndef\n", "\n# edited\n"]), judge(),
|
|
worker(append=["loop_claude/nested/edited.py", "# edited\n"]), judge(),
|
|
worker(append=["records/file_writes.py", "# edited\n"]), judge(),
|
|
worker(),
|
|
])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge"] * 5)
|
|
self.assertIn("keeping the running script", run.stderr)
|
|
reloads = [json.loads(argv[1]) for argv in run.starts if argv[:1] == ["--reloaded"]]
|
|
self.assertEqual(reloads, [{"cycle_number": cycle, "root": str(checkout.root), "task": TASK, "unfinished_evidence": "working", "route": OFF_LADDER_ROUTE, "unfinished_run": cycle, "items_taken_in": []} for cycle in (2, 3, 4)])
|
|
self.assertIn("=== cycle 5,", run.stdout)
|
|
|
|
outside = checkout.temp / "outside"
|
|
outside.mkdir()
|
|
state = {"cycle_number": 3, "root": str(checkout.root), "task": TASK, "unfinished_evidence": "", "route": None}
|
|
run = checkout.run([worker()], argv=["--reloaded", json.dumps(state)], cwd=outside)
|
|
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge"]))
|
|
self.assertEqual(run.calls[0]["cwd"], str(checkout.root))
|
|
self.assertIn("=== cycle 4,", run.stdout)
|
|
|
|
run = checkout.run([], argv=["--reloaded", "{}"])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertIn("rerun loop-claude <task>", run.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|