1141 lines
66 KiB
Python
1141 lines
66 KiB
Python
import dataclasses
|
|
import datetime
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import pty
|
|
import re
|
|
import select
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from test_env import isolated_env
|
|
from project_config.test_project_config import write_config
|
|
|
|
BIN = pathlib.Path(__file__).resolve().parents[1]
|
|
TASK = "Do the next chunk."
|
|
# How the archive records a worker that runs off the ladder.
|
|
OFF_LADDER = '{"model": "worker-model"}'
|
|
# How a reload hands that worker's route.
|
|
OFF_LADDER_ROUTE = {"rung": {"model": "worker-model", "effort": ""}, "item": 0, "likelihood": 0, "escape_and_impact": 0, "unfinished": 0, "escape_and_impact_taken_in": 0, "escalated_at": 0}
|
|
|
|
|
|
# A tmux that logs each command. It prints $FAKE_TMUX_VERSION for -V, $FAKE_TMUX_OWNER for show-options, and the file $FAKE_TMUX_WINDOWS for list-windows, where it finds no session without the file, and for any other command the ID of a new window.
|
|
FAKE_TMUX = """#!{python}
|
|
import json, os, pathlib, sys
|
|
|
|
argv = sys.argv[1:]
|
|
with open(os.environ["FAKE_TMUX_LOG"], "a") as log:
|
|
log.write(json.dumps(argv) + "\\n")
|
|
windows = pathlib.Path(os.environ["FAKE_TMUX_WINDOWS"])
|
|
if argv[0] == "-V":
|
|
print(os.environ.get("FAKE_TMUX_VERSION", "tmux 3.2"))
|
|
elif argv[0] == "show-options":
|
|
print(os.environ["FAKE_TMUX_OWNER"])
|
|
elif argv[0] == "list-windows":
|
|
if not windows.exists():
|
|
sys.exit("no server running")
|
|
print(windows.read_text(), end="")
|
|
else:
|
|
print("@7")
|
|
"""
|
|
|
|
# Plays claude from FAKE_CLAUDE_SCENARIO, one step per call, logging each call; a step for another role,
|
|
# or none left, is Ctrl-C to loop-claude, which otherwise loops forever.
|
|
FAKE_CLAUDE = """#!{python}
|
|
import json, os, pathlib, signal, sys, time
|
|
|
|
argv = sys.argv[1:]
|
|
role = "judge" if "--json-schema" in argv else "worker" if "-p" in argv else "interrupt"
|
|
scenario = pathlib.Path(os.environ["FAKE_CLAUDE_SCENARIO"])
|
|
steps = json.loads(scenario.read_text())
|
|
call = {{"argv": argv, "cwd": os.getcwd(), "role": role, "ctrl_c_ignored": signal.getsignal(signal.SIGINT) == signal.SIG_IGN, "sandboxed": os.environ.get("CLAUDE_CODE_SANDBOXED")}}
|
|
if role == "judge":
|
|
call["stdin"] = sys.stdin.read()
|
|
with open(os.environ["FAKE_CLAUDE_LOG"], "a") as log:
|
|
log.write(json.dumps(call) + "\\n")
|
|
if not steps or steps[0]["role"] != role:
|
|
parent = os.getppid()
|
|
os.kill(parent, signal.SIGINT)
|
|
while os.getppid() == parent:
|
|
time.sleep(0.05)
|
|
sys.exit(0)
|
|
step = steps.pop(0)
|
|
scenario.write_text(json.dumps(steps))
|
|
|
|
|
|
def session():
|
|
flag = "--session-id" if "--session-id" in argv else "--resume"
|
|
return argv[argv.index(flag) + 1]
|
|
|
|
|
|
def transcript():
|
|
return pathlib.Path.home() / ".claude" / "projects" / "-fake" / f"{{session()}}.jsonl"
|
|
|
|
|
|
def append(*entries):
|
|
transcript().parent.mkdir(parents=True, exist_ok=True)
|
|
with open(transcript(), "a") as handle:
|
|
handle.writelines(json.dumps(entry) + "\\n" for entry in entries)
|
|
|
|
|
|
if role == "worker":
|
|
if "--session-id" in argv and step.get("transcript", True):
|
|
append({{"type": "custom-title"}}, {{"type": "queue-operation", "timestamp": str(time.time_ns())}})
|
|
said = {{"type": "assistant", "message": {{"content": [{{"type": "text", "text": step.get("say", "working")}}]}}}}
|
|
if step.get("transcript", True):
|
|
append({{"type": "user", "promptSource": "sdk", "message": {{"content": argv[argv.index("-p") + 1]}}}})
|
|
with open(transcript(), "ab") as handle:
|
|
handle.write(step.get("raw", "").encode("latin-1"))
|
|
append(said)
|
|
if step.get("siblings"):
|
|
for sibling in (f"{{session()}}/subagents", f"{{session()}}.superseded-1/tool-results"):
|
|
(transcript().parent / sibling).mkdir(parents=True, exist_ok=True)
|
|
(transcript().parent / f"{{session()}}.orphaned-1-ab.jsonl").write_text("{{}}\\n")
|
|
if "append" in step:
|
|
edited = pathlib.Path(os.environ["FAKE_CLAUDE_EDIT"], step["append"][0])
|
|
edited.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(edited, "a") as script:
|
|
script.write(step["append"][1])
|
|
if "replace" in step:
|
|
script = pathlib.Path(os.environ["FAKE_CLAUDE_EDIT"], step["replace"][0])
|
|
script.write_text(script.read_text().replace(*step["replace"][1:]))
|
|
sys.stdout.flush()
|
|
sys.stdout.buffer.write(step.get("stream", "").encode("latin-1"))
|
|
print(json.dumps(said))
|
|
if step.get("result", True):
|
|
print(json.dumps({{"type": "result"}}))
|
|
if "signal" in step:
|
|
os.kill(os.getpid(), step["signal"])
|
|
elif role == "judge":
|
|
print(step["stdout"] if "stdout" in step else json.dumps({{"structured_output": step["verdict"]}}))
|
|
print(step.get("stderr", ""), file=sys.stderr)
|
|
else:
|
|
if step.get("ctrl_c"):
|
|
# A Ctrl-C on the terminal reaches every process in its foreground, loop-claude too. kill returns once the signal is pending in loop-claude, before this process exits and loop-claude stops waiting for it.
|
|
os.kill(os.getppid(), signal.SIGINT)
|
|
if "type" in step:
|
|
append({{"type": "user", "message": {{"content": step["type"]}}}})
|
|
if step.get("unreadable"):
|
|
transcript().chmod(0)
|
|
if step.get("vanish"):
|
|
os.remove(__file__)
|
|
sys.exit(step.get("exit", 0))
|
|
"""
|
|
|
|
# Records every start of loop-claude and turns its retry waits into log lines.
|
|
SITECUSTOMIZE = """import json, os, sys, time
|
|
|
|
if os.path.basename(sys.argv[0]) == "loop-claude":
|
|
def record(event):
|
|
with open(os.environ["LOOP_TEST_LOG"], "a") as log:
|
|
log.write(json.dumps(event) + "\\n")
|
|
record({"argv": sys.argv[1:]})
|
|
pause = time.sleep
|
|
time.sleep = lambda seconds: record({"sleep": seconds}) if seconds >= 1 else pause(seconds)
|
|
# The host's own memory doesn't matter: every read finds all of it available.
|
|
sys.path.insert(0, os.path.dirname(os.path.realpath(sys.argv[0])))
|
|
import pathlib
|
|
from machine_load import memory
|
|
memory.MEMINFO = pathlib.Path(os.environ["FAKE_MEMINFO"])
|
|
"""
|
|
|
|
PREMISES = "# Project\n\n## Goals\n\n1. Work.\n\n## Audience\n\nMe.\n"
|
|
# Item 4's L of 4 steps one rung up from the project's.
|
|
TODO = "## Items\n\n| ID | Release | Exempt | Item | L | E | I | R | S | Moves | Score |\n|---|---|---|---|---|---|---|---|---|---|---|\n| 4 | 0.1.0 | | **Do it.** | 4 | 2 | 3 | 7 | 2 | 1 by 9 | 25 |\n"
|
|
# Item 4's L of 2 steps one rung down.
|
|
CHEAP = TODO.replace("| 4 | 2 | 3 | 7 |", "| 2 | 1 | 2 | 3 |")
|
|
# Item 9 follows CHEAP's item 4, and a worker routed by item 4 may take it in: at L 4 it picks a higher rung for the worker, and at E + I 6 for the reviewers. At E + I 3, or at L 2, it picks the rung item 4 picks.
|
|
TAKEN_IN_AT_L_4 = CHEAP + "| 9 | 0.1.0 | | **Do more.** | 4 | 1 | 2 | 5 | 2 | 1 by 9 | 20 |\n"
|
|
TAKEN_IN_AT_E_PLUS_I_6 = CHEAP + "| 9 | 0.1.0 | | **Do more.** | 2 | 3 | 3 | 6 | 2 | 1 by 9 | 20 |\n"
|
|
|
|
|
|
# Waits for the interrupt's prompt before typing the answer, since loop-claude drops what was typed before it.
|
|
ANSWERING = ("Answer (", lambda: None)
|
|
|
|
|
|
def worker(**step):
|
|
return {"role": "worker", **step}
|
|
|
|
|
|
def judge(interrupt=False, finished=False, evidence="working", question="", escalation="", release_ready=False, **step):
|
|
verdict = {"answer": "", "escalation": escalation, "evidence": evidence, "finished": finished, "interrupt": interrupt, "question": question, "answered_before": "", "left_open": "", "release_ready": release_ready, "reversed": ""}
|
|
return {"role": "judge", "verdict": verdict, **step}
|
|
|
|
|
|
def interrupt(**step):
|
|
return {"role": "interrupt", **step}
|
|
|
|
|
|
def lines(path):
|
|
return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else []
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class Run:
|
|
calls: list
|
|
code: int
|
|
sleeps: list
|
|
starts: list
|
|
# stderr and stdout are without the local time that opens each message loop-claude prints, and stderr without the bell an interrupt rings; raw_stderr and raw_stdout keep them.
|
|
stderr: str
|
|
stdout: str
|
|
# Each tmux command run, as its arguments.
|
|
tmux: list
|
|
raw_stderr: str
|
|
raw_stdout: str
|
|
|
|
def roles(self):
|
|
return [call["role"] for call in self.calls]
|
|
|
|
|
|
class Checkout:
|
|
"""A temporary HOME with a git checkout at ~/code/proj, where loop-claude runs against the fake claude."""
|
|
|
|
def __init__(self, test):
|
|
self.test = test
|
|
temp = tempfile.TemporaryDirectory()
|
|
test.addCleanup(temp.cleanup)
|
|
self.temp = pathlib.Path(os.path.realpath(temp.name))
|
|
self.home = self.temp / "home"
|
|
self.root = self.home / "code" / "proj"
|
|
self.root.mkdir(parents=True)
|
|
(self.root / "README.md").write_text(PREMISES)
|
|
write_config(self.home, self.root)
|
|
self.script = BIN / "loop-claude"
|
|
self.env = isolated_env()
|
|
fake = self.temp / "fake-bin"
|
|
fake.mkdir()
|
|
(fake / "claude").write_text(FAKE_CLAUDE.format(python=sys.executable))
|
|
(fake / "claude").chmod(0o755)
|
|
(fake / "tmux").write_text(FAKE_TMUX.format(python=sys.executable))
|
|
(fake / "tmux").chmod(0o755)
|
|
(fake / "git").symlink_to(shutil.which("git"))
|
|
site = self.temp / "site"
|
|
site.mkdir()
|
|
(site / "sitecustomize.py").write_text(SITECUSTOMIZE)
|
|
(self.temp / "meminfo").write_text("MemTotal: 16 kB\nMemAvailable: 16 kB\n")
|
|
self.env.update({
|
|
"FAKE_CLAUDE_LOG": str(self.temp / "calls.jsonl"),
|
|
"FAKE_MEMINFO": str(self.temp / "meminfo"),
|
|
"FAKE_TMUX_LOG": str(self.temp / "tmux.jsonl"),
|
|
"FAKE_TMUX_WINDOWS": str(self.temp / "tmux-windows"),
|
|
"FAKE_TMUX_OWNER": str(self.root),
|
|
# The pane of the window the loop under test runs in.
|
|
"TMUX_PANE": "%3",
|
|
"FAKE_CLAUDE_SCENARIO": str(self.temp / "scenario.json"),
|
|
"GIT_CONFIG_GLOBAL": os.devnull,
|
|
"GIT_CONFIG_NOSYSTEM": "1",
|
|
"HOME": str(self.home),
|
|
"LOOP_CLAUDE_JUDGE_MODEL": "judge-model",
|
|
"LOOP_CLAUDE_WORKER_MODEL": "worker-model",
|
|
"LOOP_TEST_LOG": str(self.temp / "starts.jsonl"),
|
|
"PATH": str(fake),
|
|
"PYTHONPATH": str(site),
|
|
})
|
|
self.git("init", "--quiet")
|
|
|
|
def git(self, *args):
|
|
subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@example.com", *args], capture_output=True, check=True, cwd=self.root, env=self.env)
|
|
|
|
def commit_plan(self, todo):
|
|
"""Commits todo as todo.md on the checkout's branch, the default branch routing reads it from."""
|
|
(self.root / "todo.md").write_text(todo)
|
|
self.git("add", "todo.md")
|
|
self.git("commit", "--quiet", "-m", "plan")
|
|
|
|
def run(self, steps, argv=None, cwd=None, terminal_input=None, stdout_on_terminal=True, before_typing=None, typed_early="", chunk="no-item"):
|
|
"""Runs loop-claude with argv, or without one as the loop of the window `chunk`, on TASK. Off a terminal, a window's loop whose chunk finished is run again while steps remain, as the window it starts for the next chunk would."""
|
|
for name in ("FAKE_CLAUDE_LOG", "LOOP_TEST_LOG", "FAKE_TMUX_LOG"):
|
|
pathlib.Path(self.env[name]).unlink(missing_ok=True)
|
|
pathlib.Path(self.env["FAKE_CLAUDE_SCENARIO"]).write_text(json.dumps(steps))
|
|
command = [sys.executable, str(self.script), *(("--chunk", chunk, TASK) if argv is None else argv)]
|
|
cwd = cwd or self.root
|
|
if terminal_input is None:
|
|
code, stdout, stderr = 0, "", ""
|
|
runs = 0
|
|
while code == 0 and (runs == 0 or (argv is None and json.loads(pathlib.Path(self.env["FAKE_CLAUDE_SCENARIO"]).read_text()))):
|
|
runs += 1
|
|
done = subprocess.run(command, capture_output=True, cwd=cwd, env=self.env, stdin=subprocess.DEVNULL, text=True, timeout=60)
|
|
code, stdout, stderr = done.returncode, stdout + done.stdout, stderr + done.stderr
|
|
else:
|
|
code, stdout, stderr = self.run_on_terminal(command, cwd, terminal_input, stdout_on_terminal, before_typing, typed_early)
|
|
started = lines(pathlib.Path(self.env["LOOP_TEST_LOG"]))
|
|
return Run(
|
|
calls=lines(pathlib.Path(self.env["FAKE_CLAUDE_LOG"])),
|
|
code=code,
|
|
sleeps=[event["sleep"] for event in started if "sleep" in event],
|
|
starts=[event["argv"] for event in started if "argv" in event],
|
|
stderr=unstamped(stderr).replace("\a", ""),
|
|
stdout=unstamped(stdout),
|
|
tmux=lines(pathlib.Path(self.env["FAKE_TMUX_LOG"])),
|
|
raw_stderr=stderr,
|
|
raw_stdout=stdout,
|
|
)
|
|
|
|
def run_on_terminal(self, command, cwd, typed, stdout_on_terminal, before_typing, typed_early):
|
|
"""Types typed_early at once, then the input at once, or once stderr shows before_typing's text and its callback has run."""
|
|
controller, terminal = pty.openpty()
|
|
with tempfile.TemporaryFile() as stderr, tempfile.TemporaryFile() as stdout:
|
|
try:
|
|
proc = subprocess.Popen(command, cwd=cwd, env=self.env, stderr=stderr, stdin=terminal, stdout=terminal if stdout_on_terminal else stdout)
|
|
finally:
|
|
os.close(terminal)
|
|
try:
|
|
os.write(controller, typed_early.encode())
|
|
prompt, callback = before_typing or (None, None)
|
|
if prompt is None:
|
|
os.write(controller, typed.encode())
|
|
output, deadline = b"", time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
if select.select([controller], [], [], 0.05)[0]:
|
|
try:
|
|
chunk = os.read(controller, 4096)
|
|
except OSError:
|
|
break
|
|
if not chunk:
|
|
break
|
|
output += chunk
|
|
if prompt is not None and prompt.encode() in os.pread(stderr.fileno(), 1 << 20, 0):
|
|
callback()
|
|
os.write(controller, typed.encode())
|
|
prompt = None
|
|
code = proc.wait(timeout=10)
|
|
finally:
|
|
os.close(controller)
|
|
proc.kill()
|
|
proc.wait()
|
|
stderr.seek(0)
|
|
stdout.seek(0)
|
|
output += stdout.read()
|
|
return code, output.decode(errors="replace").replace("\r\n", "\n"), stderr.read().decode()
|
|
|
|
def claude_json(self):
|
|
return self.home / ".claude.json"
|
|
|
|
def models_file(self):
|
|
return self.home / ".claude" / "loop-claude-models.json"
|
|
|
|
def transcripts(self):
|
|
return sorted((self.home / ".claude" / "projects").glob("*/*.jsonl"))
|
|
|
|
def archive(self):
|
|
return pathlib.Path(self.env.get("LOOP_CLAUDE_TRANSCRIPTS", self.home / ".claude" / "loop-claude-transcripts"))
|
|
|
|
def archived(self):
|
|
"""Each archived transcript's outcomes, with every UTC stamp replaced by <UTC> and every setup version by <config>, after checking the copy is named by its content hash."""
|
|
outcomes = {}
|
|
for folder in (self.archive(), self.archive() / "proj", *(self.archive() / "proj").iterdir()):
|
|
self.test.assertEqual(stat.S_IMODE(folder.stat().st_mode), 0o700, folder)
|
|
for copy in sorted((self.archive() / "proj").glob("*/*.jsonl")):
|
|
self.test.assertEqual(copy.stem, hashlib.sha256(copy.read_bytes()).hexdigest())
|
|
kept = copy.with_suffix(".outcomes.md")
|
|
for archived in (copy, kept):
|
|
self.test.assertEqual(stat.S_IMODE(archived.stat().st_mode), 0o600)
|
|
stamped = re.sub(r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ ", "<UTC> ", kept.read_text(), flags=re.MULTILINE)
|
|
outcomes[copy.read_text()] = re.sub(r"^Config: .*$", "Config: <config>", stamped, flags=re.MULTILINE)
|
|
return outcomes
|
|
|
|
|
|
STAMP = r"\d{4}-\d\d-\d\d \d\d:\d\d:\d\d"
|
|
|
|
|
|
def unstamped(text):
|
|
return re.sub(rf"(?m)^{STAMP} ", "", text)
|
|
|
|
|
|
def argument(call, flag):
|
|
return call["argv"][call["argv"].index(flag) + 1] if flag in call["argv"] else None
|
|
|
|
|
|
class Premises(unittest.TestCase):
|
|
def test_refuses_to_start_without_a_task_a_checkout_or_premises(self):
|
|
checkout = Checkout(self)
|
|
for argv, code, said in (([], 1, "usage: loop-claude <task>"), (["--help"], 0, "usage: loop-claude <task>"), (["-h"], 0, "usage: loop-claude <task>")):
|
|
with self.subTest(argv=argv):
|
|
run = checkout.run([], argv=argv)
|
|
self.assertEqual(run.code, code)
|
|
self.assertIn(said, run.stdout if code == 0 else run.stderr)
|
|
outside = checkout.temp / "outside"
|
|
outside.mkdir()
|
|
run = checkout.run([], cwd=outside)
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertRegex(run.raw_stderr, rf"(?m)^{STAMP} .*not a git repository")
|
|
readmes = {
|
|
None: "{root}/README.md is missing or empty - no premises.",
|
|
b"# P\n## Goals\n1. g\xff\n## Audience\nme\n": "cannot read {root}/README.md: not UTF-8 on line 3 - no premises.",
|
|
b"# P\n## Audience\nme\n": "{root}/README.md has no ## Goals heading - no premises.",
|
|
b"# P\n## Goals\n1. g\n": "{root}/README.md has no ## Audience heading - no premises.",
|
|
b"# P\n## Goals\n\n## Audience\nme\n": "{root}/README.md has no text under ## Goals - no premises.",
|
|
b"# P\n## Goals\n1. g\n## Audience\nme\n## Audience\nyou\n": "{root}/README.md has ## Audience more than once - no premises.",
|
|
}
|
|
(checkout.root / "sub").mkdir()
|
|
for text, said in readmes.items():
|
|
with self.subTest(said=said):
|
|
readme = checkout.root / "README.md"
|
|
readme.unlink(missing_ok=True)
|
|
if text is not None:
|
|
readme.write_bytes(text)
|
|
run = checkout.run([worker()], cwd=checkout.root / "sub")
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertEqual(run.stderr.strip(), said.format(root=checkout.root))
|
|
(checkout.root / "README.md").unlink()
|
|
(checkout.root / "README.md").mkdir()
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertEqual(run.stderr.strip(), f"cannot read {checkout.root}/README.md: Is a directory - no premises.")
|
|
(checkout.home / ".claude-projects" / "proj" / "config.toml").unlink()
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertEqual(run.stderr.strip(), f"no config names the checkout {checkout.root}: write ~/.claude-projects/<key>/config.toml, as `project-config --help` shows - no premises.")
|
|
(checkout.temp / "fake-bin" / "git").unlink()
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertTrue(run.stderr.startswith("git rev-parse --show-toplevel could not start: "), run.stderr)
|
|
|
|
def test_reads_the_premises_from_the_file_the_config_names(self):
|
|
checkout = Checkout(self)
|
|
checkout.commit_plan(TODO)
|
|
(checkout.root / "README.md").write_text("# Project\n")
|
|
folder = write_config(checkout.home, checkout.root, goals="folder:goals.md")
|
|
(folder / "goals.md").write_text("## Goals\n1. Kept apart.\n\n## Audience\nThe client.\n")
|
|
run = checkout.run([worker()])
|
|
self.assertTrue(run.calls, run.stderr)
|
|
self.assertIn("## Goals\n1. Kept apart.\n\n## Audience\nThe client.", argument(run.calls[0], "-p"))
|
|
|
|
def test_makes_no_worktree_for_a_readme_without_premises(self):
|
|
checkout = Checkout(self)
|
|
checkout.commit_plan(TODO)
|
|
(checkout.root / "README.md").write_text("# Project\n")
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertFalse((checkout.home / "code" / "proj-item-4").exists())
|
|
|
|
def test_names_the_session_by_the_checkout_relative_to_code_or_else_to_the_root(self):
|
|
checkout = Checkout(self)
|
|
for root in (checkout.temp / "elsewhere", checkout.home / "code"):
|
|
with self.subTest(root=root):
|
|
root.mkdir(exist_ok=True)
|
|
(root / "README.md").write_text(PREMISES)
|
|
write_config(checkout.home, root)
|
|
checkout.git("init", "--quiet", str(root))
|
|
run = checkout.run([worker()], cwd=root)
|
|
self.assertEqual(argument(run.calls[0], "--name"), f"A {str(root).lstrip('/')} loop-claude no-item")
|
|
|
|
|
|
class Cycles(unittest.TestCase):
|
|
def test_resumes_the_session_while_unfinished_and_starts_it_fresh_after_finishing(self):
|
|
checkout = Checkout(self)
|
|
(checkout.root / "sub").mkdir()
|
|
(checkout.root / "docs").mkdir()
|
|
(checkout.root / "docs" / "decisions.md").write_text("# Decisions\n")
|
|
(checkout.home / ".claude").mkdir()
|
|
(checkout.home / ".claude" / "CLAUDE.md").write_text("# Global instructions\n")
|
|
run = checkout.run([
|
|
worker(say="started"), judge(evidence="started"),
|
|
worker(say="landed-it", siblings=True), judge(finished=True, evidence="landed-it"),
|
|
worker(say="next chunk"),
|
|
], cwd=checkout.root / "sub")
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "worker", "judge", "worker", "judge"])
|
|
workers = [call for call in run.calls if call["role"] == "worker"]
|
|
session = argument(workers[0], "--session-id")
|
|
self.assertEqual(argument(workers[0], "--name"), "A proj loop-claude no-item")
|
|
self.assertEqual(argument(workers[1], "--resume"), session)
|
|
self.assertIsNone(argument(workers[1], "--session-id"))
|
|
self.assertIsNone(argument(workers[1], "--name"))
|
|
self.assertEqual(argument(workers[2], "--session-id"), session)
|
|
self.assertTrue(argument(workers[0], "-p").startswith(f"{TASK}\n\nContinue any open draft PR"))
|
|
self.assertTrue(argument(workers[0], "-p").endswith("The chunk's plan items, or else the task above, set its scope. The goals below, " 'which rank after code health (~/.claude/CLAUDE.md → "Shared goal") unless the goals file voids it' ", choose between ways to deliver that scope, and never widen it.\n\nFrom the goals file, " f"{checkout.root}/README.md:\n## Goals\n1. Work.\n\n## Audience\nMe."))
|
|
self.assertIn("main or master", argument(workers[0], "-p"))
|
|
self.assertNotIn(TASK, argument(workers[1], "-p"))
|
|
self.assertIn("Even then, it is finished only once every answer the human typed in it", argument(workers[1], "-p"))
|
|
self.assertIn("as the work remaining:\n\n> started\n\n", argument(workers[1], "-p"))
|
|
self.assertEqual({call["cwd"] for call in run.calls}, {str(checkout.root)})
|
|
self.assertIn("1. Work.", run.calls[1]["stdin"])
|
|
self.assertIn("From docs/decisions.md:\n# Decisions\n\nFrom ~/.claude/CLAUDE.md:\n# Global instructions", run.calls[1]["stdin"])
|
|
self.assertEqual({argument(call, "--model") for call in workers}, {"worker-model"})
|
|
self.assertEqual({argument(call, "--model") for call in run.calls if call["role"] == "judge"}, {"judge-model"})
|
|
self.assertIn("landed-it", run.calls[3]["stdin"])
|
|
self.assertNotIn("landed-it", run.calls[5]["stdin"])
|
|
self.assertEqual([path.name for path in checkout.transcripts()[0].parent.iterdir()], [checkout.transcripts()[0].name])
|
|
self.assertIn("=== cycle 2, worker on worker-model ===", run.stdout)
|
|
self.assertEqual(len(re.findall(rf"(?m)^{STAMP} === cycle 2, worker on worker-model ===$", run.raw_stdout)), 1)
|
|
# The finished chunk's loop started the next chunk's window and ended; the checkout has no default branch to read a plan from.
|
|
self.assertIn("landed-it\nthe chunk finished; started no-item\n\n=== cycle 1, worker on worker-model ===\nnext chunk\n", run.stdout)
|
|
copies = {verdict.removeprefix(f"Task: {TASK}\n\n"): copy for copy, verdict in checkout.archived().items()}
|
|
self.assertEqual(sorted(copies), [f"<UTC> Stopped: Ctrl-C\n\nWorker: {OFF_LADDER}\n", f"<UTC> Verdict: finished\n\nEvidence:\n> landed-it\n\nConfig: <config>\nJudge: judge-model\nWorker: {OFF_LADDER}\n", f"<UTC> Verdict: unfinished\n\nEvidence:\n> started\n\nConfig: <config>\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
|
|
self.assertIn('"text": "landed-it"', copies[f"<UTC> Verdict: finished\n\nEvidence:\n> landed-it\n\nConfig: <config>\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
|
|
self.assertIn('"text": "next chunk"', copies[f"<UTC> Stopped: Ctrl-C\n\nWorker: {OFF_LADDER}\n"])
|
|
self.assertEqual(len(list(checkout.archive().glob(f"proj/*/session/{session}.orphaned-1-ab.jsonl"))), 1)
|
|
|
|
def test_routes_the_worker_by_its_items_l_and_steps_it_up_within_the_chunk(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(TAKEN_IN_AT_L_4)
|
|
run = checkout.run([
|
|
worker(say="one"), judge(evidence="one"),
|
|
worker(say="two"), judge(evidence="two"),
|
|
worker(say="three\nChunk items: 4 (L 2, E + I 3), 9 (L 4, E + I 5)"), judge(evidence="three"),
|
|
worker(say="four"), judge(finished=True, evidence="four"),
|
|
worker(say="next chunk"),
|
|
])
|
|
self.assertEqual(run.code, 130)
|
|
workers = [call for call in run.calls if call["role"] == "worker"]
|
|
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in workers], [
|
|
("claude-sonnet-5-5", "high"),
|
|
("claude-sonnet-5-5", "high"),
|
|
("claude-opus-5-5", "medium"),
|
|
("claude-fable-5-2", "low"),
|
|
("claude-sonnet-5-5", "high"),
|
|
])
|
|
self.assertEqual({(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "judge"}, {("judge-model", None)})
|
|
self.assertIn("loop-claude runs you on claude-sonnet-5-5 at high effort, routed by item 4's L of 2.", argument(workers[0], "-p"))
|
|
self.assertIn("on one line, as in `Chunk items: ", argument(workers[0], "-p"))
|
|
outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values())
|
|
self.assertIn('<UTC> Verdict: unfinished\n\nEvidence:\n> three\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-opus-5-5", "effort": "medium", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n\n<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n', outcomes)
|
|
self.assertIn('<UTC> Verdict: finished\n\nEvidence:\n> four\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n', outcomes)
|
|
|
|
def test_routes_on_the_ladder_the_projects_config_folder_holds_in_place_of_the_shipped_one(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-fable-5-2"
|
|
ladder = checkout.home / ".claude-projects" / "proj" / "routing-ladder.json"
|
|
ladder.write_text('[{"model": "claude-fable-5-2", "effort": "low"}, {"model": "claude-fable-5-2", "effort": "max"}]')
|
|
checkout.commit_plan(CHEAP)
|
|
run = checkout.run([worker(), judge(), worker(), judge(), worker()])
|
|
self.assertEqual(run.code, 130)
|
|
# Item 4's L of 2 steps one rung down, to Opus 5.5 on the shipped ladder; the project's ladder holds no rung below the project's rung, and one above it, which the second unfinished verdict steps to.
|
|
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "worker"], [("claude-fable-5-2", "low"), ("claude-fable-5-2", "low"), ("claude-fable-5-2", "max")])
|
|
self.assertEqual(run.stderr.count(f"routing on the project's own ladder, {ladder}.\n"), 3, "each cycle says so")
|
|
|
|
def test_says_where_the_projects_own_ladder_leaves_out_the_workers_model_and_says_nothing_on_the_shipped_one(self):
|
|
checkout = Checkout(self)
|
|
self.assertNotIn("own ladder", checkout.run([worker()]).stderr)
|
|
ladder = checkout.home / ".claude-projects" / "proj" / "routing-ladder.json"
|
|
ladder.write_text('[{"model": "claude-fable-5-2", "effort": "low"}]')
|
|
run = checkout.run([worker()])
|
|
self.assertIn(f"routing on the project's own ladder, {ladder}. worker-model is not on it, so the worker runs on worker-model unrouted; LOOP_CLAUDE_WORKER_MODEL and LOOP_CLAUDE_JUDGE_MODEL name other models for a run.\n", run.stderr)
|
|
self.assertEqual((argument(run.calls[0], "--model"), argument(run.calls[0], "--effort")), ("worker-model", None))
|
|
|
|
def test_reads_the_projects_ladder_afresh_each_cycle_and_exits_1_once_it_holds_a_fault(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-fable-5-2"
|
|
ladder = checkout.home / ".claude-projects" / "proj" / "routing-ladder.json"
|
|
ladder.write_text('[{"model": "claude-fable-5-2", "effort": "high"}]')
|
|
checkout.env["FAKE_CLAUDE_EDIT"] = str(ladder.parent)
|
|
run = checkout.run([worker(replace=["routing-ladder.json", '"high"}', '"high"},']), judge()])
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge"]))
|
|
self.assertEqual((argument(run.calls[0], "--model"), argument(run.calls[0], "--effort")), ("claude-fable-5-2", "high"))
|
|
self.assertIn(f"cannot read the routing ladder {ladder}: ", run.stderr)
|
|
|
|
def test_routes_on_and_states_the_same_item_from_the_fetched_default_branch_whatever_the_checkout_holds(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
origin = checkout.temp / "origin.git"
|
|
checkout.git("init", "--quiet", "--bare", "--initial-branch=main", str(origin))
|
|
checkout.git("symbolic-ref", "HEAD", "refs/heads/main")
|
|
checkout.commit_plan(TODO)
|
|
checkout.git("remote", "add", "origin", str(origin))
|
|
checkout.git("push", "--quiet", "--set-upstream", "origin", "main")
|
|
# Item 4 ships elsewhere, and item 6 is next; the checkout is behind origin, and holds a change of its own.
|
|
elsewhere = checkout.temp / "elsewhere"
|
|
checkout.git("clone", "--quiet", str(origin), str(elsewhere))
|
|
(elsewhere / "todo.md").write_text(TODO.replace("| 4 | 0.1.0 |", "| 6 | 0.1.0 |"))
|
|
checkout.git("-C", str(elsewhere), "commit", "--quiet", "--all", "-m", "item 4 shipped, plan item 6")
|
|
checkout.git("-C", str(elsewhere), "push", "--quiet", "origin", "main")
|
|
(checkout.root / "todo.md").write_text(TODO + "\nA change of the checkout's own.\n")
|
|
(checkout.root / "sub").mkdir()
|
|
run = checkout.run([worker()], cwd=checkout.root / "sub")
|
|
self.assertEqual(run.code, 130)
|
|
prompt = argument(run.calls[0], "-p")
|
|
self.assertIn("routed by item 6's L of 4", prompt)
|
|
self.assertIn("item 6, L 4, E + I 5", prompt)
|
|
self.assertTrue((checkout.home / "code" / "proj-item-6").is_dir())
|
|
|
|
def test_steps_the_worker_up_a_rung_when_the_judge_finds_in_the_transcript_the_escalation_trigger_it_cites(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(TODO)
|
|
run = checkout.run([
|
|
worker(say="Stability round 2: a 🔴. Trigger 3 fired; I ask for a higher rung."), judge(evidence="I ask for a higher rung.", escalation="Stability round 2: a 🔴."),
|
|
worker(say="Trigger 2 fired; I ask for a higher rung."), judge(evidence="I ask for a higher rung.", escalation="Trigger 2 fired; … a higher rung."),
|
|
worker(say="merged"), judge(finished=True, evidence="merged"),
|
|
worker(say="next chunk"),
|
|
])
|
|
self.assertEqual(run.code, 130)
|
|
workers = [call for call in run.calls if call["role"] == "worker"]
|
|
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in workers], [
|
|
("claude-fable-5-2", "low"),
|
|
("claude-fable-5-2", "medium"),
|
|
("claude-fable-5-2", "medium"),
|
|
("claude-fable-5-2", "low"),
|
|
])
|
|
outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values())
|
|
self.assertIn('<UTC> Verdict: unfinished\n\nEvidence:\n> I ask for a higher rung.\n\nEscalation:\n> Stability round 2: a 🔴.\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 4, "escape_and_impact": 5}\n\n<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', outcomes)
|
|
self.assertIn('<UTC> Verdict: unfinished\n\nEvidence:\n> I ask for a higher rung.\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', outcomes)
|
|
|
|
def test_steps_the_worker_up_once_as_the_judge_interrupts_when_the_run_also_cites_an_escalation_trigger(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(TODO)
|
|
cited = "Stability round 2: a 🔴."
|
|
run = checkout.run([
|
|
worker(say=f"{cited} Trigger 3 fired; I ask for a higher rung. Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?", escalation=cited),
|
|
interrupt(type="B"), judge(evidence="B", escalation="Trigger 3 fired; I ask for a higher rung."),
|
|
worker(say="carrying on"),
|
|
], terminal_input="/talk\n", before_typing=ANSWERING)
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "interrupt", "judge", "worker", "judge"])
|
|
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] != "judge"], [("claude-fable-5-2", "low"), ("claude-fable-5-2", "medium"), ("claude-fable-5-2", "medium")])
|
|
outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values())
|
|
self.assertIn('<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', "".join(outcomes))
|
|
self.assertNotIn("at max effort", "".join(outcomes))
|
|
self.assertIn('Verdict: interrupt\n\nA or B?\n\nEvidence:\n> Shall I use A or B?\n\nEscalation:\n> Stability round 2: a 🔴.\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 4, "escape_and_impact": 5}\n\n<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', "".join(outcomes))
|
|
|
|
def test_runs_a_cheap_items_worker_below_the_projects_rung_with_its_reviewers_on_it_and_its_judge_on_the_models_highest(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(CHEAP)
|
|
run = checkout.run([worker(say="one"), judge(finished=True, evidence="one"), worker(say="next chunk")])
|
|
self.assertEqual(run.code, 130)
|
|
first = next(call for call in run.calls if call["role"] == "worker")
|
|
self.assertEqual((argument(first, "--model"), argument(first, "--effort")), ("claude-sonnet-5-5", "high"))
|
|
ruling = next(call for call in run.calls if call["role"] == "judge")
|
|
self.assertEqual((argument(ruling, "--model"), argument(ruling, "--effort")), ("claude-opus-5-5", "xhigh"))
|
|
self.assertIn('Judge: claude-opus-5-5 at xhigh effort\nWorker: {"model": "claude-sonnet-5-5", "effort": "high", ', "".join(checkout.archived().values()))
|
|
self.assertIn("Launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool set to model opus, effort medium, the project's rung.", argument(first, "-p"))
|
|
|
|
def test_routes_the_reviewers_on_the_highest_e_plus_i_the_worker_takes_in_and_keeps_the_judge_on_the_models_highest(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(TAKEN_IN_AT_E_PLUS_I_6)
|
|
run = checkout.run([
|
|
worker(say="one\nChunk items: 4, 9"), judge(evidence="one"),
|
|
worker(say="two"), judge(finished=True, evidence="two"),
|
|
worker(say="next chunk"),
|
|
])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "judge"], [("claude-opus-5-5", "xhigh")] * 3)
|
|
workers = [call for call in run.calls if call["role"] == "worker"]
|
|
self.assertEqual({argument(call, "--effort") for call in workers}, {"high"})
|
|
self.assertIn("Launch each reviewer with the Agent tool set to model fable, effort medium, the rung picked by an item of E + I 6 taken into the chunk.", argument(workers[1], "-p"))
|
|
outcomes = "".join(checkout.archived().values())
|
|
self.assertIn('<UTC> Stepped up\n\nWorker: {"model": "claude-sonnet-5-5", "effort": "high", "item": 4, "likelihood": 2, "escape_and_impact": 3, "escape_and_impact_taken_in": 6}\n', outcomes)
|
|
self.assertIn('Judge: claude-opus-5-5 at xhigh effort\nWorker: {"model": "claude-sonnet-5-5", "effort": "high", "item": 4, "likelihood": 2, "escape_and_impact": 3, "escape_and_impact_taken_in": 6}\n', outcomes)
|
|
|
|
def test_a_rerun_resumes_an_open_chunk_on_the_rung_it_last_ran(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(TODO)
|
|
run = checkout.run([worker(), judge(), worker(), judge()])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual((argument(run.calls[-1], "--model"), argument(run.calls[-1], "--effort")), ("claude-fable-5-2", "medium"))
|
|
checkout.git("rm", "--quiet", "todo.md")
|
|
checkout.git("commit", "--quiet", "-m", "drop the plan")
|
|
run = checkout.run([worker()])
|
|
self.assertIsNotNone(argument(run.calls[0], "--resume"))
|
|
self.assertEqual((argument(run.calls[0], "--model"), argument(run.calls[0], "--effort")), ("claude-fable-5-2", "medium"))
|
|
|
|
def test_a_second_window_of_the_project_runs_its_own_session_and_leaves_the_first_ones_open_chunk_alone(self):
|
|
checkout = Checkout(self)
|
|
first = checkout.run([worker(say="first chunk"), judge(evidence="first chunk")])
|
|
session = argument(first.calls[0], "--session-id")
|
|
transcript = checkout.transcripts()[0]
|
|
kept, archived = transcript.read_bytes(), checkout.archived()
|
|
self.assertEqual(argument(first.calls[0], "--name"), "A proj loop-claude no-item")
|
|
second = checkout.run([worker(say="second chunk"), judge(evidence="second chunk")], chunk="item-2")
|
|
self.assertEqual(argument(second.calls[0], "--name"), "A proj loop-claude item-2")
|
|
self.assertNotIn(session, [argument(call, "--session-id") or argument(call, "--resume") for call in second.calls if call["role"] == "worker"])
|
|
self.assertNotIn("first chunk", second.calls[1]["stdin"])
|
|
self.assertEqual(transcript.read_bytes(), kept)
|
|
self.assertLessEqual(archived.items(), checkout.archived().items())
|
|
self.assertEqual(len(checkout.archived()), len(archived) + 1)
|
|
rerun = checkout.run([worker()])
|
|
self.assertEqual(argument(rerun.calls[0], "--resume"), session)
|
|
|
|
def test_exits_1_when_the_transcript_cannot_be_archived_and_resumes_on_the_next_run(self):
|
|
checkout = Checkout(self)
|
|
checkout.archive().parent.mkdir(parents=True)
|
|
checkout.archive().write_text("in the way")
|
|
run = checkout.run([worker(say="landed-it"), judge(finished=True, evidence="landed-it")])
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge"]))
|
|
self.assertRegex(run.stderr.strip().splitlines()[-1], r"^could not archive the transcript: .*/loop-claude-transcripts/proj: Not a directory$")
|
|
self.assertEqual(run.stderr.count("could not archive the transcript"), 1)
|
|
session = argument(run.calls[0], "--session-id")
|
|
checkout.archive().unlink()
|
|
run = checkout.run([worker(), judge(finished=True)])
|
|
self.assertEqual(argument(run.calls[0], "--resume"), session)
|
|
self.assertEqual(len(checkout.archived()), 1)
|
|
|
|
def test_stops_after_three_failures_running_retrying_a_minute_apart(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(exit=1), worker(result=False), worker(signal=9)])
|
|
self.assertEqual(run.code, 1)
|
|
self.assertEqual(run.roles(), ["worker"] * 3)
|
|
self.assertEqual(run.sleeps, [60, 60])
|
|
self.assertIn("retrying the worker in a minute: claude exited 1", run.stderr)
|
|
self.assertIn("claude exited 0 without a result event", run.stderr)
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the worker failed three times running: claude killed by signal 9")
|
|
self.assertNotIn("=== cycle 2,", run.stdout)
|
|
self.assertEqual(argument(run.calls[1], "--resume"), argument(run.calls[0], "--session-id"))
|
|
|
|
run = checkout.run([worker(exit=1), worker(exit=1), worker(), judge(), worker(exit=1), worker(exit=1)])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.sleeps, [60, 60, 60, 60])
|
|
|
|
run = checkout.run([worker(), judge(exit=1, stderr="judge down"), judge(stdout=""), judge(verdict={"interrupt": False})])
|
|
self.assertEqual(run.code, 1)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "judge", "judge"])
|
|
self.assertEqual(run.sleeps, [60, 60])
|
|
self.assertIn("retrying the judge in a minute: judge down", run.stderr)
|
|
self.assertIn("retrying the judge in a minute: the judge printed no json reply; it printed: nothing\n", run.stderr)
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], 'the judge failed three times running: the judge\'s reply has no answer of type string')
|
|
|
|
def test_waits_out_a_usage_limit_which_ends_a_run_of_failures(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["TZ"] = "UTC"
|
|
resets_at = int(time.time()) + 3600
|
|
rejected = json.dumps({"type": "rate_limit_event", "rate_limit_info": {"status": "rejected", "resetsAt": resets_at}})
|
|
limited = json.dumps({"type": "result", "api_error_status": 429, "is_error": True, "result": "You've hit your session limit"})
|
|
run = checkout.run([worker(exit=1), worker(exit=1), worker(stream=f"{rejected}\n{limited}\n", result=False, exit=1), worker(exit=1), worker(), judge(stdout=limited, exit=1), judge()])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker"] * 5 + ["judge"] * 2 + ["worker"])
|
|
self.assertEqual([round(seconds, -1) for seconds in run.sleeps], [60, 60, 3720, 60, 600])
|
|
self.assertIn(f"the worker hit a usage limit (You've hit your session limit); retrying at {datetime.datetime.fromtimestamp(resets_at + 120, datetime.timezone.utc):%Y-%m-%d %H:%M:%S}\n", run.stderr)
|
|
self.assertIn("the judge hit a usage limit (You've hit your session limit); retrying at ", run.stderr)
|
|
|
|
def test_retries_the_judge_alone_when_its_verdict_quotes_nothing_in_the_transcript(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([
|
|
worker(say="Shall I use A or B?"),
|
|
judge(interrupt=True, evidence="never said", question="A or B?"),
|
|
judge(finished=True, evidence=""),
|
|
judge(evidence="A or B"),
|
|
worker(),
|
|
])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "judge", "judge", "worker", "judge"])
|
|
self.assertEqual(run.sleeps, [60, 60])
|
|
self.assertIn("the judge quoted 'never said', which is not in the transcript, for the question: A or B?", run.stderr)
|
|
self.assertIn("retrying the judge in a minute: the judge quoted '', which is not in the transcript\n", run.stderr)
|
|
|
|
def test_retries_a_judge_claude_cannot_start(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(vanish=True)])
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
|
|
self.assertEqual(run.sleeps, [60, 60])
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the judge failed three times running: could not start claude: No such file or directory")
|
|
|
|
def test_hands_the_judge_the_tail_of_a_long_transcript(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(say="early " * 80000 + "LATE")])
|
|
self.assertIn("LATE", run.calls[1]["stdin"])
|
|
self.assertEqual(len(run.calls[1]["stdin"]), 400000)
|
|
self.assertLess(run.calls[1]["stdin"].count("early"), 80000)
|
|
self.assertTrue(run.calls[1]["stdin"].rstrip().endswith("CI not yet reported, an answer not yet recorded."))
|
|
|
|
def test_skips_a_malformed_transcript_or_stream_line(self):
|
|
checkout = Checkout(self)
|
|
raw = "\xff not UTF-8\nnot JSON\n[1]\n" + json.dumps({"type": "assistant", "message": "not an object"}) + "\n" + json.dumps({"type": "user", "message": None}) + "\n"
|
|
run = checkout.run([worker(raw=raw, say="past the noise", stream="\xff not UTF-8\n"), judge(evidence="past the noise")])
|
|
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge", "worker"]))
|
|
self.assertIn("past the noise", run.stdout)
|
|
self.assertIn("past the noise", run.calls[1]["stdin"])
|
|
|
|
@unittest.skipIf(os.geteuid() == 0, "root reads an unreadable file")
|
|
def test_exits_1_when_the_transcript_cannot_be_read(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(unreadable=True)])
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
|
|
self.assertRegex(run.stderr.strip().splitlines()[-2], r"^could not archive the transcript: .*\.jsonl: Permission denied$")
|
|
self.assertRegex(run.stderr.strip().splitlines()[-1], r"^could not read the transcript .*\.jsonl: Permission denied$")
|
|
|
|
def test_retries_a_worker_claude_cannot_start(self):
|
|
checkout = Checkout(self)
|
|
(checkout.temp / "fake-bin" / "claude").unlink()
|
|
run = checkout.run([])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertEqual(run.sleeps, [60, 60])
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the worker failed three times running: could not start claude: No such file or directory")
|
|
|
|
def test_exits_1_when_claude_leaves_no_transcript(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(transcript=False)])
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker"]))
|
|
self.assertIn("claude left no transcript for session", run.stderr)
|
|
|
|
def test_exits_2_after_six_unfinished_verdicts_running_and_a_rerun_gets_six_more(self):
|
|
checkout = Checkout(self)
|
|
six = [step for _ in range(6) for step in (worker(), judge())]
|
|
run = checkout.run(six + [worker()])
|
|
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge"] * 6))
|
|
self.assertIn("the judge ruled it unfinished 6 cycles running", run.stderr.strip().splitlines()[-1])
|
|
self.assertIn("rerun loop-claude to give it 6 more cycles", run.stderr.strip().splitlines()[-1])
|
|
stops = [outcome for outcome in checkout.archived().values() if "Stopped: " in outcome]
|
|
self.assertEqual(len(stops), 1)
|
|
self.assertIn(f"\n\nWorker: {OFF_LADDER}\n", stops[0])
|
|
run = checkout.run(six[:-2] + [worker()])
|
|
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge"] * 6))
|
|
|
|
def test_archives_the_stop_on_the_route_the_sixth_verdict_stepped_to_and_a_rerun_resumes_there(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
checkout.commit_plan(CHEAP)
|
|
run = checkout.run([step for _ in range(6) for step in (worker(), judge())] + [worker()])
|
|
self.assertEqual(run.code, 2)
|
|
stepped = '<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n\n<UTC> Stopped: '
|
|
self.assertEqual(len([outcome for outcome in checkout.archived().values() if stepped in outcome]), 1)
|
|
run = checkout.run([worker()])
|
|
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "worker"], [("claude-fable-5-2", "medium")])
|
|
|
|
def test_five_unfinished_verdicts_and_a_finished_one_do_not_add_up_to_six(self):
|
|
checkout = Checkout(self)
|
|
five = [step for _ in range(5) for step in (worker(), judge())]
|
|
run = checkout.run(five + [worker(), judge(finished=True)] + five + [worker()])
|
|
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge"] * 12))
|
|
|
|
|
|
# Items 4 and 9 with predictions that do not overlap, so both may run at once, and item 12, which overlaps item 4.
|
|
BESIDE = TAKEN_IN_AT_L_4 + "| 12 | 0.1.0 | | **Do most.** | 2 | 1 | 2 | 3 | 2 | 1 by 9 | 10 |\n\n## Details\n\n### 4. Do it.\n\nTouches:\n\n- `a.py`\n\n### 9. Do more.\n\nTouches:\n\n- `b.py`\n\n### 12. Do most.\n\nTouches:\n\n- `a.py`\n"
|
|
|
|
|
|
class Windows(unittest.TestCase):
|
|
def checkout(self, parallel=None):
|
|
"""A checkout whose plan is BESIDE, with parallel-chunks set where given."""
|
|
checkout = Checkout(self)
|
|
checkout.commit_plan(BESIDE)
|
|
if parallel:
|
|
config = checkout.home / ".claude-projects" / "proj" / "config.toml"
|
|
config.write_text(f"parallel-chunks = {parallel}\n" + config.read_text())
|
|
return checkout
|
|
|
|
def started(self, run):
|
|
"""The window each new-session or new-window made, with the command it runs."""
|
|
return [(command[0], command[command.index("-n") + 1], command[command.index("--") + 1:]) for command in run.tmux if command[0] in ("new-session", "new-window")]
|
|
|
|
def test_starts_one_chunk_in_a_window_of_the_projects_session_by_default(self):
|
|
checkout = self.checkout()
|
|
run = checkout.run([], argv=["ship", "it"])
|
|
self.assertEqual((run.code, run.calls, run.stderr), (0, [], ""))
|
|
self.assertEqual(run.tmux, [
|
|
["-V"],
|
|
["list-windows", "-t", "=loop-proj", "-F", "#{window_name}\t#{pane_dead}\t#{window_id}"],
|
|
["new-session", "-d", "-s", "loop-proj", "-n", "item-4", "-c", str(checkout.root), "-e", "LOOP_CLAUDE_JUDGE_MODEL=judge-model", "-e", "LOOP_CLAUDE_WORKER_MODEL=worker-model", "-P", "-F", "#{window_id}", "--", str(checkout.script), "--chunk", "item-4", "ship it"],
|
|
["set-option", "-t", "=loop-proj:", "@loop-root", str(checkout.root)],
|
|
["set-option", "-w", "-t", "@7", "remain-on-exit", "failed"],
|
|
])
|
|
self.assertEqual(run.stdout, "started item-4 in the tmux session loop-proj\nnot on a terminal; see the chunks with `tmux attach -t =loop-proj`\n")
|
|
|
|
def test_starts_up_to_the_configured_number_of_chunks_whose_predictions_do_not_overlap(self):
|
|
run = self.checkout(parallel=3).run([], argv=[TASK])
|
|
self.assertEqual([(made, window) for made, window, _ in self.started(run)], [("new-session", "item-4"), ("new-window", "item-9")])
|
|
self.assertIn("started item-4, item-9 in the tmux session loop-proj\n", run.stdout)
|
|
|
|
def test_starts_only_what_the_running_windows_leave_room_for(self):
|
|
checkout = self.checkout(parallel=2)
|
|
# Item 4's chunk runs and waits for the human; item 9's loop failed, so its window stays, dead.
|
|
pathlib.Path(checkout.env["FAKE_TMUX_WINDOWS"]).write_text("item-4 ?\t0\t@1\nitem-9\t1\t@2\n")
|
|
run = checkout.run([], argv=[TASK])
|
|
self.assertEqual([command for command in run.tmux if command[0] == "kill-window"], [["kill-window", "-t", "@2"]])
|
|
self.assertEqual([(made, window) for made, window, _ in self.started(run)], [("new-window", "item-9")])
|
|
pathlib.Path(checkout.env["FAKE_TMUX_WINDOWS"]).write_text("item-4\t0\t@1\nitem-9\t0\t@2\n")
|
|
run = checkout.run([], argv=[TASK])
|
|
self.assertEqual((self.started(run), run.code), ([], 0))
|
|
self.assertIn(f"the tmux session loop-proj runs every chunk that may run now, so `{TASK}` started in no new window\n", run.stdout)
|
|
|
|
def test_a_finished_chunk_starts_the_next_one_its_window_leaves_room_for(self):
|
|
checkout = self.checkout(parallel=2)
|
|
pathlib.Path(checkout.env["FAKE_TMUX_WINDOWS"]).write_text("item-4\t0\t@1\nitem-9\t0\t@2\n")
|
|
run = checkout.run([worker(say="merged"), judge(finished=True, evidence="merged")], chunk="item-9")
|
|
self.assertEqual((run.code, run.roles()), (0, ["worker", "judge"]))
|
|
# Item 9 is still in the checkout's plan, since nothing merged here; the window that leaves holds it no longer, and item 4's chunk still runs.
|
|
self.assertEqual(self.started(run), [("new-window", "item-9", [str(checkout.script), "--chunk", "item-9", TASK])])
|
|
self.assertIn("the chunk finished; started item-9\n", run.stdout)
|
|
|
|
def test_a_finished_chunk_starts_the_next_item_from_the_plan_its_merge_left_on_origin(self):
|
|
checkout = Checkout(self)
|
|
origin = checkout.temp / "origin.git"
|
|
checkout.git("init", "--quiet", "--bare", "--initial-branch=main", str(origin))
|
|
checkout.git("symbolic-ref", "HEAD", "refs/heads/main")
|
|
checkout.commit_plan(TODO)
|
|
checkout.git("remote", "add", "origin", str(origin))
|
|
checkout.git("push", "--quiet", "--set-upstream", "origin", "main")
|
|
# Item 4's chunk runs a cycle and stays open, so the rerun below resumes it and runs no chunk-preflight, which would fetch.
|
|
self.assertEqual(checkout.run([worker(), judge()], chunk="item-4").code, 130)
|
|
# Item 4's chunk merges, which leaves item 6 next on origin; the checkout fetched before it.
|
|
elsewhere = checkout.temp / "elsewhere"
|
|
checkout.git("clone", "--quiet", str(origin), str(elsewhere))
|
|
(elsewhere / "todo.md").write_text(TODO.replace("| 4 | 0.1.0 |", "| 6 | 0.1.0 |"))
|
|
checkout.git("-C", str(elsewhere), "commit", "--quiet", "--all", "-m", "Finish item 4")
|
|
checkout.git("-C", str(elsewhere), "push", "--quiet", "origin", "main")
|
|
run = checkout.run([worker(say="merged"), judge(finished=True, evidence="merged")], chunk="item-4")
|
|
self.assertEqual(self.started(run), [("new-session", "item-6", [str(checkout.script), "--chunk", "item-6", TASK])])
|
|
|
|
def test_tells_a_fresh_worker_the_items_the_other_running_chunks_hold_and_starts_from_its_windows_item(self):
|
|
checkout = self.checkout(parallel=2)
|
|
checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5"
|
|
pathlib.Path(checkout.env["FAKE_TMUX_WINDOWS"]).write_text("item-4\t0\t@1\nitem-9\t0\t@2\n")
|
|
run = checkout.run([worker()], chunk="item-9")
|
|
prompt = argument(run.calls[0], "-p")
|
|
self.assertIn("The item this chunk starts from, named with --item: item 9, L 4, E + I 3.", prompt)
|
|
self.assertIn("Other chunks of this project, running or stopped, hold item 4. Leave those items out of this chunk, even where the facts above list them as items that may join it.", prompt)
|
|
self.assertIn("routed by item 9's L of 4", prompt)
|
|
|
|
def test_refuses_to_start_on_a_tmux_older_than_needed_or_without_one(self):
|
|
checkout = self.checkout()
|
|
checkout.env["FAKE_TMUX_VERSION"] = "tmux 3.1c"
|
|
run = checkout.run([], argv=[TASK])
|
|
self.assertEqual((run.code, run.stderr, run.tmux), (1, "loop-claude runs each chunk in a tmux window, and needs tmux 3.2 or later; it found tmux 3.1c\n", [["-V"]]))
|
|
(pathlib.Path(checkout.env["PATH"]) / "tmux").unlink()
|
|
run = checkout.run([], argv=[TASK])
|
|
self.assertEqual((run.code, run.stderr), (1, "loop-claude runs each chunk in a tmux window, and needs tmux 3.2 or later; it found no tmux\n"))
|
|
|
|
def test_marks_the_window_while_an_interrupt_waits_and_rings_its_bell(self):
|
|
checkout = self.checkout()
|
|
run = checkout.run([worker(say="A or B?"), judge(interrupt=True, question="A or B?")], chunk="item-4")
|
|
self.assertEqual(run.code, 2)
|
|
self.assertEqual([command for command in run.tmux if command[0] == "rename-window"], [["rename-window", "-t", "%3", "item-4 ?"]])
|
|
self.assertEqual(run.raw_stderr.count("\a"), 1)
|
|
run = checkout.run([worker(say="A or B?"), judge(interrupt=True, question="A or B?"), worker()], chunk="item-4", terminal_input="B\n", before_typing=ANSWERING)
|
|
self.assertEqual([command for command in run.tmux if command[0] == "rename-window"], [["rename-window", "-t", "%3", "item-4 ?"], ["rename-window", "-t", "%3", "item-4"]])
|
|
|
|
|
|
class Interrupt(unittest.TestCase):
|
|
def test_exits_2_off_a_terminal_and_resumes_on_the_next_run(self):
|
|
checkout = Checkout(self)
|
|
checkout.env["LOOP_CLAUDE_TRANSCRIPTS"] = str(checkout.temp / "deep" / "archive")
|
|
steps = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
|
|
run = checkout.run(steps)
|
|
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge"]))
|
|
self.assertEqual(run.stderr.strip().splitlines()[-2:], ["A or B?", "the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal"])
|
|
[(copy, verdict)] = checkout.archived().items()
|
|
self.assertEqual(copy, (checkout.transcripts()[0]).read_text())
|
|
self.assertEqual(verdict, f"Task: {TASK}\n\n<UTC> Verdict: interrupt\n\nA or B?\n\nEvidence:\n> Shall I use A or B?\n\nConfig: <config>\nJudge: judge-model\nWorker: {OFF_LADDER}\n\n<UTC> Stopped: the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal\n\nWorker: {OFF_LADDER}\n")
|
|
self.assertEqual(stat.S_IMODE((checkout.temp / "deep").stat().st_mode), 0o700)
|
|
self.assertIn(f"transcript: {checkout.archive() / 'proj'}/", run.stderr)
|
|
self.assertRegex(run.raw_stderr, rf"(?m)^{STAMP} transcript: ")
|
|
session = argument(run.calls[0], "--session-id")
|
|
run = checkout.run(steps, terminal_input="", stdout_on_terminal=False)
|
|
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge"]))
|
|
self.assertEqual(argument(run.calls[0], "--resume"), session)
|
|
self.assertEqual(len(checkout.archived()), 2)
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the human is needed to answer the question above, but stdin or stdout is not a terminal; rerun loop-claude on a terminal")
|
|
run = checkout.run([worker(transcript=False), steps[1]])
|
|
self.assertEqual(run.code, 2)
|
|
self.assertEqual(len(checkout.archived()), 2)
|
|
|
|
def test_sends_the_dm_through_the_notifier_and_env_file_the_config_names_and_no_credential_reaches_a_prompt_a_line_printed_or_a_file_written(self):
|
|
checkout = Checkout(self)
|
|
token = "tok-3f9a1c7e-sentinel"
|
|
env_file = checkout.home / ".claude-projects" / "proj.env"
|
|
env_file.write_text(f"MATTERMOST_URL=https://127.0.0.1:1\nMATTERMOST_TOKEN={token}\nMATTERMOST_RECIPIENT=lilleman\n")
|
|
env_file.chmod(0o600)
|
|
with (checkout.home / ".claude-projects" / "proj" / "config.toml").open("a") as config:
|
|
config.write('env-file = "~/.claude-projects/proj.env"\nnotifier = "mattermost"\n')
|
|
run = checkout.run([worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")])
|
|
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge"]))
|
|
self.assertIn("could not DM you: cannot reach https://127.0.0.1:1", run.stderr)
|
|
held = [run.raw_stdout, run.raw_stderr, json.dumps(run.calls), json.dumps(run.starts)]
|
|
held += [file.read_text(errors="replace") for file in checkout.home.rglob("*") if file.is_file() and file != env_file and not file.is_symlink()]
|
|
self.assertGreater(len(held), 8, "the walk reached the archive, the state and the transcripts")
|
|
for text in held:
|
|
self.assertNotIn(token, text)
|
|
|
|
def test_resumes_the_worker_headless_with_the_answer_typed_at_its_terminal_not_with_keys_typed_before_it(self):
|
|
checkout = Checkout(self)
|
|
asked = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
|
|
run = checkout.run([*asked, worker(say="Using B."), judge(evidence="Using B.")], terminal_input="B\n", before_typing=ANSWERING, typed_early="stray\n")
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "worker", "judge", "worker"])
|
|
self.assertIn("A or B?\nAnswer (/talk opens claude, Enter stops the loop): ", run.stderr)
|
|
self.assertEqual(argument(run.calls[2], "--resume"), argument(run.calls[0], "--session-id"))
|
|
self.assertIn("[human] B\n\nUsing B.", run.calls[3]["stdin"])
|
|
self.assertNotIn("stray", run.calls[3]["stdin"])
|
|
|
|
def test_hands_the_terminal_over_and_exits_2_when_nothing_is_typed(self):
|
|
checkout = Checkout(self)
|
|
checkout.git("commit", "--allow-empty", "--quiet", "-m", "start")
|
|
claude_json = json.dumps({"projects": {str(checkout.root): {"kept": True}}})
|
|
checkout.claude_json().write_text(claude_json)
|
|
asked = [worker(say="Shall I use A or B?"), judge(interrupt=True, evidence="Shall I use A or B?", question="A or B?")]
|
|
(checkout.root / "sub").mkdir()
|
|
run = checkout.run([*asked, interrupt(type="B"), judge(evidence="B"), worker()], cwd=checkout.root / "sub", terminal_input="/talk\n", before_typing=ANSWERING)
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge", "interrupt", "judge", "worker", "judge"])
|
|
handed = run.calls[2]["argv"]
|
|
self.assertEqual(run.calls[2]["cwd"], str(checkout.root))
|
|
self.assertEqual(argument(run.calls[2], "--resume"), argument(run.calls[0], "--session-id"))
|
|
self.assertEqual(argument(run.calls[2], "--model"), "worker-model")
|
|
self.assertNotIn("-p", handed)
|
|
self.assertIn("[human] B", run.calls[3]["stdin"])
|
|
self.assertEqual(run.calls[2]["sandboxed"], "1")
|
|
self.assertEqual(checkout.claude_json().read_text(), claude_json)
|
|
|
|
for transcript in checkout.transcripts():
|
|
transcript.unlink()
|
|
run = checkout.run([*asked, interrupt()], terminal_input="/talk\n", before_typing=ANSWERING)
|
|
self.assertEqual((run.code, run.roles()), (2, ["worker", "judge", "interrupt"]))
|
|
self.assertTrue(run.stderr.strip().endswith("the human is needed to answer the question above, but the interrupt ended without anything typed"))
|
|
|
|
run = checkout.run([worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?", vanish=True)], terminal_input="/talk\n", before_typing=ANSWERING)
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge"]))
|
|
self.assertTrue(run.stderr.strip().endswith("could not start claude for the interrupt: No such file or directory"))
|
|
|
|
def test_leaves_ctrl_c_to_the_claude_open_for_talk(self):
|
|
checkout = Checkout(self)
|
|
asked = [worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?")]
|
|
run = checkout.run([*asked, interrupt(type="B", ctrl_c=True), judge(evidence="B")], terminal_input="/talk\n", before_typing=ANSWERING)
|
|
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge", "interrupt", "judge", "worker"]))
|
|
self.assertFalse(run.calls[2]["ctrl_c_ignored"])
|
|
self.assertIn("[human] B", run.calls[3]["stdin"])
|
|
|
|
@unittest.skipIf(os.geteuid() == 0, "root reads an unreadable file")
|
|
def test_exits_1_when_the_transcript_cannot_be_read_after_the_interrupt(self):
|
|
checkout = Checkout(self)
|
|
run = checkout.run([worker(say="A or B?"), judge(interrupt=True, evidence="A or B?", question="A or B?"), interrupt(type="B", unreadable=True)], terminal_input="/talk\n", before_typing=ANSWERING)
|
|
self.assertEqual((run.code, run.roles()), (1, ["worker", "judge", "interrupt"]))
|
|
self.assertRegex(run.stderr.strip().splitlines()[-1], r"^could not tell whether you typed anything: could not read the transcript .*\.jsonl: Permission denied$")
|
|
|
|
|
|
class Models(unittest.TestCase):
|
|
def catalog(self, checkout):
|
|
folder = checkout.home / ".claude" / "cache" / "model-catalog"
|
|
folder.mkdir(parents=True)
|
|
(folder / "old.json").write_text(json.dumps({"catalog": {"config": {"models": [{"id": "old-model"}]}}, "fetchedAt": 1}))
|
|
(folder / "new.json").write_text(json.dumps({"catalog": {"config": {"models": [{"id": "model-a", "name": "A"}, {"id": "model-b", "name": "B"}]}}, "fetchedAt": 2}))
|
|
|
|
def test_asks_a_project_without_a_model_on_the_terminal_and_saves_the_answer(self):
|
|
checkout = Checkout(self)
|
|
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
|
|
self.catalog(checkout)
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (2, []))
|
|
self.assertEqual(run.stderr.strip(), f'the human is needed to pick proj\'s model, or merge {{"proj": "<model>"}} into {checkout.models_file()}, but stdin or stdout is not a terminal; rerun loop-claude on a terminal')
|
|
|
|
run = checkout.run([worker()], terminal_input="\x04")
|
|
self.assertEqual((run.code, run.calls), (2, []))
|
|
self.assertEqual(run.stderr.strip().splitlines()[-1], "the human is needed to pick proj's model, but the input ended")
|
|
|
|
checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "judge-model"
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (2, []))
|
|
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"]
|
|
run = checkout.run([worker(), judge()], terminal_input="\n9\n2\n")
|
|
self.assertEqual(run.code, 130)
|
|
self.assertIn("proj runs on model-b", run.stderr)
|
|
self.assertIn("1. A (model-a)", run.stderr)
|
|
self.assertNotIn("old-model", run.stderr)
|
|
self.assertIn("pick 1 to 2", run.stderr)
|
|
self.assertEqual({argument(call, "--model") for call in run.calls}, {"model-b"})
|
|
self.assertEqual(json.loads(checkout.models_file().read_text()), {"proj": "model-b"})
|
|
self.assertEqual(stat.S_IMODE(checkout.models_file().stat().st_mode), 0o600)
|
|
|
|
checkout.env["LOOP_CLAUDE_JUDGE_MODEL"] = "judge-model"
|
|
run = checkout.run([worker(), judge()])
|
|
self.assertEqual([argument(call, "--model") for call in run.calls], ["model-b", "judge-model", "model-b"])
|
|
|
|
def test_keeps_the_models_file_mode_and_other_projects_when_saving(self):
|
|
checkout = Checkout(self)
|
|
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
|
|
checkout.models_file().parent.mkdir(parents=True)
|
|
checkout.models_file().write_text(json.dumps({"other": "model-x"}))
|
|
checkout.models_file().chmod(0o640)
|
|
def saved_elsewhere():
|
|
checkout.models_file().write_text(json.dumps({"late": "model-y", "other": "model-x"}))
|
|
|
|
run = checkout.run([worker()], terminal_input="custom-model\n", before_typing=("no model catalog cached", saved_elsewhere))
|
|
self.assertEqual(run.code, 130)
|
|
self.assertIn("no model catalog cached", run.stderr)
|
|
self.assertIn("A name claude --model takes: ", run.stderr)
|
|
self.assertEqual(json.loads(checkout.models_file().read_text()), {"late": "model-y", "other": "model-x", "proj": "custom-model"})
|
|
self.assertEqual(stat.S_IMODE(checkout.models_file().stat().st_mode), 0o640)
|
|
|
|
for broken in ("{", "[]", '{"proj": 1}'):
|
|
with self.subTest(broken=broken):
|
|
checkout.models_file().write_text(broken)
|
|
run = checkout.run([worker()])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertTrue(run.stderr.startswith(f"could not read {checkout.models_file()}: "))
|
|
self.assertEqual(checkout.models_file().read_text(), broken)
|
|
|
|
def test_offers_no_catalog_whose_models_are_not_a_list(self):
|
|
checkout = Checkout(self)
|
|
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
|
|
self.catalog(checkout)
|
|
catalog = checkout.home / ".claude" / "cache" / "model-catalog" / "new.json"
|
|
for shape in ({"catalog": {"config": {"models": []}}}, {"catalog": None}, {"catalog": {"config": []}}, {"catalog": {"config": {"models": "x"}}}):
|
|
with self.subTest(shape=shape):
|
|
catalog.write_text(json.dumps({**shape, "fetchedAt": 3}))
|
|
checkout.models_file().unlink(missing_ok=True)
|
|
run = checkout.run([worker()], terminal_input="1\n")
|
|
self.assertIn("no model catalog cached", run.stderr)
|
|
self.assertEqual(argument(run.calls[0], "--model"), "1")
|
|
|
|
@unittest.skipIf(os.geteuid() == 0, "root writes into a read-only directory")
|
|
def test_exits_1_when_the_models_file_cannot_be_written(self):
|
|
checkout = Checkout(self)
|
|
del checkout.env["LOOP_CLAUDE_JUDGE_MODEL"], checkout.env["LOOP_CLAUDE_WORKER_MODEL"]
|
|
checkout.models_file().parent.mkdir(parents=True, mode=0o500)
|
|
self.addCleanup(checkout.models_file().parent.chmod, 0o700)
|
|
run = checkout.run([worker()], terminal_input="custom-model\n")
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertIn(f"could not save the model in {checkout.models_file()}: Permission denied", run.stderr)
|
|
|
|
|
|
class Reload(unittest.TestCase):
|
|
def test_an_edit_takes_effect_at_the_next_cycle_boundary_once_it_answers_help(self):
|
|
checkout = Checkout(self)
|
|
copy = checkout.temp / "bin"
|
|
shutil.copytree(BIN, copy, ignore=shutil.ignore_patterns("__pycache__", "test_*"))
|
|
checkout.script = copy / "loop-claude"
|
|
checkout.env["FAKE_CLAUDE_EDIT"] = str(copy)
|
|
run = checkout.run([
|
|
worker(append=["loop-claude", "\ndef\n"]), judge(),
|
|
worker(replace=["loop-claude", "\ndef\n", "\n# edited\n"]), judge(),
|
|
worker(append=["loop_claude/nested/edited.py", "# edited\n"]), judge(),
|
|
worker(append=["records/file_writes.py", "# edited\n"]), judge(),
|
|
worker(),
|
|
])
|
|
self.assertEqual(run.code, 130)
|
|
self.assertEqual(run.roles(), ["worker", "judge"] * 5)
|
|
self.assertIn("keeping the running script", run.stderr)
|
|
reloads = [json.loads(argv[1]) for argv in run.starts if argv[:1] == ["--reloaded"]]
|
|
self.assertEqual(reloads, [{"cycle_number": cycle, "root": str(checkout.root), "task": TASK, "unfinished_evidence": "working", "route": OFF_LADDER_ROUTE, "unfinished_run": cycle, "chunk": "no-item"} for cycle in (2, 3, 4)])
|
|
self.assertIn("=== cycle 5,", run.stdout)
|
|
|
|
outside = checkout.temp / "outside"
|
|
outside.mkdir()
|
|
state = {"cycle_number": 3, "root": str(checkout.root), "task": TASK, "unfinished_evidence": "", "route": None, "chunk": "no-item"}
|
|
run = checkout.run([worker()], argv=["--reloaded", json.dumps(state)], cwd=outside)
|
|
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge"]))
|
|
self.assertEqual(run.calls[0]["cwd"], str(checkout.root))
|
|
self.assertIn("=== cycle 4,", run.stdout)
|
|
|
|
def test_a_loop_started_before_chunks_ran_in_windows_reloads_into_starting_the_projects_windows(self):
|
|
checkout = Checkout(self)
|
|
state = {"cycle_number": 3, "root": str(checkout.root), "task": TASK, "unfinished_evidence": "", "route": None, "slot": 0}
|
|
run = checkout.run([worker()], argv=["--reloaded", json.dumps(state)])
|
|
self.assertEqual((run.code, run.calls), (0, []))
|
|
self.assertEqual([command[0] for command in run.tmux], ["-V", "list-windows", "new-session", "set-option", "set-option"])
|
|
self.assertIn("started no-item in the tmux session loop-proj\n", run.stdout)
|
|
|
|
def test_stops_where_a_reload_hands_no_state(self):
|
|
checkout = Checkout(self)
|
|
|
|
run = checkout.run([], argv=["--reloaded", "{}"])
|
|
self.assertEqual((run.code, run.calls), (1, []))
|
|
self.assertIn("rerun loop-claude <task>", run.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|