Files
claude-config/bin/review_replay/test_review_replay.py
T

631 lines
40 KiB
Python

import contextlib
import dataclasses
import io
import json
import os
import pathlib
import re
import shutil
import subprocess
import sys
import tempfile
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from review_replay.rounds import authored_by, lost_inputs, recorded_findings, reviewed_path, Round, round_from, sample, stamp_seconds
from review_replay import changes, main, replay
from text_files.doubles import compress
from project_config.doubles import configured, write_config
from loop_effects.effects import Effects, system_effects
from claude_code.entries import entry_from
from failure import Failure
from review_rules.reviewer_context import reviewer_context
from review_rules.review_passes import REVIEWERS
from model_routing.ladder import Rung
TOOL = pathlib.Path(__file__).resolve().parents[1] / "review-replay"
REFERENCE = Rung("claude-opus-5-5", "medium")
LOW = Rung("claude-opus-5-5", "low")
MIDDLE = Rung("claude-sonnet-5-5", "high")
def entry(kind, text, **fields):
return {"type": kind, "message": {"content": [{"type": "text", "text": text}]}, **fields}
def kinds(found):
return [entry_from(one) for one in found]
def a_round(at, findings=("🔴 a",)):
return Round("stability-reviewer", "/code/p", "x", f"2026-10-14T10:00:0{at}Z", 0, "Review it.", tuple(findings))
class RecordedFindings(unittest.TestCase):
def test_takes_each_red_and_yellow_finding_with_the_lines_up_to_the_next_blank_line_or_finding(self):
report_text = "BLOCK\n\n1 🔴, 2 🟡 in all.\n\n1. 🔴 **Loop** at a.py:3\nnever ends.\n- 🟡 b.py:1 slow\n| 🟡 | c.py | odd |\n\n🟢 nit\n### 🔴 Heading\n**🟡 Bold**"
self.assertEqual(recorded_findings(report_text), ["1. 🔴 **Loop** at a.py:3\nnever ends.", "- 🟡 b.py:1 slow", "| 🟡 | c.py | odd |", "### 🔴 Heading", "**🟡 Bold**"])
def test_skips_a_count_or_verdict_line_that_names_the_marks(self):
self.assertEqual(recorded_findings("APPROVE\n\nNo 🔴 or 🟡 findings.\n0 🔴, 0 🟡"), [])
class RoundFrom(unittest.TestCase):
def test_reads_the_prompt_the_last_report_and_when_the_reviewer_started(self):
entries = [
entry("user", "Review branch x in /code/p-x.", timestamp="2026-10-14T10:00:00.500Z", cwd="/code/p-x", gitBranch="x"),
entry("assistant", "Reading."),
entry("assistant", "BLOCK\n\n🔴 a.py:1 breaks"),
]
self.assertEqual(round_from("stability-reviewer", "/code/p", 5.0, kinds(entries)), Round("stability-reviewer", "/code/p", "x", "2026-10-14T10:00:00.500Z", 5.0, "Review branch x in /code/p-x.", ("🔴 a.py:1 breaks",)))
self.assertEqual(round_from("stability-reviewer", "/code/p", 5.0, kinds(entries)).seconds, 1791972000.5)
def test_reads_a_prompt_given_as_a_string(self):
start = {"type": "user", "message": {"content": "Review branch x in /code/p-x."}, "timestamp": "2026-10-14T10:00:00Z", "gitBranch": "x"}
self.assertEqual(round_from("stability-reviewer", "/code/p", 5.0, kinds([start, entry("assistant", "🔴 a")])).prompt, "Review branch x in /code/p-x.")
def test_skips_a_round_with_no_red_or_yellow_finding_or_no_report(self):
start = entry("user", "Review.", timestamp="2026-10-14T10:00:00Z", cwd="/code/p", gitBranch="x")
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, kinds([start, entry("assistant", "APPROVE\n\n🟢 nit")])))
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, kinds([start])))
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, kinds([])))
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, kinds([{**start, "timestamp": "yesterday"}, entry("assistant", "🔴 a")])))
class ReviewedPath(unittest.TestCase):
def test_names_the_worktree_the_prompt_names_that_is_gone(self):
exists = {"/code", "/code/p", "/code/p/worktrees", "/tmp"}.__contains__
self.assertEqual(reviewed_path("Review /code/p-x/src/a.py:3-5; the diff is at /tmp/s/x.diff; /code/p is older.", "/code/p", exists), "/code/p-x")
self.assertEqual(reviewed_path("Review `/code/p/worktrees/x`.", "/code/p", exists), "/code/p/worktrees/x")
def test_names_the_checkout_when_the_prompt_names_no_gone_worktree_and_none_when_it_names_neither_or_two(self):
exists = {"/code", "/code/p"}.__contains__
self.assertEqual(reviewed_path("Review /code/p.", "/code/p", exists), "/code/p")
self.assertIsNone(reviewed_path("Review the branch.", "/code/p", exists))
self.assertIsNone(reviewed_path("Review /code/p-x and /code/p-43.", "/code/p", exists))
class LostInputs(unittest.TestCase):
def test_names_each_file_the_prompt_names_in_the_copy_that_it_lacks_other_than_a_round_report(self):
prompt = "Read /c/branch.diff and /c/ls-files.txt:3. Earlier: /c/prose-round-1.md. Write /c/principles-round-2-rerun.md; see /elsewhere/x.diff."
self.assertEqual(lost_inputs(prompt, "/c", {"ls-files.txt"}.__contains__), ["branch.diff"])
self.assertEqual(lost_inputs(prompt, "/c", {"branch.diff", "ls-files.txt"}.__contains__), [])
class AuthoredBy(unittest.TestCase):
def test_takes_the_newest_commit_authored_by_then(self):
log = [("c3", 30), ("c2", 20), ("c1", 10)]
self.assertEqual(authored_by(log, 25), "c2")
self.assertEqual(authored_by(log, 30), "c3")
self.assertIsNone(authored_by(log, 5))
def test_takes_the_newest_author_time_over_a_commit_authored_early_but_merged_late(self):
log = [("z-bump", 5), ("a-new", 30), ("m-mid", 20)]
self.assertEqual(authored_by(log, 25), "m-mid")
self.assertEqual(authored_by(log, 10), "z-bump")
def test_breaks_a_tie_by_log_order(self):
self.assertEqual(authored_by([("a-merged", 20), ("z-earlier", 20)], 25), "a-merged")
class Sample(unittest.TestCase):
def test_keeps_the_latest_rounds_in_the_order_they_ran(self):
rounds = [a_round(2), a_round(1), a_round(3)]
self.assertEqual(sample(rounds, 2), [rounds[0], rounds[2]])
class Launch(unittest.TestCase):
def setUp(self):
self.home = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
(self.home / ".claude").mkdir()
(self.home / ".claude" / "CLAUDE.md").write_text("## Shared goal\n\nHealth.\n\n## Prose\n\nShort.\n\n## Vocabulary\n\nWords.\n")
self.clone = self.home / "clone"
self.reports = self.home / "reports"
self.clone.mkdir()
self.checkout = self.home / "proj"
self.checkout.mkdir()
self.config = configured(self, self.checkout, home=self.home) / "config.toml"
def test_runs_the_reviewer_as_archived_with_claude_md(self):
self.assertEqual(replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, None), (["--agent", "prose-reviewer", *REFERENCE.flags()], "Review it."))
def test_drops_claude_md_and_hands_the_reviewer_what_the_hook_and_the_header_would(self):
(self.clone / "AGENTS.md").write_text("")
args, prompt = replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, changes.OMIT)
self.assertEqual(args, ["--agent", "prose-reviewer", *REFERENCE.flags(), "--settings", json.dumps({"env": {"CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1"}})])
self.assertEqual(prompt, f"{reviewer_context(self.home)}\n\nReview it.\nPrinciples: read {self.clone}/AGENTS.md first, for the project principles and the decision titles.")
def test_names_a_record_outside_the_checkout_where_it_stands(self):
self.config.write_text(self.config.read_text().replace('"checkout:AGENTS.md"', '"folder:AGENTS.md"'))
(self.clone / "AGENTS.md").write_text("")
(self.config.parent / "AGENTS.md").write_text("")
prompt = replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, changes.OMIT)[1]
self.assertTrue(prompt.endswith(f"Principles: read {self.config.parent}/AGENTS.md first, for the project principles and the decision titles."), prompt)
def test_names_the_decision_log_in_the_clone_or_where_it_stands_outside_the_checkout_without_agents_md(self):
places = {"checkout:docs/decisions.md": self.clone / "docs" / "decisions.md", "folder:decisions.md": self.config.parent / "decisions.md"}
for setting, decisions in places.items():
with self.subTest(setting):
self.config.write_text(self.config.read_text().replace('"checkout:docs/decisions.md"', f'"{setting}"'))
decisions.parent.mkdir(exist_ok=True)
decisions.write_text("1. A principle.\n\n# Decisions\n")
prompt = replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, changes.OMIT)[1]
self.assertTrue(prompt.endswith(f"Principles: read {decisions} first, up to `# Decisions`, for the project principles and the decision titles."), prompt)
def test_fails_without_a_config_for_the_checkout(self):
self.config.unlink()
self.assertIn("no config names the checkout", replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, changes.OMIT).reason)
def test_fails_without_the_sections_to_hand_over(self):
(self.home / ".claude" / "CLAUDE.md").write_text("")
self.assertIsInstance(replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, changes.OMIT), Failure)
class Match(unittest.TestCase):
def run_match(self, stdout):
self.asked = []
def claude(args, cwd, prompt):
self.asked.append((args, cwd, prompt))
return subprocess.CompletedProcess(args, 0, stdout, "")
return replay.match(("🔴 a", "🟡 b"), "the other report", "/code/p", REFERENCE, dataclasses.replace(system_effects(pathlib.Path.home(), os.environ), claude=claude))
def test_asks_for_the_numbered_findings_the_other_report_raises_as_json_with_no_tools(self):
self.run_match("{}")
[(args, cwd, prompt)] = self.asked
self.assertEqual(cwd, "/code/p")
self.assertIn("1. 🔴 a\n\n2. 🟡 b\n\nThe other report:\nthe other report", prompt)
self.assertEqual(json.loads(args[args.index("--json-schema") + 1]), replay.MATCH_SCHEMA)
self.assertEqual(args[args.index("--tools") + 1], "")
self.assertEqual(args[args.index("--model"):args.index("--model") + 4], REFERENCE.flags())
def test_takes_the_numbers_in_range_and_fails_on_a_reply_of_another_shape(self):
self.assertEqual(self.run_match(json.dumps({"type": "result", "structured_output": {"found": [2, 0, 3, 1]}})), {0, 1})
for output in ({"found": ["1"]}, {"found": 1}, [1]):
with self.subTest(output):
self.assertIsInstance(self.run_match(json.dumps({"type": "result", "structured_output": output})), Failure)
def unused(*_):
raise AssertionError("not called")
def run_git(cwd, *args, at=None):
"""at, when given, is the commit's author and committer time, so git log lists the commits in a fixed order."""
env = {**os.environ, "GIT_AUTHOR_DATE": at, "GIT_COMMITTER_DATE": at} if at else None
subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], cwd=cwd, check=True, capture_output=True, env=env)
class Main(unittest.TestCase):
"""The tool on a temporary home: a chunk that began after the checkout's first commit, and whose stability reviewer reviewed branch x at its second, from a worktree since removed; a prose reviewer's files sort before the stability reviewer's."""
def setUp(self):
self.home = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
self.archive = self.home / ".claude" / "loop-claude-transcripts"
(self.home / ".claude").mkdir()
(self.home / ".claude" / "loop-claude-models.json").write_text(json.dumps({"proj": "claude-opus-5-5", "other": "claude-sonnet-5-5"}))
origin, self.checkout = self.home / "origin.git", self.home / "code" / "proj"
run_git(self.home, "init", "-q", "--bare", "-b", "main", str(origin))
run_git(self.home, "clone", "-q", str(origin), str(self.checkout))
for n, stamp in enumerate(("2026-10-14T09:00:00Z", "2026-10-14T09:30:00Z", "2026-10-14T11:00:00Z")):
(self.checkout / "a.txt").write_text(f"{n}\n")
run_git(self.checkout, "add", "a.txt")
run_git(self.checkout, "commit", "-qm", f"c{n}", at=stamp)
run_git(self.checkout, "push", "-q", "origin", "main")
run_git(self.checkout, "remote", "set-head", "origin", "main")
write_config(self.home, self.checkout)
for key in ("proj", "other"):
self.write_chunk(key, "c1", "2026-10-14T10:00:00Z")
(self.home / ".claude" / "CLAUDE.md").write_text("## Shared goal\n\nHealth.\n\n## Prose\n\nShort.\n\n## Vocabulary\n\nWords.\n")
# The fixture holds one round.
self.limits = {"enough_rounds": 1}
self.reviews = []
self.review_settings = []
self.agents = set()
self.matched_in = []
def write_chunk(self, key, chunk, at, began="2026-10-14T09:15:00Z"):
"""A chunk that began at began, whose stability round began at at."""
subagents = self.archive / key / chunk / "session" / "s" / "subagents"
subagents.mkdir(parents=True, exist_ok=True)
(subagents.parents[2] / "t0.jsonl").write_text(json.dumps({"type": "summary"}) + "\n" + json.dumps(entry("user", "Do the next chunk.", timestamp=began)) + "\n")
(subagents / "agent-a.meta.json").write_text(json.dumps({"agentType": "prose-reviewer"}))
(subagents / "agent-b.meta.json").write_text(json.dumps({"agentType": "stability-reviewer"}))
gone = self.home / "code" / "proj-x"
lines = [entry("user", f"Review branch x in {gone}.", timestamp=at, cwd=str(gone), gitBranch="x"), entry("assistant", "BLOCK\n\n🔴 a.txt:1 wrong\n\n🟡 slow")]
for name in ("agent-a.jsonl", "agent-b.jsonl"):
(subagents / name).write_text("".join(json.dumps(line) + "\n" for line in lines))
def rewrite_round(self, change):
path = self.archive / "proj" / "c1" / "session" / "s" / "subagents" / "agent-b.jsonl"
lines = [json.loads(line) for line in path.read_text().splitlines()]
change(lines[0])
path.write_text("".join(json.dumps(line) + "\n" for line in lines))
def set_branch(self, branch):
self.rewrite_round(lambda first: first.update(gitBranch=branch))
def set_prompt(self, prompt):
self.rewrite_round(lambda first: first["message"]["content"][0].update(text=prompt))
def run_main(self, argv, found, review=lambda rung: {"type": "result", "result": f"review on {rung.model} {rung.effort}"}, env={}):
"""found(rung) is the indexes of the recorded findings the reviewer on rung finds; review(rung), that reviewer's reply."""
def claude(args, cwd, prompt):
rung = Rung(args[args.index("--model") + 1], args[args.index("--effort") + 1])
if "--agent" in args:
shown = [subprocess.run(["git", *command], cwd=cwd, capture_output=True, text=True).stdout for command in (["show", "HEAD:a.txt"], ["branch", "--show-current"], ["show", "origin/main:a.txt"])]
self.reviews.append((rung, *shown, prompt, cwd))
self.review_settings.append(json.loads(args[args.index("--settings") + 1])["env"] if "--settings" in args else None)
self.agents.add(args[args.index("--agent") + 1])
result = review(rung)
else:
self.matched_in.append(cwd)
reviewed = Rung(*re.search(r"review on (\S+) (\S+)", prompt).groups())
result = found(reviewed)
result = result if "api_error_status" in result else {"type": "result", "structured_output": {"found": [n + 1 for n in result["found"]]}}
return subprocess.CompletedProcess(args, 0, json.dumps(result), "")
effects = Effects(claude=claude, claude_on_terminal=unused, discard_typed=unused, dm=unused, execv=unused, git=system_effects(pathlib.Path.home(), os.environ).git, input=unused, local_time=unused, memory=unused, now=lambda: 1790000000.0, on_terminal=unused, python=unused, replace_file=unused, sleep=unused, stream_claude=unused, timer=unused, tmux=unused, tmux_on_terminal=unused)
out, err = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
code = main.main(argv, env, self.home, effects, **self.limits)
return code, out.getvalue(), err.getvalue()
def test_keeps_each_reviewer_and_match_run_and_the_printed_report_in_one_folder_of_the_archive(self):
code, out, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0, 1]})
self.assertEqual(code, 0, err)
folder = self.home / ".claude" / "loop-claude-transcripts" / "replays" / "review-replay-stability-claude-opus-5-5-20260921T141320Z"
self.assertEqual((folder / "result.md").read_text(encoding="utf-8"), out)
runs = [json.loads(path.read_text(encoding="utf-8").splitlines()[0]) for path in folder.glob("*.jsonl")]
# A run that repeats another word for word, as the fakes' do, is kept once.
self.assertEqual({run["prompt"] for run in runs if "--agent" in run["args"]}, {prompt for *_, prompt, _ in self.reviews})
self.assertTrue(any("--json-schema" in run["args"] for run in runs))
def test_without_claude_md_replays_each_round_three_times_with_it_and_once_without_on_the_project_rung(self):
finds = iter(([0, 1], [0], [0, 1], [1]))
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"found": next(finds)})
self.assertEqual(code, 0, err)
self.assertEqual([rung for rung, *_ in self.reviews], [REFERENCE] * 4)
context = reviewer_context(self.home)
self.assertEqual([prompt.startswith(context) for *_, prompt, _ in self.reviews], [False, False, False, True])
self.assertEqual(out, (
f"Replayed 1 stability rounds of claude-opus-5-5 projects, which hold 2 🔴 and 🟡 findings, four times on {REFERENCE}: three {changes.OMIT.unchanged}, one {changes.OMIT.changed}.\n"
"With CLAUDE.md the three replays found 2, 1 and 2 of them, and missed 0, 1 and 0 of the findings that both of the other two found.\n"
"Without CLAUDE.md it found 1 of them, and missed 1 of the 1 findings that both of the first two found:\n"
f" {self.checkout} 2026-10-14T10:00:00Z: 🔴 a.txt:1 wrong\n"
"It passes: 1 missed is no more than the most a replay with CLAUDE.md missed, 1.\n"
))
with_them = f"replaying 1 stability rounds with CLAUDE.md on {REFERENCE}\n"
self.assertEqual(err, with_them * 3 + f"replaying 1 stability rounds without CLAUDE.md on {REFERENCE}\n")
def test_a_change_is_refused_where_the_replays_would_inherit_its_variable_from_the_environment(self):
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--bash-ceiling"], lambda rung: {"found": [0]}, env={"BASH_MAX_OUTPUT_LENGTH": "6000"})
self.assertEqual((code, out, self.reviews), (1, "", []))
self.assertIn("BASH_MAX_OUTPUT_LENGTH", err)
code, _, err = self.run_main(["stability", "claude-opus-5-5", "--bash-ceiling"], lambda rung: {"found": [0]}, env={"CLAUDE_CODE_FILE_READ_MAX_OUTPUT_TOKENS": "5000"})
self.assertEqual(code, 0, err)
def test_a_bash_ceiling_replays_each_round_three_unchanged_and_once_under_the_ceiling_with_nothing_else_changed(self):
finds = iter(([0, 1], [0], [0, 1], [1]))
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--bash-ceiling"], lambda rung: {"found": next(finds)})
self.assertEqual(code, 0, err)
self.assertEqual(self.review_settings, [None, None, None, {"BASH_MAX_OUTPUT_LENGTH": "6000"}])
context = reviewer_context(self.home)
self.assertEqual([prompt.startswith(context) for *_, prompt, _ in self.reviews], [False] * 4)
self.assertEqual(out, (
f"Replayed 1 stability rounds of claude-opus-5-5 projects, which hold 2 🔴 and 🟡 findings, four times on {REFERENCE}: three as archived, one under a Bash ceiling of 6,000 characters.\n"
"As archived the three replays found 2, 1 and 2 of them, and missed 0, 1 and 0 of the findings that both of the other two found.\n"
"Under a Bash ceiling of 6,000 characters it found 1 of them, and missed 1 of the 1 findings that both of the first two found:\n"
f" {self.checkout} 2026-10-14T10:00:00Z: 🔴 a.txt:1 wrong\n"
"It passes: 1 missed is no more than the most a replay as archived missed, 1.\n"
))
def test_a_read_limit_hands_the_one_replay_under_it_the_limit_in_tokens(self):
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--read-limit"], lambda rung: {"found": [0]})
self.assertEqual(code, 0, err)
self.assertEqual(self.review_settings, [None, None, None, {"CLAUDE_CODE_FILE_READ_MAX_OUTPUT_TOKENS": "5000"}])
self.assertIn("four times on", out)
self.assertEqual(out.splitlines()[-2], "Under a Read limit of 5,000 tokens it found 1 of them, and missed 0 of the 1 findings that both of the first two found.")
def test_without_claude_md_hands_the_replay_without_it_the_setting_that_turns_the_instruction_files_off(self):
self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"found": [0]})
self.assertEqual(self.review_settings, [None, None, None, {"CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1"}])
def test_without_claude_md_fails_where_it_missed_more_than_every_replay_with_it(self):
finds = iter(([0, 1], [0, 1], [0, 1], []))
_, out, _ = self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"found": next(finds)})
self.assertEqual(out.splitlines()[-1], "It fails: 2 missed is more than the most a replay with CLAUDE.md missed, 0.")
def test_without_claude_md_says_so_where_the_reviewer_missed_nothing(self):
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"found": [0]})
self.assertEqual((code, out.splitlines()[-2:]), (0, ["Without CLAUDE.md it found 1 of them, and missed 0 of the 1 findings that both of the first two found.", "It passes: 0 missed is no more than the most a replay with CLAUDE.md missed, 0."]))
def test_without_claude_md_says_there_is_no_evidence_where_the_first_two_replays_share_no_finding(self):
finds = iter(([0], [1], [1], [1]))
_, out, _ = self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"found": next(finds)})
self.assertEqual(out.splitlines()[-1], "Without CLAUDE.md it found 1 of them. No evidence: the first two replays with CLAUDE.md found none in common, so there is no finding to hold it to.")
def test_without_claude_md_leaves_out_a_round_that_a_replay_failed_on(self):
failing = {}
def review(rung):
# claude is tried three times running, so the first replay fails only where all three tries do.
failing["tries"] = failing.get("tries", 0) + 1
return [] if failing["tries"] <= 3 else {"type": "result", "result": f"review on {rung.model} {rung.effort}"}
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"found": [0, 1]}, review=review)
self.assertEqual(code, 0, err)
self.assertEqual(out, (
f"Replayed 0 stability rounds of claude-opus-5-5 projects, which hold 0 🔴 and 🟡 findings, four times on {REFERENCE}: three {changes.OMIT.unchanged}, one {changes.OMIT.changed}.\n"
"Left out 1 more, which a replay failed on; stderr says why.\n"
"With CLAUDE.md the three replays found 0, 0 and 0 of them, and missed 0, 0 and 0 of the findings that both of the other two found.\n"
"Without CLAUDE.md it found 0 of them. No evidence: the first two replays with CLAUDE.md found none in common, so there is no finding to hold it to.\n"
))
self.assertIn(f" {self.checkout} 2026-10-14T10:00:00Z with CLAUDE.md failed, so the round is left out: claude printed no json reply", err)
self.assertNotIn("without CLAUDE.md failed", err)
def test_without_claude_md_stops_at_a_usage_limit(self):
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"type": "result", "api_error_status": 429, "result": "You've hit your limit"})
self.assertEqual((code, out), (1, ""))
self.assertIn("review-replay: claude hit a usage limit: You've hit your limit", err)
self.assertEqual(len(self.reviews), 1)
def test_replays_the_models_rounds_on_the_commit_reviewed_from_the_project_rung_down(self):
code, out, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [1] if rung == LOW else [0, 1]})
self.assertEqual(code, 0, err)
self.assertEqual([rung for rung, *_ in self.reviews], [REFERENCE, REFERENCE, LOW])
_, reviewed, branch, base, prompt, cwd = self.reviews[0]
# The default branch stands where it stood when the chunk began, so the prompt's diff against it shows the branch's change.
self.assertEqual((reviewed, branch, base), ("1\n", "x\n", "0\n"))
self.assertEqual(prompt, f"Review branch x in {cwd}.")
self.assertFalse(pathlib.Path(cwd).exists())
self.assertIn("The project's rung found 2 and then 2 of the 2, 100% on average, so a rung qualifies finding 2 or more of them", out)
self.assertIn(f"claude-opus-5-5 at low effort found 1, 50%, and missed 1 such 🔴: it doesn't qualify.\n {self.checkout} 2026-10-14T10:00:00Z: 🔴 a.txt:1 wrong\n", out)
self.assertTrue(out.endswith("qualifies: none below the project's rung\n"), out)
self.assertEqual(set(self.matched_in), {str(self.checkout)})
self.assertTrue(out.startswith("Replayed 1 stability rounds of claude-opus-5-5 projects,"), out)
def test_replays_the_prose_pass_with_the_prose_reviewer(self):
code, out, err = self.run_main(["prose", "claude-opus-5-5"], lambda rung: {"found": [0, 1]})
self.assertEqual(code, 0, err)
self.assertEqual(self.agents, {"prose-reviewer"})
self.assertTrue(out.startswith("Replayed 1 prose rounds"), out)
def test_tries_each_rung_down_to_the_ladders_lowest_below_the_lowest_a_worker_routes_to(self):
code, out, err = self.run_main(["prose", "claude-opus-5-5"], lambda rung: {"found": [0, 1]})
self.assertEqual(code, 0, err)
self.assertEqual([rung for rung, *_ in self.reviews], [REFERENCE, REFERENCE, LOW, MIDDLE])
self.assertTrue(out.endswith("qualifies: claude-sonnet-5-5 at high effort\n"), out)
def test_counts_a_failed_replay_on_a_lower_rung_as_finding_nothing(self):
code, out, err = self.run_main(["prose", "claude-opus-5-5"], lambda rung: {"found": [0, 1]}, review=lambda rung: {"type": "result"} if rung == LOW else {"type": "result", "result": f"review on {rung.model} {rung.effort}"})
self.assertEqual(code, 0, err)
self.assertIn(f"replaying 1 prose rounds on {REFERENCE}\n", err)
self.assertIn(f"replaying 1 prose rounds on {LOW}\n", err)
self.assertIn(f"on {LOW} failed: the reviewer replied with no report", err)
self.assertIn("claude-opus-5-5 at low effort found 0, 0%, and missed 1 such 🔴: it doesn't qualify.", out)
def review_folder_with_round_1(self):
"""The chunk's review folder. The chunk began at 09:15 and the replayed round 2 at 10:00. It holds round 1's report, written at 09:15:00; round 2's report, written after 10:00; the prose pass's report, written at 10:00:00; and an earlier attempt's report, from before 09:15."""
folder = self.home / ".local" / "state" / "larv-review" / "code" / "proj-x"
folder.mkdir(parents=True)
(folder / "stability-round-1.md").write_text("round 1\n")
began = stamp_seconds("2026-10-14T09:15:00Z")
os.utime(folder / "stability-round-1.md", (began, began))
(folder / "stability-round-2.md").write_text("🔴 a.txt:1 wrong\n")
(folder / "prose-round-1.md").write_text("later\n")
round_began = stamp_seconds("2026-10-14T10:00:00Z")
os.utime(folder / "prose-round-1.md", (round_began, round_began))
(folder / "stability-round-3.md").write_text("🔴 a.txt:1 wrong\n")
attempt = stamp_seconds("2026-10-14T09:00:00Z")
os.utime(folder / "stability-round-3.md", (attempt, attempt))
return folder
def test_replays_rounds_whose_archived_transcripts_are_compressed(self):
for copy in self.archive.rglob("*.jsonl"):
self.assertIsNone(compress(copy))
code, out, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [1] if rung == LOW else [0, 1]})
self.assertEqual(code, 0, err)
self.assertTrue(out.startswith("Replayed 1 stability rounds of claude-opus-5-5 projects,"), out)
def test_copies_a_compressed_report_from_before_the_round_under_its_plain_name(self):
folder = self.review_folder_with_round_1()
self.assertIsNone(compress(folder / "stability-round-1.md"))
self.set_prompt(f"Review branch x in {self.home}/code/proj-x. Write your report to ~/.local/state/larv-review/code/proj-x/stability-round-2.md; the last round's is ~/.local/state/larv-review/code/proj-x/stability-round-1.md.")
seen = []
def review(rung):
named = pathlib.Path(re.search(r"report to (\S+)/stability-round-2\.md", self.reviews[-1][4]).group(1))
seen.append((sorted(path.name for path in named.iterdir()), (named / "stability-round-1.md").read_text()))
return {"type": "result", "result": f"review on {rung.model} {rung.effort}"}
code, _, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0, 1]}, review)
self.assertEqual(code, 0, err)
self.assertEqual(seen[0], (["stability-round-1.md"], "round 1\n"))
def test_points_the_prompts_review_folder_at_a_copy_of_the_reports_from_before_the_round_that_it_removes(self):
folder = self.review_folder_with_round_1()
self.set_prompt(f"Review branch x in {self.home}/code/proj-x. Write your report to ~/.local/state/larv-review/code/proj-x/stability-round-2.md; the last round's is ~/.local/state/larv-review/code/proj-x/stability-round-1.md.")
seen = []
def review(rung):
named = pathlib.Path(re.search(r"report to (\S+)/stability-round-2\.md", self.reviews[-1][4]).group(1))
seen.append((named, sorted(path.name for path in named.iterdir()), (named / "stability-round-1.md").read_text()))
(named / "stability-round-2.md").write_text("replayed\n")
return {"type": "result", "result": f"review on {rung.model} {rung.effort}"}
code, _, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0, 1]}, review)
self.assertEqual(code, 0, err)
named, copied, earlier = seen[0]
self.assertEqual(named.parent, folder.parents[1], "the copy sits in the reports folder, where rule-hook lets the reviewer write")
self.assertNotEqual(named, folder)
self.assertEqual(copied, ["stability-round-1.md"], "the round's own report, later ones and an earlier attempt's would hand the replayed reviewer its findings")
self.assertEqual(earlier, "round 1\n")
self.assertFalse(named.exists())
self.assertEqual(sorted(path.name for path in folder.iterdir()), ["prose-round-1.md", "stability-round-1.md", "stability-round-2.md", "stability-round-3.md"])
self.assertEqual((folder / "stability-round-2.md").read_text(), "🔴 a.txt:1 wrong\n")
def test_points_the_review_folders_absolute_path_at_the_copy(self):
folder = self.review_folder_with_round_1()
self.set_prompt(f"Review branch x in {self.home}/code/proj-x. Write your report to {folder}/stability-round-2.md.")
code, _, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0, 1]})
self.assertEqual(code, 0, err)
prompt = self.reviews[0][4]
self.assertNotIn(str(folder), prompt)
self.assertRegex(prompt, rf"report to {folder.parents[1]}/review-replay-\w+/stability-round-2\.md\.$")
def test_replays_no_round_whose_prompt_still_names_the_review_folder_after_the_rewrite(self):
self.review_folder_with_round_1()
self.set_prompt(f"Review branch x in {self.home}/code/proj-x. Write your report to $HOME/.local/state/larv-review/code/proj-x/stability-round-2.md.")
code, out, err = self.run_main(["stability", "claude-opus-5-5"], unused)
self.assertEqual(code, 0, err)
self.assertEqual(self.reviews, [])
self.assertIn(f"on {REFERENCE} failed: the prompt names the review folder in a spelling the replay does not redirect", err)
self.assertIn("Left out 1 more, which a replay on the project's rung failed on", out)
self.assertTrue(out.endswith("No rung tried: fewer than 1 rounds were replayed on the project's rung without a failure.\n"), out)
def test_replays_a_round_whose_prompt_names_an_input_written_before_it_began_naming_the_copy(self):
folder = self.review_folder_with_round_1()
(folder / "branch.diff").write_text("the diff\n")
before = stamp_seconds("2026-10-14T09:50:00Z")
os.utime(folder / "branch.diff", (before, before))
self.set_prompt(f"Review branch x in {self.home}/code/proj-x. The diff is ~/.local/state/larv-review/code/proj-x/branch.diff. Write your report to ~/.local/state/larv-review/code/proj-x/stability-round-2.md.")
code, _, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0, 1]})
self.assertEqual(code, 0, err)
self.assertRegex(self.reviews[0][4], rf"The diff is {folder.parents[1]}/review-replay-\w+/branch\.diff\.")
def test_replays_no_round_whose_prompt_names_an_input_the_folder_holds_only_as_rewritten_after_the_round_began(self):
folder = self.review_folder_with_round_1()
(folder / "branch.diff").write_text("a later diff\n")
self.set_prompt(f"Review branch x in {self.home}/code/proj-x. The diff is ~/.local/state/larv-review/code/proj-x/branch.diff. Write your report to ~/.local/state/larv-review/code/proj-x/stability-round-2.md.")
code, out, err = self.run_main(["stability", "claude-opus-5-5"], unused)
self.assertEqual(code, 0, err)
self.assertEqual(self.reviews, [])
self.assertIn(f"on {REFERENCE} failed: the prompt names branch.diff, which the review folder did not hold from the chunk's start until the round's", err)
def test_fails_the_replay_and_leaves_no_folder_behind_when_it_cannot_copy_the_reports(self):
folder = self.review_folder_with_round_1()
(folder / "stability-round-1.md").chmod(0)
self.set_prompt(f"Review branch x in {self.home}/code/proj-x. Write your report to ~/.local/state/larv-review/code/proj-x/stability-round-2.md.")
scratch = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
self.limits["scratch"] = scratch
code, _, err = self.run_main(["stability", "claude-opus-5-5"], unused)
self.assertEqual(code, 0, err)
self.assertEqual(self.reviews, [])
self.assertIn(f"on {REFERENCE} failed: could not prepare the replay: ", err)
self.assertEqual(list(scratch.iterdir()), [], "the clone is removed")
self.assertEqual([path.name for path in folder.parent.parent.iterdir()], ["code"], "the copy is removed")
def test_replays_the_rounds_of_every_chunk(self):
self.write_chunk("proj", "c2", "2026-10-14T10:30:00Z")
code, out, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0, 1]})
self.assertEqual(code, 0, err)
self.assertTrue(out.startswith("Replayed 2 stability rounds"), out)
def test_leaves_out_a_round_that_either_replay_on_the_project_rung_failed_on(self):
self.write_chunk("proj", "c2", "2026-10-14T10:30:00Z")
self.limits["at_once"] = 1
# One round at a time, three tries on a round reply with no report: the first replay's on its first round, then the second replay's on its first, after the first replay's two reviews. Every other review replies.
for failing in (range(3), range(2, 5)):
with self.subTest(failing=failing):
calls = iter(range(100))
code, out, err = self.run_main(["prose", "claude-opus-5-5"], lambda rung: {"found": [0, 1]}, review=lambda rung: {"type": "result"} if next(calls) in failing else {"type": "result", "result": f"review on {rung.model} {rung.effort}"})
self.assertEqual(code, 0, err)
self.assertTrue(out.startswith("Replayed 1 prose rounds"), out)
self.assertIn("Left out 1 more", out)
def test_tries_no_lower_rung_where_the_project_rung_found_nothing(self):
code, out, err = self.run_main(["prose", "claude-opus-5-5"], lambda rung: {"found": []})
self.assertEqual(code, 0, err)
self.assertEqual([rung for rung, *_ in self.reviews], [REFERENCE, REFERENCE])
self.assertTrue(out.endswith("No rung tried: the project's rung found too few of them to hold a rung to.\n"), out)
def test_checks_out_the_commit_detached_when_the_reviewer_started_on_the_default_branch(self):
self.set_branch("main")
self.assertEqual(self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0]})[0], 0)
self.assertEqual(self.reviews[0][1:3], ("1\n", ""))
def test_sets_the_default_branch_to_its_own_line_and_not_to_a_branch_merged_after_the_chunk_began(self):
# Branch side's commit was authored before the chunk began, but the default branch held it only once merged, after.
run_git(self.checkout, "reset", "-q", "--hard", "HEAD~2")
run_git(self.checkout, "checkout", "-qb", "side")
(self.checkout / "a.txt").write_text("s\n")
run_git(self.checkout, "commit", "-qam", "s", at="2026-10-14T09:10:00Z")
run_git(self.checkout, "checkout", "-q", "main")
run_git(self.checkout, "merge", "-q", "--no-ff", "-m", "m", "side", at="2026-10-14T09:40:00Z")
run_git(self.checkout, "push", "-qf", "origin", "main")
self.assertEqual(self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0]})[0], 0)
self.assertEqual(self.reviews[0][1:4], ("s\n", "x\n", "0\n"))
def test_leaves_out_a_round_whose_review_on_the_project_rung_has_no_report(self):
code, out, err = self.run_main(["stability", "claude-opus-5-5"], unused, review=lambda rung: {"type": "result"})
self.assertEqual(code, 0, err)
# Each of the two replays tries claude three times.
self.assertEqual([rung for rung, *_ in self.reviews], [REFERENCE] * 6)
self.assertIn(f" {self.checkout} 2026-10-14T10:00:00Z on {REFERENCE} failed: the reviewer replied with no report", err)
self.assertIn("Left out 1 more, which a replay on the project's rung failed on", out)
self.assertTrue(out.endswith("No rung tried: fewer than 1 rounds were replayed on the project's rung without a failure.\n"), out)
def assert_no_round(self):
code, _, err = self.run_main(["stability", "claude-opus-5-5"], unused)
self.assertEqual(code, 1)
self.assertEqual(err.splitlines(), [f"review-replay: no archived stability round of projects on claude-opus-5-5 in {self.archive} holds a 🔴 or 🟡 on code it can find"])
def test_skips_a_chunk_with_no_transcript_copy(self):
(self.archive / "proj" / "c1" / "t0.jsonl").unlink()
self.assert_no_round()
def test_skips_a_chunk_whose_transcript_copy_has_no_timestamp(self):
(self.archive / "proj" / "c1" / "t0.jsonl").write_text(json.dumps({"type": "summary"}) + "\n")
self.assert_no_round()
def assert_left_out(self, reason, root=None):
code, _, err = self.run_main(["stability", "claude-opus-5-5"], unused)
self.assertEqual(code, 1)
self.assertIn(f"{root or self.checkout} 2026-10-14T10:00:00Z is left out: {reason}", err)
def test_leaves_out_a_round_whose_prompt_names_no_worktree(self):
self.set_prompt("Review the branch.")
self.assert_left_out("the prompt names neither one removed worktree nor the checkout")
def test_leaves_out_a_round_whose_checkout_names_no_default_branch(self):
run_git(self.checkout, "remote", "set-head", "origin", "-d")
self.assert_left_out("origin records no default branch")
def test_leaves_out_a_round_whose_checkout_is_gone(self):
shutil.rmtree(self.checkout)
# With no folder under ~/code, the key names /proj.
self.assert_left_out("the project has no checkout", "/proj")
def test_leaves_out_a_round_whose_chunk_began_before_the_checkouts_first_commit(self):
self.write_chunk("proj", "c1", "2026-10-14T10:00:00Z", began="2026-10-14T08:00:00Z")
self.assert_left_out("no commit was authored by then")
def test_exits_1_on_a_usage_limit_bad_arguments_or_no_round_to_replay(self):
code, _, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"type": "result", "api_error_status": 429, "result": "You've hit your limit"})
self.assertEqual(code, 1)
self.assertIn("review-replay: claude hit a usage limit: You've hit your limit", err)
for argv in ([], ["claude-opus-5-5"], ["full scan", "claude-opus-5-5"], ["stability", "off-ladder"], ["stability", "claude-opus-5-5", "x"]):
with self.subTest(argv=argv):
self.assertEqual(self.run_main(argv, unused)[0], 1)
code, _, err = self.run_main(["stability", "claude-sonnet-5-5"], unused)
self.assertEqual(code, 1)
self.assertIn("no archived stability round of projects on claude-sonnet-5-5", err)
def test_prints_the_help(self):
code, out, _ = self.run_main(["-h"], unused)
self.assertEqual((code, out.splitlines()[2]), (0, "usage: review-replay <pass> <model> [--omit-claude-md | --bash-ceiling | --read-limit]"))
self.assertIn(f"{', '.join(list(REVIEWERS)[:-1])} or {list(REVIEWERS)[-1]}.", " ".join(out.split()))
def test_the_launcher_prints_the_help(self):
ran = subprocess.run([sys.executable, str(TOOL), "--help"], capture_output=True, text=True)
self.assertEqual(ran.returncode, 0)
self.assertIn("usage: review-replay <pass> <model>", ran.stdout)
if __name__ == "__main__":
unittest.main()