631 lines
40 KiB
Python
631 lines
40 KiB
Python
import contextlib
|
|
import dataclasses
|
|
import io
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from review_replay.rounds import authored_by, lost_inputs, recorded_findings, reviewed_path, Round, round_from, sample, stamp_seconds
|
|
from review_replay import changes, main, replay
|
|
from text_files.doubles import compress
|
|
from project_config.doubles import configured, write_config
|
|
from loop_effects.effects import Effects, system_effects
|
|
from claude_code.entries import entry_from
|
|
from failure import Failure
|
|
from review_rules.reviewer_context import reviewer_context
|
|
from review_rules.review_passes import REVIEWERS
|
|
from model_routing.ladder import Rung
|
|
|
|
TOOL = pathlib.Path(__file__).resolve().parents[1] / "review-replay"
|
|
REFERENCE = Rung("claude-opus-5-5", "medium")
|
|
LOW = Rung("claude-opus-5-5", "low")
|
|
MIDDLE = Rung("claude-sonnet-5-5", "high")
|
|
|
|
|
|
def entry(kind, text, **fields):
|
|
return {"type": kind, "message": {"content": [{"type": "text", "text": text}]}, **fields}
|
|
|
|
|
|
def kinds(found):
|
|
return [entry_from(one) for one in found]
|
|
|
|
|
|
def a_round(at, findings=("🔴 a",)):
|
|
return Round("stability-reviewer", "/code/p", "x", f"2026-10-14T10:00:0{at}Z", 0, "Review it.", tuple(findings))
|
|
|
|
|
|
class RecordedFindings(unittest.TestCase):
|
|
def test_takes_each_red_and_yellow_finding_with_the_lines_up_to_the_next_blank_line_or_finding(self):
|
|
report_text = "BLOCK\n\n1 🔴, 2 🟡 in all.\n\n1. 🔴 **Loop** at a.py:3\nnever ends.\n- 🟡 b.py:1 slow\n| 🟡 | c.py | odd |\n\n🟢 nit\n### 🔴 Heading\n**🟡 Bold**"
|
|
self.assertEqual(recorded_findings(report_text), ["1. 🔴 **Loop** at a.py:3\nnever ends.", "- 🟡 b.py:1 slow", "| 🟡 | c.py | odd |", "### 🔴 Heading", "**🟡 Bold**"])
|
|
|
|
def test_skips_a_count_or_verdict_line_that_names_the_marks(self):
|
|
self.assertEqual(recorded_findings("APPROVE\n\nNo 🔴 or 🟡 findings.\n0 🔴, 0 🟡"), [])
|
|
|
|
|
|
class RoundFrom(unittest.TestCase):
|
|
def test_reads_the_prompt_the_last_report_and_when_the_reviewer_started(self):
|
|
entries = [
|
|
entry("user", "Review branch x in /code/p-x.", timestamp="2026-10-14T10:00:00.500Z", cwd="/code/p-x", gitBranch="x"),
|
|
entry("assistant", "Reading."),
|
|
entry("assistant", "BLOCK\n\n🔴 a.py:1 breaks"),
|
|
]
|
|
self.assertEqual(round_from("stability-reviewer", "/code/p", 5.0, kinds(entries)), Round("stability-reviewer", "/code/p", "x", "2026-10-14T10:00:00.500Z", 5.0, "Review branch x in /code/p-x.", ("🔴 a.py:1 breaks",)))
|
|
self.assertEqual(round_from("stability-reviewer", "/code/p", 5.0, kinds(entries)).seconds, 1791972000.5)
|
|
|
|
def test_reads_a_prompt_given_as_a_string(self):
|
|
start = {"type": "user", "message": {"content": "Review branch x in /code/p-x."}, "timestamp": "2026-10-14T10:00:00Z", "gitBranch": "x"}
|
|
self.assertEqual(round_from("stability-reviewer", "/code/p", 5.0, kinds([start, entry("assistant", "🔴 a")])).prompt, "Review branch x in /code/p-x.")
|
|
|
|
def test_skips_a_round_with_no_red_or_yellow_finding_or_no_report(self):
|
|
start = entry("user", "Review.", timestamp="2026-10-14T10:00:00Z", cwd="/code/p", gitBranch="x")
|
|
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, kinds([start, entry("assistant", "APPROVE\n\n🟢 nit")])))
|
|
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, kinds([start])))
|
|
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, kinds([])))
|
|
self.assertIsNone(round_from("stability-reviewer", "/code/p", 5.0, kinds([{**start, "timestamp": "yesterday"}, entry("assistant", "🔴 a")])))
|
|
|
|
|
|
class ReviewedPath(unittest.TestCase):
|
|
def test_names_the_worktree_the_prompt_names_that_is_gone(self):
|
|
exists = {"/code", "/code/p", "/code/p/worktrees", "/tmp"}.__contains__
|
|
self.assertEqual(reviewed_path("Review /code/p-x/src/a.py:3-5; the diff is at /tmp/s/x.diff; /code/p is older.", "/code/p", exists), "/code/p-x")
|
|
self.assertEqual(reviewed_path("Review `/code/p/worktrees/x`.", "/code/p", exists), "/code/p/worktrees/x")
|
|
|
|
def test_names_the_checkout_when_the_prompt_names_no_gone_worktree_and_none_when_it_names_neither_or_two(self):
|
|
exists = {"/code", "/code/p"}.__contains__
|
|
self.assertEqual(reviewed_path("Review /code/p.", "/code/p", exists), "/code/p")
|
|
self.assertIsNone(reviewed_path("Review the branch.", "/code/p", exists))
|
|
self.assertIsNone(reviewed_path("Review /code/p-x and /code/p-43.", "/code/p", exists))
|
|
|
|
|
|
class LostInputs(unittest.TestCase):
|
|
def test_names_each_file_the_prompt_names_in_the_copy_that_it_lacks_other_than_a_round_report(self):
|
|
prompt = "Read /c/branch.diff and /c/ls-files.txt:3. Earlier: /c/prose-round-1.md. Write /c/principles-round-2-rerun.md; see /elsewhere/x.diff."
|
|
self.assertEqual(lost_inputs(prompt, "/c", {"ls-files.txt"}.__contains__), ["branch.diff"])
|
|
self.assertEqual(lost_inputs(prompt, "/c", {"branch.diff", "ls-files.txt"}.__contains__), [])
|
|
|
|
|
|
class AuthoredBy(unittest.TestCase):
|
|
def test_takes_the_newest_commit_authored_by_then(self):
|
|
log = [("c3", 30), ("c2", 20), ("c1", 10)]
|
|
self.assertEqual(authored_by(log, 25), "c2")
|
|
self.assertEqual(authored_by(log, 30), "c3")
|
|
self.assertIsNone(authored_by(log, 5))
|
|
|
|
def test_takes_the_newest_author_time_over_a_commit_authored_early_but_merged_late(self):
|
|
log = [("z-bump", 5), ("a-new", 30), ("m-mid", 20)]
|
|
self.assertEqual(authored_by(log, 25), "m-mid")
|
|
self.assertEqual(authored_by(log, 10), "z-bump")
|
|
|
|
def test_breaks_a_tie_by_log_order(self):
|
|
self.assertEqual(authored_by([("a-merged", 20), ("z-earlier", 20)], 25), "a-merged")
|
|
|
|
|
|
class Sample(unittest.TestCase):
|
|
def test_keeps_the_latest_rounds_in_the_order_they_ran(self):
|
|
rounds = [a_round(2), a_round(1), a_round(3)]
|
|
self.assertEqual(sample(rounds, 2), [rounds[0], rounds[2]])
|
|
|
|
|
|
class Launch(unittest.TestCase):
|
|
def setUp(self):
|
|
self.home = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
|
|
(self.home / ".claude").mkdir()
|
|
(self.home / ".claude" / "CLAUDE.md").write_text("## Shared goal\n\nHealth.\n\n## Prose\n\nShort.\n\n## Vocabulary\n\nWords.\n")
|
|
self.clone = self.home / "clone"
|
|
self.reports = self.home / "reports"
|
|
self.clone.mkdir()
|
|
self.checkout = self.home / "proj"
|
|
self.checkout.mkdir()
|
|
self.config = configured(self, self.checkout, home=self.home) / "config.toml"
|
|
|
|
def test_runs_the_reviewer_as_archived_with_claude_md(self):
|
|
self.assertEqual(replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, None), (["--agent", "prose-reviewer", *REFERENCE.flags()], "Review it."))
|
|
|
|
def test_drops_claude_md_and_hands_the_reviewer_what_the_hook_and_the_header_would(self):
|
|
(self.clone / "AGENTS.md").write_text("")
|
|
args, prompt = replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, changes.OMIT)
|
|
self.assertEqual(args, ["--agent", "prose-reviewer", *REFERENCE.flags(), "--settings", json.dumps({"env": {"CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1"}})])
|
|
self.assertEqual(prompt, f"{reviewer_context(self.home)}\n\nReview it.\nPrinciples: read {self.clone}/AGENTS.md first, for the project principles and the decision titles.")
|
|
|
|
def test_names_a_record_outside_the_checkout_where_it_stands(self):
|
|
self.config.write_text(self.config.read_text().replace('"checkout:AGENTS.md"', '"folder:AGENTS.md"'))
|
|
(self.clone / "AGENTS.md").write_text("")
|
|
(self.config.parent / "AGENTS.md").write_text("")
|
|
prompt = replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, changes.OMIT)[1]
|
|
self.assertTrue(prompt.endswith(f"Principles: read {self.config.parent}/AGENTS.md first, for the project principles and the decision titles."), prompt)
|
|
|
|
def test_names_the_decision_log_in_the_clone_or_where_it_stands_outside_the_checkout_without_agents_md(self):
|
|
places = {"checkout:docs/decisions.md": self.clone / "docs" / "decisions.md", "folder:decisions.md": self.config.parent / "decisions.md"}
|
|
for setting, decisions in places.items():
|
|
with self.subTest(setting):
|
|
self.config.write_text(self.config.read_text().replace('"checkout:docs/decisions.md"', f'"{setting}"'))
|
|
decisions.parent.mkdir(exist_ok=True)
|
|
decisions.write_text("1. A principle.\n\n# Decisions\n")
|
|
prompt = replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, changes.OMIT)[1]
|
|
self.assertTrue(prompt.endswith(f"Principles: read {decisions} first, up to `# Decisions`, for the project principles and the decision titles."), prompt)
|
|
|
|
def test_fails_without_a_config_for_the_checkout(self):
|
|
self.config.unlink()
|
|
self.assertIn("no config names the checkout", replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, changes.OMIT).reason)
|
|
|
|
def test_fails_without_the_sections_to_hand_over(self):
|
|
(self.home / ".claude" / "CLAUDE.md").write_text("")
|
|
self.assertIsInstance(replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.checkout), replay.Copies(str(self.clone), str(self.reports)), self.home, changes.OMIT), Failure)
|
|
|
|
|
|
class Match(unittest.TestCase):
|
|
def run_match(self, stdout):
|
|
self.asked = []
|
|
|
|
def claude(args, cwd, prompt):
|
|
self.asked.append((args, cwd, prompt))
|
|
return subprocess.CompletedProcess(args, 0, stdout, "")
|
|
|
|
return replay.match(("🔴 a", "🟡 b"), "the other report", "/code/p", REFERENCE, dataclasses.replace(system_effects(pathlib.Path.home(), os.environ), claude=claude))
|
|
|
|
def test_asks_for_the_numbered_findings_the_other_report_raises_as_json_with_no_tools(self):
|
|
self.run_match("{}")
|
|
[(args, cwd, prompt)] = self.asked
|
|
self.assertEqual(cwd, "/code/p")
|
|
self.assertIn("1. 🔴 a\n\n2. 🟡 b\n\nThe other report:\nthe other report", prompt)
|
|
self.assertEqual(json.loads(args[args.index("--json-schema") + 1]), replay.MATCH_SCHEMA)
|
|
self.assertEqual(args[args.index("--tools") + 1], "")
|
|
self.assertEqual(args[args.index("--model"):args.index("--model") + 4], REFERENCE.flags())
|
|
|
|
def test_takes_the_numbers_in_range_and_fails_on_a_reply_of_another_shape(self):
|
|
self.assertEqual(self.run_match(json.dumps({"type": "result", "structured_output": {"found": [2, 0, 3, 1]}})), {0, 1})
|
|
for output in ({"found": ["1"]}, {"found": 1}, [1]):
|
|
with self.subTest(output):
|
|
self.assertIsInstance(self.run_match(json.dumps({"type": "result", "structured_output": output})), Failure)
|
|
|
|
|
|
def unused(*_):
|
|
raise AssertionError("not called")
|
|
|
|
|
|
def run_git(cwd, *args, at=None):
|
|
"""at, when given, is the commit's author and committer time, so git log lists the commits in a fixed order."""
|
|
env = {**os.environ, "GIT_AUTHOR_DATE": at, "GIT_COMMITTER_DATE": at} if at else None
|
|
subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], cwd=cwd, check=True, capture_output=True, env=env)
|
|
|
|
|
|
class Main(unittest.TestCase):
|
|
"""The tool on a temporary home: a chunk that began after the checkout's first commit, and whose stability reviewer reviewed branch x at its second, from a worktree since removed; a prose reviewer's files sort before the stability reviewer's."""
|
|
|
|
def setUp(self):
|
|
self.home = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
|
|
self.archive = self.home / ".claude" / "loop-claude-transcripts"
|
|
(self.home / ".claude").mkdir()
|
|
(self.home / ".claude" / "loop-claude-models.json").write_text(json.dumps({"proj": "claude-opus-5-5", "other": "claude-sonnet-5-5"}))
|
|
origin, self.checkout = self.home / "origin.git", self.home / "code" / "proj"
|
|
run_git(self.home, "init", "-q", "--bare", "-b", "main", str(origin))
|
|
run_git(self.home, "clone", "-q", str(origin), str(self.checkout))
|
|
for n, stamp in enumerate(("2026-10-14T09:00:00Z", "2026-10-14T09:30:00Z", "2026-10-14T11:00:00Z")):
|
|
(self.checkout / "a.txt").write_text(f"{n}\n")
|
|
run_git(self.checkout, "add", "a.txt")
|
|
run_git(self.checkout, "commit", "-qm", f"c{n}", at=stamp)
|
|
run_git(self.checkout, "push", "-q", "origin", "main")
|
|
run_git(self.checkout, "remote", "set-head", "origin", "main")
|
|
write_config(self.home, self.checkout)
|
|
for key in ("proj", "other"):
|
|
self.write_chunk(key, "c1", "2026-10-14T10:00:00Z")
|
|
(self.home / ".claude" / "CLAUDE.md").write_text("## Shared goal\n\nHealth.\n\n## Prose\n\nShort.\n\n## Vocabulary\n\nWords.\n")
|
|
# The fixture holds one round.
|
|
self.limits = {"enough_rounds": 1}
|
|
self.reviews = []
|
|
self.review_settings = []
|
|
self.agents = set()
|
|
self.matched_in = []
|
|
|
|
def write_chunk(self, key, chunk, at, began="2026-10-14T09:15:00Z"):
|
|
"""A chunk that began at began, whose stability round began at at."""
|
|
subagents = self.archive / key / chunk / "session" / "s" / "subagents"
|
|
subagents.mkdir(parents=True, exist_ok=True)
|
|
(subagents.parents[2] / "t0.jsonl").write_text(json.dumps({"type": "summary"}) + "\n" + json.dumps(entry("user", "Do the next chunk.", timestamp=began)) + "\n")
|
|
(subagents / "agent-a.meta.json").write_text(json.dumps({"agentType": "prose-reviewer"}))
|
|
(subagents / "agent-b.meta.json").write_text(json.dumps({"agentType": "stability-reviewer"}))
|
|
gone = self.home / "code" / "proj-x"
|
|
lines = [entry("user", f"Review branch x in {gone}.", timestamp=at, cwd=str(gone), gitBranch="x"), entry("assistant", "BLOCK\n\n🔴 a.txt:1 wrong\n\n🟡 slow")]
|
|
for name in ("agent-a.jsonl", "agent-b.jsonl"):
|
|
(subagents / name).write_text("".join(json.dumps(line) + "\n" for line in lines))
|
|
|
|
def rewrite_round(self, change):
|
|
path = self.archive / "proj" / "c1" / "session" / "s" / "subagents" / "agent-b.jsonl"
|
|
lines = [json.loads(line) for line in path.read_text().splitlines()]
|
|
change(lines[0])
|
|
path.write_text("".join(json.dumps(line) + "\n" for line in lines))
|
|
|
|
def set_branch(self, branch):
|
|
self.rewrite_round(lambda first: first.update(gitBranch=branch))
|
|
|
|
def set_prompt(self, prompt):
|
|
self.rewrite_round(lambda first: first["message"]["content"][0].update(text=prompt))
|
|
|
|
def run_main(self, argv, found, review=lambda rung: {"type": "result", "result": f"review on {rung.model} {rung.effort}"}, env={}):
|
|
"""found(rung) is the indexes of the recorded findings the reviewer on rung finds; review(rung), that reviewer's reply."""
|
|
|
|
def claude(args, cwd, prompt):
|
|
rung = Rung(args[args.index("--model") + 1], args[args.index("--effort") + 1])
|
|
if "--agent" in args:
|
|
shown = [subprocess.run(["git", *command], cwd=cwd, capture_output=True, text=True).stdout for command in (["show", "HEAD:a.txt"], ["branch", "--show-current"], ["show", "origin/main:a.txt"])]
|
|
self.reviews.append((rung, *shown, prompt, cwd))
|
|
self.review_settings.append(json.loads(args[args.index("--settings") + 1])["env"] if "--settings" in args else None)
|
|
self.agents.add(args[args.index("--agent") + 1])
|
|
result = review(rung)
|
|
else:
|
|
self.matched_in.append(cwd)
|
|
reviewed = Rung(*re.search(r"review on (\S+) (\S+)", prompt).groups())
|
|
result = found(reviewed)
|
|
result = result if "api_error_status" in result else {"type": "result", "structured_output": {"found": [n + 1 for n in result["found"]]}}
|
|
return subprocess.CompletedProcess(args, 0, json.dumps(result), "")
|
|
|
|
effects = Effects(claude=claude, claude_on_terminal=unused, discard_typed=unused, dm=unused, execv=unused, git=system_effects(pathlib.Path.home(), os.environ).git, input=unused, local_time=unused, memory=unused, now=lambda: 1790000000.0, on_terminal=unused, python=unused, replace_file=unused, sleep=unused, stream_claude=unused, timer=unused, tmux=unused, tmux_on_terminal=unused)
|
|
out, err = io.StringIO(), io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
code = main.main(argv, env, self.home, effects, **self.limits)
|
|
return code, out.getvalue(), err.getvalue()
|
|
|
|
def test_keeps_each_reviewer_and_match_run_and_the_printed_report_in_one_folder_of_the_archive(self):
|
|
code, out, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0, 1]})
|
|
self.assertEqual(code, 0, err)
|
|
folder = self.home / ".claude" / "loop-claude-transcripts" / "replays" / "review-replay-stability-claude-opus-5-5-20260921T141320Z"
|
|
self.assertEqual((folder / "result.md").read_text(encoding="utf-8"), out)
|
|
runs = [json.loads(path.read_text(encoding="utf-8").splitlines()[0]) for path in folder.glob("*.jsonl")]
|
|
# A run that repeats another word for word, as the fakes' do, is kept once.
|
|
self.assertEqual({run["prompt"] for run in runs if "--agent" in run["args"]}, {prompt for *_, prompt, _ in self.reviews})
|
|
self.assertTrue(any("--json-schema" in run["args"] for run in runs))
|
|
|
|
def test_without_claude_md_replays_each_round_three_times_with_it_and_once_without_on_the_project_rung(self):
|
|
finds = iter(([0, 1], [0], [0, 1], [1]))
|
|
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"found": next(finds)})
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual([rung for rung, *_ in self.reviews], [REFERENCE] * 4)
|
|
context = reviewer_context(self.home)
|
|
self.assertEqual([prompt.startswith(context) for *_, prompt, _ in self.reviews], [False, False, False, True])
|
|
self.assertEqual(out, (
|
|
f"Replayed 1 stability rounds of claude-opus-5-5 projects, which hold 2 🔴 and 🟡 findings, four times on {REFERENCE}: three {changes.OMIT.unchanged}, one {changes.OMIT.changed}.\n"
|
|
"With CLAUDE.md the three replays found 2, 1 and 2 of them, and missed 0, 1 and 0 of the findings that both of the other two found.\n"
|
|
"Without CLAUDE.md it found 1 of them, and missed 1 of the 1 findings that both of the first two found:\n"
|
|
f" {self.checkout} 2026-10-14T10:00:00Z: 🔴 a.txt:1 wrong\n"
|
|
"It passes: 1 missed is no more than the most a replay with CLAUDE.md missed, 1.\n"
|
|
))
|
|
with_them = f"replaying 1 stability rounds with CLAUDE.md on {REFERENCE}\n"
|
|
self.assertEqual(err, with_them * 3 + f"replaying 1 stability rounds without CLAUDE.md on {REFERENCE}\n")
|
|
|
|
def test_a_change_is_refused_where_the_replays_would_inherit_its_variable_from_the_environment(self):
|
|
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--bash-ceiling"], lambda rung: {"found": [0]}, env={"BASH_MAX_OUTPUT_LENGTH": "6000"})
|
|
self.assertEqual((code, out, self.reviews), (1, "", []))
|
|
self.assertIn("BASH_MAX_OUTPUT_LENGTH", err)
|
|
code, _, err = self.run_main(["stability", "claude-opus-5-5", "--bash-ceiling"], lambda rung: {"found": [0]}, env={"CLAUDE_CODE_FILE_READ_MAX_OUTPUT_TOKENS": "5000"})
|
|
self.assertEqual(code, 0, err)
|
|
|
|
def test_a_bash_ceiling_replays_each_round_three_unchanged_and_once_under_the_ceiling_with_nothing_else_changed(self):
|
|
finds = iter(([0, 1], [0], [0, 1], [1]))
|
|
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--bash-ceiling"], lambda rung: {"found": next(finds)})
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual(self.review_settings, [None, None, None, {"BASH_MAX_OUTPUT_LENGTH": "6000"}])
|
|
context = reviewer_context(self.home)
|
|
self.assertEqual([prompt.startswith(context) for *_, prompt, _ in self.reviews], [False] * 4)
|
|
self.assertEqual(out, (
|
|
f"Replayed 1 stability rounds of claude-opus-5-5 projects, which hold 2 🔴 and 🟡 findings, four times on {REFERENCE}: three as archived, one under a Bash ceiling of 6,000 characters.\n"
|
|
"As archived the three replays found 2, 1 and 2 of them, and missed 0, 1 and 0 of the findings that both of the other two found.\n"
|
|
"Under a Bash ceiling of 6,000 characters it found 1 of them, and missed 1 of the 1 findings that both of the first two found:\n"
|
|
f" {self.checkout} 2026-10-14T10:00:00Z: 🔴 a.txt:1 wrong\n"
|
|
"It passes: 1 missed is no more than the most a replay as archived missed, 1.\n"
|
|
))
|
|
|
|
def test_a_read_limit_hands_the_one_replay_under_it_the_limit_in_tokens(self):
|
|
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--read-limit"], lambda rung: {"found": [0]})
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual(self.review_settings, [None, None, None, {"CLAUDE_CODE_FILE_READ_MAX_OUTPUT_TOKENS": "5000"}])
|
|
self.assertIn("four times on", out)
|
|
self.assertEqual(out.splitlines()[-2], "Under a Read limit of 5,000 tokens it found 1 of them, and missed 0 of the 1 findings that both of the first two found.")
|
|
|
|
def test_without_claude_md_hands_the_replay_without_it_the_setting_that_turns_the_instruction_files_off(self):
|
|
self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"found": [0]})
|
|
self.assertEqual(self.review_settings, [None, None, None, {"CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1"}])
|
|
|
|
def test_without_claude_md_fails_where_it_missed_more_than_every_replay_with_it(self):
|
|
finds = iter(([0, 1], [0, 1], [0, 1], []))
|
|
_, out, _ = self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"found": next(finds)})
|
|
self.assertEqual(out.splitlines()[-1], "It fails: 2 missed is more than the most a replay with CLAUDE.md missed, 0.")
|
|
|
|
def test_without_claude_md_says_so_where_the_reviewer_missed_nothing(self):
|
|
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"found": [0]})
|
|
self.assertEqual((code, out.splitlines()[-2:]), (0, ["Without CLAUDE.md it found 1 of them, and missed 0 of the 1 findings that both of the first two found.", "It passes: 0 missed is no more than the most a replay with CLAUDE.md missed, 0."]))
|
|
|
|
def test_without_claude_md_says_there_is_no_evidence_where_the_first_two_replays_share_no_finding(self):
|
|
finds = iter(([0], [1], [1], [1]))
|
|
_, out, _ = self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"found": next(finds)})
|
|
self.assertEqual(out.splitlines()[-1], "Without CLAUDE.md it found 1 of them. No evidence: the first two replays with CLAUDE.md found none in common, so there is no finding to hold it to.")
|
|
|
|
def test_without_claude_md_leaves_out_a_round_that_a_replay_failed_on(self):
|
|
failing = {}
|
|
|
|
def review(rung):
|
|
# claude is tried three times running, so the first replay fails only where all three tries do.
|
|
failing["tries"] = failing.get("tries", 0) + 1
|
|
return [] if failing["tries"] <= 3 else {"type": "result", "result": f"review on {rung.model} {rung.effort}"}
|
|
|
|
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"found": [0, 1]}, review=review)
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual(out, (
|
|
f"Replayed 0 stability rounds of claude-opus-5-5 projects, which hold 0 🔴 and 🟡 findings, four times on {REFERENCE}: three {changes.OMIT.unchanged}, one {changes.OMIT.changed}.\n"
|
|
"Left out 1 more, which a replay failed on; stderr says why.\n"
|
|
"With CLAUDE.md the three replays found 0, 0 and 0 of them, and missed 0, 0 and 0 of the findings that both of the other two found.\n"
|
|
"Without CLAUDE.md it found 0 of them. No evidence: the first two replays with CLAUDE.md found none in common, so there is no finding to hold it to.\n"
|
|
))
|
|
self.assertIn(f" {self.checkout} 2026-10-14T10:00:00Z with CLAUDE.md failed, so the round is left out: claude printed no json reply", err)
|
|
self.assertNotIn("without CLAUDE.md failed", err)
|
|
|
|
def test_without_claude_md_stops_at_a_usage_limit(self):
|
|
code, out, err = self.run_main(["stability", "claude-opus-5-5", "--omit-claude-md"], lambda rung: {"type": "result", "api_error_status": 429, "result": "You've hit your limit"})
|
|
self.assertEqual((code, out), (1, ""))
|
|
self.assertIn("review-replay: claude hit a usage limit: You've hit your limit", err)
|
|
self.assertEqual(len(self.reviews), 1)
|
|
|
|
def test_replays_the_models_rounds_on_the_commit_reviewed_from_the_project_rung_down(self):
|
|
code, out, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [1] if rung == LOW else [0, 1]})
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual([rung for rung, *_ in self.reviews], [REFERENCE, REFERENCE, LOW])
|
|
_, reviewed, branch, base, prompt, cwd = self.reviews[0]
|
|
# The default branch stands where it stood when the chunk began, so the prompt's diff against it shows the branch's change.
|
|
self.assertEqual((reviewed, branch, base), ("1\n", "x\n", "0\n"))
|
|
self.assertEqual(prompt, f"Review branch x in {cwd}.")
|
|
self.assertFalse(pathlib.Path(cwd).exists())
|
|
self.assertIn("The project's rung found 2 and then 2 of the 2, 100% on average, so a rung qualifies finding 2 or more of them", out)
|
|
self.assertIn(f"claude-opus-5-5 at low effort found 1, 50%, and missed 1 such 🔴: it doesn't qualify.\n {self.checkout} 2026-10-14T10:00:00Z: 🔴 a.txt:1 wrong\n", out)
|
|
self.assertTrue(out.endswith("qualifies: none below the project's rung\n"), out)
|
|
self.assertEqual(set(self.matched_in), {str(self.checkout)})
|
|
self.assertTrue(out.startswith("Replayed 1 stability rounds of claude-opus-5-5 projects,"), out)
|
|
|
|
def test_replays_the_prose_pass_with_the_prose_reviewer(self):
|
|
code, out, err = self.run_main(["prose", "claude-opus-5-5"], lambda rung: {"found": [0, 1]})
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual(self.agents, {"prose-reviewer"})
|
|
self.assertTrue(out.startswith("Replayed 1 prose rounds"), out)
|
|
|
|
def test_tries_each_rung_down_to_the_ladders_lowest_below_the_lowest_a_worker_routes_to(self):
|
|
code, out, err = self.run_main(["prose", "claude-opus-5-5"], lambda rung: {"found": [0, 1]})
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual([rung for rung, *_ in self.reviews], [REFERENCE, REFERENCE, LOW, MIDDLE])
|
|
self.assertTrue(out.endswith("qualifies: claude-sonnet-5-5 at high effort\n"), out)
|
|
|
|
def test_counts_a_failed_replay_on_a_lower_rung_as_finding_nothing(self):
|
|
code, out, err = self.run_main(["prose", "claude-opus-5-5"], lambda rung: {"found": [0, 1]}, review=lambda rung: {"type": "result"} if rung == LOW else {"type": "result", "result": f"review on {rung.model} {rung.effort}"})
|
|
self.assertEqual(code, 0, err)
|
|
self.assertIn(f"replaying 1 prose rounds on {REFERENCE}\n", err)
|
|
self.assertIn(f"replaying 1 prose rounds on {LOW}\n", err)
|
|
self.assertIn(f"on {LOW} failed: the reviewer replied with no report", err)
|
|
self.assertIn("claude-opus-5-5 at low effort found 0, 0%, and missed 1 such 🔴: it doesn't qualify.", out)
|
|
|
|
def review_folder_with_round_1(self):
|
|
"""The chunk's review folder. The chunk began at 09:15 and the replayed round 2 at 10:00. It holds round 1's report, written at 09:15:00; round 2's report, written after 10:00; the prose pass's report, written at 10:00:00; and an earlier attempt's report, from before 09:15."""
|
|
folder = self.home / ".local" / "state" / "larv-review" / "code" / "proj-x"
|
|
folder.mkdir(parents=True)
|
|
(folder / "stability-round-1.md").write_text("round 1\n")
|
|
began = stamp_seconds("2026-10-14T09:15:00Z")
|
|
os.utime(folder / "stability-round-1.md", (began, began))
|
|
(folder / "stability-round-2.md").write_text("🔴 a.txt:1 wrong\n")
|
|
(folder / "prose-round-1.md").write_text("later\n")
|
|
round_began = stamp_seconds("2026-10-14T10:00:00Z")
|
|
os.utime(folder / "prose-round-1.md", (round_began, round_began))
|
|
(folder / "stability-round-3.md").write_text("🔴 a.txt:1 wrong\n")
|
|
attempt = stamp_seconds("2026-10-14T09:00:00Z")
|
|
os.utime(folder / "stability-round-3.md", (attempt, attempt))
|
|
return folder
|
|
|
|
def test_replays_rounds_whose_archived_transcripts_are_compressed(self):
|
|
for copy in self.archive.rglob("*.jsonl"):
|
|
self.assertIsNone(compress(copy))
|
|
code, out, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [1] if rung == LOW else [0, 1]})
|
|
self.assertEqual(code, 0, err)
|
|
self.assertTrue(out.startswith("Replayed 1 stability rounds of claude-opus-5-5 projects,"), out)
|
|
|
|
def test_copies_a_compressed_report_from_before_the_round_under_its_plain_name(self):
|
|
folder = self.review_folder_with_round_1()
|
|
self.assertIsNone(compress(folder / "stability-round-1.md"))
|
|
self.set_prompt(f"Review branch x in {self.home}/code/proj-x. Write your report to ~/.local/state/larv-review/code/proj-x/stability-round-2.md; the last round's is ~/.local/state/larv-review/code/proj-x/stability-round-1.md.")
|
|
seen = []
|
|
|
|
def review(rung):
|
|
named = pathlib.Path(re.search(r"report to (\S+)/stability-round-2\.md", self.reviews[-1][4]).group(1))
|
|
seen.append((sorted(path.name for path in named.iterdir()), (named / "stability-round-1.md").read_text()))
|
|
return {"type": "result", "result": f"review on {rung.model} {rung.effort}"}
|
|
|
|
code, _, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0, 1]}, review)
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual(seen[0], (["stability-round-1.md"], "round 1\n"))
|
|
|
|
def test_points_the_prompts_review_folder_at_a_copy_of_the_reports_from_before_the_round_that_it_removes(self):
|
|
folder = self.review_folder_with_round_1()
|
|
self.set_prompt(f"Review branch x in {self.home}/code/proj-x. Write your report to ~/.local/state/larv-review/code/proj-x/stability-round-2.md; the last round's is ~/.local/state/larv-review/code/proj-x/stability-round-1.md.")
|
|
seen = []
|
|
|
|
def review(rung):
|
|
named = pathlib.Path(re.search(r"report to (\S+)/stability-round-2\.md", self.reviews[-1][4]).group(1))
|
|
seen.append((named, sorted(path.name for path in named.iterdir()), (named / "stability-round-1.md").read_text()))
|
|
(named / "stability-round-2.md").write_text("replayed\n")
|
|
return {"type": "result", "result": f"review on {rung.model} {rung.effort}"}
|
|
|
|
code, _, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0, 1]}, review)
|
|
self.assertEqual(code, 0, err)
|
|
named, copied, earlier = seen[0]
|
|
self.assertEqual(named.parent, folder.parents[1], "the copy sits in the reports folder, where rule-hook lets the reviewer write")
|
|
self.assertNotEqual(named, folder)
|
|
self.assertEqual(copied, ["stability-round-1.md"], "the round's own report, later ones and an earlier attempt's would hand the replayed reviewer its findings")
|
|
self.assertEqual(earlier, "round 1\n")
|
|
self.assertFalse(named.exists())
|
|
self.assertEqual(sorted(path.name for path in folder.iterdir()), ["prose-round-1.md", "stability-round-1.md", "stability-round-2.md", "stability-round-3.md"])
|
|
self.assertEqual((folder / "stability-round-2.md").read_text(), "🔴 a.txt:1 wrong\n")
|
|
|
|
def test_points_the_review_folders_absolute_path_at_the_copy(self):
|
|
folder = self.review_folder_with_round_1()
|
|
self.set_prompt(f"Review branch x in {self.home}/code/proj-x. Write your report to {folder}/stability-round-2.md.")
|
|
code, _, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0, 1]})
|
|
self.assertEqual(code, 0, err)
|
|
prompt = self.reviews[0][4]
|
|
self.assertNotIn(str(folder), prompt)
|
|
self.assertRegex(prompt, rf"report to {folder.parents[1]}/review-replay-\w+/stability-round-2\.md\.$")
|
|
|
|
def test_replays_no_round_whose_prompt_still_names_the_review_folder_after_the_rewrite(self):
|
|
self.review_folder_with_round_1()
|
|
self.set_prompt(f"Review branch x in {self.home}/code/proj-x. Write your report to $HOME/.local/state/larv-review/code/proj-x/stability-round-2.md.")
|
|
code, out, err = self.run_main(["stability", "claude-opus-5-5"], unused)
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual(self.reviews, [])
|
|
self.assertIn(f"on {REFERENCE} failed: the prompt names the review folder in a spelling the replay does not redirect", err)
|
|
self.assertIn("Left out 1 more, which a replay on the project's rung failed on", out)
|
|
self.assertTrue(out.endswith("No rung tried: fewer than 1 rounds were replayed on the project's rung without a failure.\n"), out)
|
|
|
|
def test_replays_a_round_whose_prompt_names_an_input_written_before_it_began_naming_the_copy(self):
|
|
folder = self.review_folder_with_round_1()
|
|
(folder / "branch.diff").write_text("the diff\n")
|
|
before = stamp_seconds("2026-10-14T09:50:00Z")
|
|
os.utime(folder / "branch.diff", (before, before))
|
|
self.set_prompt(f"Review branch x in {self.home}/code/proj-x. The diff is ~/.local/state/larv-review/code/proj-x/branch.diff. Write your report to ~/.local/state/larv-review/code/proj-x/stability-round-2.md.")
|
|
code, _, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0, 1]})
|
|
self.assertEqual(code, 0, err)
|
|
self.assertRegex(self.reviews[0][4], rf"The diff is {folder.parents[1]}/review-replay-\w+/branch\.diff\.")
|
|
|
|
def test_replays_no_round_whose_prompt_names_an_input_the_folder_holds_only_as_rewritten_after_the_round_began(self):
|
|
folder = self.review_folder_with_round_1()
|
|
(folder / "branch.diff").write_text("a later diff\n")
|
|
self.set_prompt(f"Review branch x in {self.home}/code/proj-x. The diff is ~/.local/state/larv-review/code/proj-x/branch.diff. Write your report to ~/.local/state/larv-review/code/proj-x/stability-round-2.md.")
|
|
code, out, err = self.run_main(["stability", "claude-opus-5-5"], unused)
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual(self.reviews, [])
|
|
self.assertIn(f"on {REFERENCE} failed: the prompt names branch.diff, which the review folder did not hold from the chunk's start until the round's", err)
|
|
|
|
def test_fails_the_replay_and_leaves_no_folder_behind_when_it_cannot_copy_the_reports(self):
|
|
folder = self.review_folder_with_round_1()
|
|
(folder / "stability-round-1.md").chmod(0)
|
|
self.set_prompt(f"Review branch x in {self.home}/code/proj-x. Write your report to ~/.local/state/larv-review/code/proj-x/stability-round-2.md.")
|
|
scratch = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
|
|
self.limits["scratch"] = scratch
|
|
code, _, err = self.run_main(["stability", "claude-opus-5-5"], unused)
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual(self.reviews, [])
|
|
self.assertIn(f"on {REFERENCE} failed: could not prepare the replay: ", err)
|
|
self.assertEqual(list(scratch.iterdir()), [], "the clone is removed")
|
|
self.assertEqual([path.name for path in folder.parent.parent.iterdir()], ["code"], "the copy is removed")
|
|
|
|
def test_replays_the_rounds_of_every_chunk(self):
|
|
self.write_chunk("proj", "c2", "2026-10-14T10:30:00Z")
|
|
code, out, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0, 1]})
|
|
self.assertEqual(code, 0, err)
|
|
self.assertTrue(out.startswith("Replayed 2 stability rounds"), out)
|
|
|
|
def test_leaves_out_a_round_that_either_replay_on_the_project_rung_failed_on(self):
|
|
self.write_chunk("proj", "c2", "2026-10-14T10:30:00Z")
|
|
self.limits["at_once"] = 1
|
|
# One round at a time, three tries on a round reply with no report: the first replay's on its first round, then the second replay's on its first, after the first replay's two reviews. Every other review replies.
|
|
for failing in (range(3), range(2, 5)):
|
|
with self.subTest(failing=failing):
|
|
calls = iter(range(100))
|
|
code, out, err = self.run_main(["prose", "claude-opus-5-5"], lambda rung: {"found": [0, 1]}, review=lambda rung: {"type": "result"} if next(calls) in failing else {"type": "result", "result": f"review on {rung.model} {rung.effort}"})
|
|
self.assertEqual(code, 0, err)
|
|
self.assertTrue(out.startswith("Replayed 1 prose rounds"), out)
|
|
self.assertIn("Left out 1 more", out)
|
|
|
|
def test_tries_no_lower_rung_where_the_project_rung_found_nothing(self):
|
|
code, out, err = self.run_main(["prose", "claude-opus-5-5"], lambda rung: {"found": []})
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual([rung for rung, *_ in self.reviews], [REFERENCE, REFERENCE])
|
|
self.assertTrue(out.endswith("No rung tried: the project's rung found too few of them to hold a rung to.\n"), out)
|
|
|
|
def test_checks_out_the_commit_detached_when_the_reviewer_started_on_the_default_branch(self):
|
|
self.set_branch("main")
|
|
self.assertEqual(self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0]})[0], 0)
|
|
self.assertEqual(self.reviews[0][1:3], ("1\n", ""))
|
|
|
|
def test_sets_the_default_branch_to_its_own_line_and_not_to_a_branch_merged_after_the_chunk_began(self):
|
|
# Branch side's commit was authored before the chunk began, but the default branch held it only once merged, after.
|
|
run_git(self.checkout, "reset", "-q", "--hard", "HEAD~2")
|
|
run_git(self.checkout, "checkout", "-qb", "side")
|
|
(self.checkout / "a.txt").write_text("s\n")
|
|
run_git(self.checkout, "commit", "-qam", "s", at="2026-10-14T09:10:00Z")
|
|
run_git(self.checkout, "checkout", "-q", "main")
|
|
run_git(self.checkout, "merge", "-q", "--no-ff", "-m", "m", "side", at="2026-10-14T09:40:00Z")
|
|
run_git(self.checkout, "push", "-qf", "origin", "main")
|
|
self.assertEqual(self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"found": [0]})[0], 0)
|
|
self.assertEqual(self.reviews[0][1:4], ("s\n", "x\n", "0\n"))
|
|
|
|
def test_leaves_out_a_round_whose_review_on_the_project_rung_has_no_report(self):
|
|
code, out, err = self.run_main(["stability", "claude-opus-5-5"], unused, review=lambda rung: {"type": "result"})
|
|
self.assertEqual(code, 0, err)
|
|
# Each of the two replays tries claude three times.
|
|
self.assertEqual([rung for rung, *_ in self.reviews], [REFERENCE] * 6)
|
|
self.assertIn(f" {self.checkout} 2026-10-14T10:00:00Z on {REFERENCE} failed: the reviewer replied with no report", err)
|
|
self.assertIn("Left out 1 more, which a replay on the project's rung failed on", out)
|
|
self.assertTrue(out.endswith("No rung tried: fewer than 1 rounds were replayed on the project's rung without a failure.\n"), out)
|
|
|
|
def assert_no_round(self):
|
|
code, _, err = self.run_main(["stability", "claude-opus-5-5"], unused)
|
|
self.assertEqual(code, 1)
|
|
self.assertEqual(err.splitlines(), [f"review-replay: no archived stability round of projects on claude-opus-5-5 in {self.archive} holds a 🔴 or 🟡 on code it can find"])
|
|
|
|
def test_skips_a_chunk_with_no_transcript_copy(self):
|
|
(self.archive / "proj" / "c1" / "t0.jsonl").unlink()
|
|
self.assert_no_round()
|
|
|
|
def test_skips_a_chunk_whose_transcript_copy_has_no_timestamp(self):
|
|
(self.archive / "proj" / "c1" / "t0.jsonl").write_text(json.dumps({"type": "summary"}) + "\n")
|
|
self.assert_no_round()
|
|
|
|
def assert_left_out(self, reason, root=None):
|
|
code, _, err = self.run_main(["stability", "claude-opus-5-5"], unused)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn(f"{root or self.checkout} 2026-10-14T10:00:00Z is left out: {reason}", err)
|
|
|
|
def test_leaves_out_a_round_whose_prompt_names_no_worktree(self):
|
|
self.set_prompt("Review the branch.")
|
|
self.assert_left_out("the prompt names neither one removed worktree nor the checkout")
|
|
|
|
def test_leaves_out_a_round_whose_checkout_names_no_default_branch(self):
|
|
run_git(self.checkout, "remote", "set-head", "origin", "-d")
|
|
self.assert_left_out("origin records no default branch")
|
|
|
|
def test_leaves_out_a_round_whose_checkout_is_gone(self):
|
|
shutil.rmtree(self.checkout)
|
|
# With no folder under ~/code, the key names /proj.
|
|
self.assert_left_out("the project has no checkout", "/proj")
|
|
|
|
def test_leaves_out_a_round_whose_chunk_began_before_the_checkouts_first_commit(self):
|
|
self.write_chunk("proj", "c1", "2026-10-14T10:00:00Z", began="2026-10-14T08:00:00Z")
|
|
self.assert_left_out("no commit was authored by then")
|
|
|
|
def test_exits_1_on_a_usage_limit_bad_arguments_or_no_round_to_replay(self):
|
|
code, _, err = self.run_main(["stability", "claude-opus-5-5"], lambda rung: {"type": "result", "api_error_status": 429, "result": "You've hit your limit"})
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("review-replay: claude hit a usage limit: You've hit your limit", err)
|
|
for argv in ([], ["claude-opus-5-5"], ["full scan", "claude-opus-5-5"], ["stability", "off-ladder"], ["stability", "claude-opus-5-5", "x"]):
|
|
with self.subTest(argv=argv):
|
|
self.assertEqual(self.run_main(argv, unused)[0], 1)
|
|
code, _, err = self.run_main(["stability", "claude-sonnet-5-5"], unused)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("no archived stability round of projects on claude-sonnet-5-5", err)
|
|
|
|
def test_prints_the_help(self):
|
|
code, out, _ = self.run_main(["-h"], unused)
|
|
self.assertEqual((code, out.splitlines()[2]), (0, "usage: review-replay <pass> <model> [--omit-claude-md | --bash-ceiling | --read-limit]"))
|
|
self.assertIn(f"{', '.join(list(REVIEWERS)[:-1])} or {list(REVIEWERS)[-1]}.", " ".join(out.split()))
|
|
|
|
def test_the_launcher_prints_the_help(self):
|
|
ran = subprocess.run([sys.executable, str(TOOL), "--help"], capture_output=True, text=True)
|
|
self.assertEqual(ran.returncode, 0)
|
|
self.assertIn("usage: review-replay <pass> <model>", ran.stdout)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|