400 lines
27 KiB
Python
400 lines
27 KiB
Python
import contextlib
|
|
import io
|
|
import json
|
|
import pathlib
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from chunk_driver.prompts import ANSWER_DESTINATIONS
|
|
from transcript_archive.outcomes import ANSWERED, FINISHED, INTERRUPTED, UNFINISHED
|
|
from claude_code.entries import AssistantTurn, HumanTurn, LoopPrompt, Text
|
|
from judge_replay.replay import Case, case_from, report, reproduced, sample, verdict_kind
|
|
from text_files.doubles import compress
|
|
from project_config.doubles import configured
|
|
from loop_effects.effects import Effects
|
|
from failure import Failure
|
|
from agent_cycle.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished
|
|
from judge_replay.archived_prompt import grounds_of
|
|
from agent_cycle.judge import judge_archived, judge_rules, Rejected, TASK_HEADING, transcript_heading
|
|
from chunk_driver.drivers import LOOP_CLAUDE
|
|
from judge_replay import main
|
|
from rung_replay.search import Miss, Replay, Tried
|
|
from model_routing.ladder import Rung
|
|
|
|
TOOL = pathlib.Path(__file__).resolve().parents[1] / "judge-replay"
|
|
TRANSCRIPT = pathlib.Path("/archive/p/c/a.jsonl")
|
|
CHEAP = Rung("claude-haiku-5-5", "xhigh")
|
|
MIDDLE = Rung("claude-sonnet-5-5", "high")
|
|
REFERENCE = Rung("claude-opus-5-5", "xhigh")
|
|
|
|
|
|
def case(at, archived_kind):
|
|
return Case("/code/p", "Do the next chunk.", TRANSCRIPT, at, archived_kind)
|
|
|
|
|
|
class CaseFrom(unittest.TestCase):
|
|
def test_takes_the_task_and_the_first_verdict_on_the_copy(self):
|
|
outcomes = "Task: Do the next chunk.\nThen merge it.\n\n2026-10-14T10:00:00Z Stopped: claude failed\n\nWorker: x\n2026-10-14T11:00:00Z Verdict: unfinished\n\nEvidence:\n> on\n\nWorker: x\n2026-10-14T11:00:01Z Stepped up\n\nWorker: y\n"
|
|
self.assertEqual(case_from("/code/p", outcomes, TRANSCRIPT), Case("/code/p", "Do the next chunk.\nThen merge it.", TRANSCRIPT, "2026-10-14T11:00:00Z", UNFINISHED))
|
|
|
|
def test_reads_each_kind_of_verdict(self):
|
|
for verdict in (FINISHED, UNFINISHED, INTERRUPTED, ANSWERED):
|
|
with self.subTest(verdict=verdict):
|
|
found = case_from("/code/p", f"Task: t\n\n2026-10-14T10:00:00Z {verdict}\n\nmore\n", TRANSCRIPT)
|
|
self.assertEqual(found.archived_kind, verdict)
|
|
|
|
def test_names_the_judge_run_that_the_first_verdict_records_beside_it(self):
|
|
outcomes = f"Task: t\n\n2026-10-14T10:00:00Z Verdict: unfinished\n\nJudge: j\nJudge run: judge/{'a' * 64}.json\nWorker: x\n\n2026-10-14T11:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/{'b' * 64}.json\nWorker: x\n"
|
|
self.assertEqual(case_from("/code/p", outcomes, TRANSCRIPT).judge_run, TRANSCRIPT.parent / f"judge/{'a' * 64}.json")
|
|
self.assertIsNone(case_from("/code/p", "Task: t\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nWorker: x\n", TRANSCRIPT).judge_run)
|
|
|
|
def test_skips_a_copy_with_no_verdict(self):
|
|
self.assertIsNone(case_from("/code/p", "Task: t\n\n2026-10-14T10:00:00Z Stepped up\n\nWorker: y\n", TRANSCRIPT))
|
|
self.assertIsNone(case_from("/code/p", "not an outcomes file", TRANSCRIPT))
|
|
|
|
|
|
class Sample(unittest.TestCase):
|
|
def test_keeps_the_latest_of_each_kind_in_the_order_they_were_judged(self):
|
|
cases = [case("2026-10-14T10:00:01Z", FINISHED), case("2026-10-14T10:00:03Z", FINISHED), case("2026-10-14T10:00:02Z", FINISHED), case("2026-10-14T10:00:00Z", INTERRUPTED)]
|
|
self.assertEqual(sample(cases, 2), [cases[3], cases[2], cases[1]])
|
|
|
|
|
|
class VerdictKind(unittest.TestCase):
|
|
def test_names_each_verdict_as_the_archive_does(self):
|
|
self.assertEqual(verdict_kind(Finished("e")), FINISHED)
|
|
self.assertEqual(verdict_kind(Unfinished("e", 3)), UNFINISHED)
|
|
self.assertEqual(verdict_kind(Interrupt("e", "q?")), INTERRUPTED)
|
|
self.assertEqual(verdict_kind(QuotedAnswer("e", "q?", "a")), ANSWERED)
|
|
self.assertEqual(verdict_kind(Failure("the judge quoted x")), "failed: the judge quoted x")
|
|
|
|
|
|
class Reproduced(unittest.TestCase):
|
|
def test_holds_a_rung_to_a_verdict_only_where_the_reference_returned_the_archived_kind(self):
|
|
self.assertEqual(reproduced(case("2026-10-14T10:00:00Z", FINISHED), FINISHED), [FINISHED])
|
|
self.assertEqual(reproduced(case("2026-10-14T10:00:00Z", FINISHED), INTERRUPTED), [])
|
|
|
|
|
|
class Report(unittest.TestCase):
|
|
def test_names_whether_a_missed_verdict_was_replayed_on_its_archived_prompt(self):
|
|
kept = Case("/code/p", "Do it.", TRANSCRIPT, "2026-10-14T10:00:00Z", FINISHED, pathlib.Path("/archive/p/c/judge/run.json"))
|
|
lines = report("claude-opus-5-5", REFERENCE, [kept], Replay([UNFINISHED], [], [])).split("\n")
|
|
self.assertIn(" /code/p 2026-10-14T10:00:00Z: archived Verdict: finished, replayed Verdict: unfinished on its archived prompt", lines)
|
|
|
|
def test_counts_the_verdicts_and_names_each_miss_and_the_lowest_rung(self):
|
|
cases = [case("2026-10-14T10:00:00Z", FINISHED), case("2026-10-14T10:00:01Z", INTERRUPTED)]
|
|
replay = Replay([UNFINISHED, INTERRUPTED], [Tried(MIDDLE, []), Tried(CHEAP, [Miss(cases[1], INTERRUPTED, FINISHED)])], MIDDLE)
|
|
self.assertEqual(report("claude-opus-5-5", REFERENCE, cases, replay).splitlines(), [
|
|
"Replayed 2 verdicts of claude-opus-5-5 projects: 1 Verdict: finished, 1 Verdict: interrupt.",
|
|
"The reference judge, claude-opus-5-5 at xhigh effort, reproduced 1. It returned another kind on:",
|
|
" /code/p 2026-10-14T10:00:00Z: archived Verdict: finished, replayed Verdict: unfinished on a prompt rebuilt from today's records",
|
|
"claude-sonnet-5-5 at high effort missed 0.",
|
|
"claude-haiku-5-5 at xhigh effort missed 1:",
|
|
" /code/p 2026-10-14T10:00:01Z: archived Verdict: interrupt, replayed Verdict: finished on a prompt rebuilt from today's records",
|
|
"Lowest candidate that, like each candidate above it, returns every verdict the reference judge reproduces: claude-sonnet-5-5 at high effort",
|
|
])
|
|
missed_all = report("claude-opus-5-5", REFERENCE, cases, Replay([FINISHED, INTERRUPTED], [Tried(MIDDLE, [Miss(cases[1], INTERRUPTED, FINISHED)])], None)).splitlines()
|
|
self.assertEqual(missed_all[1], "The reference judge, claude-opus-5-5 at xhigh effort, reproduced 2.")
|
|
self.assertEqual(missed_all[-1], "Lowest candidate that, like each candidate above it, returns every verdict the reference judge reproduces: none, the project's rung misses one")
|
|
self.assertEqual(report("claude-opus-5-5", REFERENCE, cases, Replay([UNFINISHED, FINISHED], [], None)).splitlines()[-1], "No rung tried: the reference judge reproduced no verdict to hold one to.")
|
|
|
|
def test_names_each_miss_of_each_rung_the_judge_climbs_to_before_the_search_down(self):
|
|
cases = [case("2026-10-14T10:00:00Z", FINISHED), case("2026-10-14T10:00:01Z", INTERRUPTED)]
|
|
climbs = [Tried(Rung("claude-fable-5-2", "low"), [Miss(cases[1], INTERRUPTED, FINISHED)]), Tried(Rung("claude-fable-5-2", "high"), [])]
|
|
self.assertEqual(report("claude-opus-5-5", REFERENCE, cases, Replay([FINISHED, INTERRUPTED], [Tried(MIDDLE, [])], MIDDLE, climbs)).splitlines()[2:5], [
|
|
"claude-fable-5-2 at low effort, a rung a review climbs to, missed 1:",
|
|
" /code/p 2026-10-14T10:00:01Z: archived Verdict: interrupt, replayed Verdict: finished on a prompt rebuilt from today's records",
|
|
"claude-fable-5-2 at high effort, a rung a review climbs to, missed 0.",
|
|
])
|
|
|
|
|
|
def unused(*_):
|
|
raise AssertionError("not called")
|
|
|
|
|
|
class Main(unittest.TestCase):
|
|
"""The tool on a temporary home: a project on Opus 5.5 with one finished and one interrupt verdict, and one on another model."""
|
|
|
|
def setUp(self):
|
|
self.home = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory()))
|
|
(self.home / ".claude").mkdir()
|
|
(self.home / ".claude" / "loop-claude-models.json").write_text(json.dumps({"proj": "claude-opus-5-5", "other": "claude-sonnet-5-5"}))
|
|
(self.home / ".claude" / "CLAUDE.md").write_text("Be kind.\n")
|
|
for key, verdicts in (("proj", (FINISHED, INTERRUPTED)), ("other", (FINISHED,))):
|
|
(self.home / "code" / key).mkdir(parents=True)
|
|
(self.home / "code" / key / "README.md").write_text("# p\n\n## Goals\n\n1. Ship.\n\n## Audience\n\nMe.\n")
|
|
configured(self, self.home / "code" / key, home=self.home)
|
|
chunk = self.home / ".claude" / "loop-claude-transcripts" / key / "c1"
|
|
chunk.mkdir(parents=True)
|
|
for n, verdict in enumerate(verdicts):
|
|
(chunk / f"t{n}.jsonl").write_text(json.dumps({"type": "assistant", "message": {"content": [{"type": "text", "text": f"said {verdict}"}]}}) + "\n")
|
|
(chunk / f"t{n}.outcomes.md").write_text(f"Task: Do it.\n\n2026-10-14T10:00:0{n}Z {verdict}\n\nEvidence:\n> e\n")
|
|
self.calls = []
|
|
self.prompts = []
|
|
|
|
def run_main(self, argv, reply):
|
|
def claude(args, cwd, prompt):
|
|
self.calls.append((args[args.index("--model") + 1], args[args.index("--effort") + 1], cwd))
|
|
self.prompts.append(prompt)
|
|
result = reply(Rung(args[args.index("--model") + 1], args[args.index("--effort") + 1]), "said Verdict: interrupt" in prompt.rsplit("latest last:", 1)[-1])
|
|
return subprocess.CompletedProcess(args, 0, json.dumps(result), "")
|
|
|
|
effects = Effects(claude=claude, claude_on_terminal=unused, discard_typed=unused, dm=unused, execv=unused, git=unused, input=unused, local_time=unused, memory=unused, now=lambda: 1790000000.0, on_terminal=unused, python=unused, replace_file=unused, sleep=unused, stream_claude=unused, timer=unused, tmux=unused, tmux_on_terminal=unused)
|
|
out, err = io.StringIO(), io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
code = main.main(argv, {}, self.home, effects)
|
|
return code, out.getvalue(), err.getvalue()
|
|
|
|
@staticmethod
|
|
def verdict(interrupt, evidence):
|
|
output = {"answer": "", "escalation": "", "evidence": evidence, "finished": not interrupt, "interrupt": interrupt, "question": "Which?" if interrupt else "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}
|
|
return {"type": "result", "structured_output": output}
|
|
|
|
def test_replays_the_models_projects_on_the_reference_then_on_each_rung_the_judge_climbs_to_then_from_the_projects_rung_down_and_prints_the_lowest_that_returns_all(self):
|
|
# Sonnet and Fable at medium miss the interrupt; Opus returns both.
|
|
def reply(rung, interrupted):
|
|
return self.verdict(interrupted and rung not in (MIDDLE, Rung("claude-fable-5-2", "medium")), f"said {INTERRUPTED if interrupted else FINISHED}")
|
|
|
|
code, out, err = self.run_main(["claude-opus-5-5"], reply)
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual([prompt for prompt in self.prompts if f"writing it into {ANSWER_DESTINATIONS}," not in prompt], [])
|
|
self.assertEqual({cwd for _, _, cwd in self.calls}, {str(self.home / "code" / "proj")})
|
|
self.assertEqual([(model, effort) for model, effort, _ in self.calls[::2]], [
|
|
("claude-opus-5-5", "xhigh"),
|
|
("claude-fable-5-2", "low"), ("claude-fable-5-2", "medium"), ("claude-fable-5-2", "high"),
|
|
("claude-opus-5-5", "medium"), ("claude-sonnet-5-5", "high"),
|
|
])
|
|
self.assertIn("The reference judge, claude-opus-5-5 at xhigh effort, reproduced 2.", out)
|
|
self.assertIn("claude-fable-5-2 at low effort, a rung a review climbs to, missed 0.\nclaude-fable-5-2 at medium effort, a rung a review climbs to, missed 1:", out)
|
|
self.assertTrue(out.endswith("returns every verdict the reference judge reproduces: claude-opus-5-5 at medium effort\n"))
|
|
for said in ("The task the worker was given:\nDo it.", "1. Ship.", "From ~/.claude/CLAUDE.md:\nBe kind."):
|
|
with self.subTest(said=said):
|
|
self.assertIn(said, self.prompts[0])
|
|
# The premises head the prompt and the records follow them.
|
|
self.assertRegex(self.prompts[0], r"(?s)From the goals file, \S*/README\.md:\n## Goals\n1\. Ship\..*The records:.*Be kind\.")
|
|
# The replay shows the judge the whole transcript, with no step-up marked in it.
|
|
self.assertNotIn("[step]", self.prompts[0])
|
|
|
|
def test_offline_replays_every_verdict_through_the_code_deciders_without_calling_the_judge(self):
|
|
code, out, err = self.run_main(["claude-opus-5-5", "--offline"], unused)
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual(self.calls, [])
|
|
self.assertIn("Replayed 2 archived verdicts of claude-opus-5-5 projects offline, without sending anything to an outside service. The archive gives them as 1 finished, 1 interrupt.", out)
|
|
self.assertIn("\"?\" rule, on the whole archive: decided 2 of 2: finished 2 (of these, archived interrupt 1). Missed interrupts: 1.", out)
|
|
folder = self.home / ".claude" / "loop-claude-transcripts" / "replays" / "judge-replay-claude-opus-5-5-offline-20260921T141320Z"
|
|
self.assertEqual((folder / "result.md").read_text(encoding="utf-8"), out)
|
|
|
|
def test_offline_exits_1_where_no_transcript_can_be_read(self):
|
|
for path in (self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1").glob("*.jsonl"):
|
|
path.unlink()
|
|
code, out, err = self.run_main(["claude-opus-5-5", "--offline"], unused)
|
|
self.assertEqual((code, out), (1, ""))
|
|
self.assertEqual(err.splitlines()[-1], "judge-replay: no archived verdict has a transcript that can be read")
|
|
|
|
def test_offline_skips_each_verdict_whose_transcript_cannot_be_read_and_says_how_many_on_stderr_and_in_the_report(self):
|
|
kept = self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1"
|
|
(kept / "t2.outcomes.md").write_text(f"Task: Do it.\n\n2026-10-14T10:00:02Z {FINISHED}\n\nEvidence:\n> e\n")
|
|
(kept / "t0.jsonl").unlink()
|
|
code, out, err = self.run_main(["claude-opus-5-5", "--offline"], unused)
|
|
self.assertEqual(code, 0)
|
|
self.assertEqual(len(err.splitlines()), 2)
|
|
self.assertTrue(all(line.startswith("judge-replay: skipped a verdict: could not read the transcript ") for line in err.splitlines()))
|
|
self.assertIn("Replayed 1 archived verdicts of claude-opus-5-5 projects offline", out)
|
|
self.assertIn("Skipped 2 verdicts whose transcript could not be read.", out)
|
|
|
|
def test_offline_keeps_its_report_in_the_same_folder_wherever_the_flag_stands(self):
|
|
code, out, err = self.run_main(["--offline", "claude-opus-5-5"], unused)
|
|
self.assertEqual(code, 0, err)
|
|
folder = self.home / ".claude" / "loop-claude-transcripts" / "replays" / "judge-replay-claude-opus-5-5-offline-20260921T141320Z"
|
|
self.assertEqual((folder / "result.md").read_text(encoding="utf-8"), out)
|
|
|
|
def test_offline_says_the_usage_on_arguments_that_name_no_model(self):
|
|
code, out, err = self.run_main(["--offline"], unused)
|
|
self.assertEqual((code, out), (1, ""))
|
|
self.assertEqual(err, "judge-replay: cannot read the arguments --offline; usage: judge-replay <model> [--offline]; <model> is a model on the routing ladder\n")
|
|
|
|
def test_keeps_each_run_and_the_printed_report_in_one_folder_of_the_archive_beside_the_projects(self):
|
|
code, out, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
|
|
self.assertEqual(code, 0, err)
|
|
folder = self.home / ".claude" / "loop-claude-transcripts" / "replays" / "judge-replay-claude-opus-5-5-20260921T141320Z"
|
|
self.assertEqual((folder / "result.md").read_text(encoding="utf-8"), out)
|
|
runs = [json.loads(path.read_text(encoding="utf-8").splitlines()[0]) for path in folder.glob("*.jsonl")]
|
|
self.assertEqual({run["prompt"] for run in runs}, set(self.prompts))
|
|
|
|
def test_hands_the_judge_the_prompt_the_archive_kept_and_rebuilds_the_prompt_of_a_verdict_with_none(self):
|
|
kept = self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1"
|
|
prompt = "From the goals file, OLD premises\n\nThe records: none\n\nThe task the worker was given:\nDo it.\n\nWhat the worker did and said, what the human typed on [human] lines and what loop-claude told it on [loop] lines, latest last:\nsaid Verdict: finished\n\nOLD rules"
|
|
(kept / "judge").mkdir()
|
|
(kept / "judge" / "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json").write_text(json.dumps({"prompt": prompt, "reply": {"type": "result"}}))
|
|
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json\nWorker: x\n")
|
|
code, out, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
|
|
self.assertEqual(code, 0, err)
|
|
self.assertIn("The reference judge, claude-opus-5-5 at xhigh effort, reproduced 2.", out)
|
|
# The first replay, on the finished verdict, reads the archived prompt; the second, on the interrupt, is rebuilt from the records of today.
|
|
self.assertEqual(self.prompts[0], prompt)
|
|
self.assertEqual(self.calls[0][2], self.calls[1][2])
|
|
self.assertTrue(self.calls[0][2].endswith("/code/proj"))
|
|
self.assertIn("The task the worker was given:\nDo it.", self.prompts[1])
|
|
self.assertIn("1. Ship.", self.prompts[1])
|
|
|
|
def test_a_verdict_whose_archived_judge_run_cannot_be_read_replays_as_failed(self):
|
|
kept = self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1"
|
|
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.json\nWorker: x\n")
|
|
code, out, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
|
|
self.assertEqual(code, 0, err)
|
|
self.assertIn("The reference judge, claude-opus-5-5 at xhigh effort, reproduced 1. It returned another kind on:", out)
|
|
self.assertIn("could not read the archived judge run", out)
|
|
|
|
def test_tries_no_rung_below_the_lowest_a_worker_routes_to_where_every_rung_returns_every_verdict(self):
|
|
code, out, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual([(model, effort) for model, effort, _ in self.calls[::2]][-1], ("claude-sonnet-5-5", "high"))
|
|
self.assertTrue(out.endswith("returns every verdict the reference judge reproduces: claude-sonnet-5-5 at high effort\n"), out)
|
|
|
|
def test_a_fable_project_tries_no_rung_below_opus_at_medium_and_its_report_names_fable(self):
|
|
(self.home / ".claude" / "loop-claude-models.json").write_text(json.dumps({"proj": "claude-opus-5-5", "other": "claude-fable-5-2"}))
|
|
code, out, err = self.run_main(["claude-fable-5-2"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
|
|
self.assertEqual(code, 0, err)
|
|
self.assertEqual([(model, effort) for model, effort, _ in self.calls][-1], ("claude-opus-5-5", "medium"))
|
|
self.assertTrue(out.splitlines()[0].startswith("Replayed 1 verdicts of claude-fable-5-2 projects"), out)
|
|
self.assertTrue(out.endswith("returns every verdict the reference judge reproduces: claude-opus-5-5 at medium effort\n"), out)
|
|
|
|
def test_stops_at_the_projects_rung_and_names_none_when_it_misses(self):
|
|
code, out, _ = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted and rung == REFERENCE, f"said {INTERRUPTED if interrupted else FINISHED}"))
|
|
self.assertEqual(code, 0)
|
|
# The reference and the three rungs a review climbs to each replay both verdicts first.
|
|
self.assertEqual({(model, effort) for model, effort, _ in self.calls[8:]}, {("claude-opus-5-5", "medium")})
|
|
self.assertTrue(out.endswith(": none, the project's rung misses one\n"))
|
|
|
|
def test_replays_a_verdict_whose_transcript_is_gone_as_failed(self):
|
|
(self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1" / "t0.jsonl").unlink()
|
|
code, out, _ = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED}"))
|
|
self.assertEqual(code, 0)
|
|
self.assertIn("replayed failed: could not read the transcript", out)
|
|
|
|
def test_replays_a_verdict_whose_transcript_is_compressed(self):
|
|
self.assertIsNone(compress(self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1" / "t0.jsonl"))
|
|
code, out, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
|
|
self.assertEqual(code, 0, err)
|
|
self.assertNotIn("replayed failed", out)
|
|
self.assertIn("The reference judge, claude-opus-5-5 at xhigh effort, reproduced 2.", out)
|
|
|
|
def test_tries_the_projects_rung_first_and_retries_a_failed_reply_as_the_loop_does(self):
|
|
def reply(rung, interrupted):
|
|
return self.verdict(interrupted, "not in the transcript" if rung.model != "claude-opus-5-5" else f"said {INTERRUPTED if interrupted else FINISHED}")
|
|
|
|
code, out, _ = self.run_main(["claude-opus-5-5"], reply)
|
|
self.assertEqual(code, 0)
|
|
self.assertEqual(sum(model == "claude-sonnet-5-5" for model, _, _ in self.calls), 6)
|
|
self.assertIn("claude-opus-5-5 at medium effort missed 0.", out)
|
|
self.assertTrue(out.endswith(": claude-opus-5-5 at medium effort\n"))
|
|
|
|
def test_exits_1_on_a_usage_limit(self):
|
|
code, _, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: {"type": "result", "api_error_status": 429, "result": "You've hit your limit"})
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("usage limit: You've hit your limit", err)
|
|
|
|
def test_exits_1_on_bad_arguments(self):
|
|
for argv in ([], ["off-ladder"], ["claude-opus-5-5", "x"]):
|
|
with self.subTest(argv=argv):
|
|
self.assertEqual(self.run_main(argv, unused)[0], 1)
|
|
|
|
def test_exits_1_for_a_model_no_project_runs_on(self):
|
|
code, _, err = self.run_main(["claude-fable-5-2"], unused)
|
|
self.assertEqual(code, 1)
|
|
self.assertIn("no archived verdicts of projects on claude-fable-5-2", err)
|
|
|
|
def test_exits_1_on_an_unreadable_models_file(self):
|
|
(self.home / ".claude" / "loop-claude-models.json").write_text("[")
|
|
self.assertEqual(self.run_main(["claude-opus-5-5"], unused)[0], 1)
|
|
|
|
def test_prints_the_help(self):
|
|
code, out, _ = self.run_main(["-h"], unused)
|
|
self.assertEqual((code, out.splitlines()[2]), (0, "usage: judge-replay <model> [--offline]"))
|
|
|
|
def test_the_launcher_prints_the_help(self):
|
|
ran = subprocess.run([sys.executable, str(TOOL), "--help"], capture_output=True, text=True)
|
|
self.assertEqual(ran.returncode, 0)
|
|
self.assertIn("usage: judge-replay <model>", ran.stdout)
|
|
|
|
def test_exits_1_on_a_usage_limit_met_on_an_archived_prompt(self):
|
|
kept = self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1"
|
|
(kept / "judge").mkdir()
|
|
(kept / "judge" / "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json").write_text(json.dumps({"prompt": "P", "reply": {}}))
|
|
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json\nWorker: x\n")
|
|
code, _, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: {"type": "result", "api_error_status": 429, "result": "You've hit your limit"})
|
|
self.assertEqual((code, "usage limit: You've hit your limit" in err), (1, True))
|
|
|
|
|
|
class JudgeArchivedRuns(unittest.TestCase):
|
|
"""judge_archived on a prompt as judge_prompt writes it: the premises, the task, the transcript the judge was shown, and the rules."""
|
|
PROMPT = f"HEAD: Ship fast.\n\n{TASK_HEADING}Do it.\n\n{transcript_heading(LOOP_CLAUDE)}Worker chose option B.\n[human] Use Postgres.\n\n{judge_rules(ANSWER_DESTINATIONS, LOOP_CLAUDE)}"
|
|
ENTRIES = [AssistantTurn(parts=(Text("Worker chose option B."),)), HumanTurn(text="Use Postgres."), LoopPrompt(text="Keep going.")]
|
|
|
|
def judged(self, prompt=PROMPT, **fields):
|
|
"""What judge_archived returns on prompt where the judge's structured output is fields over an unfinished verdict, and the cwd claude ran in."""
|
|
output = {"answer": "", "escalation": "", "evidence": "Worker chose option B.", "finished": False, "interrupt": False, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False, **fields}
|
|
cwds = []
|
|
|
|
def claude(args, cwd, prompt):
|
|
cwds.append(cwd)
|
|
return subprocess.CompletedProcess(args, 0, json.dumps({"type": "result", "structured_output": output, "total_cost_usd": 0.25}), "")
|
|
|
|
effects = Effects(claude=claude, claude_on_terminal=unused, discard_typed=unused, dm=unused, execv=unused, git=unused, input=unused, local_time=unused, memory=unused, now=lambda: 1790000000.0, on_terminal=unused, python=unused, replace_file=unused, sleep=unused, stream_claude=unused, timer=unused, tmux=unused, tmux_on_terminal=unused)
|
|
grounds = grounds_of(prompt, self.ENTRIES)
|
|
return (grounds if isinstance(grounds, Failure) else judge_archived("/code/p", prompt, grounds, len(self.ENTRIES), REFERENCE, LOOP_CLAUDE, effects)), cwds
|
|
|
|
def test_runs_in_the_projects_folder_and_stamps_the_verdict_with_the_entries_read(self):
|
|
verdict, cwds = self.judged()
|
|
self.assertEqual((cwds, verdict.entries_read), (["/code/p"], 3))
|
|
|
|
def test_checks_the_evidence_against_the_transcript_the_judge_was_shown_and_not_the_premises(self):
|
|
self.assertIsInstance(self.judged(evidence="Ship fast.")[0], Rejected)
|
|
self.assertIsInstance(self.judged(evidence="Worker chose option B.")[0], Unfinished)
|
|
|
|
def test_keeps_the_cost_and_the_run_of_a_rejected_reply(self):
|
|
rejected, _ = self.judged(evidence="Never said.")
|
|
self.assertEqual((rejected.cost, rejected.run.prompt), (0.25, self.PROMPT))
|
|
|
|
def test_keeps_the_cost_of_a_reply_rejected_for_its_shape(self):
|
|
rejected, _ = self.judged(finished="yes")
|
|
self.assertEqual((type(rejected), rejected.cost), (Rejected, 0.25))
|
|
|
|
def test_checks_neither_the_task_nor_the_rules_as_the_transcript_or_the_premises(self):
|
|
self.assertIsInstance(self.judged(evidence=judge_rules(ANSWER_DESTINATIONS, LOOP_CLAUDE))[0], Rejected)
|
|
self.assertIsInstance(self.judged(answer="Do it.", question="Which?")[0], Interrupt)
|
|
|
|
def test_splits_the_prompt_at_the_first_transcript_heading(self):
|
|
quoting = self.PROMPT.replace("Worker chose", f"Quoted: {transcript_heading(LOOP_CLAUDE)}Worker chose", 1)
|
|
self.assertIsInstance(self.judged(quoting, evidence="Quoted:")[0], Unfinished)
|
|
|
|
def test_splits_the_prompt_at_the_first_task_heading(self):
|
|
quoting = self.PROMPT.replace("Do it.", f"Quote:\n\n{TASK_HEADING}Ship slowly.", 1)
|
|
self.assertIsInstance(self.judged(quoting, answer="Ship fast.", question="Which?")[0], QuotedAnswer)
|
|
self.assertIsInstance(self.judged(quoting, answer="Quote:", question="Which?")[0], Interrupt)
|
|
|
|
def test_fails_before_the_judge_runs_on_a_prompt_without_a_heading(self):
|
|
for prompt in ("No headings at all.", f"{TASK_HEADING}Do it.", f"\n\n{TASK_HEADING}Do it.\n\nNo transcript heading."):
|
|
with self.subTest(prompt=prompt):
|
|
failure, cwds = self.judged(prompt)
|
|
self.assertEqual((type(failure), cwds), (Failure, []))
|
|
self.assertIn("lacks the heading", failure.reason)
|
|
|
|
def test_checks_a_quoted_answer_against_the_premises(self):
|
|
self.assertIsInstance(self.judged(answer="Ship fast.", question="Which?")[0], QuotedAnswer)
|
|
self.assertIsInstance(self.judged(answer="Use Postgres.", question="Which?")[0], Interrupt)
|
|
|
|
def test_keeps_an_escalation_that_the_worker_wrote_after_the_start_of_the_transcript(self):
|
|
self.assertEqual(self.judged(escalation="Worker chose option B.")[0].escalation, "Worker chose option B.")
|
|
self.assertEqual(self.judged(escalation="Use Postgres.")[0].escalation, "")
|
|
|
|
def test_names_the_typed_answer_that_holds_a_reversal(self):
|
|
self.assertEqual(self.judged(reversed="Use Postgres.")[0].judged.reversed, "Use Postgres.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|