Kill the surviving mutants of the offline replay's report, skipped verdicts and classifier bounds

This commit is contained in:
2026-11-18 10:33:34 +01:00
parent e4b3d56b70
commit dff9051d8c
2 changed files with 50 additions and 2 deletions
+18 -2
View File
@@ -188,8 +188,24 @@ class Main(unittest.TestCase):
for path in (self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1").glob("*.jsonl"):
path.unlink()
code, out, err = self.run_main(["claude-opus-5-5", "--offline"], unused)
self.assertEqual(code, 1)
self.assertIn("no archived verdict has a transcript that can be read", err)
self.assertEqual((code, out), (1, ""))
self.assertEqual(err.splitlines()[-1], "judge-replay: no archived verdict has a transcript that can be read")
def test_offline_skips_each_verdict_whose_transcript_cannot_be_read_and_says_how_many_on_stderr_and_in_the_report(self):
kept = self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1"
(kept / "t2.outcomes.md").write_text(f"Task: Do it.\n\n2026-10-14T10:00:02Z {FINISHED}\n\nEvidence:\n> e\n")
(kept / "t0.jsonl").unlink()
code, out, err = self.run_main(["claude-opus-5-5", "--offline"], unused)
self.assertEqual(code, 0)
self.assertEqual(len(err.splitlines()), 2)
self.assertTrue(all(line.startswith("judge-replay: skipped a verdict: could not read the transcript ") for line in err.splitlines()))
self.assertIn("Replayed 1 archived verdicts of claude-opus-5-5 projects offline", out)
self.assertIn("Skipped 2 verdicts whose transcript could not be read.", out)
def test_offline_says_the_usage_on_arguments_that_name_no_model(self):
code, out, err = self.run_main(["--offline"], unused)
self.assertEqual((code, out), (1, ""))
self.assertEqual(err, "judge-replay: cannot read the arguments --offline; usage: judge-replay <model> [--offline]; <model> is a model on the routing ladder\n")
def test_keeps_each_run_and_the_printed_report_in_one_folder_of_the_archive_beside_the_projects(self):
code, out, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
+32
View File
@@ -72,6 +72,13 @@ class Classifier(unittest.TestCase):
self.assertNotEqual(decide(lookalike), FINISHED)
def test_abstains_where_a_lookalike_of_each_kind_sits_in_the_samples_of_the_other(self):
decide = self.trained([sample(INTERRUPTED, "merged pull request 7 and removed the worktree, no findings left"), sample(FINISHED, "I recommend option 7; tell me which of the options you want")])
for text in ("merged pull request", "I recommend option", "zzz qqq"):
with self.subTest(text=text):
self.assertIsNone(decide(text))
class Folds(unittest.TestCase):
def test_deals_samples_to_folds_in_turn_so_each_is_tested_once(self):
samples = [sample(FINISHED, str(n)) for n in range(7)]
@@ -111,6 +118,31 @@ class Report(unittest.TestCase):
self.assertNotIn("rule, on held-out", text)
self.assertNotIn("rule, on the latest fifth", text)
def test_words_the_whole_report_line_by_line_with_the_wrong_kinds_by_name(self):
answers = {"f": FINISHED, "i1": FINISHED, "i2": FINISHED, "u": FINISHED, "q": None}
samples = [sample(FINISHED, "f"), sample(INTERRUPTED, "i1"), sample(INTERRUPTED, "i2"), sample(UNFINISHED, "u"), sample(ANSWERED, "q")]
text = offline_report("m", samples, {"rule": lambda trained: lambda said: answers[said]}, {})
self.assertEqual(text.splitlines(), [
"Replayed 5 archived verdicts of m projects offline, without sending anything to an outside service. The archive gives them as 1 finished, 2 interrupt, 1 quoted answer, 1 unfinished.",
"rule, on the whole archive: decided 4 of 5: finished 4 (of these, archived interrupt 2, unfinished 1). Missed interrupts: 2.",
"rule, on the whole archive, does not decide finished with no missed interrupt: it decided 2 archived interrupt(s) as finished.",
])
def test_says_a_method_that_decided_nothing_decided_0_and_names_no_kind(self):
text = offline_report("m", self.samples(), {"rule": lambda trained: lambda said: None}, {})
self.assertEqual(text.splitlines()[1:], ["rule, on the whole archive: decided 0 of 4. Missed interrupts: 0."])
def test_tests_the_latest_fifth_of_the_samples_after_learning_from_the_earlier_ones(self):
trained = []
def learner(samples):
trained.append(len(samples))
return lambda said: FINISHED
text = offline_report("m", [sample(FINISHED, str(n)) for n in range(7)], {}, {"learner": learner}, folds=2)
self.assertEqual(trained, [3, 4, 6])
self.assertEqual(text.splitlines()[-1], "learner, on the latest fifth after learning from the rest: decided 1 of 1: finished 1. Missed interrupts: 0.")
def test_counts_the_verdicts_it_skipped_only_where_it_skipped_some(self):
self.assertNotIn("Skipped", offline_report("m", self.samples(), {}, {}))
self.assertIn("Skipped 2 verdicts whose transcript could not be read.", offline_report("m", self.samples(), {}, {}, skipped=2))