Kill the surviving mutants of the offline replay's report, skipped verdicts and classifier bounds
This commit is contained in:
@@ -188,8 +188,24 @@ class Main(unittest.TestCase):
|
||||
for path in (self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1").glob("*.jsonl"):
|
||||
path.unlink()
|
||||
code, out, err = self.run_main(["claude-opus-5-5", "--offline"], unused)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("no archived verdict has a transcript that can be read", err)
|
||||
self.assertEqual((code, out), (1, ""))
|
||||
self.assertEqual(err.splitlines()[-1], "judge-replay: no archived verdict has a transcript that can be read")
|
||||
|
||||
def test_offline_skips_each_verdict_whose_transcript_cannot_be_read_and_says_how_many_on_stderr_and_in_the_report(self):
|
||||
kept = self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1"
|
||||
(kept / "t2.outcomes.md").write_text(f"Task: Do it.\n\n2026-10-14T10:00:02Z {FINISHED}\n\nEvidence:\n> e\n")
|
||||
(kept / "t0.jsonl").unlink()
|
||||
code, out, err = self.run_main(["claude-opus-5-5", "--offline"], unused)
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(len(err.splitlines()), 2)
|
||||
self.assertTrue(all(line.startswith("judge-replay: skipped a verdict: could not read the transcript ") for line in err.splitlines()))
|
||||
self.assertIn("Replayed 1 archived verdicts of claude-opus-5-5 projects offline", out)
|
||||
self.assertIn("Skipped 2 verdicts whose transcript could not be read.", out)
|
||||
|
||||
def test_offline_says_the_usage_on_arguments_that_name_no_model(self):
|
||||
code, out, err = self.run_main(["--offline"], unused)
|
||||
self.assertEqual((code, out), (1, ""))
|
||||
self.assertEqual(err, "judge-replay: cannot read the arguments --offline; usage: judge-replay <model> [--offline]; <model> is a model on the routing ladder\n")
|
||||
|
||||
def test_keeps_each_run_and_the_printed_report_in_one_folder_of_the_archive_beside_the_projects(self):
|
||||
code, out, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
|
||||
|
||||
@@ -72,6 +72,13 @@ class Classifier(unittest.TestCase):
|
||||
self.assertNotEqual(decide(lookalike), FINISHED)
|
||||
|
||||
|
||||
def test_abstains_where_a_lookalike_of_each_kind_sits_in_the_samples_of_the_other(self):
|
||||
decide = self.trained([sample(INTERRUPTED, "merged pull request 7 and removed the worktree, no findings left"), sample(FINISHED, "I recommend option 7; tell me which of the options you want")])
|
||||
for text in ("merged pull request", "I recommend option", "zzz qqq"):
|
||||
with self.subTest(text=text):
|
||||
self.assertIsNone(decide(text))
|
||||
|
||||
|
||||
class Folds(unittest.TestCase):
|
||||
def test_deals_samples_to_folds_in_turn_so_each_is_tested_once(self):
|
||||
samples = [sample(FINISHED, str(n)) for n in range(7)]
|
||||
@@ -111,6 +118,31 @@ class Report(unittest.TestCase):
|
||||
self.assertNotIn("rule, on held-out", text)
|
||||
self.assertNotIn("rule, on the latest fifth", text)
|
||||
|
||||
def test_words_the_whole_report_line_by_line_with_the_wrong_kinds_by_name(self):
|
||||
answers = {"f": FINISHED, "i1": FINISHED, "i2": FINISHED, "u": FINISHED, "q": None}
|
||||
samples = [sample(FINISHED, "f"), sample(INTERRUPTED, "i1"), sample(INTERRUPTED, "i2"), sample(UNFINISHED, "u"), sample(ANSWERED, "q")]
|
||||
text = offline_report("m", samples, {"rule": lambda trained: lambda said: answers[said]}, {})
|
||||
self.assertEqual(text.splitlines(), [
|
||||
"Replayed 5 archived verdicts of m projects offline, without sending anything to an outside service. The archive gives them as 1 finished, 2 interrupt, 1 quoted answer, 1 unfinished.",
|
||||
"rule, on the whole archive: decided 4 of 5: finished 4 (of these, archived interrupt 2, unfinished 1). Missed interrupts: 2.",
|
||||
"rule, on the whole archive, does not decide finished with no missed interrupt: it decided 2 archived interrupt(s) as finished.",
|
||||
])
|
||||
|
||||
def test_says_a_method_that_decided_nothing_decided_0_and_names_no_kind(self):
|
||||
text = offline_report("m", self.samples(), {"rule": lambda trained: lambda said: None}, {})
|
||||
self.assertEqual(text.splitlines()[1:], ["rule, on the whole archive: decided 0 of 4. Missed interrupts: 0."])
|
||||
|
||||
def test_tests_the_latest_fifth_of_the_samples_after_learning_from_the_earlier_ones(self):
|
||||
trained = []
|
||||
|
||||
def learner(samples):
|
||||
trained.append(len(samples))
|
||||
return lambda said: FINISHED
|
||||
|
||||
text = offline_report("m", [sample(FINISHED, str(n)) for n in range(7)], {}, {"learner": learner}, folds=2)
|
||||
self.assertEqual(trained, [3, 4, 6])
|
||||
self.assertEqual(text.splitlines()[-1], "learner, on the latest fifth after learning from the rest: decided 1 of 1: finished 1. Missed interrupts: 0.")
|
||||
|
||||
def test_counts_the_verdicts_it_skipped_only_where_it_skipped_some(self):
|
||||
self.assertNotIn("Skipped", offline_report("m", self.samples(), {}, {}))
|
||||
self.assertIn("Skipped 2 verdicts whose transcript could not be read.", offline_report("m", self.samples(), {}, {}, skipped=2))
|
||||
|
||||
Reference in New Issue
Block a user