Average the offline classifier over five shuffles, count a pass only on both tests, and fix the import order

This commit is contained in:
2026-11-18 11:31:43 +01:00
parent 02288f955b
commit 76b6e029b9
5 changed files with 65 additions and 28 deletions
+6 -6
View File
@@ -44,8 +44,8 @@ kind, which are its missed interrupts: a "?" rule, a decision table, and a class
(judge_replay/offline.py). The rule and the table run on the whole archive; the table's rows were read from
it, so its run isn't a held-out test. The classifier runs on held-out folds, each tested by a model trained on
the other folds, and on the latest fifth of the verdicts, tested by a model trained on the earlier four
fifths. The report is kept in replays/judge-replay-<model>-offline-<UTC stamp>/result.md. A verdict whose
transcript cannot be read is skipped, and the report counts the skipped. Exit 1, saying why, on bad
fifths. The classifier trains 330 regressions in Python, about 12 minutes on 1,100 verdicts. The report is
kept in replays/judge-replay-<model>-offline-<UTC stamp>/result.md. A verdict whose transcript cannot be read is skipped, and the report counts the skipped. Exit 1, saying why, on bad
arguments, an unreadable archive or models file, no archived verdict of the model's projects, or none with a
transcript that can be read.
"""
@@ -58,11 +58,11 @@ from transcript_archive.read import archived_outcomes
from rung_replay.archive import archiving, print_report, replay_folder
from rung_replay.command import arguments_or_exit, down_from, failed, limited_exit, projects_on
from loop_claude.cycle.judge import Brief, judge_archived, judge_once
from judge_replay.offline import final_message, LEARNERS, offline_report, RULES, Sample
from judge_replay.replay import Case, case_from, Kind, report, reproduced, sample, verdict_kind
from transcript_archive.project_key import checkout_of
from loop_effects.effects import Effects
from failure import Failure
from judge_replay.offline import final_message, LEARNERS, offline_report, RULES, Sample
from rung_replay.search import first_reply, replayed
from judge_replay.archived_prompt import grounds_of
from transcript_archive.judge_run import JudgeRun
@@ -96,7 +96,7 @@ def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, eff
if not cases:
return failed(TOOL, f"no archived verdicts of projects on {project.rung.model} in {archive_root(home, env)}; run loop-claude on a project on that model first")
if OFFLINE in argv:
return offline(project.rung.model, argv, cases, archive_root(home, env), effects)
return offline(project.rung.model, cases, archive_root(home, env), effects)
cases = sample(cases, PER_KIND)
folder = replay_folder(archive_root(home, env), TOOL, argv, effects.now())
effects = archiving(effects, folder)
@@ -109,7 +109,7 @@ def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, eff
return 0
def offline(model: str, argv: list[str], cases: list[Case], root: pathlib.Path, effects: Effects) -> int:
def offline(model: str, cases: list[Case], root: pathlib.Path, effects: Effects) -> int:
"""Replays the verdicts of cases offline, oldest first, and keeps the report in the archive as a judge replay does."""
samples, skipped = [], 0
for found in sorted(cases, key=lambda found: (found.at, found.root, str(found.transcript))):
@@ -121,7 +121,7 @@ def offline(model: str, argv: list[str], cases: list[Case], root: pathlib.Path,
samples.append(Sample(found.archived_kind, final_message(entries)))
if not samples:
return failed(TOOL, "no archived verdict has a transcript that can be read")
print_report(replay_folder(root, TOOL, argv, effects.now()), offline_report(model, samples, RULES, LEARNERS, skipped=skipped))
print_report(replay_folder(root, TOOL, [model, OFFLINE], effects.now()), offline_report(model, samples, RULES, LEARNERS, skipped=skipped))
return 0
+22 -17
View File
@@ -5,8 +5,8 @@ import random
import re
import typing
from transcript_archive.outcomes import FINISHED, INTERRUPTED, UNFINISHED
from claude_code.entries import AssistantTurn, Entry, Text
from transcript_archive.outcomes import FINISHED, INTERRUPTED, UNFINISHED
from judge_replay.replay import Kind
@@ -26,6 +26,8 @@ INNER_FOLDS = 5
# A word in fewer messages than this is left out of the classifier.
MIN_MESSAGES = 3
EPOCHS = 8
# The classifier averages this many regressions, each shuffling the samples in its own order: one shuffle alone moved the missed interrupts of a run between 0 and 2.
SEEDS = 5
RATE = 0.05
DECAY = 1e-3
# The latest 1/LATEST of the archive is the second test: the verdicts a decider would meet next.
@@ -63,14 +65,14 @@ def words_of(text: str) -> frozenset[str]:
return frozenset(words) | {f"{first} {second}" for first, second in zip(words, words[1:])}
def logistic(samples: typing.Sequence[Sample]) -> typing.Callable[[str], float]:
"""The chance that a message is an interrupt, from a logistic regression over its words that the samples train."""
def logistic(samples: typing.Sequence[Sample], seed: int = 0) -> typing.Callable[[str], float]:
"""The chance that a message is an interrupt, from a logistic regression over its words that the samples train, shuffled in the order seed gives."""
seen = collections.Counter(word for found in samples for word in words_of(found.text))
vocabulary = {word for word, count in seen.items() if count >= MIN_MESSAGES}
data = [(words_of(found.text) & vocabulary, 1.0 if found.kind == INTERRUPTED else 0.0) for found in samples]
weights: dict[str, float] = collections.defaultdict(float)
bias = 0.0
shuffler = random.Random(0)
shuffler = random.Random(seed)
def chance(words: typing.Iterable[str]) -> float:
return 1 / (1 + math.exp(-max(-30.0, min(30.0, bias + sum(weights[word] for word in words)))))
@@ -85,6 +87,12 @@ def logistic(samples: typing.Sequence[Sample]) -> typing.Callable[[str], float]:
return lambda text: chance(words_of(text) & vocabulary)
def averaged(samples: typing.Sequence[Sample]) -> typing.Callable[[str], float]:
"""The mean chance of SEEDS regressions that differ only in the order they shuffle the samples."""
regressions = [logistic(samples, seed) for seed in range(SEEDS)]
return lambda text: sum(regression(text) for regression in regressions) / SEEDS
def classifier(samples: typing.Sequence[Sample]) -> Decider:
"""Finished where the chance of an interrupt is below one half and below that of every interrupt among the samples; interrupt where it is at least one half and above that of every other kind; otherwise the judge's model decides.
@@ -92,10 +100,10 @@ def classifier(samples: typing.Sequence[Sample]) -> Decider:
interrupts = [found for found in samples if found.kind == INTERRUPTED]
if not interrupts or len(interrupts) == len(samples):
return lambda text: None
held_out = [(tested.kind, chance) for trained, tests in folds_of(samples, INNER_FOLDS) for chance, tested in zip(map(logistic(trained), (found.text for found in tests)), tests)]
held_out = [(tested.kind, chance) for trained, tests in folds_of(samples, INNER_FOLDS) for chance, tested in zip(map(averaged(trained), (found.text for found in tests)), tests)]
below = min(chance for kind, chance in held_out if kind == INTERRUPTED)
above = max((chance for kind, chance in held_out if kind != INTERRUPTED), default=0.0)
chance = logistic(samples)
above = max(chance for kind, chance in held_out if kind != INTERRUPTED)
chance = averaged(samples)
def decide(text: str) -> Kind | None:
found = chance(text)
@@ -132,19 +140,16 @@ def offline_report(model: str, samples: typing.Sequence[Sample], rules: typing.M
if skipped:
lines.append(f"Skipped {skipped} verdicts whose transcript could not be read.")
for name, rule in rules.items():
lines += decided_lines(f"{name}, on the whole archive", evaluate(rule, [], samples), kinds)
found = evaluate(rule, [], samples)
lines += [result_line(f"{name}, on the whole archive", found), *decides_lines(f"{name}, on the whole archive", found)]
split = len(samples) - max(1, len(samples) // LATEST)
for name, learner in learners.items():
held_out = sum((evaluate(learner, trained, tested) for trained, tested in folds_of(samples, folds)), Counts())
lines += decided_lines(f"{name}, on held-out folds", held_out, kinds)
lines.append(result_line(f"{name}, on the latest fifth after learning from the rest", evaluate(learner, samples[:split], samples[split:])))
latest = evaluate(learner, samples[:split], samples[split:])
lines += [result_line(f"{name}, on held-out folds", held_out), result_line(f"{name}, on the latest fifth after learning from the rest", latest), *decides_lines(f"{name}, on both tests", held_out + latest)]
return "\n".join(lines)
def decided_lines(how: str, found: Counts, kinds: collections.Counter[Kind]) -> list[str]:
return [result_line(how, found), *decides_lines(how, found, kinds)]
def result_line(how: str, found: Counts) -> str:
decided = collections.Counter({kind: sum(count for (_, said), count in found.items() if said == kind) for kind in {said for _, said in found if said}})
missed = sum(count for (archived, said), count in found.items() if archived == INTERRUPTED and said not in (None, INTERRUPTED))
@@ -158,13 +163,13 @@ def wrong_kinds(decided: Kind, found: Counts) -> str:
return f" (of these, archived {', '.join(f'{kind} {count}' for kind, count in wrong)})" if wrong else ""
def decides_lines(how: str, found: Counts, kinds: collections.Counter[Kind]) -> list[str]:
"""For finished and for unfinished that the decider decided at all: whether it decided no archived interrupt as that kind, and how many of the kind it got right."""
def decides_lines(how: str, found: Counts) -> list[str]:
"""For finished and for unfinished that the decider decided at all: whether it decided no archived interrupt as that kind."""
lines = []
for kind in (FINISHED, UNFINISHED):
decided = sum(count for (_, said), count in found.items() if said == kind)
if decided and not found[(INTERRUPTED, kind)]:
lines.append(f"{how}, decides {plain(kind)} with no missed interrupt: {found[(kind, kind)]} of {kinds[kind]} {plain(kind)}.")
lines.append(f"{how}, decides {plain(kind)} with no missed interrupt.")
elif decided:
lines.append(f"{how}, does not decide {plain(kind)} with no missed interrupt: it decided {found[(INTERRUPTED, kind)]} archived interrupt(s) as {plain(kind)}.")
return lines
+6
View File
@@ -202,6 +202,12 @@ class Main(unittest.TestCase):
self.assertIn("Replayed 1 archived verdicts of claude-opus-5-5 projects offline", out)
self.assertIn("Skipped 2 verdicts whose transcript could not be read.", out)
def test_offline_keeps_its_report_in_the_same_folder_wherever_the_flag_stands(self):
code, out, err = self.run_main(["--offline", "claude-opus-5-5"], unused)
self.assertEqual(code, 0, err)
folder = self.home / ".claude" / "loop-claude-transcripts" / "replays" / "judge-replay-claude-opus-5-5-offline-20260921T141320Z"
self.assertEqual((folder / "result.md").read_text(encoding="utf-8"), out)
def test_offline_says_the_usage_on_arguments_that_name_no_model(self):
code, out, err = self.run_main(["--offline"], unused)
self.assertEqual((code, out), (1, ""))
+29 -3
View File
@@ -7,7 +7,7 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from transcript_archive.outcomes import ANSWERED, FINISHED, INTERRUPTED, UNFINISHED
from claude_code.entries import AssistantTurn, HumanTurn, Text, ToolCall
from judge_replay.offline import classifier, decision_table, evaluate, final_message, folds_of, LEARNERS, offline_report, question_mark, RULES, Sample, words_of
from judge_replay.offline import averaged, classifier, decision_table, evaluate, final_message, folds_of, LEARNERS, logistic, offline_report, question_mark, RULES, Sample, words_of
def turn(*parts):
@@ -58,6 +58,21 @@ class DecisionTable(unittest.TestCase):
self.assertIsNone(decision_table([])("I stopped at the gate."))
class Logistic(unittest.TestCase):
def test_gives_a_message_with_no_learned_word_about_the_share_of_interrupts(self):
chance = logistic([sample(INTERRUPTED, "")] * 10 + [sample(FINISHED, "")] * 30)
self.assertAlmostEqual(chance(""), 0.25, delta=0.05)
def test_learns_a_word_found_in_as_few_messages_as_the_minimum(self):
chance = logistic([sample(INTERRUPTED, "ask")] * 3 + [sample(FINISHED, "")] * 3)
self.assertGreater(chance("ask"), chance(""))
def test_averaged_is_the_mean_of_the_regressions_that_shuffle_in_each_seed_order(self):
samples = [sample(INTERRUPTED if n % 3 else FINISHED, f"word{n % 4} ask" if n % 3 else f"word{n % 4}") for n in range(30)]
mean = sum(logistic(samples, seed)("ask") for seed in range(5)) / 5
self.assertAlmostEqual(averaged(samples)("ask"), mean)
class Classifier(unittest.TestCase):
def trained(self, extra=()):
asking = [sample(INTERRUPTED, f"I recommend option {n}; tell me which of the options you want") for n in range(12)]
@@ -69,6 +84,9 @@ class Classifier(unittest.TestCase):
self.assertEqual(decide("I recommend option 3; tell me which you want"), INTERRUPTED)
self.assertEqual(decide("merged pull request 99 and removed the worktree, no findings left"), FINISHED)
def test_decides_finished_on_a_message_of_words_only_finished_verdicts_held(self):
self.assertEqual(self.trained()("merged pull request"), FINISHED)
def test_abstains_where_it_learned_no_interrupt_or_no_other_kind(self):
self.assertIsNone(classifier([sample(FINISHED, "merged")])("merged"))
self.assertIsNone(classifier([sample(INTERRUPTED, "which?")])("which?"))
@@ -115,7 +133,7 @@ class Report(unittest.TestCase):
def test_says_a_method_that_decided_a_kind_and_missed_no_interrupt_decides_that_kind(self):
text = offline_report("claude-opus-5-5", [sample(FINISHED, "Merged."), sample(INTERRUPTED, "I recommend merging.")], {"decision table": decision_table}, {})
self.assertIn("Missed interrupts: 0.", text)
self.assertIn("decision table, on the whole archive, decides finished with no missed interrupt: 1 of 1 finished.", text)
self.assertIn("decision table, on the whole archive, decides finished with no missed interrupt.", text)
def test_runs_a_learner_once_for_each_fold_and_once_on_the_latest_fifth_and_a_rule_only_on_the_whole_archive(self):
samples = [sample(INTERRUPTED if n % 2 else FINISHED, "I recommend it" if n % 2 else "merged it") for n in range(20)]
@@ -149,7 +167,15 @@ class Report(unittest.TestCase):
text = offline_report("m", [sample(FINISHED, str(n)) for n in range(7)], {}, {"learner": learner}, folds=2)
self.assertEqual(trained, [3, 4, 6])
self.assertEqual(text.splitlines()[-1], "learner, on the latest fifth after learning from the rest: decided 1 of 1: finished 1. Missed interrupts: 0.")
self.assertEqual(text.splitlines()[-2:], ["learner, on the latest fifth after learning from the rest: decided 1 of 1: finished 1. Missed interrupts: 0.", "learner, on both tests, decides finished with no missed interrupt."])
def test_says_a_learner_that_missed_an_interrupt_on_either_test_does_not_decide_the_kind(self):
def learner(samples):
return lambda said: FINISHED if said == "miss" else None
samples = [sample(FINISHED, "ok")] * 4 + [sample(INTERRUPTED, "miss")]
text = offline_report("m", samples, {}, {"learner": learner}, folds=2)
self.assertEqual(text.splitlines()[-1], "learner, on both tests, does not decide finished with no missed interrupt: it decided 2 archived interrupt(s) as finished.")
def test_counts_the_verdicts_it_skipped_only_where_it_skipped_some(self):
self.assertNotIn("Skipped", offline_report("m", self.samples(), {}, {}))
+2 -2
View File
@@ -658,9 +658,9 @@ On 2026-11-17, `bin/judge-replay claude-opus-5-5 --offline` replayed all 1,104 a
- **A "?" rule** (interrupt where the message holds a "?", finished where it holds none) decided every verdict and missed 197 interrupts.
- **A decision table**, the rows of `DECISION_TABLE` in `bin/judge_replay/offline.py`. Its rows were read from this archive's interrupts, so its run isn't a held-out test. It decided 974 and missed 55. Most of its misses are a worker reporting a merged chunk while a release is ready to cut, which depends on the plan; the rest ask for an answer in words no row names.
- **A classifier**, a logistic regression over the message's words, trained on the archive. It decides finished only where its chance of an interrupt is below that of every interrupt among its training folds when held out, and interrupt only where it is above that of every other kind. On held-out folds it decided 93 and missed 2 interrupts. On the latest fifth, learning from the rest, it decided 25 and missed 1.
- **A classifier**, a logistic regression over the message's words, trained on the archive. It decides finished only where its chance of an interrupt is below that of every interrupt among its training folds when held out, and interrupt only where it is above that of every other kind. It averages five regressions that differ only in the order they shuffle the samples, since a single shuffle moved its missed interrupts between 0 and 2 on the same archive. On held-out folds it decided 103 (92 finished, 11 interrupt) and missed none. On the latest fifth, learning from the rest, it decided 29, all as finished, and missed 1 interrupt.
No method meets the bar of no missed interrupt, so no item is filed for a verdict kind, and the judge's model keeps the verdict kind. It also keeps the escalation it quotes and whether a release is ready to cut, which this replay didn't test.
No method meets the bar of no missed interrupt on every test: the classifier met it on held-out folds and missed one interrupt on the latest fifth, the newest verdicts. So no item is filed for a verdict kind, and the judge's model keeps the verdict kind. A classifier's pass counts only where it holds on both tests, since one test alone moved with the shuffle order and with a single added verdict. It also keeps the escalation it quotes and whether a release is ready to cut, which this replay didn't test.
Valid while the archived verdicts hold interrupts that read like a finished chunk. Replay with `judge-replay <model> --offline` when the archive has doubled, or when a decider reads more than the final message.