Replay the judge on archived verdicts to find the cheapest rung that returns them

This commit is contained in:
2026-10-15 07:37:06 +02:00
parent 8412a48842
commit 2b2ec63608
10 changed files with 327 additions and 9 deletions
+3
View File
@@ -24,6 +24,9 @@
!bin/file_writes.py
!bin/git_command.py
!bin/git_repo.py
!bin/judge-replay
!bin/judge_replay/
!bin/judge_replay/*.py
!bin/larv-review
!bin/larv_review/
!bin/larv_review/*.py
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env python3
import os
import pathlib
import sys
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
from loop_claude.effects import system_effects
from judge_replay import main
sys.exit(main.main(sys.argv[1:], os.environ, pathlib.Path.home(), system_effects()))
View File
+117
View File
@@ -0,0 +1,117 @@
"""Replay loop-claude's judge on archived verdicts, to find the cheapest rung that returns them: docs/decisions.md → `routing:`.
usage: judge-replay <model>
<model> is a project model on the routing ladder, such as claude-opus-5-5. The tool replays the
verdicts archived for the projects that ~/.claude/loop-claude-models.json runs on it: the latest
15 of each kind, finished, unfinished, interrupt and quoted answer. Each transcript copy in the
archive is what the judge read for the verdict beside it, so the replay hands the judge that copy,
with the task, premises and records each project holds now.
It first replays every verdict on today's judge: the model at its settings' effort. A verdict
today's judge returns another kind on rests on the records and judge rules of its day, so no rung
is held to it. It then replays the rest on each rung from the one the lowest R + C picks up to the
project's rung, cheapest first, and stops at the first rung that returns all of them. It prints
each verdict a rung missed, and the cheapest rung that missed none.
A verdict whose project or transcript cannot be read replays as failed, and so does a reply the loop
would reject. Exit 1, saying why, on bad arguments, an unreadable archive or models file, or a
usage limit.
"""
import concurrent.futures
import pathlib
import sys
import typing
from transcript_archive.location import archive_root
from judge_replay.replay import Case, case_from, checkout_root, replayed, sample, verdict_kind
from loop_claude.effects import Effects
from failure import Failure
from loop_claude.cycle.judge import judge_args, judge_prompt, run_judge
from model_routing.ladder import LADDER, picked, position, project_rung, Rung
from loop_claude.cycle.retry import Limited
from loop_claude.project.model_choice import read_models_file
from loop_claude.project.premises import read_premises
from loop_claude.project.records import read_records
from claude_code.transcript import read_transcript, render
from loop_claude.stop import Stop
PER_KIND = 15
# claude runs at a time; each judge reads up to JUDGE_PROMPT_CHARS.
AT_ONCE = 4
LOWEST_R_PLUS_C = 2
USAGE = "usage: judge-replay <model>, a model on the routing ladder"
class UsageLimit(Exception):
pass
def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, effects: Effects) -> int:
if argv[:1] in (["-h"], ["--help"]):
print(__doc__.strip())
return 0
anchor = project_rung(argv[0]) if len(argv) == 1 else None
if anchor is None:
print(f"judge-replay: cannot read the arguments{' ' + ' '.join(argv) if argv else ''}; {USAGE}", file=sys.stderr)
return 1
cases = archived_cases(argv[0], archive_root(home, env), home)
if isinstance(cases, Failure):
print(f"judge-replay: {cases.reason}", file=sys.stderr)
return 1
cases = sample(cases, PER_KIND)
candidates = list(LADDER[position(picked(anchor, LOWEST_R_PLUS_C)):position(anchor) + 1])
try:
replay = replayed(cases, Rung(argv[0]), candidates, lambda on, rung: rule_all(on, rung, home / ".claude", effects))
except UsageLimit as limit:
print(f"judge-replay: {limit}", file=sys.stderr)
return 1
kinds = {kind: sum(found.on_record == kind for found in cases) for kind in dict.fromkeys(found.on_record for found in cases)}
print(f"Replayed {len(cases)} verdicts of {argv[0]} projects: {', '.join(f'{count} {kind}' for kind, count in kinds.items())}.")
print(f"Today's judge, {argv[0]}, reproduced {len(cases) - len(replay.unreproduced)}.{' It returned another kind on:' if replay.unreproduced else ''}")
print_missed(replay.unreproduced)
for tried in replay.tried:
print(f"{tried.rung} missed {len(tried.missed) or 'none'}{':' if tried.missed else '.'}")
print_missed(tried.missed)
print(f"Cheapest rung that returns every verdict today's judge reproduces: {replay.cheapest or 'none up to the project rung'}")
return 0
def archived_cases(model: str, root: pathlib.Path, home: pathlib.Path) -> list[Case] | Failure:
models = read_models_file(home / ".claude" / "loop-claude-models.json")
if isinstance(models, Stop):
return Failure(models.reason)
found = []
try:
for outcomes in root.rglob("*.outcomes.md"):
key = str(outcomes.parent.parent.relative_to(root))
if models.get(key) == model and (case := case_from(checkout_root(key, home), outcomes.read_text(encoding="utf-8"), outcomes.with_name(outcomes.name.removesuffix(".outcomes.md") + ".jsonl"))):
found.append(case)
except (OSError, UnicodeDecodeError) as failure:
return Failure(f"could not read the archive {root}: {failure}")
return found
def rule_all(cases: list[Case], rung: Rung, claude_dir: pathlib.Path, effects: Effects) -> list[str]:
print(f"replaying {len(cases)} verdicts on {rung}", file=sys.stderr, flush=True)
with concurrent.futures.ThreadPoolExecutor(AT_ONCE) as pool:
return list(pool.map(lambda found: rule(found, rung, claude_dir, effects), cases))
def rule(case: Case, rung: Rung, claude_dir: pathlib.Path, effects: Effects) -> str:
premises, records, entries = read_premises(case.root), read_records(case.root, claude_dir), read_transcript(str(case.transcript))
for unread in (premises, records):
if isinstance(unread, Stop):
return verdict_kind(Failure(unread.reason))
if isinstance(entries, Failure):
return verdict_kind(entries)
prompt, shown = judge_prompt(case.task, premises, records, render(entries))
verdict = run_judge(case.root, judge_args(rung), prompt, shown, f"{premises}\n\n{records}", effects)
if isinstance(verdict, Limited):
raise UsageLimit(f"{rung} hit a usage limit: {verdict.said}; replay again once it resets")
return verdict_kind(verdict)
def print_missed(missed: list[tuple[Case, str]]) -> None:
for found, kind in missed:
print(f" {found.root} {found.at}: on record {found.on_record}, replayed {kind}")
+100
View File
@@ -0,0 +1,100 @@
"""Which archived verdicts to replay, and the cheapest rung that returns them: docs/decisions.md → `routing:`."""
import collections
import dataclasses
import pathlib
import re
import typing
from failure import Failure
from loop_claude.cycle.judge import Finished, Interrupt, QuotedAnswer, Unfinished, Verdict
from model_routing.ladder import Rung
from utc_stamp import STAMPED
# The first verdict archived beside a transcript copy: the judge ruled on that copy as it stands.
FIRST_VERDICT = re.compile(rf"^({STAMPED}) Verdict: (finished|unfinished|interrupt|quoted answer)$", re.MULTILINE)
TASK = re.compile(rf"\ATask: (.*?)\n\n{STAMPED} ", re.DOTALL)
@dataclasses.dataclass(frozen=True)
class Case:
"""One archived verdict: on_record is its kind, as the archive words it, and at its UTC stamp."""
root: str
task: str
transcript: pathlib.Path
at: str
on_record: str
@dataclasses.dataclass(frozen=True)
class Tried:
"""A rung, and each case it returned another kind of verdict on, with that kind."""
rung: Rung
missed: list[tuple[Case, str]]
@dataclasses.dataclass(frozen=True)
class Replay:
"""unreproduced holds each case today's judge returned another kind on; tried, each rung tried, cheapest first; cheapest, the first rung that missed none, or None."""
unreproduced: list[tuple[Case, str]]
tried: list[Tried]
cheapest: Rung | None
def case_from(root: str, outcomes: str, transcript: pathlib.Path) -> Case | None:
"""The case an outcomes file holds; None when it holds no verdict."""
task, verdict = TASK.match(outcomes), FIRST_VERDICT.search(outcomes)
if not (task and verdict):
return None
return Case(root, task[1], transcript, verdict[1], verdict[2])
def checkout_root(key: str, home: pathlib.Path) -> str:
"""The checkout an archive key names: the inverse of project_key."""
under_code = home / "code" / key
return str(under_code) if under_code.is_dir() else f"/{key}"
def sample(cases: list[Case], per_kind: int) -> list[Case]:
"""The latest per_kind cases of each kind of verdict, in the order they were judged."""
seen, kept = collections.Counter(), []
for found in sorted(cases, key=lambda found: found.at, reverse=True):
seen[found.on_record] += 1
if seen[found.on_record] <= per_kind:
kept.append(found)
return kept[::-1]
def verdict_kind(verdict: Verdict | Failure) -> str:
match verdict:
case Finished():
return "finished"
case Unfinished():
return "unfinished"
case Interrupt():
return "interrupt"
case QuotedAnswer():
return "quoted answer"
case Failure(reason):
return f"failed: {reason}"
case _:
typing.assert_never(verdict)
def replayed(cases: list[Case], today: Rung, candidates: list[Rung], rule_all: typing.Callable[[list[Case], Rung], list[str]]) -> Replay:
"""Replays the cases on today's rung, then only those it reproduces on each candidate, cheapest first, until one returns every verdict.
A verdict today's judge does not reproduce rests on the records and judge rules of its day, so no rung is held to it.
"""
reproduced, unreproduced = [], []
for found, kind in zip(cases, rule_all(cases, today)):
if kind == found.on_record:
reproduced.append(found)
else:
unreproduced.append((found, kind))
tried = []
for rung in candidates:
missed = [(found, kind) for found, kind in zip(reproduced, rule_all(reproduced, rung)) if kind != found.on_record]
tried.append(Tried(rung, missed))
if not missed:
return Replay(unreproduced, tried, rung)
return Replay(unreproduced, tried, None)
+83
View File
@@ -0,0 +1,83 @@
import pathlib
import sys
import tempfile
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from judge_replay.replay import Case, case_from, checkout_root, replayed, sample, verdict_kind
from failure import Failure
from loop_claude.cycle.judge import Finished, Interrupt, QuotedAnswer, Unfinished
from model_routing.ladder import Rung
TRANSCRIPT = pathlib.Path("/archive/p/c/a.jsonl")
CHEAP = Rung("claude-haiku-5-5", "xhigh")
MIDDLE = Rung("claude-sonnet-5-5", "high")
TODAY = Rung("claude-opus-5-5")
def case(at, on_record):
return Case("/code/p", "Do the next chunk.", TRANSCRIPT, at, on_record)
class CaseFrom(unittest.TestCase):
def test_takes_the_task_and_the_first_verdict_on_the_copy(self):
outcomes = "Task: Do the next chunk.\nThen merge it.\n\n2026-10-14T10:00:00Z Stopped: claude failed\n\nWorker: x\n2026-10-14T11:00:00Z Verdict: unfinished\n\nEvidence:\n> on\n\nWorker: x\n2026-10-14T11:00:01Z Stepped up\n\nWorker: y\n"
self.assertEqual(case_from("/code/p", outcomes, TRANSCRIPT), Case("/code/p", "Do the next chunk.\nThen merge it.", TRANSCRIPT, "2026-10-14T11:00:00Z", "unfinished"))
def test_reads_each_kind_of_verdict(self):
for kind in ("finished", "unfinished", "interrupt", "quoted answer"):
found = case_from("/code/p", f"Task: t\n\n2026-10-14T10:00:00Z Verdict: {kind}\n\nmore\n", TRANSCRIPT)
self.assertEqual(found.on_record, kind)
def test_skips_a_copy_with_no_verdict(self):
self.assertIsNone(case_from("/code/p", "Task: t\n\n2026-10-14T10:00:00Z Stepped up\n\nWorker: y\n", TRANSCRIPT))
self.assertIsNone(case_from("/code/p", "not an outcomes file", TRANSCRIPT))
class CheckoutRoot(unittest.TestCase):
def test_finds_a_key_under_code_or_else_under_the_root(self):
with tempfile.TemporaryDirectory() as home:
(pathlib.Path(home) / "code" / "log").mkdir(parents=True)
self.assertEqual(checkout_root("log", pathlib.Path(home)), f"{home}/code/log")
self.assertEqual(checkout_root("home/me/.claude", pathlib.Path(home)), "/home/me/.claude")
class Sample(unittest.TestCase):
def test_keeps_the_latest_of_each_kind_in_the_order_they_were_judged(self):
cases = [case("2026-10-14T10:00:01Z", "finished"), case("2026-10-14T10:00:03Z", "finished"), case("2026-10-14T10:00:02Z", "finished"), case("2026-10-14T10:00:00Z", "interrupt")]
self.assertEqual(sample(cases, 2), [cases[3], cases[2], cases[1]])
class VerdictKind(unittest.TestCase):
def test_names_each_verdict_as_the_archive_does(self):
self.assertEqual(verdict_kind(Finished("e")), "finished")
self.assertEqual(verdict_kind(Unfinished("e", 3)), "unfinished")
self.assertEqual(verdict_kind(Interrupt("e", "q?")), "interrupt")
self.assertEqual(verdict_kind(QuotedAnswer("e", "q?", "a")), "quoted answer")
self.assertEqual(verdict_kind(Failure("the judge quoted x")), "failed: the judge quoted x")
class Replayed(unittest.TestCase):
def test_tries_rungs_cheapest_first_on_the_verdicts_todays_judge_reproduces_and_stops_at_the_first_that_returns_all(self):
cases = [case("2026-10-14T10:00:00Z", "finished"), case("2026-10-14T10:00:01Z", "interrupt"), case("2026-10-14T10:00:02Z", "finished")]
replies = {TODAY: ["finished", "interrupt", "unfinished"], CHEAP: ["finished", "finished"], MIDDLE: ["finished", "interrupt"]}
asked = []
def rule_all(on, rung):
asked.append((len(on), rung))
return replies[rung]
replay = replayed(cases, TODAY, [CHEAP, MIDDLE, Rung("claude-opus-5-5", "medium")], rule_all)
self.assertEqual(replay.unreproduced, [(cases[2], "unfinished")])
self.assertEqual([(tried.rung, tried.missed) for tried in replay.tried], [(CHEAP, [(cases[1], "finished")]), (MIDDLE, [])])
self.assertEqual(replay.cheapest, MIDDLE)
self.assertEqual(asked, [(3, TODAY), (2, CHEAP), (2, MIDDLE)])
def test_names_no_rung_when_every_candidate_misses_a_verdict(self):
replay = replayed([case("2026-10-14T10:00:00Z", "finished")], TODAY, [CHEAP], lambda on, rung: ["finished"] if rung == TODAY else ["interrupt"])
self.assertIsNone(replay.cheapest)
if __name__ == "__main__":
unittest.main()
+9 -5
View File
@@ -9,6 +9,7 @@ from failure import Failure
from claude_code.entries import hit_limit, limit_said, parse_entry, structured_output
from loop_claude.project.records import HUMAN_RULE_NAMES
from loop_claude.cycle.retry import Limited, retried
from model_routing.ladder import Rung
from loop_claude.stop import Stop
from todo_table import TOP_SCORE
@@ -119,20 +120,23 @@ def verdict_from(output: object) -> Verdict | Failure:
return Unfinished(output["evidence"], output["bundled"])
def judge(root: str, task: str, premises: str, records: str, rendered_transcript: str, model: str, effects: Effects) -> Verdict | Stop:
def judge(root: str, task: str, premises: str, records: str, rendered_transcript: str, rung: Rung, effects: Effects) -> Verdict | Stop:
prompt, shown = judge_prompt(task, premises, records, rendered_transcript)
return retried("judge", lambda: run_judge(root, judge_args(rung), prompt, shown, f"{premises}\n\n{records}", effects), effects)
def judge_args(rung: Rung) -> list[str]:
# --json-schema has claude check its reply against the schema and return it as the json result's structured_output.
# No tools, so the verdict rests on the transcript grounded() checks it against; no persistence, so its runs stay out of the session picker.
args = [
return [
"-p",
"--json-schema", json.dumps(JUDGE_SCHEMA),
"--model", model,
*rung.flags(),
"--no-session-persistence",
"--output-format", "json",
"--permission-mode", "bypassPermissions",
"--tools", "",
]
prompt, shown = judge_prompt(task, premises, records, rendered_transcript)
return retried("judge", lambda: run_judge(root, args, prompt, shown, f"{premises}\n\n{records}", effects), effects)
def run_judge(root: str, args: list[str], prompt: str, shown: str, quotable: str, effects: Effects) -> Verdict | Failure | Limited:
+2 -2
View File
@@ -23,7 +23,7 @@ from loop_claude.stop import Stop
@dataclasses.dataclass(frozen=True)
class Cycle:
effects: Effects
judge_model: str
judge_rung: Rung
paths: Paths
premises: str
project_archive: pathlib.Path
@@ -103,7 +103,7 @@ def rule(cycle: Cycle, transcript_path: str) -> Ruled | Stop:
entries = read_transcript(transcript_path)
if isinstance(entries, Failure):
return Stop(1, entries.reason)
verdict = judge(cycle.root, cycle.task, cycle.premises, cycle.records, render(entries), cycle.judge_model, cycle.effects)
verdict = judge(cycle.root, cycle.task, cycle.premises, cycle.records, render(entries), cycle.judge_rung, cycle.effects)
return verdict if isinstance(verdict, Stop) else Ruled(verdict)
+1 -1
View File
@@ -177,7 +177,7 @@ def run_loop(argv: list[str], env: typing.Mapping[str, str], paths: Paths, launc
loop_run = routed(loop_run, anchor, lambda: archived_route(archive_folder, session), lambda: plan_item(root))
route = loop_run.route or Route(Rung(models.worker))
print(f"\n=== cycle {loop_run.cycle_number}, worker on {route.rung} ===", flush=True)
verdict = run_cycle(Cycle(anchor=anchor, effects=effects, judge_model=models.judge, paths=paths, premises=premises, project_archive=archive_folder, records=records, root=loop_run.root, route=route, session=session, task=loop_run.task, unfinished_evidence=loop_run.unfinished_evidence))
verdict = run_cycle(Cycle(anchor=anchor, effects=effects, judge_rung=Rung(models.judge), paths=paths, premises=premises, project_archive=archive_folder, records=records, root=loop_run.root, route=route, session=session, task=loop_run.task, unfinished_evidence=loop_run.unfinished_evidence))
if isinstance(verdict, Stop):
return verdict
before, loop_run = loop_run.route, after_verdict(loop_run, anchor, verdict)
+1 -1
View File
@@ -175,7 +175,7 @@ class Cycles(unittest.TestCase):
self.archive = project_archive(str(self.home / "code" / "proj"), self.home, {})
def cycle(self, fake, records="", unfinished_evidence=""):
return run.Cycle(anchor=None, effects=fake.effects(), judge_model="judge-model", paths=self.paths, premises="## Goals", project_archive=self.archive, records=records, root=str(self.home), route=Route(Rung("worker-model")), session=self.session, task="Do it.", unfinished_evidence=unfinished_evidence)
return run.Cycle(anchor=None, effects=fake.effects(), judge_rung=Rung("judge-model"), paths=self.paths, premises="## Goals", project_archive=self.archive, records=records, root=str(self.home), route=Route(Rung("worker-model")), session=self.session, task="Do it.", unfinished_evidence=unfinished_evidence)
def archived_outcomes(self):
return sorted(path.read_text() for path in self.archive.glob("*/*.outcomes.md"))