Files
claude-config/bin/judge_replay/main.py
T

178 lines
10 KiB
Python

"""Replay loop-claude's judge on archived verdicts, to find the lowest rung that returns them: docs/decisions.md → `routing: on a model judge-replay covered`.
usage: judge-replay <model> [--offline]
<model> is a project model on the routing ladder this setup ships, such as claude-opus-5-5. The tool replays the
verdicts archived for the projects that ~/.claude/loop-claude-models.json runs on it: the latest
15 of each kind, finished, unfinished, interrupt and quoted answer. Where the archive kept the judge's own
prompt (docs/decisions.md → `loop-claude: the archive keeps the raw transcript and each judge run, …`), the
replay hands the judge that prompt, which holds what the judge read then. For a verdict from before the
archive kept it, the replay builds the prompt from the transcript copy beside the verdict, with the task,
premises and records each project holds now.
It first replays every verdict on the reference judge: the model at xhigh effort, the rung the
judge ran on before loop-claude routed it. A verdict the reference judge returns another kind on
may rest, where its prompt was rebuilt, on records or judge rules that have changed since; where the archived prompt was
replayed, the judge's own variance is the cause. Either way no rung is held to it. It
then replays the rest on each rung above the model's cheapest that a chunk's E + I raises a
review to, except the reference judge's rung. Then it replays them on each candidate rung,
highest first: from the model's cheapest rung down to the lowest rung a worker routes to. It stops
at the first candidate that misses one. It prints each verdict the reference judge did not
reproduce, each verdict a rung missed, and the lowest candidate that, like each candidate above
it, missed none.
Record each run in docs/decisions.md → `routing: on a model judge-replay covered`. Where the
model's cheapest rung misses no interrupt, the run is the evidence to add the model to
JUDGE_REPLAYED in bin/agent_cycle/routing.py, which routes that model's judge. judge_rung there
keeps the judge to its own model's rungs; a rung above them that misses no interrupt is the
evidence to let the judge climb onto it.
It keeps each run and the report it prints in loop-claude's archive, in
replays/judge-replay-<model>-<UTC stamp>/ under ~/.claude/loop-claude-transcripts or $LOOP_CLAUDE_TRANSCRIPTS.
A run's .jsonl file starts with a line holding claude's arguments, working directory, prompt, exit code and
stderr, then holds claude's stream-json output; result.md holds the report.
Like the loop, it tries a judge three times running before it counts the reply as failed. A
verdict whose project, transcript or archived judge run cannot be read replays as failed too. Exit 1, saying why, on
bad arguments, an unreadable archive or models file, no archived verdict of the model's projects,
or a usage limit.
With --offline it doesn't call a judge, so the runs, retries and usage limit above don't apply. It reads every
archived verdict of the model's projects, not a sample. It counts how many of them each of three code
deciders decides from the worker's final message, and how many archived interrupts it decides as another
kind, which are its missed interrupts: a "?" rule, a decision table, and a classifier trained on the archive
(judge_replay/offline.py). The rule and the table run on the whole archive; the table's rows were read from
it, so its run isn't a held-out test. The classifier runs on held-out folds, each tested by a model trained on
the other folds, and on the latest fifth of the verdicts, tested by a model trained on the earlier four
fifths. The classifier trains 330 regressions in Python, about 12 minutes on 1,100 verdicts. The report is
kept in replays/judge-replay-<model>-offline-<UTC stamp>/result.md. A verdict whose transcript cannot be read is skipped, and the report counts the skipped. Exit 1, saying why, on bad
arguments, an unreadable archive or models file, no archived verdict of the model's projects, or none with a
transcript that can be read.
"""
import pathlib
import sys
import typing
from chunk_driver.prompts import ANSWER_DESTINATIONS
from transcript_archive.location import archive_root, transcript_of
from transcript_archive.read import archived_outcomes
from rung_replay.archive import archiving, replay_folder
from rung_replay.command import arguments_or_exit, down_from, failed, limited_exit, print_report, projects_on, say
from rung_replay.search import AT_ONCE, first_reply, replayed
from agent_cycle.judge import Brief, judge_archived, judge_once
from judge_replay.replay import Case, case_from, Kind, report, reproduced, sample, verdict_kind
from transcript_archive.project_key import checkout_of
from loop_effects.effects import Effects
from failure import Failure
from judge_replay.offline import final_message, LEARNERS, offline_report, RULES, Sample
from judge_replay.archived_prompt import grounds_of
from transcript_archive.judge_run import JudgeRun
from model_routing.ladder import Ladder, ProjectRung, REVIEW_STEPS, Rung
from claude_code.reply import Limited
from chunk_driver.drivers import LOOP_CLAUDE
from model_routing.ladder_file import read_ladder, SHIPPED
from chunk_driver.project.premises import read_premises
from chunk_driver.project.records import read_records, RECORDS_NAMED
from claude_code.transcript import read_transcript
from agent_cycle.verdicts import Verdict
TOOL = "judge-replay"
USAGE = f"{TOOL} <model> [--offline]; <model> is a model on the routing ladder"
OFFLINE = "--offline"
PER_KIND = 15
# The effort settings.json gave every judge before loop-claude routed it; the archived verdicts came from it.
REFERENCE_EFFORT = "xhigh"
def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, effects: Effects) -> int:
# The shipped ladder, not a project's: the replay spans every project on the model.
ladder = read_ladder(SHIPPED)
if isinstance(ladder, Failure):
return failed(TOOL, ladder.reason)
project = arguments_or_exit(TOOL, argv, __doc__, USAGE, lambda argv: model_argument(ladder, [arg for arg in argv if arg != OFFLINE]))
if isinstance(project, int):
return project
cases = archived_cases(project.rung.model, archive_root(home, env), home)
if isinstance(cases, Failure):
return failed(TOOL, cases.reason)
if not cases:
return failed(TOOL, f"no archived verdicts of projects on {project.rung.model} in {archive_root(home, env)}; run loop-claude on a project on that model first")
if OFFLINE in argv:
return offline(project.rung.model, cases, archive_root(home, env), effects)
cases = sample(cases, PER_KIND)
folder = replay_folder(archive_root(home, env), TOOL, argv, effects.now())
effects = archiving(effects, folder, say)
reference = Rung(project.rung.model, REFERENCE_EFFORT)
# The candidates stop at the lowest rung a worker routes to: no route uses a rung below it for the judge.
replay = replayed(cases, reference, down_from(ladder, project.rung, project.lowest_worker_rung()), lambda found, rung: rule(found, rung, home, effects), reproduced, "verdicts", say, AT_ONCE, climbs(project, reference))
if isinstance(replay, Limited):
return limited_exit(TOOL, replay)
print_report(folder, report(project.rung.model, reference, cases, replay))
return 0
def offline(model: str, cases: list[Case], root: pathlib.Path, effects: Effects) -> int:
"""Replays the verdicts of cases offline, oldest first, and keeps the report in the archive as a judge replay does."""
samples, skipped = [], 0
for found in sorted(cases, key=lambda found: (found.at, found.root, str(found.transcript))):
entries = read_transcript(str(found.transcript))
if isinstance(entries, Failure):
skipped += 1
print(f"{TOOL}: skipped a verdict: {entries.reason}", file=sys.stderr)
else:
samples.append(Sample(found.archived_kind, final_message(entries)))
if not samples:
return failed(TOOL, "no archived verdict has a transcript that can be read")
print_report(replay_folder(root, TOOL, [model, OFFLINE], effects.now()), offline_report(model, samples, RULES, LEARNERS, skipped=skipped))
return 0
def model_argument(ladder: Ladder, argv: list[str]) -> ProjectRung | None:
"""The project's rung on ladder of the one model argv names; None for any other argv."""
return ladder.project_rung(argv[0]) if len(argv) == 1 else None
def climbs(project: ProjectRung, reference: Rung) -> list[Rung]:
"""The rungs above the project's rung that an E + I raises a review to, except the reference's, lowest first."""
return sorted({project.review_floor(score) for score in REVIEW_STEPS} - {project.rung, reference}, key=project.ladder.position)
def archived_cases(model: str, root: pathlib.Path, home: pathlib.Path) -> list[Case] | Failure:
projects = projects_on(model, home)
if isinstance(projects, Failure):
return projects
listed = archived_outcomes(root)
if isinstance(listed, Failure):
return listed
found = []
try:
for key, outcomes in listed:
case = case_from(checkout_of(key, home), outcomes.read_text(encoding="utf-8"), transcript_of(outcomes)) if key in projects else None
found += [case] if case else []
except (OSError, UnicodeDecodeError) as failure:
return Failure(f"could not read the archive {root}: {failure}")
return found
def rule(case: Case, rung: Rung, home: pathlib.Path, effects: Effects) -> Kind | Limited:
entries = read_transcript(str(case.transcript))
if isinstance(entries, Failure):
return verdict_kind(entries)
if case.judge_run:
kept = JudgeRun.read(case.judge_run)
if isinstance(kept, Failure):
return verdict_kind(kept)
grounds = grounds_of(kept.prompt, entries)
if isinstance(grounds, Failure):
return verdict_kind(grounds)
verdict: Verdict | Failure | Limited = first_reply(lambda: judge_archived(case.root, kept.prompt, grounds, len(entries), rung, LOOP_CLAUDE, effects))
return verdict if isinstance(verdict, Limited) else verdict_kind(verdict)
premises, records = read_premises(case.root, home), read_records(case.root, home, home / ".claude")
if isinstance(premises, Failure):
return verdict_kind(premises)
if isinstance(records, Failure):
return verdict_kind(records)
# 0 for the route's mark, as before any step-up: the replay compares verdict kinds, and a mark changes only which escalation quotes count.
verdict = first_reply(lambda: judge_once(case.root, Brief(task=case.task, premises=premises, records=records, records_named=RECORDS_NAMED, answer_destinations=ANSWER_DESTINATIONS, entries=entries, escalated_at=0, driver=LOOP_CLAUDE), rung, effects))
return verdict if isinstance(verdict, Limited) else verdict_kind(verdict)