178 lines
10 KiB
Python
178 lines
10 KiB
Python
"""Replay loop-claude's judge on archived verdicts, to find the lowest rung that returns them: docs/decisions.md → `routing: on a model judge-replay covered`.
|
|
|
|
usage: judge-replay <model> [--offline]
|
|
|
|
<model> is a project model on the routing ladder this setup ships, such as claude-opus-5-5. The tool replays the
|
|
verdicts archived for the projects that ~/.claude/loop-claude-models.json runs on it: the latest
|
|
15 of each kind, finished, unfinished, interrupt and quoted answer. Where the archive kept the judge's own
|
|
prompt (docs/decisions.md → `loop-claude: the archive keeps the raw transcript and each judge run, …`), the
|
|
replay hands the judge that prompt, which holds what the judge read then. For a verdict from before the
|
|
archive kept it, the replay builds the prompt from the transcript copy beside the verdict, with the task,
|
|
premises and records each project holds now.
|
|
|
|
It first replays every verdict on the reference judge: the model at xhigh effort, the rung the
|
|
judge ran on before loop-claude routed it. A verdict the reference judge returns another kind on
|
|
may rest, where its prompt was rebuilt, on records or judge rules that have changed since; where the archived prompt was
|
|
replayed, the judge's own variance is the cause. Either way no rung is held to it. It
|
|
then replays the rest on each rung above the model's cheapest that a chunk's E + I raises a
|
|
review to, except the reference judge's rung. Then it replays them on each candidate rung,
|
|
highest first: from the model's cheapest rung down to the lowest rung a worker routes to. It stops
|
|
at the first candidate that misses one. It prints each verdict the reference judge did not
|
|
reproduce, each verdict a rung missed, and the lowest candidate that, like each candidate above
|
|
it, missed none.
|
|
|
|
Record each run in docs/decisions.md → `routing: on a model judge-replay covered`. Where the
|
|
model's cheapest rung misses no interrupt, the run is the evidence to add the model to
|
|
JUDGE_REPLAYED in bin/agent_cycle/routing.py, which routes that model's judge. judge_rung there
|
|
keeps the judge to its own model's rungs; a rung above them that misses no interrupt is the
|
|
evidence to let the judge climb onto it.
|
|
|
|
It keeps each run and the report it prints in loop-claude's archive, in
|
|
replays/judge-replay-<model>-<UTC stamp>/ under ~/.claude/loop-claude-transcripts or $LOOP_CLAUDE_TRANSCRIPTS.
|
|
A run's .jsonl file starts with a line holding claude's arguments, working directory, prompt, exit code and
|
|
stderr, then holds claude's stream-json output; result.md holds the report.
|
|
|
|
Like the loop, it tries a judge three times running before it counts the reply as failed. A
|
|
verdict whose project, transcript or archived judge run cannot be read replays as failed too. Exit 1, saying why, on
|
|
bad arguments, an unreadable archive or models file, no archived verdict of the model's projects,
|
|
or a usage limit.
|
|
|
|
With --offline it doesn't call a judge, so the runs, retries and usage limit above don't apply. It reads every
|
|
archived verdict of the model's projects, not a sample. It counts how many of them each of three code
|
|
deciders decides from the worker's final message, and how many archived interrupts it decides as another
|
|
kind, which are its missed interrupts: a "?" rule, a decision table, and a classifier trained on the archive
|
|
(judge_replay/offline.py). The rule and the table run on the whole archive; the table's rows were read from
|
|
it, so its run isn't a held-out test. The classifier runs on held-out folds, each tested by a model trained on
|
|
the other folds, and on the latest fifth of the verdicts, tested by a model trained on the earlier four
|
|
fifths. The classifier trains 330 regressions in Python, about 12 minutes on 1,100 verdicts. The report is
|
|
kept in replays/judge-replay-<model>-offline-<UTC stamp>/result.md. A verdict whose transcript cannot be read is skipped, and the report counts the skipped. Exit 1, saying why, on bad
|
|
arguments, an unreadable archive or models file, no archived verdict of the model's projects, or none with a
|
|
transcript that can be read.
|
|
"""
|
|
import pathlib
|
|
import sys
|
|
import typing
|
|
|
|
from chunk_driver.prompts import ANSWER_DESTINATIONS
|
|
from transcript_archive.location import archive_root, transcript_of
|
|
from transcript_archive.read import archived_outcomes
|
|
from rung_replay.archive import archiving, replay_folder
|
|
from rung_replay.command import arguments_or_exit, down_from, failed, limited_exit, print_report, projects_on, say
|
|
from rung_replay.search import AT_ONCE, first_reply, replayed
|
|
from agent_cycle.judge import Brief, judge_archived, judge_once
|
|
from judge_replay.replay import Case, case_from, Kind, report, reproduced, sample, verdict_kind
|
|
from transcript_archive.project_key import checkout_of
|
|
from loop_effects.effects import Effects
|
|
from failure import Failure
|
|
from judge_replay.offline import final_message, LEARNERS, offline_report, RULES, Sample
|
|
from judge_replay.archived_prompt import grounds_of
|
|
from transcript_archive.judge_run import JudgeRun
|
|
from model_routing.ladder import Ladder, ProjectRung, REVIEW_STEPS, Rung
|
|
from claude_code.reply import Limited
|
|
from chunk_driver.drivers import LOOP_CLAUDE
|
|
from model_routing.ladder_file import read_ladder, SHIPPED
|
|
from chunk_driver.project.premises import read_premises
|
|
from chunk_driver.project.records import read_records, RECORDS_NAMED
|
|
from claude_code.transcript import read_transcript
|
|
from agent_cycle.verdicts import Verdict
|
|
|
|
TOOL = "judge-replay"
|
|
USAGE = f"{TOOL} <model> [--offline]; <model> is a model on the routing ladder"
|
|
OFFLINE = "--offline"
|
|
PER_KIND = 15
|
|
# The effort settings.json gave every judge before loop-claude routed it; the archived verdicts came from it.
|
|
REFERENCE_EFFORT = "xhigh"
|
|
|
|
|
|
def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, effects: Effects) -> int:
|
|
# The shipped ladder, not a project's: the replay spans every project on the model.
|
|
ladder = read_ladder(SHIPPED)
|
|
if isinstance(ladder, Failure):
|
|
return failed(TOOL, ladder.reason)
|
|
project = arguments_or_exit(TOOL, argv, __doc__, USAGE, lambda argv: model_argument(ladder, [arg for arg in argv if arg != OFFLINE]))
|
|
if isinstance(project, int):
|
|
return project
|
|
cases = archived_cases(project.rung.model, archive_root(home, env), home)
|
|
if isinstance(cases, Failure):
|
|
return failed(TOOL, cases.reason)
|
|
if not cases:
|
|
return failed(TOOL, f"no archived verdicts of projects on {project.rung.model} in {archive_root(home, env)}; run loop-claude on a project on that model first")
|
|
if OFFLINE in argv:
|
|
return offline(project.rung.model, cases, archive_root(home, env), effects)
|
|
cases = sample(cases, PER_KIND)
|
|
folder = replay_folder(archive_root(home, env), TOOL, argv, effects.now())
|
|
effects = archiving(effects, folder, say)
|
|
reference = Rung(project.rung.model, REFERENCE_EFFORT)
|
|
# The candidates stop at the lowest rung a worker routes to: no route uses a rung below it for the judge.
|
|
replay = replayed(cases, reference, down_from(ladder, project.rung, project.lowest_worker_rung()), lambda found, rung: rule(found, rung, home, effects), reproduced, "verdicts", say, AT_ONCE, climbs(project, reference))
|
|
if isinstance(replay, Limited):
|
|
return limited_exit(TOOL, replay)
|
|
print_report(folder, report(project.rung.model, reference, cases, replay))
|
|
return 0
|
|
|
|
|
|
def offline(model: str, cases: list[Case], root: pathlib.Path, effects: Effects) -> int:
|
|
"""Replays the verdicts of cases offline, oldest first, and keeps the report in the archive as a judge replay does."""
|
|
samples, skipped = [], 0
|
|
for found in sorted(cases, key=lambda found: (found.at, found.root, str(found.transcript))):
|
|
entries = read_transcript(str(found.transcript))
|
|
if isinstance(entries, Failure):
|
|
skipped += 1
|
|
print(f"{TOOL}: skipped a verdict: {entries.reason}", file=sys.stderr)
|
|
else:
|
|
samples.append(Sample(found.archived_kind, final_message(entries)))
|
|
if not samples:
|
|
return failed(TOOL, "no archived verdict has a transcript that can be read")
|
|
print_report(replay_folder(root, TOOL, [model, OFFLINE], effects.now()), offline_report(model, samples, RULES, LEARNERS, skipped=skipped))
|
|
return 0
|
|
|
|
|
|
def model_argument(ladder: Ladder, argv: list[str]) -> ProjectRung | None:
|
|
"""The project's rung on ladder of the one model argv names; None for any other argv."""
|
|
return ladder.project_rung(argv[0]) if len(argv) == 1 else None
|
|
|
|
|
|
def climbs(project: ProjectRung, reference: Rung) -> list[Rung]:
|
|
"""The rungs above the project's rung that an E + I raises a review to, except the reference's, lowest first."""
|
|
return sorted({project.review_floor(score) for score in REVIEW_STEPS} - {project.rung, reference}, key=project.ladder.position)
|
|
|
|
|
|
def archived_cases(model: str, root: pathlib.Path, home: pathlib.Path) -> list[Case] | Failure:
|
|
projects = projects_on(model, home)
|
|
if isinstance(projects, Failure):
|
|
return projects
|
|
listed = archived_outcomes(root)
|
|
if isinstance(listed, Failure):
|
|
return listed
|
|
found = []
|
|
try:
|
|
for key, outcomes in listed:
|
|
case = case_from(checkout_of(key, home), outcomes.read_text(encoding="utf-8"), transcript_of(outcomes)) if key in projects else None
|
|
found += [case] if case else []
|
|
except (OSError, UnicodeDecodeError) as failure:
|
|
return Failure(f"could not read the archive {root}: {failure}")
|
|
return found
|
|
|
|
|
|
def rule(case: Case, rung: Rung, home: pathlib.Path, effects: Effects) -> Kind | Limited:
|
|
entries = read_transcript(str(case.transcript))
|
|
if isinstance(entries, Failure):
|
|
return verdict_kind(entries)
|
|
if case.judge_run:
|
|
kept = JudgeRun.read(case.judge_run)
|
|
if isinstance(kept, Failure):
|
|
return verdict_kind(kept)
|
|
grounds = grounds_of(kept.prompt, entries)
|
|
if isinstance(grounds, Failure):
|
|
return verdict_kind(grounds)
|
|
verdict: Verdict | Failure | Limited = first_reply(lambda: judge_archived(case.root, kept.prompt, grounds, len(entries), rung, LOOP_CLAUDE, effects))
|
|
return verdict if isinstance(verdict, Limited) else verdict_kind(verdict)
|
|
premises, records = read_premises(case.root, home), read_records(case.root, home, home / ".claude")
|
|
if isinstance(premises, Failure):
|
|
return verdict_kind(premises)
|
|
if isinstance(records, Failure):
|
|
return verdict_kind(records)
|
|
# 0 for the route's mark, as before any step-up: the replay compares verdict kinds, and a mark changes only which escalation quotes count.
|
|
verdict = first_reply(lambda: judge_once(case.root, Brief(task=case.task, premises=premises, records=records, records_named=RECORDS_NAMED, answer_destinations=ANSWER_DESTINATIONS, entries=entries, escalated_at=0, driver=LOOP_CLAUDE), rung, effects))
|
|
return verdict if isinstance(verdict, Limited) else verdict_kind(verdict)
|