Replay the judge on archived verdicts to find the cheapest rung that returns them
This commit is contained in:
@@ -24,6 +24,9 @@
|
||||
!bin/file_writes.py
|
||||
!bin/git_command.py
|
||||
!bin/git_repo.py
|
||||
!bin/judge-replay
|
||||
!bin/judge_replay/
|
||||
!bin/judge_replay/*.py
|
||||
!bin/larv-review
|
||||
!bin/larv_review/
|
||||
!bin/larv_review/*.py
|
||||
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
||||
|
||||
from loop_claude.effects import system_effects
|
||||
from judge_replay import main
|
||||
|
||||
sys.exit(main.main(sys.argv[1:], os.environ, pathlib.Path.home(), system_effects()))
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Replay loop-claude's judge on archived verdicts, to find the cheapest rung that returns them: docs/decisions.md → `routing:`.
|
||||
|
||||
usage: judge-replay <model>
|
||||
|
||||
<model> is a project model on the routing ladder, such as claude-opus-5-5. The tool replays the
|
||||
verdicts archived for the projects that ~/.claude/loop-claude-models.json runs on it: the latest
|
||||
15 of each kind, finished, unfinished, interrupt and quoted answer. Each transcript copy in the
|
||||
archive is what the judge read for the verdict beside it, so the replay hands the judge that copy,
|
||||
with the task, premises and records each project holds now.
|
||||
|
||||
It first replays every verdict on today's judge: the model at its settings' effort. A verdict
|
||||
today's judge returns another kind on rests on the records and judge rules of its day, so no rung
|
||||
is held to it. It then replays the rest on each rung from the one the lowest R + C picks up to the
|
||||
project's rung, cheapest first, and stops at the first rung that returns all of them. It prints
|
||||
each verdict a rung missed, and the cheapest rung that missed none.
|
||||
|
||||
A verdict whose project or transcript cannot be read replays as failed, and so does a reply the loop
|
||||
would reject. Exit 1, saying why, on bad arguments, an unreadable archive or models file, or a
|
||||
usage limit.
|
||||
"""
|
||||
import concurrent.futures
|
||||
import pathlib
|
||||
import sys
|
||||
import typing
|
||||
|
||||
from transcript_archive.location import archive_root
|
||||
from judge_replay.replay import Case, case_from, checkout_root, replayed, sample, verdict_kind
|
||||
from loop_claude.effects import Effects
|
||||
from failure import Failure
|
||||
from loop_claude.cycle.judge import judge_args, judge_prompt, run_judge
|
||||
from model_routing.ladder import LADDER, picked, position, project_rung, Rung
|
||||
from loop_claude.cycle.retry import Limited
|
||||
from loop_claude.project.model_choice import read_models_file
|
||||
from loop_claude.project.premises import read_premises
|
||||
from loop_claude.project.records import read_records
|
||||
from claude_code.transcript import read_transcript, render
|
||||
from loop_claude.stop import Stop
|
||||
|
||||
PER_KIND = 15
|
||||
# claude runs at a time; each judge reads up to JUDGE_PROMPT_CHARS.
|
||||
AT_ONCE = 4
|
||||
LOWEST_R_PLUS_C = 2
|
||||
USAGE = "usage: judge-replay <model>, a model on the routing ladder"
|
||||
|
||||
|
||||
class UsageLimit(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, effects: Effects) -> int:
|
||||
if argv[:1] in (["-h"], ["--help"]):
|
||||
print(__doc__.strip())
|
||||
return 0
|
||||
anchor = project_rung(argv[0]) if len(argv) == 1 else None
|
||||
if anchor is None:
|
||||
print(f"judge-replay: cannot read the arguments{' ' + ' '.join(argv) if argv else ''}; {USAGE}", file=sys.stderr)
|
||||
return 1
|
||||
cases = archived_cases(argv[0], archive_root(home, env), home)
|
||||
if isinstance(cases, Failure):
|
||||
print(f"judge-replay: {cases.reason}", file=sys.stderr)
|
||||
return 1
|
||||
cases = sample(cases, PER_KIND)
|
||||
candidates = list(LADDER[position(picked(anchor, LOWEST_R_PLUS_C)):position(anchor) + 1])
|
||||
try:
|
||||
replay = replayed(cases, Rung(argv[0]), candidates, lambda on, rung: rule_all(on, rung, home / ".claude", effects))
|
||||
except UsageLimit as limit:
|
||||
print(f"judge-replay: {limit}", file=sys.stderr)
|
||||
return 1
|
||||
kinds = {kind: sum(found.on_record == kind for found in cases) for kind in dict.fromkeys(found.on_record for found in cases)}
|
||||
print(f"Replayed {len(cases)} verdicts of {argv[0]} projects: {', '.join(f'{count} {kind}' for kind, count in kinds.items())}.")
|
||||
print(f"Today's judge, {argv[0]}, reproduced {len(cases) - len(replay.unreproduced)}.{' It returned another kind on:' if replay.unreproduced else ''}")
|
||||
print_missed(replay.unreproduced)
|
||||
for tried in replay.tried:
|
||||
print(f"{tried.rung} missed {len(tried.missed) or 'none'}{':' if tried.missed else '.'}")
|
||||
print_missed(tried.missed)
|
||||
print(f"Cheapest rung that returns every verdict today's judge reproduces: {replay.cheapest or 'none up to the project rung'}")
|
||||
return 0
|
||||
|
||||
|
||||
def archived_cases(model: str, root: pathlib.Path, home: pathlib.Path) -> list[Case] | Failure:
|
||||
models = read_models_file(home / ".claude" / "loop-claude-models.json")
|
||||
if isinstance(models, Stop):
|
||||
return Failure(models.reason)
|
||||
found = []
|
||||
try:
|
||||
for outcomes in root.rglob("*.outcomes.md"):
|
||||
key = str(outcomes.parent.parent.relative_to(root))
|
||||
if models.get(key) == model and (case := case_from(checkout_root(key, home), outcomes.read_text(encoding="utf-8"), outcomes.with_name(outcomes.name.removesuffix(".outcomes.md") + ".jsonl"))):
|
||||
found.append(case)
|
||||
except (OSError, UnicodeDecodeError) as failure:
|
||||
return Failure(f"could not read the archive {root}: {failure}")
|
||||
return found
|
||||
|
||||
|
||||
def rule_all(cases: list[Case], rung: Rung, claude_dir: pathlib.Path, effects: Effects) -> list[str]:
|
||||
print(f"replaying {len(cases)} verdicts on {rung}", file=sys.stderr, flush=True)
|
||||
with concurrent.futures.ThreadPoolExecutor(AT_ONCE) as pool:
|
||||
return list(pool.map(lambda found: rule(found, rung, claude_dir, effects), cases))
|
||||
|
||||
|
||||
def rule(case: Case, rung: Rung, claude_dir: pathlib.Path, effects: Effects) -> str:
|
||||
premises, records, entries = read_premises(case.root), read_records(case.root, claude_dir), read_transcript(str(case.transcript))
|
||||
for unread in (premises, records):
|
||||
if isinstance(unread, Stop):
|
||||
return verdict_kind(Failure(unread.reason))
|
||||
if isinstance(entries, Failure):
|
||||
return verdict_kind(entries)
|
||||
prompt, shown = judge_prompt(case.task, premises, records, render(entries))
|
||||
verdict = run_judge(case.root, judge_args(rung), prompt, shown, f"{premises}\n\n{records}", effects)
|
||||
if isinstance(verdict, Limited):
|
||||
raise UsageLimit(f"{rung} hit a usage limit: {verdict.said}; replay again once it resets")
|
||||
return verdict_kind(verdict)
|
||||
|
||||
|
||||
def print_missed(missed: list[tuple[Case, str]]) -> None:
|
||||
for found, kind in missed:
|
||||
print(f" {found.root} {found.at}: on record {found.on_record}, replayed {kind}")
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Which archived verdicts to replay, and the cheapest rung that returns them: docs/decisions.md → `routing:`."""
|
||||
import collections
|
||||
import dataclasses
|
||||
import pathlib
|
||||
import re
|
||||
import typing
|
||||
|
||||
from failure import Failure
|
||||
from loop_claude.cycle.judge import Finished, Interrupt, QuotedAnswer, Unfinished, Verdict
|
||||
from model_routing.ladder import Rung
|
||||
from utc_stamp import STAMPED
|
||||
|
||||
# The first verdict archived beside a transcript copy: the judge ruled on that copy as it stands.
|
||||
FIRST_VERDICT = re.compile(rf"^({STAMPED}) Verdict: (finished|unfinished|interrupt|quoted answer)$", re.MULTILINE)
|
||||
TASK = re.compile(rf"\ATask: (.*?)\n\n{STAMPED} ", re.DOTALL)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Case:
|
||||
"""One archived verdict: on_record is its kind, as the archive words it, and at its UTC stamp."""
|
||||
root: str
|
||||
task: str
|
||||
transcript: pathlib.Path
|
||||
at: str
|
||||
on_record: str
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Tried:
|
||||
"""A rung, and each case it returned another kind of verdict on, with that kind."""
|
||||
rung: Rung
|
||||
missed: list[tuple[Case, str]]
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Replay:
|
||||
"""unreproduced holds each case today's judge returned another kind on; tried, each rung tried, cheapest first; cheapest, the first rung that missed none, or None."""
|
||||
unreproduced: list[tuple[Case, str]]
|
||||
tried: list[Tried]
|
||||
cheapest: Rung | None
|
||||
|
||||
|
||||
def case_from(root: str, outcomes: str, transcript: pathlib.Path) -> Case | None:
|
||||
"""The case an outcomes file holds; None when it holds no verdict."""
|
||||
task, verdict = TASK.match(outcomes), FIRST_VERDICT.search(outcomes)
|
||||
if not (task and verdict):
|
||||
return None
|
||||
return Case(root, task[1], transcript, verdict[1], verdict[2])
|
||||
|
||||
|
||||
def checkout_root(key: str, home: pathlib.Path) -> str:
|
||||
"""The checkout an archive key names: the inverse of project_key."""
|
||||
under_code = home / "code" / key
|
||||
return str(under_code) if under_code.is_dir() else f"/{key}"
|
||||
|
||||
|
||||
def sample(cases: list[Case], per_kind: int) -> list[Case]:
|
||||
"""The latest per_kind cases of each kind of verdict, in the order they were judged."""
|
||||
seen, kept = collections.Counter(), []
|
||||
for found in sorted(cases, key=lambda found: found.at, reverse=True):
|
||||
seen[found.on_record] += 1
|
||||
if seen[found.on_record] <= per_kind:
|
||||
kept.append(found)
|
||||
return kept[::-1]
|
||||
|
||||
|
||||
def verdict_kind(verdict: Verdict | Failure) -> str:
|
||||
match verdict:
|
||||
case Finished():
|
||||
return "finished"
|
||||
case Unfinished():
|
||||
return "unfinished"
|
||||
case Interrupt():
|
||||
return "interrupt"
|
||||
case QuotedAnswer():
|
||||
return "quoted answer"
|
||||
case Failure(reason):
|
||||
return f"failed: {reason}"
|
||||
case _:
|
||||
typing.assert_never(verdict)
|
||||
|
||||
|
||||
def replayed(cases: list[Case], today: Rung, candidates: list[Rung], rule_all: typing.Callable[[list[Case], Rung], list[str]]) -> Replay:
|
||||
"""Replays the cases on today's rung, then only those it reproduces on each candidate, cheapest first, until one returns every verdict.
|
||||
|
||||
A verdict today's judge does not reproduce rests on the records and judge rules of its day, so no rung is held to it.
|
||||
"""
|
||||
reproduced, unreproduced = [], []
|
||||
for found, kind in zip(cases, rule_all(cases, today)):
|
||||
if kind == found.on_record:
|
||||
reproduced.append(found)
|
||||
else:
|
||||
unreproduced.append((found, kind))
|
||||
tried = []
|
||||
for rung in candidates:
|
||||
missed = [(found, kind) for found, kind in zip(reproduced, rule_all(reproduced, rung)) if kind != found.on_record]
|
||||
tried.append(Tried(rung, missed))
|
||||
if not missed:
|
||||
return Replay(unreproduced, tried, rung)
|
||||
return Replay(unreproduced, tried, None)
|
||||
@@ -0,0 +1,83 @@
|
||||
import pathlib
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from judge_replay.replay import Case, case_from, checkout_root, replayed, sample, verdict_kind
|
||||
from failure import Failure
|
||||
from loop_claude.cycle.judge import Finished, Interrupt, QuotedAnswer, Unfinished
|
||||
from model_routing.ladder import Rung
|
||||
|
||||
TRANSCRIPT = pathlib.Path("/archive/p/c/a.jsonl")
|
||||
CHEAP = Rung("claude-haiku-5-5", "xhigh")
|
||||
MIDDLE = Rung("claude-sonnet-5-5", "high")
|
||||
TODAY = Rung("claude-opus-5-5")
|
||||
|
||||
|
||||
def case(at, on_record):
|
||||
return Case("/code/p", "Do the next chunk.", TRANSCRIPT, at, on_record)
|
||||
|
||||
|
||||
class CaseFrom(unittest.TestCase):
|
||||
def test_takes_the_task_and_the_first_verdict_on_the_copy(self):
|
||||
outcomes = "Task: Do the next chunk.\nThen merge it.\n\n2026-10-14T10:00:00Z Stopped: claude failed\n\nWorker: x\n2026-10-14T11:00:00Z Verdict: unfinished\n\nEvidence:\n> on\n\nWorker: x\n2026-10-14T11:00:01Z Stepped up\n\nWorker: y\n"
|
||||
self.assertEqual(case_from("/code/p", outcomes, TRANSCRIPT), Case("/code/p", "Do the next chunk.\nThen merge it.", TRANSCRIPT, "2026-10-14T11:00:00Z", "unfinished"))
|
||||
|
||||
def test_reads_each_kind_of_verdict(self):
|
||||
for kind in ("finished", "unfinished", "interrupt", "quoted answer"):
|
||||
found = case_from("/code/p", f"Task: t\n\n2026-10-14T10:00:00Z Verdict: {kind}\n\nmore\n", TRANSCRIPT)
|
||||
self.assertEqual(found.on_record, kind)
|
||||
|
||||
def test_skips_a_copy_with_no_verdict(self):
|
||||
self.assertIsNone(case_from("/code/p", "Task: t\n\n2026-10-14T10:00:00Z Stepped up\n\nWorker: y\n", TRANSCRIPT))
|
||||
self.assertIsNone(case_from("/code/p", "not an outcomes file", TRANSCRIPT))
|
||||
|
||||
|
||||
class CheckoutRoot(unittest.TestCase):
|
||||
def test_finds_a_key_under_code_or_else_under_the_root(self):
|
||||
with tempfile.TemporaryDirectory() as home:
|
||||
(pathlib.Path(home) / "code" / "log").mkdir(parents=True)
|
||||
self.assertEqual(checkout_root("log", pathlib.Path(home)), f"{home}/code/log")
|
||||
self.assertEqual(checkout_root("home/me/.claude", pathlib.Path(home)), "/home/me/.claude")
|
||||
|
||||
|
||||
class Sample(unittest.TestCase):
|
||||
def test_keeps_the_latest_of_each_kind_in_the_order_they_were_judged(self):
|
||||
cases = [case("2026-10-14T10:00:01Z", "finished"), case("2026-10-14T10:00:03Z", "finished"), case("2026-10-14T10:00:02Z", "finished"), case("2026-10-14T10:00:00Z", "interrupt")]
|
||||
self.assertEqual(sample(cases, 2), [cases[3], cases[2], cases[1]])
|
||||
|
||||
|
||||
class VerdictKind(unittest.TestCase):
|
||||
def test_names_each_verdict_as_the_archive_does(self):
|
||||
self.assertEqual(verdict_kind(Finished("e")), "finished")
|
||||
self.assertEqual(verdict_kind(Unfinished("e", 3)), "unfinished")
|
||||
self.assertEqual(verdict_kind(Interrupt("e", "q?")), "interrupt")
|
||||
self.assertEqual(verdict_kind(QuotedAnswer("e", "q?", "a")), "quoted answer")
|
||||
self.assertEqual(verdict_kind(Failure("the judge quoted x")), "failed: the judge quoted x")
|
||||
|
||||
|
||||
class Replayed(unittest.TestCase):
|
||||
def test_tries_rungs_cheapest_first_on_the_verdicts_todays_judge_reproduces_and_stops_at_the_first_that_returns_all(self):
|
||||
cases = [case("2026-10-14T10:00:00Z", "finished"), case("2026-10-14T10:00:01Z", "interrupt"), case("2026-10-14T10:00:02Z", "finished")]
|
||||
replies = {TODAY: ["finished", "interrupt", "unfinished"], CHEAP: ["finished", "finished"], MIDDLE: ["finished", "interrupt"]}
|
||||
asked = []
|
||||
|
||||
def rule_all(on, rung):
|
||||
asked.append((len(on), rung))
|
||||
return replies[rung]
|
||||
|
||||
replay = replayed(cases, TODAY, [CHEAP, MIDDLE, Rung("claude-opus-5-5", "medium")], rule_all)
|
||||
self.assertEqual(replay.unreproduced, [(cases[2], "unfinished")])
|
||||
self.assertEqual([(tried.rung, tried.missed) for tried in replay.tried], [(CHEAP, [(cases[1], "finished")]), (MIDDLE, [])])
|
||||
self.assertEqual(replay.cheapest, MIDDLE)
|
||||
self.assertEqual(asked, [(3, TODAY), (2, CHEAP), (2, MIDDLE)])
|
||||
|
||||
def test_names_no_rung_when_every_candidate_misses_a_verdict(self):
|
||||
replay = replayed([case("2026-10-14T10:00:00Z", "finished")], TODAY, [CHEAP], lambda on, rung: ["finished"] if rung == TODAY else ["interrupt"])
|
||||
self.assertIsNone(replay.cheapest)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -9,6 +9,7 @@ from failure import Failure
|
||||
from claude_code.entries import hit_limit, limit_said, parse_entry, structured_output
|
||||
from loop_claude.project.records import HUMAN_RULE_NAMES
|
||||
from loop_claude.cycle.retry import Limited, retried
|
||||
from model_routing.ladder import Rung
|
||||
from loop_claude.stop import Stop
|
||||
from todo_table import TOP_SCORE
|
||||
|
||||
@@ -119,20 +120,23 @@ def verdict_from(output: object) -> Verdict | Failure:
|
||||
return Unfinished(output["evidence"], output["bundled"])
|
||||
|
||||
|
||||
def judge(root: str, task: str, premises: str, records: str, rendered_transcript: str, model: str, effects: Effects) -> Verdict | Stop:
|
||||
def judge(root: str, task: str, premises: str, records: str, rendered_transcript: str, rung: Rung, effects: Effects) -> Verdict | Stop:
|
||||
prompt, shown = judge_prompt(task, premises, records, rendered_transcript)
|
||||
return retried("judge", lambda: run_judge(root, judge_args(rung), prompt, shown, f"{premises}\n\n{records}", effects), effects)
|
||||
|
||||
|
||||
def judge_args(rung: Rung) -> list[str]:
|
||||
# --json-schema has claude check its reply against the schema and return it as the json result's structured_output.
|
||||
# No tools, so the verdict rests on the transcript grounded() checks it against; no persistence, so its runs stay out of the session picker.
|
||||
args = [
|
||||
return [
|
||||
"-p",
|
||||
"--json-schema", json.dumps(JUDGE_SCHEMA),
|
||||
"--model", model,
|
||||
*rung.flags(),
|
||||
"--no-session-persistence",
|
||||
"--output-format", "json",
|
||||
"--permission-mode", "bypassPermissions",
|
||||
"--tools", "",
|
||||
]
|
||||
prompt, shown = judge_prompt(task, premises, records, rendered_transcript)
|
||||
return retried("judge", lambda: run_judge(root, args, prompt, shown, f"{premises}\n\n{records}", effects), effects)
|
||||
|
||||
|
||||
def run_judge(root: str, args: list[str], prompt: str, shown: str, quotable: str, effects: Effects) -> Verdict | Failure | Limited:
|
||||
|
||||
@@ -23,7 +23,7 @@ from loop_claude.stop import Stop
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Cycle:
|
||||
effects: Effects
|
||||
judge_model: str
|
||||
judge_rung: Rung
|
||||
paths: Paths
|
||||
premises: str
|
||||
project_archive: pathlib.Path
|
||||
@@ -103,7 +103,7 @@ def rule(cycle: Cycle, transcript_path: str) -> Ruled | Stop:
|
||||
entries = read_transcript(transcript_path)
|
||||
if isinstance(entries, Failure):
|
||||
return Stop(1, entries.reason)
|
||||
verdict = judge(cycle.root, cycle.task, cycle.premises, cycle.records, render(entries), cycle.judge_model, cycle.effects)
|
||||
verdict = judge(cycle.root, cycle.task, cycle.premises, cycle.records, render(entries), cycle.judge_rung, cycle.effects)
|
||||
return verdict if isinstance(verdict, Stop) else Ruled(verdict)
|
||||
|
||||
|
||||
|
||||
@@ -177,7 +177,7 @@ def run_loop(argv: list[str], env: typing.Mapping[str, str], paths: Paths, launc
|
||||
loop_run = routed(loop_run, anchor, lambda: archived_route(archive_folder, session), lambda: plan_item(root))
|
||||
route = loop_run.route or Route(Rung(models.worker))
|
||||
print(f"\n=== cycle {loop_run.cycle_number}, worker on {route.rung} ===", flush=True)
|
||||
verdict = run_cycle(Cycle(anchor=anchor, effects=effects, judge_model=models.judge, paths=paths, premises=premises, project_archive=archive_folder, records=records, root=loop_run.root, route=route, session=session, task=loop_run.task, unfinished_evidence=loop_run.unfinished_evidence))
|
||||
verdict = run_cycle(Cycle(anchor=anchor, effects=effects, judge_rung=Rung(models.judge), paths=paths, premises=premises, project_archive=archive_folder, records=records, root=loop_run.root, route=route, session=session, task=loop_run.task, unfinished_evidence=loop_run.unfinished_evidence))
|
||||
if isinstance(verdict, Stop):
|
||||
return verdict
|
||||
before, loop_run = loop_run.route, after_verdict(loop_run, anchor, verdict)
|
||||
|
||||
@@ -175,7 +175,7 @@ class Cycles(unittest.TestCase):
|
||||
self.archive = project_archive(str(self.home / "code" / "proj"), self.home, {})
|
||||
|
||||
def cycle(self, fake, records="", unfinished_evidence=""):
|
||||
return run.Cycle(anchor=None, effects=fake.effects(), judge_model="judge-model", paths=self.paths, premises="## Goals", project_archive=self.archive, records=records, root=str(self.home), route=Route(Rung("worker-model")), session=self.session, task="Do it.", unfinished_evidence=unfinished_evidence)
|
||||
return run.Cycle(anchor=None, effects=fake.effects(), judge_rung=Rung("judge-model"), paths=self.paths, premises="## Goals", project_archive=self.archive, records=records, root=str(self.home), route=Route(Rung("worker-model")), session=self.session, task="Do it.", unfinished_evidence=unfinished_evidence)
|
||||
|
||||
def archived_outcomes(self):
|
||||
return sorted(path.read_text() for path in self.archive.glob("*/*.outcomes.md"))
|
||||
|
||||
Reference in New Issue
Block a user