Archive each judge run's prompt and json reply beside the verdict it gave, and replay from the archived prompt
CI / tests (pull_request) Failing after 2m45s

This commit is contained in:
2026-11-16 23:42:36 +01:00
parent 5bb49dc623
commit e1d2eae7f4
14 changed files with 248 additions and 46 deletions
+17 -7
View File
@@ -5,8 +5,10 @@ usage: judge-replay <model>
<model> is a project model on the routing ladder this setup ships, such as claude-opus-5-5. The tool replays the
verdicts archived for the projects that ~/.claude/loop-claude-models.json runs on it: the latest
15 of each kind, finished, unfinished, interrupt and quoted answer. Each transcript copy in the
archive is what the judge read for the verdict beside it, so the replay hands the judge that copy,
with the task, premises and records each project holds now.
archive is what the judge read for the verdict beside it. Where the archive kept the judge's own prompt
(docs/decisions.md → `loop-claude: the archive keeps the raw transcript`), the replay hands the judge that
prompt; for a verdict from before the archive kept it, the replay hands the judge the copy, with the task,
premises and records each project holds now.
It first replays every verdict on the reference judge: the model at xhigh effort, the rung the
judge ran on before loop-claude routed it. A verdict the reference judge returns another kind on
@@ -32,10 +34,11 @@ or a usage limit.
import pathlib
import typing
from transcript_archive.judge_run import JudgeRun
from transcript_archive.location import archive_root, transcript_of
from transcript_archive.read import archived_outcomes
from rung_replay.command import arguments_or_exit, down_from, failed, limited_exit, projects_on
from loop_claude.cycle.judge import Brief, judge_once
from loop_claude.cycle.judge import Brief, judge_archived, judge_once
from judge_replay.replay import Case, case_from, Kind, report, reproduced, sample, verdict_kind
from transcript_archive.project_key import checkout_of
from loop_effects.effects import Effects
@@ -107,13 +110,20 @@ def archived_cases(model: str, root: pathlib.Path, home: pathlib.Path) -> list[C
def rule(case: Case, rung: Rung, home: pathlib.Path, effects: Effects) -> Kind | Limited:
premises, records, entries = read_premises(case.root, home), read_records(case.root, home, home / ".claude"), read_transcript(str(case.transcript))
entries = read_transcript(str(case.transcript))
if isinstance(entries, Failure):
return verdict_kind(entries)
if case.judge_run:
kept = JudgeRun.read(case.judge_run)
if isinstance(kept, Failure):
return verdict_kind(kept)
verdict: Verdict | Failure | Limited = first_reply(lambda: judge_archived(case.root, kept.prompt, entries, rung, effects))
return verdict if isinstance(verdict, Limited) else verdict_kind(verdict)
premises, records = read_premises(case.root, home), read_records(case.root, home, home / ".claude")
if isinstance(premises, Failure):
return verdict_kind(premises)
if isinstance(records, Failure):
return verdict_kind(records)
if isinstance(entries, Failure):
return verdict_kind(entries)
# 0 for the route's mark, as before any step-up: the replay compares verdict kinds, and a mark changes only which escalation quotes count.
verdict: Verdict | Failure | Limited = first_reply(lambda: judge_once(case.root, Brief(case.task, premises, records, entries, 0), rung, effects))
verdict = first_reply(lambda: judge_once(case.root, Brief(case.task, premises, records, entries, 0), rung, effects))
return verdict if isinstance(verdict, Limited) else verdict_kind(verdict)
+5 -2
View File
@@ -4,7 +4,7 @@ import dataclasses
import pathlib
import typing
from transcript_archive.outcomes import ANSWERED, FINISHED, first_verdict, INTERRUPTED, task_of, UNFINISHED
from transcript_archive.outcomes import ANSWERED, FINISHED, first_verdict, INTERRUPTED, judge_run_at, task_of, UNFINISHED
from failure import Failure
from loop_claude.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished, Verdict
from rung_replay.search import Replay
@@ -22,6 +22,8 @@ class Case:
transcript: pathlib.Path
at: str
archived_kind: Kind
# The judge run the archive kept beside the verdict, which holds the prompt the judge read; None where it kept none.
judge_run: pathlib.Path | None = None
def case_from(root: str, outcomes: str, transcript: pathlib.Path) -> Case | None:
@@ -32,7 +34,8 @@ def case_from(root: str, outcomes: str, transcript: pathlib.Path) -> Case | None
task, verdict = task_of(outcomes), first_verdict(outcomes)
if task is None or verdict is None:
return None
return Case(root, task, transcript, *verdict)
run = judge_run_at(outcomes, verdict[0])
return Case(root, task, transcript, *verdict, transcript.parent / run if run else None)
def sample(cases: list[Case], per_kind: int) -> list[Case]:
+27
View File
@@ -41,6 +41,11 @@ class CaseFrom(unittest.TestCase):
found = case_from("/code/p", f"Task: t\n\n2026-10-14T10:00:00Z {verdict}\n\nmore\n", TRANSCRIPT)
self.assertEqual(found.archived_kind, verdict)
def test_names_the_judge_run_that_the_first_verdict_records_beside_it(self):
outcomes = f"Task: t\n\n2026-10-14T10:00:00Z Verdict: unfinished\n\nJudge: j\nJudge run: judge/{'a' * 64}.json\nWorker: x\n\n2026-10-14T11:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/{'b' * 64}.json\nWorker: x\n"
self.assertEqual(case_from("/code/p", outcomes, TRANSCRIPT).judge_run, TRANSCRIPT.parent / f"judge/{'a' * 64}.json")
self.assertIsNone(case_from("/code/p", "Task: t\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nWorker: x\n", TRANSCRIPT).judge_run)
def test_skips_a_copy_with_no_verdict(self):
self.assertIsNone(case_from("/code/p", "Task: t\n\n2026-10-14T10:00:00Z Stepped up\n\nWorker: y\n", TRANSCRIPT))
self.assertIsNone(case_from("/code/p", "not an outcomes file", TRANSCRIPT))
@@ -161,6 +166,28 @@ class Main(unittest.TestCase):
# The replay shows the judge the whole transcript, with no step-up marked in it.
self.assertNotIn("[step]", self.prompts[0])
def test_hands_the_judge_the_prompt_the_archive_kept_and_rebuilds_the_prompt_of_a_verdict_with_none(self):
kept = self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1"
prompt = "From the goals file, OLD premises\n\nThe records: none\n\nThe task the worker was given:\nDo it.\n\nWhat the worker did and said, what the human typed on [human] lines and what loop-claude told it on [loop] lines, latest last:\nsaid Verdict: finished\n\nOLD rules"
(kept / "judge").mkdir()
(kept / "judge" / "run.json").write_text(json.dumps({"prompt": prompt, "reply": {"type": "result"}}))
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/run.json\nWorker: x\n")
code, out, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
self.assertEqual(code, 0, err)
self.assertIn("The reference judge, claude-opus-5-5 at xhigh effort, reproduced 2.", out)
# The first replay, on the finished verdict, reads the archived prompt; the second, on the interrupt, is rebuilt from the records of today.
self.assertEqual(self.prompts[0], prompt)
self.assertIn("The task the worker was given:\nDo it.", self.prompts[1])
self.assertIn("1. Ship.", self.prompts[1])
def test_replays_a_verdict_whose_archived_judge_run_cannot_be_read_as_failed(self):
kept = self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1"
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/gone.json\nWorker: x\n")
code, out, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
self.assertEqual(code, 0, err)
self.assertIn("The reference judge, claude-opus-5-5 at xhigh effort, reproduced 1. It returned another kind on:", out)
self.assertIn("could not read the archived judge run", out)
def test_tries_no_rung_below_the_lowest_a_worker_routes_to_where_every_rung_returns_every_verdict(self):
code, out, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
self.assertEqual(code, 0, err)
+23 -7
View File
@@ -9,6 +9,7 @@ from loop_effects.effects import Effects
from claude_code.entries import Entry, structured_output
from loop_claude import escalation_mark
from failure import Failure
from transcript_archive.judge_run import JudgeRun
from loop_claude.verdicts import Finished, Interrupt, Judged, QuotedAnswer, Unfinished, Verdict
from claude_code.transcript import human_answers
from loop_claude.quotes import plain, verbatim
@@ -64,6 +65,8 @@ FINISHED_RULES = (
"continue: an open PR, a review not yet run, CI not yet reported, an answer not yet recorded."
)
JUDGE_RULES = "\n\n".join((INTERRUPT_RULES, QUOTE_RULES, FINISHED_RULES))
# Opens the transcript in the prompt; the premises, the records and the task come before it, the rules after the transcript.
TRANSCRIPT_HEADING = "What the worker did and said, what the human typed on [human] lines and what loop-claude told it on [loop] lines, latest last:\n"
@dataclasses.dataclass(frozen=True)
@@ -94,10 +97,15 @@ JUDGE_SCHEMA = {
@dataclasses.dataclass(frozen=True)
class Rejected(Failure):
"""A judge reply loop-claude rejected: reason says why, reply holds the reply as json, and cost is what the run cost in dollars, 0 where its reply records none."""
reply: str
"""A judge reply loop-claude rejected: reason says why, run holds the prompt and the reply, and cost is what the run cost in dollars, 0 where its reply records none."""
run: JudgeRun
cost: float = 0.0
@property
def reply(self) -> str:
"""The reply as json."""
return json.dumps(self.run.reply, ensure_ascii=False)
class Grounds(typing.NamedTuple):
"""The texts quotes_checked checks the judge's quotes against. A quote must appear in its text as written, since a quote reworded or stitched together can say what its source does not; only the evidence may join passages with an ellipsis, each passage in order."""
@@ -155,6 +163,14 @@ def judge_once(root: str, brief: Brief, rung: Rung, effects: Effects) -> Verdict
return verdict if isinstance(verdict, Failure | Limited) else escalation_mark.stamped(verdict, len(brief.entries))
def judge_archived(root: str, prompt: str, entries: list[Entry], rung: Rung, effects: Effects) -> Verdict | Failure | Limited:
"""One judge run on a prompt the archive kept from an earlier run, as judge-replay replays it: the judge reads what it read then, and its quotes are checked against that prompt's own text, in place of the records of today. The prompt's rules count as part of the transcript, a slack that lets an evidence quote the rules pass."""
head, _, shown = prompt.partition(TRANSCRIPT_HEADING)
grounds = Grounds(shown, escalation_mark.after_mark(entries, 0), head, tuple(human_answers(entries, typed_answer)))
verdict = run_judge(root, judge_args(rung), prompt, grounds, effects)
return verdict if isinstance(verdict, Failure | Limited) else escalation_mark.stamped(verdict, len(entries))
def judge_args(rung: Rung) -> list[str]:
# --json-schema has claude check its reply against the schema and return it as the json result's structured_output.
# No tools, so the verdict rests on the transcript grounded() checks it against.
@@ -172,14 +188,14 @@ def run_judge(root: str, args: list[str], prompt: str, grounds: Grounds, effects
return reply
output = structured_output(reply)
verdict = verdict_from(output)
replied = json.dumps(reply, ensure_ascii=False)
run = JudgeRun(prompt, reply)
cost = cost_of(reply)
if isinstance(verdict, Failure):
return Rejected(verdict.reason, replied, cost)
return Rejected(verdict.reason, run, cost)
checked = quotes_checked(verdict, grounds)
if isinstance(checked, Failure):
return Rejected(checked.reason, replied, cost)
return dataclasses.replace(checked, judged=Judged(cost, tokens_of(reply.get("usage")), reversal_of(output, grounds.typed)))
return Rejected(checked.reason, run, cost)
return dataclasses.replace(checked, judged=Judged(cost, tokens_of(reply.get("usage")), reversal_of(output, grounds.typed), run))
def reversal_of(output: object, typed: tuple[str, ...]) -> str:
@@ -256,7 +272,7 @@ def judge_prompt(brief: Brief) -> tuple[str, Grounds]:
f"From the goals file, {brief.premises}\n\n"
f"The records: the project's working rules and decisions, then the human's rules for every project:\n{brief.records or 'none'}\n\n"
f"The task the worker was given:\n{brief.task}\n\n"
"What the worker did and said, what the human typed on [human] lines and what loop-claude told it on [loop] lines, latest last:\n"
f"{TRANSCRIPT_HEADING}"
)
room = max(0, JUDGE_PROMPT_CHARS - len(head) - len(JUDGE_RULES) - 2)
transcript = escalation_mark.transcript(brief.entries, brief.escalated_at, typed_answer)
+12 -11
View File
@@ -3,6 +3,7 @@ import pathlib
import typing
from loop_claude.cycle.states import Active, Archived, Closing, Ended, Event, HumanTalked, HumanTyped, Interrupting, Judging, QUOTED_ANSWERS_PER_CYCLE, Ruled, State, to_the_human, transition, TypedAnswer, WorkerRan, Working
from transcript_archive.judge_run import JudgeRun
from transcript_archive.write import archive
from loop_claude.cycle.worker import AUTO_COMPACT_WINDOW, COMPACT_WINDOW, run_worker, worker_args
from loop_claude.cycle.judge import Brief, judge, Rejected
@@ -98,9 +99,9 @@ def archived_stop(project_archive: pathlib.Path, session: Session, task: str, ro
return Stop(1, f"{failure.reason}\n{stop.reason}".rstrip()) if failure and failure.reason != stop.reason else stop
def archived(project_archive: pathlib.Path, transcript_path: str, task: str, outcome: str, effects: Effects) -> Failure | None:
def archived(project_archive: pathlib.Path, transcript_path: str, task: str, outcome: str, effects: Effects, judge_run: JudgeRun | None = None) -> Failure | None:
"""transcript_archive.write.archive, stamped now and saying its notes through effects. A cycle and close.py, which has no Cycle, both archive through it."""
return archive(project_archive, transcript_path, task, outcome, effects.now(), lambda text: say(text, effects))
return archive(project_archive, transcript_path, task, outcome, effects.now(), lambda text: say(text, effects), judge_run)
def judge_of(cycle: Cycle) -> Rung:
@@ -167,7 +168,7 @@ def rule(cycle: Cycle, transcript_path: str, quoted_answers: int) -> Ruled | Sto
if isinstance(annotation, Failure):
return Stop(1, annotation.reason)
stepped = ruling_step_up(cycle, verdict)
if failure := archive_stepped(cycle, transcript_path, ruling_outcome(cycle, verdict, quoted_answers, annotation), stepped):
if failure := archive_stepped(cycle, transcript_path, ruling_outcome(cycle, verdict, quoted_answers, annotation), stepped, verdict.judged.run):
return Stop(1, failure.reason)
return Ruled(verdict, stepped.after if stepped else None)
@@ -187,13 +188,13 @@ def annotation_of(cycle: Cycle, transcript_path: str, entries: list[Entry], judg
def ruling_outcome(cycle: Cycle, ruling: Interrupt | QuotedAnswer, quoted_answers: int, annotation: outcomes.Annotation) -> str:
"""The archive's outcome for a ruling: interrupted where it goes to the human, with the question put to them, and else answered from the premises or the records."""
if isinstance(ruling, QuotedAnswer) and not to_the_human(ruling, quoted_answers):
return outcomes.answered(question=ruling.question, sources=prompts.QUOTE_SOURCES, answer=ruling.answer, evidence=ruling.evidence, escalation=ruling.escalation, judge=str(judge_of(cycle)), worker=cycle.route.worker_line(), judge_cost=ruling.judged.cost, annotation=annotation)
return outcomes.interrupted(question_for(ruling), ruling.evidence, str(judge_of(cycle)), ruling.escalation, cycle.route.worker_line(), ruling.judged.cost, annotation)
return outcomes.answered(question=ruling.question, sources=prompts.QUOTE_SOURCES, answer=ruling.answer, evidence=ruling.evidence, escalation=ruling.escalation, judge=str(judge_of(cycle)), worker=cycle.route.worker_line(), judge_cost=ruling.judged.cost, annotation=annotation, judge_run=ruling.judged.run)
return outcomes.interrupted(question_for(ruling), ruling.evidence, str(judge_of(cycle)), ruling.escalation, cycle.route.worker_line(), ruling.judged.cost, annotation, ruling.judged.run)
def keep_rejection(cycle: Cycle, transcript_path: str, rejected: Rejected) -> Stop | None:
"""Archives a judge reply loop-claude rejected, so the archive shows what the judge returned before a retry or an exit 1."""
if unkept := archived(cycle.project_archive, transcript_path, cycle.task, outcomes.rejected(rejected.reason, rejected.reply, str(judge_of(cycle)), rejected.cost), cycle.effects):
if unkept := archived(cycle.project_archive, transcript_path, cycle.task, outcomes.rejected(rejected.reason, rejected.reply, str(judge_of(cycle)), rejected.cost, rejected.run), cycle.effects, rejected.run):
return Stop(1, unkept.reason)
return None
@@ -211,9 +212,9 @@ def ask(cycle: Cycle, transcript_path: str, ruling: Interrupt | QuotedAnswer) ->
return HumanTyped(TypedAnswer(ruling.question, answer))
def archive_stepped(cycle: Cycle, transcript_path: str, outcome: str, step_up: StepUp | None) -> Failure | None:
"""Archives the outcome of a ruling, then the step-up it made, if it made one. The first failure is returned."""
if failure := archived(cycle.project_archive, transcript_path, cycle.task, outcome, cycle.effects):
def archive_stepped(cycle: Cycle, transcript_path: str, outcome: str, step_up: StepUp | None, judge_run: JudgeRun | None) -> Failure | None:
"""Archives the outcome of a ruling and the judge run that gave it, then the step-up it made, if it made one. The first failure is returned."""
if failure := archived(cycle.project_archive, transcript_path, cycle.task, outcome, cycle.effects, judge_run):
return failure
return archived_step(cycle.project_archive, transcript_path, cycle.task, step_up, cycle.effects)
@@ -236,8 +237,8 @@ def close(cycle: Cycle, transcript_path: str, verdict: Finished | Unfinished) ->
annotation = entries if isinstance(entries, Failure) else annotation_of(cycle, transcript_path, entries, verdict.judged)
if isinstance(annotation, Failure):
return Stop(1, annotation.reason)
outcome = outcomes.finished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle)), verdict.judged.cost, annotation) if isinstance(verdict, Finished) else outcomes.unfinished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle)), verdict.escalation, verdict.judged.cost, annotation)
if failure := archived(cycle.project_archive, transcript_path, cycle.task, outcome, cycle.effects):
outcome = outcomes.finished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle)), verdict.judged.cost, annotation, verdict.judged.run) if isinstance(verdict, Finished) else outcomes.unfinished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle)), verdict.escalation, verdict.judged.cost, annotation, verdict.judged.run)
if failure := archived(cycle.project_archive, transcript_path, cycle.task, outcome, cycle.effects, verdict.judged.run):
return Stop(1, failure.reason)
if isinstance(verdict, Finished) and (failure := delete_session(cycle.session)):
return Stop(1, failure.reason)
+3 -1
View File
@@ -342,7 +342,9 @@ class Checkout:
for archived in (copy, kept):
self.test.assertEqual(stat.S_IMODE(archived.stat().st_mode), 0o600)
stamped = re.sub(r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ ", "<UTC> ", kept.read_text(), flags=re.MULTILINE)
outcomes[copy.read_text()] = re.sub(r"^Config: .*$", "Config: <config>", stamped, flags=re.MULTILINE)
configured = re.sub(r"^Config: .*$", "Config: <config>", stamped, flags=re.MULTILINE)
# The judge's run is named by the hash of its prompt, which test_loop_claude.py checks.
outcomes[copy.read_text()] = re.sub(r"^Judge run: judge/[0-9a-f]{64}\.json\n", "", configured, flags=re.MULTILINE)
return outcomes
+31 -1
View File
@@ -174,6 +174,9 @@ def ran(stdout="", code=0, stderr=""):
return subprocess.CompletedProcess([], code, stdout, stderr)
JUDGE_RUN_LINE = re.compile(r"^Judge run: judge/[0-9a-f]{64}\.json\n", re.MULTILINE)
def judged(verdict, evidence="working", answer="", escalation="", release_ready=False, cost=None, reversed=""):
reply = {"answer": answer, "escalation": escalation, "evidence": evidence, "finished": verdict == "finished", "interrupt": verdict == "interrupt", "question": "Which one?" if verdict in ("interrupt", "quoted") else "", "answered_before": "", "left_open": "", "release_ready": release_ready, "reversed": reversed}
return ran(json.dumps({"structured_output": reply, **({} if cost is None else {"total_cost_usd": cost})}))
@@ -289,7 +292,8 @@ class Cycles(unittest.TestCase):
return run.Cycle(config=CONFIG, window=WINDOW, effects=fake.effects(), facts=facts, judge_model="judge-model", ladder=LADDER, premises="## Goals", project=None, project_archive=self.archive, records=records, root=str(self.home), route=Route(Rung("worker-model")), session=self.session, task="Do it.", plan=plan_of(""), unfinished_evidence=unfinished_evidence)
def archived_outcomes(self):
return sorted(path.read_text() for path in self.archive.glob("*/*.outcomes.md"))
"""Each outcomes file's text without its `Judge run:` lines, which name a file by the hash of a prompt: the tests of the judge's runs check them."""
return sorted(JUDGE_RUN_LINE.sub("", path.read_text()) for path in self.archive.glob("*/*.outcomes.md"))
def test_a_finished_chunk_is_archived_and_its_session_deleted(self):
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
@@ -383,6 +387,32 @@ Worker: {OFF_LADDER}
f"{stamp} Judge reply rejected: the judge quoted 'never said', which is not in the transcript\n\nReply:\n> {misquoted.stdout}\n\nJudge: judge-model\n\n"
f"{stamp} Verdict: finished\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
def test_archives_each_judge_runs_prompt_and_reply_beside_the_rejection_interrupt_and_verdict_it_gave(self):
rejected, interrupted, finished = ran(json.dumps({"structured_output": {"interrupt": False}})), judged("interrupt", cost=0.25), judged("finished")
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[rejected, interrupted, finished], inputs=["Use B."])
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
prompts = [stdin for args, stdin in fake.made("claude")[:3]]
[chunk] = self.archive.iterdir()
kept = "\n".join(path.read_text() for path in sorted(chunk.glob("*.outcomes.md"), key=lambda path: path.stat().st_mtime_ns))
named = re.findall(r"^Judge run: (judge/[0-9a-f]{64}\.json)$", kept, re.MULTILINE)
self.assertEqual(len(named), 3, kept)
runs = [json.loads((chunk / name).read_text()) for name in named]
self.assertEqual([found["prompt"] for found in runs], prompts)
self.assertEqual([found["reply"] for found in runs], [json.loads(reply.stdout) for reply in (rejected, interrupted, finished)])
self.assertEqual(sorted(path.name for path in (chunk / "judge").iterdir()), sorted(name.removeprefix("judge/") for name in named))
def test_stops_when_a_judge_run_cannot_be_archived(self):
self.archive.mkdir(parents=True)
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
def replace_file(path, data):
return Failure("disk full") if "/judge/" in str(path) else real_replace_file(path, data)
real_replace_file = write.replace_file
with unittest.mock.patch("transcript_archive.write.replace_file", replace_file):
stop = run.run_cycle(self.cycle(fake))
self.assertEqual(stop.code, 1)
self.assertRegex(stop.reason, r"^could not archive the transcript: .*/judge/[0-9a-f]{64}\.json: disk full$")
def test_archives_what_each_judge_run_cost_on_an_interrupt_a_quoted_answer_and_a_rejected_misquote(self):
for first, then, inputs in (
(judged("interrupt", cost=0.25), judged("finished"), ["Use B."]),
+3
View File
@@ -2,6 +2,7 @@
import dataclasses
from claude_code.tokens import Tokens
from transcript_archive.judge_run import JudgeRun
@dataclasses.dataclass(frozen=True)
@@ -22,6 +23,8 @@ class Judged:
tokens: Tokens = Tokens()
# The human's whole typed answer that holds the judge's quote of a reversal of a choice the worker took on its own recommendation; empty for none, and where no typed answer holds the quote.
reversed: str = ""
# The prompt it read and its reply, which the archive keeps beside the verdict. Left out of comparisons and reprs, so that equal verdicts stay equal whatever prompt they came from.
run: JudgeRun | None = dataclasses.field(default=None, compare=False, repr=False)
@dataclasses.dataclass(frozen=True)
+4
View File
@@ -3,6 +3,7 @@
<archive root>/<project key>/<chunk>/<sha256>.jsonl
<archive root>/<project key>/<chunk>/<sha256>.outcomes.md
<archive root>/<project key>/<chunk>/session/...
<archive root>/<project key>/<chunk>/judge/<sha256>.json
- The archive root is $LOOP_CLAUDE_TRANSCRIPTS, or ~/.claude/loop-claude-transcripts (location.archive_root).
- The project key is the checkout's path relative to ~/code, or to / outside it (project_key.project_key).
@@ -19,6 +20,9 @@
grew from it. A file Claude Code rewrote whole lands beside its copy as <name>.<hash of its content>
(write.copy_for).
- judge/ holds each judge run an outcome names on its `Judge run:` line: the prompt the judge read and claude's
json reply, named by the hash of that content (judge_run.JudgeRun). judge-replay reads the prompt back.
write.archive writes the layout, at every interrupt and as every cycle ends. read.py and the tools that
measure finished chunks read it.
"""
+32
View File
@@ -0,0 +1,32 @@
import dataclasses
import hashlib
import json
import pathlib
from failure import Failure
# The folder of a chunk's judge runs, beside its transcript copies.
FOLDER = "judge"
@dataclasses.dataclass(frozen=True)
class JudgeRun:
"""The prompt a judge run read and claude's json reply to it. The judge has no tools and one turn, so these two are its whole transcript."""
prompt: str
reply: dict
def content(self) -> bytes:
return json.dumps({"prompt": self.prompt, "reply": self.reply}, ensure_ascii=False, indent=1).encode()
@property
def file(self) -> str:
"""Where the archive keeps the run, relative to its chunk's folder. The name is the hash of the content, so a run archived twice is kept once."""
return f"{FOLDER}/{hashlib.sha256(self.content()).hexdigest()}.json"
@staticmethod
def read(path: pathlib.Path) -> "JudgeRun | Failure":
try:
kept = json.loads(path.read_text(encoding="utf-8"))
return JudgeRun(kept["prompt"], kept["reply"])
except (OSError, ValueError, KeyError, TypeError) as failure:
return Failure(f"could not read the archived judge run {path}: {failure}")
+26 -13
View File
@@ -5,6 +5,7 @@ from model_routing.prices import PRICES
from text_files.markdown_text import quoted
from text_files.utc_stamp import STAMPED
from claude_code.tokens import Tokens
from transcript_archive.judge_run import JudgeRun
from model_routing.route import WorkerLine
TASK = "Task: "
@@ -29,6 +30,9 @@ JUDGE = "Judge: "
# What the judge's run cost, in dollars, on the line after JUDGE; absent where the verdict recorded none.
JUDGE_COST = "Judge cost: "
JUDGE_COST_AT = re.compile(rf"^{re.escape(JUDGE)}(.+)\n{re.escape(JUDGE_COST)}([0-9.]+)$", re.MULTILINE)
# The judge run the verdict came from, as a path in the chunk's folder (judge_run.JudgeRun.file), on the line after the judge's rung and cost; absent where the archive kept none.
JUDGE_RUN = "Judge run: "
JUDGE_RUN_AT = re.compile(rf"^{re.escape(JUDGE_RUN)}(.+)$", re.MULTILINE)
# The claude-config release and commit that ran the cycle, on the line before JUDGE, such as `Config: v0.8.0 (<sha>)`.
@@ -84,9 +88,18 @@ def spent_stamps(outcomes: str) -> list[str]:
return [stamp for stamp, outcome in each_outcome(outcomes) if f"\n{SPENT}" in outcome]
def judge_lines(judge: str, judge_cost: float) -> str:
"""The judge's rung, and what its run cost where that is known."""
return f"{JUDGE}{judge}" + (f"\n{JUDGE_COST}{judge_cost:.4f}" if judge_cost else "")
def judge_lines(judge: str, judge_cost: float, judge_run: JudgeRun | None) -> str:
"""The judge's rung, what its run cost where that is known, and the archived run where there is one."""
return f"{JUDGE}{judge}" + (f"\n{JUDGE_COST}{judge_cost:.4f}" if judge_cost else "") + (f"\n{JUDGE_RUN}{judge_run.file}" if judge_run else "")
def judge_run_at(outcomes: str, stamp: str) -> str | None:
"""The path of the judge run that the outcome stamped stamp names; None where it names none."""
for found, outcome in each_outcome(outcomes):
# The last such line: an interrupt's question, which comes first, may hold one.
if found == stamp and (named := JUDGE_RUN_AT.findall(outcome)):
return named[-1]
return None
def judge_costs(outcomes: str) -> list[tuple[str, float]]:
@@ -98,31 +111,31 @@ def escalation_block(escalation: str) -> str:
return f"Escalation:\n{quoted(escalation)}\n\n" if escalation else ""
def finished(worker: WorkerLine, evidence: str, judge: str, judge_cost: float = 0.0, annotation: Annotation = Annotation()) -> str:
return f"{FINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{annotation_lines(annotation)}{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
def finished(worker: WorkerLine, evidence: str, judge: str, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
return f"{FINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{annotation_lines(annotation)}{judge_lines(judge, judge_cost, judge_run)}\n{WORKER}{worker}"
def unfinished(worker: WorkerLine, evidence: str, judge: str, escalation: str, judge_cost: float = 0.0, annotation: Annotation = Annotation()) -> str:
def unfinished(worker: WorkerLine, evidence: str, judge: str, escalation: str, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
"""escalation is the judge's quote showing that the worker's cited escalation trigger fired; empty for none."""
escalated = escalation_block(escalation)
return f"{UNFINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
return f"{UNFINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost, judge_run)}\n{WORKER}{worker}"
def interrupted(question: str, evidence: str, judge: str, escalation: str, worker: WorkerLine, judge_cost: float = 0.0, annotation: Annotation = Annotation()) -> str:
def interrupted(question: str, evidence: str, judge: str, escalation: str, worker: WorkerLine, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
"""escalation is the judge's quote showing that the worker's cited escalation trigger fired; empty for none. worker is the line naming the route the worker ran on until the interrupt."""
escalated = escalation_block(escalation)
return f"{INTERRUPTED}\n\n{question}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
return f"{INTERRUPTED}\n\n{question}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost, judge_run)}\n{WORKER}{worker}"
def answered(*, question: str, sources: str, answer: str, evidence: str, escalation: str, judge: str, worker: WorkerLine, judge_cost: float = 0.0, annotation: Annotation = Annotation()) -> str:
def answered(*, question: str, sources: str, answer: str, evidence: str, escalation: str, judge: str, worker: WorkerLine, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
"""escalation and worker are as on interrupted. Every parameter is keyword-only, since a swap of two str ones would still type-check."""
escalated = escalation_block(escalation)
return f"{ANSWERED}\n\n{question}\n\n{sources} says:\n{quoted(answer)}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost)}\n{WORKER}{worker}"
return f"{ANSWERED}\n\n{question}\n\n{sources} says:\n{quoted(answer)}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost, judge_run)}\n{WORKER}{worker}"
def rejected(why: str, reply: str, judge: str, judge_cost: float = 0.0) -> str:
def rejected(why: str, reply: str, judge: str, judge_cost: float = 0.0, judge_run: JudgeRun | None = None) -> str:
"""A judge reply loop-claude rejected, with the reason and the reply as json."""
return f"{REJECTED}: {why}\n\nReply:\n{quoted(reply)}\n\n{judge_lines(judge, judge_cost)}"
return f"{REJECTED}: {why}\n\nReply:\n{quoted(reply)}\n\n{judge_lines(judge, judge_cost, judge_run)}"
def stepped_up(worker: WorkerLine) -> str:
@@ -1,6 +1,7 @@
import contextlib
import hashlib
import io
import json
import pathlib
import sys
import tempfile
@@ -11,6 +12,7 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from transcript_archive.project_key import checkout_of, project_key
from failure import Failure
from transcript_archive import outcomes, read, write
from transcript_archive.judge_run import JudgeRun
from transcript_archive.location import project_archive
from text_files.utc_stamp import utc
@@ -31,6 +33,49 @@ class Archive(unittest.TestCase):
self.assertEqual(kept_outcomes.read_text(), "Task: Do it.\n\n2026-09-21T14:13:20Z Stopped: Ctrl-C\n\nWorker: m\n")
class JudgeRuns(unittest.TestCase):
judged = JudgeRun("The prompt.", {"structured_output": {"finished": True}, "total_cost_usd": 0.5})
def test_a_run_is_kept_once_in_the_chunk_folder_under_the_hash_of_its_content(self):
with tempfile.TemporaryDirectory() as root:
kept = pathlib.Path(root, "archive")
transcript = pathlib.Path(root, "t.jsonl")
transcript.write_text('{"timestamp": "2026-10-03T00:00:00Z"}\n')
with contextlib.redirect_stderr(io.StringIO()):
for offset in (0, 1):
outcome = outcomes.finished("w", "merged", "j", judge_run=self.judged)
self.assertIsNone(write.archive(kept, str(transcript), "Do it.", outcome, NOW + offset, judge_run=self.judged))
(folder,) = kept.glob("*/")
(file,) = folder.glob("judge/*")
self.assertEqual(file, folder / self.judged.file)
self.assertEqual(file.name, hashlib.sha256(file.read_bytes()).hexdigest() + ".json")
self.assertEqual(json.loads(file.read_text()), {"prompt": "The prompt.", "reply": {"structured_output": {"finished": True}, "total_cost_usd": 0.5}})
self.assertEqual(JudgeRun.read(file), self.judged)
def test_names_the_run_on_the_line_after_the_judge_lines_and_reads_the_name_back(self):
outcome = outcomes.finished("w", "e", "j", judge_cost=0.5, judge_run=self.judged)
self.assertEqual(outcome, f"Verdict: finished\n\nEvidence:\n> e\n\nJudge: j\nJudge cost: 0.5000\nJudge run: {self.judged.file}\nWorker: w")
self.assertEqual(outcomes.rejected("why", "{}", "j", judge_run=self.judged), f"Judge reply rejected: why\n\nReply:\n> {{}}\n\nJudge: j\nJudge run: {self.judged.file}")
self.assertEqual(outcomes.workers(f"{utc(NOW)} {outcome}\n"), [(utc(NOW), "w")])
self.assertEqual(outcomes.judge_costs(f"{utc(NOW)} {outcome}\n"), [("j", 0.5)])
kept = f"Task: Do it.\n\n{utc(NOW)} {outcomes.stopped('w', 'Ctrl-C')}\n\n{utc(NOW + 1)} {outcome}\n\n{utc(NOW + 2)} {outcomes.finished('w', 'e', 'j')}\n"
self.assertEqual(outcomes.judge_run_at(kept, utc(NOW + 1)), self.judged.file)
self.assertIsNone(outcomes.judge_run_at(kept, utc(NOW + 2)))
def test_writes_no_outcome_where_the_run_cannot_be_kept(self):
with tempfile.TemporaryDirectory() as root:
kept = pathlib.Path(root, "archive")
transcript = pathlib.Path(root, "t.jsonl")
transcript.write_text('{"timestamp": "2026-10-03T00:00:00Z"}\n')
with contextlib.redirect_stderr(io.StringIO()):
self.assertIsNone(write.archive(kept, str(transcript), "Do it.", outcomes.stopped("w", "x"), NOW))
(folder,) = kept.glob("*/")
(folder / "judge").write_text("a file where the folder should be")
failure = write.archive(kept, str(transcript), "Do it.", outcomes.finished("w", "e", "j", judge_run=self.judged), NOW + 1, judge_run=self.judged)
self.assertIsInstance(failure, Failure)
self.assertNotIn("Verdict: finished", next(folder.glob("*.outcomes.md")).read_text())
class CopyFor(unittest.TestCase):
def test_skips_a_version_already_kept_beside_its_copy(self):
with tempfile.TemporaryDirectory() as temp:
+18 -2
View File
@@ -4,6 +4,7 @@ import pathlib
import typing
from transcript_archive.location import chunk_folder, outcomes_of
from transcript_archive.judge_run import JudgeRun
from failure import Failure
from text_files.file_writes import make_private, replace_file
from claude_code.session import session_files
@@ -11,8 +12,8 @@ from transcript_archive.outcomes import TASK
from text_files.utc_stamp import utc
def archive(project_archive: pathlib.Path, path: str, task: str, outcome: str, at: float, note: typing.Callable[[str], None] = lambda text: None) -> Failure | None:
"""Keeps the transcript at path, its outcomes file and its session's files in project_archive, in the layout this package's docstring gives; the failure that stopped it, or None. The outcome is stamped and appended to the outcomes file, which opens with the task only when the copy is new. Where a new copy lands, note is told where."""
def archive(project_archive: pathlib.Path, path: str, task: str, outcome: str, at: float, note: typing.Callable[[str], None] = lambda text: None, judge_run: JudgeRun | None = None) -> Failure | None:
"""Keeps the transcript at path, its outcomes file and its session's files in project_archive, in the layout this package's docstring gives; the failure that stopped it, or None. The outcome is stamped and appended to the outcomes file, which opens with the task only when the copy is new. Where a new copy lands, note is told where. A judge_run the outcome names is kept first, so the outcome never names a run the archive lacks."""
try:
with open(path, "rb") as handle:
raw = handle.read()
@@ -23,6 +24,8 @@ def archive(project_archive: pathlib.Path, path: str, task: str, outcome: str, a
for sibling in session_files(path):
if sibling != path and (unkept := mirror(pathlib.Path(sibling), folder / "session" / os.path.basename(sibling))):
return unkept
if judge_run and (unkept := keep_judge_run(folder, judge_run)):
return unkept
copy = folder / f"{hashlib.sha256(raw).hexdigest()}.jsonl"
outcomes = outcomes_of(copy)
stamped = f"{utc(at)} {outcome}"
@@ -36,6 +39,19 @@ def archive(project_archive: pathlib.Path, path: str, task: str, outcome: str, a
return None
def keep_judge_run(folder: pathlib.Path, run: JudgeRun) -> Failure | None:
kept = folder / run.file
if kept.exists():
return None
try:
make_private(kept.parent)
except OSError as failure:
return unarchived(failure.filename, failure.strerror)
if unwritten := replace_file(kept, run.content()):
return unarchived(kept, unwritten.reason)
return None
def unarchived(where: object, why: str | None) -> Failure:
return Failure(f"could not archive the transcript: {where}: {why}")
+2 -2
View File
@@ -184,14 +184,14 @@ Valid while claude takes `--session-id` and resumes by it, and the human runs lo
## loop-claude: the archive keeps the raw transcript, and a failed archive stops the loop
2026-09-28, lilleman. Serves goals 1 and 3: saving each transcript locally is the first step of backing it up, and it shows afterwards what an unattended run did. A copy of the `.jsonl`, deduplicated by content hash, with its outcomes beside it; a rendering drops tool results, and a second-resolution stamp names two states within one second as one file. A failed archive exits 1, since a loop that loses its record silently fails the backup goal. Valid while transcripts stay small enough to copy whole.
2026-09-28, lilleman. Serves goals 1 and 3: saving each transcript locally is the first step of backing it up, and it shows afterwards what an unattended run did. A copy of the `.jsonl`, deduplicated by content hash, with its outcomes beside it; a rendering drops tool results, and a second-resolution stamp names two states within one second as one file. A failed archive exits 1, since a loop that loses its record silently fails the backup goal. The judge's runs are archived too, as `judge/<sha256>.json` in the chunk's folder: the prompt it read and its json reply, which are its whole transcript, since it runs with no tools and one turn. The run is kept whole, deduplicated by hash, and the outcome of the verdict, interrupt or rejected reply it produced names it on a `Judge run:` line. It is kept whole because a prompt is at most `JUDGE_PROMPT_CHARS`, 400,000 characters, against copies of up to 14.8 MB: measured 2026-11-16 over the archive's 140 copies, 603 MB, 136 recorded verdicts times that ceiling is under a tenth of it. `judge-replay` replays a verdict on its archived prompt where there is one. Valid while transcripts, and the judge's prompts, stay small enough to copy whole.
## loop-claude: each verdict records the claude-config release and commit that ran it, the raw tokens each thread spent by model and kind since the previous outcome that recorded a spend, and the human's typed answer the judge quotes as reversing a choice the worker took on its own recommendation
2026-11-12, lilleman, todo item 305. Serves goals 1, 1.2 and 1.4. A change to the setup can be judged only by comparing chunks run before it with chunks run after, so each verdict, interrupt and quoted answer in loop-claude's archive records three things beside its judge lines.
- `Config: <release> (<commit>)`: `git describe` and the commit of `~/.claude`, so outcomes group by the setup that produced them. Where git can't tell, it records `unknown` and why, on one line.
- A `Spent since` block: the raw tokens each thread spent, by model and by kind (input, cache write 5m, cache write 1h, cache read, output), then the list prices in force, from `bin/model_routing/prices.py`. The threads are the worker's main thread, each subagent by its agent type and description, and the judge run that ruled. Each message counts once, by its id. The stretch runs from the previous outcome that recorded a spend, so what a `/talk` or a stopped run spent between rulings is counted once. It keeps tokens, since prices change. The prices recorded beside them give what the stretch cost when it ran. Claude Code doesn't report thinking apart from output, so it is left in output. The judge's tokens come from its json reply's `usage`, until item 303 archives the judge's own transcript.
- A `Spent since` block: the raw tokens each thread spent, by model and by kind (input, cache write 5m, cache write 1h, cache read, output), then the list prices in force, from `bin/model_routing/prices.py`. The threads are the worker's main thread, each subagent by its agent type and description, and the judge run that ruled. Each message counts once, by its id. The stretch runs from the previous outcome that recorded a spend, so what a `/talk` or a stopped run spent between rulings is counted once. It keeps tokens, since prices change. The prices recorded beside them give what the stretch cost when it ran. Claude Code doesn't report thinking apart from output, so it is left in output. The judge's tokens come from its json reply's `usage`, which the archived judge run holds too.
- A `Reversed:` quote: the human's typed answer that overrides a choice the worker took on its own recommendation and recorded. The judge quotes the latest such `[human]` line; loop-claude keeps the whole typed answer that holds the quote verbatim, and drops a quote no typed answer holds. `score-outcomes` counts the distinct answers per finished chunk. That count is goal 1.2's counter-signal, and the 2026-11-08 entry of `loop-claude: a judge decides each interrupt…` is valid only while it stays low. It counts only chunks that record a Config line, since earlier judges didn't look for reversals.
Valid while score-outcomes reads the reversals from the archive.