Answer stability round 1: 1 🔴, 5 🟡 (0 misread); imports sorted, the replay cuts the task and the rules from its grounds and fails on a missing heading, outcome parameters keyword-only, the mtime order dropped

This commit is contained in:
2026-11-17 01:13:39 +01:00
parent b5afb4c36a
commit 2600fbb254
11 changed files with 93 additions and 39 deletions
+15
View File
@@ -0,0 +1,15 @@
from claude_code.entries import Entry
from claude_code.transcript import human_answers
from failure import Failure
from loop_claude import escalation_mark
from loop_claude.cycle.judge import Grounds, JUDGE_RULES, TASK_HEADING, TRANSCRIPT_HEADING
from loop_claude.cycle.prompts import typed_answer
def grounds_of(prompt: str, entries: list[Entry]) -> Grounds | Failure:
"""What the quotes of a judge run on prompt are checked against, as judge_prompt would have made it for the transcript's entries: the premises and the records before the task are quotable, the transcript the judge was shown comes before the rules. The prompt is split on the headings judge_prompt writes, at the first of each; a Failure where it lacks one. Rules of an earlier wording stay part of the transcript, a slack that lets an evidence quote them pass."""
before_task, task_heading, after_task = prompt.partition(f"\n\n{TASK_HEADING}")
_, transcript_heading, shown = after_task.partition(TRANSCRIPT_HEADING)
if not (task_heading and transcript_heading):
return Failure(f"the archived prompt lacks the heading {TASK_HEADING.strip()!r} or {TRANSCRIPT_HEADING.strip()!r}, which judge_prompt writes")
return Grounds(shown.removesuffix(f"\n\n{JUDGE_RULES}"), escalation_mark.after_mark(entries, 0), before_task, tuple(human_answers(entries, typed_answer)))
+6 -2
View File
@@ -34,7 +34,6 @@ or a usage limit.
import pathlib
import typing
from transcript_archive.judge_run import JudgeRun
from transcript_archive.location import archive_root, transcript_of
from transcript_archive.read import archived_outcomes
from rung_replay.command import arguments_or_exit, down_from, failed, limited_exit, projects_on
@@ -44,6 +43,8 @@ from transcript_archive.project_key import checkout_of
from loop_effects.effects import Effects
from failure import Failure
from rung_replay.search import first_reply, replayed
from judge_replay.archived_prompt import grounds_of
from transcript_archive.judge_run import JudgeRun
from model_routing.ladder import Ladder, REVIEW_STEPS, Rung
from claude_code.reply import Limited
from model_routing.ladder_file import read_ladder, SHIPPED
@@ -117,7 +118,10 @@ def rule(case: Case, rung: Rung, home: pathlib.Path, effects: Effects) -> Kind |
kept = JudgeRun.read(case.judge_run)
if isinstance(kept, Failure):
return verdict_kind(kept)
verdict: Verdict | Failure | Limited = first_reply(lambda: judge_archived(case.root, kept.prompt, entries, rung, effects))
grounds = grounds_of(kept.prompt, entries)
if isinstance(grounds, Failure):
return verdict_kind(grounds)
verdict: Verdict | Failure | Limited = first_reply(lambda: judge_archived(case.root, kept.prompt, grounds, len(entries), rung, effects))
return verdict if isinstance(verdict, Limited) else verdict_kind(verdict)
premises, records = read_premises(case.root, home), read_records(case.root, home, home / ".claude")
if isinstance(premises, Failure):
+33 -12
View File
@@ -10,13 +10,14 @@ import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from transcript_archive.outcomes import ANSWERED, FINISHED, INTERRUPTED, UNFINISHED
from claude_code.entries import AssistantTurn, HumanTurn, LoopPrompt, Text
from judge_replay.replay import Case, case_from, report, reproduced, sample, verdict_kind
from project_config.test_project_config import configured
from loop_effects.effects import Effects
from failure import Failure
from loop_claude.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished
from loop_claude.cycle.judge import TRANSCRIPT_HEADING, Rejected, judge_archived
from claude_code.entries import AssistantTurn, HumanTurn, LoopPrompt, Text
from judge_replay.archived_prompt import grounds_of
from loop_claude.cycle.judge import judge_archived, JUDGE_RULES, Rejected, TASK_HEADING, TRANSCRIPT_HEADING
from judge_replay import main
from rung_replay.search import Miss, Replay, Tried
from model_routing.ladder import Rung
@@ -172,8 +173,8 @@ class Main(unittest.TestCase):
kept = self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1"
prompt = "From the goals file, OLD premises\n\nThe records: none\n\nThe task the worker was given:\nDo it.\n\nWhat the worker did and said, what the human typed on [human] lines and what loop-claude told it on [loop] lines, latest last:\nsaid Verdict: finished\n\nOLD rules"
(kept / "judge").mkdir()
(kept / "judge" / "run.json").write_text(json.dumps({"prompt": prompt, "reply": {"type": "result"}}))
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/run.json\nWorker: x\n")
(kept / "judge" / "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json").write_text(json.dumps({"prompt": prompt, "reply": {"type": "result"}}))
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json\nWorker: x\n")
code, out, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
self.assertEqual(code, 0, err)
self.assertIn("The reference judge, claude-opus-5-5 at xhigh effort, reproduced 2.", out)
@@ -186,7 +187,7 @@ class Main(unittest.TestCase):
def test_replays_a_verdict_whose_archived_judge_run_cannot_be_read_as_failed(self):
kept = self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1"
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/gone.json\nWorker: x\n")
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.json\nWorker: x\n")
code, out, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
self.assertEqual(code, 0, err)
self.assertIn("The reference judge, claude-opus-5-5 at xhigh effort, reproduced 1. It returned another kind on:", out)
@@ -260,19 +261,19 @@ class Main(unittest.TestCase):
def test_exits_1_on_a_usage_limit_met_on_an_archived_prompt(self):
kept = self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1"
(kept / "judge").mkdir()
(kept / "judge" / "run.json").write_text(json.dumps({"prompt": "P", "reply": {}}))
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/run.json\nWorker: x\n")
(kept / "judge" / "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json").write_text(json.dumps({"prompt": "P", "reply": {}}))
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json\nWorker: x\n")
code, _, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: {"type": "result", "api_error_status": 429, "result": "You've hit your limit"})
self.assertEqual((code, "usage limit: You've hit your limit" in err), (1, True))
class JudgeArchivedRuns(unittest.TestCase):
"""judge_archived on a prompt of two parts, as the archive keeps it: HEAD, the premises, and what follows TRANSCRIPT_HEADING, the transcript the judge was shown."""
PROMPT = f"HEAD: Ship fast.\n\n{TRANSCRIPT_HEADING}Worker chose option B.\n[human] Use Postgres.\n"
"""judge_archived on a prompt as judge_prompt writes it: the premises, the task, the transcript the judge was shown, and the rules."""
PROMPT = f"HEAD: Ship fast.\n\n{TASK_HEADING}Do it.\n\n{TRANSCRIPT_HEADING}Worker chose option B.\n[human] Use Postgres.\n\n{JUDGE_RULES}"
ENTRIES = [AssistantTurn(parts=(Text("Worker chose option B."),)), HumanTurn(text="Use Postgres."), LoopPrompt(text="Keep going.")]
def judged(self, **fields):
"""What judge_archived returns where the judge's structured output is fields over an unfinished verdict, and the cwd claude ran in."""
def judged(self, prompt=PROMPT, **fields):
"""What judge_archived returns on prompt where the judge's structured output is fields over an unfinished verdict, and the cwd claude ran in."""
output = {"answer": "", "escalation": "", "evidence": "Worker chose option B.", "finished": False, "interrupt": False, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False, **fields}
cwds = []
@@ -281,7 +282,8 @@ class JudgeArchivedRuns(unittest.TestCase):
return subprocess.CompletedProcess(args, 0, json.dumps({"type": "result", "structured_output": output, "total_cost_usd": 0.25}), "")
effects = Effects(claude=claude, claude_on_terminal=unused, discard_typed=unused, dm=unused, execv=unused, git=unused, input=unused, local_time=unused, memory=unused, now=unused, on_terminal=unused, python=unused, sleep=unused, stream_claude=unused, tmux=unused, tmux_on_terminal=unused)
return judge_archived("/code/p", self.PROMPT, self.ENTRIES, REFERENCE, effects), cwds
grounds = grounds_of(prompt, self.ENTRIES)
return (grounds if isinstance(grounds, Failure) else judge_archived("/code/p", prompt, grounds, len(self.ENTRIES), REFERENCE, effects)), cwds
def test_runs_in_the_projects_folder_and_stamps_the_verdict_with_the_entries_read(self):
verdict, cwds = self.judged()
@@ -295,6 +297,25 @@ class JudgeArchivedRuns(unittest.TestCase):
rejected, _ = self.judged(evidence="Never said.")
self.assertEqual((rejected.cost, rejected.run.prompt), (0.25, self.PROMPT))
def test_keeps_the_cost_of_a_reply_rejected_for_its_shape(self):
rejected, _ = self.judged(finished="yes")
self.assertEqual((type(rejected), rejected.cost), (Rejected, 0.25))
def test_checks_neither_the_task_nor_the_rules_as_the_transcript_or_the_premises(self):
self.assertIsInstance(self.judged(evidence=JUDGE_RULES)[0], Rejected)
self.assertIsInstance(self.judged(answer="Do it.", question="Which?")[0], Interrupt)
def test_splits_the_prompt_at_the_first_transcript_heading(self):
quoting = self.PROMPT.replace("Worker chose", f"Quoted: {TRANSCRIPT_HEADING}Worker chose", 1)
self.assertIsInstance(self.judged(quoting, evidence="Quoted:")[0], Unfinished)
def test_fails_before_the_judge_runs_on_a_prompt_without_a_heading(self):
for prompt in ("No headings at all.", f"{TASK_HEADING}Do it.", f"\n\n{TASK_HEADING}Do it.\n\nNo transcript heading."):
with self.subTest(prompt=prompt):
failure, cwds = self.judged(prompt)
self.assertEqual((type(failure), cwds), (Failure, []))
self.assertIn("lacks the heading", failure.reason)
def test_checks_a_quoted_answer_against_the_premises(self):
self.assertIsInstance(self.judged(answer="Ship fast.", question="Which?")[0], QuotedAnswer)
self.assertIsInstance(self.judged(answer="Use Postgres.", question="Which?")[0], Interrupt)
+7 -8
View File
@@ -9,9 +9,9 @@ from loop_effects.effects import Effects
from claude_code.entries import Entry, structured_output
from loop_claude import escalation_mark
from failure import Failure
from transcript_archive.judge_run import JudgeRun
from loop_claude.verdicts import Finished, Interrupt, Judged, QuotedAnswer, Unfinished, Verdict
from claude_code.transcript import human_answers
from transcript_archive.judge_run import JudgeRun
from loop_claude.quotes import plain, verbatim
from loop_claude.cycle.retry import retried
from model_routing.ladder import Rung
@@ -65,7 +65,8 @@ FINISHED_RULES = (
"continue: an open PR, a review not yet run, CI not yet reported, an answer not yet recorded."
)
JUDGE_RULES = "\n\n".join((INTERRUPT_RULES, QUOTE_RULES, FINISHED_RULES))
# Opens the transcript in the prompt; the premises, the records and the task come before it, the rules after the transcript.
# Open the task and the transcript in the prompt; the premises and the records come before the task, the rules after the transcript.
TASK_HEADING = "The task the worker was given:\n"
TRANSCRIPT_HEADING = "What the worker did and said, what the human typed on [human] lines and what loop-claude told it on [loop] lines, latest last:\n"
@@ -163,12 +164,10 @@ def judge_once(root: str, brief: Brief, rung: Rung, effects: Effects) -> Verdict
return verdict if isinstance(verdict, Failure | Limited) else escalation_mark.stamped(verdict, len(brief.entries))
def judge_archived(root: str, prompt: str, entries: list[Entry], rung: Rung, effects: Effects) -> Verdict | Failure | Limited:
"""One judge run on a prompt the archive kept from an earlier run, as judge-replay replays it: the judge reads what it read then, and its quotes are checked against that prompt's own text, in place of the records of today. The prompt's rules count as part of the transcript, a slack that lets an evidence quote the rules pass."""
head, _, shown = prompt.partition(TRANSCRIPT_HEADING)
grounds = Grounds(shown, escalation_mark.after_mark(entries, 0), head, tuple(human_answers(entries, typed_answer)))
def judge_archived(root: str, prompt: str, grounds: Grounds, entries_read: int, rung: Rung, effects: Effects) -> Verdict | Failure | Limited:
"""One judge run on a prompt the archive kept from an earlier run, as judge-replay replays it: the judge reads what it read then, and its quotes are checked against grounds, which judge_replay.archived_prompt makes from that prompt in place of the records of today."""
verdict = run_judge(root, judge_args(rung), prompt, grounds, effects)
return verdict if isinstance(verdict, Failure | Limited) else escalation_mark.stamped(verdict, len(entries))
return verdict if isinstance(verdict, Failure | Limited) else escalation_mark.stamped(verdict, entries_read)
def judge_args(rung: Rung) -> list[str]:
@@ -271,7 +270,7 @@ def judge_prompt(brief: Brief) -> tuple[str, Grounds]:
head = (
f"From the goals file, {brief.premises}\n\n"
f"The records: the project's working rules and decisions, then the human's rules for every project:\n{brief.records or 'none'}\n\n"
f"The task the worker was given:\n{brief.task}\n\n"
f"{TASK_HEADING}{brief.task}\n\n"
f"{TRANSCRIPT_HEADING}"
)
room = max(0, JUDGE_PROMPT_CHARS - len(head) - len(JUDGE_RULES) - 2)
+4 -4
View File
@@ -3,7 +3,6 @@ import pathlib
import typing
from loop_claude.cycle.states import Active, Archived, Closing, Ended, Event, HumanTalked, HumanTyped, Interrupting, Judging, QUOTED_ANSWERS_PER_CYCLE, Ruled, State, to_the_human, transition, TypedAnswer, WorkerRan, Working
from transcript_archive.judge_run import JudgeRun
from transcript_archive.write import archive
from loop_claude.cycle.worker import AUTO_COMPACT_WINDOW, COMPACT_WINDOW, run_worker, worker_args
from loop_claude.cycle.judge import Brief, judge, Rejected
@@ -15,6 +14,7 @@ from failure import Failure
from loop_claude.verdicts import Finished, Interrupt, Judged, QuotedAnswer, Unfinished, Verdict
from loop_claude.cycle.interrupt import interrupt, Talked, WorkerSession
from loop_claude.routing import judge_rung, next_route, step_up, StepUp
from transcript_archive.judge_run import JudgeRun
from model_routing.ladder import Ladder, Rung
from transcript_archive.read import last_spent
from transcript_archive import outcomes
@@ -189,12 +189,12 @@ def ruling_outcome(cycle: Cycle, ruling: Interrupt | QuotedAnswer, quoted_answer
"""The archive's outcome for a ruling: interrupted where it goes to the human, with the question put to them, and else answered from the premises or the records."""
if isinstance(ruling, QuotedAnswer) and not to_the_human(ruling, quoted_answers):
return outcomes.answered(question=ruling.question, sources=prompts.QUOTE_SOURCES, answer=ruling.answer, evidence=ruling.evidence, escalation=ruling.escalation, judge=str(judge_of(cycle)), worker=cycle.route.worker_line(), judge_cost=ruling.judged.cost, annotation=annotation, judge_run=ruling.judged.run)
return outcomes.interrupted(question_for(ruling), ruling.evidence, str(judge_of(cycle)), ruling.escalation, cycle.route.worker_line(), ruling.judged.cost, annotation, ruling.judged.run)
return outcomes.interrupted(question_for(ruling), ruling.evidence, str(judge_of(cycle)), ruling.escalation, cycle.route.worker_line(), judge_cost=ruling.judged.cost, annotation=annotation, judge_run=ruling.judged.run)
def keep_rejection(cycle: Cycle, transcript_path: str, rejected: Rejected) -> Stop | None:
"""Archives a judge reply loop-claude rejected, so the archive shows what the judge returned before a retry or an exit 1."""
if unkept := archived(cycle.project_archive, transcript_path, cycle.task, outcomes.rejected(rejected.reason, rejected.reply, str(judge_of(cycle)), rejected.cost, rejected.run), cycle.effects, rejected.run):
if unkept := archived(cycle.project_archive, transcript_path, cycle.task, outcomes.rejected(rejected.reason, rejected.reply, str(judge_of(cycle)), judge_cost=rejected.cost, judge_run=rejected.run), cycle.effects, rejected.run):
return Stop(1, unkept.reason)
return None
@@ -237,7 +237,7 @@ def close(cycle: Cycle, transcript_path: str, verdict: Finished | Unfinished) ->
annotation = entries if isinstance(entries, Failure) else annotation_of(cycle, transcript_path, entries, verdict.judged)
if isinstance(annotation, Failure):
return Stop(1, annotation.reason)
outcome = outcomes.finished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle)), verdict.judged.cost, annotation, verdict.judged.run) if isinstance(verdict, Finished) else outcomes.unfinished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle)), verdict.escalation, verdict.judged.cost, annotation, verdict.judged.run)
outcome = outcomes.finished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle)), judge_cost=verdict.judged.cost, annotation=annotation, judge_run=verdict.judged.run) if isinstance(verdict, Finished) else outcomes.unfinished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle)), verdict.escalation, judge_cost=verdict.judged.cost, annotation=annotation, judge_run=verdict.judged.run)
if failure := archived(cycle.project_archive, transcript_path, cycle.task, outcome, cycle.effects, verdict.judged.run):
return Stop(1, failure.reason)
if isinstance(verdict, Finished) and (failure := delete_session(cycle.session)):
+3 -3
View File
@@ -394,12 +394,12 @@ Worker: {OFF_LADDER}
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
prompts = [stdin for args, stdin in fake.made("claude")[:3]]
[chunk] = self.archive.iterdir()
kept = "\n".join(path.read_text() for path in sorted(chunk.glob("*.outcomes.md"), key=lambda path: path.stat().st_mtime_ns))
kept = "\n".join(path.read_text() for path in chunk.glob("*.outcomes.md"))
named = re.findall(r"^Judge run: (judge/[0-9a-f]{64}\.json)$", kept, re.MULTILINE)
self.assertEqual(len(named), 3, kept)
runs = [json.loads((chunk / name).read_text()) for name in named]
self.assertEqual([found["prompt"] for found in runs], prompts)
self.assertEqual([found["reply"] for found in runs], [json.loads(reply.stdout) for reply in (rejected, interrupted, finished)])
self.assertEqual(sorted(found["prompt"] for found in runs), sorted(prompts))
self.assertEqual(sorted(json.dumps(found["reply"]) for found in runs), sorted(json.dumps(json.loads(reply.stdout)) for reply in (rejected, interrupted, finished)))
self.assertEqual(sorted(path.name for path in (chunk / "judge").iterdir()), sorted(name.removeprefix("judge/") for name in named))
def test_stops_when_a_judge_run_cannot_be_archived(self):
+1 -1
View File
@@ -1,8 +1,8 @@
"""The verdicts cycle.judge.verdict_from builds from the judge's reply; cycle.run.rule then stamps every verdict but Finished with the items the chunk takes. They sit outside cycle/, since routing reads them and cycle imports routing."""
import dataclasses
from claude_code.tokens import Tokens
from transcript_archive.judge_run import JudgeRun
from claude_code.tokens import Tokens
@dataclasses.dataclass(frozen=True)
+4 -1
View File
@@ -27,6 +27,9 @@ class JudgeRun:
def read(path: pathlib.Path) -> "JudgeRun | Failure":
try:
kept = json.loads(path.read_text(encoding="utf-8"))
return JudgeRun(kept["prompt"], kept["reply"])
prompt, reply = kept["prompt"], kept["reply"]
except (OSError, ValueError, KeyError, TypeError) as failure:
return Failure(f"could not read the archived judge run {path}: {failure}")
if not (isinstance(prompt, str) and isinstance(reply, dict)):
return Failure(f"could not read the archived judge run {path}: its prompt is no string or its reply no object")
return JudgeRun(prompt, reply)
+6 -6
View File
@@ -1,11 +1,11 @@
import re
import typing
from transcript_archive.judge_run import FOLDER, JudgeRun
from model_routing.prices import PRICES
from text_files.markdown_text import quoted
from text_files.utc_stamp import STAMPED
from claude_code.tokens import Tokens
from transcript_archive.judge_run import JudgeRun
from model_routing.route import WorkerLine
TASK = "Task: "
@@ -32,7 +32,7 @@ JUDGE_COST = "Judge cost: "
JUDGE_COST_AT = re.compile(rf"^{re.escape(JUDGE)}(.+)\n{re.escape(JUDGE_COST)}([0-9.]+)$", re.MULTILINE)
# The judge run the verdict came from, as a path in the chunk's folder (judge_run.JudgeRun.file), on the line after the judge's rung and cost; absent where the archive kept none.
JUDGE_RUN = "Judge run: "
JUDGE_RUN_AT = re.compile(rf"^{re.escape(JUDGE_RUN)}(.+)$", re.MULTILINE)
JUDGE_RUN_AT = re.compile(rf"^{re.escape(JUDGE_RUN)}({FOLDER}/[0-9a-f]{{64}}\.json)$", re.MULTILINE)
# The claude-config release and commit that ran the cycle, on the line before JUDGE, such as `Config: v0.8.0 (<sha>)`.
@@ -111,17 +111,17 @@ def escalation_block(escalation: str) -> str:
return f"Escalation:\n{quoted(escalation)}\n\n" if escalation else ""
def finished(worker: WorkerLine, evidence: str, judge: str, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
def finished(worker: WorkerLine, evidence: str, judge: str, *, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
return f"{FINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{annotation_lines(annotation)}{judge_lines(judge, judge_cost, judge_run)}\n{WORKER}{worker}"
def unfinished(worker: WorkerLine, evidence: str, judge: str, escalation: str, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
def unfinished(worker: WorkerLine, evidence: str, judge: str, escalation: str, *, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
"""escalation is the judge's quote showing that the worker's cited escalation trigger fired; empty for none."""
escalated = escalation_block(escalation)
return f"{UNFINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost, judge_run)}\n{WORKER}{worker}"
def interrupted(question: str, evidence: str, judge: str, escalation: str, worker: WorkerLine, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
def interrupted(question: str, evidence: str, judge: str, escalation: str, worker: WorkerLine, *, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
"""escalation is the judge's quote showing that the worker's cited escalation trigger fired; empty for none. worker is the line naming the route the worker ran on until the interrupt."""
escalated = escalation_block(escalation)
return f"{INTERRUPTED}\n\n{question}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost, judge_run)}\n{WORKER}{worker}"
@@ -133,7 +133,7 @@ def answered(*, question: str, sources: str, answer: str, evidence: str, escalat
return f"{ANSWERED}\n\n{question}\n\n{sources} says:\n{quoted(answer)}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost, judge_run)}\n{WORKER}{worker}"
def rejected(why: str, reply: str, judge: str, judge_cost: float = 0.0, judge_run: JudgeRun | None = None) -> str:
def rejected(why: str, reply: str, judge: str, *, judge_cost: float = 0.0, judge_run: JudgeRun | None = None) -> str:
"""A judge reply loop-claude rejected, with the reason and the reply as json."""
return f"{REJECTED}: {why}\n\nReply:\n{quoted(reply)}\n\n{judge_lines(judge, judge_cost, judge_run)}"
@@ -11,8 +11,8 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from transcript_archive.project_key import checkout_of, project_key
from failure import Failure
from transcript_archive import outcomes, read, write
from transcript_archive.judge_run import JudgeRun
from transcript_archive import outcomes, read, write
from transcript_archive.location import project_archive
from text_files.utc_stamp import utc
@@ -66,6 +66,18 @@ class JudgeRuns(unittest.TestCase):
run = JudgeRun("Prompt é.", {"a": 1})
self.assertEqual(run.content(), b'{\n "prompt": "Prompt \xc3\xa9.",\n "reply": {\n "a": 1\n }\n}')
def test_reads_a_file_that_is_no_run_as_a_failure(self):
with tempfile.TemporaryDirectory() as root:
for text in ("not json", '{"prompt": "P"}', '{"prompt": 1, "reply": {}}', '{"prompt": "P", "reply": []}'):
with self.subTest(text=text):
path = pathlib.Path(root, "run.json")
path.write_text(text)
self.assertRegex(JudgeRun.read(path).reason, f"^could not read the archived judge run {path}: ")
def test_reads_no_path_that_leaves_the_judge_folder_from_an_outcome(self):
outcome = f"{utc(NOW)} Verdict: finished\n\nJudge run: judge/../../secret.json\n"
self.assertIsNone(outcomes.judge_run_at(outcome, utc(NOW)))
def test_names_the_run_on_each_kind_of_outcome_that_has_a_judge(self):
named = f"Judge: j\nJudge run: {self.judged.file}\n"
kept = [
+1 -1
View File
@@ -4,8 +4,8 @@ import pathlib
import typing
from transcript_archive.location import chunk_folder, outcomes_of
from transcript_archive.judge_run import JudgeRun
from failure import Failure
from transcript_archive.judge_run import JudgeRun
from text_files.file_writes import make_private, replace_file
from claude_code.session import session_files
from transcript_archive.outcomes import TASK