Answer stability round 1: 1 🔴, 5 🟡 (0 misread); imports sorted, the replay cuts the task and the rules from its grounds and fails on a missing heading, outcome parameters keyword-only, the mtime order dropped
This commit is contained in:
@@ -0,0 +1,15 @@
|
||||
from claude_code.entries import Entry
|
||||
from claude_code.transcript import human_answers
|
||||
from failure import Failure
|
||||
from loop_claude import escalation_mark
|
||||
from loop_claude.cycle.judge import Grounds, JUDGE_RULES, TASK_HEADING, TRANSCRIPT_HEADING
|
||||
from loop_claude.cycle.prompts import typed_answer
|
||||
|
||||
|
||||
def grounds_of(prompt: str, entries: list[Entry]) -> Grounds | Failure:
|
||||
"""What the quotes of a judge run on prompt are checked against, as judge_prompt would have made it for the transcript's entries: the premises and the records before the task are quotable, the transcript the judge was shown comes before the rules. The prompt is split on the headings judge_prompt writes, at the first of each; a Failure where it lacks one. Rules of an earlier wording stay part of the transcript, a slack that lets an evidence quote them pass."""
|
||||
before_task, task_heading, after_task = prompt.partition(f"\n\n{TASK_HEADING}")
|
||||
_, transcript_heading, shown = after_task.partition(TRANSCRIPT_HEADING)
|
||||
if not (task_heading and transcript_heading):
|
||||
return Failure(f"the archived prompt lacks the heading {TASK_HEADING.strip()!r} or {TRANSCRIPT_HEADING.strip()!r}, which judge_prompt writes")
|
||||
return Grounds(shown.removesuffix(f"\n\n{JUDGE_RULES}"), escalation_mark.after_mark(entries, 0), before_task, tuple(human_answers(entries, typed_answer)))
|
||||
@@ -34,7 +34,6 @@ or a usage limit.
|
||||
import pathlib
|
||||
import typing
|
||||
|
||||
from transcript_archive.judge_run import JudgeRun
|
||||
from transcript_archive.location import archive_root, transcript_of
|
||||
from transcript_archive.read import archived_outcomes
|
||||
from rung_replay.command import arguments_or_exit, down_from, failed, limited_exit, projects_on
|
||||
@@ -44,6 +43,8 @@ from transcript_archive.project_key import checkout_of
|
||||
from loop_effects.effects import Effects
|
||||
from failure import Failure
|
||||
from rung_replay.search import first_reply, replayed
|
||||
from judge_replay.archived_prompt import grounds_of
|
||||
from transcript_archive.judge_run import JudgeRun
|
||||
from model_routing.ladder import Ladder, REVIEW_STEPS, Rung
|
||||
from claude_code.reply import Limited
|
||||
from model_routing.ladder_file import read_ladder, SHIPPED
|
||||
@@ -117,7 +118,10 @@ def rule(case: Case, rung: Rung, home: pathlib.Path, effects: Effects) -> Kind |
|
||||
kept = JudgeRun.read(case.judge_run)
|
||||
if isinstance(kept, Failure):
|
||||
return verdict_kind(kept)
|
||||
verdict: Verdict | Failure | Limited = first_reply(lambda: judge_archived(case.root, kept.prompt, entries, rung, effects))
|
||||
grounds = grounds_of(kept.prompt, entries)
|
||||
if isinstance(grounds, Failure):
|
||||
return verdict_kind(grounds)
|
||||
verdict: Verdict | Failure | Limited = first_reply(lambda: judge_archived(case.root, kept.prompt, grounds, len(entries), rung, effects))
|
||||
return verdict if isinstance(verdict, Limited) else verdict_kind(verdict)
|
||||
premises, records = read_premises(case.root, home), read_records(case.root, home, home / ".claude")
|
||||
if isinstance(premises, Failure):
|
||||
|
||||
@@ -10,13 +10,14 @@ import unittest
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from transcript_archive.outcomes import ANSWERED, FINISHED, INTERRUPTED, UNFINISHED
|
||||
from claude_code.entries import AssistantTurn, HumanTurn, LoopPrompt, Text
|
||||
from judge_replay.replay import Case, case_from, report, reproduced, sample, verdict_kind
|
||||
from project_config.test_project_config import configured
|
||||
from loop_effects.effects import Effects
|
||||
from failure import Failure
|
||||
from loop_claude.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished
|
||||
from loop_claude.cycle.judge import TRANSCRIPT_HEADING, Rejected, judge_archived
|
||||
from claude_code.entries import AssistantTurn, HumanTurn, LoopPrompt, Text
|
||||
from judge_replay.archived_prompt import grounds_of
|
||||
from loop_claude.cycle.judge import judge_archived, JUDGE_RULES, Rejected, TASK_HEADING, TRANSCRIPT_HEADING
|
||||
from judge_replay import main
|
||||
from rung_replay.search import Miss, Replay, Tried
|
||||
from model_routing.ladder import Rung
|
||||
@@ -172,8 +173,8 @@ class Main(unittest.TestCase):
|
||||
kept = self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1"
|
||||
prompt = "From the goals file, OLD premises\n\nThe records: none\n\nThe task the worker was given:\nDo it.\n\nWhat the worker did and said, what the human typed on [human] lines and what loop-claude told it on [loop] lines, latest last:\nsaid Verdict: finished\n\nOLD rules"
|
||||
(kept / "judge").mkdir()
|
||||
(kept / "judge" / "run.json").write_text(json.dumps({"prompt": prompt, "reply": {"type": "result"}}))
|
||||
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/run.json\nWorker: x\n")
|
||||
(kept / "judge" / "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json").write_text(json.dumps({"prompt": prompt, "reply": {"type": "result"}}))
|
||||
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json\nWorker: x\n")
|
||||
code, out, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
|
||||
self.assertEqual(code, 0, err)
|
||||
self.assertIn("The reference judge, claude-opus-5-5 at xhigh effort, reproduced 2.", out)
|
||||
@@ -186,7 +187,7 @@ class Main(unittest.TestCase):
|
||||
|
||||
def test_replays_a_verdict_whose_archived_judge_run_cannot_be_read_as_failed(self):
|
||||
kept = self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1"
|
||||
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/gone.json\nWorker: x\n")
|
||||
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.json\nWorker: x\n")
|
||||
code, out, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
|
||||
self.assertEqual(code, 0, err)
|
||||
self.assertIn("The reference judge, claude-opus-5-5 at xhigh effort, reproduced 1. It returned another kind on:", out)
|
||||
@@ -260,19 +261,19 @@ class Main(unittest.TestCase):
|
||||
def test_exits_1_on_a_usage_limit_met_on_an_archived_prompt(self):
|
||||
kept = self.home / ".claude" / "loop-claude-transcripts" / "proj" / "c1"
|
||||
(kept / "judge").mkdir()
|
||||
(kept / "judge" / "run.json").write_text(json.dumps({"prompt": "P", "reply": {}}))
|
||||
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/run.json\nWorker: x\n")
|
||||
(kept / "judge" / "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json").write_text(json.dumps({"prompt": "P", "reply": {}}))
|
||||
(kept / "t0.outcomes.md").write_text("Task: Do it.\n\n2026-10-14T10:00:00Z Verdict: finished\n\nJudge: j\nJudge run: judge/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json\nWorker: x\n")
|
||||
code, _, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: {"type": "result", "api_error_status": 429, "result": "You've hit your limit"})
|
||||
self.assertEqual((code, "usage limit: You've hit your limit" in err), (1, True))
|
||||
|
||||
|
||||
class JudgeArchivedRuns(unittest.TestCase):
|
||||
"""judge_archived on a prompt of two parts, as the archive keeps it: HEAD, the premises, and what follows TRANSCRIPT_HEADING, the transcript the judge was shown."""
|
||||
PROMPT = f"HEAD: Ship fast.\n\n{TRANSCRIPT_HEADING}Worker chose option B.\n[human] Use Postgres.\n"
|
||||
"""judge_archived on a prompt as judge_prompt writes it: the premises, the task, the transcript the judge was shown, and the rules."""
|
||||
PROMPT = f"HEAD: Ship fast.\n\n{TASK_HEADING}Do it.\n\n{TRANSCRIPT_HEADING}Worker chose option B.\n[human] Use Postgres.\n\n{JUDGE_RULES}"
|
||||
ENTRIES = [AssistantTurn(parts=(Text("Worker chose option B."),)), HumanTurn(text="Use Postgres."), LoopPrompt(text="Keep going.")]
|
||||
|
||||
def judged(self, **fields):
|
||||
"""What judge_archived returns where the judge's structured output is fields over an unfinished verdict, and the cwd claude ran in."""
|
||||
def judged(self, prompt=PROMPT, **fields):
|
||||
"""What judge_archived returns on prompt where the judge's structured output is fields over an unfinished verdict, and the cwd claude ran in."""
|
||||
output = {"answer": "", "escalation": "", "evidence": "Worker chose option B.", "finished": False, "interrupt": False, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False, **fields}
|
||||
cwds = []
|
||||
|
||||
@@ -281,7 +282,8 @@ class JudgeArchivedRuns(unittest.TestCase):
|
||||
return subprocess.CompletedProcess(args, 0, json.dumps({"type": "result", "structured_output": output, "total_cost_usd": 0.25}), "")
|
||||
|
||||
effects = Effects(claude=claude, claude_on_terminal=unused, discard_typed=unused, dm=unused, execv=unused, git=unused, input=unused, local_time=unused, memory=unused, now=unused, on_terminal=unused, python=unused, sleep=unused, stream_claude=unused, tmux=unused, tmux_on_terminal=unused)
|
||||
return judge_archived("/code/p", self.PROMPT, self.ENTRIES, REFERENCE, effects), cwds
|
||||
grounds = grounds_of(prompt, self.ENTRIES)
|
||||
return (grounds if isinstance(grounds, Failure) else judge_archived("/code/p", prompt, grounds, len(self.ENTRIES), REFERENCE, effects)), cwds
|
||||
|
||||
def test_runs_in_the_projects_folder_and_stamps_the_verdict_with_the_entries_read(self):
|
||||
verdict, cwds = self.judged()
|
||||
@@ -295,6 +297,25 @@ class JudgeArchivedRuns(unittest.TestCase):
|
||||
rejected, _ = self.judged(evidence="Never said.")
|
||||
self.assertEqual((rejected.cost, rejected.run.prompt), (0.25, self.PROMPT))
|
||||
|
||||
def test_keeps_the_cost_of_a_reply_rejected_for_its_shape(self):
|
||||
rejected, _ = self.judged(finished="yes")
|
||||
self.assertEqual((type(rejected), rejected.cost), (Rejected, 0.25))
|
||||
|
||||
def test_checks_neither_the_task_nor_the_rules_as_the_transcript_or_the_premises(self):
|
||||
self.assertIsInstance(self.judged(evidence=JUDGE_RULES)[0], Rejected)
|
||||
self.assertIsInstance(self.judged(answer="Do it.", question="Which?")[0], Interrupt)
|
||||
|
||||
def test_splits_the_prompt_at_the_first_transcript_heading(self):
|
||||
quoting = self.PROMPT.replace("Worker chose", f"Quoted: {TRANSCRIPT_HEADING}Worker chose", 1)
|
||||
self.assertIsInstance(self.judged(quoting, evidence="Quoted:")[0], Unfinished)
|
||||
|
||||
def test_fails_before_the_judge_runs_on_a_prompt_without_a_heading(self):
|
||||
for prompt in ("No headings at all.", f"{TASK_HEADING}Do it.", f"\n\n{TASK_HEADING}Do it.\n\nNo transcript heading."):
|
||||
with self.subTest(prompt=prompt):
|
||||
failure, cwds = self.judged(prompt)
|
||||
self.assertEqual((type(failure), cwds), (Failure, []))
|
||||
self.assertIn("lacks the heading", failure.reason)
|
||||
|
||||
def test_checks_a_quoted_answer_against_the_premises(self):
|
||||
self.assertIsInstance(self.judged(answer="Ship fast.", question="Which?")[0], QuotedAnswer)
|
||||
self.assertIsInstance(self.judged(answer="Use Postgres.", question="Which?")[0], Interrupt)
|
||||
|
||||
@@ -9,9 +9,9 @@ from loop_effects.effects import Effects
|
||||
from claude_code.entries import Entry, structured_output
|
||||
from loop_claude import escalation_mark
|
||||
from failure import Failure
|
||||
from transcript_archive.judge_run import JudgeRun
|
||||
from loop_claude.verdicts import Finished, Interrupt, Judged, QuotedAnswer, Unfinished, Verdict
|
||||
from claude_code.transcript import human_answers
|
||||
from transcript_archive.judge_run import JudgeRun
|
||||
from loop_claude.quotes import plain, verbatim
|
||||
from loop_claude.cycle.retry import retried
|
||||
from model_routing.ladder import Rung
|
||||
@@ -65,7 +65,8 @@ FINISHED_RULES = (
|
||||
"continue: an open PR, a review not yet run, CI not yet reported, an answer not yet recorded."
|
||||
)
|
||||
JUDGE_RULES = "\n\n".join((INTERRUPT_RULES, QUOTE_RULES, FINISHED_RULES))
|
||||
# Opens the transcript in the prompt; the premises, the records and the task come before it, the rules after the transcript.
|
||||
# Open the task and the transcript in the prompt; the premises and the records come before the task, the rules after the transcript.
|
||||
TASK_HEADING = "The task the worker was given:\n"
|
||||
TRANSCRIPT_HEADING = "What the worker did and said, what the human typed on [human] lines and what loop-claude told it on [loop] lines, latest last:\n"
|
||||
|
||||
|
||||
@@ -163,12 +164,10 @@ def judge_once(root: str, brief: Brief, rung: Rung, effects: Effects) -> Verdict
|
||||
return verdict if isinstance(verdict, Failure | Limited) else escalation_mark.stamped(verdict, len(brief.entries))
|
||||
|
||||
|
||||
def judge_archived(root: str, prompt: str, entries: list[Entry], rung: Rung, effects: Effects) -> Verdict | Failure | Limited:
|
||||
"""One judge run on a prompt the archive kept from an earlier run, as judge-replay replays it: the judge reads what it read then, and its quotes are checked against that prompt's own text, in place of the records of today. The prompt's rules count as part of the transcript, a slack that lets an evidence quote the rules pass."""
|
||||
head, _, shown = prompt.partition(TRANSCRIPT_HEADING)
|
||||
grounds = Grounds(shown, escalation_mark.after_mark(entries, 0), head, tuple(human_answers(entries, typed_answer)))
|
||||
def judge_archived(root: str, prompt: str, grounds: Grounds, entries_read: int, rung: Rung, effects: Effects) -> Verdict | Failure | Limited:
|
||||
"""One judge run on a prompt the archive kept from an earlier run, as judge-replay replays it: the judge reads what it read then, and its quotes are checked against grounds, which judge_replay.archived_prompt makes from that prompt in place of the records of today."""
|
||||
verdict = run_judge(root, judge_args(rung), prompt, grounds, effects)
|
||||
return verdict if isinstance(verdict, Failure | Limited) else escalation_mark.stamped(verdict, len(entries))
|
||||
return verdict if isinstance(verdict, Failure | Limited) else escalation_mark.stamped(verdict, entries_read)
|
||||
|
||||
|
||||
def judge_args(rung: Rung) -> list[str]:
|
||||
@@ -271,7 +270,7 @@ def judge_prompt(brief: Brief) -> tuple[str, Grounds]:
|
||||
head = (
|
||||
f"From the goals file, {brief.premises}\n\n"
|
||||
f"The records: the project's working rules and decisions, then the human's rules for every project:\n{brief.records or 'none'}\n\n"
|
||||
f"The task the worker was given:\n{brief.task}\n\n"
|
||||
f"{TASK_HEADING}{brief.task}\n\n"
|
||||
f"{TRANSCRIPT_HEADING}"
|
||||
)
|
||||
room = max(0, JUDGE_PROMPT_CHARS - len(head) - len(JUDGE_RULES) - 2)
|
||||
|
||||
@@ -3,7 +3,6 @@ import pathlib
|
||||
import typing
|
||||
|
||||
from loop_claude.cycle.states import Active, Archived, Closing, Ended, Event, HumanTalked, HumanTyped, Interrupting, Judging, QUOTED_ANSWERS_PER_CYCLE, Ruled, State, to_the_human, transition, TypedAnswer, WorkerRan, Working
|
||||
from transcript_archive.judge_run import JudgeRun
|
||||
from transcript_archive.write import archive
|
||||
from loop_claude.cycle.worker import AUTO_COMPACT_WINDOW, COMPACT_WINDOW, run_worker, worker_args
|
||||
from loop_claude.cycle.judge import Brief, judge, Rejected
|
||||
@@ -15,6 +14,7 @@ from failure import Failure
|
||||
from loop_claude.verdicts import Finished, Interrupt, Judged, QuotedAnswer, Unfinished, Verdict
|
||||
from loop_claude.cycle.interrupt import interrupt, Talked, WorkerSession
|
||||
from loop_claude.routing import judge_rung, next_route, step_up, StepUp
|
||||
from transcript_archive.judge_run import JudgeRun
|
||||
from model_routing.ladder import Ladder, Rung
|
||||
from transcript_archive.read import last_spent
|
||||
from transcript_archive import outcomes
|
||||
@@ -189,12 +189,12 @@ def ruling_outcome(cycle: Cycle, ruling: Interrupt | QuotedAnswer, quoted_answer
|
||||
"""The archive's outcome for a ruling: interrupted where it goes to the human, with the question put to them, and else answered from the premises or the records."""
|
||||
if isinstance(ruling, QuotedAnswer) and not to_the_human(ruling, quoted_answers):
|
||||
return outcomes.answered(question=ruling.question, sources=prompts.QUOTE_SOURCES, answer=ruling.answer, evidence=ruling.evidence, escalation=ruling.escalation, judge=str(judge_of(cycle)), worker=cycle.route.worker_line(), judge_cost=ruling.judged.cost, annotation=annotation, judge_run=ruling.judged.run)
|
||||
return outcomes.interrupted(question_for(ruling), ruling.evidence, str(judge_of(cycle)), ruling.escalation, cycle.route.worker_line(), ruling.judged.cost, annotation, ruling.judged.run)
|
||||
return outcomes.interrupted(question_for(ruling), ruling.evidence, str(judge_of(cycle)), ruling.escalation, cycle.route.worker_line(), judge_cost=ruling.judged.cost, annotation=annotation, judge_run=ruling.judged.run)
|
||||
|
||||
|
||||
def keep_rejection(cycle: Cycle, transcript_path: str, rejected: Rejected) -> Stop | None:
|
||||
"""Archives a judge reply loop-claude rejected, so the archive shows what the judge returned before a retry or an exit 1."""
|
||||
if unkept := archived(cycle.project_archive, transcript_path, cycle.task, outcomes.rejected(rejected.reason, rejected.reply, str(judge_of(cycle)), rejected.cost, rejected.run), cycle.effects, rejected.run):
|
||||
if unkept := archived(cycle.project_archive, transcript_path, cycle.task, outcomes.rejected(rejected.reason, rejected.reply, str(judge_of(cycle)), judge_cost=rejected.cost, judge_run=rejected.run), cycle.effects, rejected.run):
|
||||
return Stop(1, unkept.reason)
|
||||
return None
|
||||
|
||||
@@ -237,7 +237,7 @@ def close(cycle: Cycle, transcript_path: str, verdict: Finished | Unfinished) ->
|
||||
annotation = entries if isinstance(entries, Failure) else annotation_of(cycle, transcript_path, entries, verdict.judged)
|
||||
if isinstance(annotation, Failure):
|
||||
return Stop(1, annotation.reason)
|
||||
outcome = outcomes.finished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle)), verdict.judged.cost, annotation, verdict.judged.run) if isinstance(verdict, Finished) else outcomes.unfinished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle)), verdict.escalation, verdict.judged.cost, annotation, verdict.judged.run)
|
||||
outcome = outcomes.finished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle)), judge_cost=verdict.judged.cost, annotation=annotation, judge_run=verdict.judged.run) if isinstance(verdict, Finished) else outcomes.unfinished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle)), verdict.escalation, judge_cost=verdict.judged.cost, annotation=annotation, judge_run=verdict.judged.run)
|
||||
if failure := archived(cycle.project_archive, transcript_path, cycle.task, outcome, cycle.effects, verdict.judged.run):
|
||||
return Stop(1, failure.reason)
|
||||
if isinstance(verdict, Finished) and (failure := delete_session(cycle.session)):
|
||||
|
||||
@@ -394,12 +394,12 @@ Worker: {OFF_LADDER}
|
||||
self.assertEqual(run.run_cycle(self.cycle(fake)).verdict, Finished("working"))
|
||||
prompts = [stdin for args, stdin in fake.made("claude")[:3]]
|
||||
[chunk] = self.archive.iterdir()
|
||||
kept = "\n".join(path.read_text() for path in sorted(chunk.glob("*.outcomes.md"), key=lambda path: path.stat().st_mtime_ns))
|
||||
kept = "\n".join(path.read_text() for path in chunk.glob("*.outcomes.md"))
|
||||
named = re.findall(r"^Judge run: (judge/[0-9a-f]{64}\.json)$", kept, re.MULTILINE)
|
||||
self.assertEqual(len(named), 3, kept)
|
||||
runs = [json.loads((chunk / name).read_text()) for name in named]
|
||||
self.assertEqual([found["prompt"] for found in runs], prompts)
|
||||
self.assertEqual([found["reply"] for found in runs], [json.loads(reply.stdout) for reply in (rejected, interrupted, finished)])
|
||||
self.assertEqual(sorted(found["prompt"] for found in runs), sorted(prompts))
|
||||
self.assertEqual(sorted(json.dumps(found["reply"]) for found in runs), sorted(json.dumps(json.loads(reply.stdout)) for reply in (rejected, interrupted, finished)))
|
||||
self.assertEqual(sorted(path.name for path in (chunk / "judge").iterdir()), sorted(name.removeprefix("judge/") for name in named))
|
||||
|
||||
def test_stops_when_a_judge_run_cannot_be_archived(self):
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
"""The verdicts cycle.judge.verdict_from builds from the judge's reply; cycle.run.rule then stamps every verdict but Finished with the items the chunk takes. They sit outside cycle/, since routing reads them and cycle imports routing."""
|
||||
import dataclasses
|
||||
|
||||
from claude_code.tokens import Tokens
|
||||
from transcript_archive.judge_run import JudgeRun
|
||||
from claude_code.tokens import Tokens
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
|
||||
@@ -27,6 +27,9 @@ class JudgeRun:
|
||||
def read(path: pathlib.Path) -> "JudgeRun | Failure":
|
||||
try:
|
||||
kept = json.loads(path.read_text(encoding="utf-8"))
|
||||
return JudgeRun(kept["prompt"], kept["reply"])
|
||||
prompt, reply = kept["prompt"], kept["reply"]
|
||||
except (OSError, ValueError, KeyError, TypeError) as failure:
|
||||
return Failure(f"could not read the archived judge run {path}: {failure}")
|
||||
if not (isinstance(prompt, str) and isinstance(reply, dict)):
|
||||
return Failure(f"could not read the archived judge run {path}: its prompt is no string or its reply no object")
|
||||
return JudgeRun(prompt, reply)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import re
|
||||
import typing
|
||||
|
||||
from transcript_archive.judge_run import FOLDER, JudgeRun
|
||||
from model_routing.prices import PRICES
|
||||
from text_files.markdown_text import quoted
|
||||
from text_files.utc_stamp import STAMPED
|
||||
from claude_code.tokens import Tokens
|
||||
from transcript_archive.judge_run import JudgeRun
|
||||
from model_routing.route import WorkerLine
|
||||
|
||||
TASK = "Task: "
|
||||
@@ -32,7 +32,7 @@ JUDGE_COST = "Judge cost: "
|
||||
JUDGE_COST_AT = re.compile(rf"^{re.escape(JUDGE)}(.+)\n{re.escape(JUDGE_COST)}([0-9.]+)$", re.MULTILINE)
|
||||
# The judge run the verdict came from, as a path in the chunk's folder (judge_run.JudgeRun.file), on the line after the judge's rung and cost; absent where the archive kept none.
|
||||
JUDGE_RUN = "Judge run: "
|
||||
JUDGE_RUN_AT = re.compile(rf"^{re.escape(JUDGE_RUN)}(.+)$", re.MULTILINE)
|
||||
JUDGE_RUN_AT = re.compile(rf"^{re.escape(JUDGE_RUN)}({FOLDER}/[0-9a-f]{{64}}\.json)$", re.MULTILINE)
|
||||
|
||||
|
||||
# The claude-config release and commit that ran the cycle, on the line before JUDGE, such as `Config: v0.8.0 (<sha>)`.
|
||||
@@ -111,17 +111,17 @@ def escalation_block(escalation: str) -> str:
|
||||
return f"Escalation:\n{quoted(escalation)}\n\n" if escalation else ""
|
||||
|
||||
|
||||
def finished(worker: WorkerLine, evidence: str, judge: str, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
|
||||
def finished(worker: WorkerLine, evidence: str, judge: str, *, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
|
||||
return f"{FINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{annotation_lines(annotation)}{judge_lines(judge, judge_cost, judge_run)}\n{WORKER}{worker}"
|
||||
|
||||
|
||||
def unfinished(worker: WorkerLine, evidence: str, judge: str, escalation: str, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
|
||||
def unfinished(worker: WorkerLine, evidence: str, judge: str, escalation: str, *, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
|
||||
"""escalation is the judge's quote showing that the worker's cited escalation trigger fired; empty for none."""
|
||||
escalated = escalation_block(escalation)
|
||||
return f"{UNFINISHED}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost, judge_run)}\n{WORKER}{worker}"
|
||||
|
||||
|
||||
def interrupted(question: str, evidence: str, judge: str, escalation: str, worker: WorkerLine, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
|
||||
def interrupted(question: str, evidence: str, judge: str, escalation: str, worker: WorkerLine, *, judge_cost: float = 0.0, annotation: Annotation = Annotation(), judge_run: JudgeRun | None = None) -> str:
|
||||
"""escalation is the judge's quote showing that the worker's cited escalation trigger fired; empty for none. worker is the line naming the route the worker ran on until the interrupt."""
|
||||
escalated = escalation_block(escalation)
|
||||
return f"{INTERRUPTED}\n\n{question}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost, judge_run)}\n{WORKER}{worker}"
|
||||
@@ -133,7 +133,7 @@ def answered(*, question: str, sources: str, answer: str, evidence: str, escalat
|
||||
return f"{ANSWERED}\n\n{question}\n\n{sources} says:\n{quoted(answer)}\n\nEvidence:\n{quoted(evidence)}\n\n{escalated}{annotation_lines(annotation)}{judge_lines(judge, judge_cost, judge_run)}\n{WORKER}{worker}"
|
||||
|
||||
|
||||
def rejected(why: str, reply: str, judge: str, judge_cost: float = 0.0, judge_run: JudgeRun | None = None) -> str:
|
||||
def rejected(why: str, reply: str, judge: str, *, judge_cost: float = 0.0, judge_run: JudgeRun | None = None) -> str:
|
||||
"""A judge reply loop-claude rejected, with the reason and the reply as json."""
|
||||
return f"{REJECTED}: {why}\n\nReply:\n{quoted(reply)}\n\n{judge_lines(judge, judge_cost, judge_run)}"
|
||||
|
||||
|
||||
@@ -11,8 +11,8 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from transcript_archive.project_key import checkout_of, project_key
|
||||
from failure import Failure
|
||||
from transcript_archive import outcomes, read, write
|
||||
from transcript_archive.judge_run import JudgeRun
|
||||
from transcript_archive import outcomes, read, write
|
||||
from transcript_archive.location import project_archive
|
||||
from text_files.utc_stamp import utc
|
||||
|
||||
@@ -66,6 +66,18 @@ class JudgeRuns(unittest.TestCase):
|
||||
run = JudgeRun("Prompt é.", {"a": 1})
|
||||
self.assertEqual(run.content(), b'{\n "prompt": "Prompt \xc3\xa9.",\n "reply": {\n "a": 1\n }\n}')
|
||||
|
||||
def test_reads_a_file_that_is_no_run_as_a_failure(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
for text in ("not json", '{"prompt": "P"}', '{"prompt": 1, "reply": {}}', '{"prompt": "P", "reply": []}'):
|
||||
with self.subTest(text=text):
|
||||
path = pathlib.Path(root, "run.json")
|
||||
path.write_text(text)
|
||||
self.assertRegex(JudgeRun.read(path).reason, f"^could not read the archived judge run {path}: ")
|
||||
|
||||
def test_reads_no_path_that_leaves_the_judge_folder_from_an_outcome(self):
|
||||
outcome = f"{utc(NOW)} Verdict: finished\n\nJudge run: judge/../../secret.json\n"
|
||||
self.assertIsNone(outcomes.judge_run_at(outcome, utc(NOW)))
|
||||
|
||||
def test_names_the_run_on_each_kind_of_outcome_that_has_a_judge(self):
|
||||
named = f"Judge: j\nJudge run: {self.judged.file}\n"
|
||||
kept = [
|
||||
|
||||
@@ -4,8 +4,8 @@ import pathlib
|
||||
import typing
|
||||
|
||||
from transcript_archive.location import chunk_folder, outcomes_of
|
||||
from transcript_archive.judge_run import JudgeRun
|
||||
from failure import Failure
|
||||
from transcript_archive.judge_run import JudgeRun
|
||||
from text_files.file_writes import make_private, replace_file
|
||||
from claude_code.session import session_files
|
||||
from transcript_archive.outcomes import TASK
|
||||
|
||||
Reference in New Issue
Block a user