Files
claude-config/bin/agent_cycle/judge.py
T

315 lines
20 KiB
Python

import dataclasses
import json
import re
import typing
from agent_cycle.worker_prompts import answers_recorded, typed_answer
from claude_code.reply import claude_reply, Limited
from loop_effects.effects import Effects
from claude_code.entries import Entry, structured_output
from agent_cycle import escalation_mark
from failure import Failure
from agent_cycle.verdicts import Finished, Interrupt, Judged, QuotedAnswer, Unfinished, Verdict
from claude_code.transcript import human_answers
from transcript_archive.judge_run import JudgeRun
from agent_cycle.quotes import plain, verbatim
from agent_cycle.retry import retried
from model_routing.ladder import Rung
from agent_cycle.stop import Stop
from claude_code.tokens import tokens_of
# At 3 to 4 characters a token, 100,000 to 133,000 tokens, which leaves a 200,000-token context room for the reply; the largest session rendered here was 364,000 characters on 2026-10-03.
JUDGE_PROMPT_CHARS = 400000
INTERRUPT_RULES = (
"""Decide whether the human must now be interrupted. ~/.claude/CLAUDE.md → "Working loop", "Ask, don't guess" says """
"which choices are reserved for the human. interrupt=true when any of these appear:\n"
"- the worker asks the human about a reserved choice, or makes one itself, and neither the premises nor the "
"records settle it; a choice the worker names a rule as asking the human's approval for counts as reserved, "
"since some such rules, such as larv-review's, lie outside the records\n"
"- it asks the human anything else without recommending an answer, and neither the premises "
"nor the records settle the matter\n"
"- it asks again a question a [loop] line already answered\n"
"- it is blocked on something only the human can supply: a credential, a version bump, a choice "
"outside the repo, or a release ready to cut, which only the human may publish\n"
"- it flags that the goals need changing\n"
"- it flags that the plan or the task itself is unclear or missing, and recommends no way on\n\n"
"interrupt=false otherwise. A choice in the repo that no rule reserves for the human is the worker's to make, "
"so its taking its own recommendation on one is no reason to interrupt. Where it asks about such a choice and "
"recommends an answer, quote the sentence of that rule telling it to take the option it would recommend, even "
"where a [loop] line quoted that sentence for another question. A run that ends {asks_higher_rung} "
"is no reason to interrupt either; one that asks the human's yes to a higher rung is.\n\n"
"The human may have replied inside the transcript, on a [human] line: judge only what "
"is still open, never a question they already answered there. Where a [human] line from earlier in the "
"chunk answered the question the worker asks now, interrupt only if that line's own words leave open what "
"the worker asks: quote the line in answered_before, those words in left_open, and end the question with them. "
"Otherwise interrupt=false and finished=false, with that line as the evidence: the worker is shown it as the "
"work remaining.\n\n"
"Where the worker asked or flagged something itself, the question is that one, in the worker's own "
"wording; never construct another from facts it stated, except to end it with what an earlier answer left open."
)
QUOTE_RULES = (
"Where the premises or the records answer the worker's own question and no [loop] line has "
"quoted that answer yet, set interrupt=false and quote the passage that answers it; the worker is "
"then given that passage in place of the human's reply. An interrupt outranks a quoted answer, and either "
"outranks finished."
)
FINISHED_RULES = (
"The chunk is the piece of work the worker is on, shipped as one pull request.\n\n"
"finished=true when the chunk is complete: merged, or nothing remains that the worker could do on "
"its own. Even then, it is finished only once {answers_recorded}.\n\n"
"An answer counts as recorded where the premises or the records above already hold it, the human's rules "
"(~/.claude/CLAUDE.md, ~/.claude/rules/, the project's review standards, each headed by its path) included. "
"It also counts where the transcript shows the worker writing it into {answer_destinations}, "
"but not into one of the human's rules.\n\n"
"finished=false when work remains that a further run can "
"continue: an open PR, a review not yet run, CI not yet reported, an answer not yet recorded."
)
def judge_rules(answer_destinations: str, driver: str) -> str:
"""The rules that end the judge's prompt. answer_destinations names where the worker of driver, the entry point, may record an answer the human typed."""
return "\n\n".join((INTERRUPT_RULES.format(asks_higher_rung=escalation_mark.asks_higher_rung(driver)), QUOTE_RULES, FINISHED_RULES.format(answers_recorded=answers_recorded(answer_destinations), answer_destinations=answer_destinations)))
# Open the task and the transcript in the prompt; the premises and the records come before the task, the rules after the transcript.
# judge_replay.archived_prompt splits each archived prompt on them, so rewording one makes every run archived before it replay as failed.
TASK_HEADING = "The task the worker was given:\n"
# Between the task and the tail that quotes_checked searches, so that no fragment of a quote runs across the join. plain() keeps it, and the judge's quotes hold no NUL.
PASSAGE_BREAK = "\x00"
def transcript_heading(driver: str) -> str:
return f"What the worker did and said, what the human typed on [human] lines and what {driver} told it on [loop] lines, latest last:\n"
@dataclasses.dataclass(frozen=True)
class Reply:
answer: str = dataclasses.field(metadata={"description": "With interrupt false, the passage of the premises or the records that answers the worker's question, quoted verbatim, without elisions. Empty otherwise."})
# Routes the worker one rung up, as the judge interrupts or quotes an answer, or on the next cycle: agent_cycle/escalation_mark.py.
escalation: str = dataclasses.field(metadata={"description": "Empty unless the chunk is unfinished, interrupt is true or answer quotes a passage, and the worker's run ends {asks_higher_rung}, citing an escalation trigger. Then: a verbatim quote, without elisions, of the transcript passage where the trigger fired, such as a review round's commit title or the worker's report of the finding or failing test. {quote_after_step} Never a later mention of the trigger, and never the request itself."})
evidence: str = dataclasses.field(metadata={"description": "A verbatim quote of the one passage of the task or the transcript the verdict rests on, without elisions: what triggers the interrupt; for a quoted answer, the worker's question; else what shows the chunk finished or the work remaining."})
finished: bool = dataclasses.field(metadata={"description": "Whether the chunk is finished."})
interrupt: bool = dataclasses.field(metadata={"description": "Whether the human must now be interrupted."})
answered_before: str = dataclasses.field(metadata={"description": "With interrupt true, a [human] line from earlier in the chunk that answered the question the worker asks now, quoted verbatim, without elisions. Empty otherwise."})
left_open: str = dataclasses.field(metadata={"description": "With answered_before set, the words of that line that leave open what the question asks, quoted verbatim; the question ends with them. Empty otherwise."})
question: str = dataclasses.field(metadata={"description": "With interrupt true, the single question to put to the human. With a quoted answer, the worker's question that the answer field answers. Empty otherwise."})
# True opens claude's session at once, in place of the one-line answer (run.ask). The judge decides it, since loop-claude computes the plan's standing (release_standing.chunk_item) only when it routes a chunk on the ladder afresh: it would miss a release ready to cut in a resumed chunk or off the ladder. A wrong ruling costs the human one typed /talk, or answering in claude, which stops the loop if they leave it without typing anything.
release_ready: bool = dataclasses.field(metadata={"description": "With interrupt true, whether the question is about a release ready to cut, which only the human may publish. False otherwise."})
# Finds the answer that reverses a choice the worker took on its own recommendation. score-outcomes counts these answers per chunk: they are the counter-signal the 2026-11-08 entry of docs/decisions.md → `loop-claude: a judge decides each interrupt and each quoted answer…` stays valid by.
reversed: str = dataclasses.field(metadata={"description": "The latest [human] line that overrides a choice the worker took on its own recommendation and recorded with its reason, quoted verbatim, without elisions. Empty where none does; a line that answers a question the worker left open doesn't override anything."})
REPLY_TYPES = typing.get_type_hints(Reply)
JSON_TYPES: dict[object, dict[str, typing.Any]] = {bool: {"type": "boolean"}, int: {"type": "integer"}, str: {"type": "string"}}
def judge_schema(driver: str) -> dict[str, typing.Any]:
"""The schema of the judge's reply, each field described from Reply's metadata, with the words that name driver, the entry point, filled in."""
asks, quote = escalation_mark.asks_higher_rung(driver), escalation_mark.quote_after_step(driver)
described = {field.name: field.metadata["description"].replace("{asks_higher_rung}", asks).replace("{quote_after_step}", quote) for field in dataclasses.fields(Reply)}
return {
"properties": {name: {"description": description, **JSON_TYPES[REPLY_TYPES[name]]} for name, description in described.items()},
"required": list(REPLY_TYPES),
"type": "object",
}
@dataclasses.dataclass(frozen=True)
class Rejected(Failure):
"""A judge reply loop-claude rejected: reason says why, run holds the prompt and the reply, and cost is what the run cost in dollars, 0 where its reply records none."""
run: JudgeRun
cost: float = 0.0
@property
def reply(self) -> str:
"""The reply as json."""
return json.dumps(self.run.reply, ensure_ascii=False)
class Grounds(typing.NamedTuple):
"""The texts quotes_checked checks the judge's quotes against. A quote must appear in its text as written, since a quote reworded or stitched together can say what its source does not; only the evidence may join passages with an ellipsis, each passage in order."""
# The transcript as the judge saw it.
shown: str
# The worker's own lines after the escalation mark, as escalation_mark.after_mark renders them, including those the tail in shown cuts off.
after_mark: str
# The premises and the records.
quotable: str
# Each answer the human typed, whole, latest last (transcript.human_answers).
typed: tuple[str, ...] = ()
# The worker's task, which the judge reads above shown and may quote as the evidence: the worker's first prompt holds it, but the tail in shown cuts that off once the session grows.
task: str = ""
class Brief(typing.NamedTuple):
"""What the judge reads: the worker's task, the premises and records the worker is judged by, and the transcript's entries."""
task: str
premises: str
records: str
# What the records are, in their order, as WorkerPrompts.records_named has it.
records_named: str
# Where the worker may record an answer the human typed, as the entry point names it.
answer_destinations: str
entries: list[Entry]
# How many of the entries the session held when an escalation last stepped the worker up a rung, 0 before any.
escalated_at: int
# The entry point's name, such as `do-epic`, as WorkerPrompts.driver has it.
driver: str
def verdict_from(output: object) -> Verdict | Failure:
if not isinstance(output, dict):
return Failure("the judge returned no reply")
for name, kind in REPLY_TYPES.items():
if not fits(output.get(name), kind):
return Failure(f"the judge's reply has no {name} of type {JSON_TYPES[kind]['type']}")
if output["interrupt"]:
if not output["question"].strip():
return Failure("the judge interrupted without a question")
return Interrupt(output["evidence"], output["question"], escalation=output["escalation"], release_ready=output["release_ready"], answered_before=output["answered_before"], left_open=output["left_open"])
if output["answer"].strip():
if not output["question"].strip():
return Failure("the judge quoted an answer without a question")
return QuotedAnswer(output["evidence"], output["question"], output["answer"], escalation=output["escalation"])
if output["finished"]:
return Finished(output["evidence"])
return Unfinished(output["evidence"], escalation=output["escalation"])
def fits(value: object, kind: object) -> bool:
"""Whether value, read from the judge's JSON, is of kind. type(), not isinstance(): JSON true is no integer."""
return type(value) is kind
def judge(root: str, brief: Brief, rung: Rung, effects: Effects, keep: typing.Callable[[Rejected], Stop | None]) -> Verdict | Stop:
return retried("judge", lambda: judge_once(root, brief, rung, effects), effects, lambda failure: keep(failure) if isinstance(failure, Rejected) else None)
def judge_once(root: str, brief: Brief, rung: Rung, effects: Effects) -> Verdict | Failure | Limited:
"""One judge run on the brief, as judge retries it; judge-replay replays it."""
prompt, grounds = judge_prompt(brief)
verdict = run_judge(root, judge_args(rung, brief.driver), prompt, grounds, effects)
return verdict if isinstance(verdict, Failure | Limited) else escalation_mark.stamped(verdict, len(brief.entries))
def judge_archived(root: str, prompt: str, grounds: Grounds, entries_read: int, rung: Rung, driver: str, effects: Effects) -> Verdict | Failure | Limited:
"""One judge run on a prompt the archive kept from an earlier run, as judge-replay replays it: the judge reads what it read then, and its quotes are checked against grounds, which judge_replay.archived_prompt makes from that prompt in place of the records of today."""
verdict = run_judge(root, judge_args(rung, driver), prompt, grounds, effects)
return verdict if isinstance(verdict, Failure | Limited) else escalation_mark.stamped(verdict, entries_read)
def judge_args(rung: Rung, driver: str) -> list[str]:
# --json-schema has claude check its reply against the schema and return it as the json result's structured_output.
# No tools, so the verdict rests on the transcript grounded() checks it against.
return [
"--json-schema", json.dumps(judge_schema(driver)),
*rung.flags(),
"--tools", "",
]
def run_judge(root: str, args: list[str], prompt: str, grounds: Grounds, effects: Effects) -> Verdict | Failure | Limited:
"""One judge run, each quote in its verdict checked against grounds as Grounds says."""
reply = claude_reply(effects.claude, args, root, prompt, "the judge")
if isinstance(reply, Failure | Limited):
return reply
output = structured_output(reply)
verdict = verdict_from(output)
run = JudgeRun(prompt, reply)
cost = cost_of(reply)
if isinstance(verdict, Failure):
return Rejected(verdict.reason, run, cost)
checked = quotes_checked(verdict, grounds)
if isinstance(checked, Failure):
return Rejected(checked.reason, run, cost)
return dataclasses.replace(checked, judged=Judged(cost, tokens_of(reply.get("usage")), reversal_of(output, grounds.typed), run))
def reversal_of(output: object, typed: tuple[str, ...]) -> str:
"""The latest answer the human typed that holds the reply's reversed quote, whole, so the same answer archives the same way however the judge quotes it; empty where none does. verdict_from has checked the field is a str."""
quote = output.get("reversed") if isinstance(output, dict) else None
return next((answer for answer in reversed(typed) if isinstance(quote, str) and verbatim(quote, answer)), "")
def cost_of(reply: dict) -> float:
"""What the run behind claude's json reply cost in dollars; 0 where the reply records none."""
cost = reply.get("total_cost_usd")
return float(cost) if isinstance(cost, int | float) and not isinstance(cost, bool) and cost > 0 else 0.0
def quotes_checked(verdict: Verdict, grounds: Grounds) -> Verdict | Failure:
"""The verdict with each of its quotes checked against the text it must appear in, as written:
| Quote | Must be in | Where it isn't | Checked by |
|---|---|---|---|
| the evidence | grounds.task then grounds.shown, the evidence's fragments in order | a Failure: run_judge rejects the reply, and judge retries it | grounded |
| an interrupt's answered_before | grounds.shown | the interrupt stands, as if no earlier answer was found | re_ask_checked |
| its left_open | answered_before and the question | the interrupt becomes an Unfinished that quotes answered_before back to the worker | re_ask_checked |
| a quoted answer | grounds.quotable | the question goes to the human as an interrupt, which keeps the escalation the answer carried | answer_checked |
| the escalation | grounds.shown and grounds.after_mark | the escalation is dropped, so the worker's rung holds | escalation_mark.checked |
| the reversal | one of grounds.typed | the reversal is dropped; run_judge, after this check | reversal_of |
Only a misquoted evidence fails. A Failure for any other would exit an unattended loop, so each changes the verdict instead.
"""
if not grounded(verdict.evidence, f"{grounds.task}{PASSAGE_BREAK}{grounds.shown}"):
return misquoted_evidence(verdict)
verdict = re_ask_checked(verdict, grounds.shown)
verdict = answer_checked(verdict, grounds.quotable)
return escalation_mark.checked(verdict, after=grounds.after_mark, shown=grounds.shown)
def grounded(evidence: str, searched: str) -> bool:
"""Whether the searched text holds the evidence word for word, once emphasis marks are stripped and whitespace is reflowed in both. The judge joins passages with an ellipsis though told to quote one, so each fragment between ellipses must come after the one before it."""
searched = plain(searched)
fragments = [fragment.strip() for fragment in re.split(r"\.\.\.|…", plain(evidence)) if fragment.strip()]
after = 0
for fragment in fragments:
at = searched.find(fragment, after)
if at < 0:
return False
after = at + len(fragment)
return bool(fragments)
def misquoted_evidence(verdict: Verdict) -> Failure:
asked = f", for the question: {verdict.question}" if isinstance(verdict, Interrupt | QuotedAnswer) else ""
return Failure(f"the judge quoted {verdict.evidence!r}, which is not in the task or the transcript{asked}")
def re_ask_checked(verdict: Verdict, shown: str) -> Verdict:
"""The verdict, or an Unfinished quoting the human's earlier answer back to the worker, where it is an Interrupt that re-asks what that answer settles: its left_open isn't in both the answer and the question."""
if not isinstance(verdict, Interrupt) or not verbatim(verdict.answered_before, shown):
return verdict
if verbatim(verdict.left_open, verdict.answered_before) and verbatim(verdict.left_open, verdict.question):
return verdict
return Unfinished(verdict.answered_before, escalation=verdict.escalation)
def answer_checked(verdict: Verdict, quotable: str) -> Verdict:
"""The verdict; where it is a quoted answer that the premises and the records don't hold, an Interrupt on the same question instead."""
if isinstance(verdict, QuotedAnswer) and not verbatim(verdict.answer, quotable):
return Interrupt(verdict.evidence, verdict.question, escalation=verdict.escalation)
return verdict
def judge_prompt(brief: Brief) -> tuple[str, Grounds]:
"""The prompt, and what the judge's quotes are checked against."""
head = (
f"From the goals file, {brief.premises}\n\n"
f"The records: {brief.records_named}:\n{brief.records or 'none'}\n\n"
f"{TASK_HEADING}{brief.task}\n\n"
f"{transcript_heading(brief.driver)}"
)
rules = judge_rules(brief.answer_destinations, brief.driver)
room = max(0, JUDGE_PROMPT_CHARS - len(head) - len(rules) - 2)
transcript = escalation_mark.transcript(brief.entries, brief.escalated_at, typed_answer, brief.driver)
# The tail, where an interrupt or a finish lands; the session's transcript grows cycle over cycle.
shown = transcript[max(0, len(transcript) - room):]
grounds = Grounds(
shown=shown,
after_mark=escalation_mark.after_mark(brief.entries, brief.escalated_at),
quotable=f"{brief.premises}\n\n{brief.records}",
typed=tuple(human_answers(brief.entries, typed_answer)),
task=brief.task,
)
return f"{head}{shown}\n\n{rules}", grounds