Review the judge's fix for quoting the worker's task as evidence, and close the gaps that review found #647

Merged
lilleman merged 9 commits from item-460 into main 2026-11-20 00:15:30 +01:00
9 changed files with 28 additions and 29 deletions
+9 -7
View File
@@ -74,6 +74,8 @@ def judge_rules(answer_destinations: str, driver: str) -> str:
# Open the task and the transcript in the prompt; the premises and the records come before the task, the rules after the transcript.
# judge_replay.archived_prompt splits each archived prompt on them, so rewording one makes every run archived before it replay as failed.
TASK_HEADING = "The task the worker was given:\n"
# Between the task and the tail that quotes_checked searches, so that no fragment of a quote runs across the join. plain() keeps it, and the judge's quotes hold no NUL.
PASSAGE_BREAK = "\x00"
def transcript_heading(driver: str) -> str:
@@ -85,7 +87,7 @@ class Reply:
answer: str = dataclasses.field(metadata={"description": "With interrupt false, the passage of the premises or the records that answers the worker's question, quoted verbatim, without elisions. Empty otherwise."})
# Routes the worker one rung up, as the judge interrupts or quotes an answer, or on the next cycle: agent_cycle/escalation_mark.py.
escalation: str = dataclasses.field(metadata={"description": "Empty unless the chunk is unfinished, interrupt is true or answer quotes a passage, and the worker's run ends {asks_higher_rung}, citing an escalation trigger. Then: a verbatim quote, without elisions, of the transcript passage where the trigger fired, such as a review round's commit title or the worker's report of the finding or failing test. {quote_after_step} Never a later mention of the trigger, and never the request itself."})
evidence: str = dataclasses.field(metadata={"description": "A verbatim quote of the one transcript passage the verdict rests on, without elisions: what triggers the interrupt; for a quoted answer, the worker's question; else what shows the chunk finished or the work remaining."})
evidence: str = dataclasses.field(metadata={"description": "A verbatim quote of the one passage of the task or the transcript the verdict rests on, without elisions: what triggers the interrupt; for a quoted answer, the worker's question; else what shows the chunk finished or the work remaining."})
finished: bool = dataclasses.field(metadata={"description": "Whether the chunk is finished."})
interrupt: bool = dataclasses.field(metadata={"description": "Whether the human must now be interrupted."})
answered_before: str = dataclasses.field(metadata={"description": "With interrupt true, a [human] line from earlier in the chunk that answered the question the worker asks now, quoted verbatim, without elisions. Empty otherwise."})
@@ -248,20 +250,20 @@ def quotes_checked(verdict: Verdict, grounds: Grounds) -> Verdict | Failure:
Only a misquoted evidence fails. A Failure for any other would exit an unattended loop, so each changes the verdict instead.
"""
if not grounded(verdict.evidence, f"{grounds.task}\n\n{grounds.shown}"):
if not grounded(verdict.evidence, f"{grounds.task}{PASSAGE_BREAK}{grounds.shown}"):
return misquoted_evidence(verdict)
verdict = re_ask_checked(verdict, grounds.shown)
verdict = answer_checked(verdict, grounds.quotable)
return escalation_mark.checked(verdict, after=grounds.after_mark, shown=grounds.shown)
def grounded(evidence: str, rendered_transcript: str) -> bool:
"""Whether the transcript holds the evidence word for word, once emphasis marks are stripped and whitespace is reflowed in both. The judge joins passages with an ellipsis though told to quote one, so each fragment between ellipses must come after the one before it."""
transcript = plain(rendered_transcript)
def grounded(evidence: str, searched: str) -> bool:
"""Whether the searched text holds the evidence word for word, once emphasis marks are stripped and whitespace is reflowed in both. The judge joins passages with an ellipsis though told to quote one, so each fragment between ellipses must come after the one before it."""
searched = plain(searched)
fragments = [fragment.strip() for fragment in re.split(r"\.\.\.|…", plain(evidence)) if fragment.strip()]
after = 0
for fragment in fragments:
at = transcript.find(fragment, after)
at = searched.find(fragment, after)
if at < 0:
return False
after = at + len(fragment)
@@ -271,7 +273,7 @@ def grounded(evidence: str, rendered_transcript: str) -> bool:
def misquoted_evidence(verdict: Verdict) -> Failure:
asked = f", for the question: {verdict.question}" if isinstance(verdict, Interrupt | QuotedAnswer) else ""
return Failure(f"the judge quoted {verdict.evidence!r}, which is not in the transcript{asked}")
return Failure(f"the judge quoted {verdict.evidence!r}, which is not in the task or the transcript{asked}")
def re_ask_checked(verdict: Verdict, shown: str) -> Verdict:
+2 -1
View File
@@ -124,13 +124,14 @@ class Verdict(unittest.TestCase):
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", "the project's decisions", "the plan or a comment", kinds(*entries), 1, "loop-claude"), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", evidence="round 2: 🔴")]).effects())
self.assertEqual(verdict, Unfinished("round 2: 🔴", entries_read=2))
def test_takes_the_task_as_evidence_where_the_tail_it_shows_cuts_off_the_workers_first_prompt(self):
def test_takes_the_task_as_evidence_where_the_tail_it_shows_cuts_off_the_workers_first_prompt_but_no_quote_across_the_join(self):
entries = [human("Do the next chunk. " + "x" * judge.JUDGE_PROMPT_CHARS), assistant({"type": "text", "text": "Merged."})]
def judged_on(evidence):
return judge.judge_once("/r", judge.Brief("Do the next chunk.", "## Goals", "", "the project's decisions", "the plan or a comment", kinds(*entries), 0, "loop-claude"), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", evidence=evidence)] * 3).effects())
self.assertEqual(judged_on("Do the next chunk."), Unfinished("Do the next chunk.", entries_read=2))
self.assertEqual(judged_on("Do the next chunk. ... Merged."), Unfinished("Do the next chunk. ... Merged.", entries_read=2))
self.assertIsInstance(judged_on("Do the next release."), Failure)
self.assertIsInstance(judged_on("next chunk. xxx"), Failure)
class JudgePrompt(unittest.TestCase):
+1 -1
View File
@@ -188,7 +188,7 @@ Worker: {OFF_LADDER}
self.assertEqual(self.archived_outcomes(), [
"Task: Do it.\n\n"
f"{stamp} Judge reply rejected: the judge's reply has no answer of type string\n\nReply:\n> {unfilled.stdout}\n\nJudge: judge-model\n\n"
f"{stamp} Judge reply rejected: the judge quoted 'never said', which is not in the transcript\n\nReply:\n> {misquoted.stdout}\n\nJudge: judge-model\n\n"
f"{stamp} Judge reply rejected: the judge quoted 'never said', which is not in the task or the transcript\n\nReply:\n> {misquoted.stdout}\n\nJudge: judge-model\n\n"
f"{stamp} Verdict: finished\n\nEvidence:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
def test_archives_each_judge_runs_prompt_and_reply_beside_the_rejection_interrupt_and_verdict_it_gave(self):
+1 -1
View File
@@ -9,7 +9,7 @@ from agent_cycle.worker_prompts import typed_answer
def grounds_of(prompt: str, entries: list[Entry]) -> Grounds | Failure:
"""What the quotes of a judge run on prompt are checked against, as judge_prompt would have made it for the transcript's entries: the premises and the records before the task are quotable, the transcript the judge was shown comes before the rules. The prompt is split on the headings judge_prompt writes, at the first of each; a Failure where it lacks one. Rules of an earlier wording stay part of the transcript, a slack that lets an evidence quote them pass."""
"""What the quotes of a judge run on prompt are checked against, as judge_prompt would have made it for the transcript's entries: the premises and the records before the task are quotable, the task is quotable as the evidence, and the transcript the judge was shown comes before the rules. The prompt is split on the headings judge_prompt writes, at the first of each; a Failure where it lacks one. Rules of an earlier wording stay part of the transcript, a slack that lets an evidence quote them pass."""
before_task, task_heading, after_task = prompt.partition(f"\n\n{TASK_HEADING}")
heading = transcript_heading(LOOP_CLAUDE)
task, found_heading, shown = after_task.partition(heading)
+8
View File
@@ -403,6 +403,14 @@ class JudgeArchivedRuns(unittest.TestCase):
self.assertIsInstance(self.judged(evidence="Do it.")[0], Unfinished)
self.assertIsInstance(self.judged(evidence="Do it. ... Use Postgres.")[0], Unfinished)
def test_splits_the_prompt_into_the_premises_the_task_and_the_transcript_at_the_first_heading_of_each(self):
heading = transcript_heading(LOOP_CLAUDE)
again = self.PROMPT.replace("Worker chose option B.", f"Worker chose option B. {heading}Again.", 1)
for prompt in (self.PROMPT, again):
grounds = grounds_of(prompt, self.ENTRIES)
self.assertEqual((grounds.quotable, grounds.task), ("HEAD: Ship fast.", "Do it."))
self.assertEqual(grounds_of(self.PROMPT, self.ENTRIES).shown, "Worker chose option B.\n[human] Use Postgres.")
def test_keeps_the_cost_and_the_run_of_a_rejected_reply(self):
rejected, _ = self.judged(evidence="Never said.")
self.assertEqual((rejected.cost, rejected.run.prompt), (0.25, self.PROMPT))
+2 -2
View File
@@ -99,8 +99,8 @@ class JudgeFailures(unittest.TestCase):
("verdict_from", "'the judge interrupted without a question'", unasked),
("verdict_from", "'the judge quoted an answer without a question'", unasked),
("run_judge", "verdict.reason", field),
("run_judge", "checked.reason", "quotes a passage the transcript does not hold"),
("misquoted_evidence", "f'the judge quoted {verdict.evidence!r}, which is not in the transcript", "quotes a passage the transcript does not hold"),
("run_judge", "checked.reason", "quotes a passage that neither the task nor the transcript holds"),
("misquoted_evidence", "f'the judge quoted {verdict.evidence!r}, which is not in the task or the transcript", "quotes a passage that neither the task nor the transcript holds"),
("claude_reply", "f'could not start claude:", started),
("claude_reply", "proc.stderr.strip() or f'{who} exited", exited),
("claude_reply", 'f"{who} printed no json reply', unparsed),
+3 -2
View File
@@ -183,8 +183,9 @@ Exit 1, saying why, when:
- the worker or the judge fails three times running, retried a minute apart within the cycle. A
worker fails when claude cannot start, exits non-zero, is killed, or ends without a result event;
a judge when claude cannot start, exits non-zero or prints no json reply, or its reply misses or
mistypes a field, interrupts or quotes an answer without a question, or quotes a passage the
transcript does not hold; it holds an elided passage when each fragment appears in order.
mistypes a field, interrupts or quotes an answer without a question, or quotes a passage that
neither the task nor the transcript holds, where each fragment of an elided quote must appear in
order, in the task then the transcript.
A usage limit is not a failure, and restarts the count of three: the worker or the judge is
retried 2 minutes after the limit resets, or every 10 minutes when the reset time is unknown or
more than 2 minutes past. The judge's reply gives its reset time only as text, so the judge is
+2 -2
View File
@@ -726,8 +726,8 @@ class Cycles(unittest.TestCase):
self.assertEqual(run.code, 130)
self.assertEqual(run.roles(), ["worker", "judge", "judge", "judge", "worker", "judge"])
self.assertEqual(run.sleeps, [60, 60])
self.assertIn("the judge quoted 'never said', which is not in the transcript, for the question: A or B?", run.stderr)
self.assertIn("retrying the judge in a minute: the judge quoted '', which is not in the transcript\n", run.stderr)
self.assertIn("the judge quoted 'never said', which is not in the task or the transcript, for the question: A or B?", run.stderr)
self.assertIn("retrying the judge in a minute: the judge quoted '', which is not in the task or the transcript\n", run.stderr)
def test_retries_a_judge_claude_cannot_start(self):
checkout = Checkout(self)
-13
View File
@@ -12,7 +12,6 @@
| ID | Release | Blocked by | Exempt | Item | L | E | I | R | S | Moves | Score |
|---|---|---|---|---|---|---|---|---|---|---|---|
| 460 | 0.14.0 | | | **Review the judge's fix for quoting the worker's task as evidence with every larv-review pass it fires, and adjust it to their findings.** | 2 | 2 | 3 | 5 | 2 | code health by 3, 1.1 by 5 | 7.2 |
| 463 | 0.14.0 | | defect | **Show the judge the transcript's tail whatever the size of the records.** | 2 | 2 | 3 | 5 | 3 | 1.2 by 6, 1.5 by 4, 1.4 by 2 | 4.2 |
| 449 | 0.14.0 | | | **Record do-epic's repo picks and sub-task summaries in the team's own spelling, such as `core-4` and `core-4: Bygg …`, as its config sets them.** | 2 | 2 | 2 | 4 | 3 | 1.6 by 6 | 2.0 |
| 453 | 0.14.0 | | | **Make the issue types that do-epic and `todo-item add` file in Jira settings, since a Jira project such as TW has no Task type.** | 2 | 2 | 1 | 3 | 2 | 1.6 by 5 | 1.8 |
@@ -78,18 +77,6 @@
## Details
### 460. Review the judge's fix for quoting the worker's task as evidence with every larv-review pass it fires, and adjust it to their findings.
The fix shipped without any larv-review pass, reviewer or comprehension check, at lilleman's request on 2026-11-19, so loop-claude could run unattended again: PR #641, commit "Let the judge quote the worker's task as evidence". Before it, the judge read the task above the transcript's tail but `quotes_checked` checked the evidence only against that tail. Once a session outgrew the tail, a verdict quoting the task, such as `Do the next chunk in 0.13.0`, was rejected three times running, and the loop exited after each merged chunk (items 448, 455 and 451). `Grounds` now carries the task, and the evidence must appear in the task, then the tail.
Run the passes the change fires: security, stability (with `~/.claude/rules/this-machine/cross-repo-review.md` where it applies), prose for the comments and docstrings it changed, and overrides if `rule-hook overrides` prints a line. Run the comprehension check where `larv-review` says one is due. Also weigh whether the evidence schema's description ("a verbatim quote of the one transcript passage") should name the task too, and whether `misquoted_evidence`'s message should say "the task or the transcript".
Touches:
- `bin/agent_cycle/judge.py`: `Grounds` (small), `quotes_checked` (small), `judge_prompt` (small)
- `bin/judge_replay/archived_prompt.py`: `grounds_of` (small)
- `bin/agent_cycle/test_judge.py`
- `bin/judge_replay/test_judge_replay.py`
### 463. Show the judge the transcript's tail whatever the size of the records.
`judge_prompt` in `bin/agent_cycle/judge.py` gives the transcript only what `JUDGE_PROMPT_CHARS = 400000` leaves after the premises, the records and the rules. In claude-config those now take about 519,000 to 523,000 characters, so `room` clamps to 0 and the judge reads the task above an empty transcript. Every judge prompt archived for claude-config since at least 2026-11-19 17:16 shows none: