Let the judge quote the worker's task as evidence
CI / tests (pull_request) Has been cancelled

This commit is contained in:
Lilleman
2026-11-19 21:21:54 +01:00
parent b9afedfd80
commit 14123912d9
4 changed files with 19 additions and 4 deletions
+5 -2
View File
@@ -134,6 +134,8 @@ class Grounds(typing.NamedTuple):
quotable: str
# Each answer the human typed, whole, latest last (transcript.human_answers).
typed: tuple[str, ...] = ()
# The worker's task, which the judge reads above shown and may quote as the evidence: the worker's first prompt holds it, but the tail in shown cuts that off once the session grows.
task: str = ""
class Brief(typing.NamedTuple):
@@ -237,7 +239,7 @@ def quotes_checked(verdict: Verdict, grounds: Grounds) -> Verdict | Failure:
| Quote | Must be in | Where it isn't | Checked by |
|---|---|---|---|
| the evidence | grounds.shown, the evidence's fragments in order | a Failure: run_judge rejects the reply, and judge retries it | grounded |
| the evidence | grounds.task then grounds.shown, the evidence's fragments in order | a Failure: run_judge rejects the reply, and judge retries it | grounded |
| an interrupt's answered_before | grounds.shown | the interrupt stands, as if no earlier answer was found | re_ask_checked |
| its left_open | answered_before and the question | the interrupt becomes an Unfinished that quotes answered_before back to the worker | re_ask_checked |
| a quoted answer | grounds.quotable | the question goes to the human as an interrupt, which keeps the escalation the answer carried | answer_checked |
@@ -246,7 +248,7 @@ def quotes_checked(verdict: Verdict, grounds: Grounds) -> Verdict | Failure:
Only a misquoted evidence fails. A Failure for any other would exit an unattended loop, so each changes the verdict instead.
"""
if not grounded(verdict.evidence, grounds.shown):
if not grounded(verdict.evidence, f"{grounds.task}\n\n{grounds.shown}"):
return misquoted_evidence(verdict)
verdict = re_ask_checked(verdict, grounds.shown)
verdict = answer_checked(verdict, grounds.quotable)
@@ -306,5 +308,6 @@ def judge_prompt(brief: Brief) -> tuple[str, Grounds]:
after_mark=escalation_mark.after_mark(brief.entries, brief.escalated_at),
quotable=f"{brief.premises}\n\n{brief.records}",
typed=tuple(human_answers(brief.entries, typed_answer)),
task=brief.task,
)
return f"{head}{shown}\n\n{rules}", grounds
+8
View File
@@ -124,6 +124,14 @@ class Verdict(unittest.TestCase):
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", "the project's decisions", "the plan or a comment", kinds(*entries), 1, "loop-claude"), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", evidence="round 2: 🔴")]).effects())
self.assertEqual(verdict, Unfinished("round 2: 🔴", entries_read=2))
def test_takes_the_task_as_evidence_where_the_tail_it_shows_cuts_off_the_workers_first_prompt(self):
entries = [human("Do the next chunk. " + "x" * judge.JUDGE_PROMPT_CHARS), assistant({"type": "text", "text": "Merged."})]
def judged_on(evidence):
return judge.judge_once("/r", judge.Brief("Do the next chunk.", "## Goals", "", "the project's decisions", "the plan or a comment", kinds(*entries), 0, "loop-claude"), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", evidence=evidence)] * 3).effects())
self.assertEqual(judged_on("Do the next chunk."), Unfinished("Do the next chunk.", entries_read=2))
self.assertEqual(judged_on("Do the next chunk. ... Merged."), Unfinished("Do the next chunk. ... Merged.", entries_read=2))
self.assertIsInstance(judged_on("Do the next release."), Failure)
class JudgePrompt(unittest.TestCase):
def brief(self, entries, escalated_at, answer_destinations="the plan or a comment"):
+2 -2
View File
@@ -12,7 +12,7 @@ def grounds_of(prompt: str, entries: list[Entry]) -> Grounds | Failure:
"""What the quotes of a judge run on prompt are checked against, as judge_prompt would have made it for the transcript's entries: the premises and the records before the task are quotable, the transcript the judge was shown comes before the rules. The prompt is split on the headings judge_prompt writes, at the first of each; a Failure where it lacks one. Rules of an earlier wording stay part of the transcript, a slack that lets an evidence quote them pass."""
before_task, task_heading, after_task = prompt.partition(f"\n\n{TASK_HEADING}")
heading = transcript_heading(LOOP_CLAUDE)
_, found_heading, shown = after_task.partition(heading)
task, found_heading, shown = after_task.partition(heading)
if not (task_heading and found_heading):
return Failure(f"the archived prompt lacks the heading {TASK_HEADING.strip()!r} or {heading.strip()!r}, which judge_prompt writes")
return Grounds(shown.removesuffix(f"\n\n{judge_rules(ANSWER_DESTINATIONS, LOOP_CLAUDE)}"), escalation_mark.after_mark(entries, 0), before_task, tuple(human_answers(entries, typed_answer)))
return Grounds(shown.removesuffix(f"\n\n{judge_rules(ANSWER_DESTINATIONS, LOOP_CLAUDE)}"), escalation_mark.after_mark(entries, 0), before_task, tuple(human_answers(entries, typed_answer)), task.removesuffix("\n\n"))
+4
View File
@@ -399,6 +399,10 @@ class JudgeArchivedRuns(unittest.TestCase):
self.assertIsInstance(self.judged(evidence="Ship fast.")[0], Rejected)
self.assertIsInstance(self.judged(evidence="Worker chose option B.")[0], Unfinished)
def test_takes_the_task_as_evidence(self):
self.assertIsInstance(self.judged(evidence="Do it.")[0], Unfinished)
self.assertIsInstance(self.judged(evidence="Do it. ... Use Postgres.")[0], Unfinished)
def test_keeps_the_cost_and_the_run_of_a_rejected_reply(self):
rejected, _ = self.judged(evidence="Never said.")
self.assertEqual((rejected.cost, rejected.run.prompt), (0.25, self.PROMPT))