Files
claude-config/bin/agent_cycle/test_judge.py
T

220 lines
18 KiB
Python

import dataclasses
import json
import pathlib
import sys
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from loop_effects.doubles import assistant, FakeEffects, human, judged, kinds
from failure import Failure
from agent_cycle.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished
from agent_cycle import judge
from agent_cycle.doubles import PROMPTS
from model_routing.route import Route
from model_routing.ladder import Rung
from agent_cycle.judge import verdict_from
class Grounded(unittest.TestCase):
def test_every_verdict_must_quote_the_transcript(self):
self.assertTrue(judge.grounded("Needs you: x", "so.\n\nNeeds you: x\n"))
self.assertFalse(judge.grounded("invented", "the transcript"))
self.assertFalse(judge.grounded("", "the transcript"))
self.assertFalse(judge.grounded(" ", "the transcript"))
transcript = "done.\n\n**Your step:**\n```\n./run.sh\n```\nRun it."
self.assertTrue(judge.grounded("Your step:\n```\n./run.sh\n```\nRun it.", transcript))
self.assertTrue(judge.grounded("Your step: ``` ./run.sh ``` Run it.", transcript))
self.assertFalse(judge.grounded("**", transcript))
def test_an_elided_quote_holds_when_its_fragments_appear_in_order(self):
transcript = "**Decision for you:** security, or not.\n\nI recommend one more round of each.\n\nI recommend skipping it: it costs."
self.assertTrue(judge.grounded("**Decision for you:** security, … ... I recommend one more round of each ... I recommend skipping it: …", transcript))
self.assertFalse(judge.grounded("I recommend skipping it ... Decision for you", transcript))
self.assertFalse(judge.grounded("Decision for you ... invented", transcript))
self.assertFalse(judge.grounded("... …", transcript))
self.assertTrue(judge.grounded("a ... b ... c", "x a b a c"))
class Verdict(unittest.TestCase):
def test_needs_every_field_the_schema_requires(self):
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": False, "evidence": "e", "finished": True, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Finished("e"))
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": False, "evidence": "e", "finished": False, "question": "ignored", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Unfinished("e"))
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": True, "evidence": "e", "finished": True, "question": "Which one?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Interrupt("e", "Which one?"))
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": True, "evidence": "e", "finished": False, "question": "Cut 0.2.0?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": True}), Interrupt("e", "Cut 0.2.0?", release_ready=True))
self.assertEqual(verdict_from({"escalation": "", "answered_before": "", "left_open": "", "reversed": "", "answer": "", "interrupt": True, "evidence": "e", "finished": False, "question": "Cut 0.2.0?"}), Failure("the judge's reply has no release_ready of type boolean"))
self.assertEqual(verdict_from({"escalation": "", "answer": "A.", "interrupt": False, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), QuotedAnswer("e", "Which one?", "A."))
self.assertEqual(verdict_from({"escalation": "", "answer": "A.", "interrupt": True, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Interrupt("e", "Which one?"))
self.assertEqual(verdict_from({"escalation": "", "answer": "A.", "interrupt": False, "evidence": "e", "finished": True, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Failure("the judge quoted an answer without a question"))
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": "no", "evidence": "", "finished": True, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Failure("the judge's reply has no interrupt of type boolean"))
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": False, "evidence": None, "finished": True, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Failure("the judge's reply has no evidence of type string"))
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": True, "evidence": "x", "finished": False, "question": " ", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Failure("the judge interrupted without a question"))
self.assertEqual(verdict_from(None), Failure("the judge returned no reply"))
def test_carries_the_earlier_answer_an_interrupt_names(self):
reply = {"answer": "", "escalation": "", "interrupt": True, "evidence": "e", "finished": False, "release_ready": False, "answered_before": "Grant one more round.", "left_open": "one more", "question": "Which one?", "reversed": ""}
self.assertEqual(verdict_from(reply), Interrupt("e", "Which one?", answered_before="Grant one more round.", left_open="one more"))
self.assertEqual(verdict_from({**reply, "interrupt": False, "finished": True}), Finished("e"))
def test_takes_an_escalation_quote_on_an_unfinished_verdict(self):
reply = {"answer": "", "escalation": "round 2: 🔴", "interrupt": False, "evidence": "e", "finished": False, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}
self.assertEqual(verdict_from(reply), Unfinished("e", "round 2: 🔴"))
self.assertEqual(verdict_from({**reply, "finished": True}), Finished("e"))
self.assertEqual(verdict_from({**reply, "escalation": None}), Failure("the judge's reply has no escalation of type string"))
def test_takes_an_escalation_quote_on_an_interrupt(self):
reply = {"answer": "", "escalation": "round 2: 🔴", "interrupt": True, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}
self.assertEqual(verdict_from(reply), Interrupt("e", "Which one?", "round 2: 🔴"))
def test_drops_an_escalation_quote_the_transcript_does_not_hold_word_for_word_from_an_interrupt_or_an_unfinished_verdict(self):
shown = "The agent: round 2: 🔴. Which one?"
self.assertEqual(judge.quotes_checked(Interrupt("e", "q", "round 2: 🔴"), judge.Grounds(shown, shown, "")), Interrupt("e", "q", "round 2: 🔴"))
self.assertEqual(judge.quotes_checked(Interrupt("e", "q", "round 2: … 🔴"), judge.Grounds(shown, shown, "")), Interrupt("e", "q"))
self.assertEqual(judge.quotes_checked(Unfinished("e", "round 3: 🔴"), judge.Grounds(shown, shown, "")), Unfinished("e"))
def test_turns_an_interrupt_that_re_asks_a_human_answer_into_that_answer_quoted_back(self):
answer = "Grant one more round; ask again if a 🔴 survives it."
shown = f"[loop] Asked.\n\n[human] {answer}\n\n[step] Round 2: a 🔴 survived."
grounds = judge.Grounds(shown, shown, "")
asking = "Grant a third round? Your answer left open: a 🔴 survives it"
kept = Interrupt("[step] Round 2", asking, answered_before=answer, left_open="a 🔴 survives it")
self.assertEqual(judge.quotes_checked(kept, grounds), kept)
re_asked = Interrupt("[step] Round 2", "Grant a third round?", "Round 2: a 🔴", answered_before=answer, left_open="a 🔴 survives it")
self.assertEqual(judge.quotes_checked(re_asked, grounds), Unfinished(answer, "Round 2: a 🔴"))
self.assertEqual(judge.quotes_checked(dataclasses.replace(kept, left_open=""), grounds), Unfinished(answer))
self.assertEqual(judge.quotes_checked(dataclasses.replace(kept, left_open="a 🔴 survived"), grounds), Unfinished(answer))
invented = dataclasses.replace(re_asked, answered_before="Grant two more rounds.")
self.assertEqual(judge.quotes_checked(invented, grounds), invented)
unnamed = Interrupt("[step] Round 2", "Grant a third round?")
self.assertEqual(judge.quotes_checked(unnamed, grounds), unnamed)
def test_takes_an_escalation_quote_on_a_quoted_answer(self):
reply = {"answer": "A.", "escalation": "round 2: 🔴", "interrupt": False, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}
self.assertEqual(verdict_from(reply), QuotedAnswer("e", "Which one?", "A.", "round 2: 🔴"))
def test_drops_an_escalation_quote_the_transcript_does_not_hold_word_for_word_from_a_quoted_answer(self):
shown = "The agent: round 2: 🔴. Which one?"
self.assertEqual(judge.quotes_checked(QuotedAnswer("e", "q", "A.", "round 2: 🔴"), judge.Grounds(shown, shown, "A.")), QuotedAnswer("e", "q", "A.", "round 2: 🔴"))
self.assertEqual(judge.quotes_checked(QuotedAnswer("e", "q", "A.", "round 2: … 🔴"), judge.Grounds(shown, shown, "A.")), QuotedAnswer("e", "q", "A."))
def test_puts_a_quoted_answer_the_records_do_not_hold_to_the_human_with_the_escalation_it_carried(self):
self.assertEqual(judge.quotes_checked(QuotedAnswer("e", "q", "not there", "round 2: 🔴"), judge.Grounds("The agent: round 2: 🔴.", "The agent: round 2: 🔴.", "A.")), Interrupt("e", "q", "round 2: 🔴"))
def test_puts_a_quoted_answer_the_records_do_not_hold_word_for_word_to_the_human(self):
self.assertEqual(judge.quotes_checked(QuotedAnswer("s", "q", "not there"), judge.Grounds("shown", "shown", "the records")), Interrupt("s", "q"))
self.assertEqual(judge.quotes_checked(QuotedAnswer("s", "q", "the records"), judge.Grounds("shown", "shown", "From the records: the records")), QuotedAnswer("s", "q", "the records"))
def test_drops_an_escalation_quote_found_only_in_the_transcript_entries_before_the_workers_last_step_up(self):
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "round 3: 🔴"}), assistant({"type": "text", "text": "working"})]
def judged_on(mark, escalation):
return judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", "the project's decisions", "the plan or a comment", kinds(*entries), mark, "loop-claude"), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation=escalation)]).effects())
self.assertEqual(judged_on(0, "round 2: 🔴"), Unfinished("working", "round 2: 🔴", entries_read=3))
self.assertEqual(judged_on(1, "round 2: 🔴"), Unfinished("working", entries_read=3))
self.assertEqual(judged_on(1, "round 3: 🔴"), Unfinished("working", "round 3: 🔴", entries_read=3))
self.assertEqual(judged_on(1, "round 2: 🔴\n\nround 3: 🔴"), Unfinished("working", entries_read=3))
self.assertEqual(judged_on(3, "round 3: 🔴"), Unfinished("working", entries_read=3))
def test_drops_an_escalation_quote_that_only_loop_claudes_prompts_or_the_humans_lines_repeat_after_the_step_up(self):
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), {**human("as the work remaining: round 2: 🔴"), "promptSource": "sdk"}, human("round 2: 🔴 again"), assistant({"type": "text", "text": "working"})]
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", "the project's decisions", "the plan or a comment", kinds(*entries), 1, "loop-claude"), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation="round 2: 🔴")]).effects())
self.assertEqual(verdict, Unfinished("working", entries_read=4))
def test_takes_evidence_from_before_the_workers_last_step_up(self):
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "working"})]
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", "the project's decisions", "the plan or a comment", kinds(*entries), 1, "loop-claude"), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", evidence="round 2: 🔴")]).effects())
self.assertEqual(verdict, Unfinished("round 2: 🔴", entries_read=2))
def test_takes_the_task_as_evidence_where_the_tail_it_shows_cuts_off_the_workers_first_prompt_but_no_quote_across_the_join(self):
entries = [human("Do the next chunk. " + "x" * judge.JUDGE_PROMPT_CHARS), assistant({"type": "text", "text": "Merged."})]
def judged_on(evidence):
return judge.judge_once("/r", judge.Brief("Do the next chunk.", "## Goals", "", "the project's decisions", "the plan or a comment", kinds(*entries), 0, "loop-claude"), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", evidence=evidence)] * 3).effects())
self.assertEqual(judged_on("Do the next chunk."), Unfinished("Do the next chunk.", entries_read=2))
self.assertEqual(judged_on("Do the next chunk. ... Merged."), Unfinished("Do the next chunk. ... Merged.", entries_read=2))
self.assertIsInstance(judged_on("Do the next release."), Failure)
self.assertIsInstance(judged_on("next chunk. xxx"), Failure)
class JudgePrompt(unittest.TestCase):
def brief(self, entries, escalated_at, answer_destinations="the plan or a comment", records=""):
"""What the judge of a-driver reads of a transcript holding entries."""
return judge.Brief("Do it.", "## Goals", records, "the project's decisions", answer_destinations, kinds(*entries), escalated_at, "a-driver")
def test_shows_the_transcripts_tail_and_returns_what_it_showed(self):
entries = [assistant({"type": "text", "text": "old " + "x" * judge.JUDGE_PROMPT_CHARS}), assistant({"type": "text", "text": "new"})]
prompt, grounds = judge.judge_prompt(self.brief(entries, 0))
self.assertTrue(grounds.shown.endswith("\n\nnew"))
self.assertNotIn("old", grounds.shown)
self.assertIn(grounds.shown, prompt)
self.assertLessEqual(len(prompt), judge.JUDGE_PROMPT_CHARS)
self.assertEqual(grounds.after_mark, "old " + "x" * judge.JUDGE_PROMPT_CHARS + "\n\nnew")
self.assertEqual(grounds.quotable, "## Goals\n\n")
# A transcript longer than the floor: an old message, then the last one.
LONG = [assistant({"type": "text", "text": "old " + "x" * judge.TRANSCRIPT_FLOOR}), assistant({"type": "text", "text": "Merged."})]
def overhead(self):
"""The characters of the judge's prompt that are neither the records nor the transcript."""
prompt, grounds = judge.judge_prompt(self.brief(self.LONG, 0, records="r"))
return len(prompt) - len(grounds.shown) - 1
def test_fails_naming_the_shortfall_where_the_records_leave_less_than_the_transcripts_floor(self):
failed = judge.judge_prompt(self.brief(self.LONG, 0, records="r" * (judge.JUDGE_PROMPT_CHARS + 1)))
self.assertIsInstance(failed, Failure)
for fact in ("400,000", "400,001", f"{judge.TRANSCRIPT_FLOOR:,}", f"by at least {judge.TRANSCRIPT_FLOOR + self.overhead() + 1:,}"):
self.assertIn(fact, failed.reason)
def test_judge_once_returns_that_failure_without_starting_claude(self):
failed = judge.judge_once("/r", self.brief(self.LONG, 0, records="r" * judge.JUDGE_PROMPT_CHARS), Rung("judge-model"), FakeEffects(claude=[]).effects())
self.assertIsInstance(failed, Failure)
self.assertIn(f"{judge.TRANSCRIPT_FLOOR:,}", failed.reason)
def test_shows_more_than_the_floor_of_the_transcripts_tail_where_the_records_leave_room(self):
prompt, grounds = judge.judge_prompt(self.brief(self.LONG, 0, records="r" * (judge.JUDGE_PROMPT_CHARS - judge.TRANSCRIPT_FLOOR - 10000)))
self.assertTrue(grounds.shown.endswith("\n\nMerged."))
self.assertGreaterEqual(len(grounds.shown), judge.TRANSCRIPT_FLOOR)
self.assertLessEqual(len(prompt), judge.JUDGE_PROMPT_CHARS)
def test_shows_the_floor_where_the_records_leave_exactly_that(self):
at_floor = judge.JUDGE_PROMPT_CHARS - self.overhead() - judge.TRANSCRIPT_FLOOR
self.assertEqual(len(judge.judge_prompt(self.brief(self.LONG, 0, records="r" * at_floor))[1].shown), judge.TRANSCRIPT_FLOOR)
self.assertIsInstance(judge.judge_prompt(self.brief(self.LONG, 0, records="r" * (at_floor + 1))), Failure)
def test_shows_a_transcript_shorter_than_the_floor_whole(self):
_, grounds = judge.judge_prompt(self.brief([assistant({"type": "text", "text": "Merged."})], 0, records="r" * (judge.JUDGE_PROMPT_CHARS - 10000)))
self.assertEqual(grounds.shown, "Merged.")
def test_the_judge_prompt_and_schema_call_the_worker_the_worker(self):
prompt, _ = judge.judge_prompt(self.brief([assistant({"type": "text", "text": "done"})], 1))
self.assertNotRegex(prompt + json.dumps(judge.judge_schema("a-driver")), r"\b[Aa]gent")
def test_the_judge_prompt_names_the_entry_points_answer_destinations_in_both_rules_on_a_recorded_answer(self):
prompt, _ = judge.judge_prompt(self.brief([assistant({"type": "text", "text": "done"})], 0, "the ledger"))
self.assertIn("is recorded in the ledger at the one place it governs", prompt)
self.assertIn("the worker writing it into the ledger, but not into one of the human's rules", prompt)
def test_shows_the_judge_where_the_worker_last_stepped_up_and_checks_an_escalation_only_after_it(self):
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "round 3: 🔴"})]
prompt, grounds = judge.judge_prompt(self.brief(entries, 1))
self.assertRegex(prompt, "round 2: 🔴\n\n\\[step\\] [^\n]+\n\nround 3: 🔴\n\n")
self.assertEqual(grounds.after_mark, "round 3: 🔴")
prompt, grounds = judge.judge_prompt(self.brief(entries, 0))
self.assertNotIn("stepped the worker up", prompt)
self.assertEqual(grounds.shown, "round 2: 🔴\n\nround 3: 🔴")
self.assertEqual(grounds.after_mark, "round 2: 🔴\n\nround 3: 🔴")
def test_drops_an_escalation_quote_from_after_the_step_up_that_the_tail_it_shows_cuts_off(self):
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "old " + "x" * judge.JUDGE_PROMPT_CHARS}), assistant({"type": "text", "text": "new"})]
_, grounds = judge.judge_prompt(self.brief(entries, 1))
self.assertEqual(judge.quotes_checked(Unfinished("e", "old x", entries_read=3), grounds), Unfinished("e", entries_read=3))
self.assertEqual(judge.quotes_checked(Unfinished("e", "new", entries_read=3), grounds), Unfinished("e", "new", entries_read=3))
def test_shows_a_typed_answer_before_the_step_up_as_the_humans(self):
typed = PROMPTS.typed("Which one?", "Use B.", Route(Rung("worker-model")), None, False)
entries = [{**human(typed), "promptSource": "sdk"}, assistant({"type": "text", "text": "round 3: 🔴"})]
_, grounds = judge.judge_prompt(self.brief(entries, 1))
self.assertIn("\n\n[human] Use B.\n\n[step] ", grounds.shown)
if __name__ == "__main__":
unittest.main()