226 lines
18 KiB
Python
226 lines
18 KiB
Python
import dataclasses
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from loop_effects.doubles import assistant, FakeEffects, human, judged, kinds
|
|
from failure import Failure
|
|
from agent_cycle.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished
|
|
from agent_cycle import judge
|
|
from agent_cycle.doubles import PROMPTS
|
|
from model_routing.route import Route
|
|
from model_routing.ladder import Rung
|
|
from agent_cycle.judge import verdict_from
|
|
|
|
|
|
class Grounded(unittest.TestCase):
|
|
def test_every_verdict_must_quote_the_transcript(self):
|
|
self.assertTrue(judge.grounded("Needs you: x", "so.\n\nNeeds you: x\n"))
|
|
self.assertFalse(judge.grounded("invented", "the transcript"))
|
|
self.assertFalse(judge.grounded("", "the transcript"))
|
|
self.assertFalse(judge.grounded(" ", "the transcript"))
|
|
transcript = "done.\n\n**Your step:**\n```\n./run.sh\n```\nRun it."
|
|
self.assertTrue(judge.grounded("Your step:\n```\n./run.sh\n```\nRun it.", transcript))
|
|
self.assertTrue(judge.grounded("Your step: ``` ./run.sh ``` Run it.", transcript))
|
|
self.assertFalse(judge.grounded("**", transcript))
|
|
|
|
def test_an_elided_quote_holds_when_its_fragments_appear_in_order(self):
|
|
transcript = "**Decision for you:** security, or not.\n\nI recommend one more round of each.\n\nI recommend skipping it: it costs."
|
|
self.assertTrue(judge.grounded("**Decision for you:** security, … ... I recommend one more round of each ... I recommend skipping it: …", transcript))
|
|
self.assertFalse(judge.grounded("I recommend skipping it ... Decision for you", transcript))
|
|
self.assertFalse(judge.grounded("Decision for you ... invented", transcript))
|
|
self.assertFalse(judge.grounded("... …", transcript))
|
|
self.assertTrue(judge.grounded("a ... b ... c", "x a b a c"))
|
|
|
|
|
|
class Verdict(unittest.TestCase):
|
|
def test_needs_every_field_the_schema_requires(self):
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": False, "evidence": "e", "finished": True, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Finished("e"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": False, "evidence": "e", "finished": False, "question": "ignored", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Unfinished("e"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": True, "evidence": "e", "finished": True, "question": "Which one?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Interrupt("e", "Which one?"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": True, "evidence": "e", "finished": False, "question": "Cut 0.2.0?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": True}), Interrupt("e", "Cut 0.2.0?", release_ready=True))
|
|
self.assertEqual(verdict_from({"escalation": "", "answered_before": "", "left_open": "", "reversed": "", "answer": "", "interrupt": True, "evidence": "e", "finished": False, "question": "Cut 0.2.0?"}), Failure("the judge's reply has no release_ready of type boolean"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "A.", "interrupt": False, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), QuotedAnswer("e", "Which one?", "A."))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "A.", "interrupt": True, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Interrupt("e", "Which one?"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "A.", "interrupt": False, "evidence": "e", "finished": True, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Failure("the judge quoted an answer without a question"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": "no", "evidence": "", "finished": True, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Failure("the judge's reply has no interrupt of type boolean"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": False, "evidence": None, "finished": True, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Failure("the judge's reply has no evidence of type string"))
|
|
self.assertEqual(verdict_from({"escalation": "", "answer": "", "interrupt": True, "evidence": "x", "finished": False, "question": " ", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}), Failure("the judge interrupted without a question"))
|
|
self.assertEqual(verdict_from(None), Failure("the judge returned no reply"))
|
|
|
|
def test_carries_the_earlier_answer_an_interrupt_names(self):
|
|
reply = {"answer": "", "escalation": "", "interrupt": True, "evidence": "e", "finished": False, "release_ready": False, "answered_before": "Grant one more round.", "left_open": "one more", "question": "Which one?", "reversed": ""}
|
|
self.assertEqual(verdict_from(reply), Interrupt("e", "Which one?", answered_before="Grant one more round.", left_open="one more"))
|
|
self.assertEqual(verdict_from({**reply, "interrupt": False, "finished": True}), Finished("e"))
|
|
|
|
def test_takes_an_escalation_quote_on_an_unfinished_verdict(self):
|
|
reply = {"answer": "", "escalation": "round 2: 🔴", "interrupt": False, "evidence": "e", "finished": False, "question": "", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}
|
|
self.assertEqual(verdict_from(reply), Unfinished("e", "round 2: 🔴"))
|
|
self.assertEqual(verdict_from({**reply, "finished": True}), Finished("e"))
|
|
self.assertEqual(verdict_from({**reply, "escalation": None}), Failure("the judge's reply has no escalation of type string"))
|
|
|
|
def test_takes_an_escalation_quote_on_an_interrupt(self):
|
|
reply = {"answer": "", "escalation": "round 2: 🔴", "interrupt": True, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}
|
|
self.assertEqual(verdict_from(reply), Interrupt("e", "Which one?", "round 2: 🔴"))
|
|
|
|
def test_drops_an_escalation_quote_the_transcript_does_not_hold_word_for_word_from_an_interrupt_or_an_unfinished_verdict(self):
|
|
shown = "The agent: round 2: 🔴. Which one?"
|
|
self.assertEqual(judge.quotes_checked(Interrupt("e", "q", "round 2: 🔴"), judge.Grounds(shown, shown, "")), Interrupt("e", "q", "round 2: 🔴"))
|
|
self.assertEqual(judge.quotes_checked(Interrupt("e", "q", "round 2: … 🔴"), judge.Grounds(shown, shown, "")), Interrupt("e", "q"))
|
|
self.assertEqual(judge.quotes_checked(Unfinished("e", "round 3: 🔴"), judge.Grounds(shown, shown, "")), Unfinished("e"))
|
|
|
|
def test_turns_an_interrupt_that_re_asks_a_human_answer_into_that_answer_quoted_back(self):
|
|
answer = "Grant one more round; ask again if a 🔴 survives it."
|
|
shown = f"[loop] Asked.\n\n[human] {answer}\n\n[step] Round 2: a 🔴 survived."
|
|
grounds = judge.Grounds(shown, shown, "")
|
|
asking = "Grant a third round? Your answer left open: a 🔴 survives it"
|
|
kept = Interrupt("[step] Round 2", asking, answered_before=answer, left_open="a 🔴 survives it")
|
|
self.assertEqual(judge.quotes_checked(kept, grounds), kept)
|
|
re_asked = Interrupt("[step] Round 2", "Grant a third round?", "Round 2: a 🔴", answered_before=answer, left_open="a 🔴 survives it")
|
|
self.assertEqual(judge.quotes_checked(re_asked, grounds), Unfinished(answer, "Round 2: a 🔴"))
|
|
self.assertEqual(judge.quotes_checked(dataclasses.replace(kept, left_open=""), grounds), Unfinished(answer))
|
|
self.assertEqual(judge.quotes_checked(dataclasses.replace(kept, left_open="a 🔴 survived"), grounds), Unfinished(answer))
|
|
invented = dataclasses.replace(re_asked, answered_before="Grant two more rounds.")
|
|
self.assertEqual(judge.quotes_checked(invented, grounds), invented)
|
|
unnamed = Interrupt("[step] Round 2", "Grant a third round?")
|
|
self.assertEqual(judge.quotes_checked(unnamed, grounds), unnamed)
|
|
|
|
def test_takes_an_escalation_quote_on_a_quoted_answer(self):
|
|
reply = {"answer": "A.", "escalation": "round 2: 🔴", "interrupt": False, "evidence": "e", "finished": False, "question": "Which one?", "answered_before": "", "left_open": "", "reversed": "", "release_ready": False}
|
|
self.assertEqual(verdict_from(reply), QuotedAnswer("e", "Which one?", "A.", "round 2: 🔴"))
|
|
|
|
def test_drops_an_escalation_quote_the_transcript_does_not_hold_word_for_word_from_a_quoted_answer(self):
|
|
shown = "The agent: round 2: 🔴. Which one?"
|
|
self.assertEqual(judge.quotes_checked(QuotedAnswer("e", "q", "A.", "round 2: 🔴"), judge.Grounds(shown, shown, "A.")), QuotedAnswer("e", "q", "A.", "round 2: 🔴"))
|
|
self.assertEqual(judge.quotes_checked(QuotedAnswer("e", "q", "A.", "round 2: … 🔴"), judge.Grounds(shown, shown, "A.")), QuotedAnswer("e", "q", "A."))
|
|
|
|
def test_puts_a_quoted_answer_the_records_do_not_hold_to_the_human_with_the_escalation_it_carried(self):
|
|
self.assertEqual(judge.quotes_checked(QuotedAnswer("e", "q", "not there", "round 2: 🔴"), judge.Grounds("The agent: round 2: 🔴.", "The agent: round 2: 🔴.", "A.")), Interrupt("e", "q", "round 2: 🔴"))
|
|
|
|
def test_puts_a_quoted_answer_the_records_do_not_hold_word_for_word_to_the_human(self):
|
|
self.assertEqual(judge.quotes_checked(QuotedAnswer("s", "q", "not there"), judge.Grounds("shown", "shown", "the records")), Interrupt("s", "q"))
|
|
self.assertEqual(judge.quotes_checked(QuotedAnswer("s", "q", "the records"), judge.Grounds("shown", "shown", "From the records: the records")), QuotedAnswer("s", "q", "the records"))
|
|
|
|
def test_drops_an_escalation_quote_found_only_in_the_transcript_entries_before_the_workers_last_step_up(self):
|
|
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "round 3: 🔴"}), assistant({"type": "text", "text": "working"})]
|
|
def judged_on(mark, escalation):
|
|
return judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", "the project's decisions", "the plan or a comment", kinds(*entries), mark, "loop-claude"), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation=escalation)]).effects())
|
|
self.assertEqual(judged_on(0, "round 2: 🔴"), Unfinished("working", "round 2: 🔴", entries_read=3))
|
|
self.assertEqual(judged_on(1, "round 2: 🔴"), Unfinished("working", entries_read=3))
|
|
self.assertEqual(judged_on(1, "round 3: 🔴"), Unfinished("working", "round 3: 🔴", entries_read=3))
|
|
self.assertEqual(judged_on(1, "round 2: 🔴\n\nround 3: 🔴"), Unfinished("working", entries_read=3))
|
|
self.assertEqual(judged_on(3, "round 3: 🔴"), Unfinished("working", entries_read=3))
|
|
|
|
def test_drops_an_escalation_quote_that_only_loop_claudes_prompts_or_the_humans_lines_repeat_after_the_step_up(self):
|
|
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), {**human("as the work remaining: round 2: 🔴"), "promptSource": "sdk"}, human("round 2: 🔴 again"), assistant({"type": "text", "text": "working"})]
|
|
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", "the project's decisions", "the plan or a comment", kinds(*entries), 1, "loop-claude"), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation="round 2: 🔴")]).effects())
|
|
self.assertEqual(verdict, Unfinished("working", entries_read=4))
|
|
|
|
def test_takes_evidence_from_before_the_workers_last_step_up(self):
|
|
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "working"})]
|
|
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", "the project's decisions", "the plan or a comment", kinds(*entries), 1, "loop-claude"), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", evidence="round 2: 🔴")]).effects())
|
|
self.assertEqual(verdict, Unfinished("round 2: 🔴", entries_read=2))
|
|
|
|
def test_takes_the_task_as_evidence_where_the_tail_it_shows_cuts_off_the_workers_first_prompt_but_no_quote_across_the_join(self):
|
|
entries = [human("Do the next chunk. " + "x" * judge.JUDGE_PROMPT_CHARS), assistant({"type": "text", "text": "Merged."})]
|
|
def judged_on(evidence):
|
|
return judge.judge_once("/r", judge.Brief("Do the next chunk.", "## Goals", "", "the project's decisions", "the plan or a comment", kinds(*entries), 0, "loop-claude"), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", evidence=evidence)] * 3).effects())
|
|
self.assertEqual(judged_on("Do the next chunk."), Unfinished("Do the next chunk.", entries_read=2))
|
|
self.assertEqual(judged_on("Do the next chunk. ... Merged."), Unfinished("Do the next chunk. ... Merged.", entries_read=2))
|
|
self.assertIsInstance(judged_on("Do the next release."), Failure)
|
|
self.assertIsInstance(judged_on("next chunk. xxx"), Failure)
|
|
|
|
|
|
class JudgePrompt(unittest.TestCase):
|
|
def brief(self, entries, escalated_at, answer_destinations="the plan or a comment", records=""):
|
|
"""What the judge of a-driver reads of a transcript holding entries."""
|
|
return judge.Brief("Do it.", "## Goals", records, "the project's decisions", answer_destinations, kinds(*entries), escalated_at, "a-driver")
|
|
|
|
def test_runs_with_no_tools_and_loads_no_mcp_server(self):
|
|
args = judge.judge_args(Rung("claude-opus-5-5", "medium"), "a-driver")
|
|
self.assertEqual(args[args.index("--tools") + 1 :], ["", "--strict-mcp-config"])
|
|
self.assertNotIn("--mcp-config", args)
|
|
|
|
def test_shows_the_transcripts_tail_and_returns_what_it_showed(self):
|
|
entries = [assistant({"type": "text", "text": "old " + "x" * judge.JUDGE_PROMPT_CHARS}), assistant({"type": "text", "text": "new"})]
|
|
prompt, grounds = judge.judge_prompt(self.brief(entries, 0))
|
|
self.assertTrue(grounds.shown.endswith("\n\nnew"))
|
|
self.assertNotIn("old", grounds.shown)
|
|
self.assertIn(grounds.shown, prompt)
|
|
self.assertLessEqual(len(prompt), judge.JUDGE_PROMPT_CHARS)
|
|
self.assertEqual(grounds.after_mark, "old " + "x" * judge.JUDGE_PROMPT_CHARS + "\n\nnew")
|
|
self.assertEqual(grounds.quotable, "## Goals\n\n")
|
|
|
|
# A transcript longer than the floor: an old message, then the last one.
|
|
LONG = [assistant({"type": "text", "text": "old " + "x" * judge.TRANSCRIPT_FLOOR}), assistant({"type": "text", "text": "Merged."})]
|
|
|
|
def overhead(self):
|
|
"""The characters of the judge's prompt that are neither the records nor the transcript."""
|
|
prompt, grounds = judge.judge_prompt(self.brief(self.LONG, 0, records="r"))
|
|
return len(prompt) - len(grounds.shown) - 1
|
|
|
|
def test_fails_naming_the_shortfall_where_the_records_leave_less_than_the_transcripts_floor(self):
|
|
failed = judge.judge_prompt(self.brief(self.LONG, 0, records="r" * (judge.JUDGE_PROMPT_CHARS + 1)))
|
|
self.assertIsInstance(failed, Failure)
|
|
for fact in ("leave 0 of the judge's 400,000", "400,001", f"{judge.TRANSCRIPT_FLOOR:,}", f"by at least {judge.TRANSCRIPT_FLOOR + self.overhead() + 1:,}"):
|
|
with self.subTest(fact=fact):
|
|
self.assertIn(fact, failed.reason)
|
|
|
|
def test_judge_once_returns_that_failure_without_starting_claude(self):
|
|
failed = judge.judge_once("/r", self.brief(self.LONG, 0, records="r" * judge.JUDGE_PROMPT_CHARS), Rung("judge-model"), FakeEffects(claude=[]).effects())
|
|
self.assertIsInstance(failed, Failure)
|
|
self.assertIn(f"{judge.TRANSCRIPT_FLOOR:,}", failed.reason)
|
|
|
|
def test_shows_the_whole_of_a_transcript_longer_than_the_floor_where_the_records_leave_room_for_it(self):
|
|
whole = "old " + "x" * judge.TRANSCRIPT_FLOOR + "\n\nMerged."
|
|
prompt, grounds = judge.judge_prompt(self.brief(self.LONG, 0, records="r" * (judge.JUDGE_PROMPT_CHARS - self.overhead() - len(whole))))
|
|
self.assertEqual(grounds.shown, whole)
|
|
self.assertLessEqual(len(prompt), judge.JUDGE_PROMPT_CHARS)
|
|
|
|
def test_shows_the_floor_where_the_records_leave_exactly_that(self):
|
|
at_floor = judge.JUDGE_PROMPT_CHARS - self.overhead() - judge.TRANSCRIPT_FLOOR
|
|
self.assertEqual(len(judge.judge_prompt(self.brief(self.LONG, 0, records="r" * at_floor))[1].shown), judge.TRANSCRIPT_FLOOR)
|
|
self.assertIsInstance(judge.judge_prompt(self.brief(self.LONG, 0, records="r" * (at_floor + 1))), Failure)
|
|
|
|
def test_shows_a_transcript_shorter_than_the_floor_whole(self):
|
|
_, grounds = judge.judge_prompt(self.brief([assistant({"type": "text", "text": "Merged."})], 0, records="r" * (judge.JUDGE_PROMPT_CHARS - 10000)))
|
|
self.assertEqual(grounds.shown, "Merged.")
|
|
|
|
def test_the_judge_prompt_and_schema_call_the_worker_the_worker(self):
|
|
prompt, _ = judge.judge_prompt(self.brief([assistant({"type": "text", "text": "done"})], 1))
|
|
self.assertNotRegex(prompt + json.dumps(judge.judge_schema("a-driver")), r"\b[Aa]gent")
|
|
|
|
def test_the_judge_prompt_names_the_entry_points_answer_destinations_in_both_rules_on_a_recorded_answer(self):
|
|
prompt, _ = judge.judge_prompt(self.brief([assistant({"type": "text", "text": "done"})], 0, "the ledger"))
|
|
self.assertIn("is recorded in the ledger at the one place it governs", prompt)
|
|
self.assertIn("the worker writing it into the ledger, but not into one of the human's rules", prompt)
|
|
|
|
def test_shows_the_judge_where_the_worker_last_stepped_up_and_checks_an_escalation_only_after_it(self):
|
|
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "round 3: 🔴"})]
|
|
prompt, grounds = judge.judge_prompt(self.brief(entries, 1))
|
|
self.assertRegex(prompt, "round 2: 🔴\n\n\\[step\\] [^\n]+\n\nround 3: 🔴\n\n")
|
|
self.assertEqual(grounds.after_mark, "round 3: 🔴")
|
|
prompt, grounds = judge.judge_prompt(self.brief(entries, 0))
|
|
self.assertNotIn("stepped the worker up", prompt)
|
|
self.assertEqual(grounds.shown, "round 2: 🔴\n\nround 3: 🔴")
|
|
self.assertEqual(grounds.after_mark, "round 2: 🔴\n\nround 3: 🔴")
|
|
|
|
def test_drops_an_escalation_quote_from_after_the_step_up_that_the_tail_it_shows_cuts_off(self):
|
|
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "old " + "x" * judge.JUDGE_PROMPT_CHARS}), assistant({"type": "text", "text": "new"})]
|
|
_, grounds = judge.judge_prompt(self.brief(entries, 1))
|
|
self.assertEqual(judge.quotes_checked(Unfinished("e", "old x", entries_read=3), grounds), Unfinished("e", entries_read=3))
|
|
self.assertEqual(judge.quotes_checked(Unfinished("e", "new", entries_read=3), grounds), Unfinished("e", "new", entries_read=3))
|
|
|
|
def test_shows_a_typed_answer_before_the_step_up_as_the_humans(self):
|
|
typed = PROMPTS.typed("Which one?", "Use B.", Route(Rung("worker-model")), None, False)
|
|
entries = [{**human(typed), "promptSource": "sdk"}, assistant({"type": "text", "text": "round 3: 🔴"})]
|
|
_, grounds = judge.judge_prompt(self.brief(entries, 1))
|
|
self.assertIn("\n\n[human] Use B.\n\n[step] ", grounds.shown)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|