Files

121 lines
12 KiB
Python

import pathlib
import sys
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from model_routing.ladder import ProjectRung, Rung
from chunk_driver import prompts
from model_routing.ladder_file import read_ladder, SHIPPED
from model_routing.route import Route
LADDER = read_ladder(SHIPPED)
# The ladder's rungs around the project's: the one below it, its own, and those one to three steps up.
SONNET = Rung("claude-sonnet-5-5", "high")
OPUS = Rung("claude-opus-5-5", "medium")
FABLE_LOW = Rung("claude-fable-5-2", "low")
FABLE_MEDIUM = Rung("claude-fable-5-2", "medium")
OPUS_XHIGH = Rung("claude-opus-5-5", "xhigh")
OPUS_PROJECT = ProjectRung(LADDER, OPUS)
# The sentence with which an entry point that exists only here asks its worker to name the chunk's items.
NAMING = "Name the items."
SCAN = " Run a full scan only from such a subagent, since the scan's agents run on the model that starts them."
SEATS = " Launch each comprehension seat and the security pass's subagent with the Agent tool set to model opus, effort medium, the project's rung."
MARKDOWN = " Where your branch changes only Markdown, none of it `CLAUDE.md`, `AGENTS.md`, `workflow-bench/*-prompt.md` or under `agents/`, `principles/`, `rules/` or `skills/`, launch each reviewer on the project's rung instead, with the Agent tool set to model opus, effort medium."
def held(route, project=OPUS_PROJECT):
"""The rung sentences a-driver's worker reads on route, each after a space."""
return prompts.rungs_held("a-driver", route, project)
def resumed(route, project=OPUS_PROJECT):
"""The prompt that resumes a-driver's worker on route, with no evidence of work remaining."""
return prompts.resumed("a-driver", NAMING, "", route, project)
class Resumed(unittest.TestCase):
def test_resumes_the_worker_saying_when_the_chunk_is_finished(self):
self.assertTrue(resumed(Route(Rung("m")), None).startswith("Continue the task; the previous run ended before the chunk was finished. The chunk is finished once"))
class ReviewLaunches(unittest.TestCase):
def test_keeps_every_review_on_the_projects_rung_while_the_worker_runs_below_it(self):
below = held(Route(SONNET, 3, 2, 4))
self.assertTrue(below.startswith(f" Launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool set to model opus, effort medium, the project's rung.{SCAN} Your rung is below the project's: launch each subagent that files or scores plan items on the rung `~/.claude/bin/subagent-rung --plan claude-opus-5-5 <L>` prints. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger"), below)
self.assertNotIn("Launch each", held(Route(OPUS, 3, 2, 4)))
def test_launches_reviewers_on_the_rung_the_items_e_plus_i_picks_above_the_workers(self):
level = held(Route(OPUS, 3, 2, 6))
self.assertTrue(level.startswith(f" Launch each reviewer with the Agent tool set to model fable, effort medium, the rung item 3's E + I of 6 picks. Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{MARKDOWN} When an escalation trigger"), level)
below = held(Route(SONNET, 3, 2, 5))
self.assertTrue(below.startswith(f" Launch each reviewer with the Agent tool set to model fable, effort low, the rung item 3's E + I of 5 picks.{SEATS}{SCAN} Your rung is below the project's:"), below)
self.assertNotIn("Launch each reviewer", held(Route(FABLE_MEDIUM, 3, 4, 6)))
def test_names_the_reviewers_rung_each_higher_e_plus_i_an_item_taken_in_picks(self):
taking_in = " Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8."
self.assertTrue(held(Route(OPUS, 3, 2, 4)).startswith(f"{taking_in} When an escalation trigger"))
self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.", held(Route(OPUS, 3, 2, 6)))
self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.", held(Route(FABLE_MEDIUM, 3, 4, 4)))
self.assertNotIn("Where an item you take into", held(Route(OPUS, 3, 2, 8)))
self.assertNotIn("Where an item you take into", held(Route(Rung("m")), None))
def test_launches_reviewers_on_the_rung_the_highest_e_plus_i_taken_in_picks(self):
self.assertIn(f"a-driver runs you on claude-opus-5-5 at medium effort. Launch each reviewer with the Agent tool set to model opus, effort xhigh, the rung picked by an item of E + I 7 taken into the chunk. Where an item you take into the chunk has an E + I of 8 or more, launch each reviewer with the Agent tool set to `model fable, effort high` from 8.{MARKDOWN} When an escalation trigger", resumed(Route(OPUS, 3, 2, 5, escape_and_impact_taken_in=7)))
self.assertIn(" Launch each reviewer with the Agent tool set to model fable, effort low, the rung picked by an item of E + I 5 taken into the chunk.", held(Route(OPUS, escape_and_impact_taken_in=5)))
def test_keeps_seats_and_the_security_pass_on_the_projects_rung_while_the_worker_runs_above_it(self):
above = Route(FABLE_LOW, 3, 3, 4)
sentences = f"{SEATS}{SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{MARKDOWN} When an escalation trigger"
self.assertIn(f"{prompts.ANSWERS_RECORDED}. {NAMING} a-driver runs you on claude-fable-5-2 at low effort.{sentences}", resumed(above))
self.assertNotIn("the project's", resumed(Route(OPUS, 3, 2, 4)))
self.assertNotIn("runs you on", resumed(Route(Rung("m")), None))
self.assertTrue(held(above).startswith(sentences), held(above))
def test_lets_a_markdown_only_branch_review_on_the_projects_rung_where_the_reviewers_run_above_it(self):
self.assertIn(MARKDOWN, held(Route(OPUS, 3, 2, 7)))
self.assertIn(MARKDOWN, held(Route(FABLE_LOW, 3, 3, 4)))
self.assertNotIn("only Markdown", held(Route(OPUS, 3, 2, 4)))
class Escalation(unittest.TestCase):
def test_names_the_rungs_an_escalation_trigger_lets_the_worker_hand_a_task_to_at_most_two_above_its_items(self):
routed = held(Route(OPUS, 3, 2, 4))
self.assertTrue(routed.startswith(" Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort low`, one rung above yours; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number."), routed)
self.assertIn(" When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort medium`, one rung above yours, and on each further trigger one rung higher, up to `model opus, effort xhigh`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", held(Route(FABLE_LOW, 3, 3, 4)))
self.assertIn(" When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort xhigh`, one rung above yours; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", held(Route(FABLE_MEDIUM, 3, 3, 4)))
self.assertNotIn("escalation", held(Route(Rung("claude-fable-5-2", "max"), 3, 4, 8)))
self.assertIn(" An escalation trigger in `~/.claude/CLAUDE.md` lets you hand the hard part of a task to a subagent with the Agent tool set to `model fable, effort high`, one rung above yours, only once the human agrees; a-driver will not resume you higher for a trigger either, so ask the human for that yes.", held(Route(OPUS_XHIGH, 3, 3, 4)))
self.assertIn("set to `model fable, effort max`, one rung above yours, only once the human agrees;", held(Route(Rung("claude-fable-5-2", "high"), 3, 4, 5)), "one rung below the top still names the rung above")
self.assertNotIn("rerun", held(Route(Rung("claude-opus-5-5", "low"), 3, 3, 4)), "a worker off the ladder is told no rerun rung")
self.assertIn("hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort high`, one rung above yours;", held(Route(OPUS_XHIGH, 3, 4, 5)))
self.assertNotIn("escalation", held(Route(Rung("m")), None))
self.assertNotIn("escalation", held(Route(Rung("claude-opus-5-5", "low"), 3, 3, 4)))
unrouted = held(Route(OPUS))
self.assertTrue(unrouted.startswith(" Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort low`, one rung above yours, and on each further trigger one rung higher, up to `model fable, effort medium`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number. Where the hard part is the rest of the chunk, end your run instead, asking a-driver for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `model fable, effort medium`. A reviewer"), unrouted)
def test_offers_the_worker_a_higher_rung_for_the_rest_of_the_chunk_up_to_the_escalation_cap(self):
offer = lambda cap: f" Where the hard part is the rest of the chunk, end your run instead, asking a-driver for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `{cap}`."
self.assertIn(f"`Escalation trigger <n>:`, n the trigger's number.{offer('model fable, effort low')}", held(Route(OPUS, 3, 2, 4)))
self.assertIn(f"`Escalation trigger <n>:`, n the trigger's number.{offer('model opus, effort xhigh')}", held(Route(FABLE_LOW, 3, 3, 4)))
self.assertNotIn("Where the hard part", held(Route(OPUS_XHIGH, 3, 3, 4)))
class Reruns(unittest.TestCase):
def test_names_the_rung_a_reviewer_or_seat_reruns_on_one_above_its_own(self):
both = " A reviewer or comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort low`."
self.assertTrue(resumed(Route(OPUS, 3, 2, 4)).endswith(both))
self.assertTrue(resumed(Route(SONNET, 3, 2, 4)).endswith(both))
self.assertTrue(held(Route(OPUS)).endswith(both))
self.assertTrue(resumed(Route(FABLE_LOW, 3, 3, 4)).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort medium`, and a comprehension seat with `model fable, effort low`."))
self.assertTrue(resumed(Route(OPUS, 3, 2, 7)).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort high`, and a comprehension seat with `model fable, effort low`."))
self.assertTrue(resumed(Route(Rung("claude-fable-5-2", "max"), 3, 4, 8)).endswith(" A comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort low`."))
self.assertNotIn("rerun", resumed(Route(Rung("m")), None))
self.assertNotIn("rerun", held(Route(Rung("m", "high"), 3, 4, 5), None))
fable = Rung("claude-fable-5-2", "max")
self.assertNotIn("rerun", resumed(Route(fable), ProjectRung(LADDER, fable)))
if __name__ == "__main__":
unittest.main()