Answer stability round 1: 0 🔴, 3 🟡 (0 misread); the prompt tests route on rungs the new steps reach, and pin the rung the human must agree to
This commit is contained in:
@@ -127,7 +127,7 @@ def escalation(route: Route, project: ProjectRung | None) -> list[str]:
|
||||
"""The prompt's escalation sentences, the marker's among them. None off the ladder or on its top rung. Where escalation_rungs opens none, the worker escalates only with the human's yes.
|
||||
Else they name the rungs escalation_rungs opens to a subagent, and offer to resume the worker one rung up for the rest of the chunk, up to the highest of those rungs.
|
||||
|
||||
For a worker one rung above the project's, routed by item 3 of L 4, they read: `When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `<rung>`, one rung above yours, and on each further trigger one rung higher, up to `<rung>`; above that, ask the human first.`, `Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.` and `Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `<rung>`.`
|
||||
For a worker one rung above the project's, routed by item 3 of L 3, they read: `When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `<rung>`, one rung above yours, and on each further trigger one rung higher, up to `<rung>`; above that, ask the human first.`, `Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.` and `Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `<rung>`.`
|
||||
"""
|
||||
if project is None or route.rung not in project.ladder.rungs[:-1]:
|
||||
return []
|
||||
|
||||
@@ -1760,12 +1760,12 @@ class WorkerArgs(unittest.TestCase):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
below = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 4), OPUS_PROJECT, self.FACTS)
|
||||
self.assertIn(f"item 3's L of 2. Launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool set to model opus, effort medium, the project's rung.{self.SCAN} Your rung is below the project's: launch each subagent that files or scores plan items on the rung `~/.claude/bin/subagent-rung --plan claude-opus-5-5 <L>` prints. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger", below)
|
||||
self.assertNotIn("Launch each", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), OPUS_PROJECT, self.FACTS))
|
||||
self.assertNotIn("Launch each", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS))
|
||||
|
||||
def test_launches_reviewers_on_the_rung_the_items_e_plus_i_picks_above_the_workers(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
level = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 6), OPUS_PROJECT, self.FACTS)
|
||||
self.assertIn(f"item 3's L of 3. Launch each reviewer with the Agent tool set to model fable, effort medium, the rung item 3's E + I of 6 picks. Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", level)
|
||||
level = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 6), OPUS_PROJECT, self.FACTS)
|
||||
self.assertIn(f"item 3's L of 2. Launch each reviewer with the Agent tool set to model fable, effort medium, the rung item 3's E + I of 6 picks. Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", level)
|
||||
below = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 5), OPUS_PROJECT, self.FACTS)
|
||||
self.assertIn(f"item 3's L of 2. Launch each reviewer with the Agent tool set to model fable, effort low, the rung item 3's E + I of 5 picks.{self.SEATS}{self.SCAN} Your rung is below the project's:", below)
|
||||
self.assertNotIn("Launch each reviewer", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 4, 6), OPUS_PROJECT))
|
||||
@@ -1773,35 +1773,35 @@ class WorkerArgs(unittest.TestCase):
|
||||
def test_names_the_reviewers_rung_each_higher_e_plus_i_an_item_taken_in_picks(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
taking_in = " Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8."
|
||||
self.assertIn(f"item 3's L of 3.{taking_in} When an escalation trigger", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), OPUS_PROJECT, self.FACTS))
|
||||
self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.", prompts.resumed("", Route(opus, 3, 3, 6), OPUS_PROJECT))
|
||||
self.assertIn(f"item 3's L of 2.{taking_in} When an escalation trigger", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS))
|
||||
self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.", prompts.resumed("", Route(opus, 3, 2, 6), OPUS_PROJECT))
|
||||
self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 4, 4), OPUS_PROJECT))
|
||||
self.assertNotIn("Where an item you take into", prompts.resumed("", Route(opus, 3, 3, 8), OPUS_PROJECT))
|
||||
self.assertNotIn("Where an item you take into", prompts.resumed("", Route(opus, 3, 2, 8), OPUS_PROJECT))
|
||||
self.assertNotIn("Where an item you take into", prompts.resumed("", Route(Rung("m")), None))
|
||||
|
||||
def test_launches_reviewers_on_the_rung_the_highest_e_plus_i_taken_in_picks(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
self.assertIn(f"loop-claude runs you on claude-opus-5-5 at medium effort. Launch each reviewer with the Agent tool set to model opus, effort xhigh, the rung picked by an item of E + I 7 taken into the chunk. Where an item you take into the chunk has an E + I of 8 or more, launch each reviewer with the Agent tool set to `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", prompts.resumed("", Route(opus, 3, 3, 5, escape_and_impact_taken_in=7), OPUS_PROJECT))
|
||||
self.assertIn(f"loop-claude runs you on claude-opus-5-5 at medium effort. Launch each reviewer with the Agent tool set to model opus, effort xhigh, the rung picked by an item of E + I 7 taken into the chunk. Where an item you take into the chunk has an E + I of 8 or more, launch each reviewer with the Agent tool set to `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", prompts.resumed("", Route(opus, 3, 2, 5, escape_and_impact_taken_in=7), OPUS_PROJECT))
|
||||
self.assertIn(" Launch each reviewer with the Agent tool set to model fable, effort low, the rung picked by an item of E + I 5 taken into the chunk.", prompts.resumed("", Route(opus, escape_and_impact_taken_in=5), OPUS_PROJECT))
|
||||
|
||||
def test_keeps_seats_and_the_security_pass_on_the_projects_rung_while_the_worker_runs_above_it(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
above = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-fable-5-2", "low"), 3, 4, 4), OPUS_PROJECT, self.FACTS)
|
||||
above = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT, self.FACTS)
|
||||
stepped_up = prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT)
|
||||
self.assertIn(f"{prompts.ANSWERS_RECORDED}. {prompts.NAME_THE_ITEMS} loop-claude runs you on claude-fable-5-2 at low effort.{self.SEATS}{self.SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", stepped_up)
|
||||
self.assertNotIn("the project's", prompts.resumed("", Route(opus, 3, 3, 4), OPUS_PROJECT))
|
||||
self.assertNotIn("the project's", prompts.resumed("", Route(opus, 3, 2, 4), OPUS_PROJECT))
|
||||
self.assertNotIn("runs you on", prompts.resumed("", Route(Rung("m")), None))
|
||||
self.assertIn(f"item 3's L of 4.{self.SEATS}{self.SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", above)
|
||||
self.assertIn(f"item 3's L of 3.{self.SEATS}{self.SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", above)
|
||||
|
||||
def test_names_the_rungs_an_escalation_trigger_lets_the_worker_hand_a_task_to_at_most_two_above_its_items(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
fresh = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), OPUS_PROJECT, self.FACTS)
|
||||
self.assertIn("item 3's L of 3. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort low`, one rung above yours, and on each further trigger one rung higher, up to `model opus, effort xhigh`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", fresh)
|
||||
fresh = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS)
|
||||
self.assertIn("item 3's L of 2. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort low`, one rung above yours; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", fresh)
|
||||
self.assertIn(" When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort medium`, one rung above yours, and on each further trigger one rung higher, up to `model opus, effort xhigh`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT))
|
||||
self.assertIn(" When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort xhigh`, one rung above yours; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 3, 4), OPUS_PROJECT))
|
||||
self.assertNotIn("escalation", prompts.resumed("", Route(Rung("claude-fable-5-2", "max"), 3, 4, 8), OPUS_PROJECT))
|
||||
self.assertIn("only once the human agrees", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 3, 4), OPUS_PROJECT))
|
||||
self.assertIn("only once the human agrees", prompts.resumed("", Route(Rung("claude-fable-5-2", "high"), 3, 4, 5), OPUS_PROJECT), "one rung below the top still names the rung above")
|
||||
self.assertIn(" An escalation trigger in `~/.claude/CLAUDE.md` lets you hand the hard part of a task to a subagent with the Agent tool set to `model fable, effort high`, one rung above yours, only once the human agrees;", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 3, 4), OPUS_PROJECT))
|
||||
self.assertIn("set to `model fable, effort max`, one rung above yours, only once the human agrees;", prompts.resumed("", Route(Rung("claude-fable-5-2", "high"), 3, 4, 5), OPUS_PROJECT), "one rung below the top still names the rung above")
|
||||
self.assertNotIn("rerun", prompts.resumed("", Route(Rung("claude-opus-5-5", "low"), 3, 3, 4), OPUS_PROJECT), "a worker off the ladder is told no rerun rung")
|
||||
self.assertIn("hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort high`, one rung above yours;", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 4, 5), OPUS_PROJECT))
|
||||
self.assertNotIn("escalation", prompts.resumed("", Route(Rung("m")), None))
|
||||
@@ -1810,30 +1810,30 @@ class WorkerArgs(unittest.TestCase):
|
||||
|
||||
def test_offers_the_worker_a_higher_rung_for_the_rest_of_the_chunk_up_to_the_escalation_cap(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
offer = " Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `model opus, effort xhigh`."
|
||||
self.assertIn(f"`Escalation trigger <n>:`, n the trigger's number.{offer}", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), OPUS_PROJECT, self.FACTS))
|
||||
self.assertIn(f"`Escalation trigger <n>:`, n the trigger's number.{offer}", prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT))
|
||||
offer = lambda cap: f" Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `{cap}`."
|
||||
self.assertIn(f"`Escalation trigger <n>:`, n the trigger's number.{offer('model fable, effort low')}", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS))
|
||||
self.assertIn(f"`Escalation trigger <n>:`, n the trigger's number.{offer('model opus, effort xhigh')}", prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT))
|
||||
self.assertNotIn("Where the hard part", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 3, 4), OPUS_PROJECT))
|
||||
|
||||
def test_lets_a_markdown_only_branch_review_on_the_projects_rung_where_the_reviewers_run_above_it(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
markdown = " Where your branch changes only Markdown, none of it `CLAUDE.md`, `AGENTS.md`, `workflow-bench/*-prompt.md` or under `agents/`, `principles/`, `rules/` or `skills/`, launch each reviewer on the project's rung instead, with the Agent tool set to model opus, effort medium."
|
||||
self.assertIn(markdown, prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 7), OPUS_PROJECT, self.FACTS))
|
||||
self.assertIn(markdown, prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 4, 4), OPUS_PROJECT))
|
||||
self.assertNotIn("only Markdown", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), OPUS_PROJECT, self.FACTS))
|
||||
self.assertIn(markdown, prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 7), OPUS_PROJECT, self.FACTS))
|
||||
self.assertIn(markdown, prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT))
|
||||
self.assertNotIn("only Markdown", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS))
|
||||
|
||||
def test_names_the_rung_a_reviewer_or_seat_reruns_on_one_above_its_own(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
both = " A reviewer or comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort low`."
|
||||
self.assertTrue(prompts.resumed("", Route(opus, 3, 3, 4), OPUS_PROJECT).endswith(both))
|
||||
self.assertTrue(prompts.resumed("", Route(opus, 3, 2, 4), OPUS_PROJECT).endswith(both))
|
||||
self.assertTrue(prompts.resumed("", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 4), OPUS_PROJECT).endswith(both))
|
||||
self.assertTrue(prompts.fresh("Do it.", "## Goals", Route(opus), OPUS_PROJECT, self.FACTS).split(f"\n\n{self.FACTS}")[0].endswith(both))
|
||||
self.assertTrue(prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 4, 4), OPUS_PROJECT).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort medium`, and a comprehension seat with `model fable, effort low`."))
|
||||
self.assertTrue(prompts.resumed("", Route(opus, 3, 3, 7), OPUS_PROJECT).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort high`, and a comprehension seat with `model fable, effort low`."))
|
||||
self.assertTrue(prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort medium`, and a comprehension seat with `model fable, effort low`."))
|
||||
self.assertTrue(prompts.resumed("", Route(opus, 3, 2, 7), OPUS_PROJECT).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort high`, and a comprehension seat with `model fable, effort low`."))
|
||||
self.assertTrue(prompts.resumed("", Route(Rung("claude-fable-5-2", "max"), 3, 4, 8), OPUS_PROJECT).endswith(" A comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort low`."))
|
||||
self.assertNotIn("rerun", prompts.resumed("", Route(Rung("m")), None))
|
||||
self.assertNotIn("rerun", prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), None, self.FACTS))
|
||||
self.assertIn(f"{both} After your first read", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), OPUS_PROJECT, self.FACTS))
|
||||
self.assertIn(f"{both} After your first read", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS))
|
||||
fable = Rung("claude-fable-5-2", "max")
|
||||
self.assertNotIn("rerun", prompts.resumed("", Route(fable), ProjectRung(LADDER, fable)))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user