Answer architecture round 1: 0 🔴, 5 🟡; one Agent-tool spelling of a rung, subagent-rung in its own package, its bounds tested, and the project's rung named again after each unfinished verdict
This commit is contained in:
@@ -54,6 +54,8 @@
|
||||
!bin/shell_command.py
|
||||
!bin/statusline.py
|
||||
!bin/subagent-rung
|
||||
!bin/subagent_rung/
|
||||
!bin/subagent_rung/*.py
|
||||
!bin/test_*.py
|
||||
!bin/todo_table.py
|
||||
!bin/transcript_archive/
|
||||
|
||||
@@ -21,18 +21,23 @@ def fresh(task: str, premises: str, route: Route, anchor: Rung | None) -> str:
|
||||
|
||||
|
||||
def routed_by(route: Route, anchor: Rung | None) -> str:
|
||||
said = f"loop-claude runs you on {route.rung}, routed by item {route.item}'s R + C of {route.risk_and_complexity}. Name each todo.md item this chunk takes, with its R + C, whether or not it is item {route.item}."
|
||||
return f"loop-claude runs you on {route.rung}, routed by item {route.item}'s R + C of {route.risk_and_complexity}. Name each todo.md item this chunk takes, with its R + C, whether or not it is item {route.item}.{off_the_projects_rung(route, anchor)}"
|
||||
|
||||
|
||||
def off_the_projects_rung(route: Route, anchor: Rung | None) -> str:
|
||||
"""Which subagents to launch on the project's rung, anchor, while the worker runs off it; empty while it runs on it."""
|
||||
if anchor is None or route.rung == anchor:
|
||||
return said
|
||||
return ""
|
||||
# Below the project's rung, the reviewers stay on it to hold quality; above it, the seats score every chunk on one rung, and the security pass spends no more than the project chose.
|
||||
below = highest(route.rung, anchor) == anchor
|
||||
launched = "each reviewer, each comprehension seat and the security pass's subagent" if below else "each comprehension seat and the security pass's subagent"
|
||||
return f"{said} Your rung is {'below' if below else 'above'} the project's, {anchor}: launch {launched} with the Agent tool's model `{anchor.family()}` and effort `{anchor.effort}`, and run a full scan only from such a subagent, since the scan's agents run on the model that starts them."
|
||||
return f" Your rung is {'below' if below else 'above'} the project's, {anchor}: launch {launched} with the Agent tool's {anchor.agent_args()}, and run a full scan only from such a subagent, since the scan's agents run on the model that starts them."
|
||||
|
||||
|
||||
def resumed(unfinished_evidence: str) -> str:
|
||||
def resumed(unfinished_evidence: str, route: Route, anchor: Rung | None) -> str:
|
||||
"""Names the project's rung again, since a step-up between cycles may have taken the worker above it."""
|
||||
ruled = f" A judge ruled the chunk unfinished, quoting this passage of your transcript as the work remaining:\n\n{quoted(unfinished_evidence)}\n\n" if unfinished_evidence else " "
|
||||
return f"Continue the task; the previous run ended before the chunk was finished.{ruled}The chunk is finished once it is merged, or nothing remains that you could do on your own. Even then, it is finished only once {ANSWERS_RECORDED}."
|
||||
return f"Continue the task; the previous run ended before the chunk was finished.{ruled}The chunk is finished once it is merged, or nothing remains that you could do on your own. Even then, it is finished only once {ANSWERS_RECORDED}.{off_the_projects_rung(route, anchor)}"
|
||||
|
||||
|
||||
def answered(question: str, answer: str) -> str:
|
||||
|
||||
@@ -88,7 +88,7 @@ def worker_start(cycle: Cycle, answered: QuotedAnswer | None) -> list[str]:
|
||||
if answered is not None:
|
||||
return worker_args(cycle.session, cycle.route.rung, prompts.answered(answered.question, answered.answer), resume=True)
|
||||
if session_open(cycle.session):
|
||||
return worker_args(cycle.session, cycle.route.rung, prompts.resumed(cycle.unfinished_evidence), resume=True)
|
||||
return worker_args(cycle.session, cycle.route.rung, prompts.resumed(cycle.unfinished_evidence, cycle.route, cycle.anchor), resume=True)
|
||||
return worker_args(cycle.session, cycle.route.rung, prompts.fresh(cycle.task, cycle.premises, cycle.route, cycle.anchor), resume=False)
|
||||
|
||||
|
||||
|
||||
@@ -49,7 +49,8 @@ such an item, or with a C missing, the chunk runs on the model's cheapest rung.
|
||||
prompt names the rung and the item, and asks the worker to name each item the chunk takes, with its
|
||||
R + C. When that rung is below the model's cheapest, the prompt also has the worker launch each
|
||||
reviewer, each comprehension seat and the security pass in subagents on the cheapest rung; when it
|
||||
is above, each comprehension seat and the security pass. The
|
||||
is above, each comprehension seat and the security pass. The prompt after each unfinished verdict
|
||||
says this again, since a step-up may have taken the worker above the cheapest rung. The
|
||||
judge reports the highest R + C the worker states, and the next cycle steps up to the rung that
|
||||
R + C picks, never down. Every second unfinished verdict on a chunk steps it up one rung. Each
|
||||
finished or unfinished verdict, stop and step-up in the archive names the worker's rung, with the
|
||||
|
||||
@@ -403,7 +403,7 @@ class Cycles(unittest.TestCase):
|
||||
self.assertEqual(run.code, 130)
|
||||
first = next(call for call in run.calls if call["role"] == "worker")
|
||||
self.assertEqual((argument(first, "--model"), argument(first, "--effort")), ("claude-sonnet-5-5", "high"))
|
||||
self.assertIn("Your rung is below the project's, claude-opus-5-5 at medium effort: launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool's model `opus` and effort `medium`", argument(first, "-p"))
|
||||
self.assertIn("Your rung is below the project's, claude-opus-5-5 at medium effort: launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool's model opus, effort medium", argument(first, "-p"))
|
||||
|
||||
def test_a_rerun_resumes_an_open_chunk_on_the_rung_it_last_ran(self):
|
||||
checkout = Checkout(self)
|
||||
|
||||
@@ -540,20 +540,23 @@ class WorkerArgs(unittest.TestCase):
|
||||
def test_keeps_reviewers_on_the_projects_rung_while_the_worker_runs_below_it(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
below = prompts.fresh("Do it.", "## Goals", routing.Route(Rung("claude-sonnet-5-5", "high"), 3, 5), opus)
|
||||
self.assertIn("whether or not it is item 3. Your rung is below the project's, claude-opus-5-5 at medium effort: launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool's model `opus` and effort `medium`, and run a full scan only from such a subagent, since the scan's agents run on the model that starts them.\n\n", below)
|
||||
self.assertIn("whether or not it is item 3. Your rung is below the project's, claude-opus-5-5 at medium effort: launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool's model opus, effort medium, and run a full scan only from such a subagent, since the scan's agents run on the model that starts them.\n\n", below)
|
||||
self.assertNotIn("the project's", prompts.fresh("Do it.", "## Goals", routing.Route(opus, 3, 9), opus))
|
||||
|
||||
def test_keeps_seats_and_the_security_pass_on_the_projects_rung_while_the_worker_runs_above_it(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
above = prompts.fresh("Do it.", "## Goals", routing.Route(Rung("claude-opus-5-5", "high"), 3, 11), opus)
|
||||
self.assertIn("whether or not it is item 3. Your rung is above the project's, claude-opus-5-5 at medium effort: launch each comprehension seat and the security pass's subagent with the Agent tool's model `opus` and effort `medium`, and run a full scan only from such a subagent, since the scan's agents run on the model that starts them.\n\n", above)
|
||||
stepped_up = prompts.resumed("", routing.Route(Rung("claude-opus-5-5", "high"), 3, 9), opus)
|
||||
self.assertTrue(stepped_up.endswith(" Your rung is above the project's, claude-opus-5-5 at medium effort: launch each comprehension seat and the security pass's subagent with the Agent tool's model opus, effort medium, and run a full scan only from such a subagent, since the scan's agents run on the model that starts them."), stepped_up)
|
||||
self.assertNotIn("the project's", prompts.resumed("", routing.Route(opus, 3, 9), opus))
|
||||
self.assertIn("whether or not it is item 3. Your rung is above the project's, claude-opus-5-5 at medium effort: launch each comprehension seat and the security pass's subagent with the Agent tool's model opus, effort medium, and run a full scan only from such a subagent, since the scan's agents run on the model that starts them.\n\n", above)
|
||||
|
||||
def test_opens_a_fresh_session_by_name_and_resumes_an_open_one(self):
|
||||
opened = session.session_named("A proj loop-claude", pathlib.Path("/h/.claude/projects"))
|
||||
fresh = worker.worker_args(opened, Rung("m"), prompts.fresh("Do it.", "## Goals\n1. g\n\n## Audience\nme", routing.Route(Rung("m")), None), resume=False)
|
||||
self.assertEqual(fresh[fresh.index("--session-id") + 1], str(opened.id))
|
||||
self.assertEqual(fresh[fresh.index("--name") + 1], "A proj loop-claude")
|
||||
resumed = worker.worker_args(opened, Rung("m"), prompts.resumed(""), resume=True)
|
||||
resumed = worker.worker_args(opened, Rung("m"), prompts.resumed("", routing.Route(Rung("m")), None), resume=True)
|
||||
self.assertEqual(resumed[resumed.index("--resume") + 1], str(opened.id))
|
||||
self.assertNotIn("--session-id", resumed)
|
||||
self.assertTrue(resumed[resumed.index("-p") + 1].startswith("Continue the task; the previous run ended before the chunk was finished. The chunk is finished once"))
|
||||
|
||||
@@ -11,9 +11,9 @@ class Rung:
|
||||
def flags(self) -> list[str]:
|
||||
return ["--model", self.model, *(["--effort", self.effort] if self.effort else [])]
|
||||
|
||||
def family(self) -> str:
|
||||
"""The model's family, such as opus, which the Agent tool takes as its model and runs that family's latest model; each rung names its family's latest, so the two match."""
|
||||
return self.model.split("-")[1]
|
||||
def agent_args(self) -> str:
|
||||
"""The rung as the Agent tool takes it, which runs the named family's latest model; each rung names its family's latest, so the two match."""
|
||||
return f"model {self.model.split('-')[1]}, effort {self.effort}"
|
||||
|
||||
# The archive records this wording, and loop_claude.routing.ROUTED reads it back to resume a chunk on its rung.
|
||||
def __str__(self) -> str:
|
||||
|
||||
+2
-2
@@ -4,6 +4,6 @@ import sys
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
||||
|
||||
from model_routing import subagent
|
||||
from subagent_rung import main
|
||||
|
||||
sys.exit(subagent.main(sys.argv[1:]))
|
||||
sys.exit(main.main(sys.argv[1:]))
|
||||
|
||||
@@ -21,10 +21,7 @@ USAGE = "usage: subagent-rung <model> <R+C>, R+C a whole number from 2 to 20"
|
||||
def agent_args(model: str, risk_and_complexity: int) -> str | None:
|
||||
"""The Agent tool's model and effort for the rung risk_and_complexity picks from model's cheapest rung; None for a model off the ladder."""
|
||||
anchor = project_rung(model)
|
||||
if anchor is None:
|
||||
return None
|
||||
rung = picked(anchor, risk_and_complexity)
|
||||
return f"model {rung.family()}, effort {rung.effort}"
|
||||
return picked(anchor, risk_and_complexity).agent_args() if anchor else None
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
@@ -5,7 +5,7 @@ import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from model_routing import subagent
|
||||
from subagent_rung import main
|
||||
|
||||
TOOL = pathlib.Path(__file__).resolve().parents[1] / "subagent-rung"
|
||||
|
||||
@@ -15,16 +15,16 @@ class SubagentRung(unittest.TestCase):
|
||||
return subprocess.run([sys.executable, str(TOOL), *args], capture_output=True, text=True)
|
||||
|
||||
def test_prints_the_agent_tools_model_and_effort_for_the_rung_an_r_plus_c_picks(self):
|
||||
self.assertEqual(subagent.agent_args("claude-opus-5-5", 9), "model opus, effort medium")
|
||||
self.assertEqual(subagent.agent_args("claude-opus-5-5", 7), "model sonnet, effort high")
|
||||
self.assertEqual(subagent.agent_args("claude-opus-5-5", 20), "model fable, effort max")
|
||||
self.assertEqual(subagent.agent_args("claude-haiku-5-5", 2), "model haiku, effort low")
|
||||
self.assertEqual(main.agent_args("claude-opus-5-5", 9), "model opus, effort medium")
|
||||
self.assertEqual(main.agent_args("claude-opus-5-5", 7), "model sonnet, effort high")
|
||||
self.assertEqual(main.agent_args("claude-opus-5-5", 20), "model fable, effort max")
|
||||
self.assertEqual(main.agent_args("claude-haiku-5-5", 2), "model haiku, effort low")
|
||||
|
||||
def test_picks_nothing_for_a_model_off_the_ladder(self):
|
||||
self.assertIsNone(subagent.agent_args("claude-opus-5", 9))
|
||||
self.assertIsNone(main.agent_args("claude-opus-5", 9))
|
||||
|
||||
def test_refuses_an_r_plus_c_outside_2_to_20_or_a_wrong_count_of_arguments(self):
|
||||
for argv in (["claude-opus-5-5", "1"], ["claude-opus-5-5", "21"], ["claude-opus-5-5", "x"], ["claude-opus-5-5"], ["9", "claude-opus-5-5"]):
|
||||
for argv in (["claude-opus-5-5", "1"], ["claude-opus-5-5", "02"], ["claude-opus-5-5", "21"], ["claude-opus-5-5", "x"], ["claude-opus-5-5"], ["9", "claude-opus-5-5"]):
|
||||
result = self.run_tool(*argv)
|
||||
self.assertEqual((result.returncode, result.stdout), (1, ""), argv)
|
||||
self.assertIn("usage: subagent-rung <model> <R+C>", result.stderr)
|
||||
@@ -35,7 +35,10 @@ class SubagentRung(unittest.TestCase):
|
||||
off = self.run_tool("claude-opus-5", "12")
|
||||
self.assertEqual((off.returncode, off.stdout), (0, ""))
|
||||
self.assertEqual(off.stderr, "subagent-rung: claude-opus-5 is not on the routing ladder; pass the Agent tool no model or effort\n")
|
||||
self.assertIn("usage: subagent-rung <model> <R+C>", self.run_tool("--help").stdout)
|
||||
for bound, rung in (("2", "model haiku, effort xhigh\n"), ("20", "model fable, effort max\n")):
|
||||
self.assertEqual(self.run_tool("claude-opus-5-5", bound).stdout, rung)
|
||||
for flag in ("-h", "--help"):
|
||||
self.assertIn("usage: subagent-rung <model> <R+C>", self.run_tool(flag).stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
+1
-1
@@ -178,7 +178,7 @@ The rungs, least competent first: Haiku 5.5 at low, medium, high, xhigh and max;
|
||||
|
||||
A chunk's worker runs on the rung its item picks, below the project's rung too. Only chunks run below the project's rung can prove that a cheaper worker holds quality (goal 1.1). So on 2026-10-14 lilleman chose a trial on real chunks: a replay of past chunks spends on work nothing ships (goal 1.4). Review holds quality meanwhile. While the worker runs below the project's rung, loop-claude's prompt has it launch each reviewer, each comprehension seat and the security pass's subagent on the project's rung, through the Agent tool's model and effort. It runs a full scan only from such a subagent, because the `claude-security` plugin's researchers and verifiers inherit the model that starts them. A subagent that cannot run the scan leaves it unavailable, so larv-review runs `/security-review` in its place. The cost of a cheaper worker shows as extra review rounds and unfinished verdicts, which `todo.md` item 94 is to measure, interrupting lilleman when a rung takes more of either than the rung above. A project's rung is its model's cheapest rung, which for Opus 5.5 is its default effort, medium. lilleman chose medium knowing `settings.json` runs Opus 5.5 at xhigh, so a routed worker runs below the effort an interactive session takes.
|
||||
|
||||
A model off the ladder routes nothing: claude runs it at the effort its settings give. R + C steps a task from the project's rung: 2–4 three rungs down, 5–6 two down, 7–8 one down, 9–10 none, 11–12 one up, 13–14 two up, 15–16 three up, 17 and above four up. A step past either end stops there. The bands spread the R + C that `todo.md`'s items held on 2026-10-14, 2 to 12, across the rungs, so most tasks step down. S never moves a rung, so a huge task with low R and C runs cheap. While the worker runs above the project's rung, the prompt has it launch the comprehension seats and the security pass's subagent on the project's rung too: the seats then score every chunk on one rung, and the security pass spends no more than the project chose. The other reviewers inherit the worker's higher rung until item 90 routes them. `score-outcomes` had too few finished chunks to ground the bands, and a change to the ladder or the bands is lilleman's.
|
||||
A model off the ladder routes nothing: claude runs it at the effort its settings give. R + C steps a task from the project's rung: 2–4 three rungs down, 5–6 two down, 7–8 one down, 9–10 none, 11–12 one up, 13–14 two up, 15–16 three up, 17 and above four up. A step past either end stops there. The bands spread the R + C that `todo.md`'s items held on 2026-10-14, 2 to 12, across the rungs, so most tasks step down. S never moves a rung, so a huge task with low R and C runs cheap. While the worker runs above the project's rung, loop-claude's prompt has it launch the comprehension seats and the security pass's subagent on the project's rung too, and the prompt after each unfinished verdict says so again, since a step-up may have taken the worker above it: the seats then score every chunk on one rung, and the security pass spends no more than the project chose. The other reviewers inherit the worker's higher rung until item 90 routes them. `score-outcomes` had too few finished chunks to ground the bands, and a change to the ladder or the bands is lilleman's.
|
||||
|
||||
The worker delegates a task only when it needs just the task's conclusion: a search or read across many files, or work independent enough to run in parallel. Edits, and whatever it must check line by line, stay in its own thread, since it would read a delegated diff again to check it. Measured over archived worker transcripts, reports cost a median 8% of the worker's peak context, and `sed -n` and `grep` output made up 41% of its Bash results, so delegated searches save the most context. It scores each delegated task's R and C as for an item, and `bin/subagent-rung` turns them into the Agent tool's model and effort, counted from the project's rung. The rule names both, because the Agent tool sets a subagent's effort only when instructions ask. `CLAUDE.md` states the rule, since it holds in every session.
|
||||
|
||||
|
||||
@@ -87,7 +87,7 @@ Release 0.2.0 takes items 125, 114 and 124 first, whatever their scores: Shape i
|
||||
|
||||
### 94. Every 20th cycle, have loop-claude measure token spend and routing outcomes, file them here, and interrupt the human only when they call for a routing change.
|
||||
|
||||
Count the cycles across all projects in loop-claude's archive, which holds every project's transcripts on the machine. Measure in code, as `bin/reading-cost` does, so the check itself costs no model run (goal 1.4): tokens by project, subagent type, rung and tool, and per routed chunk its review rounds and unfinished verdicts. A loop on another project cannot commit here, so it writes the figures to the archive, and this repo's next chunk files or refreshes one plan item from them. The ladder's thresholds and the delegation rule are decisions, so a change to them is the human's. Interrupt when chunks on a rung took more review rounds or unfinished verdicts than chunks on the rung above, or when subagent reports fill more of the worker's peak context than the searches they replaced would have; never twice for the same figures (goal 1.2).
|
||||
Count the cycles across all projects in loop-claude's archive, which holds every project's transcripts on the machine. Measure in code, as `bin/reading-cost` does, so the check itself costs no model run (goal 1.4): tokens by project, subagent type, rung and tool, and per routed chunk its review rounds and unfinished verdicts, and the rungs its worker delegated on, so a cheap subagent whose conclusions cost review rounds shows. A loop on another project cannot commit here, so it writes the figures to the archive, and this repo's next chunk files or refreshes one plan item from them. The ladder's thresholds and the delegation rule are decisions, so a change to them is the human's. Interrupt when chunks on a rung took more review rounds or unfinished verdicts than chunks on the rung above, or when subagent reports fill more of the worker's peak context than the searches they replaced would have; never twice for the same figures (goal 1.2).
|
||||
|
||||
### 90. Route each reviewer by the chunk's R + C, once a replay of past chunks shows the routed reviewer finds the 🔴 and 🟡 findings on record.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user