Answer product-owner round 1: 0 🔴, 2 🟡 (0 misread); each cycle on a project's own ladder says so and says where the worker's model is off it, and the errors name a model's full name and the ladder's file

This commit is contained in:
2026-11-15 19:42:49 +01:00
parent 2ecec90ec5
commit a62992d877
7 changed files with 60 additions and 27 deletions
+1 -1
View File
@@ -2,7 +2,7 @@
## Unreleased
- The routing ladder is now a file, `bin/model_routing/routing-ladder.json`, and a project may run on a ladder of its own: a `routing-ladder.json` in its config's folder, `~/.claude-projects/<key>/`, replaces the shipped ladder whole for that project. loop-claude, `subagent-rung` and rule-hook read the project's ladder; the routing figures, `judge-replay` and `review-replay` read the shipped one. `subagent-rung` now reads the config of the checkout it runs in, and exits 1 where that config or the ladder holds a fault; loop-claude stops on a faulty ladder as it does on a faulty config. `loop-claude --help` shows the file's shape, and README → Install → "Set up a project" says when to write one.
- The routing ladder is now a file, `bin/model_routing/routing-ladder.json`, and a project may run on a ladder of its own: a `routing-ladder.json` in its config's folder, `~/.claude-projects/<key>/`, replaces the shipped ladder whole for that project, and loop-claude says so as each cycle starts. loop-claude, `subagent-rung` and rule-hook read the project's ladder; the routing figures, `judge-replay` and `review-replay` read the shipped one. `subagent-rung` now reads the config of the checkout it runs in, and exits 1 where that config or the ladder holds a fault; loop-claude stops on a faulty ladder as it does on a faulty config. `loop-claude --help` shows the file's shape, and README → Install → "Set up a project" says when to write one.
## 0.10.0
+13 -2
View File
@@ -17,7 +17,7 @@ from loop_claude.loop.usage import HELP
from loop_claude.loop.this_loop import Loop
from loop_state.paths import Paths
from transcript_archive.project_key import project_key
from model_routing.ladder_file import project_ladder
from model_routing.ladder_file import project_ladder_file, read_ladder, SHIPPED
from loop_claude.project.premises import read_premises
from loop_claude.project.records import read_records
from claude_code.session import session_named
@@ -81,7 +81,10 @@ def next_cycle(loop_run: LoopRun, loop: Loop) -> Cycle | Stop:
models = get_or_ask_models(project_key(loop.root, loop.paths.home), loop.paths, loop.env, loop.effects)
if isinstance(models, Stop):
return models
ladder = project_ladder(loop.root, loop.paths.home)
ladder_file = project_ladder_file(loop.root, loop.paths.home)
if isinstance(ladder_file, Failure):
return Stop(1, ladder_file.reason)
ladder = read_ladder(ladder_file)
if isinstance(ladder, Failure):
return Stop(1, ladder.reason)
archive_folder = project_archive(loop.root, loop.paths.home, loop.env)
@@ -90,12 +93,20 @@ def next_cycle(loop_run: LoopRun, loop: Loop) -> Cycle | Stop:
project = ladder.project_rung(models.worker)
route = routed(loop_run.route, ladder, project, models.worker, lambda: archived_route(archive_folder, loop), item)
say(f"\n=== cycle {loop_run.cycle_number}, worker on {route.rung} ===", loop.effects, stdout=True)
if ladder_file != SHIPPED:
say(own_ladder(ladder_file, models.worker, project is None), loop.effects)
todo = fetched_todo(loop)
show_taken_in = shown_items(loop, todo, route.item)
show_taken_in(loop_run.items_taken_in)
return Cycle(config=setup_version(loop.paths.claude_dir, loop.effects.git), effects=loop.effects, facts=facts, judge_model=models.judge, ladder=ladder, premises=premises, project=project, project_archive=archive_folder, questions=loop.listed.folder, records=records, root=loop.root, route=route, session=loop.session, show_taken_in=show_taken_in, task=loop_run.task, todo=todo, unfinished_evidence=loop_run.unfinished_evidence)
def own_ladder(file: pathlib.Path, model: str, unrouted: bool) -> str:
"""What a cycle says of a project that runs on a ladder of its own, so that a file left in place after its reason passed doesn't go unseen, nor one that leaves out the project's model, whose work the ladder then doesn't route."""
routes = f" {model} is not on it, so the worker runs on {model} unrouted; LOOP_CLAUDE_WORKER_MODEL and LOOP_CLAUDE_JUDGE_MODEL name other models for a run." if unrouted else ""
return f"routing on the project's own ladder, {file}.{routes}"
def loop_run_from(argv: list[str], cwd: pathlib.Path, effects: Effects) -> LoopRun | Stop:
if argv[:1] != [RELOAD_FLAG]:
task = " ".join(argv)
+18 -16
View File
@@ -68,22 +68,8 @@ entry, as on its first run, is asked for one on the terminal; loop-inbox doesn't
question. LOOP_CLAUDE_JUDGE_MODEL overrides the model for the judge, LOOP_CLAUDE_WORKER_MODEL for
the worker and the session /talk opens; with both set, nothing is asked.
The routing ladder is a JSON list of rungs, least competent first, each a model at an effort of low,
medium, high, xhigh or max:
[
{"model": "claude-opus-5-5", "effort": "medium"},
{"model": "claude-fable-5-2", "effort": "low"}
]
This setup ships one, bin/model_routing/routing-ladder.json. A project that keeps a
routing-ladder.json in its config's folder, ~/.claude-projects/<key>/, runs on that ladder
instead, whole: write there every rung the project may run on. loop-claude reads the project's
ladder as each cycle starts, and never commits the file. subagent-rung and rule-hook read the same
ladder; the routing figures, judge-replay and review-replay, which span every project, read the
shipped one.
Where the project's model is on its routing ladder, the worker and the session /talk opens run on
The routing ladder is a list of rungs, least competent first, each a model at an effort. Where the
project's model is on it, the worker and the session /talk opens run on
the rung that the chunk's todo.md item picks, and the judge of some
models on another; a chunk no item routes runs on the project's model. The worker's prompts name the
rungs it launches its reviews on. The worker's rung steps up as a chunk stays unfinished, when the
@@ -97,6 +83,22 @@ bin/model_routing/route.py); a rerun resumes an open chunk on the latest route n
todo.md or an archive that cannot be read or parsed is said on stderr, and the chunk is routed
without it.
The ladder is a JSON file, each rung a model's full name and an effort of low, medium, high, xhigh
or max:
[
{"model": "claude-opus-5-5", "effort": "medium"},
{"model": "claude-fable-5-2", "effort": "low"}
]
It holds one model of each family, such as opus, since the Agent tool launches a subagent or a
review by its family's name. This setup ships one, bin/model_routing/routing-ladder.json. A project
that keeps a routing-ladder.json in its config's folder, ~/.claude-projects/<key>/, runs on that
ladder instead, whole: write there every rung the project may run on. loop-claude reads the
project's ladder as each cycle starts, says on stderr where the project runs on its own, and never
commits the file. subagent-rung and rule-hook read the same ladder; the routing figures,
judge-replay and review-replay, which span every project, read the shipped one.
Once 20 cycles have closed across all projects in the archive since the last measurement, the next
loop to close one measures every finished chunk there, with no model run, and writes
routing-figures.md at the archive's root (bin/routing_figures/). Each call the figures make for a
+11 -1
View File
@@ -480,8 +480,18 @@ class Cycles(unittest.TestCase):
checkout.commit_plan(CHEAP)
run = checkout.run([worker(), judge(), worker(), judge(), worker()])
self.assertEqual(run.code, 130)
# Item 4's L of 2 steps one rung down, to Opus 5.5 on the shipped ladder; the project's holds no rung below its own, and one above it, which the second unfinished verdict steps to.
# Item 4's L of 2 steps one rung down, to Opus 5.5 on the shipped ladder; the project's ladder holds no rung below the project's rung, and one above it, which the second unfinished verdict steps to.
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "worker"], [("claude-fable-5-2", "low"), ("claude-fable-5-2", "low"), ("claude-fable-5-2", "max")])
self.assertEqual(run.stderr.count(f"routing on the project's own ladder, {ladder}.\n"), 3, "each cycle says so")
def test_says_where_the_projects_own_ladder_leaves_out_the_workers_model_and_says_nothing_on_the_shipped_one(self):
checkout = Checkout(self)
self.assertNotIn("own ladder", checkout.run([worker()]).stderr)
ladder = checkout.home / ".claude-projects" / "proj" / "routing-ladder.json"
ladder.write_text('[{"model": "claude-fable-5-2", "effort": "low"}]')
run = checkout.run([worker()])
self.assertIn(f"routing on the project's own ladder, {ladder}. worker-model is not on it, so the worker runs on worker-model unrouted; LOOP_CLAUDE_WORKER_MODEL and LOOP_CLAUDE_JUDGE_MODEL name other models for a run.\n", run.stderr)
self.assertEqual((argument(run.calls[0], "--model"), argument(run.calls[0], "--effort")), ("worker-model", None))
def test_reads_the_projects_ladder_afresh_each_cycle_and_exits_1_once_it_holds_a_fault(self):
checkout = Checkout(self)
+8
View File
@@ -2066,6 +2066,13 @@ class HelpLimits(unittest.TestCase):
self.assertIn(f"rerun to give it {close.UNFINISHED_LIMIT} more", said)
class OwnLadder(unittest.TestCase):
def test_a_cycle_on_a_projects_own_ladder_names_its_file_and_says_where_the_workers_model_is_off_it(self):
file = pathlib.Path("/h/.claude-projects/acme/routing-ladder.json")
self.assertEqual(main.own_ladder(file, "claude-fable-5-2", False), f"routing on the project's own ladder, {file}.")
self.assertEqual(main.own_ladder(file, "claude-opus-5-5", True), f"routing on the project's own ladder, {file}. claude-opus-5-5 is not on it, so the worker runs on claude-opus-5-5 unrouted; LOOP_CLAUDE_WORKER_MODEL and LOOP_CLAUDE_JUDGE_MODEL name other models for a run.")
class ExitContract(unittest.TestCase):
def test_every_exit_1_stop_has_its_clause_in_help(self):
read, git, premises_ = "a transcript cannot be read or archived", "git cannot start or finds no checkout", "no config names the checkout, its config cannot be read, or the goals file holds no premises"
@@ -2090,6 +2097,7 @@ class ExitContract(unittest.TestCase):
("loop/close.py", "closed_cycle", "1", "failure.reason", read),
("loop/main.py", "next_cycle", "1", "premises.reason", premises_),
("loop/main.py", "next_cycle", "1", "records.reason", unreadable),
("loop/main.py", "next_cycle", "1", "ladder_file.reason", premises_),
("loop/main.py", "next_cycle", "1", "ladder.reason", "the project's routing ladder cannot be read or holds a fault"),
("loop/main.py", "loop_run_from", "0 if task else 1", "HELP", "no task is given"),
("loop/main.py", "loop_run_from", "1", 'f"the edited loop-claude cannot read the state', "a reload cannot read the state the running script handed it"),
+6 -5
View File
@@ -7,8 +7,8 @@ a defect, a level from 1 to 4 scored as for a todo.md item. It prints the rung t
the model's cheapest rung, the project's rung, in the Agent tool's words, such as `model sonnet,
effort high`. --plan, for a task that files or scores todo.md items, prints the project's rung where
that L picks a lower one, as ~/.claude/rules/todo-items.md lets only the project's rung or above
plan. For a model off the routing ladder, the tool doesn't print anything on stdout, says so on stderr and
exits 0; launch the subagent with no model or effort.
plan. For a model off the routing ladder, the tool doesn't print anything on stdout, says so on stderr,
naming the ladder's file, and exits 0; launch the subagent with no model or effort.
It reads the routing ladder of the project whose checkout it runs in: routing-ladder.json in the
folder of the project's config, ~/.claude-projects/<key>/, or else the one this setup ships, as in
@@ -24,7 +24,7 @@ import sys
from failure import Failure
from model_routing.ladder import Ladder
from release_plan.todo_table import LEVELS
from model_routing.ladder_file import project_ladder
from model_routing.ladder_file import project_ladder_file, read_ladder
USAGE = f"usage: subagent-rung [--plan] <model> <L>, L a level from 1 to {LEVELS['L']}"
@@ -47,13 +47,14 @@ def main(argv: list[str], home: pathlib.Path) -> int:
print(f"subagent-rung: cannot read the arguments{' ' + ' '.join(argv) if argv else ''}; {USAGE}", file=sys.stderr)
return 1
model, likelihood = operands
ladder = project_ladder(None, home)
file = project_ladder_file(None, home)
ladder = file if isinstance(file, Failure) else read_ladder(file)
if isinstance(ladder, Failure):
print(f"subagent-rung: {ladder.reason}", file=sys.stderr)
return 1
args = agent_args(ladder, model, int(likelihood), planning)
if args is None:
print(f"subagent-rung: {model} is not on the routing ladder; pass the Agent tool no model or effort", file=sys.stderr)
print(f"subagent-rung: {model} is not on the routing ladder {file}; pass the Agent tool no model or effort", file=sys.stderr)
else:
print(args)
return 0
+3 -2
View File
@@ -39,7 +39,8 @@ class SubagentRung(unittest.TestCase):
# On the shipped ladder an L of 1 steps Fable 5.2 down to Opus 5.5, and an L of 4 up to its medium effort.
self.assertEqual([self.run_tool("claude-fable-5-2", level, cwd=checkout / "nested").stdout for level in ("1", "4")], ["model fable, effort low\n", "model fable, effort max\n"])
off = self.run_tool("claude-opus-5-5", "3", cwd=checkout)
self.assertEqual((off.returncode, off.stdout, off.stderr), (0, "", "subagent-rung: claude-opus-5-5 is not on the routing ladder; pass the Agent tool no model or effort\n"))
own = self.home / ".claude-projects" / "acme" / "routing-ladder.json"
self.assertEqual((off.returncode, off.stdout, off.stderr), (0, "", f"subagent-rung: claude-opus-5-5 is not on the routing ladder {own}; pass the Agent tool no model or effort\n"))
def test_exits_1_naming_the_file_where_the_projects_ladder_holds_a_fault(self):
checkout = self.project('[{"model": "claude-fable-5-2", "effort": "low"},]')
@@ -76,7 +77,7 @@ class SubagentRung(unittest.TestCase):
def test_the_launcher_prints_nothing_and_says_why_and_exits_0_off_the_ladder(self):
off = self.run_tool("claude-opus-5", "4")
self.assertEqual((off.returncode, off.stdout), (0, ""))
self.assertEqual(off.stderr, "subagent-rung: claude-opus-5 is not on the routing ladder; pass the Agent tool no model or effort\n")
self.assertEqual(off.stderr, f"subagent-rung: claude-opus-5 is not on the routing ladder {SHIPPED}; pass the Agent tool no model or effort\n")
def test_the_launcher_accepts_an_l_of_1(self):
self.assertEqual(self.run_tool("claude-opus-5-5", "1").stdout, "model sonnet, effort high\n")