Answer product-owner round 1: 0 🔴, 2 🟡 (0 misread); each cycle on a project's own ladder says so and says where the worker's model is off it, and the errors name a model's full name and the ladder's file
This commit is contained in:
+1
-1
@@ -2,7 +2,7 @@
|
||||
|
||||
## Unreleased
|
||||
|
||||
- The routing ladder is now a file, `bin/model_routing/routing-ladder.json`, and a project may run on a ladder of its own: a `routing-ladder.json` in its config's folder, `~/.claude-projects/<key>/`, replaces the shipped ladder whole for that project. loop-claude, `subagent-rung` and rule-hook read the project's ladder; the routing figures, `judge-replay` and `review-replay` read the shipped one. `subagent-rung` now reads the config of the checkout it runs in, and exits 1 where that config or the ladder holds a fault; loop-claude stops on a faulty ladder as it does on a faulty config. `loop-claude --help` shows the file's shape, and README → Install → "Set up a project" says when to write one.
|
||||
- The routing ladder is now a file, `bin/model_routing/routing-ladder.json`, and a project may run on a ladder of its own: a `routing-ladder.json` in its config's folder, `~/.claude-projects/<key>/`, replaces the shipped ladder whole for that project, and loop-claude says so as each cycle starts. loop-claude, `subagent-rung` and rule-hook read the project's ladder; the routing figures, `judge-replay` and `review-replay` read the shipped one. `subagent-rung` now reads the config of the checkout it runs in, and exits 1 where that config or the ladder holds a fault; loop-claude stops on a faulty ladder as it does on a faulty config. `loop-claude --help` shows the file's shape, and README → Install → "Set up a project" says when to write one.
|
||||
|
||||
## 0.10.0
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ from loop_claude.loop.usage import HELP
|
||||
from loop_claude.loop.this_loop import Loop
|
||||
from loop_state.paths import Paths
|
||||
from transcript_archive.project_key import project_key
|
||||
from model_routing.ladder_file import project_ladder
|
||||
from model_routing.ladder_file import project_ladder_file, read_ladder, SHIPPED
|
||||
from loop_claude.project.premises import read_premises
|
||||
from loop_claude.project.records import read_records
|
||||
from claude_code.session import session_named
|
||||
@@ -81,7 +81,10 @@ def next_cycle(loop_run: LoopRun, loop: Loop) -> Cycle | Stop:
|
||||
models = get_or_ask_models(project_key(loop.root, loop.paths.home), loop.paths, loop.env, loop.effects)
|
||||
if isinstance(models, Stop):
|
||||
return models
|
||||
ladder = project_ladder(loop.root, loop.paths.home)
|
||||
ladder_file = project_ladder_file(loop.root, loop.paths.home)
|
||||
if isinstance(ladder_file, Failure):
|
||||
return Stop(1, ladder_file.reason)
|
||||
ladder = read_ladder(ladder_file)
|
||||
if isinstance(ladder, Failure):
|
||||
return Stop(1, ladder.reason)
|
||||
archive_folder = project_archive(loop.root, loop.paths.home, loop.env)
|
||||
@@ -90,12 +93,20 @@ def next_cycle(loop_run: LoopRun, loop: Loop) -> Cycle | Stop:
|
||||
project = ladder.project_rung(models.worker)
|
||||
route = routed(loop_run.route, ladder, project, models.worker, lambda: archived_route(archive_folder, loop), item)
|
||||
say(f"\n=== cycle {loop_run.cycle_number}, worker on {route.rung} ===", loop.effects, stdout=True)
|
||||
if ladder_file != SHIPPED:
|
||||
say(own_ladder(ladder_file, models.worker, project is None), loop.effects)
|
||||
todo = fetched_todo(loop)
|
||||
show_taken_in = shown_items(loop, todo, route.item)
|
||||
show_taken_in(loop_run.items_taken_in)
|
||||
return Cycle(config=setup_version(loop.paths.claude_dir, loop.effects.git), effects=loop.effects, facts=facts, judge_model=models.judge, ladder=ladder, premises=premises, project=project, project_archive=archive_folder, questions=loop.listed.folder, records=records, root=loop.root, route=route, session=loop.session, show_taken_in=show_taken_in, task=loop_run.task, todo=todo, unfinished_evidence=loop_run.unfinished_evidence)
|
||||
|
||||
|
||||
def own_ladder(file: pathlib.Path, model: str, unrouted: bool) -> str:
|
||||
"""What a cycle says of a project that runs on a ladder of its own, so that a file left in place after its reason passed doesn't go unseen, nor one that leaves out the project's model, whose work the ladder then doesn't route."""
|
||||
routes = f" {model} is not on it, so the worker runs on {model} unrouted; LOOP_CLAUDE_WORKER_MODEL and LOOP_CLAUDE_JUDGE_MODEL name other models for a run." if unrouted else ""
|
||||
return f"routing on the project's own ladder, {file}.{routes}"
|
||||
|
||||
|
||||
def loop_run_from(argv: list[str], cwd: pathlib.Path, effects: Effects) -> LoopRun | Stop:
|
||||
if argv[:1] != [RELOAD_FLAG]:
|
||||
task = " ".join(argv)
|
||||
|
||||
@@ -68,22 +68,8 @@ entry, as on its first run, is asked for one on the terminal; loop-inbox doesn't
|
||||
question. LOOP_CLAUDE_JUDGE_MODEL overrides the model for the judge, LOOP_CLAUDE_WORKER_MODEL for
|
||||
the worker and the session /talk opens; with both set, nothing is asked.
|
||||
|
||||
The routing ladder is a JSON list of rungs, least competent first, each a model at an effort of low,
|
||||
medium, high, xhigh or max:
|
||||
|
||||
[
|
||||
{"model": "claude-opus-5-5", "effort": "medium"},
|
||||
{"model": "claude-fable-5-2", "effort": "low"}
|
||||
]
|
||||
|
||||
This setup ships one, bin/model_routing/routing-ladder.json. A project that keeps a
|
||||
routing-ladder.json in its config's folder, ~/.claude-projects/<key>/, runs on that ladder
|
||||
instead, whole: write there every rung the project may run on. loop-claude reads the project's
|
||||
ladder as each cycle starts, and never commits the file. subagent-rung and rule-hook read the same
|
||||
ladder; the routing figures, judge-replay and review-replay, which span every project, read the
|
||||
shipped one.
|
||||
|
||||
Where the project's model is on its routing ladder, the worker and the session /talk opens run on
|
||||
The routing ladder is a list of rungs, least competent first, each a model at an effort. Where the
|
||||
project's model is on it, the worker and the session /talk opens run on
|
||||
the rung that the chunk's todo.md item picks, and the judge of some
|
||||
models on another; a chunk no item routes runs on the project's model. The worker's prompts name the
|
||||
rungs it launches its reviews on. The worker's rung steps up as a chunk stays unfinished, when the
|
||||
@@ -97,6 +83,22 @@ bin/model_routing/route.py); a rerun resumes an open chunk on the latest route n
|
||||
todo.md or an archive that cannot be read or parsed is said on stderr, and the chunk is routed
|
||||
without it.
|
||||
|
||||
The ladder is a JSON file, each rung a model's full name and an effort of low, medium, high, xhigh
|
||||
or max:
|
||||
|
||||
[
|
||||
{"model": "claude-opus-5-5", "effort": "medium"},
|
||||
{"model": "claude-fable-5-2", "effort": "low"}
|
||||
]
|
||||
|
||||
It holds one model of each family, such as opus, since the Agent tool launches a subagent or a
|
||||
review by its family's name. This setup ships one, bin/model_routing/routing-ladder.json. A project
|
||||
that keeps a routing-ladder.json in its config's folder, ~/.claude-projects/<key>/, runs on that
|
||||
ladder instead, whole: write there every rung the project may run on. loop-claude reads the
|
||||
project's ladder as each cycle starts, says on stderr where the project runs on its own, and never
|
||||
commits the file. subagent-rung and rule-hook read the same ladder; the routing figures,
|
||||
judge-replay and review-replay, which span every project, read the shipped one.
|
||||
|
||||
Once 20 cycles have closed across all projects in the archive since the last measurement, the next
|
||||
loop to close one measures every finished chunk there, with no model run, and writes
|
||||
routing-figures.md at the archive's root (bin/routing_figures/). Each call the figures make for a
|
||||
|
||||
@@ -480,8 +480,18 @@ class Cycles(unittest.TestCase):
|
||||
checkout.commit_plan(CHEAP)
|
||||
run = checkout.run([worker(), judge(), worker(), judge(), worker()])
|
||||
self.assertEqual(run.code, 130)
|
||||
# Item 4's L of 2 steps one rung down, to Opus 5.5 on the shipped ladder; the project's holds no rung below its own, and one above it, which the second unfinished verdict steps to.
|
||||
# Item 4's L of 2 steps one rung down, to Opus 5.5 on the shipped ladder; the project's ladder holds no rung below the project's rung, and one above it, which the second unfinished verdict steps to.
|
||||
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "worker"], [("claude-fable-5-2", "low"), ("claude-fable-5-2", "low"), ("claude-fable-5-2", "max")])
|
||||
self.assertEqual(run.stderr.count(f"routing on the project's own ladder, {ladder}.\n"), 3, "each cycle says so")
|
||||
|
||||
def test_says_where_the_projects_own_ladder_leaves_out_the_workers_model_and_says_nothing_on_the_shipped_one(self):
|
||||
checkout = Checkout(self)
|
||||
self.assertNotIn("own ladder", checkout.run([worker()]).stderr)
|
||||
ladder = checkout.home / ".claude-projects" / "proj" / "routing-ladder.json"
|
||||
ladder.write_text('[{"model": "claude-fable-5-2", "effort": "low"}]')
|
||||
run = checkout.run([worker()])
|
||||
self.assertIn(f"routing on the project's own ladder, {ladder}. worker-model is not on it, so the worker runs on worker-model unrouted; LOOP_CLAUDE_WORKER_MODEL and LOOP_CLAUDE_JUDGE_MODEL name other models for a run.\n", run.stderr)
|
||||
self.assertEqual((argument(run.calls[0], "--model"), argument(run.calls[0], "--effort")), ("worker-model", None))
|
||||
|
||||
def test_reads_the_projects_ladder_afresh_each_cycle_and_exits_1_once_it_holds_a_fault(self):
|
||||
checkout = Checkout(self)
|
||||
|
||||
@@ -2066,6 +2066,13 @@ class HelpLimits(unittest.TestCase):
|
||||
self.assertIn(f"rerun to give it {close.UNFINISHED_LIMIT} more", said)
|
||||
|
||||
|
||||
class OwnLadder(unittest.TestCase):
|
||||
def test_a_cycle_on_a_projects_own_ladder_names_its_file_and_says_where_the_workers_model_is_off_it(self):
|
||||
file = pathlib.Path("/h/.claude-projects/acme/routing-ladder.json")
|
||||
self.assertEqual(main.own_ladder(file, "claude-fable-5-2", False), f"routing on the project's own ladder, {file}.")
|
||||
self.assertEqual(main.own_ladder(file, "claude-opus-5-5", True), f"routing on the project's own ladder, {file}. claude-opus-5-5 is not on it, so the worker runs on claude-opus-5-5 unrouted; LOOP_CLAUDE_WORKER_MODEL and LOOP_CLAUDE_JUDGE_MODEL name other models for a run.")
|
||||
|
||||
|
||||
class ExitContract(unittest.TestCase):
|
||||
def test_every_exit_1_stop_has_its_clause_in_help(self):
|
||||
read, git, premises_ = "a transcript cannot be read or archived", "git cannot start or finds no checkout", "no config names the checkout, its config cannot be read, or the goals file holds no premises"
|
||||
@@ -2090,6 +2097,7 @@ class ExitContract(unittest.TestCase):
|
||||
("loop/close.py", "closed_cycle", "1", "failure.reason", read),
|
||||
("loop/main.py", "next_cycle", "1", "premises.reason", premises_),
|
||||
("loop/main.py", "next_cycle", "1", "records.reason", unreadable),
|
||||
("loop/main.py", "next_cycle", "1", "ladder_file.reason", premises_),
|
||||
("loop/main.py", "next_cycle", "1", "ladder.reason", "the project's routing ladder cannot be read or holds a fault"),
|
||||
("loop/main.py", "loop_run_from", "0 if task else 1", "HELP", "no task is given"),
|
||||
("loop/main.py", "loop_run_from", "1", 'f"the edited loop-claude cannot read the state', "a reload cannot read the state the running script handed it"),
|
||||
|
||||
@@ -7,8 +7,8 @@ a defect, a level from 1 to 4 scored as for a todo.md item. It prints the rung t
|
||||
the model's cheapest rung, the project's rung, in the Agent tool's words, such as `model sonnet,
|
||||
effort high`. --plan, for a task that files or scores todo.md items, prints the project's rung where
|
||||
that L picks a lower one, as ~/.claude/rules/todo-items.md lets only the project's rung or above
|
||||
plan. For a model off the routing ladder, the tool doesn't print anything on stdout, says so on stderr and
|
||||
exits 0; launch the subagent with no model or effort.
|
||||
plan. For a model off the routing ladder, the tool doesn't print anything on stdout, says so on stderr,
|
||||
naming the ladder's file, and exits 0; launch the subagent with no model or effort.
|
||||
|
||||
It reads the routing ladder of the project whose checkout it runs in: routing-ladder.json in the
|
||||
folder of the project's config, ~/.claude-projects/<key>/, or else the one this setup ships, as in
|
||||
@@ -24,7 +24,7 @@ import sys
|
||||
from failure import Failure
|
||||
from model_routing.ladder import Ladder
|
||||
from release_plan.todo_table import LEVELS
|
||||
from model_routing.ladder_file import project_ladder
|
||||
from model_routing.ladder_file import project_ladder_file, read_ladder
|
||||
|
||||
USAGE = f"usage: subagent-rung [--plan] <model> <L>, L a level from 1 to {LEVELS['L']}"
|
||||
|
||||
@@ -47,13 +47,14 @@ def main(argv: list[str], home: pathlib.Path) -> int:
|
||||
print(f"subagent-rung: cannot read the arguments{' ' + ' '.join(argv) if argv else ''}; {USAGE}", file=sys.stderr)
|
||||
return 1
|
||||
model, likelihood = operands
|
||||
ladder = project_ladder(None, home)
|
||||
file = project_ladder_file(None, home)
|
||||
ladder = file if isinstance(file, Failure) else read_ladder(file)
|
||||
if isinstance(ladder, Failure):
|
||||
print(f"subagent-rung: {ladder.reason}", file=sys.stderr)
|
||||
return 1
|
||||
args = agent_args(ladder, model, int(likelihood), planning)
|
||||
if args is None:
|
||||
print(f"subagent-rung: {model} is not on the routing ladder; pass the Agent tool no model or effort", file=sys.stderr)
|
||||
print(f"subagent-rung: {model} is not on the routing ladder {file}; pass the Agent tool no model or effort", file=sys.stderr)
|
||||
else:
|
||||
print(args)
|
||||
return 0
|
||||
|
||||
@@ -39,7 +39,8 @@ class SubagentRung(unittest.TestCase):
|
||||
# On the shipped ladder an L of 1 steps Fable 5.2 down to Opus 5.5, and an L of 4 up to its medium effort.
|
||||
self.assertEqual([self.run_tool("claude-fable-5-2", level, cwd=checkout / "nested").stdout for level in ("1", "4")], ["model fable, effort low\n", "model fable, effort max\n"])
|
||||
off = self.run_tool("claude-opus-5-5", "3", cwd=checkout)
|
||||
self.assertEqual((off.returncode, off.stdout, off.stderr), (0, "", "subagent-rung: claude-opus-5-5 is not on the routing ladder; pass the Agent tool no model or effort\n"))
|
||||
own = self.home / ".claude-projects" / "acme" / "routing-ladder.json"
|
||||
self.assertEqual((off.returncode, off.stdout, off.stderr), (0, "", f"subagent-rung: claude-opus-5-5 is not on the routing ladder {own}; pass the Agent tool no model or effort\n"))
|
||||
|
||||
def test_exits_1_naming_the_file_where_the_projects_ladder_holds_a_fault(self):
|
||||
checkout = self.project('[{"model": "claude-fable-5-2", "effort": "low"},]')
|
||||
@@ -76,7 +77,7 @@ class SubagentRung(unittest.TestCase):
|
||||
def test_the_launcher_prints_nothing_and_says_why_and_exits_0_off_the_ladder(self):
|
||||
off = self.run_tool("claude-opus-5", "4")
|
||||
self.assertEqual((off.returncode, off.stdout), (0, ""))
|
||||
self.assertEqual(off.stderr, "subagent-rung: claude-opus-5 is not on the routing ladder; pass the Agent tool no model or effort\n")
|
||||
self.assertEqual(off.stderr, f"subagent-rung: claude-opus-5 is not on the routing ladder {SHIPPED}; pass the Agent tool no model or effort\n")
|
||||
|
||||
def test_the_launcher_accepts_an_l_of_1(self):
|
||||
self.assertEqual(self.run_tool("claude-opus-5-5", "1").stdout, "model sonnet, effort high\n")
|
||||
|
||||
Reference in New Issue
Block a user