From 0a20b6d4f65934465cfd40769bc892cf137c6b94 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 15 Nov 2026 18:51:59 +0100 Subject: [PATCH] Read the routing ladder from routing-ladder.json, which a project replaces with a file of its own in its config's folder --- .gitignore | 1 + CHANGELOG.md | 4 + README.md | 8 +- bin/judge_replay/main.py | 23 +- bin/loop_claude/cycle/prompts.py | 60 +++--- bin/loop_claude/cycle/run.py | 16 +- bin/loop_claude/loop/chunk_start.py | 8 +- bin/loop_claude/loop/close.py | 10 +- bin/loop_claude/loop/main.py | 15 +- bin/loop_claude/loop/routing_check.py | 9 +- bin/loop_claude/loop/usage.py | 17 +- bin/loop_claude/routing.py | 68 +++--- bin/loop_claude/test_black_box.py | 24 +++ bin/loop_claude/test_loop_claude.py | 227 ++++++++++---------- bin/model_routing/ladder.py | 110 +++++----- bin/model_routing/ladder_file.py | 57 +++++ bin/model_routing/review_choice.py | 14 +- bin/model_routing/test_ladder.py | 82 +++---- bin/model_routing/test_ladder_file.py | 130 +++++++++++ bin/model_routing/test_prices.py | 4 +- bin/model_routing/test_review_choice.py | 6 +- bin/model_routing/test_worker_rungs.py | 30 ++- bin/model_routing/worker_rungs.py | 17 +- bin/project_config/main.py | 4 +- bin/project_config/settings.py | 7 +- bin/project_config/test_project_config.py | 10 +- bin/review_replay/main.py | 19 +- bin/review_replay/recall.py | 12 +- bin/review_replay/test_recall.py | 8 +- bin/routing_figures/calls.py | 8 +- bin/routing_figures/render.py | 14 +- bin/routing_figures/test_routing_figures.py | 50 ++--- bin/rule_hook/main.py | 4 +- bin/rule_hook/reviewer_rung.py | 49 +++-- bin/rule_hook/test_rule_hook.py | 64 +++++- bin/rung_replay/command.py | 8 +- bin/rung_replay/test_rung_replay.py | 6 +- bin/subagent-rung | 2 +- bin/subagent_rung/main.py | 28 ++- bin/subagent_rung/test_subagent_rung.py | 53 ++++- docs/decisions.md | 6 +- routing-ladder.json | 9 + 42 files changed, 848 insertions(+), 453 deletions(-) create mode 100644 bin/model_routing/ladder_file.py create mode 100644 bin/model_routing/test_ladder_file.py create mode 100644 routing-ladder.json diff --git a/.gitignore b/.gitignore index fcfa7f7e..df3b60f5 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ !CLAUDE.md !README.md !ci.sh +!routing-ladder.json !settings.json !todo.md diff --git a/CHANGELOG.md b/CHANGELOG.md index fb83ee35..4234814e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- The routing ladder is now a file, `routing-ladder.json` at this repo's root, and a project may run on a ladder of its own: a `routing-ladder.json` in its config's folder, `~/.claude-projects//`, replaces the shipped ladder whole for that project. loop-claude, `subagent-rung` and rule-hook read the project's ladder; the routing figures, `judge-replay` and `review-replay` read the shipped one. `subagent-rung` now reads the config of the checkout it runs in, and exits 1 where that config or the ladder holds a fault; loop-claude stops on a faulty ladder as it does on a faulty config. `loop-claude --help` shows the file's shape, and README → Install → "Set up a project" says when to write one. + ## 0.10.0 - The loop commits each change it makes to a file in the config's folder, such as a plan kept there, in that folder's git repo, titled as it titles its commits in the checkout. rule-hook ranks the plan on such a commit; before, it refused the commit. diff --git a/README.md b/README.md index 0d7ebdf0..93741814 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Claude Code configuration -My personal `~/.claude`, version controlled: `CLAUDE.md` and `rules/` for instructions, the words under `CLAUDE.md` → "Vocabulary"; `.claude/rules/` for the rules that load in this repo alone; `agents/` and `skills/`; `principles/` and `skills/larv-review/standards/` for what the reviewer agents hold projects to; `docs/` for decisions and the comprehension history; `settings.json` for the harness and `.claude/settings.json` for the harness inside this repo; `CHANGELOG.md` and `todo.md`; the tools in `bin/`; `workflow-bench/`, the scripts and prompts of the 2026-09-30 bench of larv-review and of the 2026-11-12 bench of the advisor tool; `reviewer-merge-trials/`, the 2026-11-09 trials of merging reviewers, with their method, results and scripts; `judge-field-trials/`, the 2026-11-11 trials of which facts code could read in place of loop-claude's judge, with their results and scripts; `stability-skip-count/`, the 2026-11-12 count of archived stability rounds, with its script, from which larv-review took when to skip a stability round 2; and `presentation/`, a page that explains this setup to newcomers, with the script that prints it to PDF. +My personal `~/.claude`, version controlled: `CLAUDE.md` and `rules/` for instructions, the words under `CLAUDE.md` → "Vocabulary"; `.claude/rules/` for the rules that load in this repo alone; `agents/` and `skills/`; `principles/` and `skills/larv-review/standards/` for what the reviewer agents hold projects to; `docs/` for decisions and the comprehension history; `settings.json` for the harness and `.claude/settings.json` for the harness inside this repo; `routing-ladder.json` for the models and efforts loop-claude routes work to; `CHANGELOG.md` and `todo.md`; the tools in `bin/`; `workflow-bench/`, the scripts and prompts of the 2026-09-30 bench of larv-review and of the 2026-11-12 bench of the advisor tool; `reviewer-merge-trials/`, the 2026-11-09 trials of merging reviewers, with their method, results and scripts; `judge-field-trials/`, the 2026-11-11 trials of which facts code could read in place of loop-claude's judge, with their results and scripts; `stability-skip-count/`, the 2026-11-12 count of archived stability rounds, with its script, from which larv-review took when to skip a stability round 2; and `presentation/`, a page that explains this setup to newcomers, with the script that prints it to PDF. ## Goals @@ -106,6 +106,8 @@ cd ~/code/acme-billing && ~/.claude/bin/project-config The last line prints each setting's path once the config is complete. +A project runs on the routing ladder this setup ships, `routing-ladder.json` at this repo's root: the rungs, each a model at an effort, that loop-claude routes workers, subagents and reviews to, least competent first. To run one project on other rungs, as when an outage or a client's account leaves it fewer models, copy that file to `~/.claude-projects//routing-ladder.json` and cut or change its rungs. The project's file replaces the shipped ladder whole. loop-claude reads it as each cycle starts, and `subagent-rung` and rule-hook each time they run. The loop never commits it, so commit it in the folder's repo yourself only where it should last. Run the project on a model its ladder holds, as with `LOOP_CLAUDE_WORKER_MODEL` and `LOOP_CLAUDE_JUDGE_MODEL`: loop-claude doesn't route a model off the ladder, and runs it at the effort its settings give. `loop-claude --help` shows the file's shape. + ## Update Move `~/.claude` to the newest release: @@ -153,7 +155,7 @@ Install puts only `loop-claude` and `loop-inbox` on `PATH`; call the others by t - `score-outcomes [L E I S]` shows what each finished item's chunk measured beside the scores the item was guessed at, to score a new item by. Per chunk it also shows code health's two counter-signals, misread findings and later fixes, and how many of the human's answers reversed a worker's choice; `score-outcomes --help` states the report. - `settings_filter.py` is git's clean filter for `settings.json`: it drops `"model"` and sorts the keys. `.gitattributes` sets the filter `settings` on `settings.json`, and Install's `git config` line makes this script that filter. - `statusline.py` renders Claude Code's status line; `settings.json` → `statusLine` runs it. -- `subagent-rung [--plan] ` prints the Agent tool's model and effort for a subagent's task, picked by its L from the project's model, and with `--plan` never below the project's rung; `subagent-rung --help` states the rule. +- `subagent-rung [--plan] ` prints the Agent tool's model and effort for a subagent's task, picked by its L from the project's model on the project's routing ladder, and with `--plan` never below the project's rung; `subagent-rung --help` states the rule. - `todo-item add …` files an item in a project's `todo.md`, taking `Next ID` as the item's ID, and `todo-item finish ` removes a finished item and adds the bullet passed with `--changelog` to `CHANGELOG.md`; both rank the file. `todo-item show ` prints an item's row and detail. `todo-item --help` states the flags. - `wait-for [] [--timeout ]` waits until a file a background job writes holds a line matching the pattern, and prints it. @@ -170,7 +172,7 @@ Install puts only `loop-claude` and `loop-inbox` on `PATH`; call the others by t - `loop_state/` holds the files loop-claude and loop-inbox share: the question files (`waiting/`), the files a running loop lists itself in (`running_loops.py`, `session_lock.py`), their paths, and `hand_over.py`, which opens claude for `/talk`. A change to any of them changes both tools. - `machine_load/` takes the machine's heavy slots, its shared lock files, for `run_heavy`, and reads the memory share that loop-claude waits for. It also gives `mutate_changes` a slot's CPU count. - `mattermost_dm/` sends the human a Mattermost DM, for `mattermost-dm` and for loop-claude through `loop_effects`. -- `model_routing/` holds the routing ladder, which loop-claude, `rule_hook`, both replay tools and `subagent-rung` read, and the list prices, which `routing_figures` and `transcript_archive` read. +- `model_routing/` holds the routing ladder's steps (`ladder.py`) and reads a ladder from its file, the shipped `routing-ladder.json` or a project's own (`ladder_file.py`), for loop-claude, `rule_hook`, both replay tools and `subagent-rung`. It finds a project's config folder through `project_config`. It also holds the list prices, which `routing_figures` and `transcript_archive` read. - `rank_todo.py`, `settings_filter.py` and `statusline.py` are tools of one file each, tested beside them. - `release_plan/` reads `todo.md`'s items table and release versions, says which release is next and whether it is ready to cut (`release_standing.py`), and ranks the plan (`ranking.py`, with its score formula in `formula.py` and its blocker order in `blockers.py`). `goals_file.py` reads the goals file's sections, goals and goal weights, for the ranking and for `loop_claude`'s premises, and `citations.py` holds `todo.md` and `docs/decisions.md` to each other. `plan_file.py` reads, ranks, checks and writes a `todo.md`, for `larv_review`, `rank_todo.py`, `rule_hook` and `todo_item`. - `repo_checks/` tests the repo itself. diff --git a/bin/judge_replay/main.py b/bin/judge_replay/main.py index d9613758..04f5b70b 100644 --- a/bin/judge_replay/main.py +++ b/bin/judge_replay/main.py @@ -2,7 +2,7 @@ usage: judge-replay - is a project model on the routing ladder, such as claude-opus-5-5. The tool replays the + is a project model on the routing ladder this setup ships, such as claude-opus-5-5. The tool replays the verdicts archived for the projects that ~/.claude/loop-claude-models.json runs on it: the latest 15 of each kind, finished, unfinished, interrupt and quoted answer. Each transcript copy in the archive is what the judge read for the verdict beside it, so the replay hands the judge that copy, @@ -41,8 +41,9 @@ from transcript_archive.project_key import checkout_of from loop_effects.effects import Effects from failure import Failure from rung_replay.search import first_reply, replayed +from model_routing.ladder import Ladder, REVIEW_STEPS, Rung from claude_code.reply import Limited -from model_routing.ladder import lowest_worker_rung, position, project_rung, review_floor, REVIEW_STEPS, Rung +from model_routing.ladder_file import read_ladder, SHIPPED from loop_claude.project.premises import read_premises from loop_claude.project.records import read_records from claude_code.transcript import read_transcript @@ -56,7 +57,11 @@ REFERENCE_EFFORT = "xhigh" def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, effects: Effects) -> int: - project = arguments_or_exit(TOOL, argv, __doc__, USAGE, model_argument) + # The shipped ladder, not a project's: the replay spans every project on the model. + ladder = read_ladder(SHIPPED) + if isinstance(ladder, Failure): + return failed(TOOL, ladder.reason) + project = arguments_or_exit(TOOL, argv, __doc__, USAGE, lambda argv: model_argument(ladder, argv)) if isinstance(project, int): return project cases = archived_cases(project.model, archive_root(home, env), home) @@ -67,21 +72,21 @@ def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, eff cases = sample(cases, PER_KIND) reference = Rung(project.model, REFERENCE_EFFORT) # The candidates stop at the lowest rung a worker routes to: no route uses a rung below it for the judge. - replay = replayed(cases, reference, down_from(project, lowest_worker_rung(project)), lambda found, rung: rule(found, rung, home, effects), reproduced, "verdicts", climbs(project, reference)) + replay = replayed(cases, reference, down_from(ladder, project, ladder.lowest_worker_rung(project)), lambda found, rung: rule(found, rung, home, effects), reproduced, "verdicts", climbs(ladder, project, reference)) if isinstance(replay, Limited): return limited_exit(TOOL, replay) print(report(project.model, reference, cases, replay)) return 0 -def model_argument(argv: list[str]) -> Rung | None: - """The project's rung of the one model argv names; None for any other argv.""" - return project_rung(argv[0]) if len(argv) == 1 else None +def model_argument(ladder: Ladder, argv: list[str]) -> Rung | None: + """The project's rung on ladder of the one model argv names; None for any other argv.""" + return ladder.project_rung(argv[0]) if len(argv) == 1 else None -def climbs(project: Rung, reference: Rung) -> list[Rung]: +def climbs(ladder: Ladder, project: Rung, reference: Rung) -> list[Rung]: """The rungs above the project's rung that an E + I raises a review to, except the reference's, lowest first.""" - return sorted({review_floor(project, score) for score in REVIEW_STEPS} - {project, reference}, key=position) + return sorted({ladder.review_floor(project, score) for score in REVIEW_STEPS} - {project, reference}, key=ladder.position) def archived_cases(model: str, root: pathlib.Path, home: pathlib.Path) -> list[Case] | Failure: diff --git a/bin/loop_claude/cycle/prompts.py b/bin/loop_claude/cycle/prompts.py index fc2121f1..287c4bee 100644 --- a/bin/loop_claude/cycle/prompts.py +++ b/bin/loop_claude/cycle/prompts.py @@ -1,11 +1,11 @@ """What loop-claude says to the worker each cycle. model_routing.review_choice chooses where the reviews run. review_launches and reruns put that choice into sentences, and escalation says where escalated subagents run; rung_sentences returns the sentences of all three. In a chunk's first prompt, routed_by places them after its routing sentences where an item routed the chunk, and fresh places them alone otherwise. rungs_held repeats them at the end of a prompt after a step-up or an unfinished verdict.""" from model_routing.review_choice import chosen, ReviewChoice from loop_claude.cycle.chunk_items import CHUNK_ITEMS -from model_routing.ladder import climbed, LADDER, position, Rung from model_routing.route import ESCALATION_MARKER, Route from loop_claude.routing import escalation_rungs from loop_claude.project.records import HUMAN_RULE_NAMES, RECORD_FILES from review_rules.review_passes import INSTRUCTION_FILES, INSTRUCTION_FOLDERS +from model_routing.ladder import Ladder, Rung from text_files.markdown_text import listed, quoted from claude_code.transcript import Typed @@ -29,17 +29,17 @@ NAME_THE_ITEMS = f"Name every todo.md item this chunk takes on one line, as in ` AFTER_CODE_HEALTH = 'which rank after code health (~/.claude/CLAUDE.md → "Shared goal") unless the goals file voids it' -def fresh(task: str, premises: str, route: Route, project: Rung | None, facts: str) -> str: +def fresh(task: str, premises: str, route: Route, ladder: Ladder, project: Rung | None, facts: str) -> str: """A chunk's first prompt; facts is preflight_facts' or preflight_failed's text.""" - routed = f"{routed_by(route, project)} {recount(route)}" if route.item else " ".join(rung_sentences(route, project)) + routed = f"{routed_by(route, ladder, project)} {recount(route)}" if route.item else " ".join(rung_sentences(route, ladder, project)) opening = f"{routed}\n\n" if routed else "" return f"{task}\n\nContinue any open draft PR or worktree for the chunk before creating a new one. {READING_RULE} {NAME_THE_ITEMS}\n\n{opening}{facts}\n\nThe chunk's todo.md items, or else the task above, set its scope. The goals below, {AFTER_CODE_HEALTH}, choose between ways to deliver that scope, and never widen it.\n\nFrom the goals file, {premises}" -def routed_by(route: Route, project: Rung | None) -> str: +def routed_by(route: Route, ladder: Ladder, project: Rung | None) -> str: return " ".join(( f"loop-claude runs you on {route.rung}, routed by item {route.item}'s L of {route.likelihood}.", - *rung_sentences(route, project), + *rung_sentences(route, ladder, project), )) @@ -58,12 +58,12 @@ def preflight_failed(reason: str) -> str: return f"loop-claude could not run `~/.claude/bin/chunk-preflight`, so find the chunk's facts yourself. Why it could not run: {reason}" -def review_launches(route: Route, project: Rung | None) -> list[str]: +def review_launches(route: Route, ladder: Ladder, project: Rung | None) -> list[str]: """The sentences naming where to launch reviews, in this order, each from the function that says when it applies; none off the ladder.""" if project is None: return [] - choice = chosen(project, route) - return [*shared_launches(choice, route, project), *scan_launch(choice, route), *planning_launch(route, project), *taken_in_launches(choice), *markdown_launch(choice)] + choice = chosen(ladder, project, route) + return [*shared_launches(choice, route, project), *scan_launch(choice, route), *planning_launch(route, ladder, project), *taken_in_launches(choice), *markdown_launch(choice)] def scan_launch(choice: ReviewChoice, route: Route) -> list[str]: @@ -71,9 +71,9 @@ def scan_launch(choice: ReviewChoice, route: Route) -> list[str]: return ["Run a full scan only from such a subagent, since the scan's agents run on the model that starts them."] if choice.security != route.rung else [] -def planning_launch(route: Route, project: Rung) -> list[str]: +def planning_launch(route: Route, ladder: Ladder, project: Rung) -> list[str]: """Where the worker runs below the project's rung, the sentence naming where to launch a subagent that files or scores todo.md items. On an Opus 5.5 project: `Your rung is below the project's: launch each subagent that files or scores todo.md items on the rung `~/.claude/bin/subagent-rung --plan claude-opus-5-5 ` prints.`""" - return [f"Your rung is below the project's: launch each subagent that files or scores todo.md items on the rung `~/.claude/bin/subagent-rung --plan {project.model} ` prints."] if position(route.rung) < position(project) else [] + return [f"Your rung is below the project's: launch each subagent that files or scores todo.md items on the rung `~/.claude/bin/subagent-rung --plan {project.model} ` prints."] if ladder.position(route.rung) < ladder.position(project) else [] def markdown_launch(choice: ReviewChoice) -> list[str]: @@ -118,19 +118,19 @@ def rung_named(rung: Rung, route: Route, project: Rung) -> str: return f"the rung item {route.item}'s E + I of {route.escape_and_impact} picks" -def escalation(route: Route, project: Rung | None) -> list[str]: +def escalation(route: Route, ladder: Ladder, project: Rung | None) -> list[str]: """The prompt's escalation sentences, the marker's among them. None off the ladder or on its top rung. Where escalation_rungs opens none, the worker escalates only with the human's yes. Else they name the rungs escalation_rungs opens to a subagent, and offer to resume the worker one rung up for the rest of the chunk, up to the highest of those rungs. For a worker one rung above the project's, routed by item 3 of L 4, they read: `When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to ``, one rung above yours, and on each further trigger one rung higher, up to ``; above that, ask the human first.`, `Open the subagent's prompt with `Escalation trigger :`, n the trigger's number.` and `Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to ``.` """ - if project is None or route.rung not in LADDER[:-1]: + if project is None or route.rung not in ladder.rungs[:-1]: return [] marked = f"Open the subagent's prompt with `{ESCALATION_MARKER} :`, n the trigger's number." - rungs = [f"`{rung.agent_args()}`" for rung in escalation_rungs(project, route)] + rungs = [f"`{rung.agent_args()}`" for rung in escalation_rungs(ladder, project, route)] if not rungs: return [ - f"An escalation trigger in `~/.claude/CLAUDE.md` lets you hand the hard part of a task to a subagent with the Agent tool set to `{climbed(route.rung, 1).agent_args()}`, one rung above yours, only once the human agrees; loop-claude will not resume you higher for a trigger either, so ask the human for that yes.", + f"An escalation trigger in `~/.claude/CLAUDE.md` lets you hand the hard part of a task to a subagent with the Agent tool set to `{ladder.climbed(route.rung, 1).agent_args()}`, one rung above yours, only once the human agrees; loop-claude will not resume you higher for a trigger either, so ask the human for that yes.", marked, ] further = f", and on each further trigger one rung higher, up to {rungs[-1]}" if len(rungs) > 1 else "" @@ -141,16 +141,16 @@ def escalation(route: Route, project: Rung | None) -> list[str]: ] -def reruns(route: Route, project: Rung | None) -> list[str]: +def reruns(route: Route, ladder: Ladder, project: Rung | None) -> list[str]: """The sentence naming the rung one above a reviewer's and the one above a seat's, on which each reruns when it asks, leaving out either that is already on the top rung. None off the ladder. For a worker one rung below the project's, routed by an item of E + I 4, it reads: `A reviewer or comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to ``.` """ - if project is None or route.rung not in LADDER: + if project is None or route.rung not in ladder.rungs: return [] - choice = chosen(project, route) + choice = chosen(ladder, project, route) launched = (("reviewer", choice.reviewers), ("comprehension seat", choice.seat)) - rerun = {name: climbed(rung, 1) for name, rung in launched if rung != LADDER[-1]} + rerun = {name: ladder.climbed(rung, 1) for name, rung in launched if rung != ladder.rungs[-1]} if not rerun: return [] if len(set(rerun.values())) == 1: @@ -158,31 +158,31 @@ def reruns(route: Route, project: Rung | None) -> list[str]: return [f"A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `{rerun['reviewer'].agent_args()}`, and a comprehension seat with `{rerun['comprehension seat'].agent_args()}`."] -def resumed(unfinished_evidence: str, route: Route, project: Rung | None) -> str: +def resumed(unfinished_evidence: str, route: Route, ladder: Ladder, project: Rung | None) -> str: """Names the worker's rung, so it learns whether its request for a higher rung went through, and the project's again, since a step-up between cycles may have taken the worker above it.""" on = f" loop-claude runs you on {route.rung}." if project else "" ruled = f" A judge ruled the chunk unfinished, quoting this passage of your transcript as the work remaining:\n\n{quoted(unfinished_evidence)}\n\n" if unfinished_evidence else " " - return f"Continue the task; the previous run ended before the chunk was finished.{ruled}The chunk is finished once it is merged, or nothing remains that you could do on your own. Even then, it is finished only once {ANSWERS_RECORDED}. {NAME_THE_ITEMS}{on}{rungs_held(route, project)}" + return f"Continue the task; the previous run ended before the chunk was finished.{ruled}The chunk is finished once it is merged, or nothing remains that you could do on your own. Even then, it is finished only once {ANSWERS_RECORDED}. {NAME_THE_ITEMS}{on}{rungs_held(route, ladder, project)}" -def rung_sentences(route: Route, project: Rung | None) -> list[str]: +def rung_sentences(route: Route, ladder: Ladder, project: Rung | None) -> list[str]: """The sentences that depend on the worker's rung: where its reviews run and rerun, and where its escalated subagents run.""" - return [*review_launches(route, project), *escalation(route, project), *reruns(route, project)] + return [*review_launches(route, ladder, project), *escalation(route, ladder, project), *reruns(route, ladder, project)] -def rungs_held(route: Route, project: Rung | None) -> str: +def rungs_held(route: Route, ladder: Ladder, project: Rung | None) -> str: """rung_sentences, each after a space, to follow the sentence a prompt ends on.""" - return "".join(f" {sentence}" for sentence in rung_sentences(route, project)) + return "".join(f" {sentence}" for sentence in rung_sentences(route, ladder, project)) -def answered(question: str, answer: str, route: Route, project: Rung | None, stepped: bool) -> str: +def answered(question: str, answer: str, route: Route, ladder: Ladder, project: Rung | None, stepped: bool) -> str: """stepped is whether the answer's ruling stepped the worker up.""" - return f"Your question: {question}\n\n{QUOTE_SOURCES} answers it:\n\n{answer}\n\nContinue the task.{stepped_to(route, project, stepped)}" + return f"Your question: {question}\n\n{QUOTE_SOURCES} answers it:\n\n{answer}\n\nContinue the task.{stepped_to(route, ladder, project, stepped)}" -def typed(question: str, answer: str, route: Route, project: Rung | None, stepped: bool) -> str: +def typed(question: str, answer: str, route: Route, ladder: Ladder, project: Rung | None, stepped: bool) -> str: """The prompt that resumes the worker with the human's answer to an interrupt. answer is one line, as the terminal reads it.""" - return f"{TYPED_HERE}\n{answer}\n\nYour question was: {question}\n\nContinue the task. The chunk is finished only once {ANSWERS_RECORDED}.{stepped_to(route, project, stepped)}" + return f"{TYPED_HERE}\n{answer}\n\nYour question was: {question}\n\nContinue the task. The chunk is finished only once {ANSWERS_RECORDED}.{stepped_to(route, ladder, project, stepped)}" # Opens typed's prompt, with the answer on the line after it, so typed_answer finds the answer at the prompt's start, whatever the question holds. @@ -198,9 +198,9 @@ def typed_answer(prompt: str) -> Typed | None: return Typed(rest, answer) -def stepped_to(route: Route, project: Rung | None, stepped: bool) -> str: +def stepped_to(route: Route, ladder: Ladder, project: Rung | None, stepped: bool) -> str: """The sentences naming the worker's new rung, as resumed has them, where the ruling that brought the answer stepped the worker up; empty otherwise.""" - return f" Your request for a higher rung went through: loop-claude runs you on {route.rung}. The escalation triggers that fired before this step are spent.{rungs_held(route, project)}" if stepped else "" + return f" Your request for a higher rung went through: loop-claude runs you on {route.rung}. The escalation triggers that fired before this step are spent.{rungs_held(route, ladder, project)}" if stepped else "" def past_the_cap(question: str, answer: str, cap: int) -> str: diff --git a/bin/loop_claude/cycle/run.py b/bin/loop_claude/cycle/run.py index d79502f3..91add273 100644 --- a/bin/loop_claude/cycle/run.py +++ b/bin/loop_claude/cycle/run.py @@ -14,13 +14,13 @@ from failure import Failure from loop_claude.verdicts import Finished, Interrupt, Judged, QuotedAnswer, Unfinished, Verdict from loop_claude.cycle.interrupt import interrupt, Talked, WorkerSession from loop_claude.routing import judge_rung, next_route, step_up, StepUp +from model_routing.ladder import Ladder, Rung from transcript_archive.read import last_spent from transcript_archive import outcomes from loop_claude.cycle import prompts from claude_code.transcript import read_transcript from loop_claude.cycle.retry import retried from model_routing.route import Route -from model_routing.ladder import Rung from loop_claude.cycle import spend from loop_claude.stop import Stop from model_routing import worker_rungs @@ -38,6 +38,8 @@ class Cycle: # What chunk-preflight found, or why it could not run, for a fresh worker's prompt; empty when the session was open as the cycle began. facts: str judge_model: str + # The project's routing ladder, as model_routing/ladder_file.py read it when the cycle started. + ladder: Ladder premises: str # The project's rung; None for a model off the ladder. project: Rung | None @@ -84,7 +86,7 @@ def run_cycle(cycle: Cycle) -> Stop | Concluded: def ruling_step_up(cycle: Cycle, verdict: Verdict) -> StepUp | None: """The step-up an interrupt's or a quoted answer's escalation makes, so the work after the answer runs on the stepped route; None where the route holds. A finished or unfinished verdict steps the route only as the loop starts the next cycle.""" - return step_up(cycle.route, next_route(cycle.project, cycle.route, verdict)) if isinstance(verdict, Interrupt | QuotedAnswer) else None + return step_up(cycle.route, next_route(cycle.ladder, cycle.project, cycle.route, verdict)) if isinstance(verdict, Interrupt | QuotedAnswer) else None def stopped(cycle: Cycle, stop: Stop, why: str) -> Stop: @@ -105,7 +107,7 @@ def archived(project_archive: pathlib.Path, transcript_path: str, task: str, out def judge_of(cycle: Cycle) -> Rung: """The judge's rung, which the archive names beside each verdict.""" - return judge_rung(cycle.judge_model) + return judge_rung(cycle.ladder, cycle.judge_model) def step(cycle: Cycle, state: Active) -> Event | Stop: @@ -143,12 +145,12 @@ def worker_session(cycle: Cycle, transcript_path: str) -> WorkerSession: def worker_start(cycle: Cycle, answered: QuotedAnswer | TypedAnswer | None, stepped: bool) -> list[str]: if isinstance(answered, QuotedAnswer): - return worker_args(cycle.session, cycle.route.rung, prompts.answered(answered.question, answered.answer, cycle.route, cycle.project, stepped), resume=True) + return worker_args(cycle.session, cycle.route.rung, prompts.answered(answered.question, answered.answer, cycle.route, cycle.ladder, cycle.project, stepped), resume=True) if isinstance(answered, TypedAnswer): - return worker_args(cycle.session, cycle.route.rung, prompts.typed(answered.question, answered.answer, cycle.route, cycle.project, stepped), resume=True) + return worker_args(cycle.session, cycle.route.rung, prompts.typed(answered.question, answered.answer, cycle.route, cycle.ladder, cycle.project, stepped), resume=True) if session_open(cycle.session): - return worker_args(cycle.session, cycle.route.rung, prompts.resumed(cycle.unfinished_evidence, cycle.route, cycle.project), resume=True) - return worker_args(cycle.session, cycle.route.rung, prompts.fresh(cycle.task, cycle.premises, cycle.route, cycle.project, cycle.facts), resume=False) + return worker_args(cycle.session, cycle.route.rung, prompts.resumed(cycle.unfinished_evidence, cycle.route, cycle.ladder, cycle.project), resume=True) + return worker_args(cycle.session, cycle.route.rung, prompts.fresh(cycle.task, cycle.premises, cycle.route, cycle.ladder, cycle.project, cycle.facts), resume=False) def rule(cycle: Cycle, transcript_path: str, quoted_answers: int) -> Ruled | Stop: diff --git a/bin/loop_claude/loop/chunk_start.py b/bin/loop_claude/loop/chunk_start.py index c5cd8f8b..df6cb09b 100644 --- a/bin/loop_claude/loop/chunk_start.py +++ b/bin/loop_claude/loop/chunk_start.py @@ -8,12 +8,12 @@ from loop_effects.effects import Effects, say from failure import Failure from loop_claude.routing import first_route, kept, routed_item from release_plan.todo_table import Item, items +from model_routing.ladder import Ladder, Rung from transcript_archive.read import latest_worker from project_config.main import located from loop_claude.loop.this_loop import Loop from loop_claude.cycle import prompts from model_routing.route import Route, route_from -from model_routing.ladder import Rung from claude_code.session import session_open, transcript_file from loop_state.running_loops import taken_items @@ -39,15 +39,15 @@ def fetched_todo(loop: Loop) -> str | Failure: return todo -def routed(route: Route | None, project: Rung | None, model: str, archived: typing.Callable[[], Route | None], item: typing.Callable[[], Item | None]) -> Route: - """The route the cycle runs on. project is the project's rung, and model the project's model. +def routed(route: Route | None, ladder: Ladder, project: Rung | None, model: str, archived: typing.Callable[[], Route | None], item: typing.Callable[[], Item | None]) -> Route: + """The route the cycle runs on. project is the project's rung on ladder, and model the project's model. Where project is None, the model is off the ladder, and the worker runs on it. Otherwise the chunk keeps route, or on a rerun the route its archive last names, as routing.kept allows. Else the plan item routes the chunk afresh. """ if project is None: return Route(Rung(model)) route = route or archived() - return route if route and kept(route, project) else first_route(project, item()) + return route if route and kept(route, ladder, project) else first_route(ladder, project, item()) def archived_route(archive_folder: pathlib.Path, loop: Loop) -> Route | None: diff --git a/bin/loop_claude/loop/close.py b/bin/loop_claude/loop/close.py index 9348e049..6e35251f 100644 --- a/bin/loop_claude/loop/close.py +++ b/bin/loop_claude/loop/close.py @@ -4,20 +4,20 @@ import pathlib from loop_claude.cycle.run import archived_step, archived_stop, Concluded from loop_claude.verdicts import Finished +from model_routing.ladder import Ladder, Rung from loop_claude.loop.this_loop import Loop from loop_claude.loop.reload import LoopRun from loop_claude.stop import needs_human, Stop from loop_claude.routing import next_route, step_up -from model_routing.ladder import Rung from claude_code.session import transcript_file # How many unfinished verdicts running end the loop, so a worker and a judge that disagree about a chunk cannot cycle unattended without end. A policy number, not derived from the routing steps: well above the two unfinished verdicts running that archived chunks have reached at most, and low enough that a stuck pair is stopped after six cycles. It counts every unfinished verdict, including those of cycles where the human answered an interrupt: a rerun is cheap, and a chunk that needs that many answers is worth a look. The human confirmed both the number and what it counts. UNFINISHED_LIMIT = 6 -def closed_cycle(loop_run: LoopRun, project: Rung | None, concluded: Concluded, archive_folder: pathlib.Path, loop: Loop) -> LoopRun | Stop: +def closed_cycle(loop_run: LoopRun, ladder: Ladder, project: Rung | None, concluded: Concluded, archive_folder: pathlib.Path, loop: Loop) -> LoopRun | Stop: """The loop run after a cycle concluded, with the step-up archived; the stop that needs the human, archived, once the chunk has had UNFINISHED_LIMIT unfinished verdicts running.""" - after = after_verdict(loop_run, project, concluded) + after = after_verdict(loop_run, ladder, project, concluded) # A finished chunk drops its route, so it has no step-up, and its run of unfinished verdicts restarts. if after.route is None: return after @@ -38,10 +38,10 @@ def reached_limit(loop_run: LoopRun) -> Stop | None: return needs_human("go on with the chunk", f"the judge ruled it unfinished {loop_run.unfinished_run} cycles running; finish the chunk yourself, or rerun loop-claude to give it {UNFINISHED_LIMIT} more cycles") -def after_verdict(loop_run: LoopRun, project: Rung | None, concluded: Concluded) -> LoopRun: +def after_verdict(loop_run: LoopRun, ladder: Ladder, project: Rung | None, concluded: Concluded) -> LoopRun: """The loop run the next cycle starts from. A finished chunk drops its route, its run of unfinished verdicts and the items it took in. An unfinished one takes the route routing.next_route makes of the route its cycle ended on, which an interrupt's or a quoted answer's escalation may have raised; the evidence the judge gave for the verdict, and the items the worker names on its latest `Chunk items:` line; and counts the verdict.""" verdict = concluded.verdict if isinstance(verdict, Finished): return dataclasses.replace(loop_run, route=None, unfinished_evidence="", unfinished_run=0, items_taken_in=()) - route = next_route(project, concluded.route, verdict) + route = next_route(ladder, project, concluded.route, verdict) return dataclasses.replace(loop_run, route=route, unfinished_evidence=verdict.evidence, unfinished_run=loop_run.unfinished_run + 1, items_taken_in=verdict.taken_in.items) diff --git a/bin/loop_claude/loop/main.py b/bin/loop_claude/loop/main.py index 31dd7e92..ba9f5d93 100644 --- a/bin/loop_claude/loop/main.py +++ b/bin/loop_claude/loop/main.py @@ -17,7 +17,7 @@ from loop_claude.loop.usage import HELP from loop_claude.loop.this_loop import Loop from loop_state.paths import Paths from transcript_archive.project_key import project_key -from model_routing.ladder import project_rung +from model_routing.ladder_file import project_ladder from loop_claude.project.premises import read_premises from loop_claude.project.records import read_records from claude_code.session import session_named @@ -61,7 +61,7 @@ def run_loop(argv: list[str], env: typing.Mapping[str, str], paths: Paths, launc concluded = run_cycle(cycle) if isinstance(concluded, Stop): return concluded - closed = closed_cycle(loop_run, cycle.project, concluded, cycle.project_archive, loop) + closed = closed_cycle(loop_run, cycle.ladder, cycle.project, concluded, cycle.project_archive, loop) if isinstance(closed, Stop): return closed loop_run = closed @@ -71,7 +71,7 @@ def run_loop(argv: list[str], env: typing.Mapping[str, str], paths: Paths, launc def next_cycle(loop_run: LoopRun, loop: Loop) -> Cycle | Stop: - """The cycle loop_run runs next, from the premises, the records and the project's models read afresh, and the route the chunk runs on; the stop where one of them cannot be read.""" + """The cycle loop_run runs next, from the premises, the records, the project's models and its routing ladder read afresh, and the route the chunk runs on; the stop where one of them cannot be read.""" premises = read_premises(loop.root, loop.paths.home) if isinstance(premises, Failure): return Stop(1, premises.reason) @@ -81,16 +81,19 @@ def next_cycle(loop_run: LoopRun, loop: Loop) -> Cycle | Stop: models = get_or_ask_models(project_key(loop.root, loop.paths.home), loop.paths, loop.env, loop.effects) if isinstance(models, Stop): return models + ladder = project_ladder(loop.root, loop.paths.home) + if isinstance(ladder, Failure): + return Stop(1, ladder.reason) archive_folder = project_archive(loop.root, loop.paths.home, loop.env) # After the reads above, so a loop that stops on one makes no worktree. facts, item = chunk_start(loop) - project = project_rung(models.worker) - route = routed(loop_run.route, project, models.worker, lambda: archived_route(archive_folder, loop), item) + project = ladder.project_rung(models.worker) + route = routed(loop_run.route, ladder, project, models.worker, lambda: archived_route(archive_folder, loop), item) say(f"\n=== cycle {loop_run.cycle_number}, worker on {route.rung} ===", loop.effects, stdout=True) todo = fetched_todo(loop) show_taken_in = shown_items(loop, todo, route.item) show_taken_in(loop_run.items_taken_in) - return Cycle(config=setup_version(loop.paths.claude_dir, loop.effects.git), effects=loop.effects, facts=facts, judge_model=models.judge, premises=premises, project=project, project_archive=archive_folder, questions=loop.listed.folder, records=records, root=loop.root, route=route, session=loop.session, show_taken_in=show_taken_in, task=loop_run.task, todo=todo, unfinished_evidence=loop_run.unfinished_evidence) + return Cycle(config=setup_version(loop.paths.claude_dir, loop.effects.git), effects=loop.effects, facts=facts, judge_model=models.judge, ladder=ladder, premises=premises, project=project, project_archive=archive_folder, questions=loop.listed.folder, records=records, root=loop.root, route=route, session=loop.session, show_taken_in=show_taken_in, task=loop_run.task, todo=todo, unfinished_evidence=loop_run.unfinished_evidence) def loop_run_from(argv: list[str], cwd: pathlib.Path, effects: Effects) -> LoopRun | Stop: diff --git a/bin/loop_claude/loop/routing_check.py b/bin/loop_claude/loop/routing_check.py index b7ef0cf3..08a8791a 100644 --- a/bin/loop_claude/loop/routing_check.py +++ b/bin/loop_claude/loop/routing_check.py @@ -10,6 +10,7 @@ from failure import Failure from loop_state.waiting.listing import listed from loop_claude.loop.this_loop import Loop from loop_state.paths import Paths +from model_routing.ladder_file import read_ladder, SHIPPED from routing_figures.render import render from model_routing.ladder import Rung from routing_figures import state @@ -43,9 +44,13 @@ def new_calls(root: pathlib.Path, paths: Paths, effects: Effects) -> list[Call] measured = archive.measure_archive(root, str(paths.home), raised.since, effects.git) if isinstance(measured, Failure): return measured + # The shipped ladder, not the project's: the figures span every project. + ladder = read_ladder(SHIPPED) + if isinstance(ladder, Failure): + return ladder figures = totals(measured.chunks, measured.skipped, measured.unread) - found = calls(figures) - if unwritten := archive.write_figures(root, render(figures, found, utc(effects.now()), cycle, raised.since)) or state.mark_measured(root, cycle): + found = calls(ladder, figures) + if unwritten := archive.write_figures(root, render(ladder, figures, found, utc(effects.now()), cycle, raised.since)) or state.mark_measured(root, cycle): return unwritten return [call for call in found if call not in raised.calls] diff --git a/bin/loop_claude/loop/usage.py b/bin/loop_claude/loop/usage.py index 8c9a15ea..6f003a9b 100644 --- a/bin/loop_claude/loop/usage.py +++ b/bin/loop_claude/loop/usage.py @@ -68,7 +68,21 @@ entry, as on its first run, is asked for one on the terminal; loop-inbox doesn't question. LOOP_CLAUDE_JUDGE_MODEL overrides the model for the judge, LOOP_CLAUDE_WORKER_MODEL for the worker and the session /talk opens; with both set, nothing is asked. -Where the project's model is on the routing ladder (bin/model_routing/ladder.py), the worker and the +The routing ladder is a JSON list of rungs, least competent first, each a model at an effort of low, +medium, high, xhigh or max: + + [ + {"model": "claude-opus-5-5", "effort": "medium"}, + {"model": "claude-fable-5-2", "effort": "low"} + ] + +This setup ships one, routing-ladder.json at its root. A project that keeps a routing-ladder.json in +its config's folder, ~/.claude-projects//, runs on that ladder instead, whole: write there +every rung the project may run on. loop-claude reads the project's ladder as each cycle starts, and +never commits the file. subagent-rung and rule-hook read the same ladder; the routing figures, +judge-replay and review-replay, which span every project, read the shipped one. + +Where the project's model is on its routing ladder, the worker and the session /talk opens run on the rung that the chunk's todo.md item picks, and the judge of some models on another; a chunk no item routes runs on the project's model. The worker's prompts name the rungs it launches its reviews on. The worker's rung steps up as a chunk stays unfinished, when the @@ -102,6 +116,7 @@ Exit 1, saying why, when: ~/.claude/skills/larv-review/standards/*.md where they exist, cannot be read or is not UTF-8 - no config names the checkout, its config cannot be read, or the goals file holds no premises - the models file cannot be read or written +- the project's routing ladder cannot be read or holds a fault - claude leaves no transcript - a transcript cannot be read or archived - a finished chunk's session cannot be deleted diff --git a/bin/loop_claude/routing.py b/bin/loop_claude/routing.py index 5dc6e515..f9721b47 100644 --- a/bin/loop_claude/routing.py +++ b/bin/loop_claude/routing.py @@ -30,11 +30,11 @@ Each verdict then moves the rung by one rule, and the rung never falls. The rung import dataclasses from release_plan.release_standing import chunk_item, LastCut, Next -from model_routing.ladder import climbed, highest, LADDER, position, project_rung, Rung, stepped_up, worker_rung from loop_claude import escalation_mark from failure import Failure from loop_claude.verdicts import Interrupt, QuotedAnswer, Unfinished from release_plan.todo_table import Item, items, NoItemsTable +from model_routing.ladder import Ladder, Rung from model_routing.route import Route UNFINISHED_PER_STEP = 2 @@ -56,42 +56,42 @@ def routed_item(todo: str, cut: LastCut) -> Item | Failure | None: return takes if takes and takes.likelihood and takes.escape_and_impact else None -def first_route(project: Rung, item: Item | None) -> Route: - """The route a chunk starts on: the rung its item's L picks from the project's rung, or the project's rung without an item.""" +def first_route(ladder: Ladder, project: Rung, item: Item | None) -> Route: + """The route a chunk starts on: the rung its item's L picks on ladder from the project's rung, or the project's rung without an item.""" if item is None: return Route(project) - return Route(worker_rung(project, item.likelihood), item.id, item.likelihood, item.escape_and_impact) + return Route(ladder.worker_rung(project, item.likelihood), item.id, item.likelihood, item.escape_and_impact) -def kept(route: Route, project: Rung) -> bool: +def kept(route: Route, ladder: Ladder, project: Rung) -> bool: """Whether a chunk keeps route: its rung is on the ladder and at or above item_rung.""" - return floor_rung(project, route) == route.rung + return floor_rung(ladder, project, route) == route.rung -def item_rung(project: Rung, route: Route) -> Rung: +def item_rung(ladder: Ladder, project: Rung, route: Route) -> Rung: """The rung the route's item's L picks from the project's rung, or the project's rung itself when no item routed it. The worker may run on a rung below the project's.""" - return worker_rung(project, route.likelihood) if route.item else project + return ladder.worker_rung(project, route.likelihood) if route.item else project -def floor_rung(project: Rung, route: Route, likelihood_taken_in: int = 0) -> Rung: +def floor_rung(ladder: Ladder, project: Rung, route: Route, likelihood_taken_in: int = 0) -> Rung: """The lowest rung the worker may run on: the route's own, raised to item_rung where it is lower or off the ladder, and to the rung likelihood_taken_in picks from project. likelihood_taken_in is the highest L the worker took into the chunk, 0 for none.""" - floor = highest(route.rung, item_rung(project, route)) - return highest(floor, worker_rung(project, likelihood_taken_in)) if likelihood_taken_in else floor + floor = ladder.highest(route.rung, item_rung(ladder, project, route)) + return ladder.highest(floor, ladder.worker_rung(project, likelihood_taken_in)) if likelihood_taken_in else floor -def escalation_rungs(project: Rung, route: Route) -> tuple[Rung, ...]: +def escalation_rungs(ladder: Ladder, project: Rung, route: Route) -> tuple[Rung, ...]: """The rungs an escalation trigger opens to the worker, least competent first: from one above its own up to escalation_cap; none off the ladder.""" - if route.rung not in LADDER: + if route.rung not in ladder.rungs: return () - return LADDER[position(route.rung) + 1:position(escalation_cap(project, route)) + 1] + return ladder.rungs[ladder.position(route.rung) + 1:ladder.position(escalation_cap(ladder, project, route)) + 1] -def escalation_cap(project: Rung, route: Route) -> Rung: +def escalation_cap(ladder: Ladder, project: Rung, route: Route) -> Rung: """The highest rung an escalation trigger takes the worker or a task to without the human's yes: docs/decisions.md → `routing: a worker hands a task one rung up`. It counts from item_rung, the rung the chunk started on, so a step-up between cycles narrows what the worker may climb without a yes, and never widens it. """ - return climbed(item_rung(project, route), ESCALATION_CAP) + return ladder.climbed(item_rung(ladder, project, route), ESCALATION_CAP) @dataclasses.dataclass(frozen=True) @@ -101,31 +101,31 @@ class StepUp: after: Route -def next_route(project: Rung | None, route: Route, verdict: Unfinished | Interrupt | QuotedAnswer) -> Route: +def next_route(ladder: Ladder, project: Rung | None, route: Route, verdict: Unfinished | Interrupt | QuotedAnswer) -> Route: """The route the worker runs on after the judge's verdict. Off the ladder, project is None, and the route holds. The rung never falls.""" if project is None: return route if isinstance(verdict, Unfinished): - return unfinished_route(project, route, verdict) - return escalated_route(project, route, verdict) + return unfinished_route(ladder, project, route, verdict) + return escalated_route(ladder, project, route, verdict) -def unfinished_route(project: Rung, route: Route, verdict: Unfinished) -> Route: +def unfinished_route(ladder: Ladder, project: Rung, route: Route, verdict: Unfinished) -> Route: """The route after an unfinished verdict. The rung rises to floor_rung, which the verdict's highest L taken in may raise. The verdict counts towards UNFINISHED_PER_STEP. Where steps says so, the rung steps once from the floor, and the mark moves only where the verdict escalates. Last, the route takes in the verdict's highest E + I, as Route.taking_in does. """ mark = escalation_mark.moved_to(verdict) - floor = floor_rung(project, route, verdict.taken_in.highest_likelihood) + floor = floor_rung(ladder, project, route, verdict.taken_in.highest_likelihood) counted = route.unfinished + 1 - if steps(counted, mark is not None and below_cap(project, route, floor)): - after = one_rung_up(route, floor, route.escalated_at if mark is None else mark) + if steps(counted, mark is not None and below_cap(ladder, project, route, floor)): + after = one_rung_up(ladder, route, floor, route.escalated_at if mark is None else mark) else: after = dataclasses.replace(route, rung=floor, unfinished=counted) return after.taking_in(verdict.taken_in.highest_escape_and_impact) -def escalated_route(project: Rung, route: Route, verdict: Interrupt | QuotedAnswer) -> Route: +def escalated_route(ladder: Ladder, project: Rung, route: Route, verdict: Interrupt | QuotedAnswer) -> Route: """The route after an interrupt or a quoted answer. Without an escalation it holds, since neither is an unfinished verdict. With one, the rung rises to floor_rung, which the verdict's highest L taken in may raise. Where floor_rung lies below escalation_cap, the rung steps one up from it and the mark moves, as loop_claude/escalation_mark.py says; at the cap the rung stays on floor_rung and the mark holds. @@ -133,8 +133,8 @@ def escalated_route(project: Rung, route: Route, verdict: Interrupt | QuotedAnsw mark = escalation_mark.moved_to(verdict) if mark is None: return route - floor = floor_rung(project, route, verdict.taken_in.highest_likelihood) - return one_rung_up(route, floor, mark) if below_cap(project, route, floor) else dataclasses.replace(route, rung=floor) + floor = floor_rung(ladder, project, route, verdict.taken_in.highest_likelihood) + return one_rung_up(ladder, route, floor, mark) if below_cap(ladder, project, route, floor) else dataclasses.replace(route, rung=floor) def steps(counted: int, escalates: bool) -> bool: @@ -142,9 +142,9 @@ def steps(counted: int, escalates: bool) -> bool: return counted >= UNFINISHED_PER_STEP or escalates -def below_cap(project: Rung, route: Route, floor: Rung) -> bool: +def below_cap(ladder: Ladder, project: Rung, route: Route, floor: Rung) -> bool: """Whether an escalation from floor may step the rung: floor lies below escalation_cap.""" - return position(floor) < position(escalation_cap(project, route)) + return ladder.position(floor) < ladder.position(escalation_cap(ladder, project, route)) def step_up(before: Route, after: Route) -> StepUp | None: @@ -152,13 +152,13 @@ def step_up(before: Route, after: Route) -> StepUp | None: return StepUp(before, after) if after.worker_line() != before.worker_line() else None -def one_rung_up(route: Route, rung: Rung, escalated_at: int) -> Route: +def one_rung_up(ladder: Ladder, route: Route, rung: Rung, escalated_at: int) -> Route: """The route one rung above rung, with the count of unfinished verdicts restarted and the mark at escalated_at.""" - return dataclasses.replace(route, rung=stepped_up(rung), unfinished=0, escalated_at=escalated_at) + return dataclasses.replace(route, rung=ladder.stepped_up(rung), unfinished=0, escalated_at=escalated_at) -def judge_rung(judge_model: str) -> Rung: - """The highest of the judge model's own rungs: docs/decisions.md → `routing: on a model judge-replay covered`. A model off the ladder, or not in JUDGE_REPLAYED, gets no effort flag, so claude runs it at the effort its settings give.""" - if project_rung(judge_model) is None or judge_model not in JUDGE_REPLAYED: +def judge_rung(ladder: Ladder, judge_model: str) -> Rung: + """The highest of the judge model's own rungs on ladder: docs/decisions.md → `routing: on a model judge-replay covered`. A model off the ladder, or not in JUDGE_REPLAYED, gets no effort flag, so claude runs it at the effort its settings give.""" + if ladder.project_rung(judge_model) is None or judge_model not in JUDGE_REPLAYED: return Rung(judge_model) - return highest(*(rung for rung in LADDER if rung.model == judge_model)) + return ladder.highest(*(rung for rung in ladder.rungs if rung.model == judge_model)) diff --git a/bin/loop_claude/test_black_box.py b/bin/loop_claude/test_black_box.py index 0453a4d9..7bd61357 100644 --- a/bin/loop_claude/test_black_box.py +++ b/bin/loop_claude/test_black_box.py @@ -472,6 +472,28 @@ class Cycles(unittest.TestCase): self.assertIn(' Verdict: unfinished\n\nEvidence:\n> three\n\nConfig: \nJudge: judge-model\nWorker: {"model": "claude-opus-5-5", "effort": "medium", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n\n Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n', outcomes) self.assertIn(' Verdict: finished\n\nEvidence:\n> four\n\nConfig: \nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n', outcomes) + def test_routes_on_the_ladder_the_projects_config_folder_holds_in_place_of_the_shipped_one(self): + checkout = Checkout(self) + checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-fable-5-2" + ladder = checkout.home / ".claude-projects" / "proj" / "routing-ladder.json" + ladder.write_text('[{"model": "claude-fable-5-2", "effort": "low"}, {"model": "claude-fable-5-2", "effort": "max"}]') + checkout.commit_plan(CHEAP) + run = checkout.run([worker(), judge(), worker(), judge(), worker()]) + self.assertEqual(run.code, 130) + # Item 4's L of 2 steps one rung down, to Opus 5.5 on the shipped ladder; the project's holds no rung below its own, and one above it, which the second unfinished verdict steps to. + self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "worker"], [("claude-fable-5-2", "low"), ("claude-fable-5-2", "low"), ("claude-fable-5-2", "max")]) + + def test_reads_the_projects_ladder_afresh_each_cycle_and_exits_1_once_it_holds_a_fault(self): + checkout = Checkout(self) + checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-fable-5-2" + ladder = checkout.home / ".claude-projects" / "proj" / "routing-ladder.json" + ladder.write_text('[{"model": "claude-fable-5-2", "effort": "high"}]') + checkout.env["FAKE_CLAUDE_EDIT"] = str(ladder.parent) + run = checkout.run([worker(replace=["routing-ladder.json", '"high"}', '"high"},']), judge()]) + self.assertEqual((run.code, run.roles()), (1, ["worker", "judge"])) + self.assertEqual((argument(run.calls[0], "--model"), argument(run.calls[0], "--effort")), ("claude-fable-5-2", "high")) + self.assertIn(f"cannot read the routing ladder {ladder}: ", run.stderr) + def test_routes_on_and_states_the_same_item_from_the_fetched_default_branch_whatever_the_checkout_holds(self): checkout = Checkout(self) checkout.env["LOOP_CLAUDE_WORKER_MODEL"] = "claude-opus-5-5" @@ -930,6 +952,8 @@ class Reload(unittest.TestCase): checkout = Checkout(self) copy = checkout.temp / "bin" shutil.copytree(BIN, copy, ignore=shutil.ignore_patterns("__pycache__", "test_*")) + # The copied tools read the ladder their setup ships from the folder above their bin/. + shutil.copy(BIN.parent / "routing-ladder.json", copy.parent) checkout.script = copy / "loop-claude" checkout.env["FAKE_CLAUDE_EDIT"] = str(copy) run = checkout.run([ diff --git a/bin/loop_claude/test_loop_claude.py b/bin/loop_claude/test_loop_claude.py index 8a681a8e..60b0399e 100644 --- a/bin/loop_claude/test_loop_claude.py +++ b/bin/loop_claude/test_loop_claude.py @@ -33,15 +33,16 @@ from loop_claude.verdicts import Finished, Interrupt, Judged, QuotedAnswer, Take from git_cli.git_repo import git from loop_state import hand_over, paths, running_loops, session_lock from loop_claude.cycle import interrupt, judge, prompts, retry, run, spend, states, worker -from model_routing.ladder import LADDER, Rung from claude_code.reply import Limited from loop_claude.loop import main, reload, routing_check, this_loop, usage from loop_state.waiting.talk import make_way from machine_load import memory from transcript_archive import outcomes, write from transcript_archive.location import project_archive +from model_routing.ladder_file import read_ladder, SHIPPED from model_routing.route import Route, route_from, WorkerLine from loop_claude import routing +from model_routing.ladder import Rung from loop_claude.stop import Stop from loop_state.running_loops import Taken from claude_code.tokens import Tokens, tokens_of @@ -50,6 +51,7 @@ from loop_claude.cycle.judge import verdict_from from loop_state.waiting.listing import waiting # How the archive records a worker that runs off the ladder. +LADDER = read_ladder(SHIPPED) OFF_LADDER = '{"model": "worker-model"}' # The claude-config release and commit each cycle the tests run is under. CONFIG = "v0.8.0 (abc123)" @@ -288,7 +290,7 @@ class Cycles(unittest.TestCase): self.archive = project_archive(str(self.home / "code" / "proj"), self.home, {}) def cycle(self, fake, records="", unfinished_evidence="", facts="- A preflight fact."): - return run.Cycle(config=CONFIG, effects=fake.effects(), facts=facts, judge_model="judge-model", premises="## Goals", project=None, project_archive=self.archive, questions=self.paths.waiting, records=records, root=str(self.home), route=Route(Rung("worker-model")), session=self.session, show_taken_in=lambda items: None, task="Do it.", todo="", unfinished_evidence=unfinished_evidence) + return run.Cycle(config=CONFIG, effects=fake.effects(), facts=facts, judge_model="judge-model", ladder=LADDER, premises="## Goals", project=None, project_archive=self.archive, questions=self.paths.waiting, records=records, root=str(self.home), route=Route(Rung("worker-model")), session=self.session, show_taken_in=lambda items: None, task="Do it.", todo="", unfinished_evidence=unfinished_evidence) def archived_outcomes(self): return sorted(path.read_text() for path in self.archive.glob("*/*.outcomes.md")) @@ -545,7 +547,7 @@ Spent since 2026-09-21T14:13:20Z, in tokens of input, cache write 5m, cache writ self.assertEqual(concluded.route, Route(Routing.OPUS, 179, 2, 5, escalated_at=3)) first, resumed = fake.made("stream_claude") self.assertEqual([args[args.index("--effort") + 1] for args in (first, resumed)], ["high", "medium"]) - self.assertTrue(resumed[resumed.index("-p") + 1].endswith(f"Continue the task. Your request for a higher rung went through: loop-claude runs you on {Routing.OPUS}. The escalation triggers that fired before this step are spent.{prompts.rungs_held(concluded.route, Routing.OPUS)}")) + self.assertTrue(resumed[resumed.index("-p") + 1].endswith(f"Continue the task. Your request for a higher rung went through: loop-claude runs you on {Routing.OPUS}. The escalation triggers that fired before this step are spent.{prompts.rungs_held(concluded.route, LADDER, Routing.OPUS)}")) self.assertNotIn("went through", first[first.index("-p") + 1]) answered = f"Verdict: quoted answer\n\nWhich one?\n\n{prompts.QUOTE_SOURCES} says:\n> Use A.\n\nEvidence:\n> working\n\nEscalation:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {SONNET_179}\n" stepped = f"Stepped up\n\nWorker: {OPUS_179_AT_3}\n" @@ -1134,53 +1136,53 @@ class Routing(unittest.TestCase): self.assertEqual(routing.routed_item(todo.replace(first, "") + first, NO_CUT).id, 3) def test_starts_a_chunk_on_the_rung_its_items_l_picks_below_the_projects_too(self): - self.assertEqual(routing.first_route(self.OPUS, routing.routed_item(TODO, NO_CUT)), Route(self.ONE_UP, 3, 4, 5)) + self.assertEqual(routing.first_route(LADDER, self.OPUS, routing.routed_item(TODO, NO_CUT)), Route(self.ONE_UP, 3, 4, 5)) cheap = routing.routed_item(TODO.replace("| 4 | 2 | 3 |", "| 1 | 2 | 3 |"), NO_CUT) - self.assertEqual(routing.first_route(self.OPUS, cheap), Route(Rung("claude-sonnet-5-5", "high"), 3, 1, 5)) - self.assertEqual(routing.first_route(self.OPUS, None), Route(self.OPUS)) + self.assertEqual(routing.first_route(LADDER, self.OPUS, cheap), Route(Rung("claude-sonnet-5-5", "high"), 3, 1, 5)) + self.assertEqual(routing.first_route(LADDER, self.OPUS, None), Route(self.OPUS)) def test_steps_up_a_rung_after_every_second_unfinished_verdict(self): - once = routing.next_route(self.OPUS, Route(self.OPUS, 3, 3, 4), unfinished()) + once = routing.next_route(LADDER, self.OPUS, Route(self.OPUS, 3, 3, 4), unfinished()) self.assertEqual(once, Route(self.OPUS, 3, 3, 4, unfinished=1)) - self.assertEqual(routing.next_route(self.OPUS, once, unfinished()), Route(self.ONE_UP, 3, 3, 4)) - self.assertEqual(routing.next_route(self.OPUS, Route(LADDER[-1], unfinished=1), unfinished()).rung, LADDER[-1]) + self.assertEqual(routing.next_route(LADDER, self.OPUS, once, unfinished()), Route(self.ONE_UP, 3, 3, 4)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(LADDER.rungs[-1], unfinished=1), unfinished()).rung, LADDER.rungs[-1]) def test_steps_up_to_the_rung_a_taken_in_items_l_picks_and_never_down(self): - self.assertEqual(routing.next_route(self.OPUS, Route(self.OPUS), unfinished(4)).rung, self.ONE_UP) - self.assertEqual(routing.next_route(self.OPUS, Route(self.TWO_UP), unfinished(4)).rung, self.TWO_UP) - self.assertEqual(routing.next_route(self.OPUS, Route(self.OPUS, unfinished=1), unfinished(4)).rung, self.TWO_UP) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.OPUS), unfinished(4)).rung, self.ONE_UP) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.TWO_UP), unfinished(4)).rung, self.TWO_UP) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.OPUS, unfinished=1), unfinished(4)).rung, self.TWO_UP) def test_steps_up_from_a_rung_below_the_projects_one_rung_at_a_time(self): low = Route(self.SONNET, 3, 1, 3, unfinished=1) - self.assertEqual(routing.next_route(self.OPUS, low, unfinished()).rung, self.OPUS) - self.assertEqual(routing.next_route(self.OPUS, Route(low.rung, 3, 1, 3), unfinished()).rung, low.rung) - self.assertEqual(routing.next_route(self.OPUS, Route(low.rung, 3, 1, 3), unfinished(3)).rung, self.OPUS) + self.assertEqual(routing.next_route(LADDER, self.OPUS, low, unfinished()).rung, self.OPUS) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(low.rung, 3, 1, 3), unfinished()).rung, low.rung) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(low.rung, 3, 1, 3), unfinished(3)).rung, self.OPUS) def test_steps_up_a_rung_on_an_escalation_the_judge_found_up_to_two_above_the_items_rung_marking_the_transcript_entries_it_read(self): - self.assertEqual(routing.next_route(self.OPUS, Route(self.ONE_UP, 3, 4, 5), unfinished(escalated_at=7)), Route(self.TWO_UP, 3, 4, 5, escalated_at=7)) - self.assertEqual(routing.next_route(self.OPUS, Route(self.TWO_UP, 3, 4, 5, escalated_at=7), unfinished(escalated_at=9)), Route(self.THREE_UP, 3, 4, 5, escalated_at=9)) - self.assertEqual(routing.next_route(self.OPUS, Route(self.THREE_UP, 3, 4, 5, escalated_at=9), unfinished(escalated_at=12)), Route(self.THREE_UP, 3, 4, 5, unfinished=1, escalated_at=9)) - self.assertEqual(routing.next_route(self.OPUS, Route(self.ONE_UP, 3, 4, 5, unfinished=1), unfinished(escalated_at=7)), Route(self.TWO_UP, 3, 4, 5, escalated_at=7)) - self.assertEqual(routing.next_route(self.OPUS, Route(self.TWO_UP), unfinished(escalated_at=7)), Route(self.TWO_UP, unfinished=1)) - self.assertEqual(routing.next_route(self.OPUS, Route(self.SONNET, 3, 2, 4), unfinished(4, escalated_at=7)), Route(self.ONE_UP, 3, 2, 4, unfinished=1)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.ONE_UP, 3, 4, 5), unfinished(escalated_at=7)), Route(self.TWO_UP, 3, 4, 5, escalated_at=7)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.TWO_UP, 3, 4, 5, escalated_at=7), unfinished(escalated_at=9)), Route(self.THREE_UP, 3, 4, 5, escalated_at=9)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.THREE_UP, 3, 4, 5, escalated_at=9), unfinished(escalated_at=12)), Route(self.THREE_UP, 3, 4, 5, unfinished=1, escalated_at=9)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.ONE_UP, 3, 4, 5, unfinished=1), unfinished(escalated_at=7)), Route(self.TWO_UP, 3, 4, 5, escalated_at=7)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.TWO_UP), unfinished(escalated_at=7)), Route(self.TWO_UP, unfinished=1)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.SONNET, 3, 2, 4), unfinished(4, escalated_at=7)), Route(self.ONE_UP, 3, 2, 4, unfinished=1)) def test_keeps_the_mark_through_a_step_no_escalation_took(self): - self.assertEqual(routing.next_route(self.OPUS, Route(self.OPUS, 3, 3, 4, unfinished=1, escalated_at=5), unfinished()), Route(self.ONE_UP, 3, 3, 4, escalated_at=5)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.OPUS, 3, 3, 4, unfinished=1, escalated_at=5), unfinished()), Route(self.ONE_UP, 3, 3, 4, escalated_at=5)) def test_steps_up_a_rung_as_the_judge_interrupts_up_to_two_above_the_items_rung_without_counting_an_unfinished_verdict(self): - self.assertEqual(routing.next_route(self.OPUS, Route(self.ONE_UP, 3, 4, 5, unfinished=1), interrupted(escalated_at=7)), Route(self.TWO_UP, 3, 4, 5, escalated_at=7)) - self.assertEqual(routing.next_route(self.OPUS, Route(self.TWO_UP, 3, 4, 5, escalated_at=7), interrupted(escalated_at=9)), Route(self.THREE_UP, 3, 4, 5, escalated_at=9)) - self.assertEqual(routing.next_route(self.OPUS, Route(self.THREE_UP, 3, 4, 5, unfinished=1, escalated_at=9), interrupted(escalated_at=12)), Route(self.THREE_UP, 3, 4, 5, unfinished=1, escalated_at=9)) - self.assertEqual(routing.next_route(self.OPUS, Route(self.SONNET, 3, 2, 4), interrupted(escalated_at=7)), Route(self.OPUS, 3, 2, 4, escalated_at=7)) - self.assertEqual(routing.next_route(self.OPUS, Route(self.ONE_UP, 3, 4, 5, unfinished=1, escalated_at=2), interrupted(4)), Route(self.ONE_UP, 3, 4, 5, unfinished=1, escalated_at=2)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.ONE_UP, 3, 4, 5, unfinished=1), interrupted(escalated_at=7)), Route(self.TWO_UP, 3, 4, 5, escalated_at=7)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.TWO_UP, 3, 4, 5, escalated_at=7), interrupted(escalated_at=9)), Route(self.THREE_UP, 3, 4, 5, escalated_at=9)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.THREE_UP, 3, 4, 5, unfinished=1, escalated_at=9), interrupted(escalated_at=12)), Route(self.THREE_UP, 3, 4, 5, unfinished=1, escalated_at=9)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.SONNET, 3, 2, 4), interrupted(escalated_at=7)), Route(self.OPUS, 3, 2, 4, escalated_at=7)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.ONE_UP, 3, 4, 5, unfinished=1, escalated_at=2), interrupted(4)), Route(self.ONE_UP, 3, 4, 5, unfinished=1, escalated_at=2)) def test_raises_the_worker_to_a_taken_in_items_rung_above_the_escalation_cap_without_a_step_as_the_judge_interrupts(self): - self.assertEqual(routing.next_route(self.OPUS, Route(self.OPUS, 3, 1, 3, escalated_at=2), interrupted(4, escalated_at=7)), Route(self.ONE_UP, 3, 1, 3, escalated_at=2)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.OPUS, 3, 1, 3, escalated_at=2), interrupted(4, escalated_at=7)), Route(self.ONE_UP, 3, 1, 3, escalated_at=2)) def test_steps_up_from_the_rung_a_taken_in_item_picks_as_the_judge_interrupts(self): - self.assertEqual(routing.next_route(self.OPUS, Route(self.OPUS, 3, 3, 4), interrupted(4, escalated_at=7)), Route(self.TWO_UP, 3, 3, 4, escalated_at=7)) - self.assertEqual(routing.next_route(self.OPUS, Route(self.TWO_UP, 3, 4, 5), interrupted(4, escalated_at=7)), Route(self.THREE_UP, 3, 4, 5, escalated_at=7)) - self.assertEqual(routing.next_route(self.OPUS, Route(self.THREE_UP, 3, 4, 5, unfinished=1), interrupted(4, escalated_at=7)), Route(self.THREE_UP, 3, 4, 5, unfinished=1)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.OPUS, 3, 3, 4), interrupted(4, escalated_at=7)), Route(self.TWO_UP, 3, 3, 4, escalated_at=7)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.TWO_UP, 3, 4, 5), interrupted(4, escalated_at=7)), Route(self.THREE_UP, 3, 4, 5, escalated_at=7)) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.THREE_UP, 3, 4, 5, unfinished=1), interrupted(4, escalated_at=7)), Route(self.THREE_UP, 3, 4, 5, unfinished=1)) def test_keeps_an_e_plus_i_taken_in_only_where_it_picks_a_higher_review_rung(self): taken = Route(self.OPUS, 3, 3, 4).taking_in(6) @@ -1195,14 +1197,14 @@ class Routing(unittest.TestCase): self.assertEqual(Route(self.OPUS).taking_in(0), Route(self.OPUS)) def test_a_rung_an_edit_took_off_the_ladder_counts_as_the_projects(self): - self.assertEqual(routing.next_route(self.OPUS, Route(Rung("claude-opus-5-5", "low")), unfinished()).rung, self.OPUS) - self.assertEqual(routing.next_route(self.OPUS, Route(Rung("claude-opus-5-5", "low")), unfinished(2)).rung, self.OPUS) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(Rung("claude-opus-5-5", "low")), unfinished()).rung, self.OPUS) + self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(Rung("claude-opus-5-5", "low")), unfinished(2)).rung, self.OPUS) def test_runs_the_judge_of_a_replayed_model_on_its_models_highest_rung(self): - self.assertEqual(routing.judge_rung("claude-opus-5-5"), self.THREE_UP) + self.assertEqual(routing.judge_rung(LADDER, "claude-opus-5-5"), self.THREE_UP) # A model off the ladder, or not replayed, gets no effort flag. - self.assertEqual(routing.judge_rung("judge-model"), Rung("judge-model")) - self.assertEqual(routing.judge_rung("claude-sonnet-5-5"), Rung("claude-sonnet-5-5")) + self.assertEqual(routing.judge_rung(LADDER, "judge-model"), Rung("judge-model")) + self.assertEqual(routing.judge_rung(LADDER, "claude-sonnet-5-5"), Rung("claude-sonnet-5-5")) def test_refuses_a_route_whose_item_has_levels_off_their_scales_or_levels_without_an_item(self): for item, likelihood, escape_and_impact in ((3, 0, 5), (3, 5, 5), (3, 4, 1), (3, 4, 9), (0, 4, 5), (0, 4, 0), (0, 0, 5)): @@ -1247,14 +1249,14 @@ class Routing(unittest.TestCase): archived = Route(self.TWO_UP, 3, 4, 5) def never(): raise AssertionError("not asked") - self.assertEqual(chunk_start.routed(archived, None, "worker-model", never, never), Route(Rung("worker-model"))) - self.assertEqual(chunk_start.routed(archived, self.OPUS, "claude-opus-5-5", never, never), archived) - self.assertEqual(chunk_start.routed(None, self.OPUS, "claude-opus-5-5", lambda: archived, never), archived) - self.assertEqual(chunk_start.routed(None, self.OPUS, "claude-opus-5-5", lambda: None, lambda: item), Route(self.ONE_UP, 3, 4, 5)) - self.assertEqual(chunk_start.routed(Route(self.SONNET), self.OPUS, "claude-opus-5-5", never, lambda: None), Route(self.OPUS)) + self.assertEqual(chunk_start.routed(archived, LADDER, None, "worker-model", never, never), Route(Rung("worker-model"))) + self.assertEqual(chunk_start.routed(archived, LADDER, self.OPUS, "claude-opus-5-5", never, never), archived) + self.assertEqual(chunk_start.routed(None, LADDER, self.OPUS, "claude-opus-5-5", lambda: archived, never), archived) + self.assertEqual(chunk_start.routed(None, LADDER, self.OPUS, "claude-opus-5-5", lambda: None, lambda: item), Route(self.ONE_UP, 3, 4, 5)) + self.assertEqual(chunk_start.routed(Route(self.SONNET), LADDER, self.OPUS, "claude-opus-5-5", never, lambda: None), Route(self.OPUS)) cheap = Route(self.SONNET, 4, 2, 3) - self.assertEqual(chunk_start.routed(cheap, self.OPUS, "claude-opus-5-5", never, never), cheap) - self.assertEqual(chunk_start.routed(Route(Rung("claude-haiku-5-5", "max"), 4, 2, 3), self.OPUS, "claude-opus-5-5", never, lambda: None), Route(self.OPUS)) + self.assertEqual(chunk_start.routed(cheap, LADDER, self.OPUS, "claude-opus-5-5", never, never), cheap) + self.assertEqual(chunk_start.routed(Route(Rung("claude-haiku-5-5", "max"), 4, 2, 3), LADDER, self.OPUS, "claude-opus-5-5", never, lambda: None), Route(self.OPUS)) class RouteSources(unittest.TestCase): @@ -1338,29 +1340,29 @@ class Steps(unittest.TestCase): def test_a_finished_verdict_clears_the_route_and_the_evidence(self): run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="old", route=Route(self.OPUS)) - self.assertEqual(close.after_verdict(run_with, self.OPUS, run.Concluded(Finished("x"), run_with.route)), dataclasses.replace(run_with, route=None, unfinished_evidence="")) - self.assertEqual(close.after_verdict(dataclasses.replace(run_with, unfinished_run=3), self.OPUS, run.Concluded(Finished("x"), run_with.route)).unfinished_run, 0) - self.assertEqual(close.after_verdict(run_with, self.OPUS, run.Concluded(Unfinished("new", taken_in=TakenIn(highest_likelihood=4)), run_with.route)), dataclasses.replace(run_with, route=Route(Rung("claude-fable-5-2", "low"), unfinished=1), unfinished_evidence="new", unfinished_run=1)) + self.assertEqual(close.after_verdict(run_with, LADDER, self.OPUS, run.Concluded(Finished("x"), run_with.route)), dataclasses.replace(run_with, route=None, unfinished_evidence="")) + self.assertEqual(close.after_verdict(dataclasses.replace(run_with, unfinished_run=3), LADDER, self.OPUS, run.Concluded(Finished("x"), run_with.route)).unfinished_run, 0) + self.assertEqual(close.after_verdict(run_with, LADDER, self.OPUS, run.Concluded(Unfinished("new", taken_in=TakenIn(highest_likelihood=4)), run_with.route)), dataclasses.replace(run_with, route=Route(Rung("claude-fable-5-2", "low"), unfinished=1), unfinished_evidence="new", unfinished_run=1)) def test_an_unfinished_chunk_keeps_the_items_its_verdict_found_taken_in_and_a_finished_one_drops_them(self): run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(self.OPUS), items_taken_in=(3,)) - self.assertEqual(close.after_verdict(run_with, self.OPUS, run.Concluded(Unfinished("new", taken_in=TakenIn((3, 12))), run_with.route)).items_taken_in, (3, 12)) - self.assertEqual(close.after_verdict(run_with, self.OPUS, run.Concluded(Finished("x"), run_with.route)).items_taken_in, ()) + self.assertEqual(close.after_verdict(run_with, LADDER, self.OPUS, run.Concluded(Unfinished("new", taken_in=TakenIn((3, 12))), run_with.route)).items_taken_in, (3, 12)) + self.assertEqual(close.after_verdict(run_with, LADDER, self.OPUS, run.Concluded(Finished("x"), run_with.route)).items_taken_in, ()) def test_steps_from_the_route_its_cycle_ended_on_which_a_ruling_may_have_raised(self): run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(self.OPUS, 3, 3, 4)) raised = Route(Rung("claude-fable-5-2", "low"), 3, 3, 4, escalated_at=5) - self.assertEqual(close.after_verdict(run_with, self.OPUS, run.Concluded(Unfinished("new"), raised)).route, dataclasses.replace(raised, unfinished=1)) + self.assertEqual(close.after_verdict(run_with, LADDER, self.OPUS, run.Concluded(Unfinished("new"), raised)).route, dataclasses.replace(raised, unfinished=1)) def test_keeps_the_route_its_cycle_ended_on_off_the_ladder(self): off_ladder = Route(Rung("worker-model")) run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="") - self.assertEqual(close.after_verdict(run_with, None, run.Concluded(Unfinished("new", "round 2: 🔴", TakenIn(highest_likelihood=4), entries_read=9), off_ladder)).route, off_ladder) + self.assertEqual(close.after_verdict(run_with, LADDER, None, run.Concluded(Unfinished("new", "round 2: 🔴", TakenIn(highest_likelihood=4), entries_read=9), off_ladder)).route, off_ladder) def test_an_unfinished_verdicts_escalation_marks_the_transcript_entries_the_judge_read(self): run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(self.OPUS, 3, 3, 4, escalated_at=2)) - self.assertEqual(close.after_verdict(run_with, self.OPUS, run.Concluded(Unfinished("new", "round 2: 🔴", entries_read=9), run_with.route)).route, Route(Rung("claude-fable-5-2", "low"), 3, 3, 4, escalated_at=9)) - self.assertEqual(close.after_verdict(run_with, self.OPUS, run.Concluded(Unfinished("new", entries_read=9), run_with.route)).route, Route(self.OPUS, 3, 3, 4, unfinished=1, escalated_at=2)) + self.assertEqual(close.after_verdict(run_with, LADDER, self.OPUS, run.Concluded(Unfinished("new", "round 2: 🔴", entries_read=9), run_with.route)).route, Route(Rung("claude-fable-5-2", "low"), 3, 3, 4, escalated_at=9)) + self.assertEqual(close.after_verdict(run_with, LADDER, self.OPUS, run.Concluded(Unfinished("new", entries_read=9), run_with.route)).route, Route(self.OPUS, 3, 3, 4, unfinished=1, escalated_at=2)) def test_archives_a_step_up_as_it_happens_so_a_rerun_resumes_on_it(self): with tempfile.TemporaryDirectory() as home: @@ -1372,14 +1374,14 @@ class Steps(unittest.TestCase): before = Route(self.OPUS, 3, 3, 4) with contextlib.redirect_stderr(io.StringIO()): self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW)) - after = close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), self.OPUS, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), archive, a_loop(FakeEffects().effects(), session=opened)) + after = close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), LADDER, self.OPUS, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), archive, a_loop(FakeEffects().effects(), session=opened)) self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(Rung("claude-fable-5-2", "low"), 3, 3, 4)) with contextlib.redirect_stderr(io.StringIO()): - close.closed_cycle(after, self.OPUS, run.Concluded(Finished("done"), after.route), archive, a_loop(FakeEffects().effects(), session=opened)) + close.closed_cycle(after, LADDER, self.OPUS, run.Concluded(Finished("done"), after.route), archive, a_loop(FakeEffects().effects(), session=opened)) self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(Rung("claude-fable-5-2", "low"), 3, 3, 4)) blocked = pathlib.Path(home, "blocked") blocked.write_text("") - stop = close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), self.OPUS, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), blocked, a_loop(FakeEffects().effects(), session=opened)) + stop = close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), LADDER, self.OPUS, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), blocked, a_loop(FakeEffects().effects(), session=opened)) self.assertEqual((stop.code, stop.reason.split(":")[0]), (1, "could not archive the transcript")) def test_names_no_step_up_where_only_the_count_of_unfinished_verdicts_changed(self): @@ -1397,7 +1399,7 @@ class Steps(unittest.TestCase): before = Route(self.OPUS, 3, 3, 4) with contextlib.redirect_stderr(io.StringIO()): self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW)) - close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), self.OPUS, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_escape_and_impact=6)), before), archive, a_loop(FakeEffects().effects(), session=opened)) + close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), LADDER, self.OPUS, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_escape_and_impact=6)), before), archive, a_loop(FakeEffects().effects(), session=opened)) self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.OPUS, 3, 3, 4, escape_and_impact_taken_in=6)) @@ -1443,13 +1445,13 @@ class TypedAnswers(unittest.TestCase): def test_splits_the_prompt_handing_over_the_answer_into_the_answer_on_the_line_after_its_marker_and_the_rest_and_no_other_prompt(self): route = Route(Rung("worker-model")) marked = "The human answered your question at loop-claude's terminal:\nB" - typed = prompts.typed_answer(prompts.typed("Which one?", "Use B.", route, None, False)) + typed = prompts.typed_answer(prompts.typed("Which one?", "Use B.", route, LADDER, None, False)) self.assertEqual(typed.answer, "Use B.") self.assertTrue(typed.loop_words.startswith("Your question was: Which one?\n\nContinue the task.")) - self.assertEqual(prompts.typed_answer(prompts.typed(f"Which one?\n{marked}", "Use B.", route, None, False)).answer, "Use B.") - self.assertEqual(prompts.typed_answer(prompts.typed("Which one?", "Use B.", Route(Routing.SONNET, 179, 2, 5), Routing.OPUS, True)).answer, "Use B.") - self.assertIsNone(prompts.typed_answer(prompts.answered("Which one?", marked, route, None, False))) - self.assertIsNone(prompts.typed_answer(prompts.resumed(marked, route, None))) + self.assertEqual(prompts.typed_answer(prompts.typed(f"Which one?\n{marked}", "Use B.", route, LADDER, None, False)).answer, "Use B.") + self.assertEqual(prompts.typed_answer(prompts.typed("Which one?", "Use B.", Route(Routing.SONNET, 179, 2, 5), LADDER, Routing.OPUS, True)).answer, "Use B.") + self.assertIsNone(prompts.typed_answer(prompts.answered("Which one?", marked, route, LADDER, None, False))) + self.assertIsNone(prompts.typed_answer(prompts.resumed(marked, route, LADDER, None))) class WorkerArgs(unittest.TestCase): @@ -1460,24 +1462,24 @@ class WorkerArgs(unittest.TestCase): FACTS = "- A preflight fact." def test_names_the_route_in_a_fresh_prompt_only_where_an_item_routed_it(self): - routed = prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), None, self.FACTS) + routed = prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), LADDER, None, self.FACTS) self.assertIn(f"Continue any open draft PR or worktree for the chunk before creating a new one. {prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\nloop-claude runs you on m at high effort, routed by item 3's L of 4.", routed) - unrouted = prompts.fresh("Do it.", "## Goals", Route(Rung("m")), None, self.FACTS) + unrouted = prompts.fresh("Do it.", "## Goals", Route(Rung("m")), LADDER, None, self.FACTS) self.assertIn(f"before creating a new one. {prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\n{self.FACTS}\n\nThe chunk's", unrouted) self.assertNotIn("loop-claude runs you on", unrouted) def test_has_the_routed_worker_count_its_items_size_again_and_puts_the_facts_before_the_scope(self): - routed = prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), None, self.FACTS) + routed = prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), LADDER, None, self.FACTS) self.assertIn("item 3's size again", routed) self.assertIn("escalation trigger 1", routed) self.assertIn(f"\n\n{self.FACTS}\n\nThe chunk's", routed) - self.assertNotIn("size again", prompts.fresh("Do it.", "## Goals", Route(Rung("m")), None, self.FACTS)) + self.assertNotIn("size again", prompts.fresh("Do it.", "## Goals", Route(Rung("m")), LADDER, None, self.FACTS)) def test_gives_each_fresh_worker_the_reading_rule_once(self): rule = "never read more than 80 lines of a file at once, nor take a tool output over 3,000 characters into your context" for route in (Route(Rung("m", "high"), 3, 4, 5), Route(Rung("m"))): with self.subTest(route=route): - self.assertEqual(prompts.fresh("Do it.", "## Goals", route, None, self.FACTS).count(rule), 1) + self.assertEqual(prompts.fresh("Do it.", "## Goals", route, LADDER, None, self.FACTS).count(rule), 1) def test_quotes_what_the_preflight_printed_or_has_the_worker_find_the_facts_itself(self): self.assertIn("\n\n- A fact.", prompts.preflight_facts("- A fact.")) @@ -1488,93 +1490,93 @@ class WorkerArgs(unittest.TestCase): def test_has_each_fresh_worker_continue_an_open_draft_pr_or_worktree_once_whatever_the_preflight_found(self): for facts in (prompts.preflight_facts("- A fact."), prompts.preflight_failed("git could not start")): with self.subTest(facts=facts): - self.assertEqual(prompts.fresh("Do it.", "## Goals", Route(Rung("m")), None, facts).count("Continue any open draft PR or worktree"), 1) + self.assertEqual(prompts.fresh("Do it.", "## Goals", Route(Rung("m")), LADDER, None, facts).count("Continue any open draft PR or worktree"), 1) def test_keeps_every_review_on_the_projects_rung_while_the_worker_runs_below_it(self): opus = Rung("claude-opus-5-5", "medium") - below = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 4), opus, self.FACTS) + below = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 4), LADDER, opus, self.FACTS) self.assertIn(f"item 3's L of 2. Launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool set to model opus, effort medium, the project's rung.{self.SCAN} Your rung is below the project's: launch each subagent that files or scores todo.md items on the rung `~/.claude/bin/subagent-rung --plan claude-opus-5-5 ` prints. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger", below) - self.assertNotIn("Launch each", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS)) + self.assertNotIn("Launch each", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), LADDER, opus, self.FACTS)) def test_launches_reviewers_on_the_rung_the_items_e_plus_i_picks_above_the_workers(self): opus = Rung("claude-opus-5-5", "medium") - level = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 6), opus, self.FACTS) + level = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 6), LADDER, opus, self.FACTS) self.assertIn(f"item 3's L of 3. Launch each reviewer with the Agent tool set to model fable, effort medium, the rung item 3's E + I of 6 picks. Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", level) - below = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 5), opus, self.FACTS) + below = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 5), LADDER, opus, self.FACTS) self.assertIn(f"item 3's L of 2. Launch each reviewer with the Agent tool set to model fable, effort low, the rung item 3's E + I of 5 picks.{self.SEATS}{self.SCAN} Your rung is below the project's:", below) - self.assertNotIn("Launch each reviewer", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 4, 6), opus)) + self.assertNotIn("Launch each reviewer", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 4, 6), LADDER, opus)) def test_names_the_reviewers_rung_each_higher_e_plus_i_an_item_taken_in_picks(self): opus = Rung("claude-opus-5-5", "medium") taking_in = " Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8." - self.assertIn(f"item 3's L of 3.{taking_in} When an escalation trigger", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS)) - self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.", prompts.resumed("", Route(opus, 3, 3, 6), opus)) - self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 4, 4), opus)) - self.assertNotIn("Where an item you take into", prompts.resumed("", Route(opus, 3, 3, 8), opus)) - self.assertNotIn("Where an item you take into", prompts.resumed("", Route(Rung("m")), None)) + self.assertIn(f"item 3's L of 3.{taking_in} When an escalation trigger", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), LADDER, opus, self.FACTS)) + self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.", prompts.resumed("", Route(opus, 3, 3, 6), LADDER, opus)) + self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 4, 4), LADDER, opus)) + self.assertNotIn("Where an item you take into", prompts.resumed("", Route(opus, 3, 3, 8), LADDER, opus)) + self.assertNotIn("Where an item you take into", prompts.resumed("", Route(Rung("m")), LADDER, None)) def test_launches_reviewers_on_the_rung_the_highest_e_plus_i_taken_in_picks(self): opus = Rung("claude-opus-5-5", "medium") - self.assertIn(f"loop-claude runs you on claude-opus-5-5 at medium effort. Launch each reviewer with the Agent tool set to model opus, effort xhigh, the rung picked by an item of E + I 7 taken into the chunk. Where an item you take into the chunk has an E + I of 8 or more, launch each reviewer with the Agent tool set to `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", prompts.resumed("", Route(opus, 3, 3, 5, escape_and_impact_taken_in=7), opus)) - self.assertIn(" Launch each reviewer with the Agent tool set to model fable, effort low, the rung picked by an item of E + I 5 taken into the chunk.", prompts.resumed("", Route(opus, escape_and_impact_taken_in=5), opus)) + self.assertIn(f"loop-claude runs you on claude-opus-5-5 at medium effort. Launch each reviewer with the Agent tool set to model opus, effort xhigh, the rung picked by an item of E + I 7 taken into the chunk. Where an item you take into the chunk has an E + I of 8 or more, launch each reviewer with the Agent tool set to `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", prompts.resumed("", Route(opus, 3, 3, 5, escape_and_impact_taken_in=7), LADDER, opus)) + self.assertIn(" Launch each reviewer with the Agent tool set to model fable, effort low, the rung picked by an item of E + I 5 taken into the chunk.", prompts.resumed("", Route(opus, escape_and_impact_taken_in=5), LADDER, opus)) def test_keeps_seats_and_the_security_pass_on_the_projects_rung_while_the_worker_runs_above_it(self): opus = Rung("claude-opus-5-5", "medium") - above = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-fable-5-2", "low"), 3, 4, 4), opus, self.FACTS) - stepped_up = prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), opus) + above = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-fable-5-2", "low"), 3, 4, 4), LADDER, opus, self.FACTS) + stepped_up = prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), LADDER, opus) self.assertIn(f"{prompts.ANSWERS_RECORDED}. {prompts.NAME_THE_ITEMS} loop-claude runs you on claude-fable-5-2 at low effort.{self.SEATS}{self.SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", stepped_up) - self.assertNotIn("the project's", prompts.resumed("", Route(opus, 3, 3, 4), opus)) - self.assertNotIn("runs you on", prompts.resumed("", Route(Rung("m")), None)) + self.assertNotIn("the project's", prompts.resumed("", Route(opus, 3, 3, 4), LADDER, opus)) + self.assertNotIn("runs you on", prompts.resumed("", Route(Rung("m")), LADDER, None)) self.assertIn(f"item 3's L of 4.{self.SEATS}{self.SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", above) def test_names_the_rungs_an_escalation_trigger_lets_the_worker_hand_a_task_to_at_most_two_above_its_items(self): opus = Rung("claude-opus-5-5", "medium") - fresh = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS) + fresh = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), LADDER, opus, self.FACTS) self.assertIn("item 3's L of 3. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort low`, one rung above yours, and on each further trigger one rung higher, up to `model fable, effort medium`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger :`, n the trigger's number.", fresh) - self.assertIn(" When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort medium`, one rung above yours; above that, ask the human first. Open the subagent's prompt with `Escalation trigger :`, n the trigger's number.", prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), opus)) - self.assertIn(" An escalation trigger in `~/.claude/CLAUDE.md` lets you hand the hard part of a task to a subagent with the Agent tool set to `model opus, effort xhigh`, one rung above yours, only once the human agrees; loop-claude will not resume you higher for a trigger either, so ask the human for that yes. Open the subagent's prompt with `Escalation trigger :`, n the trigger's number.", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 3, 4), opus)) - self.assertNotIn("escalation", prompts.resumed("", Route(Rung("claude-fable-5-2", "max"), 3, 4, 8), opus)) - self.assertIn("only once the human agrees", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 4, 5), opus)) - self.assertIn("hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort xhigh`, one rung above yours;", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 4, 5), opus)) - self.assertNotIn("escalation", prompts.resumed("", Route(Rung("m")), None)) - self.assertNotIn("escalation", prompts.resumed("", Route(Rung("claude-opus-5-5", "low"), 3, 3, 4), opus)) - self.assertIn(f"{prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\nWhere an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort low`, one rung above yours, and on each further trigger one rung higher, up to `model fable, effort medium`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger :`, n the trigger's number. Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `model fable, effort medium`. A reviewer", prompts.fresh("Do it.", "## Goals", Route(opus), opus, self.FACTS)) + self.assertIn(" When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort medium`, one rung above yours; above that, ask the human first. Open the subagent's prompt with `Escalation trigger :`, n the trigger's number.", prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), LADDER, opus)) + self.assertIn(" An escalation trigger in `~/.claude/CLAUDE.md` lets you hand the hard part of a task to a subagent with the Agent tool set to `model opus, effort xhigh`, one rung above yours, only once the human agrees; loop-claude will not resume you higher for a trigger either, so ask the human for that yes. Open the subagent's prompt with `Escalation trigger :`, n the trigger's number.", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 3, 4), LADDER, opus)) + self.assertNotIn("escalation", prompts.resumed("", Route(Rung("claude-fable-5-2", "max"), 3, 4, 8), LADDER, opus)) + self.assertIn("only once the human agrees", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 4, 5), LADDER, opus)) + self.assertIn("hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort xhigh`, one rung above yours;", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 4, 5), LADDER, opus)) + self.assertNotIn("escalation", prompts.resumed("", Route(Rung("m")), LADDER, None)) + self.assertNotIn("escalation", prompts.resumed("", Route(Rung("claude-opus-5-5", "low"), 3, 3, 4), LADDER, opus)) + self.assertIn(f"{prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\nWhere an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort low`, one rung above yours, and on each further trigger one rung higher, up to `model fable, effort medium`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger :`, n the trigger's number. Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `model fable, effort medium`. A reviewer", prompts.fresh("Do it.", "## Goals", Route(opus), LADDER, opus, self.FACTS)) def test_offers_the_worker_a_higher_rung_for_the_rest_of_the_chunk_up_to_the_escalation_cap(self): opus = Rung("claude-opus-5-5", "medium") offer = " Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `model fable, effort medium`." - self.assertIn(f"`Escalation trigger :`, n the trigger's number.{offer}", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS)) - self.assertIn(f"`Escalation trigger :`, n the trigger's number.{offer}", prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), opus)) - self.assertNotIn("Where the hard part", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 3, 4), opus)) + self.assertIn(f"`Escalation trigger :`, n the trigger's number.{offer}", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), LADDER, opus, self.FACTS)) + self.assertIn(f"`Escalation trigger :`, n the trigger's number.{offer}", prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), LADDER, opus)) + self.assertNotIn("Where the hard part", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 3, 4), LADDER, opus)) def test_lets_a_markdown_only_branch_review_on_the_projects_rung_where_the_reviewers_run_above_it(self): opus = Rung("claude-opus-5-5", "medium") markdown = " Where your branch changes only Markdown, none of it `CLAUDE.md`, `AGENTS.md`, `workflow-bench/*-prompt.md` or under `agents/`, `principles/`, `rules/` or `skills/`, launch each reviewer on the project's rung instead, with the Agent tool set to model opus, effort medium." - self.assertIn(markdown, prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 7), opus, self.FACTS)) - self.assertIn(markdown, prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 4, 4), opus)) - self.assertNotIn("only Markdown", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS)) + self.assertIn(markdown, prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 7), LADDER, opus, self.FACTS)) + self.assertIn(markdown, prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 4, 4), LADDER, opus)) + self.assertNotIn("only Markdown", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), LADDER, opus, self.FACTS)) def test_names_the_rung_a_reviewer_or_seat_reruns_on_one_above_its_own(self): opus = Rung("claude-opus-5-5", "medium") both = " A reviewer or comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort low`." - self.assertTrue(prompts.resumed("", Route(opus, 3, 3, 4), opus).endswith(both)) - self.assertTrue(prompts.resumed("", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 4), opus).endswith(both)) - self.assertTrue(prompts.fresh("Do it.", "## Goals", Route(opus), opus, self.FACTS).split(f"\n\n{self.FACTS}")[0].endswith(both)) - self.assertTrue(prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 4, 4), opus).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort medium`, and a comprehension seat with `model fable, effort low`.")) - self.assertTrue(prompts.resumed("", Route(opus, 3, 3, 7), opus).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort high`, and a comprehension seat with `model fable, effort low`.")) - self.assertTrue(prompts.resumed("", Route(Rung("claude-fable-5-2", "max"), 3, 4, 8), opus).endswith(" A comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort low`.")) - self.assertNotIn("rerun", prompts.resumed("", Route(Rung("m")), None)) - self.assertNotIn("rerun", prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), None, self.FACTS)) - self.assertIn(f"{both} After your first read", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), opus, self.FACTS)) + self.assertTrue(prompts.resumed("", Route(opus, 3, 3, 4), LADDER, opus).endswith(both)) + self.assertTrue(prompts.resumed("", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 4), LADDER, opus).endswith(both)) + self.assertTrue(prompts.fresh("Do it.", "## Goals", Route(opus), LADDER, opus, self.FACTS).split(f"\n\n{self.FACTS}")[0].endswith(both)) + self.assertTrue(prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 4, 4), LADDER, opus).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort medium`, and a comprehension seat with `model fable, effort low`.")) + self.assertTrue(prompts.resumed("", Route(opus, 3, 3, 7), LADDER, opus).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort high`, and a comprehension seat with `model fable, effort low`.")) + self.assertTrue(prompts.resumed("", Route(Rung("claude-fable-5-2", "max"), 3, 4, 8), LADDER, opus).endswith(" A comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort low`.")) + self.assertNotIn("rerun", prompts.resumed("", Route(Rung("m")), LADDER, None)) + self.assertNotIn("rerun", prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), LADDER, None, self.FACTS)) + self.assertIn(f"{both} After your first read", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), LADDER, opus, self.FACTS)) fable = Rung("claude-fable-5-2", "max") - self.assertNotIn("rerun", prompts.resumed("", Route(fable), fable)) + self.assertNotIn("rerun", prompts.resumed("", Route(fable), LADDER, fable)) def test_opens_a_fresh_session_by_name_and_resumes_an_open_one(self): opened = session.session_named("A proj loop-claude", pathlib.Path("/h/.claude/projects")) - fresh = worker.worker_args(opened, Rung("m"), prompts.fresh("Do it.", "## Goals\n1. g\n\n## Audience\nme", Route(Rung("m")), None, self.FACTS), resume=False) + fresh = worker.worker_args(opened, Rung("m"), prompts.fresh("Do it.", "## Goals\n1. g\n\n## Audience\nme", Route(Rung("m")), LADDER, None, self.FACTS), resume=False) self.assertEqual(fresh[fresh.index("--session-id") + 1], str(opened.id)) self.assertEqual(fresh[fresh.index("--name") + 1], "A proj loop-claude") - resumed = worker.worker_args(opened, Rung("m"), prompts.resumed("", Route(Rung("m")), None), resume=True) + resumed = worker.worker_args(opened, Rung("m"), prompts.resumed("", Route(Rung("m")), LADDER, None), resume=True) self.assertEqual(resumed[resumed.index("--resume") + 1], str(opened.id)) self.assertNotIn("--session-id", resumed) self.assertTrue(resumed[resumed.index("-p") + 1].startswith("Continue the task; the previous run ended before the chunk was finished. The chunk is finished once")) @@ -1673,7 +1675,7 @@ class JudgePrompt(unittest.TestCase): self.assertEqual(judge.quotes_checked(Unfinished("e", "new", entries_read=3), grounds), Unfinished("e", "new", entries_read=3)) def test_shows_a_typed_answer_before_the_step_up_as_the_humans(self): - typed = prompts.typed("Which one?", "Use B.", Route(Rung("worker-model")), None, False) + typed = prompts.typed("Which one?", "Use B.", Route(Rung("worker-model")), LADDER, None, False) entries = [{**human(typed), "promptSource": "sdk"}, assistant({"type": "text", "text": "round 3: 🔴"})] _, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", kinds(*entries), 1)) self.assertIn("\n\n[human] Use B.\n\n[step] ", grounds.shown) @@ -1813,7 +1815,7 @@ class ChunkItems(unittest.TestCase): self.assertEqual(chunk_items.named([said(example)]), (297, 298)) def test_every_prompt_that_starts_or_resumes_a_chunk_asks_for_the_line(self): - self.assertIn("on one line, as in `Chunk items: ", prompts.resumed("", Route(Rung("m")), None)) + self.assertIn("on one line, as in `Chunk items: ", prompts.resumed("", Route(Rung("m")), LADDER, None)) def test_reads_only_a_line_the_worker_opens_with_the_label_in_its_own_text(self): prompt = entries.LoopPrompt(text="Chunk items: 297 (L 3, E + I 5)") @@ -2086,6 +2088,7 @@ class ExitContract(unittest.TestCase): ("loop/close.py", "closed_cycle", "1", "failure.reason", read), ("loop/main.py", "next_cycle", "1", "premises.reason", premises_), ("loop/main.py", "next_cycle", "1", "records.reason", unreadable), + ("loop/main.py", "next_cycle", "1", "ladder.reason", "the project's routing ladder cannot be read or holds a fault"), ("loop/main.py", "loop_run_from", "0 if task else 1", "HELP", "no task is given"), ("loop/main.py", "loop_run_from", "1", 'f"the edited loop-claude cannot read the state', "a reload cannot read the state the running script handed it"), ("loop/reload.py", "exec_if_edited", "1", "f'could not reload loop-claude:", "python cannot start for a reload"), diff --git a/bin/model_routing/ladder.py b/bin/model_routing/ladder.py index 3dd225ed..da316a7c 100644 --- a/bin/model_routing/ladder.py +++ b/bin/model_routing/ladder.py @@ -1,4 +1,4 @@ -"""The routing ladder, and the rungs a task's L and E + I pick from the project's rung: docs/decisions.md → `routing:`.""" +"""A routing ladder, and the rungs a task's L and E + I pick on it from the project's rung: docs/decisions.md → `routing:`. model_routing/ladder_file.py reads a project's ladder from its file.""" import dataclasses import re import typing @@ -7,6 +7,8 @@ from review_rules.review_passes import ReviewedPass # How Rung.__str__ spells a rung that names an effort, and Rung.parse reads it back. WORDING = re.compile(r"(?P\S+) at (?P\S+) effort") +# Lowest first, as Claude Code's --effort takes them. +EFFORTS = ("low", "medium", "high", "xhigh", "max") # What a review launch reviews: such a pass, a comprehension seat or the security pass. ReviewKind = ReviewedPass | typing.Literal["seat", "security"] @@ -28,14 +30,6 @@ class Rung: """The rung in the Agent tool's words. The tool takes a family name, such as opus, and runs that family's latest model, which is the model each rung names.""" return f"model {self.family()}, effort {self.effort}" - @staticmethod - def from_agent(model: str, effort: str) -> "Rung": - """The rung the Agent tool's model and effort name: a family such as opus is its ladder model, at the effort given or else its cheapest. A family off the ladder, or an effort off it, gives a rung that is not on LADDER.""" - family = next((rung for rung in LADDER if rung.family() == model), None) - if family is None: - return Rung(model, effort) - return Rung(family.model, effort) if effort else family - # loop-claude hands a rung to rule-hook in environment variables in this spelling, so parse must keep reading it. def __str__(self) -> str: return f"{self.model} at {self.effort} effort" if self.effort else self.model @@ -47,76 +41,68 @@ class Rung: return Rung(found["model"], found["effort"]) if found else None -# Least competent first. -LADDER = ( - Rung("claude-sonnet-5-5", "high"), - Rung("claude-opus-5-5", "medium"), - Rung("claude-fable-5-2", "low"), - Rung("claude-fable-5-2", "medium"), - Rung("claude-opus-5-5", "xhigh"), - Rung("claude-fable-5-2", "high"), - Rung("claude-fable-5-2", "max"), -) # The rungs a task's L steps its worker from the project's rung. WORKER_STEPS = {1: -1, 2: -1, 3: 0, 4: 1} # The rungs a task's E + I steps a review up from the project's rung; an E + I under 5 steps none. REVIEW_STEPS = {5: 1, 6: 2, 7: 3, 8: 4} -def project_rung(model: str) -> Rung | None: - """The model's cheapest rung, which for Opus 5.5 is its default effort; None for a model off the ladder.""" - return next((rung for rung in LADDER if rung.model == model), None) +@dataclasses.dataclass(frozen=True) +class Ladder: + """A project's rungs, least competent first. model_routing/ladder_file.py reads one, and refuses a file that holds no rung or one rung twice, since a rung's position sets every step.""" + rungs: tuple[Rung, ...] + def from_agent(self, model: str, effort: str) -> Rung: + """The rung the Agent tool's model and effort name: a family such as opus is its ladder model, at the effort given or else its cheapest. A family off the ladder, or an effort off it, gives a rung that is not on the ladder.""" + family = next((rung for rung in self.rungs if rung.family() == model), None) + if family is None: + return Rung(model, effort) + return Rung(family.model, effort) if effort else family -def worker_rung(project: Rung, likelihood: int) -> Rung: - """The rung a task's L picks for its worker, counted from the project's rung, stopping at either end of the ladder.""" - return climbed(project, WORKER_STEPS[likelihood]) + def project_rung(self, model: str) -> Rung | None: + """The model's cheapest rung, which for Opus 5.5 is its default effort; None for a model off the ladder.""" + return next((rung for rung in self.rungs if rung.model == model), None) + def worker_rung(self, project: Rung, likelihood: int) -> Rung: + """The rung a task's L picks for its worker, counted from the project's rung, stopping at either end of the ladder.""" + return self.climbed(project, WORKER_STEPS[likelihood]) -def lowest_worker_rung(project: Rung) -> Rung: - """The lowest rung any L routes a worker to, counted from the project's rung.""" - return climbed(project, min(WORKER_STEPS.values())) + def lowest_worker_rung(self, project: Rung) -> Rung: + """The lowest rung any L routes a worker to, counted from the project's rung.""" + return self.climbed(project, min(WORKER_STEPS.values())) + def planning_rung(self, project: Rung, likelihood: int) -> Rung: + """The rung of a task that files or scores todo.md items: the one its L picks, never below the project's rung.""" + return self.highest(project, self.worker_rung(project, likelihood)) -def planning_rung(project: Rung, likelihood: int) -> Rung: - """The rung of a task that files or scores todo.md items: the one its L picks, never below the project's rung.""" - return highest(project, worker_rung(project, likelihood)) + def review_floor(self, project: Rung, escape_and_impact: int) -> Rung: + """The rung a task's E + I picks for a review, counted up from the project's rung. An E + I of 0, for no task, picks the project's rung.""" + return self.climbed(project, REVIEW_STEPS.get(escape_and_impact, 0)) + def stepped_up(self, rung: Rung) -> Rung: + """The rung above, or the top one; a rung off the ladder steps to the bottom one.""" + return self.climbed(rung, 1) -def review_floor(project: Rung, escape_and_impact: int) -> Rung: - """The rung a task's E + I picks for a review, counted up from the project's rung. An E + I of 0, for no task, picks the project's rung.""" - return climbed(project, REVIEW_STEPS.get(escape_and_impact, 0)) + def review_rung(self, project: Rung, worker: Rung, escape_and_impact: int, kind: ReviewKind, markdown_only: bool = False) -> Rung: + """The rung a review of this kind runs on in a chunk whose worker runs on worker and whose item has that E + I; markdown_only where the branch changes only Markdown no agent follows. + A pass's reviewer runs on reviewers_rung; no replay has qualified a lower rung for any pass, docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`. A seat and the security pass run on the project's rung, so the seats score every chunk on one rung, and the security pass spends no more than the project chose. + """ + return project if kind in ("seat", "security") else self.reviewers_rung(project, worker, escape_and_impact, markdown_only) -def stepped_up(rung: Rung) -> Rung: - """The rung above, or the top one; a rung off the ladder steps to the bottom one.""" - return climbed(rung, 1) + def reviewers_rung(self, project: Rung, worker: Rung, escape_and_impact: int, markdown_only: bool = False) -> Rung: + """The rung of each pass's reviewer: the higher of the worker's rung and review_floor's, which is never below the project's. + markdown_only means the branch changes only Markdown no agent follows. Such a branch holds no code or config for a higher rung to catch a defect in, so its reviewer runs on the project's rung: skills/larv-review/SKILL.md → "Markdown only". + """ + return project if markdown_only else self.highest(worker, self.review_floor(project, escape_and_impact)) -def review_rung(project: Rung, worker: Rung, escape_and_impact: int, kind: ReviewKind, markdown_only: bool = False) -> Rung: - """The rung a review of this kind runs on in a chunk whose worker runs on worker and whose item has that E + I; markdown_only where the branch changes only Markdown no agent follows. + def highest(self, *rungs: Rung) -> Rung: + """The most competent of the rungs; one off the ladder ranks below every rung on it.""" + return max(rungs, key=self.position) - A pass's reviewer runs on reviewers_rung; no replay has qualified a lower rung for any pass, docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`. A seat and the security pass run on the project's rung, so the seats score every chunk on one rung, and the security pass spends no more than the project chose. - """ - return project if kind in ("seat", "security") else reviewers_rung(project, worker, escape_and_impact, markdown_only) + def climbed(self, rung: Rung, by: int) -> Rung: + return self.rungs[max(0, min(len(self.rungs) - 1, self.position(rung) + by))] - -def reviewers_rung(project: Rung, worker: Rung, escape_and_impact: int, markdown_only: bool = False) -> Rung: - """The rung of each pass's reviewer: the higher of the worker's rung and review_floor's, which is never below the project's. - - markdown_only means the branch changes only Markdown no agent follows. Such a branch holds no code or config for a higher rung to catch a defect in, so its reviewer runs on the project's rung: skills/larv-review/SKILL.md → "Markdown only". - """ - return project if markdown_only else highest(worker, review_floor(project, escape_and_impact)) - - -def highest(*rungs: Rung) -> Rung: - """The most competent of the rungs; one off the ladder ranks below every rung on it.""" - return max(rungs, key=position) - - -def climbed(rung: Rung, by: int) -> Rung: - return LADDER[max(0, min(len(LADDER) - 1, position(rung) + by))] - - -def position(rung: Rung) -> int: - return LADDER.index(rung) if rung in LADDER else -1 + def position(self, rung: Rung) -> int: + return self.rungs.index(rung) if rung in self.rungs else -1 diff --git a/bin/model_routing/ladder_file.py b/bin/model_routing/ladder_file.py new file mode 100644 index 00000000..9ebf0226 --- /dev/null +++ b/bin/model_routing/ladder_file.py @@ -0,0 +1,57 @@ +"""Read a routing ladder from its JSON file: the one this setup ships, or the one a project keeps in its config's folder in its place. docs/decisions.md → `routing: one ladder of (model, effort) rungs`.""" +import json +import pathlib +import re + +from model_routing.ladder import EFFORTS, Ladder, Rung +from failure import Failure +from project_config.main import located +from project_config.settings import Unnamed + +# The file's name in both places. Three readers, seated as the human who writes an override in a hurry, picked a bare list of objects over a list of strings, a list of pairs and a list under a key. Two of them picked one name for both files: an override is then a copy of the shipped file with rungs cut. +NAME = "routing-ladder.json" +# The ladder this setup ships, at the root of the checkout the running tool is in. +SHIPPED = pathlib.Path(__file__).resolve().parents[2] / NAME +# A model as Rung.family reads it: the Agent tool's family name follows the first hyphen. +MODEL = re.compile(r"[^\s-]+(-[^\s-]+)+") +SHAPE = f'a JSON list of rungs, least competent first, each as {{"model": "claude-opus-5-5", "effort": "medium"}}, with an effort of {", ".join(EFFORTS[:-1])} or {EFFORTS[-1]}' + + +def project_ladder(directory: str | None, home: pathlib.Path, shipped: pathlib.Path = SHIPPED) -> Ladder | Failure: + """The ladder of the project whose checkout holds directory, else the current directory: the file NAME in its config's folder under home, or the shipped one where that folder holds none. Where no config names a checkout there, the shipped one. A Failure where the project's config or the ladder's file is faulty or cannot be read, since a tool on another ladder than the project's would route its work wrong.""" + config = located(directory, home) + if isinstance(config, Unnamed): + return read_ladder(shipped) + if isinstance(config, Failure): + return config + own = config.folder / NAME + return read_ladder(own if own.exists() else shipped) + + +def read_ladder(file: pathlib.Path) -> Ladder | Failure: + """The ladder a file holds; a Failure naming the file and its fault where it holds none.""" + try: + held = json.loads(file.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, ValueError) as failure: + return Failure(f"cannot read the routing ladder {file}: {failure}. It holds {SHAPE}") + faults = faults_of(held) + if faults: + return Failure(f"{file}: {'; '.join(faults)}. The routing ladder is {SHAPE}") + return Ladder(tuple(Rung(rung["model"], rung["effort"]) for rung in held)) + + +def faults_of(held: object) -> list[str]: + """What keeps held from being a ladder: it is no list or an empty one, or else each rung that is misspelt or repeats one above it, by its number from 1.""" + if not isinstance(held, list) or not held: + return ["it holds no list of rungs"] + faults = [] + for number, rung in enumerate(held, 1): + if not well_formed(rung): + faults.append(f"rung {number} is {json.dumps(rung)}") + elif rung in held[:number - 1]: + faults.append(f"rung {number} repeats rung {held.index(rung) + 1}") + return faults + + +def well_formed(rung: object) -> bool: + return isinstance(rung, dict) and rung.keys() == {"model", "effort"} and isinstance(rung["model"], str) and bool(MODEL.fullmatch(rung["model"])) and rung["effort"] in EFFORTS diff --git a/bin/model_routing/review_choice.py b/bin/model_routing/review_choice.py index 9af82575..d20474b9 100644 --- a/bin/model_routing/review_choice.py +++ b/bin/model_routing/review_choice.py @@ -2,7 +2,7 @@ import dataclasses from model_routing.route import HIGHEST_ESCAPE_AND_IMPACT, Route -from model_routing.ladder import review_rung, reviewers_rung, Rung +from model_routing.ladder import Ladder, Rung @dataclasses.dataclass(frozen=True) @@ -17,12 +17,12 @@ class ReviewChoice: higher: dict[Rung, int] -def chosen(project: Rung, route: Route) -> ReviewChoice: - """The rung each kind of review runs on for a chunk whose route is route, on a project whose rung is project.""" - reviewers = reviewers_rung(project, route.rung, route.reviewed_on()) +def chosen(ladder: Ladder, project: Rung, route: Route) -> ReviewChoice: + """The rung each kind of review runs on for a chunk whose route is route, on a project whose rung on ladder is project.""" + reviewers = ladder.reviewers_rung(project, route.rung, route.reviewed_on()) higher: dict[Rung, int] = {} for escape_and_impact in range(route.reviewed_on() + 1, HIGHEST_ESCAPE_AND_IMPACT + 1): - if (rung := reviewers_rung(project, route.rung, escape_and_impact)) != reviewers: + if (rung := ladder.reviewers_rung(project, route.rung, escape_and_impact)) != reviewers: higher.setdefault(rung, escape_and_impact) - markdown_only = reviewers_rung(project, route.rung, route.reviewed_on(), markdown_only=True) - return ReviewChoice(reviewers, markdown_only, review_rung(project, route.rung, route.reviewed_on(), "seat"), review_rung(project, route.rung, route.reviewed_on(), "security"), higher) + markdown_only = ladder.reviewers_rung(project, route.rung, route.reviewed_on(), markdown_only=True) + return ReviewChoice(reviewers, markdown_only, ladder.review_rung(project, route.rung, route.reviewed_on(), "seat"), ladder.review_rung(project, route.rung, route.reviewed_on(), "security"), higher) diff --git a/bin/model_routing/test_ladder.py b/bin/model_routing/test_ladder.py index 45eaee89..6e3adb4f 100644 --- a/bin/model_routing/test_ladder.py +++ b/bin/model_routing/test_ladder.py @@ -4,58 +4,58 @@ import unittest sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) -from model_routing.ladder import highest, LADDER, lowest_worker_rung, planning_rung, project_rung, review_floor, review_rung, Rung, stepped_up, worker_rung, WORKER_STEPS +from model_routing.ladder import Ladder, Rung, WORKER_STEPS from release_plan.todo_table import LEVELS +from model_routing.ladder_file import read_ladder, SHIPPED from review_rules.review_passes import REVIEWERS +# The ladder this setup ships, whose rungs the tests below name. +LADDER = read_ladder(SHIPPED) OPUS = Rung("claude-opus-5-5", "medium") FABLE_LOW = Rung("claude-fable-5-2", "low") class Ladder(unittest.TestCase): - def test_holds_each_rung_once_since_a_rung_s_position_sets_every_step(self): - self.assertEqual(len(set(LADDER)), len(LADDER)) - def test_puts_a_project_on_its_models_cheapest_rung_and_none_off_the_ladder(self): - self.assertEqual(project_rung("claude-opus-5-5"), OPUS) - self.assertEqual(project_rung("claude-sonnet-5-5"), Rung("claude-sonnet-5-5", "high")) - self.assertEqual(project_rung("claude-fable-5-2"), FABLE_LOW) - self.assertIsNone(project_rung("claude-opus-5")) - self.assertIsNone(project_rung("opus")) + self.assertEqual(LADDER.project_rung("claude-opus-5-5"), OPUS) + self.assertEqual(LADDER.project_rung("claude-sonnet-5-5"), Rung("claude-sonnet-5-5", "high")) + self.assertEqual(LADDER.project_rung("claude-fable-5-2"), FABLE_LOW) + self.assertIsNone(LADDER.project_rung("claude-opus-5")) + self.assertIsNone(LADDER.project_rung("opus")) def test_steps_a_workers_rung_by_its_l_from_the_projects_rung(self): - position = LADDER.index(OPUS) - self.assertEqual([LADDER.index(worker_rung(OPUS, likelihood)) - position for likelihood in range(1, 5)], [-1, -1, 0, 1]) + position = LADDER.rungs.index(OPUS) + self.assertEqual([LADDER.rungs.index(LADDER.worker_rung(OPUS, likelihood)) - position for likelihood in range(1, 5)], [-1, -1, 0, 1]) def test_steps_a_review_up_by_its_e_plus_i_from_the_projects_rung_and_never_down(self): - position = LADDER.index(OPUS) - self.assertEqual([LADDER.index(review_floor(OPUS, total)) - position for total in range(2, 9)], [0, 0, 0, 1, 2, 3, 4]) - self.assertEqual(review_floor(OPUS, 0), OPUS) + position = LADDER.rungs.index(OPUS) + self.assertEqual([LADDER.rungs.index(LADDER.review_floor(OPUS, total)) - position for total in range(2, 9)], [0, 0, 0, 1, 2, 3, 4]) + self.assertEqual(LADDER.review_floor(OPUS, 0), OPUS) def test_a_step_past_either_end_stops_there(self): - self.assertEqual(worker_rung(OPUS, 1), Rung("claude-sonnet-5-5", "high")) - self.assertEqual(worker_rung(LADDER[0], 1), LADDER[0]) - self.assertEqual(review_floor(Rung("claude-opus-5-5", "xhigh"), 8), LADDER[-1]) - self.assertEqual(stepped_up(LADDER[-1]), LADDER[-1]) - self.assertEqual(stepped_up(OPUS), FABLE_LOW) + self.assertEqual(LADDER.worker_rung(OPUS, 1), Rung("claude-sonnet-5-5", "high")) + self.assertEqual(LADDER.worker_rung(LADDER.rungs[0], 1), LADDER.rungs[0]) + self.assertEqual(LADDER.review_floor(Rung("claude-opus-5-5", "xhigh"), 8), LADDER.rungs[-1]) + self.assertEqual(LADDER.stepped_up(LADDER.rungs[-1]), LADDER.rungs[-1]) + self.assertEqual(LADDER.stepped_up(OPUS), FABLE_LOW) def test_steps_a_worker_for_every_level_on_ls_scale(self): self.assertEqual(set(WORKER_STEPS), set(range(1, LEVELS["L"] + 1))) def test_names_the_lowest_rung_any_l_routes_a_worker_to(self): - self.assertEqual(lowest_worker_rung(OPUS), Rung("claude-sonnet-5-5", "high")) + self.assertEqual(LADDER.lowest_worker_rung(OPUS), Rung("claude-sonnet-5-5", "high")) def test_plans_on_the_rung_l_picks_but_never_below_the_projects(self): - self.assertEqual(planning_rung(OPUS, 1), OPUS) - self.assertEqual(planning_rung(OPUS, 4), FABLE_LOW) + self.assertEqual(LADDER.planning_rung(OPUS, 1), OPUS) + self.assertEqual(LADDER.planning_rung(OPUS, 4), FABLE_LOW) def test_names_each_rung_by_a_family_the_agent_tool_takes(self): - self.assertEqual({rung.agent_args().split(",")[0] for rung in LADDER}, {"model sonnet", "model opus", "model fable"}) + self.assertEqual({rung.agent_args().split(",")[0] for rung in LADDER.rungs}, {"model sonnet", "model opus", "model fable"}) def test_ranks_a_rung_off_the_ladder_below_every_rung_on_it(self): - self.assertEqual(highest(OPUS, LADDER[-1], LADDER[0]), LADDER[-1]) - self.assertEqual(highest(Rung("claude-opus-5-5", "low"), LADDER[0]), LADDER[0]) - self.assertEqual(stepped_up(Rung("claude-opus-5-5", "low")), LADDER[0]) + self.assertEqual(LADDER.highest(OPUS, LADDER.rungs[-1], LADDER.rungs[0]), LADDER.rungs[-1]) + self.assertEqual(LADDER.highest(Rung("claude-opus-5-5", "low"), LADDER.rungs[0]), LADDER.rungs[0]) + self.assertEqual(LADDER.stepped_up(Rung("claude-opus-5-5", "low")), LADDER.rungs[0]) def test_passes_the_effort_only_where_the_rung_names_one(self): self.assertEqual(Rung("claude-opus-5-5", "high").flags(), ["--model", "claude-opus-5-5", "--effort", "high"]) @@ -64,7 +64,7 @@ class Ladder(unittest.TestCase): self.assertEqual(str(Rung("worker-model")), "worker-model") def test_parses_what_str_wrote_for_each_rung_on_the_ladder(self): - for rung in LADDER: + for rung in LADDER.rungs: with self.subTest(rung=rung): self.assertEqual(Rung.parse(str(rung)), rung) @@ -77,31 +77,31 @@ class Ladder(unittest.TestCase): self.assertIsNone(Rung.parse(text)) def test_reads_the_agent_tools_words_back_as_a_rung(self): - for rung in LADDER: + for rung in LADDER.rungs: with self.subTest(rung=rung): model, effort = (part.split(" ")[1] for part in rung.agent_args().split(", ")) - self.assertEqual(Rung.from_agent(model, effort), rung) + self.assertEqual(LADDER.from_agent(model, effort), rung) def test_reads_a_family_named_without_an_effort_as_its_cheapest_rung(self): - self.assertEqual(Rung.from_agent("opus", ""), OPUS) - self.assertEqual(Rung.from_agent("sonnet", ""), Rung("claude-sonnet-5-5", "high")) + self.assertEqual(LADDER.from_agent("opus", ""), OPUS) + self.assertEqual(LADDER.from_agent("sonnet", ""), Rung("claude-sonnet-5-5", "high")) def test_reads_an_unknown_family_or_an_effort_off_the_ladder_as_a_rung_off_it(self): for model, effort in (("gpt", "high"), ("gpt", ""), ("opus", "low"), ("opus", "high"), ("haiku", "bogus")): with self.subTest(model=model, effort=effort): - self.assertNotIn(Rung.from_agent(model, effort), LADDER) - self.assertEqual(Rung.from_agent("opus", "low"), Rung("claude-opus-5-5", "low")) + self.assertNotIn(LADDER.from_agent(model, effort), LADDER.rungs) + self.assertEqual(LADDER.from_agent("opus", "low"), Rung("claude-opus-5-5", "low")) def test_puts_reviewers_on_the_highest_of_the_projects_rung_the_workers_and_the_one_e_plus_i_picks(self): - sonnet, fable = LADDER[0], LADDER[-1] - self.assertEqual(review_rung(OPUS, sonnet, 4, "stability"), OPUS) - self.assertEqual(review_rung(OPUS, sonnet, 5, "stability"), FABLE_LOW) - self.assertEqual(review_rung(OPUS, fable, 5, "stability"), fable) - self.assertEqual(review_rung(OPUS, FABLE_LOW, 7, "stability"), Rung("claude-opus-5-5", "xhigh")) + sonnet, fable = LADDER.rungs[0], LADDER.rungs[-1] + self.assertEqual(LADDER.review_rung(OPUS, sonnet, 4, "stability"), OPUS) + self.assertEqual(LADDER.review_rung(OPUS, sonnet, 5, "stability"), FABLE_LOW) + self.assertEqual(LADDER.review_rung(OPUS, fable, 5, "stability"), fable) + self.assertEqual(LADDER.review_rung(OPUS, FABLE_LOW, 7, "stability"), Rung("claude-opus-5-5", "xhigh")) def test_keeps_the_seats_and_the_security_pass_on_the_projects_rung_whatever_the_worker_or_e_plus_i(self): - self.assertEqual([review_rung(OPUS, LADDER[-1], 8, kind) for kind in (*REVIEWERS, "seat", "security")], [LADDER[-1]] * len(REVIEWERS) + [OPUS, OPUS]) + self.assertEqual([LADDER.review_rung(OPUS, LADDER.rungs[-1], 8, kind) for kind in (*REVIEWERS, "seat", "security")], [LADDER.rungs[-1]] * len(REVIEWERS) + [OPUS, OPUS]) def test_puts_every_review_of_a_branch_changing_only_markdown_no_agent_follows_on_the_projects_rung(self): - self.assertEqual([review_rung(OPUS, LADDER[-1], 8, kind, markdown_only=True) for kind in (*REVIEWERS, "seat", "security")], [OPUS] * (len(REVIEWERS) + 2)) - self.assertEqual(review_rung(OPUS, LADDER[0], 4, "stability", markdown_only=True), OPUS) + self.assertEqual([LADDER.review_rung(OPUS, LADDER.rungs[-1], 8, kind, markdown_only=True) for kind in (*REVIEWERS, "seat", "security")], [OPUS] * (len(REVIEWERS) + 2)) + self.assertEqual(LADDER.review_rung(OPUS, LADDER.rungs[0], 4, "stability", markdown_only=True), OPUS) diff --git a/bin/model_routing/test_ladder_file.py b/bin/model_routing/test_ladder_file.py new file mode 100644 index 00000000..1fa141b2 --- /dev/null +++ b/bin/model_routing/test_ladder_file.py @@ -0,0 +1,130 @@ +import pathlib +import subprocess +import sys +import tempfile +import unittest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) + +from failure import Failure +from test_env import isolated_env +from model_routing.ladder import Ladder, Rung +from model_routing.ladder_file import NAME, project_ladder, read_ladder, SHIPPED +from project_config.test_project_config import write_config + +FABLE_ONLY = '[\n\t{"model": "claude-fable-5-2", "effort": "low"},\n\t{"model": "claude-fable-5-2", "effort": "max"}\n]\n' +FABLE_LADDER = Ladder((Rung("claude-fable-5-2", "low"), Rung("claude-fable-5-2", "max"))) +OPUS = '{"model": "claude-opus-5-5", "effort": "medium"}' + + +class LadderCase(unittest.TestCase): + def setUp(self) -> None: + made = tempfile.TemporaryDirectory() + self.addCleanup(made.cleanup) + self.home = pathlib.Path(made.name).resolve() + self.shipped = self.home / "shipped.json" + self.shipped.write_text(f"[{OPUS}]", encoding="utf-8") + + def checkout(self, name: str = "acme") -> pathlib.Path: + """A git checkout under the home, holding one commit, that no config names yet.""" + checkout = self.home / "code" / name + checkout.mkdir(parents=True) + self.git(checkout, "init", "--quiet", "-b", "main") + self.git(checkout, "commit", "--quiet", "--allow-empty", "-m", "base") + return checkout + + def git(self, cwd: pathlib.Path, *args: str) -> None: + subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@example.com", *args], capture_output=True, check=True, cwd=cwd, env=isolated_env()) + + +class ReadLadder(LadderCase): + def reason(self, text: str) -> str: + file = self.home / NAME + file.write_text(text, encoding="utf-8") + read = read_ladder(file) + assert isinstance(read, Failure), read + return read.reason + + def test_the_ladder_this_setup_ships_reads_as_a_ladder(self): + self.assertIsInstance(read_ladder(SHIPPED), Ladder) + + def test_reads_the_rungs_in_the_files_order(self): + file = self.home / NAME + file.write_text(FABLE_ONLY, encoding="utf-8") + self.assertEqual(read_ladder(file), FABLE_LADDER) + + def test_a_file_that_is_missing_or_no_json_fails_naming_the_file_and_the_shape(self): + file = self.home / NAME + missing = read_ladder(file) + self.assertIsInstance(missing, Failure) + self.assertIn(f"cannot read the routing ladder {file}: ", missing.reason) + # The comma a reader leaves after the last rung when dropping one. + trailing = self.reason(f"[\n\t{OPUS},\n]\n") + self.assertIn(f"cannot read the routing ladder {file}: ", trailing) + self.assertIn("line 2", trailing) + self.assertIn('a JSON list of rungs, least competent first, each as {"model": "claude-opus-5-5", "effort": "medium"}, with an effort of low, medium, high, xhigh or max', trailing) + file.write_bytes(b"\xff") + self.assertIn(f"cannot read the routing ladder {file}: ", read_ladder(file).reason) + + def test_a_file_holding_anything_but_a_list_of_rungs_fails(self): + for text in (OPUS, "[]", '"claude-opus-5-5 at medium effort"', "null"): + with self.subTest(text=text): + self.assertEqual(self.reason(text).split(". ")[0], f"{self.home / NAME}: it holds no list of rungs") + + def test_names_each_misspelt_rung_by_its_number(self): + misspelt = { + "an effort off the scale": '{"model": "claude-opus-5-5", "effort": "ultra"}', + "no effort": '{"model": "claude-opus-5-5"}', + "an empty effort": '{"model": "claude-opus-5-5", "effort": ""}', + "a key too many": '{"model": "claude-opus-5-5", "effort": "low", "note": "x"}', + "a model that names no family": '{"model": "opus", "effort": "low"}', + "a model with a space": '{"model": "claude-opus 5-5", "effort": "low"}', + "a model that is no text": '{"model": 5, "effort": "low"}', + "a pair": '["claude-opus-5-5", "low"]', + "the rung as a tool prints it": '"claude-opus-5-5 at low effort"', + } + for what, rung in misspelt.items(): + with self.subTest(what): + self.assertEqual(self.reason(f"[{OPUS}, {rung}]").split(". ")[0], f"{self.home / NAME}: rung 2 is {rung}") + self.assertIn("rung 1 is 1; rung 3 is 3. The routing ladder is a JSON list of rungs, least competent first", self.reason(f"[1, {OPUS}, 3]")) + + def test_a_rung_held_twice_fails_naming_both_since_a_rungs_position_sets_every_step(self): + low = '{"model": "claude-opus-5-5", "effort": "low"}' + self.assertEqual(self.reason(f"[{low}, {OPUS}, {low}]").split(". ")[0], f"{self.home / NAME}: rung 3 repeats rung 1") + + +class ProjectLadder(LadderCase): + def test_a_project_runs_on_the_ladder_its_configs_folder_holds_from_a_worktree_too(self): + checkout = self.checkout() + (write_config(self.home, checkout) / NAME).write_text(FABLE_ONLY, encoding="utf-8") + self.assertEqual(project_ladder(str(checkout), self.home, self.shipped), FABLE_LADDER) + worktree = self.home / "code" / "acme-item-7" + self.git(checkout, "worktree", "add", "--quiet", str(worktree), "-b", "item-7") + self.assertEqual(project_ladder(str(worktree), self.home, self.shipped), FABLE_LADDER) + + def test_another_projects_ladder_is_not_read(self): + checkout, other = self.checkout(), self.checkout("other") + write_config(self.home, checkout) + (write_config(self.home, other) / NAME).write_text(FABLE_ONLY, encoding="utf-8") + self.assertEqual(project_ladder(str(checkout), self.home, self.shipped), Ladder((Rung("claude-opus-5-5", "medium"),))) + + def test_a_checkout_no_config_names_and_a_folder_outside_any_checkout_run_on_the_shipped_ladder(self): + shipped = Ladder((Rung("claude-opus-5-5", "medium"),)) + self.assertEqual(project_ladder(str(self.checkout()), self.home, self.shipped), shipped) + self.assertEqual(project_ladder(str(self.home), self.home, self.shipped), shipped) + + def test_a_faulty_ladder_of_the_projects_own_fails_and_the_shipped_one_does_not_stand_in(self): + checkout = self.checkout() + own = write_config(self.home, checkout) / NAME + own.write_text("[]", encoding="utf-8") + self.assertEqual(project_ladder(str(checkout), self.home, self.shipped).reason.split(". ")[0], f"{own}: it holds no list of rungs") + + def test_a_faulty_config_fails_since_its_folder_may_hold_the_ladder(self): + checkout = self.checkout() + config = write_config(self.home, checkout) / "config.toml" + config.write_text(config.read_text(encoding="utf-8") + 'readme = "checkout:README.md"\n', encoding="utf-8") + self.assertIn("`readme` is not a setting", project_ladder(str(checkout), self.home, self.shipped).reason) + + +if __name__ == "__main__": + unittest.main() diff --git a/bin/model_routing/test_prices.py b/bin/model_routing/test_prices.py index bd627c5a..39c3e318 100644 --- a/bin/model_routing/test_prices.py +++ b/bin/model_routing/test_prices.py @@ -4,13 +4,13 @@ import unittest sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) -from model_routing.ladder import LADDER from model_routing.prices import PRICES +from model_routing.ladder_file import read_ladder, SHIPPED class Prices(unittest.TestCase): def test_every_model_on_the_ladder_has_a_price_since_a_model_without_one_is_left_out_of_every_figure(self): - self.assertEqual(sorted({rung.model for rung in LADDER} - set(PRICES)), []) + self.assertEqual(sorted({rung.model for rung in read_ladder(SHIPPED).rungs} - set(PRICES)), []) if __name__ == "__main__": diff --git a/bin/model_routing/test_review_choice.py b/bin/model_routing/test_review_choice.py index 48da394f..e99060bf 100644 --- a/bin/model_routing/test_review_choice.py +++ b/bin/model_routing/test_review_choice.py @@ -1,17 +1,19 @@ import unittest from model_routing.review_choice import chosen +from model_routing.ladder_file import read_ladder, SHIPPED from model_routing.route import Route from model_routing.ladder import Rung +LADDER = read_ladder(SHIPPED) OPUS = Rung("claude-opus-5-5", "medium") FABLE = Rung("claude-fable-5-2", "max") class Chosen(unittest.TestCase): def test_names_no_higher_rung_on_the_top_one(self): - self.assertEqual(chosen(OPUS, Route(FABLE, 3, 4, 5)).higher, {}) + self.assertEqual(chosen(LADDER, OPUS, Route(FABLE, 3, 4, 5)).higher, {}) def test_reviews_a_branch_changing_only_markdown_on_the_projects_rung_and_the_seats_and_security_pass_there_always(self): - choice = chosen(OPUS, Route(FABLE, 3, 4, 5)) + choice = chosen(LADDER, OPUS, Route(FABLE, 3, 4, 5)) self.assertEqual((choice.reviewers, choice.markdown_only, choice.seat, choice.security), (FABLE, OPUS, OPUS, OPUS)) diff --git a/bin/model_routing/test_worker_rungs.py b/bin/model_routing/test_worker_rungs.py index 47bf208a..f0074149 100644 --- a/bin/model_routing/test_worker_rungs.py +++ b/bin/model_routing/test_worker_rungs.py @@ -1,8 +1,12 @@ import unittest -from model_routing.worker_rungs import environment, handed +from model_routing.worker_rungs import environment, handed, hands_rungs +from model_routing.ladder import Ladder, Rung +from model_routing.ladder_file import read_ladder, SHIPPED from model_routing.route import Route -from model_routing.ladder import Rung + + +LADDER = read_ladder(SHIPPED) class Environment(unittest.TestCase): @@ -18,13 +22,23 @@ class Environment(unittest.TestCase): self.assertEqual(environment(None, Route(self.WORKER)), {"LOOP_CLAUDE_PROJECT_RUNG": "", "LOOP_CLAUDE_WORKER_RUNG": "claude-sonnet-5-5 at high effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "0"}) def test_handed_reads_back_what_environment_named(self): - self.assertEqual(handed(environment(self.PROJECT, Route(self.WORKER, 3, 1, 5))), (self.PROJECT, self.WORKER, 5)) - self.assertEqual(handed(environment(None, Route(self.WORKER))), (None, self.WORKER, 0)) + self.assertEqual(handed(environment(self.PROJECT, Route(self.WORKER, 3, 1, 5)), LADDER), (self.PROJECT, self.WORKER, 5)) + self.assertEqual(handed(environment(None, Route(self.WORKER)), LADDER), (None, self.WORKER, 0)) + + def test_a_rung_off_the_ladder_given_reads_as_none(self): + fable_only = Ladder((Rung("claude-fable-5-2", "low"),)) + self.assertEqual(handed(environment(self.PROJECT, Route(Rung("claude-fable-5-2", "low"))), fable_only), (None, Rung("claude-fable-5-2", "low"), 0)) + + def test_tells_whether_both_rungs_are_named_before_any_ladder_is_read(self): + self.assertTrue(hands_rungs(environment(self.PROJECT, Route(self.WORKER)))) + self.assertFalse(hands_rungs(environment(None, Route(self.WORKER)))) + self.assertFalse(hands_rungs({"LOOP_CLAUDE_PROJECT_RUNG": "claude-opus-5-5 at medium effort"})) + self.assertFalse(hands_rungs({})) def test_a_missing_or_unknown_rung_reads_as_none_and_a_missing_or_unknown_e_plus_i_as_0(self): - self.assertEqual(handed({}), (None, None, 0)) - self.assertEqual(handed({"LOOP_CLAUDE_WORKER_RUNG": "claude-haiku-5-5 bogus"}), (None, None, 0)) - self.assertEqual(handed({"LOOP_CLAUDE_WORKER_RUNG": "claude-haiku-5-5 at bogus effort"}), (None, None, 0)) + self.assertEqual(handed({}, LADDER), (None, None, 0)) + self.assertEqual(handed({"LOOP_CLAUDE_WORKER_RUNG": "claude-haiku-5-5 bogus"}, LADDER), (None, None, 0)) + self.assertEqual(handed({"LOOP_CLAUDE_WORKER_RUNG": "claude-haiku-5-5 at bogus effort"}, LADDER), (None, None, 0)) for bogus in ("", "five", "-1", "5.0", "²"): with self.subTest(bogus=bogus): - self.assertEqual(handed({"LOOP_CLAUDE_ESCAPE_AND_IMPACT": bogus})[2], 0) + self.assertEqual(handed({"LOOP_CLAUDE_ESCAPE_AND_IMPACT": bogus}, LADDER)[2], 0) diff --git a/bin/model_routing/worker_rungs.py b/bin/model_routing/worker_rungs.py index 92c0c362..a658068e 100644 --- a/bin/model_routing/worker_rungs.py +++ b/bin/model_routing/worker_rungs.py @@ -3,7 +3,7 @@ loop-claude sets these variables for its worker and for the claude an interrupt resumes; a human's shell does not, and rule-hook then checks no review launch.""" import typing -from model_routing.ladder import LADDER, Rung +from model_routing.ladder import Ladder, Rung from model_routing.route import Route PROJECT_RUNG = "LOOP_CLAUDE_PROJECT_RUNG" @@ -17,12 +17,17 @@ def environment(project: Rung | None, route: Route) -> dict[str, str]: return {PROJECT_RUNG: str(project) if project else "", WORKER_RUNG: str(route.rung), ESCAPE_AND_IMPACT: str(route.reviewed_on())} -def handed(environ: typing.Mapping[str, str]) -> tuple[Rung | None, Rung | None, int]: - """The project's and the worker's rungs, each None when unset or not on the ladder, and the reviews' E + I, 0 when unset or not a whole number. An older loop-claude hands no E + I, and 0 steps no review up.""" +def hands_rungs(environ: typing.Mapping[str, str]) -> bool: + """Whether the environment names a project's and a worker's rung, as loop-claude's does for a project on its ladder. A reader asks before it reads the project's ladder, which handed needs.""" + return bool(environ.get(PROJECT_RUNG) and environ.get(WORKER_RUNG)) + + +def handed(environ: typing.Mapping[str, str], ladder: Ladder) -> tuple[Rung | None, Rung | None, int]: + """The project's and the worker's rungs, each None when unset or not on ladder, the project's ladder, and the reviews' E + I, 0 when unset or not a whole number. An older loop-claude hands no E + I, and 0 steps no review up.""" escape_and_impact = environ.get(ESCAPE_AND_IMPACT, "") - return on_the_ladder(environ.get(PROJECT_RUNG)), on_the_ladder(environ.get(WORKER_RUNG)), int(escape_and_impact) if escape_and_impact.isdecimal() else 0 + return on_the_ladder(environ.get(PROJECT_RUNG), ladder), on_the_ladder(environ.get(WORKER_RUNG), ladder), int(escape_and_impact) if escape_and_impact.isdecimal() else 0 -def on_the_ladder(value: str | None) -> Rung | None: +def on_the_ladder(value: str | None, ladder: Ladder) -> Rung | None: rung = Rung.parse(value or "") - return rung if rung in LADDER else None + return rung if rung in ladder.rungs else None diff --git a/bin/project_config/main.py b/bin/project_config/main.py index a3fe914c..0b932651 100644 --- a/bin/project_config/main.py +++ b/bin/project_config/main.py @@ -29,7 +29,7 @@ import sys from failure import Failure from git_cli.git_repo import git, Git -from project_config.settings import load, ProjectConfig, SETTINGS +from project_config.settings import load, ProjectConfig, SETTINGS, Unnamed def main(argv: list[str]) -> int: @@ -51,7 +51,7 @@ def located(directory: str | None, home: pathlib.Path, run: Git = git) -> Projec """The config under home of the checkout that holds directory, else the current directory.""" found = run("rev-parse", "--path-format=absolute", "--show-toplevel", "--git-common-dir", cwd=directory) if isinstance(found, Failure): - return found + return Unnamed(found.reason) if len(lines := found.splitlines()) != 2: return Failure(f"git rev-parse printed {found!r}, not a checkout's top folder and git directory on two lines") # The common dir is the main checkout's .git, whichever worktree runs this. diff --git a/bin/project_config/settings.py b/bin/project_config/settings.py index 157e8e36..5beca36a 100644 --- a/bin/project_config/settings.py +++ b/bin/project_config/settings.py @@ -18,6 +18,11 @@ SPELLING = '"checkout:" for a path in the checkout, "folder:" for on Place = str | pathlib.Path +@dataclasses.dataclass(frozen=True) +class Unnamed(Failure): + """No config names a checkout here: none names the one asked for, or git finds no checkout. A reader of the routing ladder takes the shipped one there, as in a third party's clone; a config that is faulty or cannot be read is a plain Failure.""" + + @dataclasses.dataclass(frozen=True) class ProjectConfig: """folder is `~/.claude-projects/`, top is the worktree that asked, and paths holds each of SETTINGS as an absolute path.""" @@ -49,7 +54,7 @@ def load(top: pathlib.Path, main: pathlib.Path, home: pathlib.Path) -> ProjectCo if config[0].resolve() == main.resolve(): named.append((file, config[1])) if not named: - return Failure(f"no config names the checkout {main}: write ~/{PROJECTS}//{CONFIG}, as `project-config --help` shows") + return Unnamed(f"no config names the checkout {main}: write ~/{PROJECTS}//{CONFIG}, as `project-config --help` shows") if len(named) > 1: return Failure(f"{' and '.join(str(file) for file, _ in named)} each name the checkout {main}; only one config may") file, values = named[0] diff --git a/bin/project_config/test_project_config.py b/bin/project_config/test_project_config.py index a4ec0045..57a5cf2d 100644 --- a/bin/project_config/test_project_config.py +++ b/bin/project_config/test_project_config.py @@ -9,8 +9,9 @@ import unittest.mock sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) from failure import Failure -from project_config.settings import held, load, ProjectConfig, SETTINGS +from project_config.settings import held, load, ProjectConfig, SETTINGS, Unnamed from test_env import isolated_env +from project_config.main import located TOOL = pathlib.Path(__file__).resolve().parents[1] / "project-config" @@ -119,6 +120,13 @@ class LoadTest(ConfigCase): self.assertIn(str(self.checkout), reason) self.assertIn("~/.claude-projects//config.toml", reason) + def test_tells_a_checkout_no_config_names_from_one_whose_config_is_faulty(self) -> None: + self.assertIsInstance(load(self.checkout, self.checkout, self.home), Unnamed) + self.assertIsInstance(located(str(self.checkout), self.home), Unnamed, "a folder that is no checkout has no config either") + self.write("acme", CLIENT.replace('plan = "folder:todo.md"\n', "")) + self.assertNotIsInstance(load(self.checkout, self.checkout, self.home), Unnamed) + self.assertIn("`plan` is missing", self.failure()) + def test_every_missing_setting_is_named(self) -> None: kept = "".join(line + "\n" for line in CLIENT.splitlines() if not line.startswith(("plan", "changelog", "principles"))) self.write("acme", kept) diff --git a/bin/review_replay/main.py b/bin/review_replay/main.py index 92bf885c..f5386cb2 100644 --- a/bin/review_replay/main.py +++ b/bin/review_replay/main.py @@ -3,7 +3,7 @@ usage: review-replay [--omit-claude-md] is one of architecture, principles, overrides, stability, product-owner or prose. - is a project model on the routing ladder, such as claude-opus-5-5. The tool replays the + is a project model on the routing ladder this setup ships, such as claude-opus-5-5. The tool replays the latest 10 archived rounds of the pass whose report holds a 🔴 or 🟡 finding, from the projects that ~/.claude/loop-claude-models.json runs on the model. It replays each round on the code it reviewed: in a throwaway clone of the project's checkout, at the newest commit of origin/HEAD @@ -61,8 +61,9 @@ from review_replay.replay import codes_found, findings_replayed from review_replay.omit_comparison import compared from loop_effects.effects import Effects from failure import Failure +from model_routing.ladder import Ladder, Rung from claude_code.reply import Limited -from model_routing.ladder import project_rung, Rung +from model_routing.ladder_file import read_ladder, SHIPPED from review_replay.recall import recall_report, recalled from review_rules.review_passes import ReviewedPass, REVIEWERS @@ -73,7 +74,11 @@ LATEST = 10 def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, effects: Effects) -> int: - read = arguments_or_exit(TOOL, argv, __doc__, USAGE, pass_and_model) + # The shipped ladder, not a project's: the replay spans every project on the model. + ladder = read_ladder(SHIPPED) + if isinstance(ladder, Failure): + return failed(TOOL, ladder.reason) + read = arguments_or_exit(TOOL, argv, __doc__, USAGE, lambda argv: pass_and_model(ladder, argv)) if isinstance(read, int): return read reviewed, project, without = read @@ -97,17 +102,17 @@ def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, eff print(f" {found.root} {found.at} on {rung} failed: {matched.reason}", file=sys.stderr, flush=True) return matched - recall = recalled(chosen, project, replay_one, f"{reviewed} rounds") + recall = recalled(chosen, ladder, project, replay_one, f"{reviewed} rounds") if isinstance(recall, Limited): return limited_exit(TOOL, recall) print(recall_report(reviewed, project, recall)) return 0 -def pass_and_model(argv: list[str]) -> tuple[ReviewedPass, Rung, bool] | None: - """The pass, the project's rung of the model, and whether OMIT follows, as argv names them in ` [--omit-claude-md]`; None for any other argv.""" +def pass_and_model(ladder: Ladder, argv: list[str]) -> tuple[ReviewedPass, Rung, bool] | None: + """The pass, the project's rung on ladder of the model, and whether OMIT follows, as argv names them in ` [--omit-claude-md]`; None for any other argv.""" if argv[2:] not in ([], [OMIT]) or len(argv) < 2: return None reviewed = next((kind for kind in REVIEWERS if kind == argv[0]), None) - project = project_rung(argv[1]) + project = ladder.project_rung(argv[1]) return (reviewed, project, argv[2:] == [OMIT]) if reviewed and project else None diff --git a/bin/review_replay/recall.py b/bin/review_replay/recall.py index 9b75b889..cd95cc26 100644 --- a/bin/review_replay/recall.py +++ b/bin/review_replay/recall.py @@ -1,10 +1,10 @@ """The bar review-replay holds a rung to, and its report: docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`.""" import dataclasses from rung_replay.command import down_from +from model_routing.ladder import EFFORTS, Ladder, Rung from failure import Failure from fractions import Fraction from review_replay.rounds import is_red, Round -from model_routing.ladder import LADDER, Rung from claude_code.reply import Limited import math from rung_replay.search import replayed_on @@ -14,8 +14,6 @@ import typing MARGIN = Fraction(1, 10) # The fewest rounds a rung is tried on: 8 of the 10 the tool samples, so up to two may fail on the project's rung. ENOUGH_ROUNDS = 8 -# Lowest first, as Claude Code's --effort takes them. -EFFORTS = ("low", "medium", "high", "xhigh", "max") # The two replays on the project's rung. Each lists, round by round, the indexes of the findings it found. Reference = tuple[list[set[int]], list[set[int]]] @@ -39,10 +37,10 @@ class Recall: lowest: Rung | None -def candidates(project: Rung) -> list[Rung]: +def candidates(ladder: Ladder, project: Rung) -> list[Rung]: """The rungs tried below the project's, highest first: its model at each lower effort, which the ladder leaves out, then each ladder rung below it, down to the ladder's lowest. That reaches below the lowest a worker routes to, since a pass may run below the worker's rung.""" lower_efforts = [Rung(project.model, effort) for effort in reversed(EFFORTS[:EFFORTS.index(project.effort)])] - return lower_efforts + down_from(project, LADDER[0])[1:] + return lower_efforts + down_from(ladder, project, ladder.rungs[0])[1:] def enough(rounds: list[Round]) -> bool: @@ -97,7 +95,7 @@ def percent(fraction: Fraction) -> str: return f"{float(fraction):.0%}" -def recalled(chosen: list[Round], project: Rung, replay_one: typing.Callable[[Round, Rung], set[int] | Failure | Limited], what: str) -> Recall | Limited: +def recalled(chosen: list[Round], ladder: Ladder, project: Rung, replay_one: typing.Callable[[Round, Rung], set[int] | Failure | Limited], what: str) -> Recall | Limited: """Replays chosen twice on the project's rung, then each candidate, highest first, on the rounds neither of those failed on, until one doesn't qualify.""" runs: list[list[set[int] | Failure]] = [] for _ in range(2): @@ -114,7 +112,7 @@ def recalled(chosen: list[Round], project: Rung, replay_one: typing.Callable[[Ro reference[1].append(second) recalls: list[RungRecall] = [] lowest = None - for rung in candidates(project) if enough(rounds) and needed(rounds, reference) > 0 else []: + for rung in candidates(ladder, project) if enough(rounds) and needed(rounds, reference) > 0 else []: on_rung: list[set[int] | Failure] | Limited = replayed_on(rounds, rung, replay_one, what) if isinstance(on_rung, Limited): return on_rung diff --git a/bin/review_replay/test_recall.py b/bin/review_replay/test_recall.py index c7de60e9..3887986c 100644 --- a/bin/review_replay/test_recall.py +++ b/bin/review_replay/test_recall.py @@ -6,8 +6,10 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) from review_replay.recall import candidates, needed, Recall, recall_report, tried from review_replay.rounds import is_red, Round +from model_routing.ladder_file import read_ladder, SHIPPED from model_routing.ladder import Rung +LADDER = read_ladder(SHIPPED) REFERENCE = Rung("claude-opus-5-5", "medium") LOW = Rung("claude-opus-5-5", "low") SONNET = Rung("claude-sonnet-5-5", "high") @@ -27,13 +29,13 @@ TENTH = ([{0}, set()], [{0}, set()]) class Candidates(unittest.TestCase): def test_tries_the_project_models_lower_efforts_first_then_the_ladder_below_it(self): - self.assertEqual(candidates(REFERENCE), [LOW, SONNET]) + self.assertEqual(candidates(LADDER, REFERENCE), [LOW, SONNET]) def test_tries_each_ladder_rung_below_highest_first(self): - self.assertEqual(candidates(Rung("claude-fable-5-2", "low")), [REFERENCE, SONNET]) + self.assertEqual(candidates(LADDER, Rung("claude-fable-5-2", "low")), [REFERENCE, SONNET]) def test_tries_each_lower_effort_of_the_ladders_lowest_rung(self): - self.assertEqual(candidates(SONNET), [Rung("claude-sonnet-5-5", "medium"), Rung("claude-sonnet-5-5", "low")]) + self.assertEqual(candidates(LADDER, SONNET), [Rung("claude-sonnet-5-5", "medium"), Rung("claude-sonnet-5-5", "low")]) class Tried(unittest.TestCase): diff --git a/bin/routing_figures/calls.py b/bin/routing_figures/calls.py index 939d505f..769bedf4 100644 --- a/bin/routing_figures/calls.py +++ b/bin/routing_figures/calls.py @@ -6,7 +6,7 @@ import typing from routing_figures.chunk import ChunkFigures, Reports from routing_figures.dollars import DIMENSIONS, Dollars -from model_routing.ladder import position, Rung +from model_routing.ladder import Ladder, Rung # A rung or a subagent type is compared only once this many of its chunks or subagents are measured, so a single odd chunk calls for no change. MEASURED_AT_LEAST = 5 @@ -73,9 +73,9 @@ class Call: return SENTENCES[self.kind].format(*self.subjects) -def calls(figures: Figures) -> list[Call]: - """Each routing change the figures call for.""" - compared = sorted((rung for rung, chunks in figures.routed.items() if len(chunks) >= MEASURED_AT_LEAST and position(rung) >= 0), key=position) +def calls(ladder: Ladder, figures: Figures) -> list[Call]: + """Each routing change the figures call for, comparing the rungs on ladder, which is the shipped one: the figures span every project, and a project's own ladder ranks only its own chunks.""" + compared = sorted((rung for rung, chunks in figures.routed.items() if len(chunks) >= MEASURED_AT_LEAST and ladder.position(rung) >= 0), key=ladder.position) found = [] for lower, upper in zip(compared, compared[1:]): measures: tuple[tuple[Kind, typing.Callable[[ChunkFigures], int]], ...] = (("rounds", lambda chunk: chunk.rounds_past_first), ("unfinished", lambda chunk: chunk.unfinished)) diff --git a/bin/routing_figures/render.py b/bin/routing_figures/render.py index d088a62a..038bf706 100644 --- a/bin/routing_figures/render.py +++ b/bin/routing_figures/render.py @@ -6,11 +6,11 @@ import typing from routing_figures.calls import Call, Figures, mean, summed from routing_figures.chunk import ChunkFigures from routing_figures.dollars import DIMENSIONS -from model_routing.ladder import position +from model_routing.ladder import Ladder -def render(figures: Figures, found: list[Call], at: str, cycle: int, since: str) -> str: - """The figures as markdown, for the human to read. since is the latest reset's stamp, empty for none.""" +def render(ladder: Ladder, figures: Figures, found: list[Call], at: str, cycle: int, since: str) -> str: + """The figures as markdown, for the human to read. ladder orders the rungs, and since is the latest reset's stamp, empty for none.""" sections = [ "# Routing figures", f"Measured {at}, at cycle {cycle} across loop-claude's archive, from {figures.chunks} finished chunk{'' if figures.chunks == 1 else 's'}{f' that finished after the reset of {since}' if since else ''}. {basis(figures)}", @@ -19,7 +19,7 @@ def render(figures: Figures, found: list[Call], at: str, cycle: int, since: str) *[part for dimension in DIMENSIONS for part in (f"## Dollars by {dimension}", dollars_table(figures, dimension))], "## Routed chunks by the rung they started on", fixes_basis(figures), - routed_table(figures), + routed_table(ladder, figures), "## Escalations by trigger across every chunk, routed or not", f"{triggers(figures.escalated) or 'None'}.", "## Subagent reports against the tool results they read, in characters", @@ -35,8 +35,8 @@ def dollars_table(figures: Figures, dimension: str) -> str: return table((dimension.capitalize(), "Dollars"), [(label, money(dollars)) for label, dollars in sorted(figures.dollars[dimension].items(), key=lambda spent: -spent[1])]) -def routed_table(figures: Figures) -> str: - """One row per rung the routed chunks started on, from the bottom of the ladder up.""" +def routed_table(ladder: Ladder, figures: Figures) -> str: + """One row per rung the routed chunks started on, from the bottom of the ladder up, after the rungs off it.""" return table(("Rung", "Chunks", "Review rounds past each pass's first, mean", "Unfinished verdicts, mean", "Later fixes, mean", "Subagent rungs", "Escalations by trigger"), [ ( str(rung), @@ -47,7 +47,7 @@ def routed_table(figures: Figures) -> str: subagent_rungs(chunks), triggers(summed(chunks, lambda chunk: chunk.escalated)), ) - for rung, chunks in sorted(figures.routed.items(), key=lambda routed: position(routed[0])) + for rung, chunks in sorted(figures.routed.items(), key=lambda routed: ladder.position(routed[0])) ]) diff --git a/bin/routing_figures/test_routing_figures.py b/bin/routing_figures/test_routing_figures.py index 3f002460..a1989234 100644 --- a/bin/routing_figures/test_routing_figures.py +++ b/bin/routing_figures/test_routing_figures.py @@ -25,11 +25,13 @@ from failure import Failure from git_cli import git_repo from test_env import isolated_env from release_plan.todo_table import Item -from model_routing.ladder import LADDER, Rung from transcript_archive import outcomes, write +from model_routing.ladder_file import read_ladder, SHIPPED from routing_figures.render import render from model_routing.route import Route +from model_routing.ladder import Rung +LADDER = read_ladder(SHIPPED) LOW = Rung("claude-opus-5-5", "medium") HIGH = Rung("claude-fable-5-2", "low") LOW_ROUTE = Route(LOW, 7, 3, 4).worker_line() @@ -239,30 +241,30 @@ class Measured(unittest.TestCase): class Calls(unittest.TestCase): def test_calls_when_chunks_on_a_rung_took_more_rounds_or_unfinished_verdicts_than_the_rung_above(self): chunks = [figures(LOW, rounds=2)] * 5 + [figures(HIGH, unfinished=1)] * 5 - self.assertEqual(calls(totals(chunks)), [Call("rounds", (str(LOW), str(HIGH)))]) - self.assertEqual(calls(totals(chunks))[0].sentence(), f"Chunks routed onto {LOW} took more review rounds past each pass's first than chunks routed onto {HIGH}.") - [unfinished] = calls(totals([figures(LOW, unfinished=1)] * 5 + [figures(HIGH)] * 5)) + self.assertEqual(calls(LADDER, totals(chunks)), [Call("rounds", (str(LOW), str(HIGH)))]) + self.assertEqual(calls(LADDER, totals(chunks))[0].sentence(), f"Chunks routed onto {LOW} took more review rounds past each pass's first than chunks routed onto {HIGH}.") + [unfinished] = calls(LADDER, totals([figures(LOW, unfinished=1)] * 5 + [figures(HIGH)] * 5)) self.assertEqual(unfinished, Call("unfinished", (str(LOW), str(HIGH)))) self.assertEqual(unfinished.sentence(), f"Chunks routed onto {LOW} took more unfinished verdicts than chunks routed onto {HIGH}.") def test_compares_the_bottom_rungs_and_calls_nothing_on_a_tie(self): - self.assertEqual(calls(totals([figures(LADDER[0], rounds=1)] * 5 + [figures(LADDER[1])] * 5)), [Call("rounds", (str(LADDER[0]), str(LADDER[1])))]) - self.assertEqual(calls(totals([figures(LOW, rounds=1)] * 5 + [figures(HIGH, rounds=1)] * 5)), []) - self.assertEqual(calls(totals([figures(reports={"Explore": Reports(1, 20, 20)})] * 5)), []) + self.assertEqual(calls(LADDER, totals([figures(LADDER.rungs[0], rounds=1)] * 5 + [figures(LADDER.rungs[1])] * 5)), [Call("rounds", (str(LADDER.rungs[0]), str(LADDER.rungs[1])))]) + self.assertEqual(calls(LADDER, totals([figures(LOW, rounds=1)] * 5 + [figures(HIGH, rounds=1)] * 5)), []) + self.assertEqual(calls(LADDER, totals([figures(reports={"Explore": Reports(1, 20, 20)})] * 5)), []) def test_compares_only_rungs_with_five_chunks_each(self): - self.assertEqual(calls(totals([figures(LOW, rounds=2)] * 5 + [figures(HIGH)] * 4)), []) + self.assertEqual(calls(LADDER, totals([figures(LOW, rounds=2)] * 5 + [figures(HIGH)] * 4)), []) def test_calls_when_a_subagent_types_reports_outgrow_the_searches_they_ran(self): - [call] = calls(totals([figures(reports={"Explore": Reports(1, 30, 20)})] * 5)) + [call] = calls(LADDER, totals([figures(reports={"Explore": Reports(1, 30, 20)})] * 5)) self.assertEqual(call, Call("reports", ("Explore",))) self.assertEqual(call.sentence(), "Explore subagents' reports took more of the worker's context than their searches would have.") - self.assertEqual(calls(totals([figures(reports={"Explore": Reports(1, 30, 20)})] * 4)), []) - self.assertEqual(calls(totals([figures(reports={"Explore": Reports(1, 20, 30)})] * 5)), []) + self.assertEqual(calls(LADDER, totals([figures(reports={"Explore": Reports(1, 30, 20)})] * 4)), []) + self.assertEqual(calls(LADDER, totals([figures(reports={"Explore": Reports(1, 20, 30)})] * 5)), []) def test_renders_each_dimension_largest_first_and_the_calls(self): chunk = ChunkFigures({"project": collections.Counter({"a": 2.5, "b": 1234.5})}, LOW, 1, 0, collections.Counter({str(HIGH): 2}), {"Explore": Reports(2, 3, 4)}, collections.Counter(), 0.0) - text = render(totals([chunk]), [Call("reports", ("Explore",))], "2026-10-15T00:00:00Z", 40, "") + text = render(LADDER, totals([chunk]), [Call("reports", ("Explore",))], "2026-10-15T00:00:00Z", 40, "") self.assertIn("Measured 2026-10-15T00:00:00Z, at cycle 40 across loop-claude's archive, from 1 finished chunk.", text) self.assertIn("## Calls for a routing change\n\n- Explore subagents' reports took more of the worker's context than their searches would have.\n", text) self.assertIn("## Dollars by project", text) @@ -271,32 +273,32 @@ class Calls(unittest.TestCase): self.assertIn("The judge's cost counts from the verdicts and rejections that record it, so a chunk archived before they did, and a judge run that failed, add none.", text) self.assertNotIn("left out", text) self.assertNotIn("no transcript", text) - self.assertIn("The archive holds no subagent transcript for 2 of the chunks' Agent calls.", render(totals([dataclasses.replace(chunk, lacking=1)] * 2), [], "2026-10-15T00:00:00Z", 40, "")) - self.assertIn("$0.50 of the recorded cost is left out", render(totals([ChunkFigures({}, LOW, 0, 0, collections.Counter(), {}, collections.Counter(), 0.5), chunk]), [], "2026-10-15T00:00:00Z", 40, "")) + self.assertIn("The archive holds no subagent transcript for 2 of the chunks' Agent calls.", render(LADDER, totals([dataclasses.replace(chunk, lacking=1)] * 2), [], "2026-10-15T00:00:00Z", 40, "")) + self.assertIn("$0.50 of the recorded cost is left out", render(LADDER, totals([ChunkFigures({}, LOW, 0, 0, collections.Counter(), {}, collections.Counter(), 0.5), chunk]), [], "2026-10-15T00:00:00Z", 40, "")) self.assertIn(f"| {LOW} | 1 | 1.0 | 0.0 | unknown | {HIGH} 2 | |", text) two = ChunkFigures({}, HIGH, 0, 1, collections.Counter({str(HIGH): 2, str(LOW): 1}), {}, collections.Counter(), 0.0) - self.assertIn(f"| {LOW} | 1 | 1.0 | 0.0 | unknown | {HIGH} 2 | |\n| {HIGH} | 1 | 0.0 | 1.0 | unknown | {HIGH} 2, {LOW} 1 | |", render(totals([two, chunk]), [], "2026-10-15T00:00:00Z", 40, "")) + self.assertIn(f"| {LOW} | 1 | 1.0 | 0.0 | unknown | {HIGH} 2 | |\n| {HIGH} | 1 | 0.0 | 1.0 | unknown | {HIGH} 2, {LOW} 1 | |", render(LADDER, totals([two, chunk]), [], "2026-10-15T00:00:00Z", 40, "")) self.assertIn("| Explore | 2 | 3 | 4 |", text) - escalated = render(totals([figures(escalated={2: 2}), figures(escalated={2: 1, 10: 1}), figures(start=None, escalated={3: 1})]), [], "2026-10-15T00:00:00Z", 40, "") + escalated = render(LADDER, totals([figures(escalated={2: 2}), figures(escalated={2: 1, 10: 1}), figures(start=None, escalated={3: 1})]), [], "2026-10-15T00:00:00Z", 40, "") self.assertIn(f"| {LOW} | 2 | 0.0 | 0.0 | unknown | | trigger 2: 3, trigger 10: 1 |", escalated) self.assertIn("## Escalations by trigger across every chunk, routed or not\n\ntrigger 2: 3, trigger 3: 1, trigger 10: 1.\n", escalated) self.assertIn("## Escalations by trigger across every chunk, routed or not\n\nNone.\n", text) - self.assertIn("## Calls for a routing change\n\nNone.\n", render(totals([]), [], "2026-10-15T00:00:00Z", 20, "")) - self.assertIn("from 0 finished chunks that finished after the reset of 2026-10-14T00:00:00Z.", render(totals([]), [], "2026-10-15T00:00:00Z", 20, "2026-10-14T00:00:00Z")) + self.assertIn("## Calls for a routing change\n\nNone.\n", render(LADDER, totals([]), [], "2026-10-15T00:00:00Z", 20, "")) + self.assertIn("from 0 finished chunks that finished after the reset of 2026-10-14T00:00:00Z.", render(LADDER, totals([]), [], "2026-10-15T00:00:00Z", 20, "2026-10-14T00:00:00Z")) def test_renders_the_mean_later_fixes_of_the_matched_chunks_and_how_many_they_are(self): - text = render(totals([*MATCHED, figures()]), [], "2026-10-15T00:00:00Z", 40, "") + text = render(LADDER, totals([*MATCHED, figures()]), [], "2026-10-15T00:00:00Z", 40, "") self.assertIn(f"| {LOW} | 4 | 0.0 | 0.0 | 1.0 (3 known) | | |", text) self.assertIn("## Routed chunks by the rung they started on\n\nA routed chunk's later fixes are", text) self.assertNotIn("later fixes", text.split("## Calls for a routing change")[0]) def test_counts_the_routed_chunks_whose_later_fixes_are_unknown(self): - self.assertIn("The later fixes of 2 of the routed chunks are unknown", render(totals([*MATCHED, figures(), figures(), figures(start=None)]), [], "2026-10-15T00:00:00Z", 40, "")) - self.assertNotIn("are unknown", render(totals(MATCHED), [], "2026-10-15T00:00:00Z", 40, "")) + self.assertIn("The later fixes of 2 of the routed chunks are unknown", render(LADDER, totals([*MATCHED, figures(), figures(), figures(start=None)]), [], "2026-10-15T00:00:00Z", 40, "")) + self.assertNotIn("are unknown", render(LADDER, totals(MATCHED), [], "2026-10-15T00:00:00Z", 40, "")) def test_names_each_default_branch_unread(self): - self.assertIn("## Default branches unread\n\n- log: no main branch\n", render(totals(MATCHED, unread=("log: no main branch",)), [], "2026-10-15T00:00:00Z", 40, "")) - self.assertNotIn("branches unread", render(totals(MATCHED), [], "2026-10-15T00:00:00Z", 40, "")) + self.assertIn("## Default branches unread\n\n- log: no main branch\n", render(LADDER, totals(MATCHED, unread=("log: no main branch",)), [], "2026-10-15T00:00:00Z", 40, "")) + self.assertNotIn("branches unread", render(LADDER, totals(MATCHED), [], "2026-10-15T00:00:00Z", 40, "")) def test_measures_once_the_count_runs_20_past_the_last_measurement(self): self.assertEqual([due(cycle, 20) for cycle in (39, 40, 41)], [False, True, True]) @@ -428,7 +430,7 @@ class Archive(unittest.TestCase): (self.root / "broken" / "c3" / "copy.outcomes.md").write_bytes(b"\xff") found = archive.measure_archive(self.root, "/h", "", no_checkout) self.assertEqual((len(found.chunks), found.skipped), (1, ("broken", "log/c1"))) - self.assertIn("## Chunks skipped, unreadable\n\n- log/c1\n", render(totals([], ("log/c1",)), [], "2026-10-15T00:00:00Z", 20, "")) + self.assertIn("## Chunks skipped, unreadable\n\n- log/c1\n", render(LADDER, totals([], ("log/c1",)), [], "2026-10-15T00:00:00Z", 20, "")) def test_keeps_the_cycle_it_measured_at_and_reads_it_back(self): self.assertEqual(state.read_state(self.root), state.State(0, ())) diff --git a/bin/rule_hook/main.py b/bin/rule_hook/main.py index e60248ad..4c45f1d8 100644 --- a/bin/rule_hook/main.py +++ b/bin/rule_hook/main.py @@ -81,9 +81,9 @@ def main(argv: list[str], stdin: str) -> int: def breaches_of(call: Call, home: str, environ: typing.Mapping[str, str]) -> tuple[str, str, list[Breach]] | None: """The text that may carry an override, what the log names the call by, and its breaches; None for a call no rule checks.""" if call.tool == "Agent": - return text(call.given, "prompt"), text(call.given, "subagent_type"), reviewer_breaches(call, environ) + return text(call.given, "prompt"), text(call.given, "subagent_type"), reviewer_breaches(call, environ, home) if call.tool == "Skill": - return text(call.given, "args"), text(call.given, "skill"), skill_breaches(call, environ) + return text(call.given, "args"), text(call.given, "skill"), skill_breaches(call, environ, home) if call.tool == "Bash": command = text(call.given, "command") return command, command.split("\n", 1)[0], bash_breaches(command, call.cwd, home) + read_breaches(call, home) diff --git a/bin/rule_hook/reviewer_rung.py b/bin/rule_hook/reviewer_rung.py index bf021c07..df934d2a 100644 --- a/bin/rule_hook/reviewer_rung.py +++ b/bin/rule_hook/reviewer_rung.py @@ -1,4 +1,5 @@ -"""Refuse an Agent or Skill call that launches a review below the rung it needs, worked out from the rungs and the E + I loop-claude hands its worker: docs/decisions.md → `routing:`.""" +"""Refuse an Agent or Skill call that launches a review below the rung it needs, worked out on the project's ladder from the rungs and the E + I loop-claude hands its worker: docs/decisions.md → `routing:`.""" +import pathlib import typing from rule_hook.rules import Breach @@ -6,8 +7,9 @@ from rule_hook.call import Call, text from failure import Failure from git_cli import git_repo from rule_hook.git import guarded_git -from model_routing.worker_rungs import handed -from model_routing.ladder import position, review_rung, ReviewKind, Rung +from model_routing.worker_rungs import handed, hands_rungs +from model_routing.ladder import Ladder, ReviewKind, Rung +from model_routing.ladder_file import project_ladder from review_rules.review_passes import prose, REVIEWERS from claude_code.transcript import read_transcript from claude_code.entries import UserText @@ -23,29 +25,44 @@ SUBJECTS: dict[ReviewKind, str] = {"seat": "a comprehension seat", "security": " SECURITY_SKILLS = {"claude-security", "security-review"} -def reviewer_breaches(call: Call, environ: typing.Mapping[str, str]) -> list[Breach]: +def reviewer_breaches(call: Call, environ: typing.Mapping[str, str], home: str) -> list[Breach]: """The breach of an Agent call that launches a review below its rung; none where loop-claude handed no rungs.""" - project, worker, escape_and_impact = handed(environ) kind = review_kind(call.given) - if kind is None or project is None or worker is None: + if kind is None or not hands_rungs(environ): return [] - required = review_rung(project, worker, escape_and_impact, kind, markdown_only(call.cwd)) - named = named_rung(call.given, worker) - if position(named) >= position(required): + ladder = project_ladder(call.cwd, pathlib.Path(home)) + if isinstance(ladder, Failure): + return [unread(ladder, call.cwd)] + project, worker, escape_and_impact = handed(environ, ladder) + if project is None or worker is None: + return [] + required = ladder.review_rung(project, worker, escape_and_impact, kind, markdown_only(call.cwd)) + named = named_rung(call.given, worker, ladder) + if ladder.position(named) >= ladder.position(required): return [] subject = SUBJECTS.get(kind) or text(call.given, "subagent_type") return [Breach("reviewer-rung", f"{subject} would run on {named}, below {required}: launch it with {required.agent_args()}", call.cwd)] -def skill_breaches(call: Call, environ: typing.Mapping[str, str]) -> list[Breach]: +def unread(ladder: Failure, cwd: str) -> Breach: + """The breach of a review launched where loop-claude handed rungs and the project's ladder cannot be read, so nothing tells which rung the review needs.""" + return Breach("reviewer-rung", f"the rung this review needs cannot be worked out: {ladder.reason}", cwd) + + +def skill_breaches(call: Call, environ: typing.Mapping[str, str], home: str) -> list[Breach]: """The breach of a Skill call that runs the security pass below the project's rung; none where loop-claude handed no rungs. The skill runs on its caller's rung. A subagent launched as the security pass was held to the project's rung by reviewer_breaches. The worker's own rung is known, so its call is refused only below the project's rung. Any other subagent's rung is unknown, so its call is refused.""" - project, worker, _ = handed(environ) skill = text(call.given, "skill").lstrip("/").rsplit(":", 1)[-1] - if skill not in SECURITY_SKILLS or project is None or worker is None or opening_prompt(call.transcript).startswith(SECURITY_OPENER): + if skill not in SECURITY_SKILLS or not hands_rungs(environ) or opening_prompt(call.transcript).startswith(SECURITY_OPENER): return [] - if not call.subagent and position(worker) >= position(project): + ladder = project_ladder(call.cwd, pathlib.Path(home)) + if isinstance(ladder, Failure): + return [unread(ladder, call.cwd)] + project, worker, _ = handed(environ, ladder) + if project is None or worker is None: + return [] + if not call.subagent and ladder.position(worker) >= ladder.position(project): return [] runs_on = "on a rung rule-hook cannot see" if call.subagent else f"in this context on {worker}, below {project}" return [Breach("reviewer-rung", f"{skill} would run {runs_on}: invoke it from a subagent launched with {project.agent_args()}, its prompt opening with `{SECURITY_OPENER}`", call.cwd)] @@ -70,12 +87,12 @@ def review_kind(given: dict[str, typing.Any]) -> ReviewKind | None: return None -def named_rung(given: dict[str, typing.Any], worker: Rung) -> Rung: - """The rung an Agent call's model and effort name. A call naming no model runs on the worker's model, at the effort it names or else the worker's; one naming a model but no effort runs on that model's cheapest rung.""" +def named_rung(given: dict[str, typing.Any], worker: Rung, ladder: Ladder) -> Rung: + """The rung an Agent call's model and effort name. A call naming no model runs on the worker's model, at the effort it names or else the worker's; one naming a model but no effort runs on that model's cheapest rung on ladder.""" model, effort = text(given, "model"), text(given, "effort") if not model: return Rung(worker.model, effort or worker.effort) - return Rung.from_agent(model, effort) + return ladder.from_agent(model, effort) def markdown_only(directory: str) -> bool: diff --git a/bin/rule_hook/test_rule_hook.py b/bin/rule_hook/test_rule_hook.py index 503120e3..bcdcf052 100644 --- a/bin/rule_hook/test_rule_hook.py +++ b/bin/rule_hook/test_rule_hook.py @@ -216,6 +216,10 @@ class OverrideLog(unittest.TestCase): self.assertEqual(this_chunks(LogLine("t", branch, commit, "image-pin", "r", "w"), {"base", "c1"}, "chunk", on_default), expected) +# A home that holds no project config, so a review launched there is checked on the ladder this setup ships. +HOME = "/h" + + class ReviewerRung(unittest.TestCase): # A worker routed to Sonnet 5.5 at high, on a project whose rung is Opus 5.5 at medium, with its reviewers on the project's rung. ENVIRON = {"LOOP_CLAUDE_PROJECT_RUNG": "claude-opus-5-5 at medium effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-sonnet-5-5 at high effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "4"} @@ -223,7 +227,7 @@ class ReviewerRung(unittest.TestCase): ABOVE = {"LOOP_CLAUDE_PROJECT_RUNG": "claude-opus-5-5 at medium effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-fable-5-2 at max effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "4"} def refused(self, given: dict, environ: dict[str, str] = ENVIRON) -> list[str]: - return rules(reviewer_breaches(Call("Agent", given, "/r"), environ)) + return rules(reviewer_breaches(Call("Agent", given, "/r"), environ, HOME)) def test_a_branch_changing_only_markdown_reviews_on_the_projects_rung_whatever_the_e_plus_i(self): repo = tempfile.mkdtemp() @@ -238,24 +242,24 @@ class ReviewerRung(unittest.TestCase): git(repo, "commit", "-q", "-m", "prose") high = self.ENVIRON | {"LOOP_CLAUDE_ESCAPE_AND_IMPACT": "7"} on_project = {"subagent_type": "stability-reviewer", "model": "opus", "effort": "medium"} - self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high)), []) - self.assertEqual(rules(reviewer_breaches(Call("Agent", {**on_project, "model": "sonnet", "effort": "high"}, repo), high)), ["reviewer-rung"]) - self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), self.ABOVE)), [], "a worker that climbed doesn't lift a Markdown-only review either") + self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high, HOME)), []) + self.assertEqual(rules(reviewer_breaches(Call("Agent", {**on_project, "model": "sonnet", "effort": "high"}, repo), high, HOME)), ["reviewer-rung"]) + self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), self.ABOVE, HOME)), [], "a worker that climbed doesn't lift a Markdown-only review either") for instructions in ("CLAUDE.md", "AGENTS.md", "skills/review/SKILL.md", "agents/reviewer.md", "rules/todo.md", "principles/tests.md", "workflow-bench/run-prompt.md"): with self.subTest(instructions=instructions): path = pathlib.Path(repo, instructions) path.parent.mkdir(parents=True, exist_ok=True) path.write_text("Do this.\n") - self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high)), ["reviewer-rung"], "Markdown an agent follows is config") + self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high, HOME)), ["reviewer-rung"], "Markdown an agent follows is config") path.unlink() pathlib.Path(repo, "gate.yml").write_text("on: push\n") - self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high)), ["reviewer-rung"], "an uncommitted change to another file counts") + self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high, HOME)), ["reviewer-rung"], "an uncommitted change to another file counts") git(repo, "add", ".") git(repo, "commit", "-q", "-m", "config") - self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high)), ["reviewer-rung"]) + self.assertEqual(rules(reviewer_breaches(Call("Agent", on_project, repo), high, HOME)), ["reviewer-rung"]) def test_a_reviewer_below_the_project_rung_is_refused_naming_the_arguments_to_pass(self): - breaches = reviewer_breaches(Call("Agent", {"subagent_type": "stability-reviewer", "model": "sonnet", "effort": "high", "prompt": "Review."}, "/r"), self.ENVIRON) + breaches = reviewer_breaches(Call("Agent", {"subagent_type": "stability-reviewer", "model": "sonnet", "effort": "high", "prompt": "Review."}, "/r"), self.ENVIRON, HOME) self.assertEqual(rules(breaches), ["reviewer-rung"]) self.assertIn("stability-reviewer", breaches[0].detail) self.assertIn("claude-sonnet-5-5 at high effort", breaches[0].detail) @@ -312,13 +316,53 @@ class ReviewerRung(unittest.TestCase): def test_a_claude_loop_claude_did_not_start_is_not_checked(self): self.assertEqual(self.refused({"subagent_type": "stability-reviewer", "model": "haiku", "effort": "low"}, {}), []) + def project(self, ladder: str) -> tuple[str, str]: + """A checkout whose config's folder holds ladder as its routing ladder, and its home.""" + home = pathlib.Path(tempfile.mkdtemp()).resolve() + self.addCleanup(subprocess.run, ["rm", "-rf", str(home)]) + checkout = home / "code" / "acme" + checkout.mkdir(parents=True) + (configured(self, checkout, home=home) / "routing-ladder.json").write_text(ladder) + return str(checkout), str(home) + + # A project that runs on Fable 5.2 at low and at high alone, with a worker on its rung and an E + I of 5, which picks the rung one up. + TWO_RUNGS = '[{"model": "claude-fable-5-2", "effort": "low"}, {"model": "claude-fable-5-2", "effort": "high"}]' + ON_TWO_RUNGS = {"LOOP_CLAUDE_PROJECT_RUNG": "claude-fable-5-2 at low effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-fable-5-2 at low effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "5"} + + def test_a_review_is_held_to_the_rung_picked_on_the_projects_own_ladder(self): + checkout, home = self.project(self.TWO_RUNGS) + reviewer = {"subagent_type": "stability-reviewer", "model": "fable"} + self.assertEqual(rules(reviewer_breaches(Call("Agent", {**reviewer, "effort": "high"}, checkout), self.ON_TWO_RUNGS, home)), []) + # One up from Fable 5.2 at low is its medium effort on the shipped ladder, and its high effort on the project's, which holds no medium. + breaches = reviewer_breaches(Call("Agent", {**reviewer, "effort": "medium"}, checkout), self.ON_TWO_RUNGS, home) + self.assertEqual([breach.detail for breach in breaches], ["stability-reviewer would run on claude-fable-5-2 at medium effort, below claude-fable-5-2 at high effort: launch it with model fable, effort high"]) + self.assertEqual(rules(reviewer_breaches(Call("Agent", {**reviewer, "effort": "medium"}, "/r"), self.ON_TWO_RUNGS, HOME)), []) + + def test_a_security_skill_is_held_to_the_projects_rung_on_the_projects_own_ladder(self): + checkout, home = self.project('[{"model": "claude-fable-5-2", "effort": "high"}, {"model": "claude-fable-5-2", "effort": "low"}]') + # On this ladder the worker's low effort ranks above the project's high one; on the shipped ladder it ranks below. + environ = {"LOOP_CLAUDE_PROJECT_RUNG": "claude-fable-5-2 at high effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-fable-5-2 at low effort"} + self.assertEqual(skill_breaches(Call("Skill", {"skill": "security-review"}, checkout), environ, home), []) + self.assertEqual(rules(skill_breaches(Call("Skill", {"skill": "security-review"}, "/r"), environ, HOME)), ["reviewer-rung"]) + + def test_a_review_is_refused_where_the_projects_ladder_cannot_be_read_and_loop_claude_handed_rungs(self): + checkout, home = self.project("[]") + own = pathlib.Path(home, ".claude-projects", "acme", "routing-ladder.json") + reviewer = Call("Agent", {"subagent_type": "stability-reviewer", "model": "fable", "effort": "max"}, checkout) + for breaches in (reviewer_breaches(reviewer, self.ON_TWO_RUNGS, home), skill_breaches(Call("Skill", {"skill": "security-review"}, checkout), self.ON_TWO_RUNGS, home)): + self.assertEqual(rules(breaches), ["reviewer-rung"]) + self.assertTrue(breaches[0].detail.startswith(f"the rung this review needs cannot be worked out: {own}: it holds no list of rungs. "), breaches[0].detail) + self.assertEqual(reviewer_breaches(reviewer, {}, home), [], "a claude loop-claude did not start is not checked") + self.assertEqual(reviewer_breaches(Call("Agent", {"subagent_type": "Explore"}, checkout), self.ON_TWO_RUNGS, home), [], "nor is a subagent that reviews nothing") + self.assertEqual(skill_breaches(Call("Skill", {"skill": "simplify"}, checkout), self.ON_TWO_RUNGS, home), []) + def skill_refused(self, given: dict, environ: dict[str, str] = ENVIRON, subagent_prompt: str | list | None = None) -> list[Breach]: transcript = "" if subagent_prompt is not None: transcript = os.path.join(tempfile.mkdtemp(), "agent-a1.jsonl") self.addCleanup(subprocess.run, ["rm", "-rf", os.path.dirname(transcript)]) pathlib.Path(transcript).write_text(json.dumps({"type": "user", "message": {"role": "user", "content": subagent_prompt}}) + "\n") - return skill_breaches(Call("Skill", given, "/r", "general-purpose" if subagent_prompt is not None else "", transcript, subagent_prompt is not None), environ) + return skill_breaches(Call("Skill", given, "/r", "general-purpose" if subagent_prompt is not None else "", transcript, subagent_prompt is not None), environ, HOME) def test_a_security_skill_from_the_sessions_own_agent_is_refused_below_the_projects_rung(self): for skill in ("/security-review", "claude-security", "claude-security:claude-security", "security-review"): @@ -338,7 +382,7 @@ class ReviewerRung(unittest.TestCase): with self.subTest(prompt=prompt): self.assertEqual(rules(self.skill_refused({"skill": "security-review"}, self.ENVIRON, prompt)), []) self.assertEqual(rules(self.skill_refused({"skill": "security-review"}, self.ABOVE, "Invoke /security-review.")), ["reviewer-rung"], "a subagent's rung is unknown, whatever the worker's") - unreadable = skill_breaches(Call("Skill", {"skill": "security-review"}, "/r", "general-purpose", "/nonexistent/agent-a1.jsonl", True), self.ENVIRON) + unreadable = skill_breaches(Call("Skill", {"skill": "security-review"}, "/r", "general-purpose", "/nonexistent/agent-a1.jsonl", True), self.ENVIRON, HOME) self.assertEqual(rules(unreadable), ["reviewer-rung"]) def test_any_other_skill_or_a_claude_loop_claude_did_not_start_is_not_checked(self): diff --git a/bin/rung_replay/command.py b/bin/rung_replay/command.py index 3eae71b9..9d3425d4 100644 --- a/bin/rung_replay/command.py +++ b/bin/rung_replay/command.py @@ -4,7 +4,7 @@ import sys import typing from failure import Failure -from model_routing.ladder import LADDER, position, Rung +from model_routing.ladder import Ladder, Rung from claude_code.reply import Limited from loop_state.paths import paths_from from loop_claude.project.model_choice import read_models_file @@ -32,9 +32,9 @@ def projects_on(model: str, home: pathlib.Path) -> list[str] | Failure: return sorted(key for key, on in models.items() if on == model) -def down_from(top: Rung, floor: Rung) -> list[Rung]: - """top, the project's rung, and each rung below it, highest first, down to floor.""" - return list(LADDER[position(floor):position(top) + 1])[::-1] +def down_from(ladder: Ladder, top: Rung, floor: Rung) -> list[Rung]: + """top, the project's rung, and each rung below it on ladder, highest first, down to floor.""" + return list(ladder.rungs[ladder.position(floor):ladder.position(top) + 1])[::-1] def limited_exit(tool: str, limited: Limited) -> int: diff --git a/bin/rung_replay/test_rung_replay.py b/bin/rung_replay/test_rung_replay.py index 4985270f..23a8b816 100644 --- a/bin/rung_replay/test_rung_replay.py +++ b/bin/rung_replay/test_rung_replay.py @@ -10,8 +10,10 @@ from rung_replay.command import down_from from failure import Failure from rung_replay.search import first_reply, Miss, Replay, replayed, Tried from claude_code.reply import Limited +from model_routing.ladder_file import read_ladder, SHIPPED from model_routing.ladder import Rung +LADDER = read_ladder(SHIPPED) REFERENCE = Rung("claude-opus-5-5", "medium") MIDDLE = Rung("claude-sonnet-5-5", "high") CHEAP = Rung("claude-haiku-5-5", "max") @@ -73,8 +75,8 @@ class FirstReply(unittest.TestCase): class DownFrom(unittest.TestCase): def test_runs_from_the_projects_rung_down_to_the_floor_highest_first(self): - self.assertEqual(down_from(Rung("claude-fable-5-2", "low"), MIDDLE), [Rung("claude-fable-5-2", "low"), REFERENCE, MIDDLE]) - self.assertEqual(down_from(REFERENCE, MIDDLE), [REFERENCE, MIDDLE]) + self.assertEqual(down_from(LADDER, Rung("claude-fable-5-2", "low"), MIDDLE), [Rung("claude-fable-5-2", "low"), REFERENCE, MIDDLE]) + self.assertEqual(down_from(LADDER, REFERENCE, MIDDLE), [REFERENCE, MIDDLE]) if __name__ == "__main__": diff --git a/bin/subagent-rung b/bin/subagent-rung index 96224df9..5a724160 100755 --- a/bin/subagent-rung +++ b/bin/subagent-rung @@ -6,4 +6,4 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) from subagent_rung import main -sys.exit(main.main(sys.argv[1:])) +sys.exit(main.main(sys.argv[1:], pathlib.Path.home())) diff --git a/bin/subagent_rung/main.py b/bin/subagent_rung/main.py index b101f7e9..a1bfe4c7 100644 --- a/bin/subagent_rung/main.py +++ b/bin/subagent_rung/main.py @@ -5,31 +5,37 @@ usage: subagent-rung [--plan] is the project's model, such as claude-opus-5-5, and the task's L, the chance it holds a defect, a level from 1 to 4 scored as for a todo.md item. It prints the rung that L picks from the model's cheapest rung, the project's rung, in the Agent tool's words, such as `model sonnet, -effort high`. --plan, for a task that files or scores todo.md items, prints the project's rung where +effort high`. It reads the routing ladder of the project whose checkout it runs in: +routing-ladder.json in the folder of the project's config, ~/.claude-projects//, or else +the one this setup ships. loop-claude --help says how to write one. --plan, for a task that files or scores todo.md items, prints the project's rung where that L picks a lower one, as ~/.claude/rules/todo-items.md lets only the project's rung or above plan. For a model off the routing ladder, the tool doesn't print anything on stdout, says so on stderr and exits 0; launch the subagent with no model or effort. -Exit 1, saying why, on bad arguments. +Exit 1, saying why, on bad arguments, or where the project's config or its routing ladder is faulty +or cannot be read. """ +import pathlib import re import sys +from failure import Failure +from model_routing.ladder import Ladder from release_plan.todo_table import LEVELS -from model_routing.ladder import planning_rung, project_rung, worker_rung +from model_routing.ladder_file import project_ladder USAGE = f"usage: subagent-rung [--plan] , L a level from 1 to {LEVELS['L']}" -def agent_args(model: str, likelihood: int, planning: bool = False) -> str | None: - """The Agent tool's model and effort for the rung likelihood picks from model's cheapest rung, never below it when planning; None for a model off the ladder.""" - project = project_rung(model) +def agent_args(ladder: Ladder, model: str, likelihood: int, planning: bool = False) -> str | None: + """The Agent tool's model and effort for the rung likelihood picks on ladder from model's cheapest rung, never below it when planning; None for a model off the ladder.""" + project = ladder.project_rung(model) if project is None: return None - return (planning_rung if planning else worker_rung)(project, likelihood).agent_args() + return (ladder.planning_rung if planning else ladder.worker_rung)(project, likelihood).agent_args() -def main(argv: list[str]) -> int: +def main(argv: list[str], home: pathlib.Path) -> int: if argv[:1] in (["-h"], ["--help"]): print(__doc__.strip()) return 0 @@ -39,7 +45,11 @@ def main(argv: list[str]) -> int: print(f"subagent-rung: cannot read the arguments{' ' + ' '.join(argv) if argv else ''}; {USAGE}", file=sys.stderr) return 1 model, likelihood = operands - args = agent_args(model, int(likelihood), planning) + ladder = project_ladder(None, home) + if isinstance(ladder, Failure): + print(f"subagent-rung: {ladder.reason}", file=sys.stderr) + return 1 + args = agent_args(ladder, model, int(likelihood), planning) if args is None: print(f"subagent-rung: {model} is not on the routing ladder; pass the Agent tool no model or effort", file=sys.stderr) else: diff --git a/bin/subagent_rung/test_subagent_rung.py b/bin/subagent_rung/test_subagent_rung.py index f06cabc3..3202f98c 100644 --- a/bin/subagent_rung/test_subagent_rung.py +++ b/bin/subagent_rung/test_subagent_rung.py @@ -1,33 +1,66 @@ import pathlib import subprocess import sys +import tempfile import unittest sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) +from test_env import isolated_env from subagent_rung import main +from model_routing.ladder_file import read_ladder, SHIPPED +from project_config.test_project_config import write_config +LADDER = read_ladder(SHIPPED) TOOL = pathlib.Path(__file__).resolve().parents[1] / "subagent-rung" class SubagentRung(unittest.TestCase): - def run_tool(self, *args: str) -> subprocess.CompletedProcess[str]: - return subprocess.run([sys.executable, str(TOOL), *args], capture_output=True, text=True) + def setUp(self) -> None: + made = tempfile.TemporaryDirectory() + self.addCleanup(made.cleanup) + # A home with no project config, so the tool reads the ladder this setup ships, whatever the host's own projects hold. + self.home = pathlib.Path(made.name).resolve() + + def run_tool(self, *args: str, cwd: pathlib.Path | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run([sys.executable, str(TOOL), *args], capture_output=True, cwd=cwd or self.home, env={**isolated_env(), "HOME": str(self.home)}, text=True) + + def project(self, ladder: str) -> pathlib.Path: + """A checkout under the home whose config's folder holds ladder as its routing ladder.""" + checkout = self.home / "code" / "acme" + checkout.mkdir(parents=True) + subprocess.run(["git", "init", "--quiet", str(checkout)], check=True, env=isolated_env()) + (write_config(self.home, checkout) / "routing-ladder.json").write_text(ladder) + return checkout + + def test_picks_the_rung_on_the_ladder_of_the_project_it_runs_in(self): + checkout = self.project('[{"model": "claude-fable-5-2", "effort": "low"}, {"model": "claude-fable-5-2", "effort": "max"}]') + (checkout / "nested").mkdir() + # On the shipped ladder an L of 1 steps Fable 5.2 down to Opus 5.5, and an L of 4 up to its medium effort. + self.assertEqual([self.run_tool("claude-fable-5-2", level, cwd=checkout / "nested").stdout for level in ("1", "4")], ["model fable, effort low\n", "model fable, effort max\n"]) + off = self.run_tool("claude-opus-5-5", "3", cwd=checkout) + self.assertEqual((off.returncode, off.stdout, off.stderr), (0, "", "subagent-rung: claude-opus-5-5 is not on the routing ladder; pass the Agent tool no model or effort\n")) + + def test_exits_1_naming_the_file_where_the_projects_ladder_holds_a_fault(self): + checkout = self.project('[{"model": "claude-fable-5-2", "effort": "low"},]') + refused = self.run_tool("claude-fable-5-2", "3", cwd=checkout) + self.assertEqual((refused.returncode, refused.stdout), (1, "")) + self.assertTrue(refused.stderr.startswith(f"subagent-rung: cannot read the routing ladder {self.home / '.claude-projects' / 'acme' / 'routing-ladder.json'}: "), refused.stderr) def test_prints_the_agent_tools_model_and_effort_for_the_rung_an_l_picks(self): - self.assertEqual(main.agent_args("claude-opus-5-5", 3), "model opus, effort medium") - self.assertEqual(main.agent_args("claude-opus-5-5", 2), "model sonnet, effort high") - self.assertEqual(main.agent_args("claude-opus-5-5", 4), "model fable, effort low") - self.assertEqual(main.agent_args("claude-sonnet-5-5", 1), "model sonnet, effort high") - self.assertIsNone(main.agent_args("claude-haiku-5-5", 1)) + self.assertEqual(main.agent_args(LADDER, "claude-opus-5-5", 3), "model opus, effort medium") + self.assertEqual(main.agent_args(LADDER, "claude-opus-5-5", 2), "model sonnet, effort high") + self.assertEqual(main.agent_args(LADDER, "claude-opus-5-5", 4), "model fable, effort low") + self.assertEqual(main.agent_args(LADDER, "claude-sonnet-5-5", 1), "model sonnet, effort high") + self.assertIsNone(main.agent_args(LADDER, "claude-haiku-5-5", 1)) def test_a_planning_task_never_runs_below_the_projects_rung(self): - self.assertEqual(main.agent_args("claude-opus-5-5", 1, planning=True), "model opus, effort medium") - self.assertEqual(main.agent_args("claude-opus-5-5", 4, planning=True), "model fable, effort low") + self.assertEqual(main.agent_args(LADDER, "claude-opus-5-5", 1, planning=True), "model opus, effort medium") + self.assertEqual(main.agent_args(LADDER, "claude-opus-5-5", 4, planning=True), "model fable, effort low") self.assertEqual(self.run_tool("--plan", "claude-opus-5-5", "2").stdout, "model opus, effort medium\n") def test_picks_nothing_for_a_model_off_the_ladder(self): - self.assertIsNone(main.agent_args("claude-opus-5", 3)) + self.assertIsNone(main.agent_args(LADDER, "claude-opus-5", 3)) def test_refuses_an_l_outside_1_to_4_or_a_wrong_count_of_arguments(self): for argv in (["claude-opus-5-5", "0"], ["claude-opus-5-5", "01"], ["claude-opus-5-5", "5"], ["claude-opus-5-5", "x"], ["claude-opus-5-5"], ["3", "claude-opus-5-5"], ["claude-opus-5-5", "3", "--plan"], ["--planning", "claude-opus-5-5", "3"]): diff --git a/docs/decisions.md b/docs/decisions.md index 4a4e0016..da241194 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -514,9 +514,9 @@ On 2026-11-12, `bin/judge-replay claude-opus-5-5` replayed 60 verdicts, 15 of ea Valid while the archived verdicts show what a judge should return, and until a new model joins the ladder: replay then. -## routing: one ladder of (model, effort) rungs, ordered by the Artificial Analysis Intelligence Index; a task's L picks its worker's rung and its E + I its reviewers', by fixed steps counted from the cheapest rung of the project's model +## routing: one ladder of (model, effort) rungs, ordered by the Artificial Analysis Intelligence Index, shipped as `routing-ladder.json`, which a project replaces with a file of that name in its config's folder; a task's L picks its worker's rung and its E + I its reviewers', by fixed steps counted from the cheapest rung of the project's model -2026-10-14, lilleman. Serves goal 1.4, and goal 1.3: one ladder, the same in every project, read by loop-claude and by the worker through `bin/subagent-rung`. The judge's rung is set by `routing: on a model judge-replay covered`. `bin/model_routing/ladder.py` holds the rungs and the steps. +2026-10-14, lilleman. Serves goal 1.4, and goal 1.3: one ladder, the same in every project, read by loop-claude and by the worker through `bin/subagent-rung`. The judge's rung is set by `routing: on a model judge-replay covered`. `routing-ladder.json` holds the rungs and `bin/model_routing/ladder.py` the steps; the 2026-11-15 note below says when a project runs on a ladder of its own. The rungs, least competent first: Sonnet 5.5 at high; Opus 5.5 at medium; Fable 5.2 at low and medium; Opus 5.5 at xhigh; Fable 5.2 at high and max. The 2026-11-11 note below put them so. Before it they were Sonnet 5.5 at high; Opus 5.5 at medium, high, xhigh and max; Fable 5.1 at max, after the 2026-10-29 note took Haiku 5.5 off. They first were: Haiku 5.5 at low, medium, high, xhigh and max; Sonnet 5.5 at high; Opus 5.5 at medium, high, xhigh and max; Fable 5.1 at max. The order and the cut come from the Artificial Analysis Intelligence Index (https://artificialanalysis.ai/leaderboards/models), which scores each model at each effort and gives a cost per task. A rung stays only where no cheaper rung scores as high, so a step up always buys more score (goal 1.4). The first cut, from the index read 2026-10-14, dropped Sonnet 5.5 at low, medium, xhigh and max, Opus 5.5 at low, and every Fable 5.1 rung, and every tie in the index fell among the dropped rungs. lilleman put Fable 5.1 at max back on top: the index scored it under Opus 5.5 at high, but Anthropic's model catalog names Fable the most capable model, and for the hardest tasks lilleman wants the highest quality at any price. Fable 5.2 at max tops the index outright, so that exception has ended. @@ -534,6 +534,8 @@ A project's rung is its model's cheapest rung, which for Opus 5.5 is its default 2026-11-11, lilleman approved the rungs for Fable 5.2 from the index read that day. Its score and cost per task, for each effort from low to max: Sonnet 5.5 26 $0.51, 31 $0.78, 42 $1.64, 50 $4.09, 55 $10.02; Opus 5.5 35 $0.98, 48 $2.45, 52 $3.46, 57 $6.05, 59 $10.04; Fable 5.2 54 $3.10, 56 $3.80, 62 $9.08, 62 $14.83, 65 $26.54. The cut drops Opus 5.5 at high, since Fable 5.2 at low scores more for less; Opus 5.5 at max, since Fable 5.2 at high scores more for less; and Fable 5.2 at xhigh, which ties Fable 5.2 at high at a higher cost. Sonnet 5.5 at high stays the bottom rung, as the 2026-10-29 note chose. With Haiku 5.5 off, the cut alone would also keep Sonnet 5.5 at low and medium and Opus 5.5 at low, below it. On an Opus 5.5 project, an L 4 worker, an E + I 5 review and a worker's first escalation now run on Fable 5.2 at low, where they ran on Opus 5.5 at high; E + I 8 runs on Fable 5.2 at high, where it ran on Fable 5.1 at max. Three steps land on a rung the index scores lower than before: E + I 6 on Fable 5.2 at medium, 56 against Opus 5.5 at xhigh's 57; E + I 7 on Opus 5.5 at xhigh, 57 against Opus 5.5 at max's 59; and an L 3 worker's escalation stops, without lilleman's yes, at Fable 5.2 at medium, where it stopped at Opus 5.5 at xhigh. On a Sonnet 5.5 project, E + I 6 runs on Fable 5.2 at low, 54 against Opus 5.5 at high's 52; E + I 7 on Fable 5.2 at medium, 56 against Opus 5.5 at xhigh's 57; and E + I 8 on Opus 5.5 at xhigh, 57 against Opus 5.5 at max's 59. Fable 5.2 at max, on top, is reached only one rung up from Fable 5.2 at high, as by a reviewer's rerun. A Fable 5.2 project's rung is Fable 5.2 at low, so its L 1 and L 2 workers run on Opus 5.5 at medium. +2026-11-15, lilleman asked for the ladder as a file that a project can replace, on a day when an outage left only Fable 5.2 answering: the rungs were a constant in `bin/model_routing/ladder.py`, so no project could leave Sonnet 5.5 and Opus 5.5 without a release. The shipped ladder is `routing-ladder.json` at the repo's root, a JSON list of rungs, least competent first. A project that keeps a `routing-ladder.json` in its config's folder, `~/.claude-projects//`, runs on that ladder whole, and the steps count on it as on the shipped one. Every project still runs the same routing rules, and its config's folder supplies what differs, as goal 1.3 asks. lilleman chose a file per project over one per machine, so a single project can differ. The project's file is lilleman's to write, as any change to the ladder is, and the loop never commits it. loop-claude reads the project's ladder as each cycle starts; `subagent-rung` and rule-hook's `reviewer-rung` read it each time they run, from the checkout they run in. A ladder file that holds a fault stops each of them, since work routed on the shipped ladder in its place could run on a model the project cannot reach. The routing figures, `judge-replay` and `review-replay` span every project, so they read the shipped ladder: a rung only a project's own ladder holds is left out of the figures' calls, and no replay qualifies it. Three readers, seated as lilleman writing a project's file in a hurry, picked the file's shape, a bare list of `{"model", "effort"}` objects, over a list of strings, a list of pairs and a list under a key, and two of the three picked one name for both files. + Valid while the index ranks these models, and while no newer model joins the catalog. ## routing: the worker changes rung only as a chunk starts, after every second unfinished verdict, when it names a chunk item that picks a higher rung, or when it cites an escalation trigger the judge finds diff --git a/routing-ladder.json b/routing-ladder.json new file mode 100644 index 00000000..b074069f --- /dev/null +++ b/routing-ladder.json @@ -0,0 +1,9 @@ +[ + {"model": "claude-sonnet-5-5", "effort": "high"}, + {"model": "claude-opus-5-5", "effort": "medium"}, + {"model": "claude-fable-5-2", "effort": "low"}, + {"model": "claude-fable-5-2", "effort": "medium"}, + {"model": "claude-opus-5-5", "effort": "xhigh"}, + {"model": "claude-fable-5-2", "effort": "high"}, + {"model": "claude-fable-5-2", "effort": "max"} +]