Bind a project's rung to its ladder in ProjectRung, and step an L 3 worker one rung up and an L 4 worker two up #581
@@ -2,6 +2,7 @@
|
||||
|
||||
## Unreleased
|
||||
|
||||
- loop-claude and subagent-rung run an L 3 task one rung above the project's rung and an L 4 task two above it: on an Opus 5.5 project, Fable 5.2 at low and at medium.
|
||||
- Add `larv-review round-title <pass>`, which prints a review round commit's title prefix with the 🔴, 🟡 and misread counts taken from the round's report. rule-hook refuses a round commit whose counts differ from that report, where the report exists and can be counted; a report must head each finding `### 🔴` or `### 🟡`, optionally followed by ` [misread]`.
|
||||
- Add `workflow-bench/handoff_bench.py`, a bench of how a worker's step to a stronger model hands over the chunk, on six archived L 3 chunks. Sonnet 5.5 left five unfinished at its budget. On those, Opus 5.5 resuming Sonnet 5.5's session passed two and was partial on three, for a mean $0.68; a fresh session with a brief passed one and was partial on four, for $0.78. So a stepped worker still resumes its session.
|
||||
- todo-item edit --blocked-by sets the "is blocked by" links of a Jira issue the loop filed.
|
||||
|
||||
@@ -52,7 +52,7 @@ from failure import Failure
|
||||
from rung_replay.search import first_reply, replayed
|
||||
from judge_replay.archived_prompt import grounds_of
|
||||
from transcript_archive.judge_run import JudgeRun
|
||||
from model_routing.ladder import Ladder, REVIEW_STEPS, Rung
|
||||
from model_routing.ladder import Ladder, ProjectRung, REVIEW_STEPS, Rung
|
||||
from claude_code.reply import Limited
|
||||
from model_routing.ladder_file import read_ladder, SHIPPED
|
||||
from loop_claude.project.premises import read_premises
|
||||
@@ -75,31 +75,31 @@ def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, eff
|
||||
project = arguments_or_exit(TOOL, argv, __doc__, USAGE, lambda argv: model_argument(ladder, argv))
|
||||
if isinstance(project, int):
|
||||
return project
|
||||
cases = archived_cases(project.model, archive_root(home, env), home)
|
||||
cases = archived_cases(project.rung.model, archive_root(home, env), home)
|
||||
if isinstance(cases, Failure):
|
||||
return failed(TOOL, cases.reason)
|
||||
if not cases:
|
||||
return failed(TOOL, f"no archived verdicts of projects on {project.model} in {archive_root(home, env)}; run loop-claude on a project on that model first")
|
||||
return failed(TOOL, f"no archived verdicts of projects on {project.rung.model} in {archive_root(home, env)}; run loop-claude on a project on that model first")
|
||||
cases = sample(cases, PER_KIND)
|
||||
folder = replay_folder(archive_root(home, env), TOOL, argv, effects.now())
|
||||
effects = archiving(effects, folder)
|
||||
reference = Rung(project.model, REFERENCE_EFFORT)
|
||||
reference = Rung(project.rung.model, REFERENCE_EFFORT)
|
||||
# The candidates stop at the lowest rung a worker routes to: no route uses a rung below it for the judge.
|
||||
replay = replayed(cases, reference, down_from(ladder, project, ladder.lowest_worker_rung(project)), lambda found, rung: rule(found, rung, home, effects), reproduced, "verdicts", climbs(ladder, project, reference))
|
||||
replay = replayed(cases, reference, down_from(ladder, project.rung, project.lowest_worker_rung()), lambda found, rung: rule(found, rung, home, effects), reproduced, "verdicts", climbs(project, reference))
|
||||
if isinstance(replay, Limited):
|
||||
return limited_exit(TOOL, replay)
|
||||
print_report(folder, report(project.model, reference, cases, replay))
|
||||
print_report(folder, report(project.rung.model, reference, cases, replay))
|
||||
return 0
|
||||
|
||||
|
||||
def model_argument(ladder: Ladder, argv: list[str]) -> Rung | None:
|
||||
def model_argument(ladder: Ladder, argv: list[str]) -> ProjectRung | None:
|
||||
"""The project's rung on ladder of the one model argv names; None for any other argv."""
|
||||
return ladder.project_rung(argv[0]) if len(argv) == 1 else None
|
||||
|
||||
|
||||
def climbs(ladder: Ladder, project: Rung, reference: Rung) -> list[Rung]:
|
||||
def climbs(project: ProjectRung, reference: Rung) -> list[Rung]:
|
||||
"""The rungs above the project's rung that an E + I raises a review to, except the reference's, lowest first."""
|
||||
return sorted({ladder.review_floor(project, score) for score in REVIEW_STEPS} - {project, reference}, key=ladder.position)
|
||||
return sorted({project.review_floor(score) for score in REVIEW_STEPS} - {project.rung, reference}, key=project.ladder.position)
|
||||
|
||||
|
||||
def archived_cases(model: str, root: pathlib.Path, home: pathlib.Path) -> list[Case] | Failure:
|
||||
|
||||
@@ -5,8 +5,8 @@ from model_routing.route import ESCALATION_MARKER, Route
|
||||
from loop_claude.routing import escalation_rungs
|
||||
from loop_claude.project.records import HUMAN_RULE_NAMES, RECORD_FILES
|
||||
from review_rules.review_passes import INSTRUCTION_FILES, INSTRUCTION_FOLDERS
|
||||
from model_routing.ladder import Ladder, Rung
|
||||
from text_files.markdown_text import listed, quoted
|
||||
from model_routing.ladder import ProjectRung, Rung
|
||||
from claude_code.transcript import Typed
|
||||
|
||||
# Where the judge may quote an answer from, named as the human and the worker see them; of the goals file the judge sees only the premises.
|
||||
@@ -29,17 +29,17 @@ NAME_THE_ITEMS = f"Name every plan item this chunk takes on one line, as in `{CH
|
||||
AFTER_CODE_HEALTH = 'which rank after code health (~/.claude/CLAUDE.md → "Shared goal") unless the goals file voids it'
|
||||
|
||||
|
||||
def fresh(task: str, premises: str, route: Route, ladder: Ladder, project: Rung | None, facts: str) -> str:
|
||||
def fresh(task: str, premises: str, route: Route, project: ProjectRung | None, facts: str) -> str:
|
||||
"""A chunk's first prompt; facts is preflight_facts' or preflight_failed's text."""
|
||||
routed = f"{routed_by(route, ladder, project)} {recount(route)}" if route.item else " ".join(rung_sentences(route, ladder, project))
|
||||
routed = f"{routed_by(route, project)} {recount(route)}" if route.item else " ".join(rung_sentences(route, project))
|
||||
opening = f"{routed}\n\n" if routed else ""
|
||||
return f"{task}\n\nContinue any open draft PR or worktree for the chunk before creating a new one. {READING_RULE} {NAME_THE_ITEMS}\n\n{opening}{facts}\n\nThe chunk's plan items, or else the task above, set its scope. The goals below, {AFTER_CODE_HEALTH}, choose between ways to deliver that scope, and never widen it.\n\nFrom the goals file, {premises}"
|
||||
|
||||
|
||||
def routed_by(route: Route, ladder: Ladder, project: Rung | None) -> str:
|
||||
def routed_by(route: Route, project: ProjectRung | None) -> str:
|
||||
return " ".join((
|
||||
f"loop-claude runs you on {route.rung}, routed by item {route.item}'s L of {route.likelihood}.",
|
||||
*rung_sentences(route, ladder, project),
|
||||
*rung_sentences(route, project),
|
||||
))
|
||||
|
||||
|
||||
@@ -63,12 +63,12 @@ def preflight_failed(reason: str) -> str:
|
||||
return f"loop-claude could not run `~/.claude/bin/chunk-preflight`, so find the chunk's facts yourself. Why it could not run: {reason}"
|
||||
|
||||
|
||||
def review_launches(route: Route, ladder: Ladder, project: Rung | None) -> list[str]:
|
||||
def review_launches(route: Route, project: ProjectRung | None) -> list[str]:
|
||||
"""The sentences naming where to launch reviews, in this order, each from the function that says when it applies; none off the ladder."""
|
||||
if project is None:
|
||||
return []
|
||||
choice = chosen(ladder, project, route)
|
||||
return [*shared_launches(choice, route, project), *scan_launch(choice, route), *planning_launch(route, ladder, project), *taken_in_launches(choice), *markdown_launch(choice)]
|
||||
choice = chosen(project, route)
|
||||
return [*shared_launches(choice, route, project), *scan_launch(choice, route), *planning_launch(route, project), *taken_in_launches(choice), *markdown_launch(choice)]
|
||||
|
||||
|
||||
def scan_launch(choice: ReviewChoice, route: Route) -> list[str]:
|
||||
@@ -76,9 +76,9 @@ def scan_launch(choice: ReviewChoice, route: Route) -> list[str]:
|
||||
return ["Run a full scan only from such a subagent, since the scan's agents run on the model that starts them."] if choice.security != route.rung else []
|
||||
|
||||
|
||||
def planning_launch(route: Route, ladder: Ladder, project: Rung) -> list[str]:
|
||||
def planning_launch(route: Route, project: ProjectRung) -> list[str]:
|
||||
"""Where the worker runs below the project's rung, the sentence naming where to launch a subagent that files or scores plan items. On an Opus 5.5 project: `Your rung is below the project's: launch each subagent that files or scores plan items on the rung `~/.claude/bin/subagent-rung --plan claude-opus-5-5 <L>` prints.`"""
|
||||
return [f"Your rung is below the project's: launch each subagent that files or scores plan items on the rung `~/.claude/bin/subagent-rung --plan {project.model} <L>` prints."] if ladder.position(route.rung) < ladder.position(project) else []
|
||||
return [f"Your rung is below the project's: launch each subagent that files or scores plan items on the rung `~/.claude/bin/subagent-rung --plan {project.rung.model} <L>` prints."] if project.ladder.position(route.rung) < project.ladder.position(project.rung) else []
|
||||
|
||||
|
||||
def markdown_launch(choice: ReviewChoice) -> list[str]:
|
||||
@@ -90,7 +90,7 @@ def markdown_launch(choice: ReviewChoice) -> list[str]:
|
||||
return [f"Where your branch changes only Markdown, none of it {files} or under {folders}, launch {EACH_REVIEWER} on the project's rung instead, with the Agent tool set to {choice.markdown_only.agent_args()}."]
|
||||
|
||||
|
||||
def shared_launches(choice: ReviewChoice, route: Route, project: Rung) -> list[str]:
|
||||
def shared_launches(choice: ReviewChoice, route: Route, project: ProjectRung) -> list[str]:
|
||||
"""A sentence for each rung, other than the worker's, that the reviewers, the seats or the security pass run on.
|
||||
|
||||
For a worker one rung below the project's, routed by item 3 of E + I 5, they read: `Launch each reviewer with the Agent tool set to <rung>, the rung item 3's E + I of 5 picks.` and `Launch each comprehension seat and the security pass's subagent with the Agent tool set to <rung>, the project's rung.`
|
||||
@@ -114,28 +114,28 @@ def taken_in_launches(choice: ReviewChoice) -> list[str]:
|
||||
return [f"Where an item you take into the chunk has an E + I of {min(choice.higher.values())} or more, launch {EACH_REVIEWER} with the Agent tool set to {listed(picks, 'and')}."]
|
||||
|
||||
|
||||
def rung_named(rung: Rung, route: Route, project: Rung) -> str:
|
||||
def rung_named(rung: Rung, route: Route, project: ProjectRung) -> str:
|
||||
"""Why a review runs on rung: it is the project's, or the one the route's reviewed_on E + I picks, the item's or one taken in."""
|
||||
if rung == project:
|
||||
if rung == project.rung:
|
||||
return "the project's rung"
|
||||
if route.escape_and_impact_taken_in:
|
||||
return f"the rung picked by an item of E + I {route.escape_and_impact_taken_in} taken into the chunk"
|
||||
return f"the rung item {route.item}'s E + I of {route.escape_and_impact} picks"
|
||||
|
||||
|
||||
def escalation(route: Route, ladder: Ladder, project: Rung | None) -> list[str]:
|
||||
def escalation(route: Route, project: ProjectRung | None) -> list[str]:
|
||||
"""The prompt's escalation sentences, the marker's among them. None off the ladder or on its top rung. Where escalation_rungs opens none, the worker escalates only with the human's yes.
|
||||
Else they name the rungs escalation_rungs opens to a subagent, and offer to resume the worker one rung up for the rest of the chunk, up to the highest of those rungs.
|
||||
|
||||
For a worker one rung above the project's, routed by item 3 of L 4, they read: `When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `<rung>`, one rung above yours, and on each further trigger one rung higher, up to `<rung>`; above that, ask the human first.`, `Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.` and `Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `<rung>`.`
|
||||
For a worker one rung above the project's, routed by item 3 of L 3, they read: `When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `<rung>`, one rung above yours, and on each further trigger one rung higher, up to `<rung>`; above that, ask the human first.`, `Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.` and `Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `<rung>`.`
|
||||
"""
|
||||
if project is None or route.rung not in ladder.rungs[:-1]:
|
||||
if project is None or route.rung not in project.ladder.rungs[:-1]:
|
||||
return []
|
||||
marked = f"Open the subagent's prompt with `{ESCALATION_MARKER} <n>:`, n the trigger's number."
|
||||
rungs = [f"`{rung.agent_args()}`" for rung in escalation_rungs(ladder, project, route)]
|
||||
rungs = [f"`{rung.agent_args()}`" for rung in escalation_rungs(project, route)]
|
||||
if not rungs:
|
||||
return [
|
||||
f"An escalation trigger in `~/.claude/CLAUDE.md` lets you hand the hard part of a task to a subagent with the Agent tool set to `{ladder.climbed(route.rung, 1).agent_args()}`, one rung above yours, only once the human agrees; loop-claude will not resume you higher for a trigger either, so ask the human for that yes.",
|
||||
f"An escalation trigger in `~/.claude/CLAUDE.md` lets you hand the hard part of a task to a subagent with the Agent tool set to `{project.ladder.climbed(route.rung, 1).agent_args()}`, one rung above yours, only once the human agrees; loop-claude will not resume you higher for a trigger either, so ask the human for that yes.",
|
||||
marked,
|
||||
]
|
||||
further = f", and on each further trigger one rung higher, up to {rungs[-1]}" if len(rungs) > 1 else ""
|
||||
@@ -146,16 +146,16 @@ def escalation(route: Route, ladder: Ladder, project: Rung | None) -> list[str]:
|
||||
]
|
||||
|
||||
|
||||
def reruns(route: Route, ladder: Ladder, project: Rung | None) -> list[str]:
|
||||
def reruns(route: Route, project: ProjectRung | None) -> list[str]:
|
||||
"""The sentence naming the rung one above a reviewer's and the one above a seat's, on which each reruns when it asks, leaving out either that is already on the top rung. None off the ladder.
|
||||
|
||||
For a worker one rung below the project's, routed by an item of E + I 4, it reads: `A reviewer or comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to `<rung>`.`
|
||||
"""
|
||||
if project is None or route.rung not in ladder.rungs:
|
||||
if project is None or route.rung not in project.ladder.rungs:
|
||||
return []
|
||||
choice = chosen(ladder, project, route)
|
||||
choice = chosen(project, route)
|
||||
launched = (("reviewer", choice.reviewers), ("comprehension seat", choice.seat))
|
||||
rerun = {name: ladder.climbed(rung, 1) for name, rung in launched if rung != ladder.rungs[-1]}
|
||||
rerun = {name: project.ladder.climbed(rung, 1) for name, rung in launched if rung != project.ladder.rungs[-1]}
|
||||
if not rerun:
|
||||
return []
|
||||
if len(set(rerun.values())) == 1:
|
||||
@@ -163,31 +163,31 @@ def reruns(route: Route, ladder: Ladder, project: Rung | None) -> list[str]:
|
||||
return [f"A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `{rerun['reviewer'].agent_args()}`, and a comprehension seat with `{rerun['comprehension seat'].agent_args()}`."]
|
||||
|
||||
|
||||
def resumed(unfinished_evidence: str, route: Route, ladder: Ladder, project: Rung | None) -> str:
|
||||
def resumed(unfinished_evidence: str, route: Route, project: ProjectRung | None) -> str:
|
||||
"""Names the worker's rung, so it learns whether its request for a higher rung went through, and the project's again, since a step-up between cycles may have taken the worker above it."""
|
||||
on = f" loop-claude runs you on {route.rung}." if project else ""
|
||||
ruled = f" A judge ruled the chunk unfinished, quoting this passage of your transcript as the work remaining:\n\n{quoted(unfinished_evidence)}\n\n" if unfinished_evidence else " "
|
||||
return f"Continue the task; the previous run ended before the chunk was finished.{ruled}The chunk is finished once it is merged, or nothing remains that you could do on your own. Even then, it is finished only once {ANSWERS_RECORDED}. {NAME_THE_ITEMS}{on}{rungs_held(route, ladder, project)}"
|
||||
return f"Continue the task; the previous run ended before the chunk was finished.{ruled}The chunk is finished once it is merged, or nothing remains that you could do on your own. Even then, it is finished only once {ANSWERS_RECORDED}. {NAME_THE_ITEMS}{on}{rungs_held(route, project)}"
|
||||
|
||||
|
||||
def rung_sentences(route: Route, ladder: Ladder, project: Rung | None) -> list[str]:
|
||||
def rung_sentences(route: Route, project: ProjectRung | None) -> list[str]:
|
||||
"""The sentences that depend on the worker's rung: where its reviews run and rerun, and where its escalated subagents run."""
|
||||
return [*review_launches(route, ladder, project), *escalation(route, ladder, project), *reruns(route, ladder, project)]
|
||||
return [*review_launches(route, project), *escalation(route, project), *reruns(route, project)]
|
||||
|
||||
|
||||
def rungs_held(route: Route, ladder: Ladder, project: Rung | None) -> str:
|
||||
def rungs_held(route: Route, project: ProjectRung | None) -> str:
|
||||
"""rung_sentences, each after a space, to follow the sentence a prompt ends on."""
|
||||
return "".join(f" {sentence}" for sentence in rung_sentences(route, ladder, project))
|
||||
return "".join(f" {sentence}" for sentence in rung_sentences(route, project))
|
||||
|
||||
|
||||
def answered(question: str, answer: str, route: Route, ladder: Ladder, project: Rung | None, stepped: bool) -> str:
|
||||
def answered(question: str, answer: str, route: Route, project: ProjectRung | None, stepped: bool) -> str:
|
||||
"""stepped is whether the answer's ruling stepped the worker up."""
|
||||
return f"Your question: {question}\n\n{QUOTE_SOURCES} answers it:\n\n{answer}\n\nContinue the task.{stepped_to(route, ladder, project, stepped)}"
|
||||
return f"Your question: {question}\n\n{QUOTE_SOURCES} answers it:\n\n{answer}\n\nContinue the task.{stepped_to(route, project, stepped)}"
|
||||
|
||||
|
||||
def typed(question: str, answer: str, route: Route, ladder: Ladder, project: Rung | None, stepped: bool) -> str:
|
||||
def typed(question: str, answer: str, route: Route, project: ProjectRung | None, stepped: bool) -> str:
|
||||
"""The prompt that resumes the worker with the human's answer to an interrupt. answer is one line, as the terminal reads it."""
|
||||
return f"{TYPED_HERE}\n{answer}\n\nYour question was: {question}\n\nContinue the task. The chunk is finished only once {ANSWERS_RECORDED}.{stepped_to(route, ladder, project, stepped)}"
|
||||
return f"{TYPED_HERE}\n{answer}\n\nYour question was: {question}\n\nContinue the task. The chunk is finished only once {ANSWERS_RECORDED}.{stepped_to(route, project, stepped)}"
|
||||
|
||||
|
||||
# Opens typed's prompt, with the answer on the line after it, so typed_answer finds the answer at the prompt's start, whatever the question holds.
|
||||
@@ -203,9 +203,9 @@ def typed_answer(prompt: str) -> Typed | None:
|
||||
return Typed(rest, answer)
|
||||
|
||||
|
||||
def stepped_to(route: Route, ladder: Ladder, project: Rung | None, stepped: bool) -> str:
|
||||
def stepped_to(route: Route, project: ProjectRung | None, stepped: bool) -> str:
|
||||
"""The sentences naming the worker's new rung, as resumed has them, where the ruling that brought the answer stepped the worker up; empty otherwise."""
|
||||
return f" Your request for a higher rung went through: loop-claude runs you on {route.rung}. The escalation triggers that fired before this step are spent.{rungs_held(route, ladder, project)}" if stepped else ""
|
||||
return f" Your request for a higher rung went through: loop-claude runs you on {route.rung}. The escalation triggers that fired before this step are spent.{rungs_held(route, project)}" if stepped else ""
|
||||
|
||||
|
||||
def past_the_cap(question: str, answer: str, cap: int) -> str:
|
||||
|
||||
@@ -13,12 +13,12 @@ from claude_code.entries import Entry
|
||||
from failure import Failure
|
||||
from loop_claude.verdicts import Finished, Interrupt, Judged, QuotedAnswer, Unfinished, Verdict
|
||||
from loop_claude.cycle.interrupt import interrupt, Talked, WorkerSession
|
||||
from loop_claude.routing import judge_rung, next_route, step_up, StepUp
|
||||
from transcript_archive.judge_run import JudgeRun
|
||||
from model_routing.ladder import Ladder, Rung
|
||||
from transcript_archive.read import last_spent
|
||||
from loop_claude.routing import next_route, step_up, StepUp
|
||||
from transcript_archive import outcomes
|
||||
from plan_store.plans import Plan
|
||||
from model_routing.ladder import ProjectRung, Rung
|
||||
from loop_claude.cycle import prompts
|
||||
from claude_code.transcript import read_transcript
|
||||
from loop_claude.cycle.retry import retried
|
||||
@@ -40,12 +40,11 @@ class Cycle:
|
||||
effects: Effects
|
||||
# What chunk-preflight found, or why it could not run, for a fresh worker's prompt; empty when the session was open as the cycle began.
|
||||
facts: str
|
||||
judge_model: str
|
||||
# The project's routing ladder, as model_routing/ladder_file.py read it when the cycle started.
|
||||
ladder: Ladder
|
||||
# The rung routing.judge_rung picks for the project's judge model.
|
||||
judge: Rung
|
||||
premises: str
|
||||
# The project's rung; None for a model off the ladder.
|
||||
project: Rung | None
|
||||
# The project's rung on the ladder that model_routing/ladder_file.py read as the cycle started; None for a model off the ladder.
|
||||
project: ProjectRung | None
|
||||
project_archive: pathlib.Path
|
||||
records: str
|
||||
root: str
|
||||
@@ -85,7 +84,7 @@ def run_cycle(cycle: Cycle) -> Stop | Concluded:
|
||||
|
||||
def ruling_step_up(cycle: Cycle, verdict: Verdict) -> StepUp | None:
|
||||
"""The step-up an interrupt's or a quoted answer's escalation makes, so the work after the answer runs on the stepped route; None where the route holds. A finished or unfinished verdict steps the route only as the loop starts the next cycle."""
|
||||
return step_up(cycle.route, next_route(cycle.ladder, cycle.project, cycle.route, verdict)) if isinstance(verdict, Interrupt | QuotedAnswer) else None
|
||||
return step_up(cycle.route, next_route(cycle.project, cycle.route, verdict)) if isinstance(verdict, Interrupt | QuotedAnswer) else None
|
||||
|
||||
|
||||
def stopped(cycle: Cycle, stop: Stop, why: str) -> Stop:
|
||||
@@ -104,11 +103,6 @@ def archived(project_archive: pathlib.Path, transcript_path: str, task: str, out
|
||||
return archive(project_archive, transcript_path, task, outcome, effects.now(), lambda text: say(text, effects), judge_run)
|
||||
|
||||
|
||||
def judge_of(cycle: Cycle) -> Rung:
|
||||
"""The judge's rung, which the archive names beside each verdict."""
|
||||
return judge_rung(cycle.ladder, cycle.judge_model)
|
||||
|
||||
|
||||
def step(cycle: Cycle, state: Active) -> Event | Stop:
|
||||
match state:
|
||||
case Working(answered, _, stepped):
|
||||
@@ -144,12 +138,12 @@ def worker_session(cycle: Cycle, transcript_path: str) -> WorkerSession:
|
||||
|
||||
def worker_start(cycle: Cycle, answered: QuotedAnswer | TypedAnswer | None, stepped: bool) -> list[str]:
|
||||
if isinstance(answered, QuotedAnswer):
|
||||
return worker_args(cycle.session, cycle.route.rung, prompts.answered(answered.question, answered.answer, cycle.route, cycle.ladder, cycle.project, stepped), resume=True)
|
||||
return worker_args(cycle.session, cycle.route.rung, prompts.answered(answered.question, answered.answer, cycle.route, cycle.project, stepped), resume=True)
|
||||
if isinstance(answered, TypedAnswer):
|
||||
return worker_args(cycle.session, cycle.route.rung, prompts.typed(answered.question, answered.answer, cycle.route, cycle.ladder, cycle.project, stepped), resume=True)
|
||||
return worker_args(cycle.session, cycle.route.rung, prompts.typed(answered.question, answered.answer, cycle.route, cycle.project, stepped), resume=True)
|
||||
if session_open(cycle.session):
|
||||
return worker_args(cycle.session, cycle.route.rung, prompts.resumed(cycle.unfinished_evidence, cycle.route, cycle.ladder, cycle.project), resume=True)
|
||||
return worker_args(cycle.session, cycle.route.rung, prompts.fresh(cycle.task, cycle.premises, cycle.route, cycle.ladder, cycle.project, cycle.facts), resume=False)
|
||||
return worker_args(cycle.session, cycle.route.rung, prompts.resumed(cycle.unfinished_evidence, cycle.route, cycle.project), resume=True)
|
||||
return worker_args(cycle.session, cycle.route.rung, prompts.fresh(cycle.task, cycle.premises, cycle.route, cycle.project, cycle.facts), resume=False)
|
||||
|
||||
|
||||
def rule(cycle: Cycle, transcript_path: str, quoted_answers: int) -> Ruled | Stop:
|
||||
@@ -157,7 +151,7 @@ def rule(cycle: Cycle, transcript_path: str, quoted_answers: int) -> Ruled | Sto
|
||||
entries = read_transcript(transcript_path)
|
||||
if isinstance(entries, Failure):
|
||||
return Stop(1, entries.reason)
|
||||
verdict = judge(cycle.root, Brief(cycle.task, cycle.premises, cycle.records, entries, cycle.route.escalated_at), judge_of(cycle), cycle.effects, lambda rejected: keep_rejection(cycle, transcript_path, rejected))
|
||||
verdict = judge(cycle.root, Brief(cycle.task, cycle.premises, cycle.records, entries, cycle.route.escalated_at), cycle.judge, cycle.effects, lambda rejected: keep_rejection(cycle, transcript_path, rejected))
|
||||
if isinstance(verdict, Stop):
|
||||
return verdict
|
||||
if not isinstance(verdict, Finished):
|
||||
@@ -181,20 +175,20 @@ def annotation_of(cycle: Cycle, transcript_path: str, entries: list[Entry], judg
|
||||
subagents = spend.subagents(transcript_path)
|
||||
if isinstance(subagents, Failure):
|
||||
return subagents
|
||||
spent = spend.spent(since, entries, subagents, judge_of(cycle).model, judged.tokens)
|
||||
spent = spend.spent(since, entries, subagents, cycle.judge.model, judged.tokens)
|
||||
return outcomes.Annotation(cycle.config, outcomes.spent_block(since, spent), judged.reversed)
|
||||
|
||||
|
||||
def ruling_outcome(cycle: Cycle, ruling: Interrupt | QuotedAnswer, quoted_answers: int, annotation: outcomes.Annotation) -> str:
|
||||
"""The archive's outcome for a ruling: interrupted where it goes to the human, with the question put to them, and else answered from the premises or the records."""
|
||||
if isinstance(ruling, QuotedAnswer) and not to_the_human(ruling, quoted_answers):
|
||||
return outcomes.answered(question=ruling.question, sources=prompts.QUOTE_SOURCES, answer=ruling.answer, evidence=ruling.evidence, escalation=ruling.escalation, judge=str(judge_of(cycle)), worker=cycle.route.worker_line(), judge_cost=ruling.judged.cost, annotation=annotation, judge_run=ruling.judged.run)
|
||||
return outcomes.interrupted(question_for(ruling), ruling.evidence, str(judge_of(cycle)), ruling.escalation, cycle.route.worker_line(), judge_cost=ruling.judged.cost, annotation=annotation, judge_run=ruling.judged.run)
|
||||
return outcomes.answered(question=ruling.question, sources=prompts.QUOTE_SOURCES, answer=ruling.answer, evidence=ruling.evidence, escalation=ruling.escalation, judge=str(cycle.judge), worker=cycle.route.worker_line(), judge_cost=ruling.judged.cost, annotation=annotation, judge_run=ruling.judged.run)
|
||||
return outcomes.interrupted(question_for(ruling), ruling.evidence, str(cycle.judge), ruling.escalation, cycle.route.worker_line(), judge_cost=ruling.judged.cost, annotation=annotation, judge_run=ruling.judged.run)
|
||||
|
||||
|
||||
def keep_rejection(cycle: Cycle, transcript_path: str, rejected: Rejected) -> Stop | None:
|
||||
"""Archives a judge reply loop-claude rejected, so the archive shows what the judge returned before a retry or an exit 1."""
|
||||
if unkept := archived(cycle.project_archive, transcript_path, cycle.task, outcomes.rejected(rejected.reason, rejected.reply, str(judge_of(cycle)), judge_cost=rejected.cost, judge_run=rejected.run), cycle.effects, rejected.run):
|
||||
if unkept := archived(cycle.project_archive, transcript_path, cycle.task, outcomes.rejected(rejected.reason, rejected.reply, str(cycle.judge), judge_cost=rejected.cost, judge_run=rejected.run), cycle.effects, rejected.run):
|
||||
return Stop(1, unkept.reason)
|
||||
return None
|
||||
|
||||
@@ -237,7 +231,7 @@ def close(cycle: Cycle, transcript_path: str, verdict: Finished | Unfinished) ->
|
||||
annotation = entries if isinstance(entries, Failure) else annotation_of(cycle, transcript_path, entries, verdict.judged)
|
||||
if isinstance(annotation, Failure):
|
||||
return Stop(1, annotation.reason)
|
||||
outcome = outcomes.finished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle)), judge_cost=verdict.judged.cost, annotation=annotation, judge_run=verdict.judged.run) if isinstance(verdict, Finished) else outcomes.unfinished(cycle.route.worker_line(), verdict.evidence, str(judge_of(cycle)), verdict.escalation, judge_cost=verdict.judged.cost, annotation=annotation, judge_run=verdict.judged.run)
|
||||
outcome = outcomes.finished(cycle.route.worker_line(), verdict.evidence, str(cycle.judge), judge_cost=verdict.judged.cost, annotation=annotation, judge_run=verdict.judged.run) if isinstance(verdict, Finished) else outcomes.unfinished(cycle.route.worker_line(), verdict.evidence, str(cycle.judge), verdict.escalation, judge_cost=verdict.judged.cost, annotation=annotation, judge_run=verdict.judged.run)
|
||||
if failure := archived(cycle.project_archive, transcript_path, cycle.task, outcome, cycle.effects, verdict.judged.run):
|
||||
return Stop(1, failure.reason)
|
||||
if isinstance(verdict, Finished) and (failure := delete_session(cycle.session)):
|
||||
|
||||
@@ -10,11 +10,11 @@ from loop_claude.routing import first_route, kept, routed_item
|
||||
from loop_claude.loop.chunk_windows import held_beside, item_of
|
||||
from chunk_preflight.main import ITEM
|
||||
from plan.todo_table import Item
|
||||
from model_routing.ladder import Ladder, Rung
|
||||
from transcript_archive.read import latest_worker
|
||||
from project_config.main import located
|
||||
from loop_claude.loop.this_loop import Loop
|
||||
from plan_store.plans import Plan
|
||||
from model_routing.ladder import ProjectRung, Rung
|
||||
from loop_claude.cycle import prompts
|
||||
from model_routing.route import Route, route_from
|
||||
from claude_code.session import session_open, transcript_file
|
||||
@@ -31,15 +31,15 @@ def scored_plan(default: DefaultPlan | Failure, effects: Effects) -> Plan | Fail
|
||||
return plan
|
||||
|
||||
|
||||
def routed(route: Route | None, ladder: Ladder, project: Rung | None, model: str, archived: typing.Callable[[], Route | None], item: typing.Callable[[], Item | None]) -> Route:
|
||||
"""The route the cycle runs on. project is the project's rung on ladder, and model the project's model.
|
||||
def routed(route: Route | None, project: ProjectRung | None, model: str, archived: typing.Callable[[], Route | None], item: typing.Callable[[], Item | None]) -> Route:
|
||||
"""The route the cycle runs on. project is the project's rung, and model the project's model.
|
||||
|
||||
Where project is None, the model is off the ladder, and the worker runs on it. Otherwise the chunk keeps route, or on a rerun the route its archive last names, as routing.kept allows. Else the plan item routes the chunk afresh.
|
||||
"""
|
||||
if project is None:
|
||||
return Route(Rung(model))
|
||||
route = route or archived()
|
||||
return route if route and kept(route, ladder, project) else first_route(ladder, project, item())
|
||||
return route if route and kept(route, project) else first_route(project, item())
|
||||
|
||||
|
||||
def archived_route(archive_folder: pathlib.Path, loop: Loop) -> Route | None:
|
||||
|
||||
@@ -4,20 +4,20 @@ import pathlib
|
||||
|
||||
from loop_claude.cycle.run import archived_step, archived_stop, Concluded
|
||||
from loop_claude.verdicts import Finished
|
||||
from model_routing.ladder import Ladder, Rung
|
||||
from loop_claude.loop.this_loop import Loop
|
||||
from loop_claude.loop.reload import LoopRun
|
||||
from loop_claude.stop import needs_human, Stop
|
||||
from loop_claude.routing import next_route, step_up
|
||||
from model_routing.ladder import ProjectRung
|
||||
from claude_code.session import transcript_file
|
||||
|
||||
# How many unfinished verdicts running end the loop, so a worker and a judge that disagree about a chunk cannot cycle unattended without end. A policy number, not derived from the routing steps: well above the two unfinished verdicts running that archived chunks have reached at most, and low enough that a stuck pair is stopped after six cycles. It counts every unfinished verdict, including those of cycles where the human answered an interrupt: a rerun is cheap, and a chunk that needs that many answers is worth a look. The human confirmed both the number and what it counts.
|
||||
UNFINISHED_LIMIT = 6
|
||||
|
||||
|
||||
def closed_cycle(loop_run: LoopRun, ladder: Ladder, project: Rung | None, concluded: Concluded, archive_folder: pathlib.Path, loop: Loop) -> LoopRun | Stop:
|
||||
def closed_cycle(loop_run: LoopRun, project: ProjectRung | None, concluded: Concluded, archive_folder: pathlib.Path, loop: Loop) -> LoopRun | Stop:
|
||||
"""The loop run after a cycle concluded, with the step-up archived; the stop that needs the human, archived, once the chunk has had UNFINISHED_LIMIT unfinished verdicts running."""
|
||||
after = after_verdict(loop_run, ladder, project, concluded)
|
||||
after = after_verdict(loop_run, project, concluded)
|
||||
# A finished chunk drops its route, so it has no step-up, and its run of unfinished verdicts restarts.
|
||||
if after.route is None:
|
||||
return after
|
||||
@@ -38,10 +38,10 @@ def reached_limit(loop_run: LoopRun) -> Stop | None:
|
||||
return needs_human("go on with the chunk", f"the judge ruled it unfinished {loop_run.unfinished_run} cycles running; finish the chunk yourself, or rerun loop-claude to give it {UNFINISHED_LIMIT} more cycles")
|
||||
|
||||
|
||||
def after_verdict(loop_run: LoopRun, ladder: Ladder, project: Rung | None, concluded: Concluded) -> LoopRun:
|
||||
def after_verdict(loop_run: LoopRun, project: ProjectRung | None, concluded: Concluded) -> LoopRun:
|
||||
"""The loop run the next cycle starts from. A finished chunk drops its route and its run of unfinished verdicts. An unfinished one takes the route routing.next_route makes of the route its cycle ended on, which an interrupt's or a quoted answer's escalation may have raised, and the evidence the judge gave for the verdict; and counts the verdict."""
|
||||
verdict = concluded.verdict
|
||||
if isinstance(verdict, Finished):
|
||||
return dataclasses.replace(loop_run, route=None, unfinished_evidence="", unfinished_run=0)
|
||||
route = next_route(ladder, project, concluded.route, verdict)
|
||||
route = next_route(project, concluded.route, verdict)
|
||||
return dataclasses.replace(loop_run, route=route, unfinished_evidence=verdict.evidence, unfinished_run=loop_run.unfinished_run + 1)
|
||||
|
||||
@@ -16,6 +16,7 @@ from loop_claude.verdicts import Finished
|
||||
from loop_claude.project.model_choice import get_or_ask_models
|
||||
from git_cli.git_repo import Git
|
||||
from loop_claude.loop.usage import HELP
|
||||
from loop_claude.routing import judge_rung
|
||||
from loop_claude.loop.this_loop import Loop
|
||||
from loop_state.paths import Paths
|
||||
from transcript_archive.project_key import project_key
|
||||
@@ -69,12 +70,12 @@ def run_loop(argv: list[str], env: typing.Mapping[str, str], paths: Paths, launc
|
||||
concluded = run_cycle(cycle)
|
||||
if isinstance(concluded, Stop):
|
||||
return concluded
|
||||
closed = closed_cycle(loop_run, cycle.ladder, cycle.project, concluded, cycle.project_archive, loop)
|
||||
closed = closed_cycle(loop_run, cycle.project, concluded, cycle.project_archive, loop)
|
||||
if isinstance(closed, Stop):
|
||||
return closed
|
||||
loop_run = closed
|
||||
# Off the ladder the worker runs on the project's model itself, which its route names.
|
||||
if stop := check_routing(archive_root(paths.home, env), cycle.project or cycle.route.rung, loop):
|
||||
if stop := check_routing(archive_root(paths.home, env), cycle.project.rung if cycle.project else cycle.route.rung, loop):
|
||||
return stop
|
||||
if isinstance(concluded.verdict, Finished):
|
||||
return handed_on(loop_run, loop)
|
||||
@@ -124,11 +125,11 @@ def next_cycle(loop_run: LoopRun, loop: Loop) -> Cycle | Stop:
|
||||
# After the reads above, so a loop that stops on one makes no worktree.
|
||||
facts, default = chunk_start(loop)
|
||||
project = ladder.project_rung(models.worker)
|
||||
route = routed(loop_run.route, ladder, project, models.worker, lambda: archived_route(archive_folder, loop), lambda: plan_item(default, item_of(loop.window.name), loop.effects))
|
||||
route = routed(loop_run.route, project, models.worker, lambda: archived_route(archive_folder, loop), lambda: plan_item(default, item_of(loop.window.name), loop.effects))
|
||||
say(f"\n=== cycle {loop_run.cycle_number}, worker on {route.rung} ===", loop.effects, stdout=True)
|
||||
if ladder_file != SHIPPED:
|
||||
say(own_ladder(ladder_file, models.worker, project is None), loop.effects)
|
||||
return Cycle(config=setup_version(loop.paths.claude_dir, loop.effects.git), effects=loop.effects, facts=facts, judge_model=models.judge, ladder=ladder, premises=premises, project=project, project_archive=archive_folder, records=records, root=loop.root, route=route, session=loop.session, task=loop_run.task, plan=scored_plan(default, loop.effects), unfinished_evidence=loop_run.unfinished_evidence, window=loop.window)
|
||||
return Cycle(config=setup_version(loop.paths.claude_dir, loop.effects.git), effects=loop.effects, facts=facts, judge=judge_rung(ladder, models.judge), premises=premises, project=project, project_archive=archive_folder, records=records, root=loop.root, route=route, session=loop.session, task=loop_run.task, plan=scored_plan(default, loop.effects), unfinished_evidence=loop_run.unfinished_evidence, window=loop.window)
|
||||
|
||||
|
||||
def own_ladder(file: pathlib.Path, model: str, unrouted: bool) -> str:
|
||||
|
||||
+32
-30
@@ -34,7 +34,7 @@ from loop_claude import escalation_mark
|
||||
from failure import Failure
|
||||
from loop_claude.verdicts import Interrupt, QuotedAnswer, Unfinished
|
||||
from plan.todo_table import Item, NoItemsTable
|
||||
from model_routing.ladder import Ladder, Rung
|
||||
from model_routing.ladder import Ladder, ProjectRung, Rung
|
||||
from plan_store.plans import Plan
|
||||
from model_routing.route import Route
|
||||
|
||||
@@ -59,42 +59,43 @@ def routed_item(plan: Plan | Failure, cut: LastCut, item_id: int | None = None)
|
||||
return takes if takes and takes.likelihood and takes.escape_and_impact else None
|
||||
|
||||
|
||||
def first_route(ladder: Ladder, project: Rung, item: Item | None) -> Route:
|
||||
"""The route a chunk starts on: the rung its item's L picks on ladder from the project's rung, or the project's rung without an item."""
|
||||
def first_route(project: ProjectRung, item: Item | None) -> Route:
|
||||
"""The route a chunk starts on: the rung its item's L picks from the project's rung, or the project's rung without an item."""
|
||||
if item is None:
|
||||
return Route(project)
|
||||
return Route(ladder.worker_rung(project, item.likelihood), item.id, item.likelihood, item.escape_and_impact)
|
||||
return Route(project.rung)
|
||||
return Route(project.worker_rung(item.likelihood), item.id, item.likelihood, item.escape_and_impact)
|
||||
|
||||
|
||||
def kept(route: Route, ladder: Ladder, project: Rung) -> bool:
|
||||
def kept(route: Route, project: ProjectRung) -> bool:
|
||||
"""Whether a chunk keeps route: its rung is on the ladder and at or above item_rung."""
|
||||
return floor_rung(ladder, project, route) == route.rung
|
||||
return floor_rung(project, route) == route.rung
|
||||
|
||||
|
||||
def item_rung(ladder: Ladder, project: Rung, route: Route) -> Rung:
|
||||
def item_rung(project: ProjectRung, route: Route) -> Rung:
|
||||
"""The rung the route's item's L picks from the project's rung, or the project's rung itself when no item routed it. The worker may run on a rung below the project's."""
|
||||
return ladder.worker_rung(project, route.likelihood) if route.item else project
|
||||
return project.worker_rung(route.likelihood) if route.item else project.rung
|
||||
|
||||
|
||||
def floor_rung(ladder: Ladder, project: Rung, route: Route, likelihood_taken_in: int = 0) -> Rung:
|
||||
def floor_rung(project: ProjectRung, route: Route, likelihood_taken_in: int = 0) -> Rung:
|
||||
"""The lowest rung the worker may run on: the route's own, raised to item_rung where it is lower or off the ladder, and to the rung likelihood_taken_in picks from project. likelihood_taken_in is the highest L the worker took into the chunk, 0 for none."""
|
||||
floor = ladder.highest(route.rung, item_rung(ladder, project, route))
|
||||
return ladder.highest(floor, ladder.worker_rung(project, likelihood_taken_in)) if likelihood_taken_in else floor
|
||||
floor = project.ladder.highest(route.rung, item_rung(project, route))
|
||||
return project.ladder.highest(floor, project.worker_rung(likelihood_taken_in)) if likelihood_taken_in else floor
|
||||
|
||||
|
||||
def escalation_rungs(ladder: Ladder, project: Rung, route: Route) -> tuple[Rung, ...]:
|
||||
def escalation_rungs(project: ProjectRung, route: Route) -> tuple[Rung, ...]:
|
||||
"""The rungs an escalation trigger opens to the worker, least competent first: from one above its own up to escalation_cap; none off the ladder."""
|
||||
ladder = project.ladder
|
||||
if route.rung not in ladder.rungs:
|
||||
return ()
|
||||
return ladder.rungs[ladder.position(route.rung) + 1:ladder.position(escalation_cap(ladder, project, route)) + 1]
|
||||
return ladder.rungs[ladder.position(route.rung) + 1:ladder.position(escalation_cap(project, route)) + 1]
|
||||
|
||||
|
||||
def escalation_cap(ladder: Ladder, project: Rung, route: Route) -> Rung:
|
||||
def escalation_cap(project: ProjectRung, route: Route) -> Rung:
|
||||
"""The highest rung an escalation trigger takes the worker or a task to without the human's yes: docs/decisions.md → `routing: a worker hands a task one rung up`.
|
||||
|
||||
It counts from item_rung, the rung the chunk started on, so a step-up between cycles narrows what the worker may climb without a yes, and never widens it.
|
||||
"""
|
||||
return ladder.climbed(item_rung(ladder, project, route), ESCALATION_CAP)
|
||||
return project.ladder.climbed(item_rung(project, route), ESCALATION_CAP)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
@@ -104,31 +105,32 @@ class StepUp:
|
||||
after: Route
|
||||
|
||||
|
||||
def next_route(ladder: Ladder, project: Rung | None, route: Route, verdict: Unfinished | Interrupt | QuotedAnswer) -> Route:
|
||||
def next_route(project: ProjectRung | None, route: Route, verdict: Unfinished | Interrupt | QuotedAnswer) -> Route:
|
||||
"""The route the worker runs on after the judge's verdict. Off the ladder, project is None, and the route holds. The rung never falls."""
|
||||
if project is None:
|
||||
return route
|
||||
if isinstance(verdict, Unfinished):
|
||||
return unfinished_route(ladder, project, route, verdict)
|
||||
return escalated_route(ladder, project, route, verdict)
|
||||
return unfinished_route(project, route, verdict)
|
||||
return escalated_route(project, route, verdict)
|
||||
|
||||
|
||||
def unfinished_route(ladder: Ladder, project: Rung, route: Route, verdict: Unfinished) -> Route:
|
||||
"""The route after an unfinished verdict. The rung rises to floor_rung, which the verdict's highest L taken in may raise.
|
||||
def unfinished_route(project: ProjectRung, route: Route, verdict: Unfinished) -> Route:
|
||||
"""The route after an unfinished verdict, as the module docstring's table says.
|
||||
|
||||
The verdict counts towards UNFINISHED_PER_STEP. Where steps says so, the rung steps once from the floor, and the mark moves only where the verdict escalates. Last, the route takes in the verdict's highest E + I, as Route.taking_in does.
|
||||
The rung rises to floor_rung, which the verdict's highest L taken in may raise. The verdict counts towards UNFINISHED_PER_STEP. Where steps says so, the rung steps once from the floor; the mark moves only where the verdict carries an escalation. Last, the route takes in the verdict's highest E + I, as Route.taking_in does.
|
||||
"""
|
||||
floor = floor_rung(project, route, verdict.taken_in.highest_likelihood)
|
||||
mark = escalation_mark.moved_to(verdict)
|
||||
floor = floor_rung(ladder, project, route, verdict.taken_in.highest_likelihood)
|
||||
escalates = mark is not None and below_cap(project, route, floor)
|
||||
counted = route.unfinished + 1
|
||||
if steps(counted, mark is not None and below_cap(ladder, project, route, floor)):
|
||||
after = one_rung_up(ladder, route, floor, route.escalated_at if mark is None else mark)
|
||||
if steps(counted, escalates):
|
||||
after = one_rung_up(project.ladder, route, floor, route.escalated_at if mark is None else mark)
|
||||
else:
|
||||
after = dataclasses.replace(route, rung=floor, unfinished=counted)
|
||||
return after.taking_in(verdict.taken_in.highest_escape_and_impact)
|
||||
|
||||
|
||||
def escalated_route(ladder: Ladder, project: Rung, route: Route, verdict: Interrupt | QuotedAnswer) -> Route:
|
||||
def escalated_route(project: ProjectRung, route: Route, verdict: Interrupt | QuotedAnswer) -> Route:
|
||||
"""The route after an interrupt or a quoted answer. Without an escalation it holds, since neither is an unfinished verdict.
|
||||
|
||||
With one, the rung rises to floor_rung, which the verdict's highest L taken in may raise. Where floor_rung lies below escalation_cap, the rung steps one up from it and the mark moves, as loop_claude/escalation_mark.py says; at the cap the rung stays on floor_rung and the mark holds.
|
||||
@@ -136,8 +138,8 @@ def escalated_route(ladder: Ladder, project: Rung, route: Route, verdict: Interr
|
||||
mark = escalation_mark.moved_to(verdict)
|
||||
if mark is None:
|
||||
return route
|
||||
floor = floor_rung(ladder, project, route, verdict.taken_in.highest_likelihood)
|
||||
return one_rung_up(ladder, route, floor, mark) if below_cap(ladder, project, route, floor) else dataclasses.replace(route, rung=floor)
|
||||
floor = floor_rung(project, route, verdict.taken_in.highest_likelihood)
|
||||
return one_rung_up(project.ladder, route, floor, mark) if below_cap(project, route, floor) else dataclasses.replace(route, rung=floor)
|
||||
|
||||
|
||||
def steps(counted: int, escalates: bool) -> bool:
|
||||
@@ -145,9 +147,9 @@ def steps(counted: int, escalates: bool) -> bool:
|
||||
return counted >= UNFINISHED_PER_STEP or escalates
|
||||
|
||||
|
||||
def below_cap(ladder: Ladder, project: Rung, route: Route, floor: Rung) -> bool:
|
||||
def below_cap(project: ProjectRung, route: Route, floor: Rung) -> bool:
|
||||
"""Whether an escalation from floor may step the rung: floor lies below escalation_cap."""
|
||||
return ladder.position(floor) < ladder.position(escalation_cap(ladder, project, route))
|
||||
return project.ladder.position(floor) < project.ladder.position(escalation_cap(project, route))
|
||||
|
||||
|
||||
def step_up(before: Route, after: Route) -> StepUp | None:
|
||||
|
||||
@@ -150,7 +150,7 @@ if os.path.basename(sys.argv[0]) == "loop-claude":
|
||||
"""
|
||||
|
||||
PREMISES = "# Project\n\n## Goals\n\n1. Work.\n\n## Audience\n\nMe.\n"
|
||||
# Item 4's L of 4 steps one rung up from the project's.
|
||||
# Item 4's L of 4 steps two rungs up from the project's.
|
||||
TODO = "## Items\n\n| ID | Release | Exempt | Item | L | E | I | R | S | Moves | Score |\n|---|---|---|---|---|---|---|---|---|---|---|\n| 4 | 0.1.0 | | **Do it.** | 4 | 2 | 3 | 7 | 2 | 1 by 9 | 25 |\n"
|
||||
# Item 4's L of 2 steps one rung down.
|
||||
CHEAP = TODO.replace("| 4 | 2 | 3 | 7 |", "| 2 | 1 | 2 | 3 |")
|
||||
@@ -497,15 +497,15 @@ class Cycles(unittest.TestCase):
|
||||
("claude-sonnet-5-5", "high"),
|
||||
("claude-sonnet-5-5", "high"),
|
||||
("claude-opus-5-5", "medium"),
|
||||
("claude-fable-5-2", "low"),
|
||||
("claude-fable-5-2", "medium"),
|
||||
("claude-sonnet-5-5", "high"),
|
||||
])
|
||||
self.assertEqual({(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "judge"}, {("judge-model", None)})
|
||||
self.assertIn("loop-claude runs you on claude-sonnet-5-5 at high effort, routed by item 4's L of 2.", argument(workers[0], "-p"))
|
||||
self.assertIn("on one line, as in `Chunk items: ", argument(workers[0], "-p"))
|
||||
outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values())
|
||||
self.assertIn('<UTC> Verdict: unfinished\n\nEvidence:\n> three\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-opus-5-5", "effort": "medium", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n\n<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n', outcomes)
|
||||
self.assertIn('<UTC> Verdict: finished\n\nEvidence:\n> four\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n', outcomes)
|
||||
self.assertIn('<UTC> Verdict: unfinished\n\nEvidence:\n> three\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-opus-5-5", "effort": "medium", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n\n<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n', outcomes)
|
||||
self.assertIn('<UTC> Verdict: finished\n\nEvidence:\n> four\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 2, "escape_and_impact": 3}\n', outcomes)
|
||||
|
||||
def test_routes_on_the_ladder_the_projects_config_folder_holds_in_place_of_the_shipped_one(self):
|
||||
checkout = Checkout(self)
|
||||
@@ -576,14 +576,14 @@ class Cycles(unittest.TestCase):
|
||||
self.assertEqual(run.code, 130)
|
||||
workers = [call for call in run.calls if call["role"] == "worker"]
|
||||
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in workers], [
|
||||
("claude-fable-5-2", "low"),
|
||||
("claude-fable-5-2", "medium"),
|
||||
("claude-opus-5-5", "xhigh"),
|
||||
("claude-opus-5-5", "xhigh"),
|
||||
("claude-fable-5-2", "medium"),
|
||||
("claude-fable-5-2", "low"),
|
||||
])
|
||||
outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values())
|
||||
self.assertIn('<UTC> Verdict: unfinished\n\nEvidence:\n> I ask for a higher rung.\n\nEscalation:\n> Stability round 2: a 🔴.\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 4, "escape_and_impact": 5}\n\n<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', outcomes)
|
||||
self.assertIn('<UTC> Verdict: unfinished\n\nEvidence:\n> I ask for a higher rung.\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', outcomes)
|
||||
self.assertIn('<UTC> Verdict: unfinished\n\nEvidence:\n> I ask for a higher rung.\n\nEscalation:\n> Stability round 2: a 🔴.\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5}\n\n<UTC> Stepped up\n\nWorker: {"model": "claude-opus-5-5", "effort": "xhigh", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', outcomes)
|
||||
self.assertIn('<UTC> Verdict: unfinished\n\nEvidence:\n> I ask for a higher rung.\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-opus-5-5", "effort": "xhigh", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', outcomes)
|
||||
|
||||
def test_steps_the_worker_up_once_as_the_judge_interrupts_when_the_run_also_cites_an_escalation_trigger(self):
|
||||
checkout = Checkout(self)
|
||||
@@ -597,11 +597,11 @@ class Cycles(unittest.TestCase):
|
||||
], terminal_input="/talk\n", before_typing=ANSWERING)
|
||||
self.assertEqual(run.code, 130)
|
||||
self.assertEqual(run.roles(), ["worker", "judge", "interrupt", "judge", "worker", "judge"])
|
||||
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] != "judge"], [("claude-fable-5-2", "low"), ("claude-fable-5-2", "medium"), ("claude-fable-5-2", "medium")])
|
||||
self.assertEqual([(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] != "judge"], [("claude-fable-5-2", "medium"), ("claude-opus-5-5", "xhigh"), ("claude-opus-5-5", "xhigh")])
|
||||
outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values())
|
||||
self.assertIn('<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', "".join(outcomes))
|
||||
self.assertIn('<UTC> Stepped up\n\nWorker: {"model": "claude-opus-5-5", "effort": "xhigh", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', "".join(outcomes))
|
||||
self.assertNotIn("at max effort", "".join(outcomes))
|
||||
self.assertIn('Verdict: interrupt\n\nA or B?\n\nEvidence:\n> Shall I use A or B?\n\nEscalation:\n> Stability round 2: a 🔴.\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "low", "item": 4, "likelihood": 4, "escape_and_impact": 5}\n\n<UTC> Stepped up\n\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', "".join(outcomes))
|
||||
self.assertIn('Verdict: interrupt\n\nA or B?\n\nEvidence:\n> Shall I use A or B?\n\nEscalation:\n> Stability round 2: a 🔴.\n\nConfig: <config>\nJudge: judge-model\nWorker: {"model": "claude-fable-5-2", "effort": "medium", "item": 4, "likelihood": 4, "escape_and_impact": 5}\n\n<UTC> Stepped up\n\nWorker: {"model": "claude-opus-5-5", "effort": "xhigh", "item": 4, "likelihood": 4, "escape_and_impact": 5, "escalated_at": 4}\n', "".join(outcomes))
|
||||
|
||||
def test_runs_a_cheap_items_worker_below_the_projects_rung_with_its_reviewers_on_it_and_its_judge_on_the_models_highest(self):
|
||||
checkout = Checkout(self)
|
||||
@@ -640,12 +640,12 @@ class Cycles(unittest.TestCase):
|
||||
checkout.commit_plan(TODO)
|
||||
run = checkout.run([worker(), judge(), worker(), judge()])
|
||||
self.assertEqual(run.code, 130)
|
||||
self.assertEqual((argument(run.calls[-1], "--model"), argument(run.calls[-1], "--effort")), ("claude-fable-5-2", "medium"))
|
||||
self.assertEqual((argument(run.calls[-1], "--model"), argument(run.calls[-1], "--effort")), ("claude-opus-5-5", "xhigh"))
|
||||
checkout.git("rm", "--quiet", "todo.md")
|
||||
checkout.git("commit", "--quiet", "-m", "drop the plan")
|
||||
run = checkout.run([worker()])
|
||||
self.assertIsNotNone(argument(run.calls[0], "--resume"))
|
||||
self.assertEqual((argument(run.calls[0], "--model"), argument(run.calls[0], "--effort")), ("claude-fable-5-2", "medium"))
|
||||
self.assertEqual((argument(run.calls[0], "--model"), argument(run.calls[0], "--effort")), ("claude-opus-5-5", "xhigh"))
|
||||
|
||||
def test_a_second_window_of_the_project_runs_its_own_session_and_leaves_the_first_ones_open_chunk_alone(self):
|
||||
checkout = Checkout(self)
|
||||
|
||||
+137
-132
@@ -38,10 +38,10 @@ from transcript_archive import outcomes, write
|
||||
from loop_state import paths
|
||||
from plan_store.todo_md import plan_of
|
||||
from transcript_archive.location import project_archive
|
||||
from model_routing.ladder import ProjectRung, Rung
|
||||
from model_routing.ladder_file import read_ladder, SHIPPED
|
||||
from model_routing.route import Route, route_from, WorkerLine
|
||||
from loop_claude import routing
|
||||
from model_routing.ladder import Rung
|
||||
from loop_claude.stop import Stop
|
||||
from loop_claude import tmux
|
||||
from claude_code.tokens import Tokens, tokens_of
|
||||
@@ -51,6 +51,7 @@ from loop_claude.cycle.judge import verdict_from
|
||||
|
||||
# How the archive records a worker that runs off the ladder.
|
||||
LADDER = read_ladder(SHIPPED)
|
||||
OPUS_PROJECT = ProjectRung(LADDER, Rung("claude-opus-5-5", "medium"))
|
||||
OFF_LADDER = '{"model": "worker-model"}'
|
||||
# The claude-config release and commit each cycle the tests run is under.
|
||||
CONFIG = "v0.8.0 (abc123)"
|
||||
@@ -289,7 +290,7 @@ class Cycles(unittest.TestCase):
|
||||
self.archive = project_archive(str(self.home / "code" / "proj"), self.home, {})
|
||||
|
||||
def cycle(self, fake, records="", unfinished_evidence="", facts="- A preflight fact."):
|
||||
return run.Cycle(config=CONFIG, window=WINDOW, effects=fake.effects(), facts=facts, judge_model="judge-model", ladder=LADDER, premises="## Goals", project=None, project_archive=self.archive, records=records, root=str(self.home), route=Route(Rung("worker-model")), session=self.session, task="Do it.", plan=plan_of(""), unfinished_evidence=unfinished_evidence)
|
||||
return run.Cycle(config=CONFIG, window=WINDOW, effects=fake.effects(), facts=facts, judge=Rung("judge-model"), premises="## Goals", project=None, project_archive=self.archive, records=records, root=str(self.home), route=Route(Rung("worker-model")), session=self.session, task="Do it.", plan=plan_of(""), unfinished_evidence=unfinished_evidence)
|
||||
|
||||
def archived_outcomes(self):
|
||||
"""Each outcomes file's text without its `Judge run:` lines, which name a file by the hash of a prompt and its reply: the tests of the judge's runs check them."""
|
||||
@@ -341,7 +342,7 @@ Worker: {OFF_LADDER}
|
||||
|
||||
def test_a_worker_is_handed_the_projects_rung_its_own_its_reviews_e_plus_i_and_its_compaction_window_in_its_environment(self):
|
||||
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("finished")])
|
||||
cycle = dataclasses.replace(self.cycle(fake), project=Rung("claude-opus-5-5", "medium"), route=Route(Rung("claude-haiku-5-5", "max"), 163, 1, 5))
|
||||
cycle = dataclasses.replace(self.cycle(fake), project=OPUS_PROJECT, route=Route(Rung("claude-haiku-5-5", "max"), 163, 1, 5))
|
||||
self.assertEqual(run.run_cycle(cycle).verdict, Finished("working"))
|
||||
self.assertEqual(fake.worker_environments, [{"LOOP_CLAUDE_PROJECT_RUNG": "claude-opus-5-5 at medium effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-haiku-5-5 at max effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "5", "CLAUDE_CODE_AUTO_COMPACT_WINDOW": "200000"}])
|
||||
|
||||
@@ -351,7 +352,7 @@ Worker: {OFF_LADDER}
|
||||
handle.write(json.dumps(human("the second")) + "\n")
|
||||
return 0
|
||||
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt"), judged("finished")], inputs=["/talk"], on_terminal_claude=[answer])
|
||||
cycle = dataclasses.replace(self.cycle(fake), project=Rung("claude-opus-5-5", "medium"), route=Route(Rung("claude-haiku-5-5", "max"), 163, 1, 5))
|
||||
cycle = dataclasses.replace(self.cycle(fake), project=OPUS_PROJECT, route=Route(Rung("claude-haiku-5-5", "max"), 163, 1, 5))
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
self.assertEqual(run.run_cycle(cycle).verdict, Finished("working"))
|
||||
self.assertEqual(fake.terminal_environments, [{**environment, "CLAUDE_CODE_SANDBOXED": "1"} for environment in fake.worker_environments])
|
||||
@@ -554,13 +555,13 @@ Spent since 2026-09-21T14:13:20Z, in tokens of input, cache write 5m, cache writ
|
||||
|
||||
def test_a_quoted_answer_that_cites_an_escalation_trigger_steps_the_worker_up_before_it_resumes_and_archives_the_step(self):
|
||||
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[judged("quoted", answer="Use A.", escalation="working"), judged("finished")])
|
||||
cycle = dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project=Routing.OPUS, route=Route(Routing.SONNET, 179, 2, 5))
|
||||
cycle = dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project=OPUS_PROJECT, route=Route(Routing.SONNET, 179, 2, 5))
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
concluded = run.run_cycle(cycle)
|
||||
self.assertEqual(concluded.route, Route(Routing.OPUS, 179, 2, 5, escalated_at=3))
|
||||
first, resumed = fake.made("stream_claude")
|
||||
self.assertEqual([args[args.index("--effort") + 1] for args in (first, resumed)], ["high", "medium"])
|
||||
self.assertTrue(resumed[resumed.index("-p") + 1].endswith(f"Continue the task. Your request for a higher rung went through: loop-claude runs you on {Routing.OPUS}. The escalation triggers that fired before this step are spent.{prompts.rungs_held(concluded.route, LADDER, Routing.OPUS)}"))
|
||||
self.assertTrue(resumed[resumed.index("-p") + 1].endswith(f"Continue the task. Your request for a higher rung went through: loop-claude runs you on {Routing.OPUS}. The escalation triggers that fired before this step are spent.{prompts.rungs_held(concluded.route, OPUS_PROJECT)}"))
|
||||
self.assertNotIn("went through", first[first.index("-p") + 1])
|
||||
answered = f"Verdict: quoted answer\n\nWhich one?\n\n{prompts.QUOTE_SOURCES} says:\n> Use A.\n\nEvidence:\n> working\n\nEscalation:\n> working\n\nConfig: {CONFIG}\nJudge: judge-model\nWorker: {SONNET_179}\n"
|
||||
stepped = f"Stepped up\n\nWorker: {OPUS_179_AT_3}\n"
|
||||
@@ -571,7 +572,7 @@ Spent since 2026-09-21T14:13:20Z, in tokens of input, cache write 5m, cache writ
|
||||
def test_a_quoted_answer_past_the_cap_archives_the_escalation_it_carried_with_the_interrupt_and_steps_up(self):
|
||||
steps = [judged("quoted", answer="Use A.")] * 3 + [judged("quoted", answer="Use A.", escalation="working")]
|
||||
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 4, claude=steps, inputs=[""])
|
||||
cycle = dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project=Routing.OPUS, route=Route(Routing.SONNET, 179, 2, 5))
|
||||
cycle = dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project=OPUS_PROJECT, route=Route(Routing.SONNET, 179, 2, 5))
|
||||
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
||||
self.assertEqual(run.run_cycle(cycle), Stop(2, "the human is needed to answer the question above, but nothing was typed"))
|
||||
kept = "".join(self.archived_outcomes())
|
||||
@@ -579,17 +580,17 @@ Spent since 2026-09-21T14:13:20Z, in tokens of input, cache write 5m, cache writ
|
||||
self.assertIn(f"Stepped up\n\nWorker: {OPUS_179_AT_6}\n", kept)
|
||||
|
||||
def test_a_ruling_steps_the_route_only_on_an_escalation_from_the_rung_its_taken_in_l_picks(self):
|
||||
cycle = dataclasses.replace(self.cycle(FakeEffects()), project=Routing.OPUS, route=Route(Routing.OPUS, 3, 3, 4))
|
||||
cycle = dataclasses.replace(self.cycle(FakeEffects()), project=OPUS_PROJECT, route=Route(Routing.ONE_UP, 3, 3, 4))
|
||||
for ruling in (Interrupt("e", "q", entries_read=3), QuotedAnswer("e", "q", "A.", entries_read=3)):
|
||||
with self.subTest(ruling=ruling):
|
||||
self.assertIsNone(run.ruling_step_up(cycle, ruling))
|
||||
self.assertEqual(run.ruling_step_up(cycle, Interrupt("e", "q", "round 2: 🔴", TakenIn(highest_likelihood=4), entries_read=3)), routing.StepUp(cycle.route, Route(Routing.TWO_UP, 3, 3, 4, escalated_at=3)))
|
||||
self.assertEqual(run.ruling_step_up(cycle, Interrupt("e", "q", "round 2: 🔴", TakenIn(highest_likelihood=4), entries_read=3)), routing.StepUp(cycle.route, Route(Routing.THREE_UP, 3, 3, 4, escalated_at=3)))
|
||||
|
||||
def test_an_escalation_after_an_interrupt_stepped_the_worker_steps_it_again_in_the_same_cycle(self):
|
||||
streams = [Worker(self.paths.claude_code_transcripts.parent, text="round 2: 🔴"), Worker(self.paths.claude_code_transcripts.parent), Worker(self.paths.claude_code_transcripts.parent)]
|
||||
rulings = [judged("interrupt", evidence="round 2: 🔴", escalation="round 2: 🔴"), judged("quoted", answer="Use A.", escalation="working"), judged("finished")]
|
||||
fake = FakeEffects(streams=streams, claude=rulings, inputs=["Use B."])
|
||||
cycle = dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project=Routing.OPUS, route=Route(Routing.SONNET, 179, 2, 5))
|
||||
cycle = dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project=OPUS_PROJECT, route=Route(Routing.SONNET, 179, 2, 5))
|
||||
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
||||
concluded = run.run_cycle(cycle)
|
||||
self.assertEqual(concluded.route, Route(Routing.ONE_UP, 179, 2, 5, escalated_at=4))
|
||||
@@ -602,7 +603,7 @@ Spent since 2026-09-21T14:13:20Z, in tokens of input, cache write 5m, cache writ
|
||||
streams = [Worker(self.paths.claude_code_transcripts.parent, text="round 2: 🔴"), Worker(self.paths.claude_code_transcripts.parent)]
|
||||
rulings = [judged("interrupt", evidence="round 2: 🔴", escalation="round 2: 🔴"), judged("unfinished", escalation=escalation)]
|
||||
fake = FakeEffects(streams=streams, claude=rulings, inputs=["Use B."])
|
||||
cycle = dataclasses.replace(self.cycle(fake), project=Routing.OPUS, route=Route(Routing.SONNET, 179, 2, 5))
|
||||
cycle = dataclasses.replace(self.cycle(fake), project=OPUS_PROJECT, route=Route(Routing.SONNET, 179, 2, 5))
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
concluded = run.run_cycle(cycle)
|
||||
self.assertEqual(concluded, run.Concluded(verdict, Route(Routing.OPUS, 179, 2, 5, escalated_at=3)))
|
||||
@@ -626,7 +627,7 @@ Spent since 2026-09-21T14:13:20Z, in tokens of input, cache write 5m, cache writ
|
||||
|
||||
def test_an_interrupt_that_steps_the_worker_up_names_its_new_rung_in_the_prompt_that_resumes_it_with_the_typed_answer(self):
|
||||
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[judged("interrupt", escalation="working"), judged("finished")], inputs=["Use B."])
|
||||
cycle = dataclasses.replace(self.cycle(fake), project=Routing.OPUS, route=Route(Routing.SONNET, 179, 2, 5))
|
||||
cycle = dataclasses.replace(self.cycle(fake), project=OPUS_PROJECT, route=Route(Routing.SONNET, 179, 2, 5))
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
concluded = run.run_cycle(cycle)
|
||||
self.assertEqual(concluded.route, Route(Routing.OPUS, 179, 2, 5, escalated_at=3))
|
||||
@@ -1036,6 +1037,7 @@ class Routing(unittest.TestCase):
|
||||
ONE_UP = Rung("claude-fable-5-2", "low")
|
||||
TWO_UP = Rung("claude-fable-5-2", "medium")
|
||||
THREE_UP = Rung("claude-opus-5-5", "xhigh")
|
||||
FOUR_UP = Rung("claude-fable-5-2", "high")
|
||||
SONNET = Rung("claude-sonnet-5-5", "high")
|
||||
|
||||
def test_routes_on_the_next_releases_top_item_that_waits_on_no_question(self):
|
||||
@@ -1065,53 +1067,53 @@ class Routing(unittest.TestCase):
|
||||
self.assertEqual(routed_item(todo.replace(first, "") + first, NO_CUT).id, 3)
|
||||
|
||||
def test_starts_a_chunk_on_the_rung_its_items_l_picks_below_the_projects_too(self):
|
||||
self.assertEqual(routing.first_route(LADDER, self.OPUS, routed_item(TODO, NO_CUT)), Route(self.ONE_UP, 3, 4, 5))
|
||||
self.assertEqual(routing.first_route(OPUS_PROJECT, routed_item(TODO, NO_CUT)), Route(self.TWO_UP, 3, 4, 5))
|
||||
cheap = routed_item(TODO.replace("| 4 | 2 | 3 |", "| 1 | 2 | 3 |"), NO_CUT)
|
||||
self.assertEqual(routing.first_route(LADDER, self.OPUS, cheap), Route(Rung("claude-sonnet-5-5", "high"), 3, 1, 5))
|
||||
self.assertEqual(routing.first_route(LADDER, self.OPUS, None), Route(self.OPUS))
|
||||
self.assertEqual(routing.first_route(OPUS_PROJECT, cheap), Route(Rung("claude-sonnet-5-5", "high"), 3, 1, 5))
|
||||
self.assertEqual(routing.first_route(OPUS_PROJECT, None), Route(self.OPUS))
|
||||
|
||||
def test_steps_up_a_rung_after_every_second_unfinished_verdict(self):
|
||||
once = routing.next_route(LADDER, self.OPUS, Route(self.OPUS, 3, 3, 4), unfinished())
|
||||
self.assertEqual(once, Route(self.OPUS, 3, 3, 4, unfinished=1))
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, once, unfinished()), Route(self.ONE_UP, 3, 3, 4))
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(LADDER.rungs[-1], unfinished=1), unfinished()).rung, LADDER.rungs[-1])
|
||||
once = routing.next_route(OPUS_PROJECT, Route(self.ONE_UP, 3, 3, 4), unfinished())
|
||||
self.assertEqual(once, Route(self.ONE_UP, 3, 3, 4, unfinished=1))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, once, unfinished()), Route(self.TWO_UP, 3, 3, 4))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(LADDER.rungs[-1], unfinished=1), unfinished()).rung, LADDER.rungs[-1])
|
||||
|
||||
def test_steps_up_to_the_rung_a_taken_in_items_l_picks_and_never_down(self):
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.OPUS), unfinished(4)).rung, self.ONE_UP)
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.TWO_UP), unfinished(4)).rung, self.TWO_UP)
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.OPUS, unfinished=1), unfinished(4)).rung, self.TWO_UP)
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.OPUS), unfinished(4)).rung, self.TWO_UP)
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.THREE_UP), unfinished(4)).rung, self.THREE_UP)
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.OPUS, unfinished=1), unfinished(4)).rung, self.THREE_UP)
|
||||
|
||||
def test_steps_up_from_a_rung_below_the_projects_one_rung_at_a_time(self):
|
||||
low = Route(self.SONNET, 3, 1, 3, unfinished=1)
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, low, unfinished()).rung, self.OPUS)
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(low.rung, 3, 1, 3), unfinished()).rung, low.rung)
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(low.rung, 3, 1, 3), unfinished(3)).rung, self.OPUS)
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, low, unfinished()).rung, self.OPUS)
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(low.rung, 3, 1, 3), unfinished()).rung, low.rung)
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(low.rung, 3, 1, 3), unfinished(3)).rung, self.ONE_UP)
|
||||
|
||||
def test_steps_up_a_rung_on_an_escalation_the_judge_found_up_to_two_above_the_items_rung_marking_the_transcript_entries_it_read(self):
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.ONE_UP, 3, 4, 5), unfinished(escalated_at=7)), Route(self.TWO_UP, 3, 4, 5, escalated_at=7))
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.TWO_UP, 3, 4, 5, escalated_at=7), unfinished(escalated_at=9)), Route(self.THREE_UP, 3, 4, 5, escalated_at=9))
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.THREE_UP, 3, 4, 5, escalated_at=9), unfinished(escalated_at=12)), Route(self.THREE_UP, 3, 4, 5, unfinished=1, escalated_at=9))
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.ONE_UP, 3, 4, 5, unfinished=1), unfinished(escalated_at=7)), Route(self.TWO_UP, 3, 4, 5, escalated_at=7))
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.TWO_UP), unfinished(escalated_at=7)), Route(self.TWO_UP, unfinished=1))
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.SONNET, 3, 2, 4), unfinished(4, escalated_at=7)), Route(self.ONE_UP, 3, 2, 4, unfinished=1))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.TWO_UP, 3, 4, 5), unfinished(escalated_at=7)), Route(self.THREE_UP, 3, 4, 5, escalated_at=7))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.THREE_UP, 3, 4, 5, escalated_at=7), unfinished(escalated_at=9)), Route(self.FOUR_UP, 3, 4, 5, escalated_at=9))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.FOUR_UP, 3, 4, 5, escalated_at=9), unfinished(escalated_at=12)), Route(self.FOUR_UP, 3, 4, 5, unfinished=1, escalated_at=9))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.TWO_UP, 3, 4, 5, unfinished=1), unfinished(escalated_at=7)), Route(self.THREE_UP, 3, 4, 5, escalated_at=7))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.TWO_UP), unfinished(escalated_at=7)), Route(self.TWO_UP, unfinished=1))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.SONNET, 3, 2, 4), unfinished(4, escalated_at=7)), Route(self.TWO_UP, 3, 2, 4, unfinished=1))
|
||||
|
||||
def test_keeps_the_mark_through_a_step_no_escalation_took(self):
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.OPUS, 3, 3, 4, unfinished=1, escalated_at=5), unfinished()), Route(self.ONE_UP, 3, 3, 4, escalated_at=5))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.ONE_UP, 3, 3, 4, unfinished=1, escalated_at=5), unfinished()), Route(self.TWO_UP, 3, 3, 4, escalated_at=5))
|
||||
|
||||
def test_steps_up_a_rung_as_the_judge_interrupts_up_to_two_above_the_items_rung_without_counting_an_unfinished_verdict(self):
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.ONE_UP, 3, 4, 5, unfinished=1), interrupted(escalated_at=7)), Route(self.TWO_UP, 3, 4, 5, escalated_at=7))
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.TWO_UP, 3, 4, 5, escalated_at=7), interrupted(escalated_at=9)), Route(self.THREE_UP, 3, 4, 5, escalated_at=9))
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.THREE_UP, 3, 4, 5, unfinished=1, escalated_at=9), interrupted(escalated_at=12)), Route(self.THREE_UP, 3, 4, 5, unfinished=1, escalated_at=9))
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.SONNET, 3, 2, 4), interrupted(escalated_at=7)), Route(self.OPUS, 3, 2, 4, escalated_at=7))
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.ONE_UP, 3, 4, 5, unfinished=1, escalated_at=2), interrupted(4)), Route(self.ONE_UP, 3, 4, 5, unfinished=1, escalated_at=2))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.TWO_UP, 3, 4, 5, unfinished=1), interrupted(escalated_at=7)), Route(self.THREE_UP, 3, 4, 5, escalated_at=7))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.THREE_UP, 3, 4, 5, escalated_at=7), interrupted(escalated_at=9)), Route(self.FOUR_UP, 3, 4, 5, escalated_at=9))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.FOUR_UP, 3, 4, 5, unfinished=1, escalated_at=9), interrupted(escalated_at=12)), Route(self.FOUR_UP, 3, 4, 5, unfinished=1, escalated_at=9))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.SONNET, 3, 2, 4), interrupted(escalated_at=7)), Route(self.OPUS, 3, 2, 4, escalated_at=7))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.TWO_UP, 3, 4, 5, unfinished=1, escalated_at=2), interrupted(4)), Route(self.TWO_UP, 3, 4, 5, unfinished=1, escalated_at=2))
|
||||
|
||||
def test_raises_the_worker_to_a_taken_in_items_rung_above_the_escalation_cap_without_a_step_as_the_judge_interrupts(self):
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.OPUS, 3, 1, 3, escalated_at=2), interrupted(4, escalated_at=7)), Route(self.ONE_UP, 3, 1, 3, escalated_at=2))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.OPUS, 3, 1, 3, escalated_at=2), interrupted(4, escalated_at=7)), Route(self.TWO_UP, 3, 1, 3, escalated_at=2))
|
||||
|
||||
def test_steps_up_from_the_rung_a_taken_in_item_picks_as_the_judge_interrupts(self):
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.OPUS, 3, 3, 4), interrupted(4, escalated_at=7)), Route(self.TWO_UP, 3, 3, 4, escalated_at=7))
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.TWO_UP, 3, 4, 5), interrupted(4, escalated_at=7)), Route(self.THREE_UP, 3, 4, 5, escalated_at=7))
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(self.THREE_UP, 3, 4, 5, unfinished=1), interrupted(4, escalated_at=7)), Route(self.THREE_UP, 3, 4, 5, unfinished=1))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.ONE_UP, 3, 3, 4), interrupted(4, escalated_at=7)), Route(self.THREE_UP, 3, 3, 4, escalated_at=7))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.TWO_UP, 3, 4, 5), interrupted(4, escalated_at=7)), Route(self.THREE_UP, 3, 4, 5, escalated_at=7))
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(self.FOUR_UP, 3, 4, 5, unfinished=1), interrupted(4, escalated_at=7)), Route(self.FOUR_UP, 3, 4, 5, unfinished=1))
|
||||
|
||||
def test_keeps_an_e_plus_i_taken_in_only_where_it_picks_a_higher_review_rung(self):
|
||||
taken = Route(self.OPUS, 3, 3, 4).taking_in(6)
|
||||
@@ -1126,8 +1128,8 @@ class Routing(unittest.TestCase):
|
||||
self.assertEqual(Route(self.OPUS).taking_in(0), Route(self.OPUS))
|
||||
|
||||
def test_a_rung_an_edit_took_off_the_ladder_counts_as_the_projects(self):
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(Rung("claude-opus-5-5", "low")), unfinished()).rung, self.OPUS)
|
||||
self.assertEqual(routing.next_route(LADDER, self.OPUS, Route(Rung("claude-opus-5-5", "low")), unfinished(2)).rung, self.OPUS)
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(Rung("claude-opus-5-5", "low")), unfinished()).rung, self.OPUS)
|
||||
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(Rung("claude-opus-5-5", "low")), unfinished(2)).rung, self.OPUS)
|
||||
|
||||
def test_runs_the_judge_of_a_replayed_model_on_its_models_highest_rung(self):
|
||||
self.assertEqual(routing.judge_rung(LADDER, "claude-opus-5-5"), self.THREE_UP)
|
||||
@@ -1175,17 +1177,17 @@ class Routing(unittest.TestCase):
|
||||
|
||||
def test_keeps_a_chunks_route_or_its_archived_one_at_or_above_the_rung_its_item_picks_and_else_routes_afresh(self):
|
||||
item = routed_item(TODO, NO_CUT)
|
||||
archived = Route(self.TWO_UP, 3, 4, 5)
|
||||
archived = Route(self.THREE_UP, 3, 4, 5)
|
||||
def never():
|
||||
raise AssertionError("not asked")
|
||||
self.assertEqual(chunk_start.routed(archived, LADDER, None, "worker-model", never, never), Route(Rung("worker-model")))
|
||||
self.assertEqual(chunk_start.routed(archived, LADDER, self.OPUS, "claude-opus-5-5", never, never), archived)
|
||||
self.assertEqual(chunk_start.routed(None, LADDER, self.OPUS, "claude-opus-5-5", lambda: archived, never), archived)
|
||||
self.assertEqual(chunk_start.routed(None, LADDER, self.OPUS, "claude-opus-5-5", lambda: None, lambda: item), Route(self.ONE_UP, 3, 4, 5))
|
||||
self.assertEqual(chunk_start.routed(Route(self.SONNET), LADDER, self.OPUS, "claude-opus-5-5", never, lambda: None), Route(self.OPUS))
|
||||
self.assertEqual(chunk_start.routed(archived, None, "worker-model", never, never), Route(Rung("worker-model")))
|
||||
self.assertEqual(chunk_start.routed(archived, OPUS_PROJECT, "claude-opus-5-5", never, never), archived)
|
||||
self.assertEqual(chunk_start.routed(None, OPUS_PROJECT, "claude-opus-5-5", lambda: archived, never), archived)
|
||||
self.assertEqual(chunk_start.routed(None, OPUS_PROJECT, "claude-opus-5-5", lambda: None, lambda: item), Route(self.TWO_UP, 3, 4, 5))
|
||||
self.assertEqual(chunk_start.routed(Route(self.SONNET), OPUS_PROJECT, "claude-opus-5-5", never, lambda: None), Route(self.OPUS))
|
||||
cheap = Route(self.SONNET, 4, 2, 3)
|
||||
self.assertEqual(chunk_start.routed(cheap, LADDER, self.OPUS, "claude-opus-5-5", never, never), cheap)
|
||||
self.assertEqual(chunk_start.routed(Route(Rung("claude-haiku-5-5", "max"), 4, 2, 3), LADDER, self.OPUS, "claude-opus-5-5", never, lambda: None), Route(self.OPUS))
|
||||
self.assertEqual(chunk_start.routed(cheap, OPUS_PROJECT, "claude-opus-5-5", never, never), cheap)
|
||||
self.assertEqual(chunk_start.routed(Route(Rung("claude-haiku-5-5", "max"), 4, 2, 3), OPUS_PROJECT, "claude-opus-5-5", never, lambda: None), Route(self.OPUS))
|
||||
|
||||
|
||||
class RouteSources(unittest.TestCase):
|
||||
@@ -1590,27 +1592,30 @@ class StartChunks(unittest.TestCase):
|
||||
|
||||
class Steps(unittest.TestCase):
|
||||
OPUS = Rung("claude-opus-5-5", "medium")
|
||||
# The rungs an L 3 and an L 4 pick from OPUS.
|
||||
FABLE_LOW = Rung("claude-fable-5-2", "low")
|
||||
FABLE_MEDIUM = Rung("claude-fable-5-2", "medium")
|
||||
|
||||
def test_a_finished_verdict_clears_the_route_and_the_evidence(self):
|
||||
run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="old", route=Route(self.OPUS))
|
||||
self.assertEqual(close.after_verdict(run_with, LADDER, self.OPUS, run.Concluded(Finished("x"), run_with.route)), dataclasses.replace(run_with, route=None, unfinished_evidence=""))
|
||||
self.assertEqual(close.after_verdict(dataclasses.replace(run_with, unfinished_run=3), LADDER, self.OPUS, run.Concluded(Finished("x"), run_with.route)).unfinished_run, 0)
|
||||
self.assertEqual(close.after_verdict(run_with, LADDER, self.OPUS, run.Concluded(Unfinished("new", taken_in=TakenIn(highest_likelihood=4)), run_with.route)), dataclasses.replace(run_with, route=Route(Rung("claude-fable-5-2", "low"), unfinished=1), unfinished_evidence="new", unfinished_run=1))
|
||||
self.assertEqual(close.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Finished("x"), run_with.route)), dataclasses.replace(run_with, route=None, unfinished_evidence=""))
|
||||
self.assertEqual(close.after_verdict(dataclasses.replace(run_with, unfinished_run=3), OPUS_PROJECT, run.Concluded(Finished("x"), run_with.route)).unfinished_run, 0)
|
||||
self.assertEqual(close.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", taken_in=TakenIn(highest_likelihood=4)), run_with.route)), dataclasses.replace(run_with, route=Route(self.FABLE_MEDIUM, unfinished=1), unfinished_evidence="new", unfinished_run=1))
|
||||
|
||||
def test_steps_from_the_route_its_cycle_ended_on_which_a_ruling_may_have_raised(self):
|
||||
run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(self.OPUS, 3, 3, 4))
|
||||
raised = Route(Rung("claude-fable-5-2", "low"), 3, 3, 4, escalated_at=5)
|
||||
self.assertEqual(close.after_verdict(run_with, LADDER, self.OPUS, run.Concluded(Unfinished("new"), raised)).route, dataclasses.replace(raised, unfinished=1))
|
||||
raised = Route(self.FABLE_MEDIUM, 3, 3, 4, escalated_at=5)
|
||||
self.assertEqual(close.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new"), raised)).route, dataclasses.replace(raised, unfinished=1))
|
||||
|
||||
def test_keeps_the_route_its_cycle_ended_on_off_the_ladder(self):
|
||||
off_ladder = Route(Rung("worker-model"))
|
||||
run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="")
|
||||
self.assertEqual(close.after_verdict(run_with, LADDER, None, run.Concluded(Unfinished("new", "round 2: 🔴", TakenIn(highest_likelihood=4), entries_read=9), off_ladder)).route, off_ladder)
|
||||
self.assertEqual(close.after_verdict(run_with, None, run.Concluded(Unfinished("new", "round 2: 🔴", TakenIn(highest_likelihood=4), entries_read=9), off_ladder)).route, off_ladder)
|
||||
|
||||
def test_an_unfinished_verdicts_escalation_marks_the_transcript_entries_the_judge_read(self):
|
||||
run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(self.OPUS, 3, 3, 4, escalated_at=2))
|
||||
self.assertEqual(close.after_verdict(run_with, LADDER, self.OPUS, run.Concluded(Unfinished("new", "round 2: 🔴", entries_read=9), run_with.route)).route, Route(Rung("claude-fable-5-2", "low"), 3, 3, 4, escalated_at=9))
|
||||
self.assertEqual(close.after_verdict(run_with, LADDER, self.OPUS, run.Concluded(Unfinished("new", entries_read=9), run_with.route)).route, Route(self.OPUS, 3, 3, 4, unfinished=1, escalated_at=2))
|
||||
run_with = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(self.FABLE_LOW, 3, 3, 4, escalated_at=2))
|
||||
self.assertEqual(close.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", "round 2: 🔴", entries_read=9), run_with.route)).route, Route(self.FABLE_MEDIUM, 3, 3, 4, escalated_at=9))
|
||||
self.assertEqual(close.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", entries_read=9), run_with.route)).route, Route(self.FABLE_LOW, 3, 3, 4, unfinished=1, escalated_at=2))
|
||||
|
||||
def test_archives_a_step_up_as_it_happens_so_a_rerun_resumes_on_it(self):
|
||||
with tempfile.TemporaryDirectory() as home:
|
||||
@@ -1619,17 +1624,17 @@ class Steps(unittest.TestCase):
|
||||
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
|
||||
transcript.parent.mkdir(parents=True)
|
||||
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
||||
before = Route(self.OPUS, 3, 3, 4)
|
||||
before = Route(self.FABLE_LOW, 3, 3, 4)
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW))
|
||||
after = close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), LADDER, self.OPUS, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), archive, a_loop(FakeEffects().effects(), session=opened))
|
||||
self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(Rung("claude-fable-5-2", "low"), 3, 3, 4))
|
||||
after = close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), archive, a_loop(FakeEffects().effects(), session=opened))
|
||||
self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_MEDIUM, 3, 3, 4))
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
close.closed_cycle(after, LADDER, self.OPUS, run.Concluded(Finished("done"), after.route), archive, a_loop(FakeEffects().effects(), session=opened))
|
||||
self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(Rung("claude-fable-5-2", "low"), 3, 3, 4))
|
||||
close.closed_cycle(after, OPUS_PROJECT, run.Concluded(Finished("done"), after.route), archive, a_loop(FakeEffects().effects(), session=opened))
|
||||
self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_MEDIUM, 3, 3, 4))
|
||||
blocked = pathlib.Path(home, "blocked")
|
||||
blocked.write_text("")
|
||||
stop = close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), LADDER, self.OPUS, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), blocked, a_loop(FakeEffects().effects(), session=opened))
|
||||
stop = close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), blocked, a_loop(FakeEffects().effects(), session=opened))
|
||||
self.assertEqual((stop.code, stop.reason.split(":")[0]), (1, "could not archive the transcript"))
|
||||
|
||||
def test_names_no_step_up_where_only_the_count_of_unfinished_verdicts_changed(self):
|
||||
@@ -1644,11 +1649,11 @@ class Steps(unittest.TestCase):
|
||||
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
|
||||
transcript.parent.mkdir(parents=True)
|
||||
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
||||
before = Route(self.OPUS, 3, 3, 4)
|
||||
before = Route(self.FABLE_LOW, 3, 3, 4)
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW))
|
||||
close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), LADDER, self.OPUS, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_escape_and_impact=6)), before), archive, a_loop(FakeEffects().effects(), session=opened))
|
||||
self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.OPUS, 3, 3, 4, escape_and_impact_taken_in=6))
|
||||
close.closed_cycle(main.LoopRun(cycle_number=1, root="/r", task="Do it.", unfinished_evidence=""), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_escape_and_impact=6)), before), archive, a_loop(FakeEffects().effects(), session=opened))
|
||||
self.assertEqual(chunk_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_LOW, 3, 3, 4, escape_and_impact_taken_in=6))
|
||||
|
||||
|
||||
class ChunkFacts(unittest.TestCase):
|
||||
@@ -1704,13 +1709,13 @@ class TypedAnswers(unittest.TestCase):
|
||||
def test_splits_the_prompt_handing_over_the_answer_into_the_answer_on_the_line_after_its_marker_and_the_rest_and_no_other_prompt(self):
|
||||
route = Route(Rung("worker-model"))
|
||||
marked = "The human answered your question at loop-claude's terminal:\nB"
|
||||
typed = prompts.typed_answer(prompts.typed("Which one?", "Use B.", route, LADDER, None, False))
|
||||
typed = prompts.typed_answer(prompts.typed("Which one?", "Use B.", route, None, False))
|
||||
self.assertEqual(typed.answer, "Use B.")
|
||||
self.assertTrue(typed.loop_words.startswith("Your question was: Which one?\n\nContinue the task."))
|
||||
self.assertEqual(prompts.typed_answer(prompts.typed(f"Which one?\n{marked}", "Use B.", route, LADDER, None, False)).answer, "Use B.")
|
||||
self.assertEqual(prompts.typed_answer(prompts.typed("Which one?", "Use B.", Route(Routing.SONNET, 179, 2, 5), LADDER, Routing.OPUS, True)).answer, "Use B.")
|
||||
self.assertIsNone(prompts.typed_answer(prompts.answered("Which one?", marked, route, LADDER, None, False)))
|
||||
self.assertIsNone(prompts.typed_answer(prompts.resumed(marked, route, LADDER, None)))
|
||||
self.assertEqual(prompts.typed_answer(prompts.typed(f"Which one?\n{marked}", "Use B.", route, None, False)).answer, "Use B.")
|
||||
self.assertEqual(prompts.typed_answer(prompts.typed("Which one?", "Use B.", Route(Routing.SONNET, 179, 2, 5), OPUS_PROJECT, True)).answer, "Use B.")
|
||||
self.assertIsNone(prompts.typed_answer(prompts.answered("Which one?", marked, route, None, False)))
|
||||
self.assertIsNone(prompts.typed_answer(prompts.resumed(marked, route, None)))
|
||||
|
||||
|
||||
class WorkerArgs(unittest.TestCase):
|
||||
@@ -1721,24 +1726,24 @@ class WorkerArgs(unittest.TestCase):
|
||||
FACTS = "- A preflight fact."
|
||||
|
||||
def test_names_the_route_in_a_fresh_prompt_only_where_an_item_routed_it(self):
|
||||
routed = prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), LADDER, None, self.FACTS)
|
||||
routed = prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), None, self.FACTS)
|
||||
self.assertIn(f"Continue any open draft PR or worktree for the chunk before creating a new one. {prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\nloop-claude runs you on m at high effort, routed by item 3's L of 4.", routed)
|
||||
unrouted = prompts.fresh("Do it.", "## Goals", Route(Rung("m")), LADDER, None, self.FACTS)
|
||||
unrouted = prompts.fresh("Do it.", "## Goals", Route(Rung("m")), None, self.FACTS)
|
||||
self.assertIn(f"before creating a new one. {prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\n{self.FACTS}\n\nThe chunk's", unrouted)
|
||||
self.assertNotIn("loop-claude runs you on", unrouted)
|
||||
|
||||
def test_has_the_routed_worker_count_its_items_size_again_and_puts_the_facts_before_the_scope(self):
|
||||
routed = prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), LADDER, None, self.FACTS)
|
||||
routed = prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), None, self.FACTS)
|
||||
self.assertIn("item 3's size again", routed)
|
||||
self.assertIn("escalation trigger 1", routed)
|
||||
self.assertIn(f"\n\n{self.FACTS}\n\nThe chunk's", routed)
|
||||
self.assertNotIn("size again", prompts.fresh("Do it.", "## Goals", Route(Rung("m")), LADDER, None, self.FACTS))
|
||||
self.assertNotIn("size again", prompts.fresh("Do it.", "## Goals", Route(Rung("m")), None, self.FACTS))
|
||||
|
||||
def test_gives_each_fresh_worker_the_reading_rule_once(self):
|
||||
rule = "never read more than 80 lines of a file at once, nor take a tool output over 3,000 characters into your context"
|
||||
for route in (Route(Rung("m", "high"), 3, 4, 5), Route(Rung("m"))):
|
||||
with self.subTest(route=route):
|
||||
self.assertEqual(prompts.fresh("Do it.", "## Goals", route, LADDER, None, self.FACTS).count(rule), 1)
|
||||
self.assertEqual(prompts.fresh("Do it.", "## Goals", route, None, self.FACTS).count(rule), 1)
|
||||
|
||||
def test_quotes_what_the_preflight_printed_or_has_the_worker_find_the_facts_itself(self):
|
||||
self.assertIn("\n\n- A fact.", prompts.preflight_facts("- A fact."))
|
||||
@@ -1749,95 +1754,95 @@ class WorkerArgs(unittest.TestCase):
|
||||
def test_has_each_fresh_worker_continue_an_open_draft_pr_or_worktree_once_whatever_the_preflight_found(self):
|
||||
for facts in (prompts.preflight_facts("- A fact."), prompts.preflight_failed("git could not start")):
|
||||
with self.subTest(facts=facts):
|
||||
self.assertEqual(prompts.fresh("Do it.", "## Goals", Route(Rung("m")), LADDER, None, facts).count("Continue any open draft PR or worktree"), 1)
|
||||
self.assertEqual(prompts.fresh("Do it.", "## Goals", Route(Rung("m")), None, facts).count("Continue any open draft PR or worktree"), 1)
|
||||
|
||||
def test_keeps_every_review_on_the_projects_rung_while_the_worker_runs_below_it(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
below = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 4), LADDER, opus, self.FACTS)
|
||||
below = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 4), OPUS_PROJECT, self.FACTS)
|
||||
self.assertIn(f"item 3's L of 2. Launch each reviewer, each comprehension seat and the security pass's subagent with the Agent tool set to model opus, effort medium, the project's rung.{self.SCAN} Your rung is below the project's: launch each subagent that files or scores plan items on the rung `~/.claude/bin/subagent-rung --plan claude-opus-5-5 <L>` prints. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger", below)
|
||||
self.assertNotIn("Launch each", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), LADDER, opus, self.FACTS))
|
||||
self.assertNotIn("Launch each", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS))
|
||||
|
||||
def test_launches_reviewers_on_the_rung_the_items_e_plus_i_picks_above_the_workers(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
level = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 6), LADDER, opus, self.FACTS)
|
||||
self.assertIn(f"item 3's L of 3. Launch each reviewer with the Agent tool set to model fable, effort medium, the rung item 3's E + I of 6 picks. Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", level)
|
||||
below = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 5), LADDER, opus, self.FACTS)
|
||||
level = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 6), OPUS_PROJECT, self.FACTS)
|
||||
self.assertIn(f"item 3's L of 2. Launch each reviewer with the Agent tool set to model fable, effort medium, the rung item 3's E + I of 6 picks. Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", level)
|
||||
below = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 5), OPUS_PROJECT, self.FACTS)
|
||||
self.assertIn(f"item 3's L of 2. Launch each reviewer with the Agent tool set to model fable, effort low, the rung item 3's E + I of 5 picks.{self.SEATS}{self.SCAN} Your rung is below the project's:", below)
|
||||
self.assertNotIn("Launch each reviewer", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 4, 6), LADDER, opus))
|
||||
self.assertNotIn("Launch each reviewer", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 4, 6), OPUS_PROJECT))
|
||||
|
||||
def test_names_the_reviewers_rung_each_higher_e_plus_i_an_item_taken_in_picks(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
taking_in = " Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8."
|
||||
self.assertIn(f"item 3's L of 3.{taking_in} When an escalation trigger", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), LADDER, opus, self.FACTS))
|
||||
self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.", prompts.resumed("", Route(opus, 3, 3, 6), LADDER, opus))
|
||||
self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 4, 4), LADDER, opus))
|
||||
self.assertNotIn("Where an item you take into", prompts.resumed("", Route(opus, 3, 3, 8), LADDER, opus))
|
||||
self.assertNotIn("Where an item you take into", prompts.resumed("", Route(Rung("m")), LADDER, None))
|
||||
self.assertIn(f"item 3's L of 2.{taking_in} When an escalation trigger", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS))
|
||||
self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.", prompts.resumed("", Route(opus, 3, 2, 6), OPUS_PROJECT))
|
||||
self.assertIn(" Where an item you take into the chunk has an E + I of 7 or more, launch each reviewer with the Agent tool set to `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 4, 4), OPUS_PROJECT))
|
||||
self.assertNotIn("Where an item you take into", prompts.resumed("", Route(opus, 3, 2, 8), OPUS_PROJECT))
|
||||
self.assertNotIn("Where an item you take into", prompts.resumed("", Route(Rung("m")), None))
|
||||
|
||||
def test_launches_reviewers_on_the_rung_the_highest_e_plus_i_taken_in_picks(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
self.assertIn(f"loop-claude runs you on claude-opus-5-5 at medium effort. Launch each reviewer with the Agent tool set to model opus, effort xhigh, the rung picked by an item of E + I 7 taken into the chunk. Where an item you take into the chunk has an E + I of 8 or more, launch each reviewer with the Agent tool set to `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", prompts.resumed("", Route(opus, 3, 3, 5, escape_and_impact_taken_in=7), LADDER, opus))
|
||||
self.assertIn(" Launch each reviewer with the Agent tool set to model fable, effort low, the rung picked by an item of E + I 5 taken into the chunk.", prompts.resumed("", Route(opus, escape_and_impact_taken_in=5), LADDER, opus))
|
||||
self.assertIn(f"loop-claude runs you on claude-opus-5-5 at medium effort. Launch each reviewer with the Agent tool set to model opus, effort xhigh, the rung picked by an item of E + I 7 taken into the chunk. Where an item you take into the chunk has an E + I of 8 or more, launch each reviewer with the Agent tool set to `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", prompts.resumed("", Route(opus, 3, 2, 5, escape_and_impact_taken_in=7), OPUS_PROJECT))
|
||||
self.assertIn(" Launch each reviewer with the Agent tool set to model fable, effort low, the rung picked by an item of E + I 5 taken into the chunk.", prompts.resumed("", Route(opus, escape_and_impact_taken_in=5), OPUS_PROJECT))
|
||||
|
||||
def test_keeps_seats_and_the_security_pass_on_the_projects_rung_while_the_worker_runs_above_it(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
above = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-fable-5-2", "low"), 3, 4, 4), LADDER, opus, self.FACTS)
|
||||
stepped_up = prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), LADDER, opus)
|
||||
above = prompts.fresh("Do it.", "## Goals", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT, self.FACTS)
|
||||
stepped_up = prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT)
|
||||
self.assertIn(f"{prompts.ANSWERS_RECORDED}. {prompts.NAME_THE_ITEMS} loop-claude runs you on claude-fable-5-2 at low effort.{self.SEATS}{self.SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", stepped_up)
|
||||
self.assertNotIn("the project's", prompts.resumed("", Route(opus, 3, 3, 4), LADDER, opus))
|
||||
self.assertNotIn("runs you on", prompts.resumed("", Route(Rung("m")), LADDER, None))
|
||||
self.assertIn(f"item 3's L of 4.{self.SEATS}{self.SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", above)
|
||||
self.assertNotIn("the project's", prompts.resumed("", Route(opus, 3, 2, 4), OPUS_PROJECT))
|
||||
self.assertNotIn("runs you on", prompts.resumed("", Route(Rung("m")), None))
|
||||
self.assertIn(f"item 3's L of 3.{self.SEATS}{self.SCAN} Where an item you take into the chunk has an E + I of 6 or more, launch each reviewer with the Agent tool set to `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8.{self.MARKDOWN} When an escalation trigger", above)
|
||||
|
||||
def test_names_the_rungs_an_escalation_trigger_lets_the_worker_hand_a_task_to_at_most_two_above_its_items(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
fresh = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), LADDER, opus, self.FACTS)
|
||||
self.assertIn("item 3's L of 3. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort low`, one rung above yours, and on each further trigger one rung higher, up to `model fable, effort medium`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", fresh)
|
||||
self.assertIn(" When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort medium`, one rung above yours; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), LADDER, opus))
|
||||
self.assertIn(" An escalation trigger in `~/.claude/CLAUDE.md` lets you hand the hard part of a task to a subagent with the Agent tool set to `model opus, effort xhigh`, one rung above yours, only once the human agrees; loop-claude will not resume you higher for a trigger either, so ask the human for that yes. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 3, 4), LADDER, opus))
|
||||
self.assertNotIn("escalation", prompts.resumed("", Route(Rung("claude-fable-5-2", "max"), 3, 4, 8), LADDER, opus))
|
||||
self.assertIn("only once the human agrees", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 4, 5), LADDER, opus))
|
||||
self.assertIn("only once the human agrees", prompts.resumed("", Route(Rung("claude-fable-5-2", "high"), 3, 4, 5), LADDER, opus), "one rung below the top still names the rung above")
|
||||
self.assertNotIn("rerun", prompts.resumed("", Route(Rung("claude-opus-5-5", "low"), 3, 3, 4), LADDER, opus), "a worker off the ladder is told no rerun rung")
|
||||
self.assertIn("hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort xhigh`, one rung above yours;", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 4, 5), LADDER, opus))
|
||||
self.assertNotIn("escalation", prompts.resumed("", Route(Rung("m")), LADDER, None))
|
||||
self.assertNotIn("escalation", prompts.resumed("", Route(Rung("claude-opus-5-5", "low"), 3, 3, 4), LADDER, opus))
|
||||
self.assertIn(f"{prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\nWhere an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort low`, one rung above yours, and on each further trigger one rung higher, up to `model fable, effort medium`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number. Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `model fable, effort medium`. A reviewer", prompts.fresh("Do it.", "## Goals", Route(opus), LADDER, opus, self.FACTS))
|
||||
fresh = prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS)
|
||||
self.assertIn("item 3's L of 2. Where an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort low`, one rung above yours; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", fresh)
|
||||
self.assertIn(" When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort medium`, one rung above yours, and on each further trigger one rung higher, up to `model opus, effort xhigh`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT))
|
||||
self.assertIn(" When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model opus, effort xhigh`, one rung above yours; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number.", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 3, 4), OPUS_PROJECT))
|
||||
self.assertNotIn("escalation", prompts.resumed("", Route(Rung("claude-fable-5-2", "max"), 3, 4, 8), OPUS_PROJECT))
|
||||
self.assertIn(" An escalation trigger in `~/.claude/CLAUDE.md` lets you hand the hard part of a task to a subagent with the Agent tool set to `model fable, effort high`, one rung above yours, only once the human agrees;", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 3, 4), OPUS_PROJECT))
|
||||
self.assertIn("set to `model fable, effort max`, one rung above yours, only once the human agrees;", prompts.resumed("", Route(Rung("claude-fable-5-2", "high"), 3, 4, 5), OPUS_PROJECT), "one rung below the top still names the rung above")
|
||||
self.assertNotIn("rerun", prompts.resumed("", Route(Rung("claude-opus-5-5", "low"), 3, 3, 4), OPUS_PROJECT), "a worker off the ladder is told no rerun rung")
|
||||
self.assertIn("hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort high`, one rung above yours;", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 4, 5), OPUS_PROJECT))
|
||||
self.assertNotIn("escalation", prompts.resumed("", Route(Rung("m")), None))
|
||||
self.assertNotIn("escalation", prompts.resumed("", Route(Rung("claude-opus-5-5", "low"), 3, 3, 4), OPUS_PROJECT))
|
||||
self.assertIn(f"{prompts.READING_RULE} {prompts.NAME_THE_ITEMS}\n\nWhere an item you take into the chunk has an E + I of 5 or more, launch each reviewer with the Agent tool set to `model fable, effort low` from 5, `model fable, effort medium` from 6, `model opus, effort xhigh` from 7 and `model fable, effort high` from 8. When an escalation trigger in `~/.claude/CLAUDE.md` fires, hand the hard part of the task to a subagent with the Agent tool set to `model fable, effort low`, one rung above yours, and on each further trigger one rung higher, up to `model fable, effort medium`; above that, ask the human first. Open the subagent's prompt with `Escalation trigger <n>:`, n the trigger's number. Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `model fable, effort medium`. A reviewer", prompts.fresh("Do it.", "## Goals", Route(opus), OPUS_PROJECT, self.FACTS))
|
||||
|
||||
def test_offers_the_worker_a_higher_rung_for_the_rest_of_the_chunk_up_to_the_escalation_cap(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
offer = " Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `model fable, effort medium`."
|
||||
self.assertIn(f"`Escalation trigger <n>:`, n the trigger's number.{offer}", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), LADDER, opus, self.FACTS))
|
||||
self.assertIn(f"`Escalation trigger <n>:`, n the trigger's number.{offer}", prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), LADDER, opus))
|
||||
self.assertNotIn("Where the hard part", prompts.resumed("", Route(Rung("claude-fable-5-2", "medium"), 3, 3, 4), LADDER, opus))
|
||||
offer = lambda cap: f" Where the hard part is the rest of the chunk, end your run instead, asking loop-claude for a higher rung and naming a trigger that fired since your last step-up, if any: once the judge finds it in your transcript, the next cycle resumes you one rung up, up to `{cap}`."
|
||||
self.assertIn(f"`Escalation trigger <n>:`, n the trigger's number.{offer('model fable, effort low')}", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS))
|
||||
self.assertIn(f"`Escalation trigger <n>:`, n the trigger's number.{offer('model opus, effort xhigh')}", prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT))
|
||||
self.assertNotIn("Where the hard part", prompts.resumed("", Route(Rung("claude-opus-5-5", "xhigh"), 3, 3, 4), OPUS_PROJECT))
|
||||
|
||||
def test_lets_a_markdown_only_branch_review_on_the_projects_rung_where_the_reviewers_run_above_it(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
markdown = " Where your branch changes only Markdown, none of it `CLAUDE.md`, `AGENTS.md`, `workflow-bench/*-prompt.md` or under `agents/`, `principles/`, `rules/` or `skills/`, launch each reviewer on the project's rung instead, with the Agent tool set to model opus, effort medium."
|
||||
self.assertIn(markdown, prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 7), LADDER, opus, self.FACTS))
|
||||
self.assertIn(markdown, prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 4, 4), LADDER, opus))
|
||||
self.assertNotIn("only Markdown", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), LADDER, opus, self.FACTS))
|
||||
self.assertIn(markdown, prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 7), OPUS_PROJECT, self.FACTS))
|
||||
self.assertIn(markdown, prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT))
|
||||
self.assertNotIn("only Markdown", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS))
|
||||
|
||||
def test_names_the_rung_a_reviewer_or_seat_reruns_on_one_above_its_own(self):
|
||||
opus = Rung("claude-opus-5-5", "medium")
|
||||
both = " A reviewer or comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort low`."
|
||||
self.assertTrue(prompts.resumed("", Route(opus, 3, 3, 4), LADDER, opus).endswith(both))
|
||||
self.assertTrue(prompts.resumed("", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 4), LADDER, opus).endswith(both))
|
||||
self.assertTrue(prompts.fresh("Do it.", "## Goals", Route(opus), LADDER, opus, self.FACTS).split(f"\n\n{self.FACTS}")[0].endswith(both))
|
||||
self.assertTrue(prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 4, 4), LADDER, opus).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort medium`, and a comprehension seat with `model fable, effort low`."))
|
||||
self.assertTrue(prompts.resumed("", Route(opus, 3, 3, 7), LADDER, opus).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort high`, and a comprehension seat with `model fable, effort low`."))
|
||||
self.assertTrue(prompts.resumed("", Route(Rung("claude-fable-5-2", "max"), 3, 4, 8), LADDER, opus).endswith(" A comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort low`."))
|
||||
self.assertNotIn("rerun", prompts.resumed("", Route(Rung("m")), LADDER, None))
|
||||
self.assertNotIn("rerun", prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), LADDER, None, self.FACTS))
|
||||
self.assertIn(f"{both} After your first read", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 3, 4), LADDER, opus, self.FACTS))
|
||||
self.assertTrue(prompts.resumed("", Route(opus, 3, 2, 4), OPUS_PROJECT).endswith(both))
|
||||
self.assertTrue(prompts.resumed("", Route(Rung("claude-sonnet-5-5", "high"), 3, 2, 4), OPUS_PROJECT).endswith(both))
|
||||
self.assertTrue(prompts.fresh("Do it.", "## Goals", Route(opus), OPUS_PROJECT, self.FACTS).split(f"\n\n{self.FACTS}")[0].endswith(both))
|
||||
self.assertTrue(prompts.resumed("", Route(Rung("claude-fable-5-2", "low"), 3, 3, 4), OPUS_PROJECT).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort medium`, and a comprehension seat with `model fable, effort low`."))
|
||||
self.assertTrue(prompts.resumed("", Route(opus, 3, 2, 7), OPUS_PROJECT).endswith(" A reviewer that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort high`, and a comprehension seat with `model fable, effort low`."))
|
||||
self.assertTrue(prompts.resumed("", Route(Rung("claude-fable-5-2", "max"), 3, 4, 8), OPUS_PROJECT).endswith(" A comprehension seat that asks for a rerun one rung up reruns with the Agent tool set to `model fable, effort low`."))
|
||||
self.assertNotIn("rerun", prompts.resumed("", Route(Rung("m")), None))
|
||||
self.assertNotIn("rerun", prompts.fresh("Do it.", "## Goals", Route(Rung("m", "high"), 3, 4, 5), None, self.FACTS))
|
||||
self.assertIn(f"{both} After your first read", prompts.fresh("Do it.", "## Goals", Route(opus, 3, 2, 4), OPUS_PROJECT, self.FACTS))
|
||||
fable = Rung("claude-fable-5-2", "max")
|
||||
self.assertNotIn("rerun", prompts.resumed("", Route(fable), LADDER, fable))
|
||||
self.assertNotIn("rerun", prompts.resumed("", Route(fable), ProjectRung(LADDER, fable)))
|
||||
|
||||
def test_opens_a_fresh_session_by_name_and_resumes_an_open_one(self):
|
||||
opened = session.session_named("A proj loop-claude", pathlib.Path("/h/.claude/projects"))
|
||||
fresh = worker.worker_args(opened, Rung("m"), prompts.fresh("Do it.", "## Goals\n1. g\n\n## Audience\nme", Route(Rung("m")), LADDER, None, self.FACTS), resume=False)
|
||||
fresh = worker.worker_args(opened, Rung("m"), prompts.fresh("Do it.", "## Goals\n1. g\n\n## Audience\nme", Route(Rung("m")), None, self.FACTS), resume=False)
|
||||
self.assertEqual(fresh[fresh.index("--session-id") + 1], str(opened.id))
|
||||
self.assertEqual(fresh[fresh.index("--name") + 1], "A proj loop-claude")
|
||||
resumed = worker.worker_args(opened, Rung("m"), prompts.resumed("", Route(Rung("m")), LADDER, None), resume=True)
|
||||
resumed = worker.worker_args(opened, Rung("m"), prompts.resumed("", Route(Rung("m")), None), resume=True)
|
||||
self.assertEqual(resumed[resumed.index("--resume") + 1], str(opened.id))
|
||||
self.assertNotIn("--session-id", resumed)
|
||||
self.assertTrue(resumed[resumed.index("-p") + 1].startswith("Continue the task; the previous run ended before the chunk was finished. The chunk is finished once"))
|
||||
@@ -1944,7 +1949,7 @@ class JudgePrompt(unittest.TestCase):
|
||||
self.assertEqual(judge.quotes_checked(Unfinished("e", "new", entries_read=3), grounds), Unfinished("e", "new", entries_read=3))
|
||||
|
||||
def test_shows_a_typed_answer_before_the_step_up_as_the_humans(self):
|
||||
typed = prompts.typed("Which one?", "Use B.", Route(Rung("worker-model")), LADDER, None, False)
|
||||
typed = prompts.typed("Which one?", "Use B.", Route(Rung("worker-model")), None, False)
|
||||
entries = [{**human(typed), "promptSource": "sdk"}, assistant({"type": "text", "text": "round 3: 🔴"})]
|
||||
_, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", kinds(*entries), 1))
|
||||
self.assertIn("\n\n[human] Use B.\n\n[step] ", grounds.shown)
|
||||
@@ -2084,7 +2089,7 @@ class ChunkItems(unittest.TestCase):
|
||||
self.assertEqual(chunk_items.named([said(example)]), (297, 298))
|
||||
|
||||
def test_every_prompt_that_starts_or_resumes_a_chunk_asks_for_the_line(self):
|
||||
self.assertIn("on one line, as in `Chunk items: ", prompts.resumed("", Route(Rung("m")), LADDER, None))
|
||||
self.assertIn("on one line, as in `Chunk items: ", prompts.resumed("", Route(Rung("m")), None))
|
||||
|
||||
def test_reads_only_a_line_the_worker_opens_with_the_label_in_its_own_text(self):
|
||||
prompt = entries.LoopPrompt(text="Chunk items: 297 (L 3, E + I 5)")
|
||||
@@ -2315,7 +2320,7 @@ class WindowStarts(unittest.TestCase):
|
||||
fake = FakeEffects(git=["/r\n"])
|
||||
effects, env = fake.effects(), {"TMUX_PANE": "%4"}
|
||||
launcher = pathlib.Path("/b/loop-claude")
|
||||
cycle = unittest.mock.Mock(project="claude-opus-5-5")
|
||||
cycle = unittest.mock.Mock(project=OPUS_PROJECT)
|
||||
unfinished = unittest.mock.Mock(verdict=unittest.mock.Mock(spec=Unfinished))
|
||||
finished = unittest.mock.Mock(verdict=unittest.mock.Mock(spec=Finished))
|
||||
closed = [main.LoopRun(1, "/r", "do it", "", chunk="item-3"), main.LoopRun(2, "/r", "do it", "", chunk="item-3")]
|
||||
|
||||
+42
-34
@@ -42,7 +42,7 @@ class Rung:
|
||||
|
||||
|
||||
# The rungs a task's L steps its worker from the project's rung.
|
||||
WORKER_STEPS = {1: -1, 2: -1, 3: 0, 4: 1}
|
||||
WORKER_STEPS = {1: -1, 2: -1, 3: 1, 4: 2}
|
||||
# The rungs a task's E + I steps a review up from the project's rung; an E + I under 5 steps none.
|
||||
REVIEW_STEPS = {5: 1, 6: 2, 7: 3, 8: 4}
|
||||
|
||||
@@ -59,44 +59,15 @@ class Ladder:
|
||||
return Rung(model, effort)
|
||||
return Rung(family.model, effort) if effort else family
|
||||
|
||||
def project_rung(self, model: str) -> Rung | None:
|
||||
"""The model's cheapest rung, which for Opus 5.5 is its default effort; None for a model off the ladder."""
|
||||
return next((rung for rung in self.rungs if rung.model == model), None)
|
||||
|
||||
def worker_rung(self, project: Rung, likelihood: int) -> Rung:
|
||||
"""The rung a task's L picks for its worker, counted from the project's rung, stopping at either end of the ladder."""
|
||||
return self.climbed(project, WORKER_STEPS[likelihood])
|
||||
|
||||
def lowest_worker_rung(self, project: Rung) -> Rung:
|
||||
"""The lowest rung any L routes a worker to, counted from the project's rung."""
|
||||
return self.climbed(project, min(WORKER_STEPS.values()))
|
||||
|
||||
def planning_rung(self, project: Rung, likelihood: int) -> Rung:
|
||||
"""The rung of a task that files or scores todo.md items: the one its L picks, never below the project's rung."""
|
||||
return self.highest(project, self.worker_rung(project, likelihood))
|
||||
|
||||
def review_floor(self, project: Rung, escape_and_impact: int) -> Rung:
|
||||
"""The rung a task's E + I picks for a review, counted up from the project's rung. An E + I of 0, for no task, picks the project's rung."""
|
||||
return self.climbed(project, REVIEW_STEPS.get(escape_and_impact, 0))
|
||||
def project_rung(self, model: str) -> "ProjectRung | None":
|
||||
"""The model's cheapest rung on this ladder, which for Opus 5.5 is its default effort; None for a model off the ladder."""
|
||||
rung = next((rung for rung in self.rungs if rung.model == model), None)
|
||||
return ProjectRung(self, rung) if rung else None
|
||||
|
||||
def stepped_up(self, rung: Rung) -> Rung:
|
||||
"""The rung above, or the top one; a rung off the ladder steps to the bottom one."""
|
||||
return self.climbed(rung, 1)
|
||||
|
||||
def review_rung(self, project: Rung, worker: Rung, escape_and_impact: int, kind: ReviewKind, markdown_only: bool = False) -> Rung:
|
||||
"""The rung a review of this kind runs on in a chunk whose worker runs on worker and whose item has that E + I; markdown_only where the branch changes only Markdown no agent follows.
|
||||
|
||||
A pass's reviewer runs on reviewers_rung; no replay has qualified a lower rung for any pass, docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`. A seat and the security pass run on the project's rung, so the seats score every chunk on one rung, and the security pass spends no more than the project chose.
|
||||
"""
|
||||
return project if kind in ("seat", "security") else self.reviewers_rung(project, worker, escape_and_impact, markdown_only)
|
||||
|
||||
def reviewers_rung(self, project: Rung, worker: Rung, escape_and_impact: int, markdown_only: bool = False) -> Rung:
|
||||
"""The rung of each pass's reviewer: the higher of the worker's rung and review_floor's, which is never below the project's.
|
||||
|
||||
markdown_only means the branch changes only Markdown no agent follows. Such a branch holds no code or config for a higher rung to catch a defect in, so its reviewer runs on the project's rung: skills/larv-review/SKILL.md → "Markdown only".
|
||||
"""
|
||||
return project if markdown_only else self.highest(worker, self.review_floor(project, escape_and_impact))
|
||||
|
||||
def highest(self, *rungs: Rung) -> Rung:
|
||||
"""The most competent of the rungs; one off the ladder ranks below every rung on it."""
|
||||
return max(rungs, key=self.position)
|
||||
@@ -106,3 +77,40 @@ class Ladder:
|
||||
|
||||
def position(self, rung: Rung) -> int:
|
||||
return self.rungs.index(rung) if rung in self.rungs else -1
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class ProjectRung:
|
||||
"""A project's rung on the ladder it stands on, from which each task's L and E + I count their steps."""
|
||||
ladder: Ladder
|
||||
rung: Rung
|
||||
|
||||
def worker_rung(self, likelihood: int) -> Rung:
|
||||
"""The rung a task's L picks for its worker, counted from the project's rung, stopping at either end of the ladder."""
|
||||
return self.ladder.climbed(self.rung, WORKER_STEPS[likelihood])
|
||||
|
||||
def lowest_worker_rung(self) -> Rung:
|
||||
"""The lowest rung any L routes a worker to, counted from the project's rung."""
|
||||
return self.ladder.climbed(self.rung, min(WORKER_STEPS.values()))
|
||||
|
||||
def planning_rung(self, likelihood: int) -> Rung:
|
||||
"""The rung of a task that files or scores todo.md items: the one its L picks, never below the project's rung."""
|
||||
return self.ladder.highest(self.rung, self.worker_rung(likelihood))
|
||||
|
||||
def review_floor(self, escape_and_impact: int) -> Rung:
|
||||
"""The rung a task's E + I picks for a review, counted up from the project's rung. An E + I of 0, for no task, picks the project's rung."""
|
||||
return self.ladder.climbed(self.rung, REVIEW_STEPS.get(escape_and_impact, 0))
|
||||
|
||||
def review_rung(self, worker: Rung, escape_and_impact: int, kind: ReviewKind, markdown_only: bool = False) -> Rung:
|
||||
"""The rung a review of this kind runs on in a chunk whose worker runs on worker and whose item has that E + I; markdown_only where the branch changes only Markdown no agent follows.
|
||||
|
||||
A pass's reviewer runs on reviewers_rung; no replay has qualified a lower rung for any pass, docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`. A seat and the security pass run on the project's rung, so the seats score every chunk on one rung, and the security pass spends no more than the project chose.
|
||||
"""
|
||||
return self.rung if kind in ("seat", "security") else self.reviewers_rung(worker, escape_and_impact, markdown_only)
|
||||
|
||||
def reviewers_rung(self, worker: Rung, escape_and_impact: int, markdown_only: bool = False) -> Rung:
|
||||
"""The rung of each pass's reviewer: the higher of the worker's rung and review_floor's, which is never below the project's.
|
||||
|
||||
markdown_only means the branch changes only Markdown no agent follows. Such a branch holds no code or config for a higher rung to catch a defect in, so its reviewer runs on the project's rung: skills/larv-review/SKILL.md → "Markdown only".
|
||||
"""
|
||||
return self.rung if markdown_only else self.ladder.highest(worker, self.review_floor(escape_and_impact))
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
import dataclasses
|
||||
|
||||
from model_routing.route import HIGHEST_ESCAPE_AND_IMPACT, Route
|
||||
from model_routing.ladder import Ladder, Rung
|
||||
from model_routing.ladder import ProjectRung, Rung
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
@@ -17,12 +17,12 @@ class ReviewChoice:
|
||||
higher: dict[Rung, int]
|
||||
|
||||
|
||||
def chosen(ladder: Ladder, project: Rung, route: Route) -> ReviewChoice:
|
||||
"""The rung each kind of review runs on for a chunk whose route is route, on a project whose rung on ladder is project."""
|
||||
reviewers = ladder.reviewers_rung(project, route.rung, route.reviewed_on())
|
||||
def chosen(project: ProjectRung, route: Route) -> ReviewChoice:
|
||||
"""The rung each kind of review runs on for a chunk whose route is route, on a project whose rung is project."""
|
||||
reviewers = project.reviewers_rung(route.rung, route.reviewed_on())
|
||||
higher: dict[Rung, int] = {}
|
||||
for escape_and_impact in range(route.reviewed_on() + 1, HIGHEST_ESCAPE_AND_IMPACT + 1):
|
||||
if (rung := ladder.reviewers_rung(project, route.rung, escape_and_impact)) != reviewers:
|
||||
if (rung := project.reviewers_rung(route.rung, escape_and_impact)) != reviewers:
|
||||
higher.setdefault(rung, escape_and_impact)
|
||||
markdown_only = ladder.reviewers_rung(project, route.rung, route.reviewed_on(), markdown_only=True)
|
||||
return ReviewChoice(reviewers, markdown_only, ladder.review_rung(project, route.rung, route.reviewed_on(), "seat"), ladder.review_rung(project, route.rung, route.reviewed_on(), "security"), higher)
|
||||
markdown_only = project.reviewers_rung(route.rung, route.reviewed_on(), markdown_only=True)
|
||||
return ReviewChoice(reviewers, markdown_only, project.review_rung(route.rung, route.reviewed_on(), "seat"), project.review_rung(route.rung, route.reviewed_on(), "security"), higher)
|
||||
|
||||
@@ -5,37 +5,38 @@ import unittest
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from plan.todo_table import LEVELS
|
||||
from model_routing.ladder import ProjectRung, Rung, WORKER_STEPS
|
||||
from model_routing.ladder_file import read_ladder, SHIPPED
|
||||
from review_rules.review_passes import REVIEWERS
|
||||
from model_routing.ladder import Rung, WORKER_STEPS
|
||||
|
||||
# The ladder this setup ships, whose rungs the tests below name.
|
||||
LADDER = read_ladder(SHIPPED)
|
||||
OPUS = Rung("claude-opus-5-5", "medium")
|
||||
PROJECT = ProjectRung(LADDER, OPUS)
|
||||
FABLE_LOW = Rung("claude-fable-5-2", "low")
|
||||
|
||||
|
||||
class Ladder(unittest.TestCase):
|
||||
def test_puts_a_project_on_its_models_cheapest_rung_and_none_off_the_ladder(self):
|
||||
self.assertEqual(LADDER.project_rung("claude-opus-5-5"), OPUS)
|
||||
self.assertEqual(LADDER.project_rung("claude-sonnet-5-5"), Rung("claude-sonnet-5-5", "high"))
|
||||
self.assertEqual(LADDER.project_rung("claude-fable-5-2"), FABLE_LOW)
|
||||
self.assertEqual(LADDER.project_rung("claude-opus-5-5"), PROJECT)
|
||||
self.assertEqual(LADDER.project_rung("claude-sonnet-5-5"), ProjectRung(LADDER, Rung("claude-sonnet-5-5", "high")))
|
||||
self.assertEqual(LADDER.project_rung("claude-fable-5-2"), ProjectRung(LADDER, FABLE_LOW))
|
||||
self.assertIsNone(LADDER.project_rung("claude-opus-5"))
|
||||
self.assertIsNone(LADDER.project_rung("opus"))
|
||||
|
||||
def test_steps_a_workers_rung_by_its_l_from_the_projects_rung(self):
|
||||
position = LADDER.rungs.index(OPUS)
|
||||
self.assertEqual([LADDER.rungs.index(LADDER.worker_rung(OPUS, likelihood)) - position for likelihood in range(1, 5)], [-1, -1, 0, 1])
|
||||
self.assertEqual([LADDER.rungs.index(PROJECT.worker_rung(likelihood)) - position for likelihood in range(1, 5)], [-1, -1, 1, 2])
|
||||
|
||||
def test_steps_a_review_up_by_its_e_plus_i_from_the_projects_rung_and_never_down(self):
|
||||
position = LADDER.rungs.index(OPUS)
|
||||
self.assertEqual([LADDER.rungs.index(LADDER.review_floor(OPUS, total)) - position for total in range(2, 9)], [0, 0, 0, 1, 2, 3, 4])
|
||||
self.assertEqual(LADDER.review_floor(OPUS, 0), OPUS)
|
||||
self.assertEqual([LADDER.rungs.index(PROJECT.review_floor(total)) - position for total in range(2, 9)], [0, 0, 0, 1, 2, 3, 4])
|
||||
self.assertEqual(PROJECT.review_floor(0), OPUS)
|
||||
|
||||
def test_a_step_past_either_end_stops_there(self):
|
||||
self.assertEqual(LADDER.worker_rung(OPUS, 1), Rung("claude-sonnet-5-5", "high"))
|
||||
self.assertEqual(LADDER.worker_rung(LADDER.rungs[0], 1), LADDER.rungs[0])
|
||||
self.assertEqual(LADDER.review_floor(Rung("claude-opus-5-5", "xhigh"), 8), LADDER.rungs[-1])
|
||||
self.assertEqual(PROJECT.worker_rung(1), Rung("claude-sonnet-5-5", "high"))
|
||||
self.assertEqual(ProjectRung(LADDER, LADDER.rungs[0]).worker_rung(1), LADDER.rungs[0])
|
||||
self.assertEqual(ProjectRung(LADDER, Rung("claude-opus-5-5", "xhigh")).review_floor(8), LADDER.rungs[-1])
|
||||
self.assertEqual(LADDER.stepped_up(LADDER.rungs[-1]), LADDER.rungs[-1])
|
||||
self.assertEqual(LADDER.stepped_up(OPUS), FABLE_LOW)
|
||||
|
||||
@@ -43,11 +44,11 @@ class Ladder(unittest.TestCase):
|
||||
self.assertEqual(set(WORKER_STEPS), set(range(1, LEVELS["L"] + 1)))
|
||||
|
||||
def test_names_the_lowest_rung_any_l_routes_a_worker_to(self):
|
||||
self.assertEqual(LADDER.lowest_worker_rung(OPUS), Rung("claude-sonnet-5-5", "high"))
|
||||
self.assertEqual(PROJECT.lowest_worker_rung(), Rung("claude-sonnet-5-5", "high"))
|
||||
|
||||
def test_plans_on_the_rung_l_picks_but_never_below_the_projects(self):
|
||||
self.assertEqual(LADDER.planning_rung(OPUS, 1), OPUS)
|
||||
self.assertEqual(LADDER.planning_rung(OPUS, 4), FABLE_LOW)
|
||||
self.assertEqual(PROJECT.planning_rung(1), OPUS)
|
||||
self.assertEqual(PROJECT.planning_rung(4), Rung("claude-fable-5-2", "medium"))
|
||||
|
||||
def test_names_each_rung_by_a_family_the_agent_tool_takes(self):
|
||||
self.assertEqual({rung.agent_args().split(",")[0] for rung in LADDER.rungs}, {"model sonnet", "model opus", "model fable"})
|
||||
@@ -95,14 +96,14 @@ class Ladder(unittest.TestCase):
|
||||
|
||||
def test_puts_reviewers_on_the_highest_of_the_projects_rung_the_workers_and_the_one_e_plus_i_picks(self):
|
||||
sonnet, fable = LADDER.rungs[0], LADDER.rungs[-1]
|
||||
self.assertEqual(LADDER.review_rung(OPUS, sonnet, 4, "stability"), OPUS)
|
||||
self.assertEqual(LADDER.review_rung(OPUS, sonnet, 5, "stability"), FABLE_LOW)
|
||||
self.assertEqual(LADDER.review_rung(OPUS, fable, 5, "stability"), fable)
|
||||
self.assertEqual(LADDER.review_rung(OPUS, FABLE_LOW, 7, "stability"), Rung("claude-opus-5-5", "xhigh"))
|
||||
self.assertEqual(PROJECT.review_rung(sonnet, 4, "stability"), OPUS)
|
||||
self.assertEqual(PROJECT.review_rung(sonnet, 5, "stability"), FABLE_LOW)
|
||||
self.assertEqual(PROJECT.review_rung(fable, 5, "stability"), fable)
|
||||
self.assertEqual(PROJECT.review_rung(FABLE_LOW, 7, "stability"), Rung("claude-opus-5-5", "xhigh"))
|
||||
|
||||
def test_keeps_the_seats_and_the_security_pass_on_the_projects_rung_whatever_the_worker_or_e_plus_i(self):
|
||||
self.assertEqual([LADDER.review_rung(OPUS, LADDER.rungs[-1], 8, kind) for kind in (*REVIEWERS, "seat", "security")], [LADDER.rungs[-1]] * len(REVIEWERS) + [OPUS, OPUS])
|
||||
self.assertEqual([PROJECT.review_rung(LADDER.rungs[-1], 8, kind) for kind in (*REVIEWERS, "seat", "security")], [LADDER.rungs[-1]] * len(REVIEWERS) + [OPUS, OPUS])
|
||||
|
||||
def test_puts_every_review_of_a_branch_changing_only_markdown_no_agent_follows_on_the_projects_rung(self):
|
||||
self.assertEqual([LADDER.review_rung(OPUS, LADDER.rungs[-1], 8, kind, markdown_only=True) for kind in (*REVIEWERS, "seat", "security")], [OPUS] * (len(REVIEWERS) + 2))
|
||||
self.assertEqual(LADDER.review_rung(OPUS, LADDER.rungs[0], 4, "stability", markdown_only=True), OPUS)
|
||||
self.assertEqual([PROJECT.review_rung(LADDER.rungs[-1], 8, kind, markdown_only=True) for kind in (*REVIEWERS, "seat", "security")], [OPUS] * (len(REVIEWERS) + 2))
|
||||
self.assertEqual(PROJECT.review_rung(LADDER.rungs[0], 4, "stability", markdown_only=True), OPUS)
|
||||
|
||||
@@ -1,19 +1,20 @@
|
||||
import unittest
|
||||
|
||||
from model_routing.review_choice import chosen
|
||||
from model_routing.ladder import ProjectRung, Rung
|
||||
from model_routing.ladder_file import read_ladder, SHIPPED
|
||||
from model_routing.route import Route
|
||||
from model_routing.ladder import Rung
|
||||
|
||||
LADDER = read_ladder(SHIPPED)
|
||||
OPUS = Rung("claude-opus-5-5", "medium")
|
||||
PROJECT = ProjectRung(LADDER, OPUS)
|
||||
FABLE = Rung("claude-fable-5-2", "max")
|
||||
|
||||
|
||||
class Chosen(unittest.TestCase):
|
||||
def test_names_no_higher_rung_on_the_top_one(self):
|
||||
self.assertEqual(chosen(LADDER, OPUS, Route(FABLE, 3, 4, 5)).higher, {})
|
||||
self.assertEqual(chosen(PROJECT, Route(FABLE, 3, 4, 5)).higher, {})
|
||||
|
||||
def test_reviews_a_branch_changing_only_markdown_on_the_projects_rung_and_the_seats_and_security_pass_there_always(self):
|
||||
choice = chosen(LADDER, OPUS, Route(FABLE, 3, 4, 5))
|
||||
choice = chosen(PROJECT, Route(FABLE, 3, 4, 5))
|
||||
self.assertEqual((choice.reviewers, choice.markdown_only, choice.seat, choice.security), (FABLE, OPUS, OPUS, OPUS))
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import unittest
|
||||
|
||||
from model_routing.worker_rungs import environment, handed, hands_rungs
|
||||
from model_routing.ladder import Ladder, Rung
|
||||
from model_routing.ladder import Ladder, ProjectRung, Rung
|
||||
from model_routing.ladder_file import read_ladder, SHIPPED
|
||||
from model_routing.route import Route
|
||||
|
||||
@@ -10,7 +10,7 @@ LADDER = read_ladder(SHIPPED)
|
||||
|
||||
|
||||
class Environment(unittest.TestCase):
|
||||
PROJECT = Rung("claude-opus-5-5", "medium")
|
||||
PROJECT = ProjectRung(LADDER, Rung("claude-opus-5-5", "medium"))
|
||||
WORKER = Rung("claude-sonnet-5-5", "high")
|
||||
|
||||
def test_the_project_and_the_workers_rungs_and_the_e_plus_i_reviews_take_are_named_in_the_environment(self):
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
loop-claude sets these variables for its worker and for the claude an interrupt resumes; a human's shell does not, and rule-hook then checks no review launch."""
|
||||
import typing
|
||||
|
||||
from model_routing.ladder import Ladder, Rung
|
||||
from model_routing.ladder import Ladder, ProjectRung, Rung
|
||||
from model_routing.route import Route
|
||||
|
||||
PROJECT_RUNG = "LOOP_CLAUDE_PROJECT_RUNG"
|
||||
@@ -12,9 +12,9 @@ WORKER_RUNG = "LOOP_CLAUDE_WORKER_RUNG"
|
||||
ESCAPE_AND_IMPACT = "LOOP_CLAUDE_ESCAPE_AND_IMPACT"
|
||||
|
||||
|
||||
def environment(project: Rung | None, route: Route) -> dict[str, str]:
|
||||
def environment(project: ProjectRung | None, route: Route) -> dict[str, str]:
|
||||
"""The variables a worker's claude on route starts with. A project off the ladder gets an empty project rung, so none is inherited from the environment."""
|
||||
return {PROJECT_RUNG: str(project) if project else "", WORKER_RUNG: str(route.rung), ESCAPE_AND_IMPACT: str(route.reviewed_on())}
|
||||
return {PROJECT_RUNG: str(project.rung) if project else "", WORKER_RUNG: str(route.rung), ESCAPE_AND_IMPACT: str(route.reviewed_on())}
|
||||
|
||||
|
||||
def hands_rungs(environ: typing.Mapping[str, str]) -> bool:
|
||||
@@ -22,10 +22,11 @@ def hands_rungs(environ: typing.Mapping[str, str]) -> bool:
|
||||
return bool(environ.get(PROJECT_RUNG) and environ.get(WORKER_RUNG))
|
||||
|
||||
|
||||
def handed(environ: typing.Mapping[str, str], ladder: Ladder) -> tuple[Rung | None, Rung | None, int]:
|
||||
"""The project's and the worker's rungs, each None when unset or not on ladder, and the reviews' E + I, 0 when unset or not a whole number. ladder is the project's. An older loop-claude hands no E + I, and 0 steps no review up."""
|
||||
def handed(environ: typing.Mapping[str, str], ladder: Ladder) -> tuple[ProjectRung | None, Rung | None, int]:
|
||||
"""The project's rung and the worker's rung, each None when unset or not on ladder, and the reviews' E + I, 0 when unset or not a whole number. ladder is the project's. An older loop-claude hands no E + I, and 0 steps no review up."""
|
||||
escape_and_impact = environ.get(ESCAPE_AND_IMPACT, "")
|
||||
return on_the_ladder(environ.get(PROJECT_RUNG), ladder), on_the_ladder(environ.get(WORKER_RUNG), ladder), int(escape_and_impact) if escape_and_impact.isdecimal() else 0
|
||||
project = on_the_ladder(environ.get(PROJECT_RUNG), ladder)
|
||||
return ProjectRung(ladder, project) if project else None, on_the_ladder(environ.get(WORKER_RUNG), ladder), int(escape_and_impact) if escape_and_impact.isdecimal() else 0
|
||||
|
||||
|
||||
def on_the_ladder(value: str | None, ladder: Ladder) -> Rung | None:
|
||||
|
||||
@@ -123,4 +123,4 @@ def pass_and_model(ladder: Ladder, argv: list[str]) -> tuple[ReviewedPass, Rung,
|
||||
return None
|
||||
reviewed = next((kind for kind in REVIEWERS if kind == argv[0]), None)
|
||||
project = ladder.project_rung(argv[1])
|
||||
return (reviewed, project, argv[2:] == [OMIT]) if reviewed and project else None
|
||||
return (reviewed, project.rung, argv[2:] == [OMIT]) if reviewed and project else None
|
||||
|
||||
@@ -36,7 +36,7 @@ def reviewer_breaches(call: Call, environ: typing.Mapping[str, str], home: str)
|
||||
project, worker, escape_and_impact = handed(environ, ladder)
|
||||
if project is None or worker is None:
|
||||
return []
|
||||
required = ladder.review_rung(project, worker, escape_and_impact, kind, markdown_only(call.cwd))
|
||||
required = project.review_rung(worker, escape_and_impact, kind, markdown_only(call.cwd))
|
||||
named = named_rung(call.given, worker, ladder)
|
||||
if ladder.position(named) >= ladder.position(required):
|
||||
return []
|
||||
@@ -62,10 +62,10 @@ def skill_breaches(call: Call, environ: typing.Mapping[str, str], home: str) ->
|
||||
project, worker, _ = handed(environ, ladder)
|
||||
if project is None or worker is None:
|
||||
return []
|
||||
if not call.subagent and ladder.position(worker) >= ladder.position(project):
|
||||
if not call.subagent and ladder.position(worker) >= ladder.position(project.rung):
|
||||
return []
|
||||
runs_on = "on a rung rule-hook cannot see" if call.subagent else f"in this context on {worker}, below {project}"
|
||||
return [Breach("reviewer-rung", f"{skill} would run {runs_on}: invoke it from a subagent launched with {project.agent_args()}, its prompt opening with `{SECURITY_OPENER}`", call.cwd)]
|
||||
runs_on = "on a rung rule-hook cannot see" if call.subagent else f"in this context on {worker}, below {project.rung}"
|
||||
return [Breach("reviewer-rung", f"{skill} would run {runs_on}: invoke it from a subagent launched with {project.rung.agent_args()}, its prompt opening with `{SECURITY_OPENER}`", call.cwd)]
|
||||
|
||||
|
||||
def opening_prompt(transcript: str) -> str:
|
||||
|
||||
@@ -34,7 +34,7 @@ def agent_args(ladder: Ladder, model: str, likelihood: int, planning: bool = Fal
|
||||
project = ladder.project_rung(model)
|
||||
if project is None:
|
||||
return None
|
||||
return (ladder.planning_rung if planning else ladder.worker_rung)(project, likelihood).agent_args()
|
||||
return (project.planning_rung if planning else project.worker_rung)(likelihood).agent_args()
|
||||
|
||||
|
||||
def main(argv: list[str], home: pathlib.Path) -> int:
|
||||
|
||||
@@ -36,7 +36,7 @@ class SubagentRung(unittest.TestCase):
|
||||
def test_picks_the_rung_on_the_ladder_of_the_project_it_runs_in(self):
|
||||
checkout = self.project('[{"model": "claude-fable-5-2", "effort": "low"}, {"model": "claude-fable-5-2", "effort": "max"}]')
|
||||
(checkout / "nested").mkdir()
|
||||
# On the shipped ladder an L of 1 steps Fable 5.2 down to Opus 5.5, and an L of 4 up to its medium effort.
|
||||
# On the shipped ladder an L of 1 steps Fable 5.2 down to Opus 5.5, and an L of 4 up to Opus 5.5 at xhigh.
|
||||
self.assertEqual([self.run_tool("claude-fable-5-2", level, cwd=checkout / "nested").stdout for level in ("1", "4")], ["model fable, effort low\n", "model fable, effort max\n"])
|
||||
off = self.run_tool("claude-opus-5-5", "3", cwd=checkout)
|
||||
own = self.home / ".claude-projects" / "acme" / "routing-ladder.json"
|
||||
@@ -49,15 +49,15 @@ class SubagentRung(unittest.TestCase):
|
||||
self.assertTrue(refused.stderr.startswith(f"subagent-rung: cannot read the routing ladder {self.home / '.claude-projects' / 'acme' / 'routing-ladder.json'}: "), refused.stderr)
|
||||
|
||||
def test_prints_the_agent_tools_model_and_effort_for_the_rung_an_l_picks(self):
|
||||
self.assertEqual(main.agent_args(LADDER, "claude-opus-5-5", 3), "model opus, effort medium")
|
||||
self.assertEqual(main.agent_args(LADDER, "claude-opus-5-5", 3), "model fable, effort low")
|
||||
self.assertEqual(main.agent_args(LADDER, "claude-opus-5-5", 2), "model sonnet, effort high")
|
||||
self.assertEqual(main.agent_args(LADDER, "claude-opus-5-5", 4), "model fable, effort low")
|
||||
self.assertEqual(main.agent_args(LADDER, "claude-opus-5-5", 4), "model fable, effort medium")
|
||||
self.assertEqual(main.agent_args(LADDER, "claude-sonnet-5-5", 1), "model sonnet, effort high")
|
||||
self.assertIsNone(main.agent_args(LADDER, "claude-haiku-5-5", 1))
|
||||
|
||||
def test_a_planning_task_never_runs_below_the_projects_rung(self):
|
||||
self.assertEqual(main.agent_args(LADDER, "claude-opus-5-5", 1, planning=True), "model opus, effort medium")
|
||||
self.assertEqual(main.agent_args(LADDER, "claude-opus-5-5", 4, planning=True), "model fable, effort low")
|
||||
self.assertEqual(main.agent_args(LADDER, "claude-opus-5-5", 4, planning=True), "model fable, effort medium")
|
||||
self.assertEqual(self.run_tool("--plan", "claude-opus-5-5", "2").stdout, "model opus, effort medium\n")
|
||||
|
||||
def test_picks_nothing_for_a_model_off_the_ladder(self):
|
||||
@@ -72,7 +72,7 @@ class SubagentRung(unittest.TestCase):
|
||||
|
||||
def test_the_launcher_prints_the_rung_and_exits_0_on_the_ladder(self):
|
||||
on = self.run_tool("claude-opus-5-5", "4")
|
||||
self.assertEqual((on.returncode, on.stdout, on.stderr), (0, "model fable, effort low\n", ""))
|
||||
self.assertEqual((on.returncode, on.stdout, on.stderr), (0, "model fable, effort medium\n", ""))
|
||||
|
||||
def test_the_launcher_prints_nothing_and_says_why_and_exits_0_off_the_ladder(self):
|
||||
off = self.run_tool("claude-opus-5", "4")
|
||||
|
||||
+2
-2
@@ -630,7 +630,7 @@ Valid while the archived verdicts show what a judge should return, and until the
|
||||
|
||||
The rungs, least competent first: Sonnet 5.5 at high; Opus 5.5 at medium; Fable 5.2 at low and medium; Opus 5.5 at xhigh; Fable 5.2 at high and max. The 2026-11-11 note below put them so. Before it they were Sonnet 5.5 at high; Opus 5.5 at medium, high, xhigh and max; Fable 5.1 at max, after the 2026-10-29 note took Haiku 5.5 off. They first were: Haiku 5.5 at low, medium, high, xhigh and max; Sonnet 5.5 at high; Opus 5.5 at medium, high, xhigh and max; Fable 5.1 at max. The order and the cut come from the Artificial Analysis Intelligence Index (https://artificialanalysis.ai/leaderboards/models), which scores each model at each effort and gives a cost per task. A rung stays only where no cheaper rung scores as high, so a step up always buys more score (goal 1.4). The first cut, from the index read 2026-10-14, dropped Sonnet 5.5 at low, medium, xhigh and max, Opus 5.5 at low, and every Fable 5.1 rung, and every tie in the index fell among the dropped rungs. lilleman put Fable 5.1 at max back on top: the index scored it under Opus 5.5 at high, but Anthropic's model catalog names Fable the most capable model, and for the hardest tasks lilleman wants the highest quality at any price. Fable 5.2 at max tops the index outright, so that exception has ended.
|
||||
|
||||
A project's rung is its model's cheapest rung, which for Opus 5.5 is its default effort, medium. A model off the ladder doesn't route anything: claude runs it at the effort its settings give. A task's L steps its worker from the project's rung: 1 and 2 one down, 3 none, 4 one up. Its E + I steps its reviewers up from the project's rung: 4 or less none, 5 one up, 6 two up, 7 three up, 8 four up. A step past either end stops there. A worker errs more the more it changes and the newer that is, which L scores; review must look harder the further a defect would escape and the more it would break, which E + I scores (goals 1.1 and 1.4). Size raises L, so a large change runs its worker higher. The steps spread the values this repo's plan held when rescored on 2026-10-16, L 1 to 4 and E + I 2 to 7, so a worker on most items runs at or below the project's rung, and a review climbs only where a defect would escape far or break much. lilleman approved the steps on 2026-10-16 as a guess. `score-outcomes` reports how well L, E and I each order finished chunks by review rounds and 🔴 findings. A change to the ladder or the steps is lilleman's.
|
||||
A project's rung is its model's cheapest rung, which for Opus 5.5 is its default effort, medium. A model off the ladder doesn't route anything: claude runs it at the effort its settings give. A task's L steps its worker from the project's rung: 1 and 2 one down, 3 one up, 4 two up. Its E + I steps its reviewers up from the project's rung: 4 or less none, 5 one up, 6 two up, 7 three up, 8 four up. A step past either end stops there. A worker errs more the more it changes and the newer that is, which L scores; review must look harder the further a defect would escape and the more it would break, which E + I scores (goals 1.1 and 1.4). Size raises L, so a large change runs its worker higher. The steps spread the values this repo's plan held when rescored on 2026-10-16, L 1 to 4 and E + I 2 to 7, so a worker on most items runs below the project's rung, and a review climbs only where a defect would escape far or break much. lilleman approved the steps on 2026-10-16 as a guess, and raised L 3 and L 4 on 2026-11-14, as below. `score-outcomes` reports how well L, E and I each order finished chunks by review rounds and 🔴 findings. A change to the ladder or the steps is lilleman's.
|
||||
|
||||
2026-10-21, lilleman, settling the routing calls stamped 2026-10-21T20:33:29Z: L 1 steps one rung down, not two as before, so an Opus 5.5 project runs no worker on Haiku 5.5. A delegated subagent takes the same steps, so an L 1 search moves off Haiku 5.5 too: lilleman chose one set of steps for chunk workers and subagents alike, so loop-claude, `subagent-rung` and rule-hook read one table (code health). No figure yet shows whether Sonnet 5.5 searches better than Haiku 5.5, so this costs more per search for a gain nobody has measured (goal 1.4). The calls came from loop-claude's routing figures, measured at cycle 165 over 46 chunks. Chunks started on Haiku 5.5 at max took a mean 7.8 review rounds past each pass's first, against 2.8 on Sonnet 5.5 at high, and 0.2 unfinished verdicts, against 0.1. The figures also called for moving workers off Sonnet 5.5 at high, whose chunks took 0.1 unfinished verdicts against 0.0 on Opus 5.5 at medium. lilleman kept L 2 on Sonnet 5.5, since that gap rests on one or two verdicts.
|
||||
|
||||
@@ -644,7 +644,7 @@ A project's rung is its model's cheapest rung, which for Opus 5.5 is its default
|
||||
|
||||
2026-11-11, lilleman approved the rungs for Fable 5.2 from the index read that day. Its score and cost per task, for each effort from low to max: Sonnet 5.5 26 $0.51, 31 $0.78, 42 $1.64, 50 $4.09, 55 $10.02; Opus 5.5 35 $0.98, 48 $2.45, 52 $3.46, 57 $6.05, 59 $10.04; Fable 5.2 54 $3.10, 56 $3.80, 62 $9.08, 62 $14.83, 65 $26.54. The cut drops Opus 5.5 at high, since Fable 5.2 at low scores more for less; Opus 5.5 at max, since Fable 5.2 at high scores more for less; and Fable 5.2 at xhigh, which ties Fable 5.2 at high at a higher cost. Sonnet 5.5 at high stays the bottom rung, as the 2026-10-29 note chose. With Haiku 5.5 off, the cut alone would also keep Sonnet 5.5 at low and medium and Opus 5.5 at low, below it. On an Opus 5.5 project, an L 4 worker, an E + I 5 review and a worker's first escalation now run on Fable 5.2 at low, where they ran on Opus 5.5 at high; E + I 8 runs on Fable 5.2 at high, where it ran on Fable 5.1 at max. Three steps land on a rung the index scores lower than before: E + I 6 on Fable 5.2 at medium, 56 against Opus 5.5 at xhigh's 57; E + I 7 on Opus 5.5 at xhigh, 57 against Opus 5.5 at max's 59; and an L 3 worker's escalation stops, without lilleman's yes, at Fable 5.2 at medium, where it stopped at Opus 5.5 at xhigh. On a Sonnet 5.5 project, E + I 6 runs on Fable 5.2 at low, 54 against Opus 5.5 at high's 52; E + I 7 on Fable 5.2 at medium, 56 against Opus 5.5 at xhigh's 57; and E + I 8 on Opus 5.5 at xhigh, 57 against Opus 5.5 at max's 59. Fable 5.2 at max, on top, is reached only one rung up from Fable 5.2 at high, as by a reviewer's rerun. A Fable 5.2 project's rung is Fable 5.2 at low, so its L 1 and L 2 workers run on Opus 5.5 at medium.
|
||||
|
||||
2026-11-14, lilleman ruled that L 3 is to step one rung up and L 4 two up, in every project and for delegated subagents. The steps stated above, L 3 none and L 4 one up, hold until `bin/model_routing/ladder.py` changes, which `todo.md` item 393 does in 0.12.0, as lilleman chose on 2026-11-16. Then, on an Opus 5.5 project, an L 3 worker or subagent runs on Fable 5.2 at low, and an escalation stops without lilleman's yes at Opus 5.5 at xhigh. An L 4 one runs on Fable 5.2 at medium, and stops at Fable 5.2 at high. The ruling settles the two calls of the routing figures measured at 2026-11-14T00:59:50Z, which were not yet in `routing-calls-raised.md`; the chunk that settles them cites their stamp there. The figures came from 276 chunks finished since the 2026-10-29 reset. Chunks started on Opus 5.5 at medium, 79 of them, took a mean 0.8 review rounds past each pass's first, 0.3 unfinished verdicts and 0.2 later fixes. Chunks started on Fable 5.2 at low, 5 of them, took 0.4, 0.0 and 0.0. On an Opus 5.5 project only L 4 items start on Fable 5.2 at low, so it did better on harder items. Two things weaken the evidence: 5 chunks is the fewest a call allows, and the Opus 5.5 figures include chunks from before Fable 5.2 joined the ladder. On the 2026-11-11 index, an L 3 task's cost rises about 1.3 times, from $2.45 to $3.10, for fewer rounds and fewer unfinished verdicts (goal 1.4). The chunks started on Sonnet 5.5 at high, 164 of the 276, don't move.
|
||||
2026-11-14, lilleman ruled that L 3 is to step one rung up and L 4 two up, in every project and for delegated subagents. The steps take effect in 0.12.0, as lilleman chose on 2026-11-16. On an Opus 5.5 project, an L 3 worker or subagent runs on Fable 5.2 at low, and an escalation stops without lilleman's yes at Opus 5.5 at xhigh. An L 4 one runs on Fable 5.2 at medium, and stops at Fable 5.2 at high. The ruling settles the two calls that `routing-calls-raised.md` stamps 2026-11-14T13:57:34Z, raised on the routing figures measured at 2026-11-14T00:59:50Z. The figures came from 276 chunks finished since the 2026-10-29 reset. Chunks started on Opus 5.5 at medium, 79 of them, took a mean 0.8 review rounds past each pass's first, 0.3 unfinished verdicts and 0.2 later fixes. Chunks started on Fable 5.2 at low, 5 of them, took 0.4, 0.0 and 0.0. Under the old steps, only L 4 items started on Fable 5.2 at low on an Opus 5.5 project, so it did better on harder items. Two things weaken the evidence: 5 chunks is the fewest a call allows, and the Opus 5.5 figures include chunks from before Fable 5.2 joined the ladder. On the 2026-11-11 index, an L 3 task's cost rises about 1.3 times, from $2.45 to $3.10, for fewer rounds and fewer unfinished verdicts (goal 1.4). The chunks started on Sonnet 5.5 at high, 164 of the 276, don't move.
|
||||
|
||||
2026-11-15, lilleman asked for the ladder as a file that a project can replace, on a day when an outage left only Fable 5.2 answering: the rungs were a constant in `bin/model_routing/ladder.py`, so no project could leave Sonnet 5.5 and Opus 5.5 without a release. The shipped ladder is `bin/model_routing/routing-ladder.json`, a JSON list of rungs, least competent first. A project that keeps a `routing-ladder.json` in its config's folder, `~/.claude-projects/<key>/`, runs on that ladder whole, and the steps count on it as on the shipped one. Every project still runs the same routing rules and steps, so the workflow of goal 1.3 holds; the ladder joins what a project's config folder may supply, which that goal's list does not name yet. lilleman chose a file per project over one per machine, so a single project can differ. A ladder holds one model of each family, since the Agent tool launches a subagent or a review by its family's name. The project's file is lilleman's to write, as any change to the ladder is, and the loop never commits it. loop-claude reads the project's ladder as each cycle starts; `subagent-rung` and rule-hook's `reviewer-rung` read it each time they run, from the checkout they run in. A ladder file that holds a fault stops each of them, since work routed on the shipped ladder in its place could run on a model the project cannot reach. The routing figures, `judge-replay` and `review-replay` span every project, so they read the shipped ladder: a rung only a project's own ladder holds is left out of the figures' calls, and no replay qualifies it. lilleman shipped the change as 0.10.1, a patch cut from the tag v0.10.0, on the day of the outage: it adds a file format and an exit 1 to `subagent-rung`, which the technical principles' semantic versioning gives a minor, and lilleman asked for the patch "even if that breaks semver", so that the release in use could run again without taking the unreleased work on `main`.
|
||||
|
||||
|
||||
@@ -815,7 +815,7 @@ dl.gloss dt { font-weight: 700; } dl.gloss dd { margin: 0; color: var(--muted);
|
||||
<div>
|
||||
<h3>How a rung is picked</h3>
|
||||
<ul class="tight">
|
||||
<li><b>Worker:</b> L 1–2 one rung down, L 3 the project's rung, L 4 one up.</li>
|
||||
<li><b>Worker:</b> L 1–2 one rung down, L 3 one up, L 4 two up.</li>
|
||||
<li><b>Reviewers:</b> never below the project's rung or the worker's. E + I of 5 or more steps them up, one rung per point.</li>
|
||||
<li><b>Readers and security:</b> always the project's rung. <b>The judge:</b> never below it.</li>
|
||||
</ul>
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
|
||||
| ID | Release | Blocked by | Exempt | Item | L | E | I | R | S | Moves | Score |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 387 | 0.12.0 | | | **Bind a project's rung to its ladder in one value, so the routing functions and `Cycle` take one argument for both.** | 3 | 1 | 3 | 5 | 5 | code health by 3 | 2.9 |
|
||||
| 338 | 0.12.0 | | owner | **Script each step that 0.9.0's chunks repeated by hand and a program can compute, filing one item per step, and file this audit again for the next release.** | 1 | 2 | 1 | 2 | 2 | 1.4 by 3, 1.5 by 3 | 2.3 |
|
||||
| 299 | 0.12.0 | | | **Replay the archived judge verdicts through code rules and a classifier trained locally on the archive, and file an item for each verdict kind one of them decides with no missed interrupt.** | 2 | 2 | 2 | 4 | 3 | 1.4 by 3 | 1.0 |
|
||||
| 311 | 0.12.0 | | | **Hand each worker's claude a Bash output ceiling that saves longer output to a file, and refuse a Read of a session transcript under `~/.claude/projects`.** | 2 | 2 | 3 | 5 | 3 | 1.4 by 3 | 1.0 |
|
||||
@@ -20,7 +19,6 @@
|
||||
| 339 | 0.12.0 | | | **Lower the reading cost of the files a chunk reads most before its first commit.** | 2 | 2 | 1 | 3 | 3 | 1.4 by 3 | 1.0 |
|
||||
| 341 | 0.12.0 | | | **Hand each worker's claude a lower Read limit through `CLAUDE_CODE_FILE_READ_MAX_OUTPUT_TOKENS`, kept only where replays of the archived review rounds find what two unchanged replays find.** | 2 | 2 | 3 | 5 | 3 | 1.4 by 3 | 1.0 |
|
||||
| 310 | 0.12.0 | | | **Compact a waiting worker's session shortly before its prompt cache expires, only when the human has not answered an interrupt by then.** | 3 | 2 | 3 | 6 | 4 | 1.4 by 3 | 0.9 |
|
||||
| 393 | 0.12.0 | | decision | **Step an L 3 worker one rung up and an L 4 worker two up from the project's rung, as lilleman ruled on 2026-11-14.** | 2 | 1 | 3 | 4 | 4 | 1.4 by 3 | 0.9 |
|
||||
| 298 | 0.12.0 | | | **Send the judge only the decision entries and the transcript tail a verdict needs, and keep the cut only where `judge-replay` changes no verdict kind.** | 3 | 3 | 3 | 7 | 4 | 1.4 by 2, 1.5 by 1 | 0.9 |
|
||||
| 365 | 0.13.0 | | | **Have `score-outcomes` read the finished and dropped items of a plan outside the checkout, from the project repo's history or from Jira's issue changelogs.** | 3 | 2 | 2 | 5 | 4 | 1.6 by 2, 1.1 by 2 | 1.2 |
|
||||
| 368 | 0.13.0 | | | **Call a release review due, call a release ready to cut, and tell whether a chunk owes a scoring run, for a plan outside the checkout, from the project repo's history or from Jira.** | 3 | 2 | 2 | 5 | 4 | 1.6 by 2, 1.1 by 2 | 1.2 |
|
||||
@@ -67,27 +65,6 @@
|
||||
|
||||
## Details
|
||||
|
||||
### 387. Bind a project's rung to its ladder in one value, so the routing functions and `Cycle` take one argument for both.
|
||||
|
||||
Filed by PR #553, which made the routing ladder a file a project may replace. Its stability round 1 found two places the comprehension seats name as hard, touched and no easier to read: `bin/loop_claude/routing.py` `unfinished_route` took `ladder` as a fourth parameter, and `Cycle` in `bin/loop_claude/cycle/run.py` gained the field `ladder`. The patch threaded the ladder as an argument, since a wider change did not fit a patch cut during an outage.
|
||||
|
||||
12 functions in `routing.py` take `ladder` only to pass it on beside `project`, and 6 of `Ladder`'s methods in `bin/model_routing/ladder.py` take `project` first. One value that holds the project's rung with the ladder it stands on removes the extra argument from each, and gives `Cycle` back the field count it had. Make `unfinished_route` and `Cycle` easier to read while there: both are places the seats found hard.
|
||||
|
||||
Touches:
|
||||
|
||||
- `bin/model_routing/ladder.py`: `Ladder`
|
||||
- `bin/loop_claude/routing.py`
|
||||
- `bin/loop_claude/cycle/run.py`: `Cycle`
|
||||
- `bin/loop_claude/loop/main.py`
|
||||
- `bin/loop_claude/cycle/prompts.py`
|
||||
- `bin/loop_claude/loop/chunk_start.py`
|
||||
- `bin/loop_claude/loop/close.py`
|
||||
- `bin/loop_claude/escalation_mark.py`
|
||||
- `bin/model_routing/review_choice.py`
|
||||
- `bin/rule_hook/reviewer_rung.py`
|
||||
- `bin/judge_replay/main.py`
|
||||
- `bin/loop_claude/test_loop_claude.py`
|
||||
|
||||
### 338. Script each step that 0.9.0's chunks repeated by hand and a program can compute, filing one item per step, and file this audit again for the next release.
|
||||
|
||||
lilleman asked, on 2026-10-16, for a standing audit each release, so the loop keeps moving work a program does as well from the model to scripts (goals 1.4 and 1.5). This audit reads the archived transcripts of the chunks that shipped release 0.9.0 for every step that chunks repeated by hand, before, during or after review. It keeps the steps whose answer a program computes the same way each time, such as a lookup, a check, a prompt assembled for a reviewer, a cleanup. File one item per such step, with the time and tokens it cost, into the next release with items left. Leave a step that needs judgement with the model. Run `command-patterns 0.9.0` first; its table names the patterns, and `read_threads` and `calls` in `bin/command_patterns/tally.py` open the calls of one pattern, or `command-patterns 0.9.0 --calls <file>` dumps them all as JSON lines. The 0.7.0 audit found that most steps its chunks repeated were already scripted by that release's own tools, so a step a shipped item already covers is not filed.
|
||||
@@ -181,19 +158,6 @@ Touches:
|
||||
- `bin/loop_claude/test_loop_claude.py`: `FakeEffects`, `Interrupts`
|
||||
- `bin/loop_effects/test_effects.py`
|
||||
|
||||
### 393. Step an L 3 worker one rung up and an L 4 worker two up from the project's rung, as lilleman ruled on 2026-11-14.
|
||||
|
||||
lilleman ruled on 2026-11-14 (`docs/decisions.md` → the routing ladder entry, the 2026-11-14 note) that an L 3 task steps one rung up from the project's rung and an L 4 task two up, in every project and for delegated subagents. `bin/model_routing/ladder.py`'s `WORKER_STEPS` still reads `{1: -1, 2: -1, 3: 0, 4: 1}`. The 0.11.0 release review's principles pass found the gap, and lilleman chose on 2026-11-16 to file it for 0.12.0.
|
||||
|
||||
Set `WORKER_STEPS` to `{1: -1, 2: -1, 3: 1, 4: 2}`. Then re-derive, from the ruling, the expected rungs of the 26 routing tests that pin the old steps: `bin/model_routing/test_ladder.py`, `bin/subagent_rung/test_subagent_rung.py`, and the routing, step and worker-argument tests in `bin/loop_claude/test_loop_claude.py` and `bin/loop_claude/test_black_box.py`. On an Opus 5.5 project, an L 3 worker or subagent then runs on Fable 5.2 at low, and an escalation stops without lilleman's yes at Opus 5.5 at xhigh; an L 4 one runs on Fable 5.2 at medium, and stops at Fable 5.2 at high. Reword the 2026-11-14 note's "hold until" sentence once the steps change, and cite the routing figures' stamp 2026-11-14T00:59:50Z in `routing-calls-raised.md` as settled.
|
||||
|
||||
Touches:
|
||||
- `bin/model_routing/ladder.py`: `WORKER_STEPS` (small)
|
||||
- `bin/model_routing/test_ladder.py`: `Ladder`
|
||||
- `bin/subagent_rung/test_subagent_rung.py`: `SubagentRung`
|
||||
- `bin/loop_claude/test_loop_claude.py`: `Routing`, `Steps`, `WorkerArgs`
|
||||
- `bin/loop_claude/test_black_box.py`: `Models`
|
||||
|
||||
### 298. Send the judge only the decision entries and the transcript tail a verdict needs, and keep the cut only where `judge-replay` changes no verdict kind.
|
||||
|
||||
153 `Judge cost:` entries total $109.76, a median of about $0.76 a verdict at every rung, and the cost follows the size of the records, not the transcript: about 76k characters in log and 188k in claude-config. In 224 of 251 recent verdicts (89%) the evidence sits in the worker's final message. A missed interrupt is the costly error, so the bar is zero changed interrupts and zero new rejections across the replay. The judge is about 4.5% of dollars, so the saving is small; each cycle also gets faster.
|
||||
|
||||
Reference in New Issue
Block a user