Answer architecture round 1: 1 🔴, 5 🟡; track bin/model_routing, hand over the route by its rung, share the R + C mapping, record the routing item, and have the judge report R + C plainly
This commit is contained in:
@@ -35,6 +35,8 @@
|
||||
!bin/metric-bench
|
||||
!bin/metric_bench/
|
||||
!bin/metric_bench/*.py
|
||||
!bin/model_routing/
|
||||
!bin/model_routing/*.py
|
||||
!bin/project_key.py
|
||||
!bin/rank_todo.py
|
||||
!bin/reading-cost
|
||||
|
||||
@@ -11,6 +11,8 @@ from loop_claude.project.records import HUMAN_RULE_NAMES
|
||||
from loop_claude.cycle.retry import Limited, retried
|
||||
from loop_claude.stop import Stop
|
||||
|
||||
# Routes the next cycle's worker; loop-claude never steps it down, so the item the chunk was routed on may count too.
|
||||
BUNDLED = "The highest R + C the agent states for a todo.md item it took into this chunk; 0 when it states none."
|
||||
# At 3 to 4 characters a token, 100,000 to 133,000 tokens, which leaves a 200,000-token context room for the reply; the largest session rendered here was 364,000 characters on 2026-10-03.
|
||||
JUDGE_PROMPT_CHARS = 400000
|
||||
JUDGE_RULES = (
|
||||
@@ -43,15 +45,14 @@ JUDGE_RULES = (
|
||||
f"the transcript shows the agent writing it into {ANSWER_DESTINATIONS}; writing it into "
|
||||
f"{either(HUMAN_RULE_NAMES)} does not count. finished=false when work remains that a further run can "
|
||||
"continue: an open PR, a review not yet run, CI not yet reported, an answer not yet recorded.\n"
|
||||
"bundled is the highest R + C the agent says a todo.md item it took into this chunk scores, where that is "
|
||||
"higher than the R + C loop-claude's first prompt says it routed the chunk on; 0 otherwise."
|
||||
f"bundled: {BUNDLED}"
|
||||
)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Reply:
|
||||
answer: str = dataclasses.field(metadata={"description": "With interrupt false, the passage of README.md or the records that answers the agent's question, quoted verbatim, without elisions. Empty otherwise."})
|
||||
bundled: int = dataclasses.field(metadata={"description": "The highest R + C the agent says a todo.md item it took into this chunk scores, where that is higher than the R + C loop-claude's first prompt says it routed the chunk on. 0 otherwise."})
|
||||
bundled: int = dataclasses.field(metadata={"description": BUNDLED})
|
||||
evidence: str = dataclasses.field(metadata={"description": "A verbatim quote of the one transcript passage the verdict rests on, without elisions: what triggers the interrupt; for a quoted answer, the agent's question; else what shows the chunk finished or the work remaining."})
|
||||
finished: bool = dataclasses.field(metadata={"description": "Whether the chunk is finished."})
|
||||
interrupt: bool = dataclasses.field(metadata={"description": "Whether the human must now be interrupted."})
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
from loop_claude.project.records import HUMAN_RULE_NAMES, RECORD_FILES
|
||||
from todo_table import Item
|
||||
from markdown_text import quoted
|
||||
from loop_claude.cycle.routing import risk_and_complexity
|
||||
from model_routing.ladder import Rung
|
||||
from loop_claude.loop.routing import Route
|
||||
|
||||
def either(names: tuple[str, ...]) -> str:
|
||||
"""The names as `a, b or c`."""
|
||||
@@ -16,13 +14,13 @@ ANSWER_DESTINATIONS = "README.md's Goals or Audience, " + ", ".join(RECORD_FILES
|
||||
ANSWERS_RECORDED = f"every answer the human typed in it that a later chunk could need is recorded in {ANSWER_DESTINATIONS} at the one place it governs"
|
||||
|
||||
|
||||
def fresh(task: str, premises: str, routed: str = "") -> str:
|
||||
route = f"{routed}\n\n" if routed else ""
|
||||
return f"{task}\n\nContinue any open draft PR or worktree for the chunk before creating a new one.\n\n{route}The chunk's todo.md items, or else the task above, set its scope. The goals below choose between ways to deliver that scope, and never widen it.\n\nFrom README.md:\n{premises}"
|
||||
def fresh(task: str, premises: str, route: Route) -> str:
|
||||
routed = f"{routed_by(route)}\n\n" if route.item else ""
|
||||
return f"{task}\n\nContinue any open draft PR or worktree for the chunk before creating a new one.\n\n{routed}The chunk's todo.md items, or else the task above, set its scope. The goals below choose between ways to deliver that scope, and never widen it.\n\nFrom README.md:\n{premises}"
|
||||
|
||||
|
||||
def routed(worker: Rung, item: Item) -> str:
|
||||
return f"loop-claude runs you on {worker}, the rung item {item.id}'s R + C of {risk_and_complexity(item)} picks. When the chunk takes in a todo.md item whose R + C is higher, say so, naming that item and its R + C."
|
||||
def routed_by(route: Route) -> str:
|
||||
return f"loop-claude runs you on {route.rung}, the rung item {route.item}'s R + C of {route.risk_and_complexity} picks. When the chunk takes in another todo.md item, say so, naming that item and its R + C."
|
||||
|
||||
|
||||
def resumed(unfinished_evidence: str) -> str:
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
"""Which ladder rung a chunk's worker runs on: docs/decisions.md → `routing:`."""
|
||||
from failure import Failure
|
||||
from model_routing.ladder import climbed, steps
|
||||
from todo_table import Item, items
|
||||
|
||||
UNFINISHED_PER_STEP = 2
|
||||
|
||||
|
||||
def routed_item(todo: str) -> Item | None:
|
||||
"""The next release's first item that waits on no question, as rank_todo.py ranks the table; None when there is none, or it has no C."""
|
||||
found = items(todo)
|
||||
if not found or isinstance(found, Failure):
|
||||
return None
|
||||
next_release = min(item.release for item in found)
|
||||
takes = next((item for item in found if item.release == next_release and item.exempt != "question"), None)
|
||||
return takes if takes and "C" in takes.scores else None
|
||||
|
||||
|
||||
def risk_and_complexity(item: Item) -> int:
|
||||
return item.scores["R"] + item.scores["C"]
|
||||
|
||||
|
||||
def first_rung(anchor: int, item: Item | None) -> int:
|
||||
"""The rung a chunk starts on: the one its item's R + C picks from anchor, the project's rung, never below it."""
|
||||
return anchor if item is None else max(anchor, climbed(anchor, steps(risk_and_complexity(item))))
|
||||
|
||||
|
||||
def stepped(anchor: int, rung: int, unfinished: int, bundled: int) -> tuple[int, int]:
|
||||
"""The rung and the unfinished verdicts since it last stepped, after one more unfinished verdict.
|
||||
|
||||
bundled is the highest R + C the judge saw the worker say an item it took into the chunk scores, 0 for none.
|
||||
"""
|
||||
if bundled:
|
||||
rung = max(rung, climbed(anchor, steps(bundled)))
|
||||
unfinished += 1
|
||||
return (climbed(rung, 1), 0) if unfinished == UNFINISHED_PER_STEP else (rung, unfinished)
|
||||
@@ -14,8 +14,8 @@ from loop_claude.paths import Paths
|
||||
from loop_claude.cycle import prompts
|
||||
from claude_code.transcript import read_transcript, render
|
||||
from loop_claude.cycle.retry import retried
|
||||
from loop_claude.loop.routing import Route
|
||||
from loop_claude.cycle.worker import run_worker, worker_args
|
||||
from model_routing.ladder import Rung
|
||||
from loop_claude.stop import Stop
|
||||
|
||||
|
||||
@@ -30,9 +30,7 @@ class Cycle:
|
||||
root: str
|
||||
session: Session
|
||||
task: str
|
||||
worker: Rung
|
||||
# The fresh worker prompt's word on its route; empty when the chunk is under way or not routed.
|
||||
routed: str
|
||||
route: Route
|
||||
# Empty when the previous cycle's verdict was not Unfinished.
|
||||
unfinished_evidence: str
|
||||
|
||||
@@ -85,10 +83,10 @@ def work(cycle: Cycle, answered: QuotedAnswer | None) -> WorkerRan | Stop:
|
||||
|
||||
def worker_start(cycle: Cycle, answered: QuotedAnswer | None) -> list[str]:
|
||||
if answered is not None:
|
||||
return worker_args(cycle.session, cycle.worker, prompts.answered(answered.question, answered.answer), resume=True)
|
||||
return worker_args(cycle.session, cycle.route.rung, prompts.answered(answered.question, answered.answer), resume=True)
|
||||
if session_open(cycle.session):
|
||||
return worker_args(cycle.session, cycle.worker, prompts.resumed(cycle.unfinished_evidence), resume=True)
|
||||
return worker_args(cycle.session, cycle.worker, prompts.fresh(cycle.task, cycle.premises, cycle.routed), resume=False)
|
||||
return worker_args(cycle.session, cycle.route.rung, prompts.resumed(cycle.unfinished_evidence), resume=True)
|
||||
return worker_args(cycle.session, cycle.route.rung, prompts.fresh(cycle.task, cycle.premises, cycle.route), resume=False)
|
||||
|
||||
|
||||
def keep_answer(cycle: Cycle, answered: QuotedAnswer) -> Stop | None:
|
||||
@@ -111,7 +109,7 @@ def ask(cycle: Cycle, transcript_path: str, ruling: Interrupt | QuotedAnswer) ->
|
||||
# Archived now, so the transcript the question was asked on is kept with it; the human's answer then grows the transcript.
|
||||
if failure := archive(cycle.project_archive, transcript_path, cycle.task, outcomes.interrupted(question, ruling.evidence), cycle.effects.now()):
|
||||
return Stop(1, failure.reason)
|
||||
return interrupt(cycle.root, cycle.session.id, cycle.worker, transcript_path, question, cycle.paths, cycle.effects) or HumanAnswered()
|
||||
return interrupt(cycle.root, cycle.session.id, cycle.route.rung, transcript_path, question, cycle.paths, cycle.effects) or HumanAnswered()
|
||||
|
||||
|
||||
def question_for(ruling: Interrupt | QuotedAnswer) -> str:
|
||||
@@ -122,7 +120,7 @@ def question_for(ruling: Interrupt | QuotedAnswer) -> str:
|
||||
|
||||
def close(cycle: Cycle, transcript_path: str, verdict: Finished | Unfinished) -> Archived | Stop:
|
||||
finished = isinstance(verdict, Finished)
|
||||
if failure := archive(cycle.project_archive, transcript_path, cycle.task, outcomes.finished(str(cycle.worker), verdict.evidence) if finished else outcomes.unfinished(str(cycle.worker), verdict.evidence), cycle.effects.now()):
|
||||
if failure := archive(cycle.project_archive, transcript_path, cycle.task, outcomes.finished(str(cycle.route), verdict.evidence) if finished else outcomes.unfinished(str(cycle.route), verdict.evidence), cycle.effects.now()):
|
||||
return Stop(1, failure.reason)
|
||||
if finished and (failure := delete_session(cycle.session)):
|
||||
return Stop(1, failure.reason)
|
||||
|
||||
@@ -44,11 +44,12 @@ Where that model is on the routing ladder (bin/model_routing/ladder.py), the wor
|
||||
interrupt run on a rung of it instead: a model at an effort. As a chunk starts, the R + C of
|
||||
todo.md's top item in the next release, skipping a question, picks the rung from the model's
|
||||
cheapest one, and never below it; without such an item, or with a C missing, the chunk runs on the
|
||||
model's cheapest rung. The worker's first prompt names the rung and asks it to say when the chunk
|
||||
takes in an item of higher R + C. The judge reports the highest such R + C it sees the worker say,
|
||||
and the next cycle steps up to the rung that picks, never down. Every second unfinished verdict on
|
||||
a chunk steps it up one rung. A rerun routes an open chunk afresh. The archive records each
|
||||
verdict's rung.
|
||||
model's cheapest rung. The worker's first prompt names the rung and the item, and asks it to name
|
||||
each other item the chunk takes in, with its R + C. The judge reports the highest R + C it sees the
|
||||
worker state, and the next cycle steps up to the rung that picks, never down. Every second
|
||||
unfinished verdict on a chunk steps it up one rung. A rung an edit took off the ladder counts as
|
||||
the model's cheapest. A rerun routes an open chunk afresh. The archive records each verdict's rung,
|
||||
with the item and R + C that routed the chunk.
|
||||
|
||||
A change to bin/loop-claude or any bin/**/*.py takes effect at the next cycle boundary, once
|
||||
loop-claude --help succeeds: the loop reloads into the edited script.
|
||||
@@ -95,13 +96,12 @@ from loop_claude.cycle.run import Cycle, run_cycle
|
||||
from loop_claude.loop.reload import decode_handed, exec_if_edited, fingerprint, LoopRun, RELOAD_FLAG
|
||||
from loop_claude.effects import Effects
|
||||
from loop_claude.cycle.judge import Finished, Unfinished
|
||||
from loop_claude.cycle.routing import first_rung, routed_item, stepped
|
||||
from loop_claude.loop.routing import first_route, Route, routed_item, stepped
|
||||
from loop_claude.project.model_choice import get_or_ask_models
|
||||
from model_routing.ladder import LADDER, project_rung, Rung
|
||||
from loop_claude.paths import Paths
|
||||
from transcript_archive.location import project_archive
|
||||
from project_key import project_key
|
||||
from loop_claude.cycle import prompts
|
||||
from model_routing.ladder import project_rung, Rung
|
||||
from loop_claude.project.premises import read_premises
|
||||
from loop_claude.project.records import read_records
|
||||
from claude_code.session import session_named
|
||||
@@ -141,30 +141,29 @@ def run_loop(argv: list[str], env: typing.Mapping[str, str], paths: Paths, launc
|
||||
if isinstance(models, Stop):
|
||||
return models
|
||||
anchor = project_rung(models.worker)
|
||||
loop_run, worker, routed = route_worker(loop_run, anchor, models.worker)
|
||||
print(f"\n=== cycle {loop_run.cycle_number}, worker on {worker} ===", flush=True)
|
||||
verdict = run_cycle(Cycle(effects=effects, judge_model=models.judge, paths=paths, premises=premises, project_archive=archive, records=records, root=loop_run.root, routed=routed, session=session, task=loop_run.task, unfinished_evidence=loop_run.unfinished_evidence, worker=worker))
|
||||
loop_run = routed(loop_run, anchor)
|
||||
route = loop_run.route or Route(Rung(models.worker))
|
||||
print(f"\n=== cycle {loop_run.cycle_number}, worker on {route.rung} ===", flush=True)
|
||||
verdict = run_cycle(Cycle(effects=effects, judge_model=models.judge, paths=paths, premises=premises, project_archive=archive, records=records, root=loop_run.root, route=route, session=session, task=loop_run.task, unfinished_evidence=loop_run.unfinished_evidence))
|
||||
if isinstance(verdict, Stop):
|
||||
return verdict
|
||||
loop_run = after_verdict(loop_run, anchor, verdict)
|
||||
|
||||
|
||||
def route_worker(loop_run: LoopRun, anchor: int | None, project_model: str) -> tuple[LoopRun, Rung, str]:
|
||||
"""The loop run, the worker's rung, and what the fresh prompt says of its route; a chunk is routed once, as it starts."""
|
||||
def routed(loop_run: LoopRun, anchor: int | None) -> LoopRun:
|
||||
"""The loop run with its chunk's route; a chunk is routed once, as it starts, and not at all off the ladder."""
|
||||
if anchor is None:
|
||||
return loop_run, Rung(project_model), ""
|
||||
if loop_run.rung >= 0:
|
||||
return loop_run, LADDER[loop_run.rung], ""
|
||||
item = routed_item(read_todo(loop_run.root))
|
||||
loop_run = dataclasses.replace(loop_run, rung=first_rung(anchor, item), unfinished=0)
|
||||
return loop_run, LADDER[loop_run.rung], prompts.routed(LADDER[loop_run.rung], item) if item else ""
|
||||
return dataclasses.replace(loop_run, route=None)
|
||||
if loop_run.route:
|
||||
return loop_run
|
||||
return dataclasses.replace(loop_run, route=first_route(anchor, routed_item(read_todo(loop_run.root))))
|
||||
|
||||
|
||||
def after_verdict(loop_run: LoopRun, anchor: int | None, verdict: Finished | Unfinished) -> LoopRun:
|
||||
if isinstance(verdict, Finished):
|
||||
return dataclasses.replace(loop_run, rung=-1, unfinished=0, unfinished_evidence="")
|
||||
rung, unfinished = stepped(anchor, loop_run.rung, loop_run.unfinished, verdict.bundled) if anchor is not None else (loop_run.rung, 0)
|
||||
return dataclasses.replace(loop_run, rung=rung, unfinished=unfinished, unfinished_evidence=verdict.evidence)
|
||||
return dataclasses.replace(loop_run, route=None, unfinished_evidence="")
|
||||
route = stepped(anchor, loop_run.route, verdict.bundled) if anchor is not None and loop_run.route else loop_run.route
|
||||
return dataclasses.replace(loop_run, route=route, unfinished_evidence=verdict.evidence)
|
||||
|
||||
|
||||
def read_todo(root: str) -> str:
|
||||
|
||||
@@ -3,9 +3,11 @@ import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
import types
|
||||
import typing
|
||||
|
||||
from loop_claude.effects import Effects
|
||||
from loop_claude.loop.routing import Route
|
||||
from loop_claude.stop import Stop
|
||||
|
||||
RELOAD_FLAG = "--reloaded"
|
||||
@@ -20,9 +22,8 @@ class LoopRun:
|
||||
root: str
|
||||
task: str
|
||||
unfinished_evidence: str
|
||||
# The chunk's worker rung on the ladder, -1 until the chunk is routed; and the unfinished verdicts since it last stepped up.
|
||||
rung: int = -1
|
||||
unfinished: int = 0
|
||||
# None until the chunk is routed, and where the project's model is off the ladder.
|
||||
route: Route | None = None
|
||||
|
||||
|
||||
def decode_handed(handed: str, kind: type[T]) -> T | None:
|
||||
@@ -30,12 +31,24 @@ def decode_handed(handed: str, kind: type[T]) -> T | None:
|
||||
fields = json.loads(handed)
|
||||
except ValueError:
|
||||
return None
|
||||
shape = typing.get_type_hints(kind)
|
||||
if not isinstance(fields, dict) or fields.keys() != shape.keys():
|
||||
return None
|
||||
if any(type(fields[name]) is not type_ for name, type_ in shape.items()):
|
||||
return None
|
||||
return kind(**fields)
|
||||
decoded = decoded_as(fields, kind)
|
||||
return None if decoded is REJECTED else decoded
|
||||
|
||||
|
||||
REJECTED = object()
|
||||
|
||||
|
||||
def decoded_as(value: object, kind: typing.Any) -> typing.Any:
|
||||
"""value as kind, or REJECTED: a dataclass from an object holding exactly its fields, a union's first member that fits, else a value of exactly kind."""
|
||||
if isinstance(kind, types.UnionType):
|
||||
return next((found for option in typing.get_args(kind) if (found := decoded_as(value, option)) is not REJECTED), REJECTED)
|
||||
if isinstance(kind, type) and dataclasses.is_dataclass(kind):
|
||||
shape = typing.get_type_hints(kind)
|
||||
if not isinstance(value, dict) or value.keys() != shape.keys():
|
||||
return REJECTED
|
||||
fields = {name: decoded_as(value[name], type_) for name, type_ in shape.items()}
|
||||
return REJECTED if any(field is REJECTED for field in fields.values()) else kind(**fields)
|
||||
return value if type(value) is kind else REJECTED
|
||||
|
||||
|
||||
def fingerprint(launcher: pathlib.Path) -> str:
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Which ladder rung a chunk's worker runs on: docs/decisions.md → `routing:`."""
|
||||
import dataclasses
|
||||
|
||||
from model_routing.ladder import climbed, index_of, LADDER, picked, Rung
|
||||
from failure import Failure
|
||||
from todo_table import Item, items
|
||||
|
||||
UNFINISHED_PER_STEP = 2
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Route:
|
||||
"""A chunk's worker rung, the plan item whose R + C picked it (0 for none), and the unfinished verdicts since it last stepped up."""
|
||||
rung: Rung
|
||||
item: int = 0
|
||||
risk_and_complexity: int = 0
|
||||
unfinished: int = 0
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.rung}, routed by item {self.item}'s R + C of {self.risk_and_complexity}" if self.item else str(self.rung)
|
||||
|
||||
|
||||
def routed_item(todo: str) -> Item | None:
|
||||
"""The next release's first item that waits on no question, as rank_todo.py ranks the table; None when there is none, or it has no C."""
|
||||
found = items(todo)
|
||||
if not found or isinstance(found, Failure):
|
||||
return None
|
||||
next_release = min(item.release for item in found)
|
||||
takes = next((item for item in found if item.release == next_release and item.exempt != "question"), None)
|
||||
return takes if takes and takes.risk_and_complexity else None
|
||||
|
||||
|
||||
def first_route(anchor: int, item: Item | None) -> Route:
|
||||
"""The route a chunk starts on: the rung its item's R + C picks from anchor, the project's rung, never below it."""
|
||||
if item is None:
|
||||
return Route(LADDER[anchor])
|
||||
return Route(LADDER[max(anchor, picked(anchor, item.risk_and_complexity))], item.id, item.risk_and_complexity)
|
||||
|
||||
|
||||
def stepped(anchor: int, route: Route, bundled: int) -> Route:
|
||||
"""The route after one more unfinished verdict; bundled is the highest R + C the judge saw the worker take into the chunk, 0 for none.
|
||||
|
||||
A rung an edit took off the ladder counts as the project's rung.
|
||||
"""
|
||||
rung = index_of(route.rung)
|
||||
rung = max(anchor if rung is None else rung, picked(anchor, bundled) if bundled else anchor)
|
||||
unfinished = route.unfinished + 1
|
||||
if unfinished >= UNFINISHED_PER_STEP:
|
||||
return dataclasses.replace(route, rung=LADDER[climbed(rung, 1)], unfinished=0)
|
||||
return dataclasses.replace(route, rung=LADDER[rung], unfinished=unfinished)
|
||||
@@ -390,10 +390,10 @@ class Cycles(unittest.TestCase):
|
||||
("claude-opus-5-5", "high"),
|
||||
])
|
||||
self.assertEqual({(argument(call, "--model"), argument(call, "--effort")) for call in run.calls if call["role"] == "judge"}, {("judge-model", None)})
|
||||
self.assertIn("loop-claude runs you on claude-opus-5-5 at high effort, the rung item 4's R + C of 11 picks.", argument(workers[0], "-p"))
|
||||
self.assertIn("loop-claude runs you on claude-opus-5-5 at high effort, the rung item 4's R + C of 11 picks. When the chunk takes in another todo.md item, say so, naming that item and its R + C.", argument(workers[0], "-p"))
|
||||
outcomes = sorted(verdict.removeprefix(f"Task: {TASK}\n\n") for verdict in checkout.archived().values())
|
||||
self.assertIn("<UTC> Verdict: unfinished\n\nWorker: claude-opus-5-5 at xhigh effort\n\nEvidence:\n> three\n", outcomes)
|
||||
self.assertIn("<UTC> Verdict: finished\n\nWorker: claude-fable-5-1 at max effort\n\nEvidence:\n> four\n", outcomes)
|
||||
self.assertIn("<UTC> Verdict: unfinished\n\nWorker: claude-opus-5-5 at xhigh effort, routed by item 4's R + C of 11\n\nEvidence:\n> three\n", outcomes)
|
||||
self.assertIn("<UTC> Verdict: finished\n\nWorker: claude-fable-5-1 at max effort, routed by item 4's R + C of 11\n\nEvidence:\n> four\n", outcomes)
|
||||
|
||||
def test_exits_1_when_the_transcript_cannot_be_archived_and_resumes_on_the_next_run(self):
|
||||
checkout = Checkout(self)
|
||||
@@ -474,7 +474,7 @@ class Cycles(unittest.TestCase):
|
||||
self.assertIn("LATE", run.calls[1]["stdin"])
|
||||
self.assertEqual(len(run.calls[1]["stdin"]), 400000)
|
||||
self.assertLess(run.calls[1]["stdin"].count("early"), 80000)
|
||||
self.assertTrue(run.calls[1]["stdin"].rstrip().endswith("routed the chunk on; 0 otherwise."))
|
||||
self.assertTrue(run.calls[1]["stdin"].rstrip().endswith("0 when it states none."))
|
||||
|
||||
def test_skips_a_malformed_transcript_or_stream_line(self):
|
||||
checkout = Checkout(self)
|
||||
@@ -716,12 +716,12 @@ class Reload(unittest.TestCase):
|
||||
self.assertEqual(run.roles(), ["worker", "judge"] * 5)
|
||||
self.assertIn("keeping the running script", run.stderr)
|
||||
reloads = [json.loads(argv[1]) for argv in run.starts if argv[:1] == ["--reloaded"]]
|
||||
self.assertEqual(reloads, [{"cycle_number": cycle, "root": str(checkout.root), "task": TASK, "unfinished_evidence": "working", "rung": -1, "unfinished": 0} for cycle in (2, 3, 4)])
|
||||
self.assertEqual(reloads, [{"cycle_number": cycle, "root": str(checkout.root), "task": TASK, "unfinished_evidence": "working", "route": None} for cycle in (2, 3, 4)])
|
||||
self.assertIn("=== cycle 5,", run.stdout)
|
||||
|
||||
outside = checkout.temp / "outside"
|
||||
outside.mkdir()
|
||||
state = {"cycle_number": 3, "root": str(checkout.root), "task": TASK, "unfinished_evidence": "", "rung": -1, "unfinished": 0}
|
||||
state = {"cycle_number": 3, "root": str(checkout.root), "task": TASK, "unfinished_evidence": "", "route": None}
|
||||
run = checkout.run([worker()], argv=["--reloaded", json.dumps(state)], cwd=outside)
|
||||
self.assertEqual((run.code, run.roles()), (130, ["worker", "judge"]))
|
||||
self.assertEqual(run.calls[0]["cwd"], str(checkout.root))
|
||||
|
||||
@@ -20,10 +20,10 @@ from loop_claude.effects import Effects
|
||||
from claude_code import entries, session
|
||||
from failure import Failure
|
||||
from loop_claude.cycle.judge import Finished, Interrupt, QuotedAnswer, Unfinished, verdict_from
|
||||
from loop_claude.cycle import interrupt, judge, prompts, retry, routing, run, states, worker
|
||||
from loop_claude.cycle import interrupt, judge, prompts, retry, run, states, worker
|
||||
from model_routing.ladder import LADDER, project_rung, Rung
|
||||
from loop_claude.cycle.retry import Limited
|
||||
from loop_claude.loop import main, reload
|
||||
from loop_claude.loop import main, reload, routing
|
||||
from loop_claude import paths
|
||||
from transcript_archive.location import project_archive
|
||||
from loop_claude.stop import Stop
|
||||
@@ -170,7 +170,7 @@ class Cycles(unittest.TestCase):
|
||||
self.archive = project_archive(str(self.home / "code" / "proj"), self.home, {})
|
||||
|
||||
def cycle(self, fake, records="", unfinished_evidence=""):
|
||||
return run.Cycle(effects=fake.effects(), judge_model="judge-model", paths=self.paths, premises="## Goals", project_archive=self.archive, records=records, root=str(self.home), routed="", session=self.session, task="Do it.", unfinished_evidence=unfinished_evidence, worker=Rung("worker-model"))
|
||||
return run.Cycle(effects=fake.effects(), judge_model="judge-model", paths=self.paths, premises="## Goals", project_archive=self.archive, records=records, root=str(self.home), route=routing.Route(Rung("worker-model")), session=self.session, task="Do it.", unfinished_evidence=unfinished_evidence)
|
||||
|
||||
def archived_outcomes(self):
|
||||
return sorted(path.read_text() for path in self.archive.glob("*/*.outcomes.md"))
|
||||
@@ -396,6 +396,8 @@ TODO = """# todo
|
||||
|
||||
|
||||
class Routing(unittest.TestCase):
|
||||
OPUS = project_rung("claude-opus-5-5")
|
||||
|
||||
def test_routes_on_the_next_releases_top_item_that_waits_on_no_question(self):
|
||||
self.assertEqual(routing.routed_item(TODO).id, 3)
|
||||
self.assertIsNone(routing.routed_item(TODO.replace("| | **Do", "| question | **Do")))
|
||||
@@ -403,30 +405,35 @@ class Routing(unittest.TestCase):
|
||||
self.assertIsNone(routing.routed_item(TODO.replace("| 5 | 3 |", "| x | 3 |")))
|
||||
|
||||
def test_starts_a_chunk_on_the_rung_its_items_r_plus_c_picks_never_below_the_projects(self):
|
||||
opus = project_rung("claude-opus-5-5")
|
||||
item = routing.routed_item(TODO)
|
||||
self.assertEqual(LADDER[routing.first_rung(opus, item)], Rung("claude-opus-5-5", "high"))
|
||||
self.assertEqual(routing.first_route(self.OPUS, routing.routed_item(TODO)), routing.Route(Rung("claude-opus-5-5", "high"), 3, 11))
|
||||
cheap = routing.routed_item(TODO.replace("| 5 | 3 | 6 |", "| 1 | 3 | 1 |"))
|
||||
self.assertEqual(routing.first_rung(opus, cheap), opus)
|
||||
self.assertEqual(routing.first_rung(opus, None), opus)
|
||||
self.assertEqual(routing.first_route(self.OPUS, cheap), routing.Route(LADDER[self.OPUS], 3, 2))
|
||||
self.assertEqual(routing.first_route(self.OPUS, None), routing.Route(LADDER[self.OPUS]))
|
||||
|
||||
def test_steps_up_a_rung_after_every_second_unfinished_verdict(self):
|
||||
opus = project_rung("claude-opus-5-5")
|
||||
self.assertEqual(routing.stepped(opus, opus, 0, 0), (opus, 1))
|
||||
self.assertEqual(routing.stepped(opus, opus, 1, 0), (opus + 1, 0))
|
||||
self.assertEqual(routing.stepped(opus, len(LADDER) - 1, 1, 0), (len(LADDER) - 1, 0))
|
||||
route = routing.Route(LADDER[self.OPUS], 3, 9)
|
||||
once = routing.stepped(self.OPUS, route, 0)
|
||||
self.assertEqual(once, routing.Route(LADDER[self.OPUS], 3, 9, 1))
|
||||
self.assertEqual(routing.stepped(self.OPUS, once, 0), routing.Route(LADDER[self.OPUS + 1], 3, 9, 0))
|
||||
self.assertEqual(routing.stepped(self.OPUS, routing.Route(LADDER[-1], unfinished=1), 0).rung, LADDER[-1])
|
||||
|
||||
def test_steps_up_to_the_rung_a_bundled_items_r_plus_c_picks_and_never_down(self):
|
||||
opus = project_rung("claude-opus-5-5")
|
||||
self.assertEqual(routing.stepped(opus, opus, 0, 13), (opus + 2, 1))
|
||||
self.assertEqual(routing.stepped(opus, opus + 2, 0, 11), (opus + 2, 1))
|
||||
self.assertEqual(routing.stepped(opus, opus, 1, 13), (opus + 3, 0))
|
||||
self.assertEqual(routing.stepped(self.OPUS, routing.Route(LADDER[self.OPUS]), 13).rung, LADDER[self.OPUS + 2])
|
||||
self.assertEqual(routing.stepped(self.OPUS, routing.Route(LADDER[self.OPUS + 2]), 11).rung, LADDER[self.OPUS + 2])
|
||||
self.assertEqual(routing.stepped(self.OPUS, routing.Route(LADDER[self.OPUS], unfinished=1), 13).rung, LADDER[self.OPUS + 3])
|
||||
|
||||
def test_a_rung_an_edit_took_off_the_ladder_counts_as_the_projects(self):
|
||||
self.assertEqual(routing.stepped(self.OPUS, routing.Route(Rung("claude-opus-5-5", "low")), 0).rung, LADDER[self.OPUS])
|
||||
|
||||
def test_names_the_item_that_routed_the_chunk(self):
|
||||
self.assertEqual(str(routing.Route(Rung("claude-opus-5-5", "high"), 3, 11)), "claude-opus-5-5 at high effort, routed by item 3's R + C of 11")
|
||||
self.assertEqual(str(routing.Route(Rung("worker-model"))), "worker-model")
|
||||
|
||||
|
||||
class WorkerArgs(unittest.TestCase):
|
||||
def test_opens_a_fresh_session_by_name_and_resumes_an_open_one(self):
|
||||
opened = session.session_named("A proj loop-claude", pathlib.Path("/h/.claude/projects"))
|
||||
fresh = worker.worker_args(opened, Rung("m"), prompts.fresh("Do it.", "## Goals\n1. g\n\n## Audience\nme"), resume=False)
|
||||
fresh = worker.worker_args(opened, Rung("m"), prompts.fresh("Do it.", "## Goals\n1. g\n\n## Audience\nme", routing.Route(Rung("m"))), resume=False)
|
||||
self.assertEqual(fresh[fresh.index("--session-id") + 1], str(opened.id))
|
||||
self.assertEqual(fresh[fresh.index("--name") + 1], "A proj loop-claude")
|
||||
resumed = worker.worker_args(opened, Rung("m"), prompts.resumed(""), resume=True)
|
||||
@@ -535,6 +542,17 @@ class LoopRuns(unittest.TestCase):
|
||||
self.assertEqual(reload.decode_handed('{"count": 1, "name": "n"}', Stringly), Stringly(1, "n"))
|
||||
self.assertIsNone(reload.decode_handed('{"count": "1", "name": "n"}', Stringly))
|
||||
|
||||
def test_decodes_a_route_inside_the_state_or_none(self):
|
||||
state = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=routing.Route(Rung("claude-opus-5-5", "high"), 3, 11, 1))
|
||||
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(state)), main.LoopRun), state)
|
||||
unrouted = dataclasses.replace(state, route=None)
|
||||
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(unrouted)), main.LoopRun), unrouted)
|
||||
handed = dataclasses.asdict(state)
|
||||
handed["route"]["rung"]["effort"] = 1
|
||||
self.assertIsNone(reload.decode_handed(json.dumps(handed), main.LoopRun))
|
||||
del handed["route"]["rung"]
|
||||
self.assertIsNone(reload.decode_handed(json.dumps(handed), main.LoopRun))
|
||||
|
||||
def test_stops_when_the_reload_cannot_start_python(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
launcher = pathlib.Path(root, "loop-claude")
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
"""The routing ladder, and the step an R + C moves a task from the project's rung: docs/decisions.md → `routing:`."""
|
||||
import dataclasses
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Rung:
|
||||
model: str
|
||||
# Empty leaves the effort to claude's default for the model, as for a model off the ladder.
|
||||
effort: str = ""
|
||||
|
||||
def flags(self) -> list[str]:
|
||||
return ["--model", self.model, *(["--effort", self.effort] if self.effort else [])]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.model} at {self.effort} effort" if self.effort else self.model
|
||||
|
||||
|
||||
# Least competent first. The Artificial Analysis Intelligence Index orders every rung but the top one, keeping a rung only
|
||||
# where no cheaper rung scores as high; Fable 5.1 tops the ladder as Anthropic's most capable model, which the index does not show.
|
||||
LADDER = (
|
||||
Rung("claude-haiku-5-5", "low"),
|
||||
Rung("claude-haiku-5-5", "medium"),
|
||||
Rung("claude-haiku-5-5", "high"),
|
||||
Rung("claude-haiku-5-5", "xhigh"),
|
||||
Rung("claude-haiku-5-5", "max"),
|
||||
Rung("claude-sonnet-5-5", "high"),
|
||||
Rung("claude-opus-5-5", "medium"),
|
||||
Rung("claude-opus-5-5", "high"),
|
||||
Rung("claude-opus-5-5", "xhigh"),
|
||||
Rung("claude-opus-5-5", "max"),
|
||||
Rung("claude-fable-5-1", "max"),
|
||||
)
|
||||
# (highest R + C in the band, rungs it steps from the project's rung); a higher R + C steps ABOVE_THE_BANDS.
|
||||
BANDS = ((4, -3), (6, -2), (8, -1), (10, 0), (12, 1), (14, 2), (16, 3))
|
||||
ABOVE_THE_BANDS = 4
|
||||
|
||||
|
||||
def project_rung(model: str) -> int | None:
|
||||
"""The index of the model's cheapest rung, which for Opus 5.5 is its default effort; None for a model off the ladder."""
|
||||
return next((index for index, rung in enumerate(LADDER) if rung.model == model), None)
|
||||
|
||||
|
||||
def steps(risk_and_complexity: int) -> int:
|
||||
return next((step for highest, step in BANDS if risk_and_complexity <= highest), ABOVE_THE_BANDS)
|
||||
|
||||
|
||||
def climbed(index: int, by: int) -> int:
|
||||
"""The rung by steps from index, stopping at either end of the ladder."""
|
||||
return max(0, min(len(LADDER) - 1, index + by))
|
||||
|
||||
|
||||
def picked(anchor: int, risk_and_complexity: int) -> int:
|
||||
"""The rung a task's R + C picks from anchor, the project's rung."""
|
||||
return climbed(anchor, steps(risk_and_complexity))
|
||||
|
||||
|
||||
def index_of(rung: Rung) -> int | None:
|
||||
return LADDER.index(rung) if rung in LADDER else None
|
||||
@@ -0,0 +1,41 @@
|
||||
import pathlib
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from model_routing.ladder import climbed, LADDER, project_rung, Rung, steps
|
||||
|
||||
|
||||
class Ladder(unittest.TestCase):
|
||||
def test_runs_from_haiku_at_low_effort_to_fable_at_max_with_opus_at_medium_between(self):
|
||||
self.assertEqual(LADDER[0], Rung("claude-haiku-5-5", "low"))
|
||||
self.assertEqual(LADDER[-1], Rung("claude-fable-5-1", "max"))
|
||||
self.assertEqual(len(LADDER), 11)
|
||||
self.assertEqual(len(set(LADDER)), len(LADDER))
|
||||
|
||||
def test_puts_a_project_on_its_models_cheapest_rung_and_none_off_the_ladder(self):
|
||||
self.assertEqual(LADDER[project_rung("claude-opus-5-5")], Rung("claude-opus-5-5", "medium"))
|
||||
self.assertEqual(LADDER[project_rung("claude-sonnet-5-5")], Rung("claude-sonnet-5-5", "high"))
|
||||
self.assertIsNone(project_rung("claude-opus-5"))
|
||||
self.assertIsNone(project_rung("opus"))
|
||||
|
||||
def test_steps_each_band_of_r_plus_c_from_the_projects_rung(self):
|
||||
self.assertEqual([steps(total) for total in range(2, 21)], [-3, -3, -3, -2, -2, -1, -1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4])
|
||||
|
||||
def test_a_step_past_either_end_stops_there(self):
|
||||
opus = project_rung("claude-opus-5-5")
|
||||
self.assertEqual(LADDER[climbed(opus, steps(4))], Rung("claude-haiku-5-5", "xhigh"))
|
||||
self.assertEqual(LADDER[climbed(opus, steps(17))], Rung("claude-fable-5-1", "max"))
|
||||
self.assertEqual(climbed(0, -3), 0)
|
||||
self.assertEqual(climbed(len(LADDER) - 1, 1), len(LADDER) - 1)
|
||||
|
||||
def test_passes_the_effort_only_where_the_rung_names_one(self):
|
||||
self.assertEqual(Rung("claude-opus-5-5", "high").flags(), ["--model", "claude-opus-5-5", "--effort", "high"])
|
||||
self.assertEqual(Rung("worker-model").flags(), ["--model", "worker-model"])
|
||||
self.assertEqual(str(Rung("claude-opus-5-5", "high")), "claude-opus-5-5 at high effort")
|
||||
self.assertEqual(str(Rung("worker-model")), "worker-model")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -19,6 +19,11 @@ class Item:
|
||||
scores: dict[str, int]
|
||||
goals: tuple[str, ...]
|
||||
|
||||
@property
|
||||
def risk_and_complexity(self) -> int:
|
||||
"""R + C, which routes the item's chunk to a model; 0 without a C."""
|
||||
return self.scores["R"] + self.scores["C"] if "C" in self.scores else 0
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Table:
|
||||
|
||||
+1
-1
@@ -178,6 +178,6 @@ The rungs, least competent first: Haiku 5.5 at low, medium, high, xhigh and max;
|
||||
|
||||
A project's rung is its model's cheapest rung, which for Opus 5.5 is its default effort, medium. A model off the ladder routes nothing: claude runs it at its default effort. R + C steps a task from the project's rung: 2–4 three rungs down, 5–6 two down, 7–8 one down, 9–10 none, 11–12 one up, 13–14 two up, 15–16 three up, 17 and above four up. A step past either end stops there. The bands spread the R + C that `todo.md`'s items held on 2026-10-14, 2 to 12, across the rungs, so most tasks step down. S never moves a rung, so a huge task with low R and C runs cheap. The comprehension panel and the security pass stay on the project's model. `score-outcomes` had too few finished chunks to ground the bands; `todo.md` item 94 measures the routes' outcomes, and a change to the ladder or the bands is lilleman's.
|
||||
|
||||
A chunk's worker starts on the rung its item's R + C picks, never below the project's rung, until `todo.md` item 93 shows a cheaper worker costs no review rounds (goal 1.1). It steps up only as the chunk starts, after every second unfinished verdict, or when the judge reports an item taken into the chunk whose R + C picks a higher rung. A model switch inside a session discards its prompt cache.
|
||||
A chunk's worker starts on the rung its item's R + C picks, never below the project's rung, until `todo.md` item 93 shows a cheaper worker costs no review rounds (goal 1.1). It steps up only as the chunk starts, after every second unfinished verdict, or when the judge reports an item taken into the chunk whose R + C picks a higher rung. A model switch inside a session discards its prompt cache. loop-claude routes on the first row of the next release in `todo.md`'s table that waits on no question. It reads neither an order the plan states in prose nor the task, so a chunk may start on another item's rung; since the worker never goes below the project's rung, that costs money, never quality.
|
||||
|
||||
Valid while the index ranks these models, and while no newer model joins the catalog.
|
||||
|
||||
Reference in New Issue
Block a user