Answer stability round 1: 0 🔴, 4 🟡 (0 misread); refuse a ladder that names two models of one family, test the one finding a rung needs, and settle the two places in the PR summary

This commit is contained in:
2026-11-15 19:42:47 +01:00
parent 46135acab6
commit 2ecec90ec5
5 changed files with 59 additions and 17 deletions
+2 -2
View File
@@ -1,4 +1,4 @@
"""A routing ladder, and the rungs a task's L and E + I pick on it from the project's rung: docs/decisions.md → `routing:`. model_routing/ladder_file.py reads a project's ladder from its file."""
"""A routing ladder, and the rungs a task's L and E + I pick on it from the project's rung: docs/decisions.md → `routing:`."""
import dataclasses
import re
import typing
@@ -49,7 +49,7 @@ REVIEW_STEPS = {5: 1, 6: 2, 7: 3, 8: 4}
@dataclasses.dataclass(frozen=True)
class Ladder:
"""A project's rungs, least competent first. model_routing/ladder_file.py reads one, and refuses a file that holds no rung or one rung twice, since a rung's position sets every step."""
"""A project's rungs, least competent first. model_routing/ladder_file.py reads one, and refuses a file that holds no rung, one rung twice, since a rung's position sets every step, or two models of one family, since from_agent reads a family as one model."""
rungs: tuple[Rung, ...]
def from_agent(self, model: str, effort: str) -> Rung:
+24 -10
View File
@@ -1,4 +1,4 @@
"""Read a routing ladder from its JSON file: the one this setup ships, or the one a project keeps in its config's folder in its place. docs/decisions.md → `routing: one ladder of (model, effort) rungs`."""
"""Read a routing ladder from its JSON file: the one this setup ships, or a project's own, kept in its config's folder. docs/decisions.md → `routing: one ladder of (model, effort) rungs`."""
import json
import pathlib
import re
@@ -10,22 +10,28 @@ from project_config.settings import Unnamed
# The file's name in both places. Three readers, seated as the human who writes an override in a hurry, picked a bare list of objects over a list of strings, a list of pairs and a list under a key. Two of them picked one name for both files: an override is then a copy of the shipped file with rungs cut.
NAME = "routing-ladder.json"
# The ladder this setup ships, beside this module, so a copy of bin/ carries it: the tests of a reload and mutate-changes each run the tools from such a copy.
# The ladder this setup ships, beside this module, so a copy of bin/ carries it: mutate-changes and the tests of a reload each run the tools from such a copy.
SHIPPED = pathlib.Path(__file__).resolve().with_name(NAME)
# A model as Rung.family reads it: the Agent tool's family name follows the first hyphen.
MODEL = re.compile(r"[^\s-]+(-[^\s-]+)+")
SHAPE = f'a JSON list of rungs, least competent first, each as {{"model": "claude-opus-5-5", "effort": "medium"}}, with an effort of {", ".join(EFFORTS[:-1])} or {EFFORTS[-1]}'
SHAPE = f'a JSON list of rungs, least competent first, each as {{"model": "claude-opus-5-5", "effort": "medium"}}: a model\'s full name, one model for each family such as opus, and an effort of {", ".join(EFFORTS[:-1])} or {EFFORTS[-1]}'
def project_ladder(directory: str | None, home: pathlib.Path, shipped: pathlib.Path = SHIPPED) -> Ladder | Failure:
"""The ladder of the project whose checkout holds directory, else the current directory: the file NAME in its config's folder under home, or the shipped one where that folder holds none. Where no config names a checkout there, the shipped one. A Failure where the project's config or the ladder's file is faulty or cannot be read, since a tool on another ladder than the project's would route its work wrong."""
def project_ladder_file(directory: str | None, home: pathlib.Path, shipped: pathlib.Path = SHIPPED) -> pathlib.Path | Failure:
"""The file that holds the ladder of the project whose checkout holds directory, else the current directory: the file NAME in its config's folder under home, or shipped where that folder holds none. Also shipped where no config names a checkout there, or git fails there. A Failure where the project's config is faulty or cannot be read, since its folder may hold the ladder."""
config = located(directory, home)
if isinstance(config, Unnamed):
return read_ladder(shipped)
return shipped
if isinstance(config, Failure):
return config
own = config.folder / NAME
return read_ladder(own if own.exists() else shipped)
return own if own.exists() else shipped
def project_ladder(directory: str | None, home: pathlib.Path, shipped: pathlib.Path = SHIPPED) -> Ladder | Failure:
"""The ladder project_ladder_file's file holds. A Failure where that file holds a fault too: the shipped ladder never stands in for a project's own, since work routed on it could run on a model the project cannot reach."""
file = project_ladder_file(directory, home, shipped)
return file if isinstance(file, Failure) else read_ladder(file)
def read_ladder(file: pathlib.Path) -> Ladder | Failure:
@@ -33,7 +39,8 @@ def read_ladder(file: pathlib.Path) -> Ladder | Failure:
try:
held = json.loads(file.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, ValueError) as failure:
return Failure(f"cannot read the routing ladder {file}: {failure}. It holds {SHAPE}")
# An OSError's own text names the file again.
return Failure(f"cannot read the routing ladder {file}: {failure.strerror if isinstance(failure, OSError) else failure}. The routing ladder is {SHAPE}")
faults = faults_of(held)
if faults:
return Failure(f"{file}: {'; '.join(faults)}. The routing ladder is {SHAPE}")
@@ -41,15 +48,22 @@ def read_ladder(file: pathlib.Path) -> Ladder | Failure:
def faults_of(held: object) -> list[str]:
"""What keeps held from being a ladder: it is no list or an empty one, or else each rung that is misspelt or repeats one above it, by its number from 1."""
"""What keeps held from being a ladder: it is no list or an empty one, or else each rung, by its number from 1, that is misspelt, repeats one above it, or names a second model of a family."""
if not isinstance(held, list) or not held:
return ["it holds no list of rungs"]
faults = []
# The first rung of each family, by number: the Agent tool launches by family name, which Ladder.from_agent reads back as that family's one model.
families: dict[str, tuple[int, str]] = {}
for number, rung in enumerate(held, 1):
if not well_formed(rung):
faults.append(f"rung {number} is {json.dumps(rung)}")
elif rung in held[:number - 1]:
continue
family = Rung(rung["model"]).family()
first, model = families.setdefault(family, (number, rung["model"]))
if rung in held[:number - 1]:
faults.append(f"rung {number} repeats rung {held.index(rung) + 1}")
elif model != rung["model"]:
faults.append(f"rung {number} names a second model of the family {family}, after rung {first}")
return faults
+1 -1
View File
@@ -4,10 +4,10 @@ import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from model_routing.ladder import Ladder, Rung, WORKER_STEPS
from release_plan.todo_table import LEVELS
from model_routing.ladder_file import read_ladder, SHIPPED
from review_rules.review_passes import REVIEWERS
from model_routing.ladder import Rung, WORKER_STEPS
# The ladder this setup ships, whose rungs the tests below name.
LADDER = read_ladder(SHIPPED)
+20 -3
View File
@@ -9,7 +9,7 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from failure import Failure
from test_env import isolated_env
from model_routing.ladder import Ladder, Rung
from model_routing.ladder_file import NAME, project_ladder, read_ladder, SHIPPED
from model_routing.ladder_file import NAME, project_ladder, project_ladder_file, read_ladder, SHIPPED
from project_config.test_project_config import write_config
FABLE_ONLY = '[\n\t{"model": "claude-fable-5-2", "effort": "low"},\n\t{"model": "claude-fable-5-2", "effort": "max"}\n]\n'
@@ -57,12 +57,12 @@ class ReadLadder(LadderCase):
file = self.home / NAME
missing = read_ladder(file)
self.assertIsInstance(missing, Failure)
self.assertIn(f"cannot read the routing ladder {file}: ", missing.reason)
self.assertTrue(missing.reason.startswith(f"cannot read the routing ladder {file}: No such file or directory. The routing ladder is a JSON list of rungs"), missing.reason)
# The comma a reader leaves after the last rung when dropping one.
trailing = self.reason(f"[\n\t{OPUS},\n]\n")
self.assertIn(f"cannot read the routing ladder {file}: ", trailing)
self.assertIn("line 2", trailing)
self.assertIn('a JSON list of rungs, least competent first, each as {"model": "claude-opus-5-5", "effort": "medium"}, with an effort of low, medium, high, xhigh or max', trailing)
self.assertTrue(trailing.endswith('. The routing ladder is a JSON list of rungs, least competent first, each as {"model": "claude-opus-5-5", "effort": "medium"}: a model\'s full name, one model for each family such as opus, and an effort of low, medium, high, xhigh or max'), trailing)
file.write_bytes(b"\xff")
self.assertIn(f"cannot read the routing ladder {file}: ", read_ladder(file).reason)
@@ -92,6 +92,15 @@ class ReadLadder(LadderCase):
low = '{"model": "claude-opus-5-5", "effort": "low"}'
self.assertEqual(self.reason(f"[{low}, {OPUS}, {low}]").split(". ")[0], f"{self.home / NAME}: rung 3 repeats rung 1")
def test_a_second_model_of_one_family_fails_naming_the_familys_first_rung_since_a_subagent_launches_by_family(self):
older, newer = '{"model": "claude-fable-5-1", "effort": "max"}', '{"model": "claude-fable-5-2", "effort": "low"}'
self.assertEqual(self.reason(f"[{OPUS}, {older}, {newer}]").split(". ")[0], f"{self.home / NAME}: rung 3 names a second model of the family fable, after rung 2")
both = self.reason(f'[{older}, {newer}, {{"model": "claude-fable-5-2", "effort": "high"}}]').split(". ")[0]
self.assertEqual(both, f"{self.home / NAME}: rung 2 names a second model of the family fable, after rung 1; rung 3 names a second model of the family fable, after rung 1")
file = self.home / NAME
file.write_text(f'[{OPUS}, {newer}, {{"model": "claude-opus-5-5", "effort": "xhigh"}}, {{"model": "claude-fable-5-2", "effort": "max"}}]', encoding="utf-8")
self.assertIsInstance(read_ladder(file), Ladder, "one model at several efforts, between another family's rungs, is no fault")
class ProjectLadder(LadderCase):
def test_a_project_runs_on_the_ladder_its_configs_folder_holds_from_a_worktree_too(self):
@@ -102,6 +111,14 @@ class ProjectLadder(LadderCase):
self.git(checkout, "worktree", "add", "--quiet", str(worktree), "-b", "item-7")
self.assertEqual(project_ladder(str(worktree), self.home, self.shipped), FABLE_LADDER)
def test_names_the_file_it_reads_the_projects_own_or_else_the_shipped_one(self):
checkout = self.checkout()
folder = write_config(self.home, checkout)
self.assertEqual(project_ladder_file(str(checkout), self.home, self.shipped), self.shipped)
(folder / NAME).write_text(FABLE_ONLY, encoding="utf-8")
self.assertEqual(project_ladder_file(str(checkout), self.home, self.shipped), folder / NAME)
self.assertEqual(project_ladder_file(str(self.home), self.home), SHIPPED, "outside a checkout, the ladder this setup ships")
def test_another_projects_ladder_is_not_read(self):
checkout, other = self.checkout(), self.checkout("other")
write_config(self.home, checkout)
+12 -1
View File
@@ -1,10 +1,12 @@
import contextlib
import io
import pathlib
import sys
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from review_replay.recall import candidates, needed, Recall, recall_report, tried
from review_replay.recall import candidates, needed, Recall, recall_report, recalled, tried
from review_replay.rounds import is_red, Round
from model_routing.ladder_file import read_ladder, SHIPPED
from model_routing.ladder import Rung
@@ -38,6 +40,15 @@ class Candidates(unittest.TestCase):
self.assertEqual(candidates(LADDER, SONNET), [Rung("claude-sonnet-5-5", "medium"), Rung("claude-sonnet-5-5", "low")])
class Recalled(unittest.TestCase):
def test_tries_the_rungs_below_where_the_project_rung_needs_one_finding(self):
# Eight findings, of which both replays on the project's rung find one: a share of an eighth, a tenth under which leaves one finding needed.
rounds = [a_round(at, ("🟡 a",)) for at in range(8)]
with contextlib.redirect_stderr(io.StringIO()):
recall = recalled(rounds, LADDER, REFERENCE, lambda one, rung: {0} if one == rounds[0] else set(), "prose rounds")
self.assertEqual(([one.rung for one in recall.tried], recall.lowest), ([LOW, SONNET], SONNET))
class Tried(unittest.TestCase):
def test_qualifies_a_rung_no_more_than_the_margin_below_the_project_rungs_share_that_misses_no_red_found_both_times(self):
found = tried(LOW, ROUNDS, (FIRST, SECOND), [{0, 1, 2}, {0, 1, 2}])