Answer stability round 1: 0 🔴, 4 🟡 (0 misread); refuse a ladder that names two models of one family, test the one finding a rung needs, and settle the two places in the PR summary
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
"""A routing ladder, and the rungs a task's L and E + I pick on it from the project's rung: docs/decisions.md → `routing:`. model_routing/ladder_file.py reads a project's ladder from its file."""
|
||||
"""A routing ladder, and the rungs a task's L and E + I pick on it from the project's rung: docs/decisions.md → `routing:`."""
|
||||
import dataclasses
|
||||
import re
|
||||
import typing
|
||||
@@ -49,7 +49,7 @@ REVIEW_STEPS = {5: 1, 6: 2, 7: 3, 8: 4}
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Ladder:
|
||||
"""A project's rungs, least competent first. model_routing/ladder_file.py reads one, and refuses a file that holds no rung or one rung twice, since a rung's position sets every step."""
|
||||
"""A project's rungs, least competent first. model_routing/ladder_file.py reads one, and refuses a file that holds no rung, one rung twice, since a rung's position sets every step, or two models of one family, since from_agent reads a family as one model."""
|
||||
rungs: tuple[Rung, ...]
|
||||
|
||||
def from_agent(self, model: str, effort: str) -> Rung:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Read a routing ladder from its JSON file: the one this setup ships, or the one a project keeps in its config's folder in its place. docs/decisions.md → `routing: one ladder of (model, effort) rungs`."""
|
||||
"""Read a routing ladder from its JSON file: the one this setup ships, or a project's own, kept in its config's folder. docs/decisions.md → `routing: one ladder of (model, effort) rungs`."""
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
@@ -10,22 +10,28 @@ from project_config.settings import Unnamed
|
||||
|
||||
# The file's name in both places. Three readers, seated as the human who writes an override in a hurry, picked a bare list of objects over a list of strings, a list of pairs and a list under a key. Two of them picked one name for both files: an override is then a copy of the shipped file with rungs cut.
|
||||
NAME = "routing-ladder.json"
|
||||
# The ladder this setup ships, beside this module, so a copy of bin/ carries it: the tests of a reload and mutate-changes each run the tools from such a copy.
|
||||
# The ladder this setup ships, beside this module, so a copy of bin/ carries it: mutate-changes and the tests of a reload each run the tools from such a copy.
|
||||
SHIPPED = pathlib.Path(__file__).resolve().with_name(NAME)
|
||||
# A model as Rung.family reads it: the Agent tool's family name follows the first hyphen.
|
||||
MODEL = re.compile(r"[^\s-]+(-[^\s-]+)+")
|
||||
SHAPE = f'a JSON list of rungs, least competent first, each as {{"model": "claude-opus-5-5", "effort": "medium"}}, with an effort of {", ".join(EFFORTS[:-1])} or {EFFORTS[-1]}'
|
||||
SHAPE = f'a JSON list of rungs, least competent first, each as {{"model": "claude-opus-5-5", "effort": "medium"}}: a model\'s full name, one model for each family such as opus, and an effort of {", ".join(EFFORTS[:-1])} or {EFFORTS[-1]}'
|
||||
|
||||
|
||||
def project_ladder(directory: str | None, home: pathlib.Path, shipped: pathlib.Path = SHIPPED) -> Ladder | Failure:
|
||||
"""The ladder of the project whose checkout holds directory, else the current directory: the file NAME in its config's folder under home, or the shipped one where that folder holds none. Where no config names a checkout there, the shipped one. A Failure where the project's config or the ladder's file is faulty or cannot be read, since a tool on another ladder than the project's would route its work wrong."""
|
||||
def project_ladder_file(directory: str | None, home: pathlib.Path, shipped: pathlib.Path = SHIPPED) -> pathlib.Path | Failure:
|
||||
"""The file that holds the ladder of the project whose checkout holds directory, else the current directory: the file NAME in its config's folder under home, or shipped where that folder holds none. Also shipped where no config names a checkout there, or git fails there. A Failure where the project's config is faulty or cannot be read, since its folder may hold the ladder."""
|
||||
config = located(directory, home)
|
||||
if isinstance(config, Unnamed):
|
||||
return read_ladder(shipped)
|
||||
return shipped
|
||||
if isinstance(config, Failure):
|
||||
return config
|
||||
own = config.folder / NAME
|
||||
return read_ladder(own if own.exists() else shipped)
|
||||
return own if own.exists() else shipped
|
||||
|
||||
|
||||
def project_ladder(directory: str | None, home: pathlib.Path, shipped: pathlib.Path = SHIPPED) -> Ladder | Failure:
|
||||
"""The ladder project_ladder_file's file holds. A Failure where that file holds a fault too: the shipped ladder never stands in for a project's own, since work routed on it could run on a model the project cannot reach."""
|
||||
file = project_ladder_file(directory, home, shipped)
|
||||
return file if isinstance(file, Failure) else read_ladder(file)
|
||||
|
||||
|
||||
def read_ladder(file: pathlib.Path) -> Ladder | Failure:
|
||||
@@ -33,7 +39,8 @@ def read_ladder(file: pathlib.Path) -> Ladder | Failure:
|
||||
try:
|
||||
held = json.loads(file.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, ValueError) as failure:
|
||||
return Failure(f"cannot read the routing ladder {file}: {failure}. It holds {SHAPE}")
|
||||
# An OSError's own text names the file again.
|
||||
return Failure(f"cannot read the routing ladder {file}: {failure.strerror if isinstance(failure, OSError) else failure}. The routing ladder is {SHAPE}")
|
||||
faults = faults_of(held)
|
||||
if faults:
|
||||
return Failure(f"{file}: {'; '.join(faults)}. The routing ladder is {SHAPE}")
|
||||
@@ -41,15 +48,22 @@ def read_ladder(file: pathlib.Path) -> Ladder | Failure:
|
||||
|
||||
|
||||
def faults_of(held: object) -> list[str]:
|
||||
"""What keeps held from being a ladder: it is no list or an empty one, or else each rung that is misspelt or repeats one above it, by its number from 1."""
|
||||
"""What keeps held from being a ladder: it is no list or an empty one, or else each rung, by its number from 1, that is misspelt, repeats one above it, or names a second model of a family."""
|
||||
if not isinstance(held, list) or not held:
|
||||
return ["it holds no list of rungs"]
|
||||
faults = []
|
||||
# The first rung of each family, by number: the Agent tool launches by family name, which Ladder.from_agent reads back as that family's one model.
|
||||
families: dict[str, tuple[int, str]] = {}
|
||||
for number, rung in enumerate(held, 1):
|
||||
if not well_formed(rung):
|
||||
faults.append(f"rung {number} is {json.dumps(rung)}")
|
||||
elif rung in held[:number - 1]:
|
||||
continue
|
||||
family = Rung(rung["model"]).family()
|
||||
first, model = families.setdefault(family, (number, rung["model"]))
|
||||
if rung in held[:number - 1]:
|
||||
faults.append(f"rung {number} repeats rung {held.index(rung) + 1}")
|
||||
elif model != rung["model"]:
|
||||
faults.append(f"rung {number} names a second model of the family {family}, after rung {first}")
|
||||
return faults
|
||||
|
||||
|
||||
|
||||
@@ -4,10 +4,10 @@ import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from model_routing.ladder import Ladder, Rung, WORKER_STEPS
|
||||
from release_plan.todo_table import LEVELS
|
||||
from model_routing.ladder_file import read_ladder, SHIPPED
|
||||
from review_rules.review_passes import REVIEWERS
|
||||
from model_routing.ladder import Rung, WORKER_STEPS
|
||||
|
||||
# The ladder this setup ships, whose rungs the tests below name.
|
||||
LADDER = read_ladder(SHIPPED)
|
||||
|
||||
@@ -9,7 +9,7 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
from failure import Failure
|
||||
from test_env import isolated_env
|
||||
from model_routing.ladder import Ladder, Rung
|
||||
from model_routing.ladder_file import NAME, project_ladder, read_ladder, SHIPPED
|
||||
from model_routing.ladder_file import NAME, project_ladder, project_ladder_file, read_ladder, SHIPPED
|
||||
from project_config.test_project_config import write_config
|
||||
|
||||
FABLE_ONLY = '[\n\t{"model": "claude-fable-5-2", "effort": "low"},\n\t{"model": "claude-fable-5-2", "effort": "max"}\n]\n'
|
||||
@@ -57,12 +57,12 @@ class ReadLadder(LadderCase):
|
||||
file = self.home / NAME
|
||||
missing = read_ladder(file)
|
||||
self.assertIsInstance(missing, Failure)
|
||||
self.assertIn(f"cannot read the routing ladder {file}: ", missing.reason)
|
||||
self.assertTrue(missing.reason.startswith(f"cannot read the routing ladder {file}: No such file or directory. The routing ladder is a JSON list of rungs"), missing.reason)
|
||||
# The comma a reader leaves after the last rung when dropping one.
|
||||
trailing = self.reason(f"[\n\t{OPUS},\n]\n")
|
||||
self.assertIn(f"cannot read the routing ladder {file}: ", trailing)
|
||||
self.assertIn("line 2", trailing)
|
||||
self.assertIn('a JSON list of rungs, least competent first, each as {"model": "claude-opus-5-5", "effort": "medium"}, with an effort of low, medium, high, xhigh or max', trailing)
|
||||
self.assertTrue(trailing.endswith('. The routing ladder is a JSON list of rungs, least competent first, each as {"model": "claude-opus-5-5", "effort": "medium"}: a model\'s full name, one model for each family such as opus, and an effort of low, medium, high, xhigh or max'), trailing)
|
||||
file.write_bytes(b"\xff")
|
||||
self.assertIn(f"cannot read the routing ladder {file}: ", read_ladder(file).reason)
|
||||
|
||||
@@ -92,6 +92,15 @@ class ReadLadder(LadderCase):
|
||||
low = '{"model": "claude-opus-5-5", "effort": "low"}'
|
||||
self.assertEqual(self.reason(f"[{low}, {OPUS}, {low}]").split(". ")[0], f"{self.home / NAME}: rung 3 repeats rung 1")
|
||||
|
||||
def test_a_second_model_of_one_family_fails_naming_the_familys_first_rung_since_a_subagent_launches_by_family(self):
|
||||
older, newer = '{"model": "claude-fable-5-1", "effort": "max"}', '{"model": "claude-fable-5-2", "effort": "low"}'
|
||||
self.assertEqual(self.reason(f"[{OPUS}, {older}, {newer}]").split(". ")[0], f"{self.home / NAME}: rung 3 names a second model of the family fable, after rung 2")
|
||||
both = self.reason(f'[{older}, {newer}, {{"model": "claude-fable-5-2", "effort": "high"}}]').split(". ")[0]
|
||||
self.assertEqual(both, f"{self.home / NAME}: rung 2 names a second model of the family fable, after rung 1; rung 3 names a second model of the family fable, after rung 1")
|
||||
file = self.home / NAME
|
||||
file.write_text(f'[{OPUS}, {newer}, {{"model": "claude-opus-5-5", "effort": "xhigh"}}, {{"model": "claude-fable-5-2", "effort": "max"}}]', encoding="utf-8")
|
||||
self.assertIsInstance(read_ladder(file), Ladder, "one model at several efforts, between another family's rungs, is no fault")
|
||||
|
||||
|
||||
class ProjectLadder(LadderCase):
|
||||
def test_a_project_runs_on_the_ladder_its_configs_folder_holds_from_a_worktree_too(self):
|
||||
@@ -102,6 +111,14 @@ class ProjectLadder(LadderCase):
|
||||
self.git(checkout, "worktree", "add", "--quiet", str(worktree), "-b", "item-7")
|
||||
self.assertEqual(project_ladder(str(worktree), self.home, self.shipped), FABLE_LADDER)
|
||||
|
||||
def test_names_the_file_it_reads_the_projects_own_or_else_the_shipped_one(self):
|
||||
checkout = self.checkout()
|
||||
folder = write_config(self.home, checkout)
|
||||
self.assertEqual(project_ladder_file(str(checkout), self.home, self.shipped), self.shipped)
|
||||
(folder / NAME).write_text(FABLE_ONLY, encoding="utf-8")
|
||||
self.assertEqual(project_ladder_file(str(checkout), self.home, self.shipped), folder / NAME)
|
||||
self.assertEqual(project_ladder_file(str(self.home), self.home), SHIPPED, "outside a checkout, the ladder this setup ships")
|
||||
|
||||
def test_another_projects_ladder_is_not_read(self):
|
||||
checkout, other = self.checkout(), self.checkout("other")
|
||||
write_config(self.home, checkout)
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import contextlib
|
||||
import io
|
||||
import pathlib
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from review_replay.recall import candidates, needed, Recall, recall_report, tried
|
||||
from review_replay.recall import candidates, needed, Recall, recall_report, recalled, tried
|
||||
from review_replay.rounds import is_red, Round
|
||||
from model_routing.ladder_file import read_ladder, SHIPPED
|
||||
from model_routing.ladder import Rung
|
||||
@@ -38,6 +40,15 @@ class Candidates(unittest.TestCase):
|
||||
self.assertEqual(candidates(LADDER, SONNET), [Rung("claude-sonnet-5-5", "medium"), Rung("claude-sonnet-5-5", "low")])
|
||||
|
||||
|
||||
class Recalled(unittest.TestCase):
|
||||
def test_tries_the_rungs_below_where_the_project_rung_needs_one_finding(self):
|
||||
# Eight findings, of which both replays on the project's rung find one: a share of an eighth, a tenth under which leaves one finding needed.
|
||||
rounds = [a_round(at, ("🟡 a",)) for at in range(8)]
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
recall = recalled(rounds, LADDER, REFERENCE, lambda one, rung: {0} if one == rounds[0] else set(), "prose rounds")
|
||||
self.assertEqual(([one.rung for one in recall.tried], recall.lowest), ([LOW, SONNET], SONNET))
|
||||
|
||||
|
||||
class Tried(unittest.TestCase):
|
||||
def test_qualifies_a_rung_no_more_than_the_margin_below_the_project_rungs_share_that_misses_no_red_found_both_times(self):
|
||||
found = tried(LOW, ROUNDS, (FIRST, SECOND), [{0, 1, 2}, {0, 1, 2}])
|
||||
|
||||
Reference in New Issue
Block a user