Files

148 lines
7.9 KiB
Python

import pathlib
import subprocess
import sys
import tempfile
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from failure import Failure
from test_env import isolated_env
from model_routing.ladder import Ladder, Rung
from model_routing.ladder_file import NAME, project_ladder, project_ladder_file, read_ladder, SHIPPED
from project_config.doubles import write_config
FABLE_ONLY = '[\n\t{"model": "claude-fable-5-2", "effort": "low"},\n\t{"model": "claude-fable-5-2", "effort": "max"}\n]\n'
FABLE_LADDER = Ladder((Rung("claude-fable-5-2", "low"), Rung("claude-fable-5-2", "max")))
OPUS = '{"model": "claude-opus-5-5", "effort": "medium"}'
class LadderCase(unittest.TestCase):
def setUp(self) -> None:
made = tempfile.TemporaryDirectory()
self.addCleanup(made.cleanup)
self.home = pathlib.Path(made.name).resolve()
self.shipped = self.home / "shipped.json"
self.shipped.write_text(f"[{OPUS}]", encoding="utf-8")
def checkout(self, name: str = "acme") -> pathlib.Path:
"""A git checkout under the home, holding one commit, that no config names yet."""
checkout = self.home / "code" / name
checkout.mkdir(parents=True)
self.git(checkout, "init", "--quiet", "-b", "main")
self.git(checkout, "commit", "--quiet", "--allow-empty", "-m", "base")
return checkout
def git(self, cwd: pathlib.Path, *args: str) -> None:
subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@example.com", *args], capture_output=True, check=True, cwd=cwd, env=isolated_env())
class ReadLadder(LadderCase):
def reason(self, text: str) -> str:
file = self.home / NAME
file.write_text(text, encoding="utf-8")
read = read_ladder(file)
assert isinstance(read, Failure), read
return read.reason
def test_the_ladder_this_setup_ships_reads_as_a_ladder(self):
self.assertIsInstance(read_ladder(SHIPPED), Ladder)
def test_reads_the_rungs_in_the_files_order(self):
file = self.home / NAME
file.write_text(FABLE_ONLY, encoding="utf-8")
self.assertEqual(read_ladder(file), FABLE_LADDER)
def test_a_file_that_is_missing_or_no_json_fails_naming_the_file_and_the_shape(self):
file = self.home / NAME
missing = read_ladder(file)
self.assertIsInstance(missing, Failure)
self.assertTrue(missing.reason.startswith(f"cannot read the routing ladder {file}: No such file or directory. The routing ladder is a JSON list of rungs"), missing.reason)
# The comma a reader leaves after the last rung when dropping one.
trailing = self.reason(f"[\n\t{OPUS},\n]\n")
self.assertIn(f"cannot read the routing ladder {file}: ", trailing)
self.assertIn("line 2", trailing)
self.assertTrue(trailing.endswith('. The routing ladder is a JSON list of rungs, least competent first, each as {"model": "claude-opus-5-5", "effort": "medium"}: a model\'s full name, one model for each family such as opus, and an effort of low, medium, high, xhigh or max'), trailing)
file.write_bytes(b"\xff")
self.assertIn(f"cannot read the routing ladder {file}: ", read_ladder(file).reason)
def test_a_file_holding_anything_but_a_list_of_rungs_fails(self):
for text in (OPUS, "[]", '"claude-opus-5-5 at medium effort"', "null"):
with self.subTest(text=text):
self.assertEqual(self.reason(text).split(". ")[0], f"{self.home / NAME}: it holds no list of rungs")
def test_names_each_misspelt_rung_by_its_number(self):
misspelt = {
"an effort off the scale": '{"model": "claude-opus-5-5", "effort": "ultra"}',
"no effort": '{"model": "claude-opus-5-5"}',
"an empty effort": '{"model": "claude-opus-5-5", "effort": ""}',
"a key too many": '{"model": "claude-opus-5-5", "effort": "low", "note": "x"}',
"a model that names no family": '{"model": "opus", "effort": "low"}',
"a model with a space": '{"model": "claude-opus 5-5", "effort": "low"}',
"a model that is no text": '{"model": 5, "effort": "low"}',
"a pair": '["claude-opus-5-5", "low"]',
"the rung as a tool prints it": '"claude-opus-5-5 at low effort"',
}
for what, rung in misspelt.items():
with self.subTest(what):
self.assertEqual(self.reason(f"[{OPUS}, {rung}]").split(". ")[0], f"{self.home / NAME}: rung 2 is {rung}")
self.assertIn("rung 1 is 1; rung 3 is 3. The routing ladder is a JSON list of rungs, least competent first", self.reason(f"[1, {OPUS}, 3]"))
def test_a_rung_held_twice_fails_naming_both_since_a_rungs_position_sets_every_step(self):
low = '{"model": "claude-opus-5-5", "effort": "low"}'
self.assertEqual(self.reason(f"[{low}, {OPUS}, {low}]").split(". ")[0], f"{self.home / NAME}: rung 3 repeats rung 1")
def test_a_second_model_of_one_family_fails_naming_the_familys_first_rung_since_a_subagent_launches_by_family(self):
older, newer = '{"model": "claude-fable-5-1", "effort": "max"}', '{"model": "claude-fable-5-2", "effort": "low"}'
self.assertEqual(self.reason(f"[{OPUS}, {older}, {newer}]").split(". ")[0], f"{self.home / NAME}: rung 3 names a second model of the family fable, after rung 2")
both = self.reason(f'[{older}, {newer}, {{"model": "claude-fable-5-2", "effort": "high"}}]').split(". ")[0]
self.assertEqual(both, f"{self.home / NAME}: rung 2 names a second model of the family fable, after rung 1; rung 3 names a second model of the family fable, after rung 1")
file = self.home / NAME
file.write_text(f'[{OPUS}, {newer}, {{"model": "claude-opus-5-5", "effort": "xhigh"}}, {{"model": "claude-fable-5-2", "effort": "max"}}]', encoding="utf-8")
self.assertIsInstance(read_ladder(file), Ladder, "one model at several efforts, between another family's rungs, is no fault")
class ProjectLadder(LadderCase):
def test_a_project_runs_on_the_ladder_its_configs_folder_holds_from_a_worktree_too(self):
checkout = self.checkout()
(write_config(self.home, checkout) / NAME).write_text(FABLE_ONLY, encoding="utf-8")
self.assertEqual(project_ladder(str(checkout), self.home, self.shipped), FABLE_LADDER)
worktree = self.home / "code" / "acme-item-7"
self.git(checkout, "worktree", "add", "--quiet", str(worktree), "-b", "item-7")
self.assertEqual(project_ladder(str(worktree), self.home, self.shipped), FABLE_LADDER)
def test_names_the_file_it_reads_the_projects_own_or_else_the_shipped_one(self):
checkout = self.checkout()
folder = write_config(self.home, checkout)
self.assertEqual(project_ladder_file(str(checkout), self.home, self.shipped), self.shipped)
(folder / NAME).write_text(FABLE_ONLY, encoding="utf-8")
self.assertEqual(project_ladder_file(str(checkout), self.home, self.shipped), folder / NAME)
self.assertEqual(project_ladder_file(str(self.home), self.home), SHIPPED, "outside a checkout, the ladder this setup ships")
def test_another_projects_ladder_is_not_read(self):
checkout, other = self.checkout(), self.checkout("other")
write_config(self.home, checkout)
(write_config(self.home, other) / NAME).write_text(FABLE_ONLY, encoding="utf-8")
self.assertEqual(project_ladder(str(checkout), self.home, self.shipped), Ladder((Rung("claude-opus-5-5", "medium"),)))
def test_a_checkout_no_config_names_and_a_folder_outside_any_checkout_run_on_the_shipped_ladder(self):
shipped = Ladder((Rung("claude-opus-5-5", "medium"),))
self.assertEqual(project_ladder(str(self.checkout()), self.home, self.shipped), shipped)
self.assertEqual(project_ladder(str(self.home), self.home, self.shipped), shipped)
def test_a_faulty_ladder_of_the_projects_own_fails_and_the_shipped_one_does_not_stand_in(self):
checkout = self.checkout()
own = write_config(self.home, checkout) / NAME
own.write_text("[]", encoding="utf-8")
self.assertEqual(project_ladder(str(checkout), self.home, self.shipped).reason.split(". ")[0], f"{own}: it holds no list of rungs")
def test_a_faulty_config_fails_since_its_folder_may_hold_the_ladder(self):
checkout = self.checkout()
config = write_config(self.home, checkout) / "config.toml"
config.write_text(config.read_text(encoding="utf-8") + 'readme = "checkout:README.md"\n', encoding="utf-8")
self.assertIn("`readme` is not a setting", project_ladder(str(checkout), self.home, self.shipped).reason)
if __name__ == "__main__":
unittest.main()