148 lines
7.9 KiB
Python
148 lines
7.9 KiB
Python
import pathlib
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from failure import Failure
|
|
from test_env import isolated_env
|
|
from model_routing.ladder import Ladder, Rung
|
|
from model_routing.ladder_file import NAME, project_ladder, project_ladder_file, read_ladder, SHIPPED
|
|
from project_config.doubles import write_config
|
|
|
|
FABLE_ONLY = '[\n\t{"model": "claude-fable-5-2", "effort": "low"},\n\t{"model": "claude-fable-5-2", "effort": "max"}\n]\n'
|
|
FABLE_LADDER = Ladder((Rung("claude-fable-5-2", "low"), Rung("claude-fable-5-2", "max")))
|
|
OPUS = '{"model": "claude-opus-5-5", "effort": "medium"}'
|
|
|
|
|
|
class LadderCase(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
made = tempfile.TemporaryDirectory()
|
|
self.addCleanup(made.cleanup)
|
|
self.home = pathlib.Path(made.name).resolve()
|
|
self.shipped = self.home / "shipped.json"
|
|
self.shipped.write_text(f"[{OPUS}]", encoding="utf-8")
|
|
|
|
def checkout(self, name: str = "acme") -> pathlib.Path:
|
|
"""A git checkout under the home, holding one commit, that no config names yet."""
|
|
checkout = self.home / "code" / name
|
|
checkout.mkdir(parents=True)
|
|
self.git(checkout, "init", "--quiet", "-b", "main")
|
|
self.git(checkout, "commit", "--quiet", "--allow-empty", "-m", "base")
|
|
return checkout
|
|
|
|
def git(self, cwd: pathlib.Path, *args: str) -> None:
|
|
subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@example.com", *args], capture_output=True, check=True, cwd=cwd, env=isolated_env())
|
|
|
|
|
|
class ReadLadder(LadderCase):
|
|
def reason(self, text: str) -> str:
|
|
file = self.home / NAME
|
|
file.write_text(text, encoding="utf-8")
|
|
read = read_ladder(file)
|
|
assert isinstance(read, Failure), read
|
|
return read.reason
|
|
|
|
def test_the_ladder_this_setup_ships_reads_as_a_ladder(self):
|
|
self.assertIsInstance(read_ladder(SHIPPED), Ladder)
|
|
|
|
def test_reads_the_rungs_in_the_files_order(self):
|
|
file = self.home / NAME
|
|
file.write_text(FABLE_ONLY, encoding="utf-8")
|
|
self.assertEqual(read_ladder(file), FABLE_LADDER)
|
|
|
|
def test_a_file_that_is_missing_or_no_json_fails_naming_the_file_and_the_shape(self):
|
|
file = self.home / NAME
|
|
missing = read_ladder(file)
|
|
self.assertIsInstance(missing, Failure)
|
|
self.assertTrue(missing.reason.startswith(f"cannot read the routing ladder {file}: No such file or directory. The routing ladder is a JSON list of rungs"), missing.reason)
|
|
# The comma a reader leaves after the last rung when dropping one.
|
|
trailing = self.reason(f"[\n\t{OPUS},\n]\n")
|
|
self.assertIn(f"cannot read the routing ladder {file}: ", trailing)
|
|
self.assertIn("line 2", trailing)
|
|
self.assertTrue(trailing.endswith('. The routing ladder is a JSON list of rungs, least competent first, each as {"model": "claude-opus-5-5", "effort": "medium"}: a model\'s full name, one model for each family such as opus, and an effort of low, medium, high, xhigh or max'), trailing)
|
|
file.write_bytes(b"\xff")
|
|
self.assertIn(f"cannot read the routing ladder {file}: ", read_ladder(file).reason)
|
|
|
|
def test_a_file_holding_anything_but_a_list_of_rungs_fails(self):
|
|
for text in (OPUS, "[]", '"claude-opus-5-5 at medium effort"', "null"):
|
|
with self.subTest(text=text):
|
|
self.assertEqual(self.reason(text).split(". ")[0], f"{self.home / NAME}: it holds no list of rungs")
|
|
|
|
def test_names_each_misspelt_rung_by_its_number(self):
|
|
misspelt = {
|
|
"an effort off the scale": '{"model": "claude-opus-5-5", "effort": "ultra"}',
|
|
"no effort": '{"model": "claude-opus-5-5"}',
|
|
"an empty effort": '{"model": "claude-opus-5-5", "effort": ""}',
|
|
"a key too many": '{"model": "claude-opus-5-5", "effort": "low", "note": "x"}',
|
|
"a model that names no family": '{"model": "opus", "effort": "low"}',
|
|
"a model with a space": '{"model": "claude-opus 5-5", "effort": "low"}',
|
|
"a model that is no text": '{"model": 5, "effort": "low"}',
|
|
"a pair": '["claude-opus-5-5", "low"]',
|
|
"the rung as a tool prints it": '"claude-opus-5-5 at low effort"',
|
|
}
|
|
for what, rung in misspelt.items():
|
|
with self.subTest(what):
|
|
self.assertEqual(self.reason(f"[{OPUS}, {rung}]").split(". ")[0], f"{self.home / NAME}: rung 2 is {rung}")
|
|
self.assertIn("rung 1 is 1; rung 3 is 3. The routing ladder is a JSON list of rungs, least competent first", self.reason(f"[1, {OPUS}, 3]"))
|
|
|
|
def test_a_rung_held_twice_fails_naming_both_since_a_rungs_position_sets_every_step(self):
|
|
low = '{"model": "claude-opus-5-5", "effort": "low"}'
|
|
self.assertEqual(self.reason(f"[{low}, {OPUS}, {low}]").split(". ")[0], f"{self.home / NAME}: rung 3 repeats rung 1")
|
|
|
|
def test_a_second_model_of_one_family_fails_naming_the_familys_first_rung_since_a_subagent_launches_by_family(self):
|
|
older, newer = '{"model": "claude-fable-5-1", "effort": "max"}', '{"model": "claude-fable-5-2", "effort": "low"}'
|
|
self.assertEqual(self.reason(f"[{OPUS}, {older}, {newer}]").split(". ")[0], f"{self.home / NAME}: rung 3 names a second model of the family fable, after rung 2")
|
|
both = self.reason(f'[{older}, {newer}, {{"model": "claude-fable-5-2", "effort": "high"}}]').split(". ")[0]
|
|
self.assertEqual(both, f"{self.home / NAME}: rung 2 names a second model of the family fable, after rung 1; rung 3 names a second model of the family fable, after rung 1")
|
|
file = self.home / NAME
|
|
file.write_text(f'[{OPUS}, {newer}, {{"model": "claude-opus-5-5", "effort": "xhigh"}}, {{"model": "claude-fable-5-2", "effort": "max"}}]', encoding="utf-8")
|
|
self.assertIsInstance(read_ladder(file), Ladder, "one model at several efforts, between another family's rungs, is no fault")
|
|
|
|
|
|
class ProjectLadder(LadderCase):
|
|
def test_a_project_runs_on_the_ladder_its_configs_folder_holds_from_a_worktree_too(self):
|
|
checkout = self.checkout()
|
|
(write_config(self.home, checkout) / NAME).write_text(FABLE_ONLY, encoding="utf-8")
|
|
self.assertEqual(project_ladder(str(checkout), self.home, self.shipped), FABLE_LADDER)
|
|
worktree = self.home / "code" / "acme-item-7"
|
|
self.git(checkout, "worktree", "add", "--quiet", str(worktree), "-b", "item-7")
|
|
self.assertEqual(project_ladder(str(worktree), self.home, self.shipped), FABLE_LADDER)
|
|
|
|
def test_names_the_file_it_reads_the_projects_own_or_else_the_shipped_one(self):
|
|
checkout = self.checkout()
|
|
folder = write_config(self.home, checkout)
|
|
self.assertEqual(project_ladder_file(str(checkout), self.home, self.shipped), self.shipped)
|
|
(folder / NAME).write_text(FABLE_ONLY, encoding="utf-8")
|
|
self.assertEqual(project_ladder_file(str(checkout), self.home, self.shipped), folder / NAME)
|
|
self.assertEqual(project_ladder_file(str(self.home), self.home), SHIPPED, "outside a checkout, the ladder this setup ships")
|
|
|
|
def test_another_projects_ladder_is_not_read(self):
|
|
checkout, other = self.checkout(), self.checkout("other")
|
|
write_config(self.home, checkout)
|
|
(write_config(self.home, other) / NAME).write_text(FABLE_ONLY, encoding="utf-8")
|
|
self.assertEqual(project_ladder(str(checkout), self.home, self.shipped), Ladder((Rung("claude-opus-5-5", "medium"),)))
|
|
|
|
def test_a_checkout_no_config_names_and_a_folder_outside_any_checkout_run_on_the_shipped_ladder(self):
|
|
shipped = Ladder((Rung("claude-opus-5-5", "medium"),))
|
|
self.assertEqual(project_ladder(str(self.checkout()), self.home, self.shipped), shipped)
|
|
self.assertEqual(project_ladder(str(self.home), self.home, self.shipped), shipped)
|
|
|
|
def test_a_faulty_ladder_of_the_projects_own_fails_and_the_shipped_one_does_not_stand_in(self):
|
|
checkout = self.checkout()
|
|
own = write_config(self.home, checkout) / NAME
|
|
own.write_text("[]", encoding="utf-8")
|
|
self.assertEqual(project_ladder(str(checkout), self.home, self.shipped).reason.split(". ")[0], f"{own}: it holds no list of rungs")
|
|
|
|
def test_a_faulty_config_fails_since_its_folder_may_hold_the_ladder(self):
|
|
checkout = self.checkout()
|
|
config = write_config(self.home, checkout) / "config.toml"
|
|
config.write_text(config.read_text(encoding="utf-8") + 'readme = "checkout:README.md"\n', encoding="utf-8")
|
|
self.assertIn("`readme` is not a setting", project_ladder(str(checkout), self.home, self.shipped).reason)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|