Files

110 lines
6.0 KiB
Python

import pathlib
import sys
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from plan.todo_table import LEVELS
from model_routing.ladder import ProjectRung, Rung, WORKER_STEPS
from model_routing.ladder_file import read_ladder, SHIPPED
from review_rules.review_passes import REVIEWERS
# The ladder this setup ships, whose rungs the tests below name.
LADDER = read_ladder(SHIPPED)
OPUS = Rung("claude-opus-5-5", "medium")
PROJECT = ProjectRung(LADDER, OPUS)
FABLE_LOW = Rung("claude-fable-5-2", "low")
class Ladder(unittest.TestCase):
def test_puts_a_project_on_its_models_cheapest_rung_and_none_off_the_ladder(self):
self.assertEqual(LADDER.project_rung("claude-opus-5-5"), PROJECT)
self.assertEqual(LADDER.project_rung("claude-sonnet-5-5"), ProjectRung(LADDER, Rung("claude-sonnet-5-5", "high")))
self.assertEqual(LADDER.project_rung("claude-fable-5-2"), ProjectRung(LADDER, FABLE_LOW))
self.assertIsNone(LADDER.project_rung("claude-opus-5"))
self.assertIsNone(LADDER.project_rung("opus"))
def test_steps_a_workers_rung_by_its_l_from_the_projects_rung(self):
position = LADDER.rungs.index(OPUS)
self.assertEqual([LADDER.rungs.index(PROJECT.worker_rung(likelihood)) - position for likelihood in range(1, 5)], [-1, -1, 1, 2])
def test_steps_a_review_up_by_its_e_plus_i_from_the_projects_rung_and_never_down(self):
position = LADDER.rungs.index(OPUS)
self.assertEqual([LADDER.rungs.index(PROJECT.review_floor(total)) - position for total in range(2, 9)], [0, 0, 0, 1, 2, 3, 4])
self.assertEqual(PROJECT.review_floor(0), OPUS)
def test_a_step_past_either_end_stops_there(self):
self.assertEqual(PROJECT.worker_rung(1), Rung("claude-sonnet-5-5", "high"))
self.assertEqual(ProjectRung(LADDER, LADDER.rungs[0]).worker_rung(1), LADDER.rungs[0])
self.assertEqual(ProjectRung(LADDER, Rung("claude-opus-5-5", "xhigh")).review_floor(8), LADDER.rungs[-1])
self.assertEqual(LADDER.stepped_up(LADDER.rungs[-1]), LADDER.rungs[-1])
self.assertEqual(LADDER.stepped_up(OPUS), FABLE_LOW)
def test_steps_a_worker_for_every_level_on_ls_scale(self):
self.assertEqual(set(WORKER_STEPS), set(range(1, LEVELS["L"] + 1)))
def test_names_the_lowest_rung_any_l_routes_a_worker_to(self):
self.assertEqual(PROJECT.lowest_worker_rung(), Rung("claude-sonnet-5-5", "high"))
def test_plans_on_the_rung_l_picks_but_never_below_the_projects(self):
self.assertEqual(PROJECT.planning_rung(1), OPUS)
self.assertEqual(PROJECT.planning_rung(4), Rung("claude-fable-5-2", "medium"))
def test_names_each_rung_by_a_family_the_agent_tool_takes(self):
self.assertEqual({rung.agent_args().split(",")[0] for rung in LADDER.rungs}, {"model sonnet", "model opus", "model fable"})
def test_ranks_a_rung_off_the_ladder_below_every_rung_on_it(self):
self.assertEqual(LADDER.highest(OPUS, LADDER.rungs[-1], LADDER.rungs[0]), LADDER.rungs[-1])
self.assertEqual(LADDER.highest(Rung("claude-opus-5-5", "low"), LADDER.rungs[0]), LADDER.rungs[0])
self.assertEqual(LADDER.stepped_up(Rung("claude-opus-5-5", "low")), LADDER.rungs[0])
def test_passes_the_effort_only_where_the_rung_names_one(self):
self.assertEqual(Rung("claude-opus-5-5", "high").flags(), ["--model", "claude-opus-5-5", "--effort", "high"])
self.assertEqual(Rung("worker-model").flags(), ["--model", "worker-model"])
self.assertEqual(str(Rung("claude-opus-5-5", "high")), "claude-opus-5-5 at high effort")
self.assertEqual(str(Rung("worker-model")), "worker-model")
def test_parses_what_str_wrote_for_each_rung_on_the_ladder(self):
for rung in LADDER.rungs:
with self.subTest(rung=rung):
self.assertEqual(Rung.parse(str(rung)), rung)
def test_parses_a_rung_off_the_ladder_too_for_the_caller_to_judge(self):
self.assertEqual(Rung.parse("claude-opus-5-5 at low effort"), Rung("claude-opus-5-5", "low"))
def test_parses_no_other_wording_as_a_rung(self):
for text in ("", "claude-opus-5-5", "claude-opus-5-5 medium", "claude-opus-5-5 at medium", "at medium effort", "a b at medium effort"):
with self.subTest(text=text):
self.assertIsNone(Rung.parse(text))
def test_reads_the_agent_tools_words_back_as_a_rung(self):
for rung in LADDER.rungs:
with self.subTest(rung=rung):
model, effort = (part.split(" ")[1] for part in rung.agent_args().split(", "))
self.assertEqual(LADDER.from_agent(model, effort), rung)
def test_reads_a_family_named_without_an_effort_as_its_cheapest_rung(self):
self.assertEqual(LADDER.from_agent("opus", ""), OPUS)
self.assertEqual(LADDER.from_agent("sonnet", ""), Rung("claude-sonnet-5-5", "high"))
def test_reads_an_unknown_family_or_an_effort_off_the_ladder_as_a_rung_off_it(self):
for model, effort in (("gpt", "high"), ("gpt", ""), ("opus", "low"), ("opus", "high"), ("haiku", "bogus")):
with self.subTest(model=model, effort=effort):
self.assertNotIn(LADDER.from_agent(model, effort), LADDER.rungs)
self.assertEqual(LADDER.from_agent("opus", "low"), Rung("claude-opus-5-5", "low"))
self.assertEqual(LADDER.from_agent("gpt", "high"), Rung("gpt", "high"))
def test_puts_reviewers_on_the_highest_of_the_projects_rung_the_workers_and_the_one_e_plus_i_picks(self):
sonnet, fable = LADDER.rungs[0], LADDER.rungs[-1]
self.assertEqual(PROJECT.review_rung(sonnet, 4, "stability"), OPUS)
self.assertEqual(PROJECT.review_rung(sonnet, 5, "stability"), FABLE_LOW)
self.assertEqual(PROJECT.review_rung(fable, 5, "stability"), fable)
self.assertEqual(PROJECT.review_rung(FABLE_LOW, 7, "stability"), Rung("claude-opus-5-5", "xhigh"))
def test_keeps_the_seats_and_the_security_pass_on_the_projects_rung_whatever_the_worker_or_e_plus_i(self):
self.assertEqual([PROJECT.review_rung(LADDER.rungs[-1], 8, kind) for kind in (*REVIEWERS, "seat", "security")], [LADDER.rungs[-1]] * len(REVIEWERS) + [OPUS, OPUS])
def test_puts_every_review_of_a_branch_changing_only_markdown_no_agent_follows_on_the_projects_rung(self):
self.assertEqual([PROJECT.review_rung(LADDER.rungs[-1], 8, kind, markdown_only=True) for kind in (*REVIEWERS, "seat", "security")], [OPUS] * (len(REVIEWERS) + 2))
self.assertEqual(PROJECT.review_rung(LADDER.rungs[0], 4, "stability", markdown_only=True), OPUS)