110 lines
6.0 KiB
Python
110 lines
6.0 KiB
Python
import pathlib
|
|
import sys
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from plan.todo_table import LEVELS
|
|
from model_routing.ladder import ProjectRung, Rung, WORKER_STEPS
|
|
from model_routing.ladder_file import read_ladder, SHIPPED
|
|
from review_rules.review_passes import REVIEWERS
|
|
|
|
# The ladder this setup ships, whose rungs the tests below name.
|
|
LADDER = read_ladder(SHIPPED)
|
|
OPUS = Rung("claude-opus-5-5", "medium")
|
|
PROJECT = ProjectRung(LADDER, OPUS)
|
|
FABLE_LOW = Rung("claude-fable-5-2", "low")
|
|
|
|
|
|
class Ladder(unittest.TestCase):
|
|
def test_puts_a_project_on_its_models_cheapest_rung_and_none_off_the_ladder(self):
|
|
self.assertEqual(LADDER.project_rung("claude-opus-5-5"), PROJECT)
|
|
self.assertEqual(LADDER.project_rung("claude-sonnet-5-5"), ProjectRung(LADDER, Rung("claude-sonnet-5-5", "high")))
|
|
self.assertEqual(LADDER.project_rung("claude-fable-5-2"), ProjectRung(LADDER, FABLE_LOW))
|
|
self.assertIsNone(LADDER.project_rung("claude-opus-5"))
|
|
self.assertIsNone(LADDER.project_rung("opus"))
|
|
|
|
def test_steps_a_workers_rung_by_its_l_from_the_projects_rung(self):
|
|
position = LADDER.rungs.index(OPUS)
|
|
self.assertEqual([LADDER.rungs.index(PROJECT.worker_rung(likelihood)) - position for likelihood in range(1, 5)], [-1, -1, 1, 2])
|
|
|
|
def test_steps_a_review_up_by_its_e_plus_i_from_the_projects_rung_and_never_down(self):
|
|
position = LADDER.rungs.index(OPUS)
|
|
self.assertEqual([LADDER.rungs.index(PROJECT.review_floor(total)) - position for total in range(2, 9)], [0, 0, 0, 1, 2, 3, 4])
|
|
self.assertEqual(PROJECT.review_floor(0), OPUS)
|
|
|
|
def test_a_step_past_either_end_stops_there(self):
|
|
self.assertEqual(PROJECT.worker_rung(1), Rung("claude-sonnet-5-5", "high"))
|
|
self.assertEqual(ProjectRung(LADDER, LADDER.rungs[0]).worker_rung(1), LADDER.rungs[0])
|
|
self.assertEqual(ProjectRung(LADDER, Rung("claude-opus-5-5", "xhigh")).review_floor(8), LADDER.rungs[-1])
|
|
self.assertEqual(LADDER.stepped_up(LADDER.rungs[-1]), LADDER.rungs[-1])
|
|
self.assertEqual(LADDER.stepped_up(OPUS), FABLE_LOW)
|
|
|
|
def test_steps_a_worker_for_every_level_on_ls_scale(self):
|
|
self.assertEqual(set(WORKER_STEPS), set(range(1, LEVELS["L"] + 1)))
|
|
|
|
def test_names_the_lowest_rung_any_l_routes_a_worker_to(self):
|
|
self.assertEqual(PROJECT.lowest_worker_rung(), Rung("claude-sonnet-5-5", "high"))
|
|
|
|
def test_plans_on_the_rung_l_picks_but_never_below_the_projects(self):
|
|
self.assertEqual(PROJECT.planning_rung(1), OPUS)
|
|
self.assertEqual(PROJECT.planning_rung(4), Rung("claude-fable-5-2", "medium"))
|
|
|
|
def test_names_each_rung_by_a_family_the_agent_tool_takes(self):
|
|
self.assertEqual({rung.agent_args().split(",")[0] for rung in LADDER.rungs}, {"model sonnet", "model opus", "model fable"})
|
|
|
|
def test_ranks_a_rung_off_the_ladder_below_every_rung_on_it(self):
|
|
self.assertEqual(LADDER.highest(OPUS, LADDER.rungs[-1], LADDER.rungs[0]), LADDER.rungs[-1])
|
|
self.assertEqual(LADDER.highest(Rung("claude-opus-5-5", "low"), LADDER.rungs[0]), LADDER.rungs[0])
|
|
self.assertEqual(LADDER.stepped_up(Rung("claude-opus-5-5", "low")), LADDER.rungs[0])
|
|
|
|
def test_passes_the_effort_only_where_the_rung_names_one(self):
|
|
self.assertEqual(Rung("claude-opus-5-5", "high").flags(), ["--model", "claude-opus-5-5", "--effort", "high"])
|
|
self.assertEqual(Rung("worker-model").flags(), ["--model", "worker-model"])
|
|
self.assertEqual(str(Rung("claude-opus-5-5", "high")), "claude-opus-5-5 at high effort")
|
|
self.assertEqual(str(Rung("worker-model")), "worker-model")
|
|
|
|
def test_parses_what_str_wrote_for_each_rung_on_the_ladder(self):
|
|
for rung in LADDER.rungs:
|
|
with self.subTest(rung=rung):
|
|
self.assertEqual(Rung.parse(str(rung)), rung)
|
|
|
|
def test_parses_a_rung_off_the_ladder_too_for_the_caller_to_judge(self):
|
|
self.assertEqual(Rung.parse("claude-opus-5-5 at low effort"), Rung("claude-opus-5-5", "low"))
|
|
|
|
def test_parses_no_other_wording_as_a_rung(self):
|
|
for text in ("", "claude-opus-5-5", "claude-opus-5-5 medium", "claude-opus-5-5 at medium", "at medium effort", "a b at medium effort"):
|
|
with self.subTest(text=text):
|
|
self.assertIsNone(Rung.parse(text))
|
|
|
|
def test_reads_the_agent_tools_words_back_as_a_rung(self):
|
|
for rung in LADDER.rungs:
|
|
with self.subTest(rung=rung):
|
|
model, effort = (part.split(" ")[1] for part in rung.agent_args().split(", "))
|
|
self.assertEqual(LADDER.from_agent(model, effort), rung)
|
|
|
|
def test_reads_a_family_named_without_an_effort_as_its_cheapest_rung(self):
|
|
self.assertEqual(LADDER.from_agent("opus", ""), OPUS)
|
|
self.assertEqual(LADDER.from_agent("sonnet", ""), Rung("claude-sonnet-5-5", "high"))
|
|
|
|
def test_reads_an_unknown_family_or_an_effort_off_the_ladder_as_a_rung_off_it(self):
|
|
for model, effort in (("gpt", "high"), ("gpt", ""), ("opus", "low"), ("opus", "high"), ("haiku", "bogus")):
|
|
with self.subTest(model=model, effort=effort):
|
|
self.assertNotIn(LADDER.from_agent(model, effort), LADDER.rungs)
|
|
self.assertEqual(LADDER.from_agent("opus", "low"), Rung("claude-opus-5-5", "low"))
|
|
self.assertEqual(LADDER.from_agent("gpt", "high"), Rung("gpt", "high"))
|
|
|
|
def test_puts_reviewers_on_the_highest_of_the_projects_rung_the_workers_and_the_one_e_plus_i_picks(self):
|
|
sonnet, fable = LADDER.rungs[0], LADDER.rungs[-1]
|
|
self.assertEqual(PROJECT.review_rung(sonnet, 4, "stability"), OPUS)
|
|
self.assertEqual(PROJECT.review_rung(sonnet, 5, "stability"), FABLE_LOW)
|
|
self.assertEqual(PROJECT.review_rung(fable, 5, "stability"), fable)
|
|
self.assertEqual(PROJECT.review_rung(FABLE_LOW, 7, "stability"), Rung("claude-opus-5-5", "xhigh"))
|
|
|
|
def test_keeps_the_seats_and_the_security_pass_on_the_projects_rung_whatever_the_worker_or_e_plus_i(self):
|
|
self.assertEqual([PROJECT.review_rung(LADDER.rungs[-1], 8, kind) for kind in (*REVIEWERS, "seat", "security")], [LADDER.rungs[-1]] * len(REVIEWERS) + [OPUS, OPUS])
|
|
|
|
def test_puts_every_review_of_a_branch_changing_only_markdown_no_agent_follows_on_the_projects_rung(self):
|
|
self.assertEqual([PROJECT.review_rung(LADDER.rungs[-1], 8, kind, markdown_only=True) for kind in (*REVIEWERS, "seat", "security")], [OPUS] * (len(REVIEWERS) + 2))
|
|
self.assertEqual(PROJECT.review_rung(LADDER.rungs[0], 4, "stability", markdown_only=True), OPUS)
|