Files

110 lines
7.4 KiB
Python

import dataclasses
import pathlib
import sys
import types
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from agent_cycle.verdicts import Interrupt, TakenIn, Unfinished
from model_routing.ladder import ProjectRung, Rung
from model_routing.ladder_file import read_ladder, SHIPPED
from model_routing.route import Route
from agent_cycle import routing
LADDER = read_ladder(SHIPPED)
OPUS_PROJECT = ProjectRung(LADDER, Rung("claude-opus-5-5", "medium"))
# The ladder's rungs around the project's: the one below it, its own, and those one to four steps up.
SONNET = Rung("claude-sonnet-5-5", "high")
OPUS = Rung("claude-opus-5-5", "medium")
ONE_UP = Rung("claude-fable-5-2", "low")
TWO_UP = Rung("claude-fable-5-2", "medium")
THREE_UP = Rung("claude-opus-5-5", "xhigh")
FOUR_UP = Rung("claude-fable-5-2", "high")
def scored(likelihood, escape_and_impact):
"""Item 3 with that L and E + I, as routing.Scored reads an item."""
return types.SimpleNamespace(id=3, likelihood=likelihood, escape_and_impact=escape_and_impact)
def unfinished(likelihood=0, escalated_at=None):
"""An unfinished verdict: the worker took in an item of that L, 0 for none, and the judge found an escalation after reading escalated_at transcript entries, None for none."""
return Unfinished("e", escalation="round 2: 🔴" if escalated_at is not None else "", taken_in=TakenIn(highest_likelihood=likelihood), entries_read=escalated_at)
def interrupted(likelihood=0, escalated_at=None):
"""An interrupt, its arguments as unfinished's."""
return Interrupt("e", "q", escalation="round 2: 🔴" if escalated_at is not None else "", taken_in=TakenIn(highest_likelihood=likelihood), entries_read=escalated_at)
class FirstRoute(unittest.TestCase):
def test_starts_a_chunk_on_the_rung_its_items_l_picks_below_the_projects_too(self):
self.assertEqual(routing.first_route(OPUS_PROJECT, scored(4, 5)), Route(TWO_UP, 3, 4, 5))
self.assertEqual(routing.first_route(OPUS_PROJECT, scored(1, 5)), Route(SONNET, 3, 1, 5))
self.assertEqual(routing.first_route(OPUS_PROJECT, None), Route(OPUS))
class NextRoute(unittest.TestCase):
def test_steps_up_a_rung_after_every_second_unfinished_verdict(self):
once = routing.next_route(OPUS_PROJECT, Route(ONE_UP, 3, 3, 4), unfinished())
self.assertEqual(once, Route(ONE_UP, 3, 3, 4, unfinished=1))
self.assertEqual(routing.next_route(OPUS_PROJECT, once, unfinished()), Route(TWO_UP, 3, 3, 4))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(LADDER.rungs[-1], unfinished=1), unfinished()).rung, LADDER.rungs[-1])
def test_steps_up_to_the_rung_a_taken_in_items_l_picks_and_never_down(self):
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(OPUS), unfinished(4)).rung, TWO_UP)
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(THREE_UP), unfinished(4)).rung, THREE_UP)
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(OPUS, unfinished=1), unfinished(4)).rung, THREE_UP)
def test_steps_up_from_a_rung_below_the_projects_one_rung_at_a_time(self):
low = Route(SONNET, 3, 1, 3, unfinished=1)
self.assertEqual(routing.next_route(OPUS_PROJECT, low, unfinished()).rung, OPUS)
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(low.rung, 3, 1, 3), unfinished()).rung, low.rung)
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(low.rung, 3, 1, 3), unfinished(3)).rung, ONE_UP)
def test_steps_up_a_rung_on_an_escalation_the_judge_found_up_to_two_above_the_items_rung_marking_the_transcript_entries_it_read(self):
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(TWO_UP, 3, 4, 5), unfinished(escalated_at=7)), Route(THREE_UP, 3, 4, 5, escalated_at=7))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(THREE_UP, 3, 4, 5, escalated_at=7), unfinished(escalated_at=9)), Route(FOUR_UP, 3, 4, 5, escalated_at=9))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(FOUR_UP, 3, 4, 5, escalated_at=9), unfinished(escalated_at=12)), Route(FOUR_UP, 3, 4, 5, unfinished=1, escalated_at=9))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(TWO_UP, 3, 4, 5, unfinished=1), unfinished(escalated_at=7)), Route(THREE_UP, 3, 4, 5, escalated_at=7))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(TWO_UP), unfinished(escalated_at=7)), Route(TWO_UP, unfinished=1))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(SONNET, 3, 2, 4), unfinished(4, escalated_at=7)), Route(TWO_UP, 3, 2, 4, unfinished=1))
def test_keeps_the_mark_through_a_step_no_escalation_took(self):
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(ONE_UP, 3, 3, 4, unfinished=1, escalated_at=5), unfinished()), Route(TWO_UP, 3, 3, 4, escalated_at=5))
def test_steps_up_a_rung_as_the_judge_interrupts_up_to_two_above_the_items_rung_without_counting_an_unfinished_verdict(self):
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(TWO_UP, 3, 4, 5, unfinished=1), interrupted(escalated_at=7)), Route(THREE_UP, 3, 4, 5, escalated_at=7))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(THREE_UP, 3, 4, 5, escalated_at=7), interrupted(escalated_at=9)), Route(FOUR_UP, 3, 4, 5, escalated_at=9))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(FOUR_UP, 3, 4, 5, unfinished=1, escalated_at=9), interrupted(escalated_at=12)), Route(FOUR_UP, 3, 4, 5, unfinished=1, escalated_at=9))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(SONNET, 3, 2, 4), interrupted(escalated_at=7)), Route(OPUS, 3, 2, 4, escalated_at=7))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(TWO_UP, 3, 4, 5, unfinished=1, escalated_at=2), interrupted(4)), Route(TWO_UP, 3, 4, 5, unfinished=1, escalated_at=2))
def test_raises_the_worker_to_a_taken_in_items_rung_above_the_escalation_cap_without_a_step_as_the_judge_interrupts(self):
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(OPUS, 3, 1, 3, escalated_at=2), interrupted(4, escalated_at=7)), Route(TWO_UP, 3, 1, 3, escalated_at=2))
def test_steps_up_from_the_rung_a_taken_in_item_picks_as_the_judge_interrupts(self):
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(ONE_UP, 3, 3, 4), interrupted(4, escalated_at=7)), Route(THREE_UP, 3, 3, 4, escalated_at=7))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(TWO_UP, 3, 4, 5), interrupted(4, escalated_at=7)), Route(THREE_UP, 3, 4, 5, escalated_at=7))
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(FOUR_UP, 3, 4, 5, unfinished=1), interrupted(4, escalated_at=7)), Route(FOUR_UP, 3, 4, 5, unfinished=1))
def test_a_rung_an_edit_took_off_the_ladder_counts_as_the_projects(self):
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(Rung("claude-opus-5-5", "low")), unfinished()).rung, OPUS)
self.assertEqual(routing.next_route(OPUS_PROJECT, Route(Rung("claude-opus-5-5", "low")), unfinished(2)).rung, OPUS)
class StepUp(unittest.TestCase):
def test_names_no_step_up_where_only_the_count_of_unfinished_verdicts_changed(self):
route = Route(OPUS, 3, 3, 4)
self.assertIsNone(routing.step_up(route, dataclasses.replace(route, unfinished=1)))
self.assertEqual(routing.step_up(route, route.taking_in(6)), routing.StepUp(route, route.taking_in(6)))
class JudgeRung(unittest.TestCase):
def test_runs_the_judge_of_a_replayed_model_on_its_models_highest_rung(self):
self.assertEqual(routing.judge_rung(LADDER, "claude-opus-5-5"), THREE_UP)
# A model off the ladder, or not replayed, gets no effort flag.
self.assertEqual(routing.judge_rung(LADDER, "judge-model"), Rung("judge-model"))
self.assertEqual(routing.judge_rung(LADDER, "claude-sonnet-5-5"), Rung("claude-sonnet-5-5"))