125 lines
6.9 KiB
Python
125 lines
6.9 KiB
Python
import pathlib
|
|
import sys
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from rung_replay.search import AT_ONCE
|
|
from review_replay.recall import candidates, ENOUGH_ROUNDS, needed, Recall, recall_report, recalled, tried
|
|
from review_replay.rounds import is_red, Round
|
|
from model_routing.ladder_file import read_ladder, SHIPPED
|
|
from model_routing.ladder import Rung
|
|
|
|
LADDER = read_ladder(SHIPPED)
|
|
REFERENCE = Rung("claude-opus-5-5", "medium")
|
|
LOW = Rung("claude-opus-5-5", "low")
|
|
SONNET = Rung("claude-sonnet-5-5", "high")
|
|
|
|
|
|
def a_round(at, findings):
|
|
return Round("prose-reviewer", "/code/p", "x", f"2026-10-14T10:00:0{at}Z", 0, "Review it.", tuple(findings))
|
|
|
|
|
|
# Ten findings over two rounds; the project's rung found 8 and then 6, so its share is 70%, the bar 60% and a rung needs 6. Both found the first round's 🔴, and only the first found the second's.
|
|
ROUNDS = [a_round(1, ("🔴 a", "🟡 b", "🟡 c", "🟡 d", "🟡 e")), a_round(2, ("🔴 f\nmore", "🟡 g", "🟡 h", "🟡 i", "🟡 j"))]
|
|
FIRST = [{0, 1, 2, 3}, {0, 1, 2, 3}]
|
|
SECOND = [{0, 1, 2}, {1, 2, 3}]
|
|
# The project's rung found 1 and then 1 of the 10, so the bar is zero.
|
|
TENTH = ([{0}, set()], [{0}, set()])
|
|
|
|
|
|
class Candidates(unittest.TestCase):
|
|
def test_tries_the_project_models_lower_efforts_first_then_the_ladder_below_it(self):
|
|
self.assertEqual(candidates(LADDER, REFERENCE), [LOW, SONNET])
|
|
|
|
def test_tries_each_ladder_rung_below_highest_first(self):
|
|
self.assertEqual(candidates(LADDER, Rung("claude-fable-5-2", "low")), [REFERENCE, SONNET])
|
|
|
|
def test_tries_each_lower_effort_of_the_ladders_lowest_rung(self):
|
|
self.assertEqual(candidates(LADDER, SONNET), [Rung("claude-sonnet-5-5", "medium"), Rung("claude-sonnet-5-5", "low")])
|
|
|
|
|
|
class Recalled(unittest.TestCase):
|
|
def test_tries_the_rungs_below_where_the_project_rung_needs_one_finding(self):
|
|
# Eight findings, of which both replays on the project's rung find one: a share of an eighth, a tenth under which leaves one finding needed.
|
|
rounds = [a_round(at, ("🟡 a",)) for at in range(8)]
|
|
recall = recalled(rounds, LADDER, REFERENCE, lambda one, rung: {0} if one == rounds[0] else set(), "prose rounds", lambda text: None, AT_ONCE, ENOUGH_ROUNDS)
|
|
self.assertEqual(([one.rung for one in recall.tried], recall.lowest), ([LOW, SONNET], SONNET))
|
|
|
|
|
|
class Tried(unittest.TestCase):
|
|
def test_qualifies_a_rung_no_more_than_the_margin_below_the_project_rungs_share_that_misses_no_red_found_both_times(self):
|
|
found = tried(LOW, ROUNDS, (FIRST, SECOND), [{0, 1, 2}, {0, 1, 2}])
|
|
self.assertEqual((found.found, found.missed_red, found.qualifies), (6, [], True))
|
|
|
|
def test_fails_a_rung_below_the_bar(self):
|
|
found = tried(LOW, ROUNDS, (FIRST, SECOND), [{0, 1}, {0, 3}])
|
|
self.assertEqual((found.found, found.missed_red, found.qualifies), (4, [], False))
|
|
|
|
def test_fails_a_rung_that_misses_a_red_found_both_times_and_only_such_a_red(self):
|
|
above_the_bar = [{1, 2, 3, 4}, {0, 1, 2, 3}]
|
|
found = tried(LOW, ROUNDS, (FIRST, SECOND), above_the_bar)
|
|
self.assertEqual((found.found, found.missed_red, found.qualifies), (8, [(ROUNDS[0], 0)], False))
|
|
# The first round's 🔴 is found both times; a yellow found both times, such as its b, may be missed.
|
|
self.assertTrue(tried(LOW, ROUNDS, (FIRST, SECOND), [{0, 2, 3, 4}, {0, 2, 3, 4}]).qualifies)
|
|
# The second round's 🔴, which only the first replay found, may be missed too.
|
|
self.assertTrue(tried(LOW, ROUNDS, (FIRST, SECOND), [{0, 1, 2, 3}, {1, 2, 3}]).qualifies)
|
|
|
|
def test_fails_every_rung_where_the_bar_is_zero_or_below(self):
|
|
self.assertFalse(tried(LOW, ROUNDS, ([set(), set()], [set(), set()]), [{0}, {0}]).qualifies)
|
|
self.assertFalse(tried(LOW, ROUNDS, TENTH, [{0}, {0}]).qualifies)
|
|
|
|
def test_rounds_the_findings_a_rung_needs_up(self):
|
|
# 33 findings, of which the project's rung found 24 and then 23: the bar is 61.2%, which 20 findings, 60.6%, don't reach.
|
|
rounds = [a_round(1, ["🟡 a"] * 33)]
|
|
reference = ([set(range(24))], [set(range(23))])
|
|
self.assertEqual(needed(rounds, reference), 21)
|
|
self.assertFalse(tried(LOW, rounds, reference, [set(range(20))]).qualifies)
|
|
self.assertTrue(tried(LOW, rounds, reference, [set(range(21))]).qualifies)
|
|
|
|
def test_lets_a_rung_miss_a_yellow_that_mentions_a_red(self):
|
|
rounds = [a_round(1, ("### 🟡 `SKILL.md:80`: the rule says a 🔴 in round 2 escalates", "🟡 b"))]
|
|
self.assertFalse(is_red(rounds[0].findings[0]))
|
|
self.assertTrue(is_red("### 1. **🔴 a\nwhich is no 🟡"))
|
|
found = tried(LOW, rounds, ([{0, 1}], [{0, 1}]), [{1}])
|
|
self.assertEqual((found.missed_red, found.qualifies), ([], False))
|
|
self.assertEqual(tried(LOW, rounds, ([{0, 1}], [{0}]), [{1}]).missed_red, [])
|
|
|
|
|
|
class RecallReport(unittest.TestCase):
|
|
# Eight rounds, enough to try a rung.
|
|
EIGHT = (ROUNDS * 4, (FIRST * 4, SECOND * 4))
|
|
|
|
def test_states_the_bar_each_rung_tried_each_red_it_missed_and_the_lowest_that_qualifies(self):
|
|
rounds, reference = self.EIGHT
|
|
low = tried(LOW, rounds, reference, [{0, 1, 2}, {0, 1, 2}] * 4)
|
|
sonnet = tried(SONNET, rounds, reference, [{1, 2, 3, 4}, {0}] * 4)
|
|
lines = recall_report("prose", REFERENCE, Recall(rounds, reference, 1, [low, sonnet], LOW), ENOUGH_ROUNDS).splitlines()
|
|
self.assertEqual(lines, [
|
|
"Replayed 8 prose rounds of claude-opus-5-5 projects, which hold 40 🔴 and 🟡 findings, twice on the project's rung, claude-opus-5-5 at medium effort.",
|
|
"Left out 1 more, which a replay on the project's rung failed on; stderr says why.",
|
|
"The project's rung found 32 and then 24 of the 40, 70% on average, so a rung qualifies finding 24 or more of them, missing no 🔴 that both of its replays found.",
|
|
"claude-opus-5-5 at low effort found 24, 60%, and missed no such 🔴: it qualifies.",
|
|
"claude-sonnet-5-5 at high effort found 20, 50%, and missed 4 such 🔴: it doesn't qualify.",
|
|
*[" /code/p 2026-10-14T10:00:01Z: 🔴 a"] * 4,
|
|
"Lowest rung that qualifies: claude-opus-5-5 at low effort",
|
|
])
|
|
|
|
def test_says_why_no_rung_was_tried(self):
|
|
few = recall_report("prose", REFERENCE, Recall(ROUNDS, (FIRST, SECOND), 0, [], None), ENOUGH_ROUNDS).splitlines()
|
|
self.assertEqual(few[1:], ["No rung tried: fewer than 8 rounds were replayed on the project's rung without a failure."])
|
|
rounds, _ = self.EIGHT
|
|
for reference in (([set()] * 8, [set()] * 8), (TENTH[0] * 4, TENTH[1] * 4)):
|
|
too_few = recall_report("prose", REFERENCE, Recall(rounds, reference, 0, [], None), ENOUGH_ROUNDS).splitlines()
|
|
self.assertEqual(too_few[1:], ["No rung tried: the project's rung found too few of them to hold a rung to."])
|
|
|
|
def test_names_no_rung_where_none_qualifies(self):
|
|
rounds, reference = self.EIGHT
|
|
low = tried(LOW, rounds, reference, [{0}, {0}] * 4)
|
|
lines = recall_report("prose", REFERENCE, Recall(rounds, reference, 0, [low], None), ENOUGH_ROUNDS).splitlines()
|
|
self.assertEqual(lines[-2:], ["claude-opus-5-5 at low effort found 8, 20%, and missed no such 🔴: it doesn't qualify.", "Lowest rung that qualifies: none below the project's rung"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|