Files
claude-config/bin/review_replay/recall.py
T

122 lines
6.9 KiB
Python

"""The bar review-replay holds a rung to, and its report: docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`."""
import dataclasses
from rung_replay.command import down_from
from model_routing.ladder import EFFORTS, Ladder, Rung
from failure import Failure
from fractions import Fraction
from review_replay.rounds import is_red, Round
from claude_code.reply import Limited
import math
from rung_replay.search import replayed_on
import typing
# A rung qualifies with a share of the archived findings up to this far below the project's rung's. The project's rung found 69 to 78% of its own findings again, and its two replays of the same rounds differed by up to 4 points.
MARGIN = Fraction(1, 10)
# The fewest rounds a rung is tried on: 8 of the 10 the tool samples, so up to two may fail on the project's rung.
ENOUGH_ROUNDS = 8
# The two replays on the project's rung. Each lists, round by round, the indexes of the findings it found.
Reference = tuple[list[set[int]], list[set[int]]]
@dataclasses.dataclass(frozen=True)
class RungRecall:
"""A rung's replay: how many of the archived findings it found, and each 🔴 it missed that both of the project's rung's replays found, as a round and the finding's index."""
rung: Rung
found: int
missed_red: list[tuple[Round, int]]
qualifies: bool
@dataclasses.dataclass(frozen=True)
class Recall:
"""rounds, the rounds no replay on the project's rung failed on; left_out, how many more it failed on; tried, each candidate tried, highest first; lowest, the lowest that, like each above it, qualifies."""
rounds: list[Round]
reference: Reference
left_out: int
tried: list[RungRecall]
lowest: Rung | None
def candidates(ladder: Ladder, project: Rung) -> list[Rung]:
"""The rungs tried below the project's, highest first: its model at each lower effort, which the ladder leaves out, then each ladder rung below it, down to the ladder's lowest. That reaches below the lowest a worker routes to, since a pass may run below the worker's rung."""
lower_efforts = [Rung(project.model, effort) for effort in reversed(EFFORTS[:EFFORTS.index(project.effort)])]
return lower_efforts + down_from(ladder, project, ladder.rungs[0])[1:]
def count(found: list[set[int]]) -> int:
return sum(map(len, found))
def held(rounds: list[Round]) -> int:
"""How many findings rounds hold."""
return sum(len(one.findings) for one in rounds)
def mean_share(rounds: list[Round], reference: Reference) -> Fraction:
"""The mean share of rounds' findings that the project's rung's two replays found; rounds is never empty, and each holds a finding."""
return Fraction(count(reference[0]) + count(reference[1]), 2 * held(rounds))
def needed(rounds: list[Round], reference: Reference) -> int:
"""How many of rounds' findings a rung needs to find: the fewest that reach the project's rung's mean share less MARGIN. Zero or below, no rung is tried, since a rung that found none would qualify."""
return math.ceil((mean_share(rounds, reference) - MARGIN) * held(rounds))
def tried(rung: Rung, rounds: list[Round], reference: Reference, found: list[set[int]]) -> RungRecall:
"""rung's recall, from the indexes of each round's findings it found. It qualifies finding as many as needed, missing no 🔴 that both of the project's rung's replays found."""
missed_red = [(one, index) for one, first, second, got in zip(rounds, *reference, found) for index in sorted((first & second) - got) if is_red(one.findings[index])]
return RungRecall(rung, count(found), missed_red, 0 < needed(rounds, reference) <= count(found) and not missed_red)
def recall_report(reviewed: str, project: Rung, recall: Recall, enough_rounds: int) -> str:
rounds, (first, second) = recall.rounds, recall.reference
lines = [
f"Replayed {len(rounds)} {reviewed} rounds of {project.model} projects, which hold {held(rounds)} 🔴 and 🟡 findings, twice on the project's rung, {project}.",
*([f"Left out {recall.left_out} more, which a replay on the project's rung failed on; stderr says why."] if recall.left_out else []),
]
if len(rounds) < enough_rounds:
return "\n".join([*lines, f"No rung tried: fewer than {enough_rounds} rounds were replayed on the project's rung without a failure."])
if needed(rounds, recall.reference) <= 0:
return "\n".join([*lines, "No rung tried: the project's rung found too few of them to hold a rung to."])
lines.append(f"The project's rung found {count(first)} and then {count(second)} of the {held(rounds)}, {percent(mean_share(rounds, recall.reference))} on average, so a rung qualifies finding {needed(rounds, recall.reference)} or more of them, missing no 🔴 that both of its replays found.")
for one in recall.tried:
verdict = "it qualifies" if one.qualifies else "it doesn't qualify"
lines += [f"{one.rung} found {one.found}, {percent(Fraction(one.found, held(rounds)))}, and missed {len(one.missed_red) or 'no'} such 🔴: {verdict}.", *(f" {at.root} {at.at}: {at.findings[index].splitlines()[0]}" for at, index in one.missed_red)]
lowest = recall.lowest or "none below the project's rung"
lines.append(f"Lowest rung that qualifies: {lowest}")
return "\n".join(lines)
def percent(fraction: Fraction) -> str:
return f"{float(fraction):.0%}"
def recalled(chosen: list[Round], ladder: Ladder, project: Rung, replay_one: typing.Callable[[Round, Rung], set[int] | Failure | Limited], what: str, say: typing.Callable[[str], None], at_once: int, enough_rounds: int) -> Recall | Limited:
"""Replays chosen twice on the project's rung, then each candidate, highest first, on the rounds neither of those failed on, until one doesn't qualify; no candidate where fewer than enough_rounds rounds are left."""
runs: list[list[set[int] | Failure]] = []
for _ in range(2):
found: list[set[int] | Failure] | Limited = replayed_on(chosen, project, replay_one, what, say, at_once)
if isinstance(found, Limited):
return found
runs.append(found)
rounds: list[Round] = []
reference: Reference = ([], [])
for one, first, second in zip(chosen, *runs):
if not isinstance(first, Failure) and not isinstance(second, Failure):
rounds.append(one)
reference[0].append(first)
reference[1].append(second)
recalls: list[RungRecall] = []
lowest = None
for rung in candidates(ladder, project) if len(rounds) >= enough_rounds and needed(rounds, reference) > 0 else []:
on_rung: list[set[int] | Failure] | Limited = replayed_on(rounds, rung, replay_one, what, say, at_once)
if isinstance(on_rung, Limited):
return on_rung
# A failed replay on a candidate found none, so it lowers that rung's count and never lifts it to what a rung needs.
one = tried(rung, rounds, reference, [set() if isinstance(found, Failure) else found for found in on_rung])
recalls.append(one)
if not one.qualifies:
break
lowest = rung
return Recall(rounds, reference, len(chosen) - len(rounds), recalls, lowest)