122 lines
6.9 KiB
Python
122 lines
6.9 KiB
Python
"""The bar review-replay holds a rung to, and its report: docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`."""
|
|
import dataclasses
|
|
from rung_replay.command import down_from
|
|
from model_routing.ladder import EFFORTS, Ladder, Rung
|
|
from failure import Failure
|
|
from fractions import Fraction
|
|
from review_replay.rounds import is_red, Round
|
|
from claude_code.reply import Limited
|
|
import math
|
|
from rung_replay.search import replayed_on
|
|
import typing
|
|
|
|
# A rung qualifies with a share of the archived findings up to this far below the project's rung's. The project's rung found 69 to 78% of its own findings again, and its two replays of the same rounds differed by up to 4 points.
|
|
MARGIN = Fraction(1, 10)
|
|
# The fewest rounds a rung is tried on: 8 of the 10 the tool samples, so up to two may fail on the project's rung.
|
|
ENOUGH_ROUNDS = 8
|
|
# The two replays on the project's rung. Each lists, round by round, the indexes of the findings it found.
|
|
Reference = tuple[list[set[int]], list[set[int]]]
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class RungRecall:
|
|
"""A rung's replay: how many of the archived findings it found, and each 🔴 it missed that both of the project's rung's replays found, as a round and the finding's index."""
|
|
rung: Rung
|
|
found: int
|
|
missed_red: list[tuple[Round, int]]
|
|
qualifies: bool
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Recall:
|
|
"""rounds, the rounds no replay on the project's rung failed on; left_out, how many more it failed on; tried, each candidate tried, highest first; lowest, the lowest that, like each above it, qualifies."""
|
|
rounds: list[Round]
|
|
reference: Reference
|
|
left_out: int
|
|
tried: list[RungRecall]
|
|
lowest: Rung | None
|
|
|
|
|
|
def candidates(ladder: Ladder, project: Rung) -> list[Rung]:
|
|
"""The rungs tried below the project's, highest first: its model at each lower effort, which the ladder leaves out, then each ladder rung below it, down to the ladder's lowest. That reaches below the lowest a worker routes to, since a pass may run below the worker's rung."""
|
|
lower_efforts = [Rung(project.model, effort) for effort in reversed(EFFORTS[:EFFORTS.index(project.effort)])]
|
|
return lower_efforts + down_from(ladder, project, ladder.rungs[0])[1:]
|
|
|
|
|
|
def count(found: list[set[int]]) -> int:
|
|
return sum(map(len, found))
|
|
|
|
|
|
def held(rounds: list[Round]) -> int:
|
|
"""How many findings rounds hold."""
|
|
return sum(len(one.findings) for one in rounds)
|
|
|
|
|
|
def mean_share(rounds: list[Round], reference: Reference) -> Fraction:
|
|
"""The mean share of rounds' findings that the project's rung's two replays found; rounds is never empty, and each holds a finding."""
|
|
return Fraction(count(reference[0]) + count(reference[1]), 2 * held(rounds))
|
|
|
|
|
|
def needed(rounds: list[Round], reference: Reference) -> int:
|
|
"""How many of rounds' findings a rung needs to find: the fewest that reach the project's rung's mean share less MARGIN. Zero or below, no rung is tried, since a rung that found none would qualify."""
|
|
return math.ceil((mean_share(rounds, reference) - MARGIN) * held(rounds))
|
|
|
|
|
|
def tried(rung: Rung, rounds: list[Round], reference: Reference, found: list[set[int]]) -> RungRecall:
|
|
"""rung's recall, from the indexes of each round's findings it found. It qualifies finding as many as needed, missing no 🔴 that both of the project's rung's replays found."""
|
|
missed_red = [(one, index) for one, first, second, got in zip(rounds, *reference, found) for index in sorted((first & second) - got) if is_red(one.findings[index])]
|
|
return RungRecall(rung, count(found), missed_red, 0 < needed(rounds, reference) <= count(found) and not missed_red)
|
|
|
|
|
|
def recall_report(reviewed: str, project: Rung, recall: Recall, enough_rounds: int) -> str:
|
|
rounds, (first, second) = recall.rounds, recall.reference
|
|
lines = [
|
|
f"Replayed {len(rounds)} {reviewed} rounds of {project.model} projects, which hold {held(rounds)} 🔴 and 🟡 findings, twice on the project's rung, {project}.",
|
|
*([f"Left out {recall.left_out} more, which a replay on the project's rung failed on; stderr says why."] if recall.left_out else []),
|
|
]
|
|
if len(rounds) < enough_rounds:
|
|
return "\n".join([*lines, f"No rung tried: fewer than {enough_rounds} rounds were replayed on the project's rung without a failure."])
|
|
if needed(rounds, recall.reference) <= 0:
|
|
return "\n".join([*lines, "No rung tried: the project's rung found too few of them to hold a rung to."])
|
|
lines.append(f"The project's rung found {count(first)} and then {count(second)} of the {held(rounds)}, {percent(mean_share(rounds, recall.reference))} on average, so a rung qualifies finding {needed(rounds, recall.reference)} or more of them, missing no 🔴 that both of its replays found.")
|
|
for one in recall.tried:
|
|
verdict = "it qualifies" if one.qualifies else "it doesn't qualify"
|
|
lines += [f"{one.rung} found {one.found}, {percent(Fraction(one.found, held(rounds)))}, and missed {len(one.missed_red) or 'no'} such 🔴: {verdict}.", *(f" {at.root} {at.at}: {at.findings[index].splitlines()[0]}" for at, index in one.missed_red)]
|
|
lowest = recall.lowest or "none below the project's rung"
|
|
lines.append(f"Lowest rung that qualifies: {lowest}")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def percent(fraction: Fraction) -> str:
|
|
return f"{float(fraction):.0%}"
|
|
|
|
|
|
def recalled(chosen: list[Round], ladder: Ladder, project: Rung, replay_one: typing.Callable[[Round, Rung], set[int] | Failure | Limited], what: str, say: typing.Callable[[str], None], at_once: int, enough_rounds: int) -> Recall | Limited:
|
|
"""Replays chosen twice on the project's rung, then each candidate, highest first, on the rounds neither of those failed on, until one doesn't qualify; no candidate where fewer than enough_rounds rounds are left."""
|
|
runs: list[list[set[int] | Failure]] = []
|
|
for _ in range(2):
|
|
found: list[set[int] | Failure] | Limited = replayed_on(chosen, project, replay_one, what, say, at_once)
|
|
if isinstance(found, Limited):
|
|
return found
|
|
runs.append(found)
|
|
rounds: list[Round] = []
|
|
reference: Reference = ([], [])
|
|
for one, first, second in zip(chosen, *runs):
|
|
if not isinstance(first, Failure) and not isinstance(second, Failure):
|
|
rounds.append(one)
|
|
reference[0].append(first)
|
|
reference[1].append(second)
|
|
recalls: list[RungRecall] = []
|
|
lowest = None
|
|
for rung in candidates(ladder, project) if len(rounds) >= enough_rounds and needed(rounds, reference) > 0 else []:
|
|
on_rung: list[set[int] | Failure] | Limited = replayed_on(rounds, rung, replay_one, what, say, at_once)
|
|
if isinstance(on_rung, Limited):
|
|
return on_rung
|
|
# A failed replay on a candidate found none, so it lowers that rung's count and never lifts it to what a rung needs.
|
|
one = tried(rung, rounds, reference, [set() if isinstance(found, Failure) else found for found in on_rung])
|
|
recalls.append(one)
|
|
if not one.qualifies:
|
|
break
|
|
lowest = rung
|
|
return Recall(rounds, reference, len(chosen) - len(rounds), recalls, lowest)
|