73 lines
5.3 KiB
Python
73 lines
5.3 KiB
Python
"""The experiment behind review-replay's --omit-claude-md, --bash-ceiling and --read-limit: three replays of a pass's rounds unchanged, one with the change, and the report that holds the one with it to what the others found: docs/decisions.md → `agents/: a reviewer starts without CLAUDE.md only where a replay of its pass missed no more findings than the most an unchanged replay missed`."""
|
|
import typing
|
|
|
|
from review_replay.changes import Change
|
|
from failure import Failure
|
|
from claude_code.reply import Limited
|
|
from rung_replay.search import replayed_on
|
|
from review_replay.rounds import Round
|
|
from model_routing.ladder import Rung
|
|
|
|
|
|
def compared(reviewed: str, project: Rung, chosen: list[Round], change: Change, found_on: typing.Callable[[Round, Change | None], set[int] | Failure | Limited], say: typing.Callable[[str], None], at_once: int) -> str | Limited:
|
|
"""Replays chosen three times unchanged and once with change, as found_on(round, change or None) finds, and reports the comparison of the rounds no replay failed on; the Limited where claude hit its usage limit. It tells say each round a failed replay leaves out."""
|
|
runs: list[list[set[int] | Failure]] = []
|
|
for applied in (None, None, None, change):
|
|
label = change.changed if applied else change.unchanged
|
|
|
|
def replay_one(one: Round, _: Rung, applied: Change | None = applied, label: str = label) -> set[int] | Failure | Limited:
|
|
found = found_on(one, applied)
|
|
if isinstance(found, Failure):
|
|
say(f" {one.root} {one.at} {label} failed, so the round is left out: {found.reason}")
|
|
return found
|
|
|
|
found: list[set[int] | Failure] | Limited = replayed_on(chosen, project, replay_one, f"{reviewed} rounds {label}", say, at_once)
|
|
if isinstance(found, Limited):
|
|
return found
|
|
runs.append(found)
|
|
# A failed replay found no findings, which would lower the bar where it was unchanged and count as misses where it had the change.
|
|
kept = rounds_without_failure(chosen, runs)
|
|
first, second, third, without = ([found[run] for _, found in kept] for run in range(4))
|
|
return comparison(reviewed, project, change, [one for one, _ in kept], (first, second, third), without, len(chosen) - len(kept))
|
|
|
|
|
|
def rounds_without_failure(chosen: list[Round], runs: list[list[set[int] | Failure]]) -> list[tuple[Round, list[set[int]]]]:
|
|
"""Each round of chosen that no run failed on, with what each run found in it."""
|
|
kept = []
|
|
for one, *found in zip(chosen, *runs):
|
|
finds = [each for each in found if not isinstance(each, Failure)]
|
|
if len(finds) == len(found):
|
|
kept.append((one, finds))
|
|
return kept
|
|
|
|
|
|
def comparison(reviewed: str, reference: Rung, change: Change, rounds: list[Round], with_it: tuple[list[set[int]], list[set[int]], list[set[int]]], without: list[set[int]], left_out: int) -> str:
|
|
"""What the three unchanged replays and the one with change found in rounds. Each unchanged replay missed some findings the other two found: the control, the noise of an unchanged reviewer. The one with the change is held to the findings the first two found, as the third is, and passes where it missed no more than the most an unchanged replay did. with_it holds what each of the three unchanged replays found in each round, and without what the replay with change found. left_out counts the rounds a replay failed on, which rounds leaves out."""
|
|
first, second, third = with_it
|
|
|
|
def shared(one: list[set[int]], other: list[set[int]]) -> list[set[int]]:
|
|
return [found_by_one & found_by_other for found_by_one, found_by_other in zip(one, other)]
|
|
|
|
def missing(held: list[set[int]], run: list[set[int]]) -> list[tuple[Round, int]]:
|
|
return [(found_round, index) for found_round, held_in_round, found in zip(rounds, held, run) for index in sorted(held_in_round - found)]
|
|
|
|
def count(run: list[set[int]]) -> int:
|
|
return sum(len(found) for found in run)
|
|
|
|
both = shared(first, second)
|
|
missed = missing(both, without)
|
|
control = [len(missing(shared(one, other), run)) for run, one, other in ((first, second, third), (second, first, third), (third, first, second))]
|
|
passes = len(missed) <= max(control)
|
|
return "\n".join([
|
|
f"Replayed {len(rounds)} {reviewed} rounds of {reference.model} projects, which hold {sum(len(found_round.findings) for found_round in rounds)} 🔴 and 🟡 findings, four times on {reference}: three {change.unchanged}, one {change.changed}.",
|
|
*([f"Left out {left_out} more, which a replay failed on; stderr says why."] if left_out else []),
|
|
f"{capitalized(change.unchanged)} the three replays found {count(first)}, {count(second)} and {count(third)} of them, and missed {control[0]}, {control[1]} and {control[2]} of the findings that both of the other two found.",
|
|
*([f"{capitalized(change.changed)} it found {count(without)} of them, and missed {len(missed)} of the {count(both)} findings that both of the first two found{':' if missed else '.'}"] if count(both) else [f"{capitalized(change.changed)} it found {count(without)} of them. No evidence: the first two replays {change.unchanged} found none in common, so there is no finding to hold it to."]),
|
|
*(f" {found_round.root} {found_round.at}: {found_round.findings[index].splitlines()[0]}" for found_round, index in missed),
|
|
*([f"It {'passes' if passes else 'fails'}: {len(missed)} missed is {'no more' if passes else 'more'} than the most a replay {change.unchanged} missed, {max(control)}."] if count(both) else []),
|
|
])
|
|
|
|
|
|
def capitalized(words: str) -> str:
|
|
return words[0].upper() + words[1:]
|