Answer stability round 1: 0 🔴, 2 🟡 (0 misread); count the passes from the bugs' runs and drop change's duplicate parameters

This commit is contained in:
2026-11-22 17:32:04 +01:00
parent f9f96f69ae
commit f5a691ff07
+12 -13
View File
@@ -68,13 +68,12 @@ def reads_before(calls: list[ToolCall], bug_file: str) -> int | None:
def table(results: list[Result], tasks: dict[int, str]) -> list[str]:
"""A row per task and side, then one per side over every task, then the head's change against the base. Each figure is the
median over the side's runs that have figures, and Reads over those that opened the bug's file. The change is not the
difference of the two All rows: it is the mean of the bugs' own changes."""
"""A row per task and side, then one per side over every task, then the head's change against the base, as `change` says. Each
figure is the median over the side's runs that have figures, and Reads over those that opened the bug's file."""
sides = {(task, side): [result for result in results if (result.task, result.side) == (task, side)] for task in tasks for side in SIDES}
rows = [row(f"{task}: {tasks[task]}", side, sides[task, side]) for task in sorted(tasks) for side in SIDES]
base, head = ([result for result in results if result.side == side] for side in SIDES)
return [*HEADER, *rows, row("All", "base", base), row("All", "head", head), change(base, head, [(sides[task, "base"], sides[task, "head"]) for task in sorted(tasks)])]
return [*HEADER, *rows, row("All", "base", base), row("All", "head", head), change([(sides[task, "base"], sides[task, "head"]) for task in sorted(tasks)])]
def medians(results: list[Result]) -> list[float | None]:
@@ -97,21 +96,21 @@ def row(task: str, side: str, results: list[Result]) -> str:
return cells([task, side, passed, *("-" if median is None else number(median) for median in medians(results))])
def change(base: list[Result], head: list[Result], bugs: list[tuple[list[Result], list[Result]]]) -> str:
"""The head's change against the base: the pass count as the difference over every run, and each other figure as the mean
over the bugs of that bug's own change, which is its head median less its base median. Tokens and wall time show it as a share
of the base's median, turns and reads as a count. A bug whose either side lacks the figure, or whose base median is zero where
a share is shown, doesn't count towards that figure's mean. The mean of the bugs' changes swings less than the change of
the All rows' medians, which lands in whichever bug's runs sit in the middle of the pooled costs."""
passed = sum(result.passed for result in head) - sum(result.passed for result in base)
pairs = [list(zip(medians(was), medians(now))) for was, now in bugs]
def change(bugs: list[tuple[list[Result], list[Result]]]) -> str:
"""The head's change against the base, from each bug's base runs and head runs. The pass count is the difference over every
run. Each other figure is the mean, over the bugs, of the bug's own change: its head median less its base median, shown
as a share of the base median for tokens and wall time, and as a count for turns and reads. This isn't the change from the All
base row to the All head row: those medians land in whichever bug's runs sit in the middle of the pooled costs."""
passed = sum(result.passed for _, head in bugs for result in head) - sum(result.passed for base, _ in bugs for result in base)
pairs = [list(zip(medians(base), medians(head))) for base, head in bugs]
# Turns and reads are small counts, where a share misleads.
means = [mean_change([pair[column] for pair in pairs], share=column not in (4, 6)) for column in range(7)]
return cells(["All", "change", number(passed, signed=True), *(found or "-" for found in means)])
def mean_change(pairs: list[tuple[float | None, float | None]], share: bool) -> str | None:
"""The mean over the (base, head) medians of one figure, one pair per bug; None where no bug has both."""
"""The mean of the bugs' changes in one figure, from each bug's (base median, head median). A bug counts only where both
medians exist and, for a share, the base median isn't zero; None where no bug counts."""
changes = [(now - was) * (100 / was if share else 1) for was, now in pairs if was is not None and now is not None and (was or not share)]
if not changes:
return None