Answer stability round 1: 0 🔴, 2 🟡 (0 misread); count the passes from the bugs' runs and drop change's duplicate parameters
This commit is contained in:
+12
-13
@@ -68,13 +68,12 @@ def reads_before(calls: list[ToolCall], bug_file: str) -> int | None:
|
||||
|
||||
|
||||
def table(results: list[Result], tasks: dict[int, str]) -> list[str]:
|
||||
"""A row per task and side, then one per side over every task, then the head's change against the base. Each figure is the
|
||||
median over the side's runs that have figures, and Reads over those that opened the bug's file. The change is not the
|
||||
difference of the two All rows: it is the mean of the bugs' own changes."""
|
||||
"""A row per task and side, then one per side over every task, then the head's change against the base, as `change` says. Each
|
||||
figure is the median over the side's runs that have figures, and Reads over those that opened the bug's file."""
|
||||
sides = {(task, side): [result for result in results if (result.task, result.side) == (task, side)] for task in tasks for side in SIDES}
|
||||
rows = [row(f"{task}: {tasks[task]}", side, sides[task, side]) for task in sorted(tasks) for side in SIDES]
|
||||
base, head = ([result for result in results if result.side == side] for side in SIDES)
|
||||
return [*HEADER, *rows, row("All", "base", base), row("All", "head", head), change(base, head, [(sides[task, "base"], sides[task, "head"]) for task in sorted(tasks)])]
|
||||
return [*HEADER, *rows, row("All", "base", base), row("All", "head", head), change([(sides[task, "base"], sides[task, "head"]) for task in sorted(tasks)])]
|
||||
|
||||
|
||||
def medians(results: list[Result]) -> list[float | None]:
|
||||
@@ -97,21 +96,21 @@ def row(task: str, side: str, results: list[Result]) -> str:
|
||||
return cells([task, side, passed, *("-" if median is None else number(median) for median in medians(results))])
|
||||
|
||||
|
||||
def change(base: list[Result], head: list[Result], bugs: list[tuple[list[Result], list[Result]]]) -> str:
|
||||
"""The head's change against the base: the pass count as the difference over every run, and each other figure as the mean
|
||||
over the bugs of that bug's own change, which is its head median less its base median. Tokens and wall time show it as a share
|
||||
of the base's median, turns and reads as a count. A bug whose either side lacks the figure, or whose base median is zero where
|
||||
a share is shown, doesn't count towards that figure's mean. The mean of the bugs' changes swings less than the change of
|
||||
the All rows' medians, which lands in whichever bug's runs sit in the middle of the pooled costs."""
|
||||
passed = sum(result.passed for result in head) - sum(result.passed for result in base)
|
||||
pairs = [list(zip(medians(was), medians(now))) for was, now in bugs]
|
||||
def change(bugs: list[tuple[list[Result], list[Result]]]) -> str:
|
||||
"""The head's change against the base, from each bug's base runs and head runs. The pass count is the difference over every
|
||||
run. Each other figure is the mean, over the bugs, of the bug's own change: its head median less its base median, shown
|
||||
as a share of the base median for tokens and wall time, and as a count for turns and reads. This isn't the change from the All
|
||||
base row to the All head row: those medians land in whichever bug's runs sit in the middle of the pooled costs."""
|
||||
passed = sum(result.passed for _, head in bugs for result in head) - sum(result.passed for base, _ in bugs for result in base)
|
||||
pairs = [list(zip(medians(base), medians(head))) for base, head in bugs]
|
||||
# Turns and reads are small counts, where a share misleads.
|
||||
means = [mean_change([pair[column] for pair in pairs], share=column not in (4, 6)) for column in range(7)]
|
||||
return cells(["All", "change", number(passed, signed=True), *(found or "-" for found in means)])
|
||||
|
||||
|
||||
def mean_change(pairs: list[tuple[float | None, float | None]], share: bool) -> str | None:
|
||||
"""The mean over the (base, head) medians of one figure, one pair per bug; None where no bug has both."""
|
||||
"""The mean of the bugs' changes in one figure, from each bug's (base median, head median). A bug counts only where both
|
||||
medians exist and, for a share, the base median isn't zero; None where no bug counts."""
|
||||
changes = [(now - was) * (100 / was if share else 1) for was, now in pairs if was is not None and now is not None and (was or not share)]
|
||||
if not changes:
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user