From f5a691ff07873ece3c06b8a02c1f94f81dcb2ca8 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 22 Nov 2026 17:32:04 +0100 Subject: [PATCH] =?UTF-8?q?Answer=20stability=20round=201:=200=20?= =?UTF-8?q?=F0=9F=94=B4,=202=20=F0=9F=9F=A1=20(0=20misread);=20count=20the?= =?UTF-8?q?=20passes=20from=20the=20bugs'=20runs=20and=20drop=20change's?= =?UTF-8?q?=20duplicate=20parameters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/task_bench/figures.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/bin/task_bench/figures.py b/bin/task_bench/figures.py index d222553b..6eaabc1d 100644 --- a/bin/task_bench/figures.py +++ b/bin/task_bench/figures.py @@ -68,13 +68,12 @@ def reads_before(calls: list[ToolCall], bug_file: str) -> int | None: def table(results: list[Result], tasks: dict[int, str]) -> list[str]: - """A row per task and side, then one per side over every task, then the head's change against the base. Each figure is the - median over the side's runs that have figures, and Reads over those that opened the bug's file. The change is not the - difference of the two All rows: it is the mean of the bugs' own changes.""" + """A row per task and side, then one per side over every task, then the head's change against the base, as `change` says. Each + figure is the median over the side's runs that have figures, and Reads over those that opened the bug's file.""" sides = {(task, side): [result for result in results if (result.task, result.side) == (task, side)] for task in tasks for side in SIDES} rows = [row(f"{task}: {tasks[task]}", side, sides[task, side]) for task in sorted(tasks) for side in SIDES] base, head = ([result for result in results if result.side == side] for side in SIDES) - return [*HEADER, *rows, row("All", "base", base), row("All", "head", head), change(base, head, [(sides[task, "base"], sides[task, "head"]) for task in sorted(tasks)])] + return [*HEADER, *rows, row("All", "base", base), row("All", "head", head), change([(sides[task, "base"], sides[task, "head"]) for task in sorted(tasks)])] def medians(results: list[Result]) -> list[float | None]: @@ -97,21 +96,21 @@ def row(task: str, side: str, results: list[Result]) -> str: return cells([task, side, passed, *("-" if median is None else number(median) for median in medians(results))]) -def change(base: list[Result], head: list[Result], bugs: list[tuple[list[Result], list[Result]]]) -> str: - """The head's change against the base: the pass count as the difference over every run, and each other figure as the mean - over the bugs of that bug's own change, which is its head median less its base median. Tokens and wall time show it as a share - of the base's median, turns and reads as a count. A bug whose either side lacks the figure, or whose base median is zero where - a share is shown, doesn't count towards that figure's mean. The mean of the bugs' changes swings less than the change of - the All rows' medians, which lands in whichever bug's runs sit in the middle of the pooled costs.""" - passed = sum(result.passed for result in head) - sum(result.passed for result in base) - pairs = [list(zip(medians(was), medians(now))) for was, now in bugs] +def change(bugs: list[tuple[list[Result], list[Result]]]) -> str: + """The head's change against the base, from each bug's base runs and head runs. The pass count is the difference over every + run. Each other figure is the mean, over the bugs, of the bug's own change: its head median less its base median, shown + as a share of the base median for tokens and wall time, and as a count for turns and reads. This isn't the change from the All + base row to the All head row: those medians land in whichever bug's runs sit in the middle of the pooled costs.""" + passed = sum(result.passed for _, head in bugs for result in head) - sum(result.passed for base, _ in bugs for result in base) + pairs = [list(zip(medians(base), medians(head))) for base, head in bugs] # Turns and reads are small counts, where a share misleads. means = [mean_change([pair[column] for pair in pairs], share=column not in (4, 6)) for column in range(7)] return cells(["All", "change", number(passed, signed=True), *(found or "-" for found in means)]) def mean_change(pairs: list[tuple[float | None, float | None]], share: bool) -> str | None: - """The mean over the (base, head) medians of one figure, one pair per bug; None where no bug has both.""" + """The mean of the bugs' changes in one figure, from each bug's (base median, head median). A bug counts only where both + medians exist and, for a share, the base median isn't zero; None where no bug counts.""" changes = [(now - was) * (100 / was if share else 1) for was, now in pairs if was is not None and now is not None and (was or not share)] if not changes: return None