From 993295b486402db040478a6b8be62fd6d99a0ef7 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 22 Nov 2026 16:50:51 +0100 Subject: [PATCH 01/10] Stop the bench with a Failure on a run that ended on an api_error without a status --- bin/task_bench/figures.py | 2 +- bin/task_bench/test_figures.py | 4 ++++ docs/reading-cost.md | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/bin/task_bench/figures.py b/bin/task_bench/figures.py index 59875e22..6071b3cf 100644 --- a/bin/task_bench/figures.py +++ b/bin/task_bench/figures.py @@ -48,7 +48,7 @@ def figures(stream: bytes, bug_file: str) -> Figures | None | Failure: result = next((event for event in reversed(events) if event.get("type") == "result"), None) if result is None: return None - if result.get("api_error_status"): + if result.get("api_error_status") or result.get("terminal_reason") == "api_error": return Failure(f"claude could not use the API: {result.get('result')}") # {"type": "system", "subtype": "init", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330", ...} init = next((event for event in events if event.get("subtype") == "init"), {}) diff --git a/bin/task_bench/test_figures.py b/bin/task_bench/test_figures.py index f46faff4..c81c2781 100644 --- a/bin/task_bench/test_figures.py +++ b/bin/task_bench/test_figures.py @@ -69,6 +69,10 @@ class FiguresTest(unittest.TestCase): refused = {**RESULT, "is_error": True, "api_error_status": 401, "result": "Failed to authenticate. API Error: 401 OAuth access token is invalid."} self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: Failed to authenticate. API Error: 401 OAuth access token is invalid.")) + def test_a_run_that_ended_on_an_api_error_without_a_status_is_a_failure_that_quotes_claude(self): + refused = {**RESULT, "is_error": True, "terminal_reason": "api_error", "result": "CLAUDE_CODE_OAUTH_TOKEN is not a valid header value."} + self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: CLAUDE_CODE_OAUTH_TOKEN is not a valid header value.")) + def run(task: int, side: str, number: int, passed: bool, input_: int = 100, turns: int = 10, seconds: float = 60, reads: int | None = 3) -> Result: return Result(task, side, number, passed, Figures("2.1.330", "claude-haiku-5-5", Tokens(input=input_, write_5m=20, write_1h=980, read=5000, output=400), turns, seconds, reads)) diff --git a/docs/reading-cost.md b/docs/reading-cost.md index d3aa0268..556f696c 100644 --- a/docs/reading-cost.md +++ b/docs/reading-cost.md @@ -3384,3 +3384,20 @@ Read most (opens and searches): - `bin/reading_cost/measure.py` 2 - `bin/review_rules/review_folder.py` 2 - `bin/task_bench/figures.py` 2 + +## Reading cost, chunk 666aabdd77a8, finished 2026-11-22T15:50:09Z + +283046 new input tokens before the first commit. + +Read most (opens and searches): + +- `docs/decisions.md` 14 +- `bin/task_bench/main.py` 9 +- `bin/task_bench/figures.py` 4 +- `bin/task_bench/plant.py` 4 +- `bin/task_bench/test_figures.py` 4 +- `bin/task_bench/runs.py` 3 +- `CHANGELOG.md` 2 +- `README.md` 2 +- `docs/comprehension-history.md` 2 +- `docs/score-anchors.md` 2 From e38b6c344b7a618545166920c4bf334b96de3314 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 22 Nov 2026 16:52:54 +0100 Subject: [PATCH 02/10] Print the mean of the bugs' own changes as task-bench's All change --- bin/task_bench/figures.py | 33 +++++++++++++++++++++++---------- bin/task_bench/main.py | 9 +++++---- bin/task_bench/test_figures.py | 19 +++++++++++++++++++ docs/decisions.md | 8 ++++---- docs/reading-cost.md | 17 +++++++++++++++++ 5 files changed, 68 insertions(+), 18 deletions(-) diff --git a/bin/task_bench/figures.py b/bin/task_bench/figures.py index 59875e22..4a2ac957 100644 --- a/bin/task_bench/figures.py +++ b/bin/task_bench/figures.py @@ -69,10 +69,12 @@ def reads_before(calls: list[ToolCall], bug_file: str) -> int | None: def table(results: list[Result], tasks: dict[int, str]) -> list[str]: """A row per task and side, then one per side over every task, then the head's change against the base. Each figure is the - median over the side's runs that have figures, and Reads over those that opened the bug's file.""" - rows = [row(f"{task}: {tasks[task]}", side, [result for result in results if (result.task, result.side) == (task, side)]) for task in sorted(tasks) for side in SIDES] + median over the side's runs that have figures, and Reads over those that opened the bug's file. The change is not the + difference of the two All rows: it is the mean of the bugs' own changes.""" + sides = {(task, side): [result for result in results if (result.task, result.side) == (task, side)] for task in tasks for side in SIDES} + rows = [row(f"{task}: {tasks[task]}", side, sides[task, side]) for task in sorted(tasks) for side in SIDES] base, head = ([result for result in results if result.side == side] for side in SIDES) - return [*HEADER, *rows, row("All", "base", base), row("All", "head", head), change(base, head)] + return [*HEADER, *rows, row("All", "base", base), row("All", "head", head), change(base, head, [(sides[task, "base"], sides[task, "head"]) for task in sorted(tasks)])] def medians(results: list[Result]) -> list[float | None]: @@ -95,15 +97,26 @@ def row(task: str, side: str, results: list[Result]) -> str: return cells([task, side, passed, *("-" if median is None else number(median) for median in medians(results))]) -def change(base: list[Result], head: list[Result]) -> str: - """The head's row less the base's: a count as the difference, and tokens and wall time as that difference's share of the base's.""" +def change(base: list[Result], head: list[Result], bugs: list[tuple[list[Result], list[Result]]]) -> str: + """The head's change against the base: the pass count as the difference over every run, and each other figure as the mean + over the bugs of that bug's own change, which is its head median less its base median. Tokens and wall time show it as a share + of the base's median, turns and reads as a count. A bug whose either side lacks the figure, or whose base median is zero where + a share is shown, doesn't count towards that figure's mean. The mean of the bugs' changes swings less than the change of + the All rows' medians, which lands in whichever bug's runs sit in the middle of the pooled costs.""" passed = sum(result.passed for result in head) - sum(result.passed for result in base) - pairs = list(zip(medians(base), medians(head))) - shares = [None if was is None or now is None or not was else f"{number(round(100 * (now - was) / was), signed=True)} %" for was, now in pairs] - counts = [None if was is None or now is None else number(now - was, signed=True) for was, now in pairs] + pairs = [list(zip(medians(was), medians(now))) for was, now in bugs] # Turns and reads are small counts, where a share misleads. - mixed = [*shares[:4], counts[4], shares[5], counts[6]] - return cells(["All", "change", number(passed, signed=True), *(found or "-" for found in mixed)]) + means = [mean_change(column, share=index not in (4, 6)) for index, column in enumerate(zip(*pairs))] if pairs else [None] * 7 + return cells(["All", "change", number(passed, signed=True), *(found or "-" for found in means)]) + + +def mean_change(pairs: tuple[tuple[float | None, float | None], ...], share: bool) -> str | None: + """The mean over the (base, head) medians of one figure, one pair per bug; None where no bug has both.""" + changes = [(now - was) * (100 / was if share else 1) for was, now in pairs if was is not None and now is not None and (was or not share)] + if not changes: + return None + mean = statistics.mean(changes) + return f"{number(round(mean), signed=True)} %" if share else number(mean, signed=True) def number(value: float, signed: bool = False) -> str: diff --git a/bin/task_bench/main.py b/bin/task_bench/main.py index 0cb3e4f4..c6e1c591 100644 --- a/bin/task_bench/main.py +++ b/bin/task_bench/main.py @@ -36,7 +36,8 @@ example: It prints the base and the head, then a table: a row per bug and side with how many fixes passed and the medians, over the runs that finished, of the input, cache-write, cache-read and output tokens, the turns, the wall time, and the reads and searches before the run first opened the file the bug is in. The median of the reads leaves out a run that never opened that file. Then come a row -per side over every bug, and the head's change against the base. The last line names the bench's folder: `task-bench` in the +per side over every bug, and the head's change against the base: the pass count's difference, and for each other column the mean over +the bugs of that bug's own change, which isn't the difference of the two rows above it. The last line names the bench's folder: `task-bench` in the chunk's review folder. That folder holds each run's result and stream, and digest.md, which says why runs failed in a few kilobytes per bug: @@ -46,9 +47,9 @@ That folder holds each run's result and stream, and digest.md, which says why ru passing run with the median tokens. A trace has a line per tool call, such as `Read pick.go:120-180` or `Grep "readName" → 4 hits`, and the first 3 lines of what Haiku wrote between calls. -Read a change against the noise. With the defaults, a bench of a commit against itself shows, in 19 cases of 20, an All row change -under 31% in output tokens, 50% in cache-read tokens and 21% in wall time. The mean of the bugs' own changes, which you work out -from the bug rows, stays under 12%, 16% and 10%. On `fejkdata`'s bugs every run passed, so the pass rate's noise is unmeasured. +Read a change against the noise. With the defaults, a bench of a commit against itself shows, in 19 cases of 20, a change under +12% in output tokens, 16% in cache-read tokens and 10% in wall time. On `fejkdata`'s bugs every run passed, so the pass rate's noise +is unmeasured. `docs/decisions.md` → `task-bench: a bench takes` has the figures. `task-bench gather`, run in the same checkout, turns the digest into plan items. It starts a claude on the project's rung, the diff --git a/bin/task_bench/test_figures.py b/bin/task_bench/test_figures.py index f46faff4..453a64b5 100644 --- a/bin/task_bench/test_figures.py +++ b/bin/task_bench/test_figures.py @@ -102,6 +102,25 @@ class TableTest(unittest.TestCase): results = [run(1, "base", 1, True, input_=0, turns=10, seconds=60, reads=0), run(1, "head", 1, False, input_=50, turns=12, seconds=120, reads=2)] self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[-1], "| All | change | -1 | - | 0 % | 0 % | 0 % | +2 | +100 % | +2 |") + def test_the_change_is_the_mean_of_the_bugs_own_changes_not_the_change_of_the_pooled_medians(self): + results = [ + run(1, "base", 1, True, input_=100, turns=10, seconds=60, reads=3), run(1, "head", 1, True, input_=200, turns=14, seconds=90, reads=5), + run(2, "base", 1, True, input_=1000, turns=20, seconds=60, reads=7), run(2, "head", 1, True, input_=1000, turns=20, seconds=60, reads=7), + ] + rows = table(results, {1: "calc.go:4:7 ARITHMETIC_BASE", 2: "pick.go:9:3 ARITHMETIC_BASE"}) + self.assertEqual(rows[-3:], [ + "| All | base | 2/2 | 550 | 1000 | 5000 | 400 | 15 | 60 | 5 |", + "| All | head | 2/2 | 600 | 1000 | 5000 | 400 | 17 | 75 | 6 |", + "| All | change | 0 | +50 % | 0 % | 0 % | 0 % | +2 | +25 % | +1 |", + ]) + + def test_a_bug_whose_base_measured_zero_is_left_out_of_the_mean_share(self): + results = [ + run(1, "base", 1, True, input_=0), run(1, "head", 1, True, input_=50), + run(2, "base", 1, True, input_=100), run(2, "head", 1, True, input_=150), + ] + self.assertEqual(table(results, {1: "a.go:1:1 ARITHMETIC_BASE", 2: "b.go:1:1 ARITHMETIC_BASE"})[-1].split(" | ")[3], "+50 %") + def test_a_run_without_figures_counts_as_not_passed_and_a_side_without_any_shows_a_dash(self): results = [Result(1, "base", 1, False, None), run(1, "head", 1, True)] self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[2:], [ diff --git a/docs/decisions.md b/docs/decisions.md index eb24af45..e070920a 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -825,14 +825,14 @@ Valid while Claude Code loads `~/.claude/rules/*.md` in every project, a path-sc **Defaults stay 5 bugs and 5 runs.** A sixth run per side cuts the spread of the mean of the bugs' cache-read changes from 16% to 13% for 10 more runs, about 35 minutes. -**A chunk-sized change doesn't rise above the noise.** PR #490 moved the medians by 5% to 9%. The spread at 5 × 6, the largest count measured, is 11% to 13%. It falls with the square root of the runs, so seeing a 9% change takes about 13 runs per side and bug: 130 runs, 8 hours and 70M cache-read tokens. No chunk waits that long. The bench shows a change of more than about 15% in tokens, if you work out the mean of the bugs' own changes from its bug rows. The All row it prints needs more than 31% in output tokens and 50% in cache-read tokens. What to try is filed as items 514 (print the mean of the bugs' changes, which has a third to half of the All row's spread in tokens), 515 (plant bugs Haiku fails at the base, so the pass rate moves) and 516 (measure the spread of the reads and the turns, which are counts, not token sums). +**A chunk-sized change doesn't rise above the noise.** PR #490 moved the medians by 5% to 9%. The spread at 5 × 6, the largest count measured, is 11% to 13%. It falls with the square root of the runs, so seeing a 9% change takes about 13 runs per side and bug: 130 runs, 8 hours and 70M cache-read tokens. No chunk waits that long. The bench shows a change of more than about 15% in tokens. Its All change is the mean of the bugs' own changes (item 514); the change of the pooled medians, which it printed before, needed more than 31% in output tokens and 50% in cache-read tokens. What to try is filed as items 515 (plant bugs Haiku fails at the base, so the pass rate moves) and 516 (measure the spread of the reads and the turns, which are counts, not token sums). **Pilot first.** A new setup, or a project the bench hasn't run on, runs 2 bugs × 2 runs before a full bench (lilleman, 2026-11-21). A pilot takes about 40 minutes. The first pilot of this calibration passed no run, because the API refused a token that held a line break. It showed that within 15 minutes, where a full bench would have planted for 25 minutes first and printed the same. Measured on `fejkdata` (Go, 5 bugs planted in the files `larv-review places` names, seed 1, Haiku at low effort), 2026-11-21: -- **Same commit at both sides** (head `c7dd7e15`, 5 bugs × 6 runs, after a pilot of 2 bugs × 2 runs): all 68 runs passed, as did the 30 counted chunk runs below, 98 in all. So the pass rate's spread is zero and it can't show a change of any size, though it is unmeasured on harder bugs. The All row's change was -5% in output tokens, -12% in cache-read tokens and -15% in wall time. -- **Spread by bugs × runs.** Each bug's 12 runs were split at random into two sides 4,000 times. Each cell is the 95th percentile of the change's size, in percent, first for the All row's change as the bench prints it, then for the mean of the bugs' own changes: +- **Same commit at both sides** (head `c7dd7e15`, 5 bugs × 6 runs, after a pilot of 2 bugs × 2 runs): all 68 runs passed, as did the 30 counted chunk runs below, 98 in all. So the pass rate's spread is zero and it can't show a change of any size, though it is unmeasured on harder bugs. The pooled change was -5% in output tokens, -12% in cache-read tokens and -15% in wall time. +- **Spread by bugs × runs.** Each bug's 12 runs were split at random into two sides 4,000 times. Each cell is the 95th percentile of the change's size, in percent, first for the change of the All rows' pooled medians, which the bench printed before item 514, then for the mean of the bugs' own changes, which it prints now: | Bugs × runs | Output tokens | Cache-read tokens | Wall time | |---|---|---|---| @@ -842,7 +842,7 @@ Measured on `fejkdata` (Go, 5 bugs planted in the files `larv-review places` nam | 5 × 6 | 28 / 11 | 43 / 13 | 19 / 8.5 | | 3 × 6 | 28 / 14 | 40 / 17 | 16 / 12 | - The All row pools every run, so its median lands in whichever bug's runs sit in the middle of the pooled costs, and a bug's cost differs from another's by up to 8×. In tokens, the mean of the bugs' changes has a third to half of the All row's spread. In wall time it gains only from 5 × 5. The 86 at 5 × 3 wasn't traced. For the mean of the bugs' token changes, 5 × 4 and 3 × 6 give the same spread (14 and 14, 18 and 17) for 40 and 36 runs. + The All rows pool every run, so their median lands in whichever bug's runs sit in the middle of the pooled costs, and a bug's cost differs from another's by up to 8×. In tokens, the mean of the bugs' changes has a third to half of the pooled change's spread. The bench keeps the pooled All rows per side, which cost nothing, and prints the mean as the All change. In wall time it gains only from 5 × 5. The 86 at 5 × 3 wasn't traced. For the mean of the bugs' token changes, 5 × 4 and 3 × 6 give the same spread (14 and 14, 18 and 17) for 40 and 36 runs. - **One finished chunk**: `fejkdata`'s PR #490, base `33b7fa80`, head `5b9c2a14`, which the comprehension panel ruled better on Navigation, Locality and Shape and the same on Self-sufficiency. The bench planted 5 bugs and ran 38 of 50 runs before the machine restarted. The first 3 runs per bug and side, 30 runs, are the ones counted, and all passed. The mean of the bugs' changes was +4.6% in output tokens, +8.9% in cache-read tokens and +3.8% in turns, with the bugs' own changes in cache-read tokens from -41% to +47%. The two token figures sit inside the spread at 5 × 3, 17% and 22%; the spread of the turns wasn't measured. The mean of the wall times' changes, +105%, lies above the 86% of that row, from one bug whose head runs took 6.8 times as long at the median. Nobody read its streams to find out why. The head cost slightly more tokens where the panel finds it easier to read, and the wall time can't be judged, so the bench neither confirmed nor contradicted the panel. - **Cost**: 5 bugs × 6 runs took 3.9 hours and 34M cache-read, 1.2M cache-write and 0.3M output tokens; a run takes about 3.5 minutes, and planting 16 to 25 minutes. A 5 × 5 bench is about 3.3 hours and 28M cache-read tokens. Gremlins ran out of 8 GB at the base of PR #490, so the help's example takes 16 GB. diff --git a/docs/reading-cost.md b/docs/reading-cost.md index d3aa0268..556f696c 100644 --- a/docs/reading-cost.md +++ b/docs/reading-cost.md @@ -3384,3 +3384,20 @@ Read most (opens and searches): - `bin/reading_cost/measure.py` 2 - `bin/review_rules/review_folder.py` 2 - `bin/task_bench/figures.py` 2 + +## Reading cost, chunk 666aabdd77a8, finished 2026-11-22T15:50:09Z + +283046 new input tokens before the first commit. + +Read most (opens and searches): + +- `docs/decisions.md` 14 +- `bin/task_bench/main.py` 9 +- `bin/task_bench/figures.py` 4 +- `bin/task_bench/plant.py` 4 +- `bin/task_bench/test_figures.py` 4 +- `bin/task_bench/runs.py` 3 +- `CHANGELOG.md` 2 +- `README.md` 2 +- `docs/comprehension-history.md` 2 +- `docs/score-anchors.md` 2 From 5b792514a4e553f81f977a81a88ffd8a8c000c40 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 22 Nov 2026 17:09:21 +0100 Subject: [PATCH 03/10] =?UTF-8?q?Answer=20security=20round=201:=200=20?= =?UTF-8?q?=F0=9F=94=B4,=200=20=F0=9F=9F=A1=20(0=20misread);=20nothing=20f?= =?UTF-8?q?ound?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 72c481696d5aa2cf799ad3a947d63b052686cb34 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 22 Nov 2026 17:09:21 +0100 Subject: [PATCH 04/10] =?UTF-8?q?Answer=20stability=20round=201:=200=20?= =?UTF-8?q?=F0=9F=94=B4,=200=20=F0=9F=9F=A1=20(0=20misread);=20approved=20?= =?UTF-8?q?without=20findings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 0b6cc71520e75d7a23fe272ce7c024909d17cf08 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 22 Nov 2026 17:09:32 +0100 Subject: [PATCH 05/10] Finish item 513 in the plan --- CHANGELOG.md | 1 + todo.md | 12 ------------ 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90d16d4f..abd905a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- `task-bench` now stops with a Failure on a run whose final result says `terminal_reason: "api_error"`, even where it holds no `api_error_status`, so a token the API refuses no longer lets the bench run on for hours with empty runs. - `task-bench`'s `--help` now says how far a bench of a commit against itself swings, in 19 cases of 20, with the default 5 bugs and 5 runs, and that a Gremlins run on a large project needs 16 GB of memory. ## 0.15.0 diff --git a/todo.md b/todo.md index f1d78ffe..1228a6f6 100644 --- a/todo.md +++ b/todo.md @@ -14,7 +14,6 @@ |---|---|---|---|---|---|---|---|---|---|---|---| | 514 | 0.15.1 | | | **Print the mean of the bugs' own head/base changes as the table's All change, in place of the change of the medians pooled over every run.** | 2 | 2 | 1 | 3 | 3 | 1.1 by 3 | 1.1 | | 516 | 0.15.1 | | | **Record in a decision the 95th-percentile spread, at 5 bugs × 5 runs, of the reads before the first open of the bug's file and of the turns, beside the tokens'.** | 1 | 2 | 1 | 2 | 2 | 1.1 by 2 | 0.8 | -| 513 | 0.15.1 | | | **Stop the bench with a Failure on a run whose final `result` event says `terminal_reason: "api_error"`, even where its `api_error_status` is empty.** | 1 | 1 | 1 | 1 | 2 | 1.4 by 2 | 0.8 | | 515 | 0.15.1 | | | **Plant bugs that Haiku fails at the base, so the pass rate's spread over two benches of one commit is above zero, and record it in a decision.** | 3 | 2 | 1 | 4 | 5 | 1.1 by 3 | 0.7 | | 509 | 0.16.0 | | owner | **Run `task-bench` beside the scoring run in larv-review comprehension check, and record its figures in the comprehension history beside the panel scores.** | 3 | 2 | 3 | 6 | 5 | code health by 5, 1.1 by 3 | 5.6 | | 504 | 0.16.0 | | owner | **Bench new features in `task-bench`, each a spec for Haiku with acceptance tests it never sees and a reference patch that proves it can be built, kept in a `bench-tasks` folder the project config names, starting with six for fejkdata.** | 3 | 2 | 1 | 4 | 6 | 1.1 by 4, code health by 3 | 2.6 | @@ -130,17 +129,6 @@ Touches: - `docs/decisions.md`: `## task-bench: a bench takes 5 bugs and 5 runs per side, after a 2 × 2 pilot on a new setup, and at no affordable count does a chunk-sized change rise above the noise` -### 513. Stop the bench with a Failure on a run whose final `result` event says `terminal_reason: "api_error"`, even where its `api_error_status` is empty. - -**Stop the bench with a Failure on a run whose final `result` event says `terminal_reason: "api_error"`, even where its `api_error_status` is empty.** - -Today `figures()` checks only `api_error_status`. During item 503's calibration a token the API refused over a line break ended runs with `terminal_reason: "api_error"`, a message naming `CLAUDE_CODE_OAUTH_TOKEN`, and no status, so each counted as a finished, not-passed run with 0 tokens, and the bench went on for hours. `task-bench --help` promises that a run the API refused stops the bench. Write the test first: a stream whose result holds that terminal reason and no status gives a Failure that quotes claude's message. - -Touches: - -- `bin/task_bench/figures.py`: `figures` (small) -- `bin/task_bench/test_figures.py`: `FiguresTest` - ### 515. Plant bugs that Haiku fails at the base, so the pass rate's spread over two benches of one commit is above zero, and record it in a decision. **Plant bugs that Haiku fails at the base, so the pass rate's spread over two benches of one commit is above zero, and record it in a decision.** From f9f96f69ae3e20bafacc1e9c8e33070b66768bbb Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 22 Nov 2026 17:10:03 +0100 Subject: [PATCH 06/10] Build each column's mean change without a no-bugs branch --- bin/task_bench/figures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/task_bench/figures.py b/bin/task_bench/figures.py index 4a2ac957..d222553b 100644 --- a/bin/task_bench/figures.py +++ b/bin/task_bench/figures.py @@ -106,11 +106,11 @@ def change(base: list[Result], head: list[Result], bugs: list[tuple[list[Result] passed = sum(result.passed for result in head) - sum(result.passed for result in base) pairs = [list(zip(medians(was), medians(now))) for was, now in bugs] # Turns and reads are small counts, where a share misleads. - means = [mean_change(column, share=index not in (4, 6)) for index, column in enumerate(zip(*pairs))] if pairs else [None] * 7 + means = [mean_change([pair[column] for pair in pairs], share=column not in (4, 6)) for column in range(7)] return cells(["All", "change", number(passed, signed=True), *(found or "-" for found in means)]) -def mean_change(pairs: tuple[tuple[float | None, float | None], ...], share: bool) -> str | None: +def mean_change(pairs: list[tuple[float | None, float | None]], share: bool) -> str | None: """The mean over the (base, head) medians of one figure, one pair per bug; None where no bug has both.""" changes = [(now - was) * (100 / was if share else 1) for was, now in pairs if was is not None and now is not None and (was or not share)] if not changes: From f5a691ff07873ece3c06b8a02c1f94f81dcb2ca8 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 22 Nov 2026 17:32:04 +0100 Subject: [PATCH 07/10] =?UTF-8?q?Answer=20stability=20round=201:=200=20?= =?UTF-8?q?=F0=9F=94=B4,=202=20=F0=9F=9F=A1=20(0=20misread);=20count=20the?= =?UTF-8?q?=20passes=20from=20the=20bugs'=20runs=20and=20drop=20change's?= =?UTF-8?q?=20duplicate=20parameters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/task_bench/figures.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/bin/task_bench/figures.py b/bin/task_bench/figures.py index d222553b..6eaabc1d 100644 --- a/bin/task_bench/figures.py +++ b/bin/task_bench/figures.py @@ -68,13 +68,12 @@ def reads_before(calls: list[ToolCall], bug_file: str) -> int | None: def table(results: list[Result], tasks: dict[int, str]) -> list[str]: - """A row per task and side, then one per side over every task, then the head's change against the base. Each figure is the - median over the side's runs that have figures, and Reads over those that opened the bug's file. The change is not the - difference of the two All rows: it is the mean of the bugs' own changes.""" + """A row per task and side, then one per side over every task, then the head's change against the base, as `change` says. Each + figure is the median over the side's runs that have figures, and Reads over those that opened the bug's file.""" sides = {(task, side): [result for result in results if (result.task, result.side) == (task, side)] for task in tasks for side in SIDES} rows = [row(f"{task}: {tasks[task]}", side, sides[task, side]) for task in sorted(tasks) for side in SIDES] base, head = ([result for result in results if result.side == side] for side in SIDES) - return [*HEADER, *rows, row("All", "base", base), row("All", "head", head), change(base, head, [(sides[task, "base"], sides[task, "head"]) for task in sorted(tasks)])] + return [*HEADER, *rows, row("All", "base", base), row("All", "head", head), change([(sides[task, "base"], sides[task, "head"]) for task in sorted(tasks)])] def medians(results: list[Result]) -> list[float | None]: @@ -97,21 +96,21 @@ def row(task: str, side: str, results: list[Result]) -> str: return cells([task, side, passed, *("-" if median is None else number(median) for median in medians(results))]) -def change(base: list[Result], head: list[Result], bugs: list[tuple[list[Result], list[Result]]]) -> str: - """The head's change against the base: the pass count as the difference over every run, and each other figure as the mean - over the bugs of that bug's own change, which is its head median less its base median. Tokens and wall time show it as a share - of the base's median, turns and reads as a count. A bug whose either side lacks the figure, or whose base median is zero where - a share is shown, doesn't count towards that figure's mean. The mean of the bugs' changes swings less than the change of - the All rows' medians, which lands in whichever bug's runs sit in the middle of the pooled costs.""" - passed = sum(result.passed for result in head) - sum(result.passed for result in base) - pairs = [list(zip(medians(was), medians(now))) for was, now in bugs] +def change(bugs: list[tuple[list[Result], list[Result]]]) -> str: + """The head's change against the base, from each bug's base runs and head runs. The pass count is the difference over every + run. Each other figure is the mean, over the bugs, of the bug's own change: its head median less its base median, shown + as a share of the base median for tokens and wall time, and as a count for turns and reads. This isn't the change from the All + base row to the All head row: those medians land in whichever bug's runs sit in the middle of the pooled costs.""" + passed = sum(result.passed for _, head in bugs for result in head) - sum(result.passed for base, _ in bugs for result in base) + pairs = [list(zip(medians(base), medians(head))) for base, head in bugs] # Turns and reads are small counts, where a share misleads. means = [mean_change([pair[column] for pair in pairs], share=column not in (4, 6)) for column in range(7)] return cells(["All", "change", number(passed, signed=True), *(found or "-" for found in means)]) def mean_change(pairs: list[tuple[float | None, float | None]], share: bool) -> str | None: - """The mean over the (base, head) medians of one figure, one pair per bug; None where no bug has both.""" + """The mean of the bugs' changes in one figure, from each bug's (base median, head median). A bug counts only where both + medians exist and, for a share, the base median isn't zero; None where no bug counts.""" changes = [(now - was) * (100 / was if share else 1) for was, now in pairs if was is not None and now is not None and (was or not share)] if not changes: return None From 4612f715928b957686669729fe7ed94676e32a90 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 22 Nov 2026 17:32:04 +0100 Subject: [PATCH 08/10] =?UTF-8?q?Answer=20product-owner=20round=201:=200?= =?UTF-8?q?=20=F0=9F=94=B4,=200=20=F0=9F=9F=A1=20(0=20misread);=20the=20he?= =?UTF-8?q?lp=20calls=20the=20noise=20figures=20an=20All=20change,=20and?= =?UTF-8?q?=20the=20changelog=20says=20the=20row's=20meaning=20changed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/task_bench/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/task_bench/main.py b/bin/task_bench/main.py index c6e1c591..21d4c454 100644 --- a/bin/task_bench/main.py +++ b/bin/task_bench/main.py @@ -36,8 +36,8 @@ example: It prints the base and the head, then a table: a row per bug and side with how many fixes passed and the medians, over the runs that finished, of the input, cache-write, cache-read and output tokens, the turns, the wall time, and the reads and searches before the run first opened the file the bug is in. The median of the reads leaves out a run that never opened that file. Then come a row -per side over every bug, and the head's change against the base: the pass count's difference, and for each other column the mean over -the bugs of that bug's own change, which isn't the difference of the two rows above it. The last line names the bench's folder: `task-bench` in the +per side over every bug, and the head's change against the base. That last row holds the pass count's difference and, for each other +column, the mean over the bugs of each bug's own change, so it isn't the difference of the two rows above it. The last line names the bench's folder: `task-bench` in the chunk's review folder. That folder holds each run's result and stream, and digest.md, which says why runs failed in a few kilobytes per bug: @@ -47,9 +47,9 @@ That folder holds each run's result and stream, and digest.md, which says why ru passing run with the median tokens. A trace has a line per tool call, such as `Read pick.go:120-180` or `Grep "readName" → 4 hits`, and the first 3 lines of what Haiku wrote between calls. -Read a change against the noise. With the defaults, a bench of a commit against itself shows, in 19 cases of 20, a change under -12% in output tokens, 16% in cache-read tokens and 10% in wall time. On `fejkdata`'s bugs every run passed, so the pass rate's noise -is unmeasured. +Read a change against the noise. Measured on `fejkdata`, a Go project, with the defaults, a bench of a commit against itself shows, +in 19 cases of 20, an All change under 12% in output tokens, 16% in cache-read tokens and 10% in wall time. On its bugs every run +passed, so the pass rate's noise is unmeasured. `docs/decisions.md` → `task-bench: a bench takes` has the figures. `task-bench gather`, run in the same checkout, turns the digest into plan items. It starts a claude on the project's rung, the From b65710dca7ff003d3962363471eac8b8f4982a0c Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 22 Nov 2026 17:32:04 +0100 Subject: [PATCH 09/10] =?UTF-8?q?Answer=20prose=20round=201:=200=20?= =?UTF-8?q?=F0=9F=94=B4,=209=20=F0=9F=9F=A1=20(0=20misread);=20reword=20th?= =?UTF-8?q?e=20docstrings,=20help=20and=20decision,=20and=20item=20516's?= =?UTF-8?q?=20detail?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/decisions.md | 4 ++-- todo.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/decisions.md b/docs/decisions.md index e070920a..6dc91432 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -825,7 +825,7 @@ Valid while Claude Code loads `~/.claude/rules/*.md` in every project, a path-sc **Defaults stay 5 bugs and 5 runs.** A sixth run per side cuts the spread of the mean of the bugs' cache-read changes from 16% to 13% for 10 more runs, about 35 minutes. -**A chunk-sized change doesn't rise above the noise.** PR #490 moved the medians by 5% to 9%. The spread at 5 × 6, the largest count measured, is 11% to 13%. It falls with the square root of the runs, so seeing a 9% change takes about 13 runs per side and bug: 130 runs, 8 hours and 70M cache-read tokens. No chunk waits that long. The bench shows a change of more than about 15% in tokens. Its All change is the mean of the bugs' own changes (item 514); the change of the pooled medians, which it printed before, needed more than 31% in output tokens and 50% in cache-read tokens. What to try is filed as items 515 (plant bugs Haiku fails at the base, so the pass rate moves) and 516 (measure the spread of the reads and the turns, which are counts, not token sums). +**A chunk-sized change doesn't rise above the noise.** PR #490 moved the medians by 5% to 9%. The spread at 5 × 6, the largest count measured, is 11% to 13%. It falls with the square root of the runs, so seeing a 9% change takes about 13 runs per side and bug: 130 runs, 8 hours and 70M cache-read tokens. No chunk waits that long. The bench shows a change of more than about 15% in tokens, since its All change is the mean of the bugs' own changes (item 514). Before that, the change of the pooled medians needed more than 31% in output tokens and 50% in cache-read tokens. What to try is filed as items 515 (plant bugs Haiku fails at the base, so the pass rate moves) and 516 (measure the spread of the reads and the turns, which are counts, not token sums). **Pilot first.** A new setup, or a project the bench hasn't run on, runs 2 bugs × 2 runs before a full bench (lilleman, 2026-11-21). A pilot takes about 40 minutes. The first pilot of this calibration passed no run, because the API refused a token that held a line break. It showed that within 15 minutes, where a full bench would have planted for 25 minutes first and printed the same. @@ -842,7 +842,7 @@ Measured on `fejkdata` (Go, 5 bugs planted in the files `larv-review places` nam | 5 × 6 | 28 / 11 | 43 / 13 | 19 / 8.5 | | 3 × 6 | 28 / 14 | 40 / 17 | 16 / 12 | - The All rows pool every run, so their median lands in whichever bug's runs sit in the middle of the pooled costs, and a bug's cost differs from another's by up to 8×. In tokens, the mean of the bugs' changes has a third to half of the pooled change's spread. The bench keeps the pooled All rows per side, which cost nothing, and prints the mean as the All change. In wall time it gains only from 5 × 5. The 86 at 5 × 3 wasn't traced. For the mean of the bugs' token changes, 5 × 4 and 3 × 6 give the same spread (14 and 14, 18 and 17) for 40 and 36 runs. + The All rows pool every run, so their median lands in whichever bug's runs sit in the middle of the pooled costs, and a bug's cost differs from another's by up to 8×. In tokens, the mean of the bugs' changes has a third to half of the pooled change's spread. In wall time, the mean gains only from 5 × 5. The 86 at 5 × 3 wasn't traced. For the mean of the bugs' token changes, 5 × 4 and 3 × 6 give the same spread (14 and 14, 18 and 17) for 40 and 36 runs. The bench still prints the pooled All rows per side, and the mean as the All change. - **One finished chunk**: `fejkdata`'s PR #490, base `33b7fa80`, head `5b9c2a14`, which the comprehension panel ruled better on Navigation, Locality and Shape and the same on Self-sufficiency. The bench planted 5 bugs and ran 38 of 50 runs before the machine restarted. The first 3 runs per bug and side, 30 runs, are the ones counted, and all passed. The mean of the bugs' changes was +4.6% in output tokens, +8.9% in cache-read tokens and +3.8% in turns, with the bugs' own changes in cache-read tokens from -41% to +47%. The two token figures sit inside the spread at 5 × 3, 17% and 22%; the spread of the turns wasn't measured. The mean of the wall times' changes, +105%, lies above the 86% of that row, from one bug whose head runs took 6.8 times as long at the median. Nobody read its streams to find out why. The head cost slightly more tokens where the panel finds it easier to read, and the wall time can't be judged, so the bench neither confirmed nor contradicted the panel. - **Cost**: 5 bugs × 6 runs took 3.9 hours and 34M cache-read, 1.2M cache-write and 0.3M output tokens; a run takes about 3.5 minutes, and planting 16 to 25 minutes. A 5 × 5 bench is about 3.3 hours and 28M cache-read tokens. Gremlins ran out of 8 GB at the base of PR #490, so the help's example takes 16 GB. diff --git a/todo.md b/todo.md index f1d78ffe..7db19de2 100644 --- a/todo.md +++ b/todo.md @@ -124,7 +124,7 @@ Touches: **Record in a decision the 95th-percentile spread, at 5 bugs × 5 runs, of the reads before the first open of the bug's file and of the turns, beside the tokens'.** -Item 503 found no affordable count that lifts a chunk-sized change above the tokens' noise (`docs/decisions.md` → `task-bench: a bench takes 5 bugs and 5 runs per side, after a 2 × 2 pilot on a new setup, and at no affordable count does a chunk-sized change rise above the noise`). Reads and turns are counts, not token sums, so they may be steadier. Run `task-bench` with the head against itself (`--base` the head's own commit) on a Go project, 5 bugs × 5 runs after a 2 × 2 pilot. Split each bug's runs at random into two sides, as the decision describes, and record each figure's 95th-percentile spread, for the All change and for the mean of the bugs' changes. Say in the decision whether either count can see a change the tokens can't. It needs a Claude token, given as `task-bench --help` says, and about 4 hours of wall time. +Item 503 found no affordable count that lifts a chunk-sized change above the tokens' noise (`docs/decisions.md` → `task-bench: a bench takes 5 bugs and 5 runs per side, after a 2 × 2 pilot on a new setup, and at no affordable count does a chunk-sized change rise above the noise`). Reads and turns are counts, not token sums, so they may be steadier. Run `task-bench` with the head against itself (`--base` the head's own commit) on a Go project, 5 bugs × 5 runs after a 2 × 2 pilot. Split each bug's runs at random into two sides, as the decision describes, and record each figure's 95th-percentile spread, for the change of the pooled medians, which the bench printed before item 514, and for the mean of the bugs' own changes, which it prints now. Say in the decision whether either count can see a change the tokens can't. It needs a Claude token, given as `task-bench --help` says, and about 4 hours of wall time. Touches: From ab08c3c238214e52e964f6b2391a07279ca70ce2 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 22 Nov 2026 17:49:53 +0100 Subject: [PATCH 10/10] Finish item 514 in the plan --- CHANGELOG.md | 1 + todo.md | 14 -------------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abd905a9..234df3f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- `task-bench`'s table prints the mean of the bugs' own head/base changes as its All change, in place of the change of the two All rows' medians, which swung up to three times as far in tokens. The All rows per side stay, so the change row is no longer their difference, and a change from before 0.15.1 can't be compared with one from after it. - `task-bench` now stops with a Failure on a run whose final result says `terminal_reason: "api_error"`, even where it holds no `api_error_status`, so a token the API refuses no longer lets the bench run on for hours with empty runs. - `task-bench`'s `--help` now says how far a bench of a commit against itself swings, in 19 cases of 20, with the default 5 bugs and 5 runs, and that a Gremlins run on a large project needs 16 GB of memory. diff --git a/todo.md b/todo.md index d6c8077a..68248caf 100644 --- a/todo.md +++ b/todo.md @@ -12,7 +12,6 @@ | ID | Release | Blocked by | Exempt | Item | L | E | I | R | S | Moves | Score | |---|---|---|---|---|---|---|---|---|---|---|---| -| 514 | 0.15.1 | | | **Print the mean of the bugs' own head/base changes as the table's All change, in place of the change of the medians pooled over every run.** | 2 | 2 | 1 | 3 | 3 | 1.1 by 3 | 1.1 | | 516 | 0.15.1 | | | **Record in a decision the 95th-percentile spread, at 5 bugs × 5 runs, of the reads before the first open of the bug's file and of the turns, beside the tokens'.** | 1 | 2 | 1 | 2 | 2 | 1.1 by 2 | 0.8 | | 515 | 0.15.1 | | | **Plant bugs that Haiku fails at the base, so the pass rate's spread over two benches of one commit is above zero, and record it in a decision.** | 3 | 2 | 1 | 4 | 5 | 1.1 by 3 | 0.7 | | 509 | 0.16.0 | | owner | **Run `task-bench` beside the scoring run in larv-review comprehension check, and record its figures in the comprehension history beside the panel scores.** | 3 | 2 | 3 | 6 | 5 | code health by 5, 1.1 by 3 | 5.6 | @@ -106,19 +105,6 @@ ## Details -### 514. Print the mean of the bugs' own head/base changes as the table's All change, in place of the change of the medians pooled over every run. - -**Print the mean of the bugs' own head/base changes as the table's All change, in place of the change of the medians pooled over every run.** - -The All row pools every run, so its median lands in whichever bug's runs sit in the middle of the pooled costs, and its change swings by bug. Item 503 measured the pooled change's spread at up to three times the mean of the bugs' own changes: at 5 bugs × 5 runs, 31% against 12% in output tokens and 50% against 16% in cache-read tokens (`docs/decisions.md` → `task-bench: a bench takes 5 bugs and 5 runs per side, after a 2 × 2 pilot on a new setup, and at no affordable count does a chunk-sized change rise above the noise`). Keep the per-side All rows where that stays cheap. Then restate the help's paragraph "Read a change against the noise" for the one figure the table prints, and update the decision's figures and its plan-items sentence to match. - -Touches: - -- `bin/task_bench/figures.py`: `table`, `change` -- `bin/task_bench/test_figures.py`: `TableTest` -- `bin/task_bench/main.py`: `(module)` (small) -- `docs/decisions.md`: `## task-bench: a bench takes 5 bugs and 5 runs per side, after a 2 × 2 pilot on a new setup, and at no affordable count does a chunk-sized change rise above the noise` (small) - ### 516. Record in a decision the 95th-percentile spread, at 5 bugs × 5 runs, of the reads before the first open of the bug's file and of the turns, beside the tokens'. **Record in a decision the 95th-percentile spread, at 5 bugs × 5 runs, of the reads before the first open of the bug's file and of the turns, beside the tokens'.**