From 993295b486402db040478a6b8be62fd6d99a0ef7 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 22 Nov 2026 16:50:51 +0100 Subject: [PATCH 1/4] Stop the bench with a Failure on a run that ended on an api_error without a status --- bin/task_bench/figures.py | 2 +- bin/task_bench/test_figures.py | 4 ++++ docs/reading-cost.md | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/bin/task_bench/figures.py b/bin/task_bench/figures.py index 59875e22..6071b3cf 100644 --- a/bin/task_bench/figures.py +++ b/bin/task_bench/figures.py @@ -48,7 +48,7 @@ def figures(stream: bytes, bug_file: str) -> Figures | None | Failure: result = next((event for event in reversed(events) if event.get("type") == "result"), None) if result is None: return None - if result.get("api_error_status"): + if result.get("api_error_status") or result.get("terminal_reason") == "api_error": return Failure(f"claude could not use the API: {result.get('result')}") # {"type": "system", "subtype": "init", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330", ...} init = next((event for event in events if event.get("subtype") == "init"), {}) diff --git a/bin/task_bench/test_figures.py b/bin/task_bench/test_figures.py index f46faff4..c81c2781 100644 --- a/bin/task_bench/test_figures.py +++ b/bin/task_bench/test_figures.py @@ -69,6 +69,10 @@ class FiguresTest(unittest.TestCase): refused = {**RESULT, "is_error": True, "api_error_status": 401, "result": "Failed to authenticate. API Error: 401 OAuth access token is invalid."} self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: Failed to authenticate. API Error: 401 OAuth access token is invalid.")) + def test_a_run_that_ended_on_an_api_error_without_a_status_is_a_failure_that_quotes_claude(self): + refused = {**RESULT, "is_error": True, "terminal_reason": "api_error", "result": "CLAUDE_CODE_OAUTH_TOKEN is not a valid header value."} + self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: CLAUDE_CODE_OAUTH_TOKEN is not a valid header value.")) + def run(task: int, side: str, number: int, passed: bool, input_: int = 100, turns: int = 10, seconds: float = 60, reads: int | None = 3) -> Result: return Result(task, side, number, passed, Figures("2.1.330", "claude-haiku-5-5", Tokens(input=input_, write_5m=20, write_1h=980, read=5000, output=400), turns, seconds, reads)) diff --git a/docs/reading-cost.md b/docs/reading-cost.md index d3aa0268..556f696c 100644 --- a/docs/reading-cost.md +++ b/docs/reading-cost.md @@ -3384,3 +3384,20 @@ Read most (opens and searches): - `bin/reading_cost/measure.py` 2 - `bin/review_rules/review_folder.py` 2 - `bin/task_bench/figures.py` 2 + +## Reading cost, chunk 666aabdd77a8, finished 2026-11-22T15:50:09Z + +283046 new input tokens before the first commit. + +Read most (opens and searches): + +- `docs/decisions.md` 14 +- `bin/task_bench/main.py` 9 +- `bin/task_bench/figures.py` 4 +- `bin/task_bench/plant.py` 4 +- `bin/task_bench/test_figures.py` 4 +- `bin/task_bench/runs.py` 3 +- `CHANGELOG.md` 2 +- `README.md` 2 +- `docs/comprehension-history.md` 2 +- `docs/score-anchors.md` 2 From 5b792514a4e553f81f977a81a88ffd8a8c000c40 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 22 Nov 2026 17:09:21 +0100 Subject: [PATCH 2/4] =?UTF-8?q?Answer=20security=20round=201:=200=20?= =?UTF-8?q?=F0=9F=94=B4,=200=20=F0=9F=9F=A1=20(0=20misread);=20nothing=20f?= =?UTF-8?q?ound?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 72c481696d5aa2cf799ad3a947d63b052686cb34 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 22 Nov 2026 17:09:21 +0100 Subject: [PATCH 3/4] =?UTF-8?q?Answer=20stability=20round=201:=200=20?= =?UTF-8?q?=F0=9F=94=B4,=200=20=F0=9F=9F=A1=20(0=20misread);=20approved=20?= =?UTF-8?q?without=20findings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 0b6cc71520e75d7a23fe272ce7c024909d17cf08 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sun, 22 Nov 2026 17:09:32 +0100 Subject: [PATCH 4/4] Finish item 513 in the plan --- CHANGELOG.md | 1 + todo.md | 12 ------------ 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90d16d4f..abd905a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- `task-bench` now stops with a Failure on a run whose final result says `terminal_reason: "api_error"`, even where it holds no `api_error_status`, so a token the API refuses no longer lets the bench run on for hours with empty runs. - `task-bench`'s `--help` now says how far a bench of a commit against itself swings, in 19 cases of 20, with the default 5 bugs and 5 runs, and that a Gremlins run on a large project needs 16 GB of memory. ## 0.15.0 diff --git a/todo.md b/todo.md index f1d78ffe..1228a6f6 100644 --- a/todo.md +++ b/todo.md @@ -14,7 +14,6 @@ |---|---|---|---|---|---|---|---|---|---|---|---| | 514 | 0.15.1 | | | **Print the mean of the bugs' own head/base changes as the table's All change, in place of the change of the medians pooled over every run.** | 2 | 2 | 1 | 3 | 3 | 1.1 by 3 | 1.1 | | 516 | 0.15.1 | | | **Record in a decision the 95th-percentile spread, at 5 bugs × 5 runs, of the reads before the first open of the bug's file and of the turns, beside the tokens'.** | 1 | 2 | 1 | 2 | 2 | 1.1 by 2 | 0.8 | -| 513 | 0.15.1 | | | **Stop the bench with a Failure on a run whose final `result` event says `terminal_reason: "api_error"`, even where its `api_error_status` is empty.** | 1 | 1 | 1 | 1 | 2 | 1.4 by 2 | 0.8 | | 515 | 0.15.1 | | | **Plant bugs that Haiku fails at the base, so the pass rate's spread over two benches of one commit is above zero, and record it in a decision.** | 3 | 2 | 1 | 4 | 5 | 1.1 by 3 | 0.7 | | 509 | 0.16.0 | | owner | **Run `task-bench` beside the scoring run in larv-review comprehension check, and record its figures in the comprehension history beside the panel scores.** | 3 | 2 | 3 | 6 | 5 | code health by 5, 1.1 by 3 | 5.6 | | 504 | 0.16.0 | | owner | **Bench new features in `task-bench`, each a spec for Haiku with acceptance tests it never sees and a reference patch that proves it can be built, kept in a `bench-tasks` folder the project config names, starting with six for fejkdata.** | 3 | 2 | 1 | 4 | 6 | 1.1 by 4, code health by 3 | 2.6 | @@ -130,17 +129,6 @@ Touches: - `docs/decisions.md`: `## task-bench: a bench takes 5 bugs and 5 runs per side, after a 2 × 2 pilot on a new setup, and at no affordable count does a chunk-sized change rise above the noise` -### 513. Stop the bench with a Failure on a run whose final `result` event says `terminal_reason: "api_error"`, even where its `api_error_status` is empty. - -**Stop the bench with a Failure on a run whose final `result` event says `terminal_reason: "api_error"`, even where its `api_error_status` is empty.** - -Today `figures()` checks only `api_error_status`. During item 503's calibration a token the API refused over a line break ended runs with `terminal_reason: "api_error"`, a message naming `CLAUDE_CODE_OAUTH_TOKEN`, and no status, so each counted as a finished, not-passed run with 0 tokens, and the bench went on for hours. `task-bench --help` promises that a run the API refused stops the bench. Write the test first: a stream whose result holds that terminal reason and no status gives a Failure that quotes claude's message. - -Touches: - -- `bin/task_bench/figures.py`: `figures` (small) -- `bin/task_bench/test_figures.py`: `FiguresTest` - ### 515. Plant bugs that Haiku fails at the base, so the pass rate's spread over two benches of one commit is above zero, and record it in a decision. **Plant bugs that Haiku fails at the base, so the pass rate's spread over two benches of one commit is above zero, and record it in a decision.**