diff --git a/bin/task_bench/figures.py b/bin/task_bench/figures.py index 59875e22..6071b3cf 100644 --- a/bin/task_bench/figures.py +++ b/bin/task_bench/figures.py @@ -48,7 +48,7 @@ def figures(stream: bytes, bug_file: str) -> Figures | None | Failure: result = next((event for event in reversed(events) if event.get("type") == "result"), None) if result is None: return None - if result.get("api_error_status"): + if result.get("api_error_status") or result.get("terminal_reason") == "api_error": return Failure(f"claude could not use the API: {result.get('result')}") # {"type": "system", "subtype": "init", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330", ...} init = next((event for event in events if event.get("subtype") == "init"), {}) diff --git a/bin/task_bench/test_figures.py b/bin/task_bench/test_figures.py index f46faff4..c81c2781 100644 --- a/bin/task_bench/test_figures.py +++ b/bin/task_bench/test_figures.py @@ -69,6 +69,10 @@ class FiguresTest(unittest.TestCase): refused = {**RESULT, "is_error": True, "api_error_status": 401, "result": "Failed to authenticate. API Error: 401 OAuth access token is invalid."} self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: Failed to authenticate. API Error: 401 OAuth access token is invalid.")) + def test_a_run_that_ended_on_an_api_error_without_a_status_is_a_failure_that_quotes_claude(self): + refused = {**RESULT, "is_error": True, "terminal_reason": "api_error", "result": "CLAUDE_CODE_OAUTH_TOKEN is not a valid header value."} + self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: CLAUDE_CODE_OAUTH_TOKEN is not a valid header value.")) + def run(task: int, side: str, number: int, passed: bool, input_: int = 100, turns: int = 10, seconds: float = 60, reads: int | None = 3) -> Result: return Result(task, side, number, passed, Figures("2.1.330", "claude-haiku-5-5", Tokens(input=input_, write_5m=20, write_1h=980, read=5000, output=400), turns, seconds, reads)) diff --git a/docs/reading-cost.md b/docs/reading-cost.md index d3aa0268..556f696c 100644 --- a/docs/reading-cost.md +++ b/docs/reading-cost.md @@ -3384,3 +3384,20 @@ Read most (opens and searches): - `bin/reading_cost/measure.py` 2 - `bin/review_rules/review_folder.py` 2 - `bin/task_bench/figures.py` 2 + +## Reading cost, chunk 666aabdd77a8, finished 2026-11-22T15:50:09Z + +283046 new input tokens before the first commit. + +Read most (opens and searches): + +- `docs/decisions.md` 14 +- `bin/task_bench/main.py` 9 +- `bin/task_bench/figures.py` 4 +- `bin/task_bench/plant.py` 4 +- `bin/task_bench/test_figures.py` 4 +- `bin/task_bench/runs.py` 3 +- `CHANGELOG.md` 2 +- `README.md` 2 +- `docs/comprehension-history.md` 2 +- `docs/score-anchors.md` 2