Stop the bench with a Failure on a run that ended on an api_error without a status
CI / tests (pull_request) Successful in 5m21s

This commit is contained in:
2026-11-22 16:50:51 +01:00
parent e687860824
commit 993295b486
3 changed files with 22 additions and 1 deletions
+1 -1
View File
@@ -48,7 +48,7 @@ def figures(stream: bytes, bug_file: str) -> Figures | None | Failure:
result = next((event for event in reversed(events) if event.get("type") == "result"), None)
if result is None:
return None
if result.get("api_error_status"):
if result.get("api_error_status") or result.get("terminal_reason") == "api_error":
return Failure(f"claude could not use the API: {result.get('result')}")
# {"type": "system", "subtype": "init", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330", ...}
init = next((event for event in events if event.get("subtype") == "init"), {})
+4
View File
@@ -69,6 +69,10 @@ class FiguresTest(unittest.TestCase):
refused = {**RESULT, "is_error": True, "api_error_status": 401, "result": "Failed to authenticate. API Error: 401 OAuth access token is invalid."}
self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: Failed to authenticate. API Error: 401 OAuth access token is invalid."))
def test_a_run_that_ended_on_an_api_error_without_a_status_is_a_failure_that_quotes_claude(self):
refused = {**RESULT, "is_error": True, "terminal_reason": "api_error", "result": "CLAUDE_CODE_OAUTH_TOKEN is not a valid header value."}
self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: CLAUDE_CODE_OAUTH_TOKEN is not a valid header value."))
def run(task: int, side: str, number: int, passed: bool, input_: int = 100, turns: int = 10, seconds: float = 60, reads: int | None = 3) -> Result:
return Result(task, side, number, passed, Figures("2.1.330", "claude-haiku-5-5", Tokens(input=input_, write_5m=20, write_1h=980, read=5000, output=400), turns, seconds, reads))
+17
View File
@@ -3384,3 +3384,20 @@ Read most (opens and searches):
- `bin/reading_cost/measure.py` 2
- `bin/review_rules/review_folder.py` 2
- `bin/task_bench/figures.py` 2
## Reading cost, chunk 666aabdd77a8, finished 2026-11-22T15:50:09Z
283046 new input tokens before the first commit.
Read most (opens and searches):
- `docs/decisions.md` 14
- `bin/task_bench/main.py` 9
- `bin/task_bench/figures.py` 4
- `bin/task_bench/plant.py` 4
- `bin/task_bench/test_figures.py` 4
- `bin/task_bench/runs.py` 3
- `CHANGELOG.md` 2
- `README.md` 2
- `docs/comprehension-history.md` 2
- `docs/score-anchors.md` 2