Files
claude-config/bin/task_bench/test_figures.py
T
2026-11-21 01:39:12 +01:00

114 lines
6.6 KiB
Python

"""task-bench's figures: what it reads from one run's stream, and the table it prints for a bench."""
import json
import pathlib
import sys
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from failure import Failure
from task_bench.figures import Figures, figures, Result, table
from claude_code.tokens import Tokens
INIT = {"type": "system", "subtype": "init", "cwd": "/work", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330"}
USAGE = {"input_tokens": 40, "cache_creation_input_tokens": 9000, "cache_read_input_tokens": 52000, "output_tokens": 800, "cache_creation": {"ephemeral_1h_input_tokens": 9000, "ephemeral_5m_input_tokens": 0}}
RESULT = {"type": "result", "subtype": "success", "is_error": False, "num_turns": 7, "duration_ms": 61500, "usage": USAGE, "result": "Fixed."}
def call(name: str, **given: str) -> dict:
return {"type": "assistant", "message": {"id": f"msg_{name}_{sorted(given.values())}", "model": "claude-haiku-5-5", "content": [{"type": "tool_use", "id": "toolu_1", "name": name, "input": given}], "usage": {}}}
def stream(*events: dict) -> bytes:
return "\n".join(map(json.dumps, events)).encode() + b"\n"
class FiguresTest(unittest.TestCase):
def test_reads_the_version_model_tokens_turns_and_wall_time_from_the_streams_first_and_last_events(self):
found = figures(stream(INIT, RESULT), "/work/calc.go")
self.assertEqual(found, Figures("2.1.330", "claude-haiku-5-5", Tokens(input=40, write_1h=9000, read=52000, output=800), 7, 61.5, None))
def test_counts_the_reads_and_searches_before_the_first_open_of_the_file_the_bug_is_in(self):
calls = [
call("Grep", pattern="Pick"),
call("Read", file_path="/work/README.md"),
call("Bash", command="go test ./..."),
call("Bash", command="cat render/pick.go | head -40"),
call("Glob", pattern="**/*.go"),
call("Read", file_path="/work/calc.go"),
call("Read", file_path="/work/other.go"),
call("Edit", file_path="/work/calc.go", old_string="a", new_string="b"),
]
found = figures(stream(INIT, *calls, RESULT), "/work/calc.go")
self.assertEqual(found.reads, 4)
def test_a_shell_command_that_reads_the_file_opens_it(self):
found = figures(stream(INIT, call("Read", file_path="/work/README.md"), call("Bash", command="sed -n 1,40p calc.go"), RESULT), "/work/calc.go")
self.assertEqual(found.reads, 1)
def test_has_no_reads_where_the_run_never_opened_the_file(self):
self.assertIsNone(figures(stream(INIT, call("Read", file_path="/work/README.md"), RESULT), "/work/calc.go").reads)
def test_a_stream_that_ends_without_a_result_has_no_figures(self):
self.assertIsNone(figures(stream(INIT, call("Read", file_path="/work/calc.go")), "/work/calc.go"))
self.assertIsNone(figures(b"", "/work/calc.go"))
def test_a_result_without_turns_or_a_duration_counts_none(self):
found = figures(stream(INIT, {"type": "result", "usage": USAGE}), "/work/calc.go")
self.assertEqual((found.turns, found.seconds), (0, 0))
def test_a_stream_without_an_init_names_no_version_or_model(self):
found = figures(stream(RESULT), "/work/calc.go")
self.assertEqual((found.version, found.model), ("", ""))
def test_a_shell_command_that_reads_under_the_runs_home_reads_no_file_of_the_code(self):
found = figures(stream(INIT, call("Bash", command="cat ~/calc.go"), call("Read", file_path="/work/calc.go"), RESULT), "/work/calc.go")
self.assertEqual(found.reads, 1)
def test_a_run_the_api_refused_is_a_failure_that_quotes_claude(self):
refused = {**RESULT, "is_error": True, "api_error_status": 401, "result": "Failed to authenticate. API Error: 401 OAuth access token is invalid."}
self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: Failed to authenticate. API Error: 401 OAuth access token is invalid."))
def run(task: int, side: str, number: int, passed: bool, input_: int = 100, turns: int = 10, seconds: float = 60, reads: int | None = 3) -> Result:
return Result(task, side, number, passed, Figures("2.1.330", "claude-haiku-5-5", Tokens(input=input_, write_5m=20, write_1h=980, read=5000, output=400), turns, seconds, reads))
class TableTest(unittest.TestCase):
def test_prints_a_row_per_task_and_side_with_the_pass_count_and_each_median_then_all_tasks_and_the_heads_change(self):
results = [
run(1, "base", 1, True, input_=100, turns=10, seconds=60, reads=3),
run(1, "base", 2, False, input_=300, turns=20, seconds=120, reads=None),
run(1, "base", 3, True, input_=200, turns=12, seconds=90, reads=5),
run(1, "head", 1, True, input_=100, turns=8, seconds=45, reads=1),
run(1, "head", 2, True, input_=100, turns=8, seconds=45, reads=1),
run(1, "head", 3, True, input_=160, turns=9, seconds=50, reads=2),
run(2, "base", 1, False, input_=200, turns=12, seconds=90, reads=4),
run(2, "head", 1, True, input_=100, turns=9, seconds=45, reads=2),
]
self.assertEqual(table(results, {1: "calc.go:4:7 CONDITIONALS_NEGATION", 2: "pick.go:9:3 ARITHMETIC_BASE"}), [
"| Task | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |",
"|---|---|---|---|---|---|---|---|---|---|",
"| 1: calc.go:4:7 CONDITIONALS_NEGATION | base | 2/3 | 200 | 1000 | 5000 | 400 | 12 | 90 | 4 |",
"| 1: calc.go:4:7 CONDITIONALS_NEGATION | head | 3/3 | 100 | 1000 | 5000 | 400 | 8 | 45 | 1 |",
"| 2: pick.go:9:3 ARITHMETIC_BASE | base | 0/1 | 200 | 1000 | 5000 | 400 | 12 | 90 | 4 |",
"| 2: pick.go:9:3 ARITHMETIC_BASE | head | 1/1 | 100 | 1000 | 5000 | 400 | 9 | 45 | 2 |",
"| All | base | 2/4 | 200 | 1000 | 5000 | 400 | 12 | 90 | 4 |",
"| All | head | 4/4 | 100 | 1000 | 5000 | 400 | 8.5 | 45 | 1.5 |",
"| All | change | +2 | -50 % | 0 % | 0 % | 0 % | -3.5 | -50 % | -2.5 |",
])
def test_the_change_is_a_dash_where_the_base_measured_zero_and_signed_where_it_rose(self):
results = [run(1, "base", 1, True, input_=0, turns=10, seconds=60, reads=0), run(1, "head", 1, False, input_=50, turns=12, seconds=90, reads=2)]
self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[-1], "| All | change | -1 | - | 0 % | 0 % | 0 % | +2 | +50 % | +2 |")
def test_a_run_without_figures_counts_as_not_passed_and_a_side_without_any_shows_a_dash(self):
results = [Result(1, "base", 1, False, None), run(1, "head", 1, True)]
self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[2:], [
"| 1: calc.go:4:7 ARITHMETIC_BASE | base | 0/1 | - | - | - | - | - | - | - |",
"| 1: calc.go:4:7 ARITHMETIC_BASE | head | 1/1 | 100 | 1000 | 5000 | 400 | 10 | 60 | 3 |",
"| All | base | 0/1 | - | - | - | - | - | - | - |",
"| All | head | 1/1 | 100 | 1000 | 5000 | 400 | 10 | 60 | 3 |",
"| All | change | +1 | - | - | - | - | - | - | - |",
])