"""task-bench's figures: what it reads from one run's stream, and the table it prints for a bench.""" import json import pathlib import sys import unittest sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) from failure import Failure from task_bench.figures import Figures, figures, Result, table from claude_code.tokens import Tokens INIT = {"type": "system", "subtype": "init", "cwd": "/work", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330"} USAGE = {"input_tokens": 40, "cache_creation_input_tokens": 9000, "cache_read_input_tokens": 52000, "output_tokens": 800, "cache_creation": {"ephemeral_1h_input_tokens": 9000, "ephemeral_5m_input_tokens": 0}} RESULT = {"type": "result", "subtype": "success", "is_error": False, "num_turns": 7, "duration_ms": 61500, "usage": USAGE, "result": "Fixed."} def call(name: str, **given: str) -> dict: return {"type": "assistant", "message": {"id": f"msg_{name}_{sorted(given.values())}", "model": "claude-haiku-5-5", "content": [{"type": "tool_use", "id": "toolu_1", "name": name, "input": given}], "usage": {}}} def stream(*events: dict) -> bytes: return "\n".join(map(json.dumps, events)).encode() + b"\n" class FiguresTest(unittest.TestCase): def test_reads_the_version_model_tokens_turns_and_wall_time_from_the_streams_first_and_last_events(self): found = figures(stream(INIT, RESULT), "/work/calc.go") self.assertEqual(found, Figures("2.1.330", "claude-haiku-5-5", Tokens(input=40, write_1h=9000, read=52000, output=800), 7, 61.5, None)) def test_counts_the_reads_and_searches_before_the_first_open_of_the_file_the_bug_is_in(self): calls = [ call("Grep", pattern="Pick"), call("Read", file_path="/work/README.md"), call("Bash", command="go test ./..."), call("Bash", command="cat render/pick.go | head -40"), call("Glob", pattern="**/*.go"), call("Read", file_path="/work/calc.go"), call("Read", file_path="/work/other.go"), call("Edit", file_path="/work/calc.go", old_string="a", new_string="b"), ] found = figures(stream(INIT, *calls, RESULT), "/work/calc.go") self.assertEqual(found.reads, 4) def test_a_shell_command_that_reads_the_file_opens_it(self): found = figures(stream(INIT, call("Read", file_path="/work/README.md"), call("Bash", command="sed -n 1,40p calc.go"), RESULT), "/work/calc.go") self.assertEqual(found.reads, 1) def test_has_no_reads_where_the_run_never_opened_the_file(self): self.assertIsNone(figures(stream(INIT, call("Read", file_path="/work/README.md"), RESULT), "/work/calc.go").reads) def test_a_stream_that_ends_without_a_result_has_no_figures(self): self.assertIsNone(figures(stream(INIT, call("Read", file_path="/work/calc.go")), "/work/calc.go")) self.assertIsNone(figures(b"", "/work/calc.go")) def test_a_result_without_turns_or_a_duration_counts_none(self): found = figures(stream(INIT, {"type": "result", "usage": USAGE}), "/work/calc.go") self.assertEqual((found.turns, found.seconds), (0, 0)) def test_a_stream_without_an_init_names_no_version_or_model(self): found = figures(stream(RESULT), "/work/calc.go") self.assertEqual((found.version, found.model), ("", "")) def test_a_shell_command_that_reads_under_the_runs_home_reads_no_file_of_the_code(self): found = figures(stream(INIT, call("Bash", command="cat ~/calc.go"), call("Read", file_path="/work/calc.go"), RESULT), "/work/calc.go") self.assertEqual(found.reads, 1) def test_a_run_the_api_refused_is_a_failure_that_quotes_claude(self): refused = {**RESULT, "is_error": True, "api_error_status": 401, "result": "Failed to authenticate. API Error: 401 OAuth access token is invalid."} self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: Failed to authenticate. API Error: 401 OAuth access token is invalid.")) def test_a_run_that_ended_on_an_api_error_without_a_status_is_a_failure_that_quotes_claude(self): refused = {**RESULT, "is_error": True, "terminal_reason": "api_error", "result": "CLAUDE_CODE_OAUTH_TOKEN is not a valid header value."} self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: CLAUDE_CODE_OAUTH_TOKEN is not a valid header value.")) def run(task: int, side: str, number: int, passed: bool, input_: int = 100, turns: int = 10, seconds: float = 60, reads: int | None = 3) -> Result: return Result(task, side, number, passed, Figures("2.1.330", "claude-haiku-5-5", Tokens(input=input_, write_5m=20, write_1h=980, read=5000, output=400), turns, seconds, reads)) class TableTest(unittest.TestCase): def test_prints_a_row_per_task_and_side_with_the_pass_count_and_each_median_then_all_tasks_and_the_heads_change(self): results = [ run(1, "base", 1, True, input_=100, turns=10, seconds=60, reads=3), run(1, "base", 2, False, input_=300, turns=20, seconds=120, reads=None), run(1, "base", 3, True, input_=200, turns=12, seconds=90, reads=5), run(1, "head", 1, True, input_=100, turns=8, seconds=45, reads=1), run(1, "head", 2, True, input_=100, turns=8, seconds=45, reads=1), run(1, "head", 3, True, input_=160, turns=9, seconds=50, reads=2), run(2, "base", 1, False, input_=200, turns=12, seconds=90, reads=4), run(2, "head", 1, True, input_=100, turns=9, seconds=45, reads=2), ] self.assertEqual(table(results, {1: "calc.go:4:7 CONDITIONALS_NEGATION", 2: "pick.go:9:3 ARITHMETIC_BASE"}), [ "| Bug | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |", "|---|---|---|---|---|---|---|---|---|---|", "| 1: calc.go:4:7 CONDITIONALS_NEGATION | base | 2/3 | 200 | 1000 | 5000 | 400 | 12 | 90 | 4 |", "| 1: calc.go:4:7 CONDITIONALS_NEGATION | head | 3/3 | 100 | 1000 | 5000 | 400 | 8 | 45 | 1 |", "| 2: pick.go:9:3 ARITHMETIC_BASE | base | 0/1 | 200 | 1000 | 5000 | 400 | 12 | 90 | 4 |", "| 2: pick.go:9:3 ARITHMETIC_BASE | head | 1/1 | 100 | 1000 | 5000 | 400 | 9 | 45 | 2 |", "| All | base | 2/4 | 200 | 1000 | 5000 | 400 | 12 | 90 | 4 |", "| All | head | 4/4 | 100 | 1000 | 5000 | 400 | 8.5 | 45 | 1.5 |", "| All | change | +2 | -50 % | 0 % | 0 % | 0 % | -3.5 | -50 % | -2.5 |", ]) def test_the_change_is_a_dash_where_the_base_measured_zero_and_signed_where_it_rose(self): results = [run(1, "base", 1, True, input_=0, turns=10, seconds=60, reads=0), run(1, "head", 1, False, input_=50, turns=12, seconds=120, reads=2)] self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[-1], "| All | change | -1 | - | 0 % | 0 % | 0 % | +2 | +100 % | +2 |") def test_the_change_is_the_mean_of_the_bugs_own_changes_not_the_change_of_the_pooled_medians(self): results = [ run(1, "base", 1, True, input_=100, turns=10, seconds=60, reads=3), run(1, "head", 1, True, input_=200, turns=14, seconds=90, reads=5), run(2, "base", 1, True, input_=1000, turns=20, seconds=60, reads=7), run(2, "head", 1, True, input_=1000, turns=20, seconds=60, reads=7), ] rows = table(results, {1: "calc.go:4:7 ARITHMETIC_BASE", 2: "pick.go:9:3 ARITHMETIC_BASE"}) self.assertEqual(rows[-3:], [ "| All | base | 2/2 | 550 | 1000 | 5000 | 400 | 15 | 60 | 5 |", "| All | head | 2/2 | 600 | 1000 | 5000 | 400 | 17 | 75 | 6 |", "| All | change | 0 | +50 % | 0 % | 0 % | 0 % | +2 | +25 % | +1 |", ]) def test_a_bug_whose_base_measured_zero_is_left_out_of_the_mean_share(self): results = [ run(1, "base", 1, True, input_=0), run(1, "head", 1, True, input_=50), run(2, "base", 1, True, input_=100), run(2, "head", 1, True, input_=150), ] self.assertEqual(table(results, {1: "a.go:1:1 ARITHMETIC_BASE", 2: "b.go:1:1 ARITHMETIC_BASE"})[-1].split(" | ")[3], "+50 %") def test_a_run_without_figures_counts_as_not_passed_and_a_side_without_any_shows_a_dash(self): results = [Result(1, "base", 1, False, None), run(1, "head", 1, True)] self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[2:], [ "| 1: calc.go:4:7 ARITHMETIC_BASE | base | 0/1 | - | - | - | - | - | - | - |", "| 1: calc.go:4:7 ARITHMETIC_BASE | head | 1/1 | 100 | 1000 | 5000 | 400 | 10 | 60 | 3 |", "| All | base | 0/1 | - | - | - | - | - | - | - |", "| All | head | 1/1 | 100 | 1000 | 5000 | 400 | 10 | 60 | 3 |", "| All | change | +1 | - | - | - | - | - | - | - |", ]) def test_the_change_is_a_dash_where_only_the_head_has_no_figures(self): results = [run(1, "base", 1, True), Result(1, "head", 1, False, None)] self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[-1], "| All | change | -1 | - | - | - | - | - | - | - |")