141 lines
8.3 KiB
Python
141 lines
8.3 KiB
Python
"""task-bench's figures: what it reads from one run's stream, and the table it prints for a bench."""
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from failure import Failure
|
|
from task_bench.figures import Figures, figures, Result, table
|
|
from claude_code.tokens import Tokens
|
|
|
|
INIT = {"type": "system", "subtype": "init", "cwd": "/work", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330"}
|
|
USAGE = {"input_tokens": 40, "cache_creation_input_tokens": 9000, "cache_read_input_tokens": 52000, "output_tokens": 800, "cache_creation": {"ephemeral_1h_input_tokens": 9000, "ephemeral_5m_input_tokens": 0}}
|
|
RESULT = {"type": "result", "subtype": "success", "is_error": False, "num_turns": 7, "duration_ms": 61500, "usage": USAGE, "result": "Fixed."}
|
|
|
|
|
|
def call(name: str, **given: str) -> dict:
|
|
return {"type": "assistant", "message": {"id": f"msg_{name}_{sorted(given.values())}", "model": "claude-haiku-5-5", "content": [{"type": "tool_use", "id": "toolu_1", "name": name, "input": given}], "usage": {}}}
|
|
|
|
|
|
def stream(*events: dict) -> bytes:
|
|
return "\n".join(map(json.dumps, events)).encode() + b"\n"
|
|
|
|
|
|
class FiguresTest(unittest.TestCase):
|
|
def test_reads_the_version_model_tokens_turns_and_wall_time_from_the_streams_first_and_last_events(self):
|
|
found = figures(stream(INIT, RESULT), "/work/calc.go")
|
|
self.assertEqual(found, Figures("2.1.330", "claude-haiku-5-5", Tokens(input=40, write_1h=9000, read=52000, output=800), 7, 61.5, None))
|
|
|
|
def test_counts_the_reads_and_searches_before_the_first_open_of_the_file_the_bug_is_in(self):
|
|
calls = [
|
|
call("Grep", pattern="Pick"),
|
|
call("Read", file_path="/work/README.md"),
|
|
call("Bash", command="go test ./..."),
|
|
call("Bash", command="cat render/pick.go | head -40"),
|
|
call("Glob", pattern="**/*.go"),
|
|
call("Read", file_path="/work/calc.go"),
|
|
call("Read", file_path="/work/other.go"),
|
|
call("Edit", file_path="/work/calc.go", old_string="a", new_string="b"),
|
|
]
|
|
found = figures(stream(INIT, *calls, RESULT), "/work/calc.go")
|
|
self.assertEqual(found.reads, 4)
|
|
|
|
def test_a_shell_command_that_reads_the_file_opens_it(self):
|
|
found = figures(stream(INIT, call("Read", file_path="/work/README.md"), call("Bash", command="sed -n 1,40p calc.go"), RESULT), "/work/calc.go")
|
|
self.assertEqual(found.reads, 1)
|
|
|
|
def test_has_no_reads_where_the_run_never_opened_the_file(self):
|
|
self.assertIsNone(figures(stream(INIT, call("Read", file_path="/work/README.md"), RESULT), "/work/calc.go").reads)
|
|
|
|
def test_a_stream_that_ends_without_a_result_has_no_figures(self):
|
|
self.assertIsNone(figures(stream(INIT, call("Read", file_path="/work/calc.go")), "/work/calc.go"))
|
|
self.assertIsNone(figures(b"", "/work/calc.go"))
|
|
|
|
def test_a_result_without_turns_or_a_duration_counts_none(self):
|
|
found = figures(stream(INIT, {"type": "result", "usage": USAGE}), "/work/calc.go")
|
|
self.assertEqual((found.turns, found.seconds), (0, 0))
|
|
|
|
def test_a_stream_without_an_init_names_no_version_or_model(self):
|
|
found = figures(stream(RESULT), "/work/calc.go")
|
|
self.assertEqual((found.version, found.model), ("", ""))
|
|
|
|
def test_a_shell_command_that_reads_under_the_runs_home_reads_no_file_of_the_code(self):
|
|
found = figures(stream(INIT, call("Bash", command="cat ~/calc.go"), call("Read", file_path="/work/calc.go"), RESULT), "/work/calc.go")
|
|
self.assertEqual(found.reads, 1)
|
|
|
|
def test_a_run_the_api_refused_is_a_failure_that_quotes_claude(self):
|
|
refused = {**RESULT, "is_error": True, "api_error_status": 401, "result": "Failed to authenticate. API Error: 401 OAuth access token is invalid."}
|
|
self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: Failed to authenticate. API Error: 401 OAuth access token is invalid."))
|
|
|
|
def test_a_run_that_ended_on_an_api_error_without_a_status_is_a_failure_that_quotes_claude(self):
|
|
refused = {**RESULT, "is_error": True, "terminal_reason": "api_error", "result": "CLAUDE_CODE_OAUTH_TOKEN is not a valid header value."}
|
|
self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: CLAUDE_CODE_OAUTH_TOKEN is not a valid header value."))
|
|
|
|
|
|
def run(task: int, side: str, number: int, passed: bool, input_: int = 100, turns: int = 10, seconds: float = 60, reads: int | None = 3) -> Result:
|
|
return Result(task, side, number, passed, Figures("2.1.330", "claude-haiku-5-5", Tokens(input=input_, write_5m=20, write_1h=980, read=5000, output=400), turns, seconds, reads))
|
|
|
|
|
|
class TableTest(unittest.TestCase):
|
|
def test_prints_a_row_per_task_and_side_with_the_pass_count_and_each_median_then_all_tasks_and_the_heads_change(self):
|
|
results = [
|
|
run(1, "base", 1, True, input_=100, turns=10, seconds=60, reads=3),
|
|
run(1, "base", 2, False, input_=300, turns=20, seconds=120, reads=None),
|
|
run(1, "base", 3, True, input_=200, turns=12, seconds=90, reads=5),
|
|
run(1, "head", 1, True, input_=100, turns=8, seconds=45, reads=1),
|
|
run(1, "head", 2, True, input_=100, turns=8, seconds=45, reads=1),
|
|
run(1, "head", 3, True, input_=160, turns=9, seconds=50, reads=2),
|
|
run(2, "base", 1, False, input_=200, turns=12, seconds=90, reads=4),
|
|
run(2, "head", 1, True, input_=100, turns=9, seconds=45, reads=2),
|
|
]
|
|
self.assertEqual(table(results, {1: "calc.go:4:7 CONDITIONALS_NEGATION", 2: "pick.go:9:3 ARITHMETIC_BASE"}), [
|
|
"| Bug | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |",
|
|
"|---|---|---|---|---|---|---|---|---|---|",
|
|
"| 1: calc.go:4:7 CONDITIONALS_NEGATION | base | 2/3 | 200 | 1000 | 5000 | 400 | 12 | 90 | 4 |",
|
|
"| 1: calc.go:4:7 CONDITIONALS_NEGATION | head | 3/3 | 100 | 1000 | 5000 | 400 | 8 | 45 | 1 |",
|
|
"| 2: pick.go:9:3 ARITHMETIC_BASE | base | 0/1 | 200 | 1000 | 5000 | 400 | 12 | 90 | 4 |",
|
|
"| 2: pick.go:9:3 ARITHMETIC_BASE | head | 1/1 | 100 | 1000 | 5000 | 400 | 9 | 45 | 2 |",
|
|
"| All | base | 2/4 | 200 | 1000 | 5000 | 400 | 12 | 90 | 4 |",
|
|
"| All | head | 4/4 | 100 | 1000 | 5000 | 400 | 8.5 | 45 | 1.5 |",
|
|
"| All | change | +2 | -50 % | 0 % | 0 % | 0 % | -3.5 | -50 % | -2.5 |",
|
|
])
|
|
|
|
def test_the_change_is_a_dash_where_the_base_measured_zero_and_signed_where_it_rose(self):
|
|
results = [run(1, "base", 1, True, input_=0, turns=10, seconds=60, reads=0), run(1, "head", 1, False, input_=50, turns=12, seconds=120, reads=2)]
|
|
self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[-1], "| All | change | -1 | - | 0 % | 0 % | 0 % | +2 | +100 % | +2 |")
|
|
|
|
def test_the_change_is_the_mean_of_the_bugs_own_changes_not_the_change_of_the_pooled_medians(self):
|
|
results = [
|
|
run(1, "base", 1, True, input_=100, turns=10, seconds=60, reads=3), run(1, "head", 1, True, input_=200, turns=14, seconds=90, reads=5),
|
|
run(2, "base", 1, True, input_=1000, turns=20, seconds=60, reads=7), run(2, "head", 1, True, input_=1000, turns=20, seconds=60, reads=7),
|
|
]
|
|
rows = table(results, {1: "calc.go:4:7 ARITHMETIC_BASE", 2: "pick.go:9:3 ARITHMETIC_BASE"})
|
|
self.assertEqual(rows[-3:], [
|
|
"| All | base | 2/2 | 550 | 1000 | 5000 | 400 | 15 | 60 | 5 |",
|
|
"| All | head | 2/2 | 600 | 1000 | 5000 | 400 | 17 | 75 | 6 |",
|
|
"| All | change | 0 | +50 % | 0 % | 0 % | 0 % | +2 | +25 % | +1 |",
|
|
])
|
|
|
|
def test_a_bug_whose_base_measured_zero_is_left_out_of_the_mean_share(self):
|
|
results = [
|
|
run(1, "base", 1, True, input_=0), run(1, "head", 1, True, input_=50),
|
|
run(2, "base", 1, True, input_=100), run(2, "head", 1, True, input_=150),
|
|
]
|
|
self.assertEqual(table(results, {1: "a.go:1:1 ARITHMETIC_BASE", 2: "b.go:1:1 ARITHMETIC_BASE"})[-1].split(" | ")[3], "+50 %")
|
|
|
|
def test_a_run_without_figures_counts_as_not_passed_and_a_side_without_any_shows_a_dash(self):
|
|
results = [Result(1, "base", 1, False, None), run(1, "head", 1, True)]
|
|
self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[2:], [
|
|
"| 1: calc.go:4:7 ARITHMETIC_BASE | base | 0/1 | - | - | - | - | - | - | - |",
|
|
"| 1: calc.go:4:7 ARITHMETIC_BASE | head | 1/1 | 100 | 1000 | 5000 | 400 | 10 | 60 | 3 |",
|
|
"| All | base | 0/1 | - | - | - | - | - | - | - |",
|
|
"| All | head | 1/1 | 100 | 1000 | 5000 | 400 | 10 | 60 | 3 |",
|
|
"| All | change | +1 | - | - | - | - | - | - | - |",
|
|
])
|
|
|
|
def test_the_change_is_a_dash_where_only_the_head_has_no_figures(self):
|
|
results = [run(1, "base", 1, True), Result(1, "head", 1, False, None)]
|
|
self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[-1], "| All | change | -1 | - | - | - | - | - | - | - |")
|