Files
claude-config/bin/task_bench/figures.py
T
2026-11-22 16:50:51 +01:00

117 lines
5.5 KiB
Python

"""What task-bench measures: the figures of one run, read from the stream its claude printed, and the table of a bench's runs."""
import dataclasses
import statistics
import typing
from failure import Failure
from claude_code.entries import json_object, parse_entry, tool_calls, ToolCall
from reading_cost.measure import reads_of
from claude_code.tokens import Tokens, tokens_of
# Where each run's container mounts its copy of the code, and its home folder: the paths in a run's stream start there.
WORK = "/work"
HOME = "/tmp"
# The tools that read or search without a shell. A Bash call counts where `reads_of` finds a file it reads.
READING_TOOLS = frozenset({"Glob", "Grep", "Read"})
SIDES = ("base", "head")
HEADER = ["| Bug | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |", "|---|---|---|---|---|---|---|---|---|---|"]
@dataclasses.dataclass(frozen=True)
class Figures:
"""One finished run, as its stream tells it. reads counts the reads and searches before the first open of the file the bug is
in, and is None where the run never opened it."""
version: str
model: str
tokens: Tokens
turns: int
seconds: float
reads: int | None
class Result(typing.NamedTuple):
"""One run of a task at a side. figures is None where its claude ended without a result, as when it ran out of time."""
task: int
side: str
run: int
passed: bool
figures: Figures | None
def figures(stream: bytes, bug_file: str) -> Figures | None | Failure:
"""The figures of the run whose claude printed stream, with bug_file the bug's file as the run's container names it.
None where the stream holds no result. A Failure where the API refused the run, as on a token it doesn't take: such a run
doesn't measure anything, and every later run would end the same."""
events = [event for line in stream.splitlines() if (event := json_object(line))]
result = next((event for event in reversed(events) if event.get("type") == "result"), None)
if result is None:
return None
if result.get("api_error_status") or result.get("terminal_reason") == "api_error":
return Failure(f"claude could not use the API: {result.get('result')}")
# {"type": "system", "subtype": "init", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330", ...}
init = next((event for event in events if event.get("subtype") == "init"), {})
entries = [entry for line in stream.splitlines() if (entry := parse_entry(line))]
calls = [call for _, call in tool_calls(entries)]
return Figures(str(init.get("claude_code_version", "")), str(init.get("model", "")), tokens_of(result.get("usage")), int(result.get("num_turns") or 0), (result.get("duration_ms") or 0) / 1000, reads_before(calls, bug_file))
def reads_before(calls: list[ToolCall], bug_file: str) -> int | None:
count = 0
for call in calls:
read = reads_of(call, WORK, HOME).read
if bug_file in read:
return count
count += call.name in READING_TOOLS or bool(read)
return None
def table(results: list[Result], tasks: dict[int, str]) -> list[str]:
"""A row per task and side, then one per side over every task, then the head's change against the base. Each figure is the
median over the side's runs that have figures, and Reads over those that opened the bug's file."""
rows = [row(f"{task}: {tasks[task]}", side, [result for result in results if (result.task, result.side) == (task, side)]) for task in sorted(tasks) for side in SIDES]
base, head = ([result for result in results if result.side == side] for side in SIDES)
return [*HEADER, *rows, row("All", "base", base), row("All", "head", head), change(base, head)]
def medians(results: list[Result]) -> list[float | None]:
"""Input, cache write, cache read, output, turns, wall time and reads."""
measured = [result.figures for result in results if result.figures]
columns: list[list[float]] = [
[found.tokens.input for found in measured],
[found.tokens.write_5m + found.tokens.write_1h for found in measured],
[found.tokens.read for found in measured],
[found.tokens.output for found in measured],
[found.turns for found in measured],
[found.seconds for found in measured],
[found.reads for found in measured if found.reads is not None],
]
return [statistics.median(column) if column else None for column in columns]
def row(task: str, side: str, results: list[Result]) -> str:
passed = f"{sum(result.passed for result in results)}/{len(results)}"
return cells([task, side, passed, *("-" if median is None else number(median) for median in medians(results))])
def change(base: list[Result], head: list[Result]) -> str:
"""The head's row less the base's: a count as the difference, and tokens and wall time as that difference's share of the base's."""
passed = sum(result.passed for result in head) - sum(result.passed for result in base)
pairs = list(zip(medians(base), medians(head)))
shares = [None if was is None or now is None or not was else f"{number(round(100 * (now - was) / was), signed=True)} %" for was, now in pairs]
counts = [None if was is None or now is None else number(now - was, signed=True) for was, now in pairs]
# Turns and reads are small counts, where a share misleads.
mixed = [*shares[:4], counts[4], shares[5], counts[6]]
return cells(["All", "change", number(passed, signed=True), *(found or "-" for found in mixed)])
def number(value: float, signed: bool = False) -> str:
"""A whole number as it is, another with one decimal; signed, with a + before a rise."""
text = str(int(value)) if value == int(value) else f"{value:.1f}"
return f"+{text}" if signed and value > 0 else text
def cells(found: list[str]) -> str:
return f"| {' | '.join(found)} |"