Files
claude-config/bin/task_bench/figures.py
T
2026-11-22 17:32:09 +01:00

129 lines
6.5 KiB
Python

"""What task-bench measures: the figures of one run, read from the stream its claude printed, and the table of a bench's runs."""
import dataclasses
import statistics
import typing
from failure import Failure
from claude_code.entries import json_object, parse_entry, tool_calls, ToolCall
from reading_cost.measure import reads_of
from claude_code.tokens import Tokens, tokens_of
# Where each run's container mounts its copy of the code, and its home folder: the paths in a run's stream start there.
WORK = "/work"
HOME = "/tmp"
# The tools that read or search without a shell. A Bash call counts where `reads_of` finds a file it reads.
READING_TOOLS = frozenset({"Glob", "Grep", "Read"})
SIDES = ("base", "head")
HEADER = ["| Bug | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |", "|---|---|---|---|---|---|---|---|---|---|"]
@dataclasses.dataclass(frozen=True)
class Figures:
"""One finished run, as its stream tells it. reads counts the reads and searches before the first open of the file the bug is
in, and is None where the run never opened it."""
version: str
model: str
tokens: Tokens
turns: int
seconds: float
reads: int | None
class Result(typing.NamedTuple):
"""One run of a task at a side. figures is None where its claude ended without a result, as when it ran out of time."""
task: int
side: str
run: int
passed: bool
figures: Figures | None
def figures(stream: bytes, bug_file: str) -> Figures | None | Failure:
"""The figures of the run whose claude printed stream, with bug_file the bug's file as the run's container names it.
None where the stream holds no result. A Failure where the API refused the run, as on a token it doesn't take: such a run
doesn't measure anything, and every later run would end the same."""
events = [event for line in stream.splitlines() if (event := json_object(line))]
result = next((event for event in reversed(events) if event.get("type") == "result"), None)
if result is None:
return None
if result.get("api_error_status") or result.get("terminal_reason") == "api_error":
return Failure(f"claude could not use the API: {result.get('result')}")
# {"type": "system", "subtype": "init", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330", ...}
init = next((event for event in events if event.get("subtype") == "init"), {})
entries = [entry for line in stream.splitlines() if (entry := parse_entry(line))]
calls = [call for _, call in tool_calls(entries)]
return Figures(str(init.get("claude_code_version", "")), str(init.get("model", "")), tokens_of(result.get("usage")), int(result.get("num_turns") or 0), (result.get("duration_ms") or 0) / 1000, reads_before(calls, bug_file))
def reads_before(calls: list[ToolCall], bug_file: str) -> int | None:
count = 0
for call in calls:
read = reads_of(call, WORK, HOME).read
if bug_file in read:
return count
count += call.name in READING_TOOLS or bool(read)
return None
def table(results: list[Result], tasks: dict[int, str]) -> list[str]:
"""A row per task and side, then one per side over every task, then the head's change against the base, as `change` says. Each
figure is the median over the side's runs that have figures, and Reads over those that opened the bug's file."""
sides = {(task, side): [result for result in results if (result.task, result.side) == (task, side)] for task in tasks for side in SIDES}
rows = [row(f"{task}: {tasks[task]}", side, sides[task, side]) for task in sorted(tasks) for side in SIDES]
base, head = ([result for result in results if result.side == side] for side in SIDES)
return [*HEADER, *rows, row("All", "base", base), row("All", "head", head), change([(sides[task, "base"], sides[task, "head"]) for task in sorted(tasks)])]
def medians(results: list[Result]) -> list[float | None]:
"""Input, cache write, cache read, output, turns, wall time and reads."""
measured = [result.figures for result in results if result.figures]
columns: list[list[float]] = [
[found.tokens.input for found in measured],
[found.tokens.write_5m + found.tokens.write_1h for found in measured],
[found.tokens.read for found in measured],
[found.tokens.output for found in measured],
[found.turns for found in measured],
[found.seconds for found in measured],
[found.reads for found in measured if found.reads is not None],
]
return [statistics.median(column) if column else None for column in columns]
def row(task: str, side: str, results: list[Result]) -> str:
passed = f"{sum(result.passed for result in results)}/{len(results)}"
return cells([task, side, passed, *("-" if median is None else number(median) for median in medians(results))])
def change(bugs: list[tuple[list[Result], list[Result]]]) -> str:
"""The head's change against the base, from each bug's base runs and head runs. The pass count is the difference over every
run. Each other figure is the mean, over the bugs, of the bug's own change: its head median less its base median, shown
as a share of the base median for tokens and wall time, and as a count for turns and reads. This isn't the change from the All
base row to the All head row: those medians land in whichever bug's runs sit in the middle of the pooled costs."""
passed = sum(result.passed for _, head in bugs for result in head) - sum(result.passed for base, _ in bugs for result in base)
pairs = [list(zip(medians(base), medians(head))) for base, head in bugs]
# Turns and reads are small counts, where a share misleads.
means = [mean_change([pair[column] for pair in pairs], share=column not in (4, 6)) for column in range(7)]
return cells(["All", "change", number(passed, signed=True), *(found or "-" for found in means)])
def mean_change(pairs: list[tuple[float | None, float | None]], share: bool) -> str | None:
"""The mean of the bugs' changes in one figure, from each bug's (base median, head median). A bug counts only where both
medians exist and, for a share, the base median isn't zero; None where no bug counts."""
changes = [(now - was) * (100 / was if share else 1) for was, now in pairs if was is not None and now is not None and (was or not share)]
if not changes:
return None
mean = statistics.mean(changes)
return f"{number(round(mean), signed=True)} %" if share else number(mean, signed=True)
def number(value: float, signed: bool = False) -> str:
"""A whole number as it is, another with one decimal; signed, with a + before a rise."""
text = str(int(value)) if value == int(value) else f"{value:.1f}"
return f"+{text}" if signed and value > 0 else text
def cells(found: list[str]) -> str:
return f"| {' | '.join(found)} |"