130 lines
6.6 KiB
Python
130 lines
6.6 KiB
Python
"""What task-bench measures: the figures of one run, read from the stream its claude printed, and the table of a bench's runs."""
|
|
import dataclasses
|
|
import statistics
|
|
import typing
|
|
|
|
from failure import Failure
|
|
from claude_code.entries import json_object, parse_entry, tool_calls, ToolCall
|
|
from reading_cost.measure import reads_of
|
|
from claude_code.tokens import Tokens, tokens_of
|
|
|
|
# Where each run's container mounts its copy of the code, and its home folder: the paths in a run's stream start there.
|
|
WORK = "/work"
|
|
HOME = "/tmp"
|
|
# The tools that read or search without a shell. A Bash call counts where `reads_of` finds a file it reads.
|
|
READING_TOOLS = frozenset({"Glob", "Grep", "Read"})
|
|
SIDES = ("base", "head")
|
|
HEADER = ["| Bug | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |", "|---|---|---|---|---|---|---|---|---|---|"]
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Figures:
|
|
"""One finished run, as its stream tells it. reads counts the reads and searches before the first open of the file the bug is
|
|
in, and is None where the run never opened it."""
|
|
version: str
|
|
model: str
|
|
tokens: Tokens
|
|
turns: int
|
|
seconds: float
|
|
reads: int | None
|
|
|
|
|
|
class Result(typing.NamedTuple):
|
|
"""One run of a task at a side. figures is None where its claude ended without a result, as when it ran out of time."""
|
|
|
|
task: int
|
|
side: str
|
|
run: int
|
|
passed: bool
|
|
figures: Figures | None
|
|
|
|
|
|
def figures(stream: bytes, bug_file: str) -> Figures | None | Failure:
|
|
"""The figures of the run whose claude printed stream, with bug_file the bug's file as the run's container names it.
|
|
|
|
None where the stream holds no result. A Failure where the API refused the run, as on a token it doesn't take: such a run
|
|
doesn't measure anything, and every later run would end the same."""
|
|
events = [event for line in stream.splitlines() if (event := json_object(line))]
|
|
result = next((event for event in reversed(events) if event.get("type") == "result"), None)
|
|
if result is None:
|
|
return None
|
|
if result.get("api_error_status"):
|
|
return Failure(f"claude could not use the API: {result.get('result')}")
|
|
# {"type": "system", "subtype": "init", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330", ...}
|
|
init = next((event for event in events if event.get("subtype") == "init"), {})
|
|
entries = [entry for line in stream.splitlines() if (entry := parse_entry(line))]
|
|
calls = [call for _, call in tool_calls(entries)]
|
|
return Figures(str(init.get("claude_code_version", "")), str(init.get("model", "")), tokens_of(result.get("usage")), int(result.get("num_turns") or 0), (result.get("duration_ms") or 0) / 1000, reads_before(calls, bug_file))
|
|
|
|
|
|
def reads_before(calls: list[ToolCall], bug_file: str) -> int | None:
|
|
count = 0
|
|
for call in calls:
|
|
read = reads_of(call, WORK, HOME).read
|
|
if bug_file in read:
|
|
return count
|
|
count += call.name in READING_TOOLS or bool(read)
|
|
return None
|
|
|
|
|
|
def table(results: list[Result], tasks: dict[int, str]) -> list[str]:
|
|
"""A row per task and side, then one per side over every task, then the head's change against the base. Each figure is the
|
|
median over the side's runs that have figures, and Reads over those that opened the bug's file. The change is not the
|
|
difference of the two All rows: it is the mean of the bugs' own changes."""
|
|
sides = {(task, side): [result for result in results if (result.task, result.side) == (task, side)] for task in tasks for side in SIDES}
|
|
rows = [row(f"{task}: {tasks[task]}", side, sides[task, side]) for task in sorted(tasks) for side in SIDES]
|
|
base, head = ([result for result in results if result.side == side] for side in SIDES)
|
|
return [*HEADER, *rows, row("All", "base", base), row("All", "head", head), change(base, head, [(sides[task, "base"], sides[task, "head"]) for task in sorted(tasks)])]
|
|
|
|
|
|
def medians(results: list[Result]) -> list[float | None]:
|
|
"""Input, cache write, cache read, output, turns, wall time and reads."""
|
|
measured = [result.figures for result in results if result.figures]
|
|
columns: list[list[float]] = [
|
|
[found.tokens.input for found in measured],
|
|
[found.tokens.write_5m + found.tokens.write_1h for found in measured],
|
|
[found.tokens.read for found in measured],
|
|
[found.tokens.output for found in measured],
|
|
[found.turns for found in measured],
|
|
[found.seconds for found in measured],
|
|
[found.reads for found in measured if found.reads is not None],
|
|
]
|
|
return [statistics.median(column) if column else None for column in columns]
|
|
|
|
|
|
def row(task: str, side: str, results: list[Result]) -> str:
|
|
passed = f"{sum(result.passed for result in results)}/{len(results)}"
|
|
return cells([task, side, passed, *("-" if median is None else number(median) for median in medians(results))])
|
|
|
|
|
|
def change(base: list[Result], head: list[Result], bugs: list[tuple[list[Result], list[Result]]]) -> str:
|
|
"""The head's change against the base: the pass count as the difference over every run, and each other figure as the mean
|
|
over the bugs of that bug's own change, which is its head median less its base median. Tokens and wall time show it as a share
|
|
of the base's median, turns and reads as a count. A bug whose either side lacks the figure, or whose base median is zero where
|
|
a share is shown, doesn't count towards that figure's mean. The mean of the bugs' changes swings less than the change of
|
|
the All rows' medians, which lands in whichever bug's runs sit in the middle of the pooled costs."""
|
|
passed = sum(result.passed for result in head) - sum(result.passed for result in base)
|
|
pairs = [list(zip(medians(was), medians(now))) for was, now in bugs]
|
|
# Turns and reads are small counts, where a share misleads.
|
|
means = [mean_change(column, share=index not in (4, 6)) for index, column in enumerate(zip(*pairs))] if pairs else [None] * 7
|
|
return cells(["All", "change", number(passed, signed=True), *(found or "-" for found in means)])
|
|
|
|
|
|
def mean_change(pairs: tuple[tuple[float | None, float | None], ...], share: bool) -> str | None:
|
|
"""The mean over the (base, head) medians of one figure, one pair per bug; None where no bug has both."""
|
|
changes = [(now - was) * (100 / was if share else 1) for was, now in pairs if was is not None and now is not None and (was or not share)]
|
|
if not changes:
|
|
return None
|
|
mean = statistics.mean(changes)
|
|
return f"{number(round(mean), signed=True)} %" if share else number(mean, signed=True)
|
|
|
|
|
|
def number(value: float, signed: bool = False) -> str:
|
|
"""A whole number as it is, another with one decimal; signed, with a + before a rise."""
|
|
text = str(int(value)) if value == int(value) else f"{value:.1f}"
|
|
return f"+{text}" if signed and value > 0 else text
|
|
|
|
|
|
def cells(found: list[str]) -> str:
|
|
return f"| {' | '.join(found)} |"
|