Files
claude-config/bin/task_bench/digest.py
T
2026-11-21 03:56:18 +01:00

146 lines
6.5 KiB
Python

"""What task-bench distils from its runs' streams for the model that gathers plan items: a trace of each run worth reading, and
per bug the decoys, the files that misled the runs that failed. No model reads a whole stream unless the digest leaves a
question open."""
import collections
import re
import typing
from claude_code.entries import AssistantTurn, parse_entry, Text, tool_calls, ToolCall, ToolResult, ToolResults
from reading_cost.measure import reads_of
from task_bench.figures import HOME, WORK
# How much of the model's text between two calls a trace keeps, and of each line, of a search's pattern or of a shell command.
TEXT_LINES = 3
TEXT_WIDTH = 200
COMMAND_WIDTH = 120
CUT = " …"
# What Claude Code's Grep and Glob print where nothing matched: "No matches found", "No files found".
NO_HITS = re.compile(r"No \w+ found")
class Run(typing.NamedTuple):
"""One run of a bug: the name of its stream's file without the suffix, whether its fix passed, the tokens it spent, none where
it didn't finish, and its stream."""
id: str
passed: bool
tokens: int
stream: bytes
class Bug(typing.NamedTuple):
"""A bug as the digest tells it: its number and name, the file the fix changes, what the failing test printed at the base, and
its runs at both sides in the order they ran."""
named: str
file: str
message: str
runs: list[Run]
def digest(table: list[str], bugs: list[Bug], is_file: typing.Callable[[str], bool]) -> str:
"""The digest of a bench: the table of its figures, then each bug. is_file says whether a path in the project names a file."""
return "\n".join(["# task-bench digest", "", *table, *(line for bug in bugs for line in ["", *told(bug, is_file)])]) + "\n"
def told(bug: Bug, is_file: typing.Callable[[str], bool]) -> list[str]:
passing, failing = [run for run in bug.runs if run.passed], [run for run in bug.runs if not run.passed]
passed = f"{len(passing)} of the {len(bug.runs)} runs passed." if failing else f"The {counted(len(bug.runs), 'run')} passed."
misled = decoys([run.stream for run in failing], bug.file, is_file)
if not failing:
found = "No run failed, so the bug has no decoys."
elif not misled:
found = f"No decoys: no file but the one the fix changes was read by more than half the {counted(len(failing), 'failing run')}."
else:
found = "Decoys, the files more than half the failing runs read that the fix doesn't change:"
shown: list[tuple[Run, str]] = []
if failing:
shown.append((max(failing, key=lambda run: run.tokens), "the failing run that spent the most tokens"))
if passing:
# Of an even count, the lower of the two in the middle, so the trace is a run's own.
shown.append((sorted(passing, key=lambda run: run.tokens)[(len(passing) - 1) // 2], "the passing run with the median tokens"))
return [
f"## Bug {bug.named}",
"",
"The failing test printed, at the base:",
"",
*(f" {line}" for line in bug.message.splitlines()),
"",
f"{passed} {found}",
*(["", *(f"- {decoy}" for decoy in misled)] if misled else []),
*(line for run, why in shown for line in ["", f"### {run.id}, {why}, {run.tokens}", "", *trace(run.stream)]),
]
def decoys(failing: list[bytes], fixed: str, is_file: typing.Callable[[str], bool]) -> list[str]:
"""The files of the project that more than half of the failing runs' streams read, but for fixed, the file the fix changes:
each with how many runs read it, sorted by name."""
read_by = collections.Counter(path for stream in failing for path in files_read(stream))
return [f"`{path}`, read by {count} of the {counted(len(failing), 'failing run')}" for path, count in sorted(read_by.items()) if count * 2 > len(failing) and path != fixed and is_file(path)]
def files_read(stream: bytes) -> set[str]:
"""The paths in the project that the run opened or searched, a folder it searched among them."""
entries = [entry for line in stream.splitlines() if (entry := parse_entry(line))]
return {path.removeprefix(WORK + "/") for _, call in tool_calls(entries) for path in reads_of(call, WORK, HOME).read if path.startswith(WORK + "/")}
def trace(stream: bytes) -> list[str]:
"""The run in short: a line per tool call, and between calls the first TEXT_LINES lines of what the model wrote, each after `> `."""
entries = [entry for line in stream.splitlines() if (entry := parse_entry(line))]
results = {result.tool_use_id: result for entry in entries if isinstance(entry, ToolResults) for result in entry.results}
return [line for entry in entries if isinstance(entry, AssistantTurn) for part in entry.parts for line in (written(part.text) if isinstance(part, Text) else [called(part, results.get(part.id))])]
def written(text: str) -> list[str]:
lines = [line.strip() for line in text.splitlines() if line.strip()]
kept = [cut(line, TEXT_WIDTH) for line in lines[:TEXT_LINES]]
if len(lines) > TEXT_LINES and not kept[-1].endswith(CUT):
kept[-1] += CUT
return [f"> {line}" for line in kept]
def called(call: ToolCall, result: ToolResult | None) -> str:
given = call.input
searched = call.name in ("Glob", "Grep")
if call.name == "Read":
line = f"Read {in_project(given.get('file_path'))}{lines_asked(given)}"
elif searched:
line = f'{call.name} "{cut(str(given.get("pattern", "")), COMMAND_WIDTH)}"' + (f" in {in_project(given['path'])}" if given.get("path") else "")
elif call.name == "Bash":
first, _, rest = str(given.get("command", "")).strip().partition("\n")
line = f"Bash {cut(first, COMMAND_WIDTH)}" + (CUT if rest and len(first) <= COMMAND_WIDTH else "")
else:
line = f"{call.name} {in_project(given['file_path'])}" if given.get("file_path") else call.name
if result is None:
return line
if result.failed:
return f"{line} → failed"
return f"{line} → {counted(hits(result.text), 'hit')}" if searched else line
def lines_asked(given: dict) -> str:
"""The lines a Read asks for, as `:120-180`; nothing where it asks for the whole file."""
offset, limit = given.get("offset"), given.get("limit")
if not isinstance(offset, int) and not isinstance(limit, int):
return ""
first = offset if isinstance(offset, int) else 1
return f":{first}-{first + limit - 1 if isinstance(limit, int) else ''}"
def hits(found: str) -> int:
"""The lines a search printed, which name a file or a match each."""
return 0 if NO_HITS.fullmatch(found.strip()) else sum(1 for line in found.splitlines() if line.strip())
def in_project(path: object) -> str:
return str(path).removeprefix(WORK + "/")
def cut(text: str, width: int) -> str:
return text if len(text) <= width else text[:width] + CUT
def counted(count: int, noun: str) -> str:
return f"{count} {noun}{'' if count == 1 else 's'}"