"""What task-bench distils from its runs' streams for the model that gathers plan items: a trace of each run worth reading, and per bug the decoys, the files that misled the runs that failed. No model reads a whole stream unless the digest leaves a question open.""" import collections import re import typing from claude_code.entries import AssistantTurn, parse_entry, Text, tool_calls, ToolCall, ToolResult, ToolResults from reading_cost.measure import reads_of from task_bench.figures import HOME, WORK # How much of the model's text between two calls a trace keeps, and of each line, of a search's pattern or of a shell command. TEXT_LINES = 3 TEXT_WIDTH = 200 COMMAND_WIDTH = 120 CUT = " …" # What Claude Code's Grep and Glob print where nothing matched: "No matches found", "No files found". NO_HITS = re.compile(r"No \w+ found") class Run(typing.NamedTuple): """One run of a bug: the name of its stream's file without the suffix, whether its fix passed, the tokens it spent, none where it didn't finish, and its stream.""" id: str passed: bool tokens: int stream: bytes class Bug(typing.NamedTuple): """A bug as the digest tells it: its number and name, the file the fix changes, what the failing test printed at the base, and its runs at both sides in the order they ran.""" named: str file: str message: str runs: list[Run] def digest(table: list[str], bugs: list[Bug], is_file: typing.Callable[[str], bool]) -> str: """The digest of a bench: the table of its figures, then each bug. is_file says whether a path in the project names a file.""" return "\n".join(["# task-bench digest", "", *table, *(line for bug in bugs for line in ["", *told(bug, is_file)])]) + "\n" def told(bug: Bug, is_file: typing.Callable[[str], bool]) -> list[str]: passing, failing = [run for run in bug.runs if run.passed], [run for run in bug.runs if not run.passed] passed = f"{len(passing)} of the {len(bug.runs)} runs passed." if failing else f"The {counted(len(bug.runs), 'run')} passed." misled = decoys([run.stream for run in failing], bug.file, is_file) if not failing: found = "No run failed, so the bug has no decoys." elif not misled: found = f"No decoys: no file but the one the fix changes was read by more than half the {counted(len(failing), 'failing run')}." else: found = "Decoys, the files more than half the failing runs read that the fix doesn't change:" shown: list[tuple[Run, str]] = [] if failing: shown.append((max(failing, key=lambda run: run.tokens), "the failing run that spent the most tokens")) if passing: # Of an even count, the lower of the two in the middle, so the trace is a run's own. shown.append((sorted(passing, key=lambda run: run.tokens)[(len(passing) - 1) // 2], "the passing run with the median tokens")) return [ f"## Bug {bug.named}", "", "The failing test printed, at the base:", "", *(f" {line}" for line in bug.message.splitlines()), "", f"{passed} {found}", *(["", *(f"- {decoy}" for decoy in misled)] if misled else []), *(line for run, why in shown for line in ["", f"### {run.id}, {why}, {run.tokens}", "", *trace(run.stream)]), ] def decoys(failing: list[bytes], fixed: str, is_file: typing.Callable[[str], bool]) -> list[str]: """The files of the project that more than half of the failing runs' streams read, but for fixed, the file the fix changes: each with how many runs read it, sorted by name.""" read_by = collections.Counter(path for stream in failing for path in files_read(stream)) return [f"`{path}`, read by {count} of the {counted(len(failing), 'failing run')}" for path, count in sorted(read_by.items()) if count * 2 > len(failing) and path != fixed and is_file(path)] def files_read(stream: bytes) -> set[str]: """The paths in the project that the run opened or searched, a folder it searched among them.""" entries = [entry for line in stream.splitlines() if (entry := parse_entry(line))] return {path.removeprefix(WORK + "/") for _, call in tool_calls(entries) for path in reads_of(call, WORK, HOME).read if path.startswith(WORK + "/")} def trace(stream: bytes) -> list[str]: """The run in short: a line per tool call, and between calls the first TEXT_LINES lines of what the model wrote, each after `> `.""" entries = [entry for line in stream.splitlines() if (entry := parse_entry(line))] results = {result.tool_use_id: result for entry in entries if isinstance(entry, ToolResults) for result in entry.results} return [line for entry in entries if isinstance(entry, AssistantTurn) for part in entry.parts for line in (written(part.text) if isinstance(part, Text) else [called(part, results.get(part.id))])] def written(text: str) -> list[str]: lines = [line.strip() for line in text.splitlines() if line.strip()] kept = [cut(line, TEXT_WIDTH) for line in lines[:TEXT_LINES]] if len(lines) > TEXT_LINES and not kept[-1].endswith(CUT): kept[-1] += CUT return [f"> {line}" for line in kept] def called(call: ToolCall, result: ToolResult | None) -> str: given = call.input searched = call.name in ("Glob", "Grep") if call.name == "Read": line = f"Read {in_project(given.get('file_path'))}{lines_asked(given)}" elif searched: line = f'{call.name} "{cut(str(given.get("pattern", "")), COMMAND_WIDTH)}"' + (f" in {in_project(given['path'])}" if given.get("path") else "") elif call.name == "Bash": first, _, rest = str(given.get("command", "")).strip().partition("\n") line = f"Bash {cut(first, COMMAND_WIDTH)}" + (CUT if rest and len(first) <= COMMAND_WIDTH else "") else: line = f"{call.name} {in_project(given['file_path'])}" if given.get("file_path") else call.name if result is None: return line if result.failed: return f"{line} → failed" return f"{line} → {counted(hits(result.text), 'hit')}" if searched else line def lines_asked(given: dict) -> str: """The lines a Read asks for, as `:120-180`; nothing where it asks for the whole file.""" offset, limit = given.get("offset"), given.get("limit") if not isinstance(offset, int) and not isinstance(limit, int): return "" first = offset if isinstance(offset, int) else 1 return f":{first}-{first + limit - 1 if isinstance(limit, int) else ''}" def hits(found: str) -> int: """The lines a search printed, which name a file or a match each.""" return 0 if NO_HITS.fullmatch(found.strip()) else sum(1 for line in found.splitlines() if line.strip()) def in_project(path: object) -> str: return str(path).removeprefix(WORK + "/") def cut(text: str, width: int) -> str: return text if len(text) <= width else text[:width] + CUT def counted(count: int, noun: str) -> str: return f"{count} {noun}{'' if count == 1 else 's'}"