From 089e9b683b065f29a9c5de6fe98f10dac22d2165 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sat, 21 Nov 2026 01:05:50 +0100 Subject: [PATCH] Add task-bench, which has Haiku fix planted bugs at a base and a head and prints what each side took --- .gitignore | 3 + README.md | 4 + bin/mutate_changes/in_diff_tools.py | 27 ++- bin/task-bench | 15 ++ bin/task_bench/__init__.py | 0 bin/task_bench/api_proxy.py | 50 +++++ bin/task_bench/figures.py | 116 +++++++++++ bin/task_bench/main.py | 307 ++++++++++++++++++++++++++++ bin/task_bench/plant.py | 119 +++++++++++ bin/task_bench/runs.py | 171 ++++++++++++++++ bin/task_bench/test_figures.py | 97 +++++++++ bin/task_bench/test_plant.py | 130 ++++++++++++ bin/task_bench/test_task_bench.py | 299 +++++++++++++++++++++++++++ docs/reading-cost.md | 26 +++ 14 files changed, 1355 insertions(+), 9 deletions(-) create mode 100755 bin/task-bench create mode 100644 bin/task_bench/__init__.py create mode 100644 bin/task_bench/api_proxy.py create mode 100644 bin/task_bench/figures.py create mode 100644 bin/task_bench/main.py create mode 100644 bin/task_bench/plant.py create mode 100644 bin/task_bench/runs.py create mode 100644 bin/task_bench/test_figures.py create mode 100644 bin/task_bench/test_plant.py create mode 100644 bin/task_bench/test_task_bench.py diff --git a/.gitignore b/.gitignore index 3b58f19f..9733ff4c 100644 --- a/.gitignore +++ b/.gitignore @@ -155,6 +155,9 @@ !bin/subagent-rung !bin/subagent_rung/ !bin/subagent_rung/*.py +!bin/task-bench +!bin/task_bench/ +!bin/task_bench/*.py !bin/test_*.py !bin/text_files/ !bin/text_files/*.py diff --git a/README.md b/README.md index d72bab15..1580b04a 100644 --- a/README.md +++ b/README.md @@ -225,6 +225,7 @@ Install puts only `loop-claude` on `PATH`; call the others by their path, such a - `settings_filter.py` is git's clean filter for `settings.json`: it drops `"model"` and sorts the keys. `.gitattributes` sets the filter `settings` on `settings.json`, and Install's `git config` line makes this script that filter. - `statusline.py` renders Claude Code's status line; `settings.json` → `statusLine` runs it. - `subagent-rung [--plan] ` prints the Agent tool's model and effort for a subagent's task, picked by its L from the project's model on the project's routing ladder, and with `--plan` never below the project's rung; `subagent-rung --help` states the rule. +- `task-bench --image --test [--base ] [--head ]` plants bugs the tests catch where the comprehension seats found the code hardest, has Haiku fix each at the base and at the head, and prints how often the fix passes and what it took; `task-bench --help` says how it plants, runs and checks. - `todo-item` reads and changes a project's plan, whatever holds it: `list` prints the items in the plan's order and `show ` one item's row and detail; `add …` files an item under the next free ID, `edit …` sets an item's cells or detail, and `move --release ` sets an item's release; `finish ` removes a finished item and adds the bullet passed with `--changelog` to `CHANGELOG.md`, except for a plan kept in Jira. Each change ranks a markdown plan. `todo-item --help` states the flags. - `touch-predictions record ...` appends what a chunk's items were predicted to touch, under `Touches:` in their details, beside the files and parts its branch touched, to the record the config names as `touch-predictions`; `touch-predictions --help` shows the spelling and how a part is named. `chunk-preflight` reads the same predictions to name the items that may start chunks at once. - `wait-for [] [--timeout ]` waits until a file a background job writes holds a line matching the pattern, and prints it. @@ -259,6 +260,7 @@ Install puts only `loop-claude` on `PATH`; call the others by their path, such a - `review_rules/` holds the larv-review rules that several tools apply: its passes, its review folder, its round commit titles and the comprehension history's format. - `routing_figures/` measures loop-claude's archive, and each of its projects' default branches, for loop-claude's routing calls. - `rung_replay/` holds what `judge_replay` and `review_replay` share: the parallel replay runner, the argument reader, the candidate rungs and the exits, and `judge_replay`'s rung search. It reads `agent_cycle`'s retry count and `chunk_driver`'s models file, and finds that file through `loop_state`'s paths. Its `archive.py` keeps each replay's runs and its report in a `replays/` folder of loop-claude's archive. +- `task_bench/` holds `task-bench`: `plant.py` picks the bugs from Gremlins' killed mutants, `runs.py` runs docker for each fix and each check, behind `api_proxy.py`, the one way out of a fix's network, `figures.py` reads a run's stream and prints the table, and `main.py` runs the bench. - `test_env.py` serves the tests. - `text_files/` reads a Markdown file's `##` sections, quotes a passage and joins names into a list, reads the text piped into a command, stamps times in UTC, replaces a file in one step, private where it is new, and gzips a file beside its plain name, by which it reads it afterwards. - `transcript_archive/` stores loop-claude's transcripts. Its `project_key.py` names each project's archive folder, loop-claude session and models-file entry. @@ -273,6 +275,8 @@ CI runs `ci.sh` on every pull request, and `main` doesn't merge anything until i To mutation-test the lines a branch changes, run `~/.claude/bin/run-heavy ~/.claude/bin/mutate-changes --prefix 'uvx --python "$(command -v python3)" --with pytest==9.2.0 --from "mutmut>=3,<4"'` in its checkout. +To bench how hard a Go project's code is to fix a bug in, at a branch's base and at its head, run `~/.claude/bin/run-heavy ~/.claude/bin/task-bench --image --test '' --prefix ''` in its checkout, with a token from `claude setup-token` in `CLAUDE_CODE_OAUTH_TOKEN`. It needs docker, and a comprehension history that names hard places; `task-bench --help` gives a prefix. + ## Tracking a new file `.gitignore` is an allowlist. Track a file by negating it and its parent directories, then confirm with `git check-ignore -v `: diff --git a/bin/mutate_changes/in_diff_tools.py b/bin/mutate_changes/in_diff_tools.py index 010b22ee..68d1d7c1 100644 --- a/bin/mutate_changes/in_diff_tools.py +++ b/bin/mutate_changes/in_diff_tools.py @@ -59,28 +59,37 @@ def run_gremlins(top: str, base: str, run: Run, jobs: int) -> list[str] | Failur """A line with the number of mutants on the changed lines and how many of them Gremlins reports as lived or not covered, then a line per such mutant, as `::: : `. - Gremlins runs in top and mutates the Go lines the branch changes since base. It skips `*_test.go` itself. It reads the diff from - git, so `run` must hold the environment that `with_plain_git_diff` makes. Gremlins exits nonzero on a mutation score below a - threshold the project sets, which its exit code doesn't tell from a run that broke, so any nonzero exit fails the run.""" + Gremlins mutates the Go lines the branch changes since base. It reads the diff from git, so `run` must hold the environment that + `with_plain_git_diff` makes.""" + mutants = gremlins_mutants(top, ["--diff", base], run, jobs) + if isinstance(mutants, Failure): + return mutants + # Gremlins lists the mutants on the unchanged lines of a hunk too, as SKIPPED. It lists the files in no fixed order. + unkilled = sorted(mutant for mutant in mutants if mutant.status in GREMLINS_UNKILLED) + on_changed_lines = sum(mutant.status != "SKIPPED" for mutant in mutants) + return [f"gremlins: {on_changed_lines} mutants on changed lines, {len(unkilled)} not killed.", *(f"{m.path}:{m.line}:{m.column}: {m.status}: {m.mutator}" for m in unkilled)] + + +def gremlins_mutants(top: str, arguments: list[str], run: Run, jobs: int) -> list[Mutant] | Failure: + """Each mutant in the report of `gremlins unleash `, which runs in top. + + Gremlins skips `*_test.go` itself. It exits nonzero on a mutation score below a threshold the project sets, which its exit code + doesn't tell from a run that broke, so any nonzero exit fails the run.""" scratch = scratch_folder(top) if isinstance(scratch, Failure): return scratch try: report = pathlib.Path(scratch, "report.json") - ran = run(["gremlins", "unleash", "--diff", base, "--output", os.path.join(os.path.relpath(scratch, top), report.name), "--workers", str(jobs)], top) + ran = run(["gremlins", "unleash", *arguments, "--output", os.path.join(os.path.relpath(scratch, top), report.name), "--workers", str(jobs)], top) if isinstance(ran, Failure) or ran.code: return ran if isinstance(ran, Failure) else whole_output("Gremlins", ran) # Gremlins writes no report where it finds no mutant, and `null` for a list with no entry. files = (json.loads(report.read_text(encoding="utf-8"))["files"] or []) if report.is_file() else [] - mutants = [Mutant(file["file_name"], mutant["line"], mutant["column"], mutant["status"], mutant["type"]) for file in files for mutant in file["mutations"] or []] + return [Mutant(file["file_name"], mutant["line"], mutant["column"], mutant["status"], mutant["type"]) for file in files for mutant in file["mutations"] or []] except (OSError, ValueError, KeyError, TypeError) as failure: return Failure(f"could not read Gremlins' report in {scratch}: {failure}") finally: remove(scratch) - # Gremlins lists the mutants on the unchanged lines of a hunk too, as SKIPPED. It lists the files in no fixed order. - unkilled = sorted(mutant for mutant in mutants if mutant.status in GREMLINS_UNKILLED) - on_changed_lines = sum(mutant.status != "SKIPPED" for mutant in mutants) - return [f"gremlins: {on_changed_lines} mutants on changed lines, {len(unkilled)} not killed.", *(f"{m.path}:{m.line}:{m.column}: {m.status}: {m.mutator}" for m in unkilled)] def with_plain_git_diff(env: typing.Mapping[str, str]) -> dict[str, str]: diff --git a/bin/task-bench b/bin/task-bench new file mode 100755 index 00000000..5fc76ae1 --- /dev/null +++ b/bin/task-bench @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +import os +import pathlib +import signal +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from task_bench import main + +# Python ends on SIGTERM or SIGHUP without running a finally, which would leave the bench's containers, network and image. As +# SystemExit, the run unwinds as on Ctrl-C. +for number in (signal.SIGHUP, signal.SIGTERM): + signal.signal(number, lambda number, frame: sys.exit(128 + number)) +sys.exit(main.main(sys.argv[1:], os.environ)) diff --git a/bin/task_bench/__init__.py b/bin/task_bench/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bin/task_bench/api_proxy.py b/bin/task_bench/api_proxy.py new file mode 100644 index 00000000..fc1b902e --- /dev/null +++ b/bin/task_bench/api_proxy.py @@ -0,0 +1,50 @@ +"""An HTTP proxy that tunnels `CONNECT` to the one `host:port` its first argument names and answers 403 to every other request. + +task-bench runs it in a container of its own, as `python api_proxy.py []`: the containers Haiku works in sit +on a network with no route out, and reach the API through it and nothing else. It imports only the standard library, since that +container holds nothing more.""" +import asyncio +import contextlib +import sys + +PORT = 8080 + + +async def served(allowed: str, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + """Answers one client: a tunnel to `allowed` where its request is `CONNECT `, else 403.""" + with contextlib.suppress(OSError, asyncio.IncompleteReadError, asyncio.LimitOverrunError): + try: + request = (await reader.readuntil(b"\r\n\r\n")).split(b"\r\n")[0].decode("latin-1").split() + if len(request) != 3 or request[0] != "CONNECT" or request[1] != allowed: + writer.write(b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n") + await writer.drain() + return + host, _, port = allowed.rpartition(":") + there_reader, there_writer = await asyncio.open_connection(host, int(port)) + writer.write(b"HTTP/1.1 200 Connection established\r\n\r\n") + await writer.drain() + # Either side closing ends the tunnel, so the other copy is cancelled rather than left waiting. + copies = [asyncio.ensure_future(copied(reader, there_writer)), asyncio.ensure_future(copied(there_reader, writer))] + await asyncio.wait(copies, return_when=asyncio.FIRST_COMPLETED) + for copy in copies: + copy.cancel() + there_writer.close() + finally: + writer.close() + + +async def copied(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + with contextlib.suppress(OSError): + while data := await reader.read(65536): + writer.write(data) + await writer.drain() + + +async def serve(allowed: str, port: int) -> None: + server = await asyncio.start_server(lambda reader, writer: served(allowed, reader, writer), "0.0.0.0", port) + async with server: + await server.serve_forever() + + +if __name__ == "__main__": + asyncio.run(serve(sys.argv[1], int(sys.argv[2]) if len(sys.argv) > 2 else PORT)) diff --git a/bin/task_bench/figures.py b/bin/task_bench/figures.py new file mode 100644 index 00000000..fbea4fdb --- /dev/null +++ b/bin/task_bench/figures.py @@ -0,0 +1,116 @@ +"""What task-bench measures: the figures of one run, read from the stream its claude printed, and the table of a bench's runs.""" +import dataclasses +import statistics +import typing + +from claude_code.entries import json_object, parse_entry, tool_calls, ToolCall +from claude_code.tokens import Tokens, tokens_of +from failure import Failure +from reading_cost.measure import reads_of + +# Where each run's container mounts its copy of the code, and its home folder: the paths in a run's stream start there. +WORK = "/work" +HOME = "/tmp" +# The tools that read or search without a shell. A Bash call counts where `reads_of` finds a file it reads. +READING_TOOLS = frozenset({"Glob", "Grep", "Read"}) +SIDES = ("base", "head") +HEADER = ["| Task | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |", "|---|---|---|---|---|---|---|---|---|---|"] + + +@dataclasses.dataclass(frozen=True) +class Figures: + """One finished run, as its stream tells it. reads counts the reads and searches before the first open of the file the bug is + in, and is None where the run never opened it.""" + version: str + model: str + tokens: Tokens + turns: int + seconds: float + reads: int | None + + +class Result(typing.NamedTuple): + """One run of a task at a side. figures is None where its claude ended without a result, as when it ran out of time.""" + + task: int + side: str + run: int + passed: bool + figures: Figures | None + + +def figures(stream: bytes, bug_file: str) -> Figures | None | Failure: + """The figures of the run whose claude printed stream, with bug_file the bug's file as the run's container names it. + + None where the stream holds no result. A Failure where the API refused the run, as on a token it doesn't take: such a run + measures nothing, and every later run would end the same.""" + events = [event for line in stream.splitlines() if (event := json_object(line))] + result = next((event for event in reversed(events) if event.get("type") == "result"), None) + if result is None: + return None + if result.get("api_error_status"): + return Failure(f"claude could not use the API: {result.get('result')}") + # {"type": "system", "subtype": "init", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330", ...} + init = next((event for event in events if event.get("subtype") == "init"), {}) + entries = [entry for line in stream.splitlines() if (entry := parse_entry(line))] + calls = [call for _, call in tool_calls(entries)] + return Figures(str(init.get("claude_code_version", "")), str(init.get("model", "")), tokens_of(result.get("usage")), int(result.get("num_turns") or 0), (result.get("duration_ms") or 0) / 1000, reads_before(calls, bug_file)) + + +def reads_before(calls: list[ToolCall], bug_file: str) -> int | None: + count = 0 + for call in calls: + read = reads_of(call, WORK, HOME).read + if bug_file in read: + return count + count += call.name in READING_TOOLS or bool(read) + return None + + +def table(results: list[Result], tasks: dict[int, str]) -> list[str]: + """A row per task and side, then one per side over every task, then the head's change against the base. Each figure is the + median over the side's runs that have figures, and Reads over those that opened the bug's file.""" + rows = [row(f"{task}: {tasks[task]}", side, [result for result in results if (result.task, result.side) == (task, side)]) for task in sorted(tasks) for side in SIDES] + base, head = ([result for result in results if result.side == side] for side in SIDES) + return [*HEADER, *rows, row("All", "base", base), row("All", "head", head), change(base, head)] + + +def medians(results: list[Result]) -> list[float | None]: + """Input, cache write, cache read, output, turns, wall time and reads.""" + measured = [result.figures for result in results if result.figures] + columns: list[list[float]] = [ + [found.tokens.input for found in measured], + [found.tokens.write_5m + found.tokens.write_1h for found in measured], + [found.tokens.read for found in measured], + [found.tokens.output for found in measured], + [found.turns for found in measured], + [found.seconds for found in measured], + [found.reads for found in measured if found.reads is not None], + ] + return [statistics.median(column) if column else None for column in columns] + + +def row(task: str, side: str, results: list[Result]) -> str: + passed = f"{sum(result.passed for result in results)}/{len(results)}" + return cells([task, side, passed, *("-" if median is None else number(median) for median in medians(results))]) + + +def change(base: list[Result], head: list[Result]) -> str: + """The head's row less the base's: counts as they differ, and tokens and wall time as a share of the base's.""" + passed = sum(result.passed for result in head) - sum(result.passed for result in base) + pairs = list(zip(medians(base), medians(head))) + shares = [None if was is None or now is None or not was else f"{number(round(100 * (now - was) / was), signed=True)} %" for was, now in pairs] + counts = [None if was is None or now is None else number(now - was, signed=True) for was, now in pairs] + # Turns and reads are small counts, where a share misleads. + mixed = [*shares[:4], counts[4], shares[5], counts[6]] + return cells(["All", "change", number(passed, signed=True), *(found or "-" for found in mixed)]) + + +def number(value: float, signed: bool = False) -> str: + """A whole number as it is, another with one decimal; signed, with a + before a rise.""" + text = str(int(value)) if value == int(value) else f"{value:.1f}" + return f"+{text}" if signed and value > 0 else text + + +def cells(found: list[str]) -> str: + return f"| {' | '.join(found)} |" diff --git a/bin/task_bench/main.py b/bin/task_bench/main.py new file mode 100644 index 00000000..db6c0926 --- /dev/null +++ b/bin/task_bench/main.py @@ -0,0 +1,307 @@ +r"""Measure how hard a project's code is to fix a bug in, at a base commit and at a head: plant bugs its tests catch, have Haiku fix +each several times at both, and print how often the fix passes and what it took. + +usage: task-bench --image --test [--base ] [--head ] [--prefix ] [--seed ] [--bugs ] [--runs ] + +Run it in a checkout, under ~/.claude/bin/run-heavy, with a long-lived token in CLAUDE_CODE_OAUTH_TOKEN, which `claude setup-token` +prints. The head is --head, by default HEAD. The base is the merge base of the head and --base, by default the default branch: +origin's as last fetched, else main or master. `--base HEAD` benches the head against itself. No model judges anything: the bench +runs programs, and reads Haiku's own stream. + +- Plant. It copies the tree at the base and runs Gremlins there, as `gremlins unleash`, over the Go files that hold the places + `larv-review places` prints, the ones the comprehension seats found hardest. Of the mutants the tests kill, it draws --bugs, 5 by + default, in the order --seed sets, 1 by default. A drawn mutant becomes a bug where its line stands unchanged at the head, and + where, planted at each side, the tests fail on an assertion whose message says something. Others are passed over, so a change to + the very line of a bug is not benched. Go is the one language so far. +- Run. Per bug and side, --runs times, 5 by default, sides in turn: a copy of that side's tree with the bug in it, committed once, + so git shows nothing of the bug. In it, in a container, `claude -p --strict-mcp-config --disable-slash-commands + --no-session-persistence --model claude-haiku-5-5 --effort low` works with every tool, told the failing assertion's message at that + side, without the test's name, file or line. The container's Claude home is new and empty, so the project's own CLAUDE.md, or + AGENTS.md, in the copy is all the setup that loads. Its network reaches api.anthropic.com and nothing else. A fix ends after 30 + minutes. +- Check. The test command runs over the planted tree with each file that is no test as Haiku left it; a fix passes where it exits 0. + Tests Haiku changed, removed or added count for nothing. + +--image is the project's development image, such as golang:1.27.4. The bench adds Claude Code to it, as its installer installs it, +so the image needs curl and bash; the result names the version. Haiku works in that image, and --test runs in it, as sh runs it at +the copy's top, with no network: the image must hold what the tests download otherwise. +--prefix is shell text that runs Gremlins, on the host, as mutate-changes' --prefix does: sh runs ` "$@"` in the copy of the +base, which $PWD names, with `gremlins unleash ...` as "$@". For example: + + docker run --rm --memory 8g --memory-swap 8g -v "$PWD:$PWD" -w "$PWD" golang:1.27.4 \ + sh -c 'shift; exec go run github.com/go-gremlins/gremlins/cmd/gremlins@v0.6.0 "$@"' sh + +It prints the base and the head, then a table: a row per bug and side with how many fixes passed and the medians, over the runs +that finished, of the input, cache-write, cache-read and output tokens, the turns, the wall time, and the reads and searches before +the run first opened the file the bug is in; then a row per side over every bug, and the head's change against the base. Each +run's result and stream are in the folder its last line names, `task-bench` in the chunk's review folder, which each bench empties +first. + +Exit 1, saying why, on an unknown or repeated argument, a missing --image or --test, a count that is no whole number from 1, or +no token; and where git, docker or Gremlins fails, the seats' places name no Go file, the tests fail at a side without a bug, no +mutant becomes a bug, or the API refuses a run. +""" +import dataclasses +import functools +import json +import os +import pathlib +import shutil +import sys +import tarfile +import typing +import uuid + +from claude_code.tokens import Tokens +from failure import Failure +from git_cli import git_repo +from git_cli.git_repo import default_ref, Git, top +from larv_review.history_commands import read_places +from machine_load.slots import CPUS_PER_SLOT +from mutate_changes.tool_process import failed, Ran, run_tool +from review_rules.review_folder import made_review_folder +from run_heavy.capped_docker import SHARE_VARIABLE +from task_bench.figures import Figures, figures, Result, SIDES, table, WORK +from task_bench.plant import drawn, killed_mutants, line_at_head, Mutant, mutated, place_files, task_text +from task_bench.runs import checked_tree, closed, Containers, docker_cli, fix_attempted, MEMORY_MIB, opened, tests_run, TOKEN + +USAGE = "usage: task-bench --image --test [--base ] [--head ] [--prefix ] [--seed ] [--bugs ] [--runs ]" +# A panel of three readers, seated as the README's audience, picked these options on 2026-11-21, each 3 to 0. +OPTIONS = ("--base", "--bugs", "--head", "--image", "--prefix", "--runs", "--seed", "--test") +COUNTS = {"--bugs": "5", "--runs": "5", "--seed": "1"} +# The planted commit names no one, and no hook or signing key of the caller's reaches it. +COMMIT = ["-c", "user.name=task-bench", "-c", "user.email=task-bench@localhost", "-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", "commit", "--quiet", "--message", "Import"] + + +@dataclasses.dataclass(frozen=True) +class Task: + """A bug to fix: its number from 1, the mutant at the base, and per side the planted tree and what Haiku is told there.""" + number: int + mutant: Mutant + trees: dict[str, str] + messages: dict[str, str] + + def named(self) -> str: + return f"{self.mutant.path}:{self.mutant.line}:{self.mutant.column} {self.mutant.mutator}" + + +def main(argv: list[str], env: typing.Mapping[str, str]) -> int: + """Runs task-bench with the arguments argv; git, docker and Gremlins run in the environment env.""" + if argv in (["-h"], ["--help"]): + print(__doc__.strip()) + return 0 + given = options(argv, env) + if isinstance(given, Failure): + print(f"{given.reason}; {USAGE}", file=sys.stderr) + return 1 + home = pathlib.Path(env.get("HOME") or pathlib.Path.home()) + places = read_places(home) + report = places if isinstance(places, Failure) else bench(given, env, places, home) + if isinstance(report, Failure): + print(report.reason, file=sys.stderr) + return 1 + print("\n".join(report)) + return 0 + + +def options(argv: list[str], env: typing.Mapping[str, str]) -> dict[str, str] | Failure: + given: dict[str, str] = {} + rest = list(argv) + while rest: + name, equals, value = rest.pop(0).partition("=") + if name not in OPTIONS or name in given: + return Failure(f"unknown or repeated argument {name}") + if not equals and not rest: + return Failure(f"{name} needs a value") + given[name] = value if equals else rest.pop(0) + for name in ("--image", "--test"): + if not given.get(name): + return Failure(f"{name} is missing") + for name, default in COUNTS.items(): + if not given.setdefault(name, default).isdecimal() or int(given[name]) < 1: + return Failure(f"{name} takes a whole number from 1, not {given[name]}") + if not env.get(TOKEN): + return Failure(f"{TOKEN} holds no token") + return given + + +def bench(given: dict[str, str], env: typing.Mapping[str, str], places: str, home: pathlib.Path) -> list[str] | Failure: + """The report of one bench of the checkout the process runs in, with `places` what `larv-review places` prints there.""" + git = functools.partial(git_repo.git, env=env) + checkout = top(git) + if isinstance(checkout, Failure): + return checkout + sides = commits(given, checkout, git) + if isinstance(sides, Failure): + return sides + print(f"task-bench: base {sides['base'][:12]}, head {sides['head'][:12]}.", file=sys.stderr) + review = made_review_folder(checkout, home) + if isinstance(review, Failure): + return review + folder = review / "task-bench" + containers = Containers(docker_cli(env), f"task-bench-{uuid.uuid4().hex[:12]}", given["--test"], int(env.get(SHARE_VARIABLE) or MEMORY_MIB), f"{os.getuid()}:{os.getgid()}") + try: + shutil.rmtree(folder, ignore_errors=True) + trees: dict[str, str] = {} + for side, commit in sides.items(): + tree = archived(commit, folder / "trees" / side, checkout, git) + if isinstance(tree, Failure): + return tree + trees[side] = tree + files = [path for path in place_files(places) if pathlib.Path(trees["base"], path).is_file()] + if not files: + return Failure("The places the seats found hard name no Go file that the base holds, so there is nowhere to plant a bug.") + run = functools.partial(run_tool, prefix=given.get("--prefix"), env=env, killpg=os.killpg) + mutants = killed_mutants(trees["base"], files, run, CPUS_PER_SLOT) + if isinstance(mutants, Failure): + return mutants + try: + found = opened(containers, given["--image"]) or benched(given, checkout, sides, trees, mutants, containers, folder, git) + finally: + closed(containers) + except OSError as failure: + return Failure(f"could not write under {folder}: {failure}") + return found if isinstance(found, Failure) else [*found, f"Each run's result and stream: {folder}"] + + +def commits(given: dict[str, str], checkout: str, git: Git) -> dict[str, str] | Failure: + """The base's commit and the head's.""" + head = git("rev-parse", "--verify", f"{given.get('--head', 'HEAD')}^{{commit}}", cwd=checkout) + against = given.get("--base") or default_ref(cwd=checkout, run=git) + if isinstance(head, Failure): + return head + if isinstance(against, Failure): + return against + base = git("merge-base", against, head.strip(), cwd=checkout) + return base if isinstance(base, Failure) else {"base": base.strip(), "head": head.strip()} + + +def archived(commit: str, into: pathlib.Path, checkout: str, git: Git) -> str | Failure: + """Makes `into` the tree at commit, without git's own data, and returns its path.""" + into.mkdir(parents=True) + archive = into.with_suffix(".tar") + wrote = git("archive", "--format=tar", "--output", str(archive), commit, cwd=checkout) + if isinstance(wrote, Failure): + return wrote + try: + with tarfile.open(archive) as held: + held.extractall(into, filter="tar") + except tarfile.TarError as failure: + return Failure(f"could not unpack the tree at {commit}: {failure}") + archive.unlink() + return str(into) + + +def benched(given: dict[str, str], checkout: str, sides: dict[str, str], trees: dict[str, str], mutants: list[Mutant], containers: Containers, folder: pathlib.Path, git: Git) -> list[str] | Failure: + """Plants the bugs, runs each fix, writes the results into folder and returns the report.""" + for side, tree in trees.items(): + ran = tests_in_copy(containers, tree, folder) + if isinstance(ran, Failure) or ran.code: + return ran if isinstance(ran, Failure) else failed(f"The tests fail at the {side} without a planted bug: the test command", ran) + tasks: list[Task] = [] + for mutant in drawn(mutants, int(given["--seed"])): + if len(tasks) == int(given["--bugs"]): + break + task = planted(len(tasks) + 1, mutant, checkout, sides, trees, containers, folder, git) + if isinstance(task, Failure): + return task + tasks += [task] if task else [] + if not tasks: + return Failure(f"None of the {len(mutants)} mutants the tests kill becomes a bug: each is on a line the head changed, or fails no assertion with a message.") + results: list[Result] = [] + total = int(given["--runs"]) * len(tasks) * len(SIDES) + for number in range(1, int(given["--runs"]) + 1): + for task in tasks: + for side in SIDES: + result = fixed(task, side, number, containers, folder) + if isinstance(result, Failure): + return result + results.append(result) + print(f"task-bench: run {len(results)} of {total}, bug {task.number} at the {side}: {'passed' if result.passed else 'not passed'}.", file=sys.stderr) + (folder / "results.json").write_text(json.dumps({"sides": sides, "seed": int(given["--seed"]), "bugs": {task.number: task.named() for task in tasks}, "runs": [recorded(result) for result in results]}, indent="\t") + "\n", encoding="utf-8") + ran_on = sorted({f"{found.model} in Claude Code {found.version}" for result in results if (found := result.figures)}) + short = [] if len(tasks) == int(given["--bugs"]) else [f"Planted {len(tasks)} of the {given['--bugs']} bugs asked for: no other mutant the tests kill becomes one."] + return [ + f"task-bench: base {sides['base'][:12]}, head {sides['head'][:12]}. {counted(len(tasks), 'bug')}, fixed {counted(int(given['--runs']), 'time')} at each side by {' and '.join(ran_on) or 'no run that finished'}.", + *short, + "", + *table(results, {task.number: task.named() for task in tasks}), + "", + ] + + +def planted(number: int, mutant: Mutant, checkout: str, sides: dict[str, str], trees: dict[str, str], containers: Containers, folder: pathlib.Path, git: Git) -> Task | Failure | None: + """The task of fixing mutant, with its planted tree at each side; None where the mutant is no bug at both sides. + + The bug is the same change to the same line at both sides, so only the code around it differs between them.""" + diff = git("diff", "-U0", "--no-color", "--no-ext-diff", "--no-textconv", sides["base"], sides["head"], "--", mutant.path, cwd=checkout, errors="replace") + if isinstance(diff, Failure): + return diff + at_head = line_at_head(diff, mutant.line) + if at_head is None: + return None + planted_trees, messages = {}, {} + for side, placed in (("base", mutant), ("head", mutant._replace(line=at_head))): + source = pathlib.Path(trees[side], mutant.path) + changed = mutated(source.read_bytes(), placed) if source.is_file() else None + if changed is None: + return None + tree = folder / "bugs" / str(number) / side + shutil.rmtree(tree, ignore_errors=True) + shutil.copytree(trees[side], tree, symlinks=True) + (tree / mutant.path).write_bytes(changed) + ran = tests_in_copy(containers, str(tree), folder) + if isinstance(ran, Failure): + return ran + message = task_text(ran.output) if ran.code else None + if message is None: + return None + for args in (["init", "--quiet"], ["add", "--all", "--force"], COMMIT): + done = git(*args, cwd=str(tree)) + if isinstance(done, Failure): + return done + planted_trees[side], messages[side] = str(tree), message + return Task(number, mutant, planted_trees, messages) + + +def tests_in_copy(containers: Containers, tree: str, folder: pathlib.Path) -> Ran | Failure: + """The test command's run in a copy of tree that goes afterwards, so what the tests write stays out of tree.""" + copy = folder / "tested" + shutil.copytree(tree, copy, symlinks=True) + try: + return tests_run(containers, str(copy)) + finally: + shutil.rmtree(copy, ignore_errors=True) + + +def fixed(task: Task, side: str, number: int, containers: Containers, folder: pathlib.Path) -> Result | Failure: + """One run: Haiku's fix of task at side in a copy of its planted tree, then the tests over what it changed. The stream stays in + folder; the copy goes.""" + stream = folder / "runs" / f"bug-{task.number}-{side}-{number}.jsonl" + stream.parent.mkdir(exist_ok=True) + copy, check = folder / "copy", folder / "check" + try: + shutil.copytree(task.trees[side], copy, symlinks=True) + attempted = fix_attempted(containers, str(copy), task.messages[side], stream) + if isinstance(attempted, Failure): + return attempted + found = figures(stream.read_bytes(), os.path.join(WORK, task.mutant.path)) + if isinstance(found, Failure): + return found + checked_tree(task.trees[side], str(copy), str(check)) + ran = tests_run(containers, str(check)) + return Result(task.number, side, number, not isinstance(ran, Failure) and ran.code == 0, found) + finally: + shutil.rmtree(copy, ignore_errors=True) + shutil.rmtree(check, ignore_errors=True) + + +def recorded(result: Result) -> dict: + """A run as results.json holds it.""" + found = result.figures or Figures("", "", Tokens(), 0, 0, None) + measured = {"claude_code": found.version, "model": found.model, "tokens": dataclasses.asdict(found.tokens), "turns": found.turns, "seconds": found.seconds, "reads": found.reads} + return {"bug": result.task, "side": result.side, "run": result.run, "passed": result.passed, "finished": result.figures is not None, **(measured if result.figures else {})} + + +def counted(count: int, noun: str) -> str: + return f"{count} {noun}{'' if count == 1 else 's'}" diff --git a/bin/task_bench/plant.py b/bin/task_bench/plant.py new file mode 100644 index 00000000..a561171a --- /dev/null +++ b/bin/task_bench/plant.py @@ -0,0 +1,119 @@ +"""Pick the bugs task-bench plants: Gremlins' killed mutants in the Go files that hold the places the comprehension seats found hard, +re-applied from its report, each with the failing assertion's message that tells Haiku what goes wrong.""" +import os +import random +import re +import typing + +from failure import Failure +from mutate_changes.changed_lines import is_test +from mutate_changes.in_diff_tools import gremlins_mutants +from mutate_changes.tool_process import Run + +# A place as `larv-review places` prints it: `:` in backticks. +PLACE = re.compile(r"`([^`\s:]+):\d+`") +# What each of Gremlins v0.6.0's mutators writes in place of a token: `tokenMutations` in its internal/engine/mappings.go, under the +# names its report gives the mutators, from internal/mutator/mutator.go. +MUTATIONS: dict[str, dict[bytes, bytes]] = { + "ARITHMETIC_BASE": {b"+": b"-", b"*": b"/", b"/": b"*", b"%": b"*", b"-": b"+"}, + "CONDITIONALS_BOUNDARY": {b">=": b">", b">": b">=", b"<=": b"<", b"<": b"<="}, + "CONDITIONALS_NEGATION": {b"==": b"!=", b">=": b"<", b">": b"<=", b"<=": b">", b"<": b">=", b"!=": b"=="}, + "INCREMENT_DECREMENT": {b"--": b"++", b"++": b"--"}, + "INVERT_ASSIGNMENTS": {b"+=": b"-=", b"*=": b"/=", b"/=": b"*=", b"-=": b"+="}, + "INVERT_BITWISE": {b"&": b"|", b"|": b"&", b"^": b"&", b"&^": b"&", b"<<": b">>", b">>": b"<<"}, + "INVERT_BWASSIGN": {b"&=": b"|=", b"|=": b"&=", b"^=": b"&=", b"&^=": b"&=", b"<<=": b">>=", b">>=": b"<<="}, + "INVERT_LOGICAL": {b"&&": b"||", b"||": b"&&"}, + "INVERT_LOOPCTRL": {b"break": b"continue", b"continue": b"break"}, + "INVERT_NEGATIVES": {b"-": b"+"}, + "REMOVE_SELF_ASSIGNMENTS": dict.fromkeys((b"+=", b"&=", b"&^=", b"*=", b"|=", b"/=", b"%=", b"<<=", b">>=", b"-=", b"^="), b"="), +} +HUNK = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@", re.MULTILINE) +# go test's report of a failed assertion: ` pick_test.go:42: got 3, want 4`, under the `--- FAIL: TestPick (0.00s)` of its test. +ASSERTION = re.compile(r"^( *)\S*_test\.go:\d+:(.*)$") +FAILED_TEST = re.compile(r"^\s*--- FAIL: (\S+)", re.MULTILINE) + + +class Mutant(typing.NamedTuple): + """A mutant Gremlins' tests killed: the mutator and the token it changed, by its line and its column in bytes, both from 1.""" + + path: str + line: int + column: int + mutator: str + + +def place_files(places: str) -> list[str]: + """The Go files that hold a place in what `larv-review places` prints, test files left out, in the order of their paths.""" + return sorted({path for path in PLACE.findall(places) if path.endswith(".go") and not is_test(path)}) + + +def killed_mutants(top: str, files: list[str], run: Run, jobs: int) -> list[Mutant] | Failure: + """The mutants in `files` that the tests of the tree at top kill, as Gremlins finds them there. + + Gremlins takes no list of files to mutate, only patterns to leave out, so it gets one for each other Go file of the tree.""" + others = sorted(path for path in go_files(top) if path not in files) + found = gremlins_mutants(top, [argument for path in others for argument in ("--exclude-files", f"^{re.escape(path)}$")], run, jobs) + if isinstance(found, Failure): + return found + return [Mutant(mutant.path, mutant.line, mutant.column, mutant.mutator) for mutant in found if mutant.status == "KILLED" and mutant.path in files] + + +def go_files(top: str) -> list[str]: + """Each Go file under top that Gremlins would mutate, by its path from top.""" + paths = (os.path.relpath(os.path.join(folder, name), top) for folder, _, names in os.walk(top) for name in names) + return [path for path in paths if path.endswith(".go") and not is_test(path)] + + +def drawn(mutants: list[Mutant], seed: int) -> list[Mutant]: + """The mutants in the order the seed draws them, whatever order Gremlins listed them in.""" + return random.Random(seed).sample(sorted(set(mutants)), len(set(mutants))) + + +def mutated(source: bytes, mutant: Mutant) -> bytes | None: + """A file's source with the mutant applied, or None where its line and column hold no token its mutator changes.""" + lines = source.split(b"\n") + if not 1 <= mutant.line <= len(lines): + return None + line = lines[mutant.line - 1] + rest = line[mutant.column - 1:] + # The longest token first: `>=` holds `>`, and Gremlins names the mutant by the whole token. + token = max((token for token in MUTATIONS.get(mutant.mutator, {}) if rest.startswith(token)), key=len, default=None) + if token is None: + return None + lines[mutant.line - 1] = line[:mutant.column - 1] + MUTATIONS[mutant.mutator][token] + rest[len(token):] + return b"\n".join(lines) + + +def line_at_head(diff: str, line: int) -> int | None: + """The number at the head of a file's line `line` at the base, or None where the head changed or removed that line. + + diff is `git diff -U0 -- `, so each hunk holds changed lines only.""" + moved = 0 + for old_start, old_count, new_start, new_count in ((int(a), int(b or 1), int(c), int(d or 1)) for a, b, c, d in HUNK.findall(diff)): + # A hunk that only adds names the line it adds after, and one that only removes the line the gap is after. + old_end = old_start + old_count if old_count else old_start + 1 + new_end = new_start + new_count if new_count else new_start + 1 + if old_count and old_start <= line < old_end: + return None + if line >= old_end: + moved = new_end - old_end + return line + moved + + +def task_text(output: str) -> str | None: + """What Haiku is told of a planted bug: the message of the first assertion that failed in go test's output, without the test's + name, file or line, so it says what goes wrong and not where. None where no assertion failed, as on a panic, a build that + broke or a passing run, or where its message says nothing beyond the test.""" + lines = output.splitlines() + # A subtest is named `TestPick/empty_input`; a message may name the whole or the test it belongs to. + tests = {part for name in FAILED_TEST.findall(output) for part in (name, name.partition("/")[0])} + for index, line in enumerate(lines): + if found := ASSERTION.match(line): + indent = len(found[1]) + # A message's later lines are indented deeper than its first. + later = lines[index + 1:] + ends = next((at for at, text in enumerate(later) if len(text) - len(text.lstrip(" ")) <= indent), len(later)) + message = [text.strip() for text in (found[2], *later[:ends])] + kept = [text for text in message if text and "_test.go" not in text and not any(test in text for test in tests)] + return "\n".join(kept) or None + return None diff --git a/bin/task_bench/runs.py b/bin/task_bench/runs.py new file mode 100644 index 00000000..ba5c02a6 --- /dev/null +++ b/bin/task_bench/runs.py @@ -0,0 +1,171 @@ +"""Run task-bench's containers: the image that adds Claude Code to the project's, the network that reaches only the API, one fix +by Haiku in a copy of the code, and the project's tests over what it changed.""" +import contextlib +import dataclasses +import os +import pathlib +import shutil +import subprocess +import typing + +from failure import Failure +from mutate_changes.changed_lines import is_test +from mutate_changes.tool_process import failed, Ran +from task_bench.api_proxy import PORT +from task_bench.figures import HOME, WORK + +MODEL = "claude-haiku-5-5" +# No MCP server, skill, command or stored session, so a run starts from the code alone; -p needs --verbose with stream-json, and no +# one answers a permission prompt. Not --bare, which can't sign in by a long-lived token: docs/decisions.md → `Claude Code in a home +# of its own`. +CLAUDE = ["claude", "-p", "--strict-mcp-config", "--disable-slash-commands", "--no-session-persistence", "--model", MODEL, "--effort", "low", "--output-format", "stream-json", "--verbose", "--permission-mode", "bypassPermissions"] +PROMPT = "A test of this project fails with the message below. Find the bug in the code and fix it. Change no test.\n\n{message}" +TOKEN = "CLAUDE_CODE_OAUTH_TOKEN" +API = "api.anthropic.com:443" +# Holds only the proxy, which needs Python's standard library and nothing else. +PROXY_IMAGE = "python:3.15.2-slim" +# The installer puts claude in root's home, which no other user reads; each run works as the caller's user, so files it writes +# in the copy are the caller's. +DOCKERFILE = 'FROM {image}\nUSER root\nRUN curl -fsSL https://claude.ai/install.sh | bash && cp -L /root/.local/bin/claude /usr/local/bin/claude\n' +BUILD_SECONDS = 20 * 60 +# How long one fix, or one run of the tests, may take before its container is removed; the run then counts as not passed. +RUN_SECONDS = 30 * 60 +# The memory each container gets without run-heavy, which hands its slot's share in RUN_HEAVY_MEMORY, in MiB. +MEMORY_MIB = 4096 + + +class Docker(typing.Protocol): + """Runs `docker ` and waits for it, at most `seconds`. Its output goes to `stdout` and its errors to `stderr` where the + call gives them, and into the Ran otherwise.""" + + def __call__(self, args: list[str], *, seconds: float, stdin: str | None = None, stdout: typing.IO[bytes] | None = None, stderr: typing.IO[bytes] | None = None) -> Ran | Failure: ... + + +def docker_cli(env: typing.Mapping[str, str]) -> Docker: + def docker(args: list[str], *, seconds: float, stdin: str | None = None, stdout: typing.IO[bytes] | None = None, stderr: typing.IO[bytes] | None = None) -> Ran | Failure: + try: + ran = subprocess.run(["docker", *args], env=env, input=None if stdin is None else stdin.encode(), stdin=subprocess.DEVNULL if stdin is None else None, stdout=stdout or subprocess.PIPE, stderr=stderr or subprocess.STDOUT, timeout=seconds, check=False) + except OSError as failure: + return Failure(f"docker could not start: {failure.strerror}") + except subprocess.TimeoutExpired: + return Failure(f"docker {args[0]} still ran after {round(seconds / 60)} min") + return Ran(ran.returncode, (ran.stdout or b"").decode("utf-8", errors="replace")) + + return docker + + +@dataclasses.dataclass(frozen=True) +class Containers: + """What one bench holds in docker, all named after `name`, which no other bench shares.""" + docker: Docker + name: str + test: str + memory_mib: int + user: str + + @property + def image(self) -> str: + return f"{self.name}:throwaway" + + @property + def proxy(self) -> str: + return f"{self.name}-proxy" + + @property + def running(self) -> str: + """The one container that fixes or tests at a time.""" + return f"{self.name}-run" + + def limits(self) -> list[str]: + return ["--memory", f"{self.memory_mib}m", "--memory-swap", f"{self.memory_mib}m"] + + +def opened(containers: Containers, image: str) -> Failure | None: + """Builds the bench's image, the project's with Claude Code added, and starts the network and its proxy. + + The network has no route out. The proxy sits on it and on docker's default network, and tunnels to the API alone, so a run + reaches the API and nothing else: not a package index, and not the repository the code came from.""" + docker, name = containers.docker, containers.name + steps: list[tuple[list[str], float, str | None]] = [ + (["build", "--quiet", *containers.limits(), "--tag", containers.image, "-"], BUILD_SECONDS, DOCKERFILE.format(image=image)), + (["network", "create", "--internal", name], 60, None), + (["run", "--detach", "--rm", "--name", containers.proxy, "--memory", "256m", "--memory-swap", "256m", "--volume", f"{pathlib.Path(__file__).with_name('api_proxy.py')}:/api_proxy.py:ro", PROXY_IMAGE, "python", "/api_proxy.py", API], 300, None), + (["network", "connect", name, containers.proxy], 60, None), + ] + for args, seconds, stdin in steps: + ran = docker(args, seconds=seconds, stdin=stdin) + if isinstance(ran, Failure) or ran.code: + return ran if isinstance(ran, Failure) else failed(f"docker {args[0]}", ran) + return None + + +def closed(containers: Containers) -> None: + """Removes what `opened` made and any container still running, where each exists; a bench that broke half-way leaves the rest.""" + for args in (["rm", "--force", containers.running], ["rm", "--force", containers.proxy], ["network", "rm", containers.name], ["image", "rm", containers.image]): + containers.docker(args, seconds=120) + + +def tests_run(containers: Containers, tree: str) -> Ran | Failure: + """The project's test command, run at the top of tree with no network; the tests pass where it exits 0. A Failure where it still + ran after RUN_SECONDS.""" + ran = containers.docker(["run", "--rm", "--name", containers.running, "--network", "none", *in_tree(containers, tree), containers.image, "sh", "-c", containers.test], seconds=RUN_SECONDS) + if isinstance(ran, Failure): + containers.docker(["rm", "--force", containers.running], seconds=120) + return ran + + +def fix_attempted(containers: Containers, copy: str, message: str, stream: pathlib.Path) -> Failure | None: + """Has Haiku fix the bug in copy, told only the failing assertion's message, and writes what its claude prints to `stream`, and its + errors beside it. A fix that still runs after RUN_SECONDS ends there, with the stream as far as it came. + + The container's home is new and empty, so no instruction, memory, hook, skill or plugin of the caller loads; and docker hands the + token on by its name, so its value is on no command line. A Failure where docker itself could not run.""" + args = ["run", "--rm", "--name", containers.running, "--network", containers.name, *in_tree(containers, copy), "--env", TOKEN, "--env", f"HTTPS_PROXY=http://{containers.proxy}:{PORT}", "--env", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1", containers.image, *CLAUDE, PROMPT.format(message=message)] + try: + with stream.open("wb") as out, stream.with_suffix(".stderr").open("wb") as errors: + ran = containers.docker(args, seconds=RUN_SECONDS, stdout=out, stderr=errors) + except OSError as failure: + return Failure(f"could not write {stream}: {failure}") + if isinstance(ran, Failure): + containers.docker(["rm", "--force", containers.running], seconds=120) + # Out of time is the run's outcome, not the bench's failure. + return None if "still ran" in ran.reason else ran + return None + + +def in_tree(containers: Containers, tree: str) -> list[str]: + """The options that run a container in tree, mounted at WORK on every run so no path tells the sides apart, as the caller's user.""" + return [*containers.limits(), "--user", containers.user, "--env", f"HOME={HOME}", "--volume", f"{tree}:{WORK}", "--workdir", WORK] + + +def checked_tree(planted: str, copy: str, into: str) -> None: + """Makes at `into` the tree the tests check a fix in: `planted`, the code with the bug as Haiku got it, with each file that is no + test as `copy` holds it after the fix. So a test Haiku changed, removed or added counts for nothing. + + Raises OSError where a file cannot be copied.""" + shutil.copytree(planted, into, symlinks=True) + before, after = set(entries(planted)), set(entries(copy)) + for path in before - after: + if not is_test(path): + with contextlib.suppress(FileNotFoundError): + os.unlink(os.path.join(into, path)) + for path in sorted(after): + target = os.path.join(into, path) + # A link is copied as a link, never followed; and one that would stand in for a folder of the planted tree is left out. + if is_test(path) or (os.path.isdir(target) and not os.path.islink(target)): + continue + os.makedirs(os.path.dirname(target), exist_ok=True) + with contextlib.suppress(FileNotFoundError): + os.unlink(target) + shutil.copy2(os.path.join(copy, path), target, follow_symlinks=False) + + +def entries(top: str) -> list[str]: + """Each file and link under top, by its path from top, without git's own under `.git`.""" + found = [] + for folder, folders, names in os.walk(top): + if folder == top and ".git" in folders: + folders.remove(".git") + links = [name for name in folders if os.path.islink(os.path.join(folder, name))] + found += [os.path.relpath(os.path.join(folder, name), top) for name in (*names, *links)] + return found diff --git a/bin/task_bench/test_figures.py b/bin/task_bench/test_figures.py new file mode 100644 index 00000000..757e3667 --- /dev/null +++ b/bin/task_bench/test_figures.py @@ -0,0 +1,97 @@ +"""task-bench's figures: what it reads from one run's stream, and the table it prints for a bench.""" +import json +import pathlib +import sys +import unittest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) + +from claude_code.tokens import Tokens +from failure import Failure +from task_bench.figures import Figures, figures, Result, table + +INIT = {"type": "system", "subtype": "init", "cwd": "/work", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330"} +USAGE = {"input_tokens": 40, "cache_creation_input_tokens": 9000, "cache_read_input_tokens": 52000, "output_tokens": 800, "cache_creation": {"ephemeral_1h_input_tokens": 9000, "ephemeral_5m_input_tokens": 0}} +RESULT = {"type": "result", "subtype": "success", "is_error": False, "num_turns": 7, "duration_ms": 61500, "usage": USAGE, "result": "Fixed."} + + +def call(name: str, **given: str) -> dict: + return {"type": "assistant", "message": {"id": f"msg_{name}_{sorted(given.values())}", "model": "claude-haiku-5-5", "content": [{"type": "tool_use", "id": "toolu_1", "name": name, "input": given}], "usage": {}}} + + +def stream(*events: dict) -> bytes: + return "\n".join(map(json.dumps, events)).encode() + b"\n" + + +class FiguresTest(unittest.TestCase): + def test_reads_the_version_model_tokens_turns_and_wall_time_from_the_streams_first_and_last_events(self): + found = figures(stream(INIT, RESULT), "/work/calc.go") + self.assertEqual(found, Figures("2.1.330", "claude-haiku-5-5", Tokens(input=40, write_1h=9000, read=52000, output=800), 7, 61.5, None)) + + def test_counts_the_reads_and_searches_before_the_first_open_of_the_file_the_bug_is_in(self): + calls = [ + call("Grep", pattern="Pick"), + call("Read", file_path="/work/README.md"), + call("Bash", command="go test ./..."), + call("Bash", command="cat render/pick.go | head -40"), + call("Glob", pattern="**/*.go"), + call("Read", file_path="/work/calc.go"), + call("Read", file_path="/work/other.go"), + call("Edit", file_path="/work/calc.go", old_string="a", new_string="b"), + ] + found = figures(stream(INIT, *calls, RESULT), "/work/calc.go") + self.assertEqual(found.reads, 4) + + def test_a_shell_command_that_reads_the_file_opens_it(self): + found = figures(stream(INIT, call("Read", file_path="/work/README.md"), call("Bash", command="sed -n 1,40p calc.go"), RESULT), "/work/calc.go") + self.assertEqual(found.reads, 1) + + def test_has_no_reads_where_the_run_never_opened_the_file(self): + self.assertIsNone(figures(stream(INIT, call("Read", file_path="/work/README.md"), RESULT), "/work/calc.go").reads) + + def test_a_stream_that_ends_without_a_result_has_no_figures(self): + self.assertIsNone(figures(stream(INIT, call("Read", file_path="/work/calc.go")), "/work/calc.go")) + self.assertIsNone(figures(b"", "/work/calc.go")) + + def test_a_run_the_api_refused_is_a_failure_that_quotes_claude(self): + refused = {**RESULT, "is_error": True, "api_error_status": 401, "result": "Failed to authenticate. API Error: 401 OAuth access token is invalid."} + self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: Failed to authenticate. API Error: 401 OAuth access token is invalid.")) + + +def run(task: int, side: str, number: int, passed: bool, input_: int = 100, turns: int = 10, seconds: float = 60, reads: int | None = 3) -> Result: + return Result(task, side, number, passed, Figures("2.1.330", "claude-haiku-5-5", Tokens(input=input_, write_5m=20, write_1h=980, read=5000, output=400), turns, seconds, reads)) + + +class TableTest(unittest.TestCase): + def test_prints_a_row_per_task_and_side_with_the_pass_count_and_each_median_then_all_tasks_and_the_heads_change(self): + results = [ + run(1, "base", 1, True, input_=100, turns=10, seconds=60, reads=3), + run(1, "base", 2, False, input_=300, turns=20, seconds=120, reads=None), + run(1, "base", 3, True, input_=200, turns=12, seconds=90, reads=5), + run(1, "head", 1, True, input_=100, turns=8, seconds=45, reads=1), + run(1, "head", 2, True, input_=100, turns=8, seconds=45, reads=1), + run(1, "head", 3, True, input_=160, turns=9, seconds=50, reads=2), + run(2, "base", 1, False, input_=200, turns=12, seconds=90, reads=4), + run(2, "head", 1, True, input_=100, turns=9, seconds=45, reads=2), + ] + self.assertEqual(table(results, {1: "calc.go:4:7 CONDITIONALS_NEGATION", 2: "pick.go:9:3 ARITHMETIC_BASE"}), [ + "| Task | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |", + "|---|---|---|---|---|---|---|---|---|---|", + "| 1: calc.go:4:7 CONDITIONALS_NEGATION | base | 2/3 | 200 | 1000 | 5000 | 400 | 12 | 90 | 4 |", + "| 1: calc.go:4:7 CONDITIONALS_NEGATION | head | 3/3 | 100 | 1000 | 5000 | 400 | 8 | 45 | 1 |", + "| 2: pick.go:9:3 ARITHMETIC_BASE | base | 0/1 | 200 | 1000 | 5000 | 400 | 12 | 90 | 4 |", + "| 2: pick.go:9:3 ARITHMETIC_BASE | head | 1/1 | 100 | 1000 | 5000 | 400 | 9 | 45 | 2 |", + "| All | base | 2/4 | 200 | 1000 | 5000 | 400 | 12 | 90 | 4 |", + "| All | head | 4/4 | 100 | 1000 | 5000 | 400 | 8.5 | 45 | 1.5 |", + "| All | change | +2 | -50 % | 0 % | 0 % | 0 % | -3.5 | -50 % | -2.5 |", + ]) + + def test_a_run_without_figures_counts_as_not_passed_and_a_side_without_any_shows_a_dash(self): + results = [Result(1, "base", 1, False, None), run(1, "head", 1, True)] + self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[2:], [ + "| 1: calc.go:4:7 ARITHMETIC_BASE | base | 0/1 | - | - | - | - | - | - | - |", + "| 1: calc.go:4:7 ARITHMETIC_BASE | head | 1/1 | 100 | 1000 | 5000 | 400 | 10 | 60 | 3 |", + "| All | base | 0/1 | - | - | - | - | - | - | - |", + "| All | head | 1/1 | 100 | 1000 | 5000 | 400 | 10 | 60 | 3 |", + "| All | change | +1 | - | - | - | - | - | - | - |", + ]) diff --git a/bin/task_bench/test_plant.py b/bin/task_bench/test_plant.py new file mode 100644 index 00000000..059876b2 --- /dev/null +++ b/bin/task_bench/test_plant.py @@ -0,0 +1,130 @@ +"""task-bench's planter: which files it mutates, which mutants it draws, how it re-applies one and what it tells Haiku.""" +import pathlib +import sys +import unittest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) + +from task_bench.plant import drawn, line_at_head, Mutant, mutated, place_files, task_text + +PLACES = """## 2026-11-19T22:33:15Z, PR #646 at a357cc7b + +Junior A, hardest first: + +- Locality, Shape: `internal/render/pick.go:287` `parked`: three steps held at once +- Shape: `internal/render/pick_test.go:12` `TestPick`: a long table +- Self-sufficiency: `README.md:89` "Round cap": three sections +- Locality: `cli/report.go:41` the comment: points to `cli/gone.go` + +Mid A, hardest first: + +- Locality: `internal/render/pick.go:354` `resumed`: a loop +- Locality: `data-import/geo.py:3` `load`: a long chain +""" +# go test's report of an assertion that failed in a subtest, then of one whose message runs over two lines. +GO_FAILURE = """ok \texample.com/calc/cli\t0.004s +--- FAIL: TestPick (0.00s) + --- FAIL: TestPick/empty_input (0.00s) + pick_test.go:42: Pick() = 3, + want 4 + pick_test.go:50: got 1 row, want 2 +FAIL +FAIL\texample.com/calc/render\t0.012s +FAIL +""" +# testify prints the test's file and name inside the message. +TESTIFY_FAILURE = """--- FAIL: TestTotal (0.00s) + total_test.go:12: + \tError Trace:\t/work/total_test.go:12 + \tError: \tNot equal: + \t \texpected: 1 + \t \tactual : 2 + \tTest: \tTestTotal +FAIL +""" +PANIC = """--- FAIL: TestPick (0.00s) +panic: runtime error: index out of range [3] with length 3 [recovered] +FAIL\texample.com/calc/render\t0.012s +""" + + +class PlaceFilesTest(unittest.TestCase): + def test_names_each_go_file_that_holds_a_place_once_and_no_test_file_or_other_language(self): + self.assertEqual(place_files(PLACES), ["cli/report.go", "internal/render/pick.go"]) + + def test_names_none_where_the_seats_found_no_places(self): + self.assertEqual(place_files("Seats found no places hard."), []) + + +class DrawnTest(unittest.TestCase): + MUTANTS = [Mutant("a.go", line, 3, "ARITHMETIC_BASE") for line in range(1, 40)] + + def test_a_seed_draws_the_same_order_whatever_order_gremlins_listed_the_mutants_in(self): + self.assertEqual(drawn(self.MUTANTS, 7), drawn(self.MUTANTS[::-1], 7)) + + def test_draws_every_mutant_once(self): + self.assertEqual(sorted(drawn(self.MUTANTS, 7)), self.MUTANTS) + + def test_another_seed_draws_another_order(self): + self.assertNotEqual(drawn(self.MUTANTS, 7), drawn(self.MUTANTS, 8)) + + +class MutatedTest(unittest.TestCase): + SOURCE = b"package calc\n\nfunc f(a, b int) int {\n\tif a >= b {\n\t\treturn a - b\n\t}\n\tn -= 1\n\treturn a\n}\n" + + def planted(self, line: int, column: int, mutator: str, source: bytes = SOURCE) -> str | None: + found = mutated(source, Mutant("calc.go", line, column, mutator)) + return None if found is None else found.decode().splitlines()[line - 1] + + def test_swaps_the_token_at_the_line_and_column_as_gremlins_mutator_does(self): + self.assertEqual(self.planted(4, 7, "CONDITIONALS_NEGATION"), "\tif a < b {") + self.assertEqual(self.planted(4, 7, "CONDITIONALS_BOUNDARY"), "\tif a > b {") + self.assertEqual(self.planted(5, 12, "ARITHMETIC_BASE"), "\t\treturn a + b") + self.assertEqual(self.planted(7, 4, "INVERT_ASSIGNMENTS"), "\tn += 1") + self.assertEqual(self.planted(7, 4, "REMOVE_SELF_ASSIGNMENTS"), "\tn = 1") + + def test_changes_no_other_byte(self): + found = mutated(self.SOURCE, Mutant("calc.go", 5, 12, "ARITHMETIC_BASE")) + self.assertEqual(found, self.SOURCE.replace(b"a - b", b"a + b")) + + def test_counts_the_column_in_bytes_as_go_does(self): + self.assertEqual(self.planted(1, 15, "CONDITIONALS_NEGATION", 'x := "åäö" == y\n'.encode()), 'x := "åäö" != y') + + def test_is_none_where_the_place_holds_no_token_of_the_mutator(self): + self.assertIsNone(self.planted(4, 6, "CONDITIONALS_NEGATION")) + self.assertIsNone(self.planted(5, 12, "CONDITIONALS_NEGATION")) + self.assertIsNone(self.planted(40, 1, "ARITHMETIC_BASE")) + self.assertIsNone(self.planted(4, 70, "ARITHMETIC_BASE")) + self.assertIsNone(self.planted(4, 7, "A_MUTATOR_GREMLINS_ADDS_LATER")) + + +class TaskTextTest(unittest.TestCase): + def test_is_the_first_failing_assertions_message_without_the_tests_name_file_or_line(self): + self.assertEqual(task_text(GO_FAILURE), "Pick() = 3,\nwant 4") + + def test_leaves_out_each_line_that_names_the_test_or_its_file(self): + self.assertEqual(task_text(TESTIFY_FAILURE), "Error: \tNot equal:\nexpected: 1\nactual : 2") + + def test_is_none_where_no_assertion_failed_as_on_a_panic_or_a_passing_run(self): + self.assertIsNone(task_text(PANIC)) + self.assertIsNone(task_text("ok \texample.com/calc\t0.004s\n")) + + def test_is_none_where_the_message_says_nothing_beyond_the_test(self): + self.assertIsNone(task_text("--- FAIL: TestPick (0.00s)\n pick_test.go:42: TestPick failed\nFAIL\n")) + + +class LineAtHeadTest(unittest.TestCase): + # Lines 3 and 4 became one, two lines went in after line 9, and line 20 went. + DIFF = "diff --git a/calc.go b/calc.go\n--- a/calc.go\n+++ b/calc.go\n@@ -3,2 +3 @@\n-a\n-b\n+c\n@@ -9,0 +9,2 @@\n+d\n+e\n@@ -20 +20,0 @@\n-f\n" + + def test_a_line_above_every_change_keeps_its_number(self): + self.assertEqual(line_at_head(self.DIFF, 2), 2) + + def test_a_line_below_a_change_moves_by_the_lines_it_added_and_removed(self): + self.assertEqual([line_at_head(self.DIFF, line) for line in (5, 9, 10, 19, 21)], [4, 8, 11, 20, 21]) + + def test_a_line_the_head_changed_or_removed_has_none(self): + self.assertEqual([line_at_head(self.DIFF, line) for line in (3, 4, 20)], [None, None, None]) + + def test_no_diff_keeps_every_line(self): + self.assertEqual(line_at_head("", 12), 12) diff --git a/bin/task_bench/test_task_bench.py b/bin/task_bench/test_task_bench.py new file mode 100644 index 00000000..91ffd05b --- /dev/null +++ b/bin/task_bench/test_task_bench.py @@ -0,0 +1,299 @@ +"""task-bench from its command line down: a whole bench over a small repository with docker, Gremlins and claude faked, the tree a +fix is checked in, and the proxy that reaches only the API.""" +import asyncio +import contextlib +import io +import json +import os +import pathlib +import subprocess +import sys +import tempfile +import textwrap +import unittest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) + +from failure import Failure +from task_bench import api_proxy, main +from task_bench.runs import checked_tree +from test_env import isolated_env + +BASE_CODE = "package calc\n\nfunc Add(a, b int) int {\n\treturn a + b\n}\n\nfunc Twice(a int) int {\n\treturn a * 2\n}\n" +# The head moves Add down a line and rewrites Twice's line. +HEAD_CODE = "// Package calc adds.\n" + BASE_CODE.replace("a * 2", "2 * a") +# Stands in for `go test`: a shell text in a file named as a Go test, which fails as go test reports a failed assertion. +TESTS = textwrap.dedent("""\ + grep -q 'a + b' calc.go && exit 0 + printf -- '--- FAIL: TestAdd (0.00s)\\n calc_test.go:9: Add(1, 2) = -1, want 3\\nFAIL\\n' + exit 1 +""") +# Reports `+` in Add, which the head keeps, and the `*` in Twice, which the head rewrites, as killed; and a mutant no test caught. +GREMLINS = textwrap.dedent("""\ + import json, sys + lines = open("calc.go").read().split("\\n") + found = [{"line": at, "column": 11, "status": "KILLED", "type": "ARITHMETIC_BASE"} for at, line in enumerate(lines, 1) if " + " in line or " * " in line] + report = {"files": [{"file_name": "calc.go", "mutations": [*found, {**found[0], "status": "LIVED", "type": "INVERT_NEGATIVES"}]}]} + json.dump(report, open(sys.argv[sys.argv.index("--output") + 1], "w")) +""") +# Logs each call. A `run` of the bench's image runs its command in the mounted tree: sh as it is, and claude as the fake below. +DOCKER = textwrap.dedent("""\ + #!/usr/bin/env python3 + import json, os, subprocess, sys + args = sys.argv[1:] + with open(os.environ["FAKE_LOG"], "a") as log: + log.write(json.dumps(args) + "\\n") + if args[0] == "build": + sys.stdin.read() + if args[0] != "run" or "--detach" in args: + sys.exit(int(os.environ.get("FAKE_FAILS") == args[0])) + tree = args[args.index("--volume") + 1].rpartition(":")[0] + command = args[next(at for at, arg in enumerate(args) if arg.endswith(":throwaway")) + 1:] + if command[0] == "claude": + command = [sys.executable, os.environ["FAKE_CLAUDE"], command[-1]] + sys.exit(subprocess.run(command, cwd=tree).returncode) +""") +# Reads the README, then the file with the bug. At the head it fixes the bug; at the base it rewrites the test to pass instead. +CLAUDE = textwrap.dedent("""\ + import json, os, pathlib, sys + def call(name, path): + return {"type": "assistant", "message": {"id": "msg_" + path, "content": [{"type": "tool_use", "id": "toolu_" + path, "name": name, "input": {"file_path": path}}], "usage": {}}} + if os.environ.get("FAKE_REFUSED"): + print(json.dumps({"type": "result", "is_error": True, "api_error_status": 401, "result": "API Error: 401"})) + sys.exit(1) + print(json.dumps({"type": "system", "subtype": "init", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330"})) + print(json.dumps(call("Read", "/work/README.md"))) + print(json.dumps(call("Read", "/work/calc.go"))) + code = pathlib.Path("calc.go") + if code.read_text().startswith("//"): + code.write_text(code.read_text().replace("a - b", "a + b")) + else: + pathlib.Path("calc_test.go").write_text("exit 0\\n") + pathlib.Path("prompt").write_text(sys.argv[1]) + usage = {"input_tokens": 40, "cache_creation_input_tokens": 9000, "cache_read_input_tokens": 52000, "output_tokens": 800} + print(json.dumps({"type": "result", "subtype": "success", "is_error": False, "num_turns": 3, "duration_ms": 61500, "usage": usage, "result": "Fixed."})) +""") +PLACES = "## 2026-10-08T12:00:00Z, PR #1 at aaaaaaa\n\nJunior A, hardest first:\n\n- `calc.go:3` `Add`: long\n- `gone.go:1` `Gone`: removed since\n- `README.md:1` `Calc`: thin\n" + + +class BenchTest(unittest.TestCase): + def setUp(self): + folder = tempfile.TemporaryDirectory() + self.addCleanup(folder.cleanup) + self.root = pathlib.Path(folder.name).resolve() + self.home, self.repo, fakes = self.root / "home", self.root / "repo", self.root / "fakes" + for made in (self.home, self.repo, fakes): + made.mkdir() + (fakes / "docker").write_text(DOCKER) + (fakes / "docker").chmod(0o755) + (fakes / "claude.py").write_text(CLAUDE) + (fakes / "gremlins.py").write_text(GREMLINS) + self.log = self.root / "docker.log" + self.env = {**isolated_env(), "HOME": str(self.home), "PATH": f"{fakes}{os.pathsep}{os.environ['PATH']}", "FAKE_LOG": str(self.log), "FAKE_CLAUDE": str(fakes / "claude.py"), "CLAUDE_CODE_OAUTH_TOKEN": "sk-secret-token", "RUN_HEAVY_MEMORY": "2048"} + self.given = {"--image": "golang:1.27.4", "--test": "sh calc_test.go", "--prefix": f"{sys.executable} {fakes / 'gremlins.py'}", "--seed": "1", "--bugs": "2", "--runs": "2"} + self.git("init", "--quiet", "--initial-branch", "main") + for name, text in (("calc.go", BASE_CODE), ("calc_test.go", TESTS), ("README.md", "# Calc\n")): + (self.repo / name).write_text(text) + self.base = self.commit("Base") + self.git("checkout", "--quiet", "-b", "item-1") + (self.repo / "calc.go").write_text(HEAD_CODE) + self.head = self.commit("Head") + self.enterContext(contextlib.chdir(self.repo)) + + def git(self, *args: str) -> str: + return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@localhost", "-c", "commit.gpgsign=false", *args], cwd=self.repo, env=self.env, check=True, capture_output=True, text=True).stdout.strip() + + def commit(self, title: str) -> str: + self.git("add", "--all") + self.git("commit", "--quiet", "--message", title) + return self.git("rev-parse", "HEAD") + + def bench(self, **given: str): + with contextlib.redirect_stderr(io.StringIO()) as self.progress: + return main.bench({**self.given, "--base": "main", **given}, self.env, PLACES, self.home) + + def calls(self) -> list[list[str]]: + return [json.loads(line) for line in self.log.read_text().splitlines()] + + def test_plants_the_killed_mutant_the_head_left_alone_and_prints_how_each_side_fixed_it(self): + report = self.bench() + self.assertNotIsInstance(report, Failure, report) + folder = pathlib.Path(report[-1].removeprefix("Each run's result and stream: ")) + self.assertEqual(report[:-1], [ + f"task-bench: base {self.base[:12]}, head {self.head[:12]}. 1 bug, fixed 2 times at each side by claude-haiku-5-5 in Claude Code 2.1.330.", + "Planted 1 of the 2 bugs asked for: no other mutant the tests kill becomes one.", + "", + "| Task | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |", + "|---|---|---|---|---|---|---|---|---|---|", + "| 1: calc.go:4:11 ARITHMETIC_BASE | base | 0/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |", + "| 1: calc.go:4:11 ARITHMETIC_BASE | head | 2/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |", + "| All | base | 0/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |", + "| All | head | 2/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |", + "| All | change | +2 | 0 % | 0 % | 0 % | 0 % | 0 | 0 % | 0 |", + "", + ]) + self.assertTrue(folder.is_relative_to(self.home / ".local/state/larv-review") and folder.name == "task-bench", folder) + results = json.loads((folder / "results.json").read_text()) + self.assertEqual((results["sides"], results["bugs"], len(results["runs"])), ({"base": self.base, "head": self.head}, {"1": "calc.go:4:11 ARITHMETIC_BASE"}, 4)) + self.assertEqual(results["runs"][0], {"bug": 1, "side": "base", "run": 1, "passed": False, "finished": True, "claude_code": "2.1.330", "model": "claude-haiku-5-5", "tokens": {"input": 40, "write_5m": 9000, "write_1h": 0, "read": 52000, "output": 800}, "turns": 3, "seconds": 61.5, "reads": 1}) + self.assertEqual(sorted(path.name for path in (folder / "runs").iterdir() if path.suffix == ".jsonl"), ["bug-1-base-1.jsonl", "bug-1-base-2.jsonl", "bug-1-head-1.jsonl", "bug-1-head-2.jsonl"]) + self.assertIn("run 4 of 4, bug 1 at the head: passed.", self.progress.getvalue()) + + def test_each_side_gets_its_own_tree_with_the_bug_committed_once_and_no_trace_of_the_checkout(self): + folder = pathlib.Path(self.bench()[-1].removeprefix("Each run's result and stream: ")) + for side, code in (("base", BASE_CODE), ("head", HEAD_CODE)): + tree = folder / "bugs/1" / side + self.assertEqual((tree / "calc.go").read_text(), code.replace("a + b", "a - b")) + log = subprocess.run(["git", "log", "--format=%an %s"], cwd=tree, env=self.env, check=True, capture_output=True, text=True).stdout + self.assertEqual(log, "task-bench Import\n") + self.assertEqual(subprocess.run(["git", "status", "--porcelain"], cwd=tree, env=self.env, check=True, capture_output=True, text=True).stdout, "") + + def test_claude_runs_in_the_benchs_image_on_its_network_with_the_message_alone_and_the_token_by_name(self): + self.bench(**{"--runs": "1"}) + calls = self.calls() + fixes = [call for call in calls if "claude" in call] + self.assertEqual(len(fixes), 2) + name = fixes[0][fixes[0].index("--network") + 1] + self.assertRegex(name, r"^task-bench-[0-9a-f]{12}$") + image = fixes[0].index(f"{name}:throwaway") + self.assertEqual(fixes[0][image + 1:], ["claude", "-p", "--strict-mcp-config", "--disable-slash-commands", "--no-session-persistence", "--model", "claude-haiku-5-5", "--effort", "low", "--output-format", "stream-json", "--verbose", "--permission-mode", "bypassPermissions", "A test of this project fails with the message below. Find the bug in the code and fix it. Change no test.\n\nAdd(1, 2) = -1, want 3"]) + options = fixes[0][:image] + for expected in (["--memory", "2048m"], ["--memory-swap", "2048m"], ["--user", f"{os.getuid()}:{os.getgid()}"], ["--env", "HOME=/tmp"], ["--env", "CLAUDE_CODE_OAUTH_TOKEN"], ["--env", f"HTTPS_PROXY=http://{name}-proxy:8080"], ["--workdir", "/work"]): + self.assertIn(expected, [options[at:at + 2] for at in range(len(options))]) + self.assertNotIn("sk-secret-token", self.log.read_text()) + self.assertEqual(calls[0][:1] + calls[0][-3:], ["build", "--tag", f"{name}:throwaway", "-"]) + self.assertEqual(calls[1], ["network", "create", "--internal", name]) + self.assertEqual([call[:2] for call in calls[-4:]], [["rm", "--force"], ["rm", "--force"], ["network", "rm"], ["image", "rm"]]) + + def test_the_tests_run_without_a_network(self): + self.bench(**{"--runs": "1"}) + tests = [call for call in self.calls() if call[-3:] == ["sh", "-c", "sh calc_test.go"]] + # Each side without a bug, each side with it, and each side's fix. + self.assertEqual(len(tests), 6) + for call in tests: + self.assertEqual(call[call.index("--network") + 1], "none") + + def test_benches_the_head_against_itself_where_the_base_is_the_head(self): + report = self.bench(**{"--base": "HEAD", "--runs": "1", "--bugs": "1"}) + self.assertEqual(report[0], f"task-bench: base {self.head[:12]}, head {self.head[:12]}. 1 bug, fixed 1 time at each side by claude-haiku-5-5 in Claude Code 2.1.330.") + self.assertEqual(report[4:6], ["| 1: calc.go:5:11 ARITHMETIC_BASE | base | 1/1 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |", "| 1: calc.go:5:11 ARITHMETIC_BASE | head | 1/1 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |"]) + + def test_fails_where_the_places_name_no_go_file_of_the_base(self): + with contextlib.redirect_stderr(io.StringIO()): + found = main.bench({**self.given, "--base": "main"}, self.env, "- `gone.go:1` `Gone`: removed\n- `README.md:1` `Calc`: thin\n", self.home) + self.assertEqual(found, Failure("The places the seats found hard name no Go file that the base holds, so there is nowhere to plant a bug.")) + self.assertFalse(self.log.exists()) + + def test_fails_where_the_tests_fail_without_a_bug(self): + found = self.bench(**{"--test": "echo broken; exit 2"}) + self.assertIsInstance(found, Failure) + self.assertIn("The tests fail at the base without a planted bug", found.reason) + self.assertIn("broken", found.reason) + + def test_fails_where_no_mutant_becomes_a_bug(self): + # Passing tests with the bug planted: nothing to tell Haiku. + (self.repo / "calc_test.go").write_text("exit 0\n") + self.commit("Tests that catch nothing") + self.assertEqual(self.bench(), Failure("None of the 2 mutants the tests kill becomes a bug: each is on a line the head changed, or fails no assertion with a message.")) + + def test_a_run_the_api_refuses_fails_the_bench_and_still_removes_what_it_made(self): + self.env["FAKE_REFUSED"] = "1" + self.assertEqual(self.bench(), Failure("claude could not use the API: API Error: 401")) + self.assertEqual(self.calls()[-1][:2], ["image", "rm"]) + + def test_a_docker_that_fails_to_build_fails_the_bench(self): + self.env["FAKE_FAILS"] = "build" + found = self.bench() + self.assertIsInstance(found, Failure) + self.assertIn("docker build", found.reason) + + +class MainTest(unittest.TestCase): + def run_main(self, *argv: str, token: str = "t") -> tuple[int, str]: + with contextlib.redirect_stderr(io.StringIO()) as errors, contextlib.redirect_stdout(io.StringIO()): + return main.main(list(argv), {"CLAUDE_CODE_OAUTH_TOKEN": token, "HOME": "/nonexistent"}), errors.getvalue() + + def test_refuses_an_unknown_repeated_or_valueless_argument(self): + for argv, said in ((["--imgae", "x"], "unknown or repeated argument --imgae"), (["--image", "x", "--image=y"], "unknown or repeated argument --image"), (["--image"], "--image needs a value")): + code, errors = self.run_main(*argv) + self.assertEqual((code, errors.startswith(f"{said}; usage: task-bench --image")), (1, True), errors) + + def test_refuses_a_missing_image_or_test_command(self): + self.assertEqual(self.run_main("--test", "go test ./...")[1].partition(";")[0], "--image is missing") + self.assertEqual(self.run_main("--image", "golang:1.27.4")[1].partition(";")[0], "--test is missing") + + def test_refuses_a_count_that_is_no_whole_number_from_1(self): + for name, value in (("--bugs", "0"), ("--runs", "-1"), ("--seed", "x")): + self.assertEqual(self.run_main("--image", "i", "--test", "t", name, value)[1].partition(";")[0], f"{name} takes a whole number from 1, not {value}") + + def test_refuses_to_start_without_a_token(self): + self.assertEqual(self.run_main("--image", "i", "--test", "t", token="")[1].partition(";")[0], "CLAUDE_CODE_OAUTH_TOKEN holds no token") + + def test_prints_its_help(self): + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(main.main(["--help"], {}), 0) + self.assertIn("usage: task-bench --image --test ", out.getvalue()) + + +class CheckedTreeTest(unittest.TestCase): + def setUp(self): + folder = tempfile.TemporaryDirectory() + self.addCleanup(folder.cleanup) + self.root = pathlib.Path(folder.name) + + def tree(self, name: str, files: dict[str, str]) -> str: + for path, text in files.items(): + (self.root / name / path).parent.mkdir(parents=True, exist_ok=True) + (self.root / name / path).write_text(text) + return str(self.root / name) + + def held(self, name: str) -> dict[str, str]: + top = self.root / name + return {str(path.relative_to(top)): path.read_text() for path in sorted(top.rglob("*")) if path.is_file()} + + def test_takes_the_code_as_fixed_and_the_tests_as_planted(self): + planted = self.tree("planted", {"calc.go": "bug", "calc_test.go": "tests", "sub/old.go": "old", "sub/old_test.go": "old tests", ".git/HEAD": "ref"}) + copy = self.tree("copy", {"calc.go": "fixed", "calc_test.go": "rewritten", "extra_test.go": "added", "sub/new.go": "new", ".git/HEAD": "moved"}) + checked_tree(planted, copy, str(self.root / "check")) + self.assertEqual(self.held("check"), {".git/HEAD": "ref", "calc.go": "fixed", "calc_test.go": "tests", "sub/new.go": "new", "sub/old_test.go": "old tests"}) + + def test_copies_a_link_as_a_link(self): + planted = self.tree("planted", {"calc.go": "bug"}) + copy = self.tree("copy", {"calc.go": "fixed"}) + os.symlink("/etc/hostname", os.path.join(copy, "host.go")) + checked_tree(planted, copy, str(self.root / "check")) + self.assertEqual(os.readlink(self.root / "check/host.go"), "/etc/hostname") + + +class ApiProxyTest(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + async def echoed(reader, writer): + writer.write(b"api:" + await reader.read(5)) + await writer.drain() + writer.close() + + api = await asyncio.start_server(echoed, "127.0.0.1", 0) + self.allowed = f"127.0.0.1:{api.sockets[0].getsockname()[1]}" + proxy = await asyncio.start_server(lambda reader, writer: api_proxy.served(self.allowed, reader, writer), "127.0.0.1", 0) + self.port = proxy.sockets[0].getsockname()[1] + for server in (api, proxy): + self.addAsyncCleanup(server.wait_closed) + self.addCleanup(server.close) + + async def answered(self, request: bytes) -> bytes: + reader, writer = await asyncio.open_connection("127.0.0.1", self.port) + writer.write(request) + answer = await asyncio.wait_for(reader.read(), 5) + writer.close() + return answer + + async def test_tunnels_a_connect_to_the_one_allowed_target(self): + self.assertEqual(await self.answered(f"CONNECT {self.allowed} HTTP/1.1\r\n\r\nhello".encode()), b"HTTP/1.1 200 Connection established\r\n\r\napi:hello") + + async def test_answers_403_to_any_other_target_or_method(self): + for request in (b"CONNECT example.com:443 HTTP/1.1\r\n\r\n", f"GET http://{self.allowed}/ HTTP/1.1\r\n\r\n".encode(), b"junk\r\n\r\n"): + self.assertEqual(await self.answered(request), b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/reading-cost.md b/docs/reading-cost.md index 9f06acf4..d13ffa81 100644 --- a/docs/reading-cost.md +++ b/docs/reading-cost.md @@ -3296,3 +3296,29 @@ Read most (opens and searches): Read most (opens and searches): - `README.md` 1 + +## Reading cost, chunk d8a3d6c859ad, finished 2026-11-20T23:06:03Z + +162248 new input tokens before the first commit. + +Read most (opens and searches): + +- `docs/decisions.md` 9 +- `README.md` 1 +- `bin/chunk_driver/project/model_choice.py` 1 +- `bin/claude_code/model_catalog.py` 1 +- `bin/loop_state/paths.py` 1 +- `todo.md` 1 + +## Reading cost, chunk b043e7de5b94, finished 2026-11-20T23:46:55Z + +55211 new input tokens before the first commit. + +Read most (opens and searches): + +- `bin/mutate_changes/test_mutate_changes_runs.py` 4 +- `bin/mutate_changes/main.py` 2 +- `bin/mutate_changes/changed_lines.py` 1 +- `bin/mutate_changes/in_diff_tools.py` 1 +- `bin/mutate_changes/test_mutate_changes.py` 1 +- `bin/mutate_changes/tool_process.py` 1