Files
claude-config/bin/task_bench/main.py
T

308 lines
16 KiB
Python

r"""Measure how hard a project's code is to fix a bug in, at a base commit and at a head: plant bugs its tests catch, have Haiku fix
each several times at both, and print how often the fix passes and what it took.
usage: task-bench --image <image> --test <shell text> [--base <ref>] [--head <ref>] [--prefix <shell text>] [--seed <n>] [--bugs <n>] [--runs <n>]
Run it in a checkout, under ~/.claude/bin/run-heavy, with a long-lived token in CLAUDE_CODE_OAUTH_TOKEN, which `claude setup-token`
prints. The head is --head, by default HEAD. The base is the merge base of the head and --base, by default the default branch:
origin's as last fetched, else main or master. `--base HEAD` benches the head against itself. No model judges anything: the bench
runs programs, and reads Haiku's own stream.
- Plant. It copies the tree at the base and runs Gremlins there, as `gremlins unleash`, over the Go files that hold the places
`larv-review places` prints, the ones the comprehension seats found hardest. Of the mutants the tests kill, it draws --bugs, 5 by
default, in the order --seed sets, 1 by default. A drawn mutant becomes a bug where its line stands unchanged at the head, and
where, planted at each side, the tests fail on an assertion whose message says something. Others are passed over, so a change to
the very line of a bug is not benched. Go is the one language so far.
- Run. Per bug and side, --runs times, 5 by default, sides in turn: a copy of that side's tree with the bug in it, committed once,
so git shows nothing of the bug. In it, in a container, `claude -p --strict-mcp-config --disable-slash-commands
--no-session-persistence --model claude-haiku-5-5 --effort low` works with every tool, told the failing assertion's message at that
side, without the test's name, file or line. The container's Claude home is new and empty, so the project's own CLAUDE.md, or
AGENTS.md, in the copy is all the setup that loads. Its network reaches api.anthropic.com and nothing else. A fix ends after 30
minutes.
- Check. The test command runs over the planted tree with each file that is no test as Haiku left it; a fix passes where it exits 0.
Tests Haiku changed, removed or added count for nothing.
--image is the project's development image, such as golang:1.27.4. The bench adds Claude Code to it, as its installer installs it,
so the image needs curl and bash; the result names the version. Haiku works in that image, and --test runs in it, as sh runs it at
the copy's top, with no network: the image must hold what the tests download otherwise.
--prefix is shell text that runs Gremlins, on the host, as mutate-changes' --prefix does: sh runs `<prefix> "$@"` in the copy of the
base, which $PWD names, with `gremlins unleash ...` as "$@". For example:
docker run --rm --memory 8g --memory-swap 8g -v "$PWD:$PWD" -w "$PWD" golang:1.27.4 \
sh -c 'shift; exec go run github.com/go-gremlins/gremlins/cmd/gremlins@v0.6.0 "$@"' sh
It prints the base and the head, then a table: a row per bug and side with how many fixes passed and the medians, over the runs
that finished, of the input, cache-write, cache-read and output tokens, the turns, the wall time, and the reads and searches before
the run first opened the file the bug is in; then a row per side over every bug, and the head's change against the base. Each
run's result and stream are in the folder its last line names, `task-bench` in the chunk's review folder, which each bench empties
first.
Exit 1, saying why, on an unknown or repeated argument, a missing --image or --test, a count that is no whole number from 1, or
no token; and where git, docker or Gremlins fails, the seats' places name no Go file, the tests fail at a side without a bug, no
mutant becomes a bug, or the API refuses a run.
"""
import dataclasses
import functools
import json
import os
import pathlib
import shutil
import sys
import tarfile
import typing
import uuid
from claude_code.tokens import Tokens
from failure import Failure
from git_cli import git_repo
from git_cli.git_repo import default_ref, Git, top
from larv_review.history_commands import read_places
from machine_load.slots import CPUS_PER_SLOT
from mutate_changes.tool_process import failed, Ran, run_tool
from review_rules.review_folder import made_review_folder
from run_heavy.capped_docker import SHARE_VARIABLE
from task_bench.figures import Figures, figures, Result, SIDES, table, WORK
from task_bench.plant import drawn, killed_mutants, line_at_head, Mutant, mutated, place_files, task_text
from task_bench.runs import checked_tree, closed, Containers, docker_cli, fix_attempted, MEMORY_MIB, opened, tests_run, TOKEN
USAGE = "usage: task-bench --image <image> --test <shell text> [--base <ref>] [--head <ref>] [--prefix <shell text>] [--seed <n>] [--bugs <n>] [--runs <n>]"
# A panel of three readers, seated as the README's audience, picked these options on 2026-11-21, each 3 to 0.
OPTIONS = ("--base", "--bugs", "--head", "--image", "--prefix", "--runs", "--seed", "--test")
COUNTS = {"--bugs": "5", "--runs": "5", "--seed": "1"}
# The planted commit names no one, and no hook or signing key of the caller's reaches it.
COMMIT = ["-c", "user.name=task-bench", "-c", "user.email=task-bench@localhost", "-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", "commit", "--quiet", "--message", "Import"]
@dataclasses.dataclass(frozen=True)
class Task:
"""A bug to fix: its number from 1, the mutant at the base, and per side the planted tree and what Haiku is told there."""
number: int
mutant: Mutant
trees: dict[str, str]
messages: dict[str, str]
def named(self) -> str:
return f"{self.mutant.path}:{self.mutant.line}:{self.mutant.column} {self.mutant.mutator}"
def main(argv: list[str], env: typing.Mapping[str, str]) -> int:
"""Runs task-bench with the arguments argv; git, docker and Gremlins run in the environment env."""
if argv in (["-h"], ["--help"]):
print(__doc__.strip())
return 0
given = options(argv, env)
if isinstance(given, Failure):
print(f"{given.reason}; {USAGE}", file=sys.stderr)
return 1
home = pathlib.Path(env.get("HOME") or pathlib.Path.home())
places = read_places(home)
report = places if isinstance(places, Failure) else bench(given, env, places, home)
if isinstance(report, Failure):
print(report.reason, file=sys.stderr)
return 1
print("\n".join(report))
return 0
def options(argv: list[str], env: typing.Mapping[str, str]) -> dict[str, str] | Failure:
given: dict[str, str] = {}
rest = list(argv)
while rest:
name, equals, value = rest.pop(0).partition("=")
if name not in OPTIONS or name in given:
return Failure(f"unknown or repeated argument {name}")
if not equals and not rest:
return Failure(f"{name} needs a value")
given[name] = value if equals else rest.pop(0)
for name in ("--image", "--test"):
if not given.get(name):
return Failure(f"{name} is missing")
for name, default in COUNTS.items():
if not given.setdefault(name, default).isdecimal() or int(given[name]) < 1:
return Failure(f"{name} takes a whole number from 1, not {given[name]}")
if not env.get(TOKEN):
return Failure(f"{TOKEN} holds no token")
return given
def bench(given: dict[str, str], env: typing.Mapping[str, str], places: str, home: pathlib.Path) -> list[str] | Failure:
"""The report of one bench of the checkout the process runs in, with `places` what `larv-review places` prints there."""
git = functools.partial(git_repo.git, env=env)
checkout = top(git)
if isinstance(checkout, Failure):
return checkout
sides = commits(given, checkout, git)
if isinstance(sides, Failure):
return sides
print(f"task-bench: base {sides['base'][:12]}, head {sides['head'][:12]}.", file=sys.stderr)
review = made_review_folder(checkout, home)
if isinstance(review, Failure):
return review
folder = review / "task-bench"
containers = Containers(docker_cli(env), f"task-bench-{uuid.uuid4().hex[:12]}", given["--test"], int(env.get(SHARE_VARIABLE) or MEMORY_MIB), f"{os.getuid()}:{os.getgid()}")
try:
shutil.rmtree(folder, ignore_errors=True)
trees: dict[str, str] = {}
for side, commit in sides.items():
tree = archived(commit, folder / "trees" / side, checkout, git)
if isinstance(tree, Failure):
return tree
trees[side] = tree
files = [path for path in place_files(places) if pathlib.Path(trees["base"], path).is_file()]
if not files:
return Failure("The places the seats found hard name no Go file that the base holds, so there is nowhere to plant a bug.")
run = functools.partial(run_tool, prefix=given.get("--prefix"), env=env, killpg=os.killpg)
mutants = killed_mutants(trees["base"], files, run, CPUS_PER_SLOT)
if isinstance(mutants, Failure):
return mutants
try:
found = opened(containers, given["--image"]) or benched(given, checkout, sides, trees, mutants, containers, folder, git)
finally:
closed(containers)
except OSError as failure:
return Failure(f"could not write under {folder}: {failure}")
return found if isinstance(found, Failure) else [*found, f"Each run's result and stream: {folder}"]
def commits(given: dict[str, str], checkout: str, git: Git) -> dict[str, str] | Failure:
"""The base's commit and the head's."""
head = git("rev-parse", "--verify", f"{given.get('--head', 'HEAD')}^{{commit}}", cwd=checkout)
against = given.get("--base") or default_ref(cwd=checkout, run=git)
if isinstance(head, Failure):
return head
if isinstance(against, Failure):
return against
base = git("merge-base", against, head.strip(), cwd=checkout)
return base if isinstance(base, Failure) else {"base": base.strip(), "head": head.strip()}
def archived(commit: str, into: pathlib.Path, checkout: str, git: Git) -> str | Failure:
"""Makes `into` the tree at commit, without git's own data, and returns its path."""
into.mkdir(parents=True)
archive = into.with_suffix(".tar")
wrote = git("archive", "--format=tar", "--output", str(archive), commit, cwd=checkout)
if isinstance(wrote, Failure):
return wrote
try:
with tarfile.open(archive) as held:
held.extractall(into, filter="tar")
except tarfile.TarError as failure:
return Failure(f"could not unpack the tree at {commit}: {failure}")
archive.unlink()
return str(into)
def benched(given: dict[str, str], checkout: str, sides: dict[str, str], trees: dict[str, str], mutants: list[Mutant], containers: Containers, folder: pathlib.Path, git: Git) -> list[str] | Failure:
"""Plants the bugs, runs each fix, writes the results into folder and returns the report."""
for side, tree in trees.items():
ran = tests_in_copy(containers, tree, folder)
if isinstance(ran, Failure) or ran.code:
return ran if isinstance(ran, Failure) else failed(f"The tests fail at the {side} without a planted bug: the test command", ran)
tasks: list[Task] = []
for mutant in drawn(mutants, int(given["--seed"])):
if len(tasks) == int(given["--bugs"]):
break
task = planted(len(tasks) + 1, mutant, checkout, sides, trees, containers, folder, git)
if isinstance(task, Failure):
return task
tasks += [task] if task else []
if not tasks:
return Failure(f"None of the {len(mutants)} mutants the tests kill becomes a bug: each is on a line the head changed, or fails no assertion with a message.")
results: list[Result] = []
total = int(given["--runs"]) * len(tasks) * len(SIDES)
for number in range(1, int(given["--runs"]) + 1):
for task in tasks:
for side in SIDES:
result = fixed(task, side, number, containers, folder)
if isinstance(result, Failure):
return result
results.append(result)
print(f"task-bench: run {len(results)} of {total}, bug {task.number} at the {side}: {'passed' if result.passed else 'not passed'}.", file=sys.stderr)
(folder / "results.json").write_text(json.dumps({"sides": sides, "seed": int(given["--seed"]), "bugs": {task.number: task.named() for task in tasks}, "runs": [recorded(result) for result in results]}, indent="\t") + "\n", encoding="utf-8")
ran_on = sorted({f"{found.model} in Claude Code {found.version}" for result in results if (found := result.figures)})
short = [] if len(tasks) == int(given["--bugs"]) else [f"Planted {len(tasks)} of the {given['--bugs']} bugs asked for: no other mutant the tests kill becomes one."]
return [
f"task-bench: base {sides['base'][:12]}, head {sides['head'][:12]}. {counted(len(tasks), 'bug')}, fixed {counted(int(given['--runs']), 'time')} at each side by {' and '.join(ran_on) or 'no run that finished'}.",
*short,
"",
*table(results, {task.number: task.named() for task in tasks}),
"",
]
def planted(number: int, mutant: Mutant, checkout: str, sides: dict[str, str], trees: dict[str, str], containers: Containers, folder: pathlib.Path, git: Git) -> Task | Failure | None:
"""The task of fixing mutant, with its planted tree at each side; None where the mutant is no bug at both sides.
The bug is the same change to the same line at both sides, so only the code around it differs between them."""
diff = git("diff", "-U0", "--no-color", "--no-ext-diff", "--no-textconv", sides["base"], sides["head"], "--", mutant.path, cwd=checkout, errors="replace")
if isinstance(diff, Failure):
return diff
at_head = line_at_head(diff, mutant.line)
if at_head is None:
return None
planted_trees, messages = {}, {}
for side, placed in (("base", mutant), ("head", mutant._replace(line=at_head))):
source = pathlib.Path(trees[side], mutant.path)
changed = mutated(source.read_bytes(), placed) if source.is_file() else None
if changed is None:
return None
tree = folder / "bugs" / str(number) / side
shutil.rmtree(tree, ignore_errors=True)
shutil.copytree(trees[side], tree, symlinks=True)
(tree / mutant.path).write_bytes(changed)
ran = tests_in_copy(containers, str(tree), folder)
if isinstance(ran, Failure):
return ran
message = task_text(ran.output) if ran.code else None
if message is None:
return None
for args in (["init", "--quiet"], ["add", "--all", "--force"], COMMIT):
done = git(*args, cwd=str(tree))
if isinstance(done, Failure):
return done
planted_trees[side], messages[side] = str(tree), message
return Task(number, mutant, planted_trees, messages)
def tests_in_copy(containers: Containers, tree: str, folder: pathlib.Path) -> Ran | Failure:
"""The test command's run in a copy of tree that goes afterwards, so what the tests write stays out of tree."""
copy = folder / "tested"
shutil.copytree(tree, copy, symlinks=True)
try:
return tests_run(containers, str(copy))
finally:
shutil.rmtree(copy, ignore_errors=True)
def fixed(task: Task, side: str, number: int, containers: Containers, folder: pathlib.Path) -> Result | Failure:
"""One run: Haiku's fix of task at side in a copy of its planted tree, then the tests over what it changed. The stream stays in
folder; the copy goes."""
stream = folder / "runs" / f"bug-{task.number}-{side}-{number}.jsonl"
stream.parent.mkdir(exist_ok=True)
copy, check = folder / "copy", folder / "check"
try:
shutil.copytree(task.trees[side], copy, symlinks=True)
attempted = fix_attempted(containers, str(copy), task.messages[side], stream)
if isinstance(attempted, Failure):
return attempted
found = figures(stream.read_bytes(), os.path.join(WORK, task.mutant.path))
if isinstance(found, Failure):
return found
checked_tree(task.trees[side], str(copy), str(check))
ran = tests_run(containers, str(check))
return Result(task.number, side, number, not isinstance(ran, Failure) and ran.code == 0, found)
finally:
shutil.rmtree(copy, ignore_errors=True)
shutil.rmtree(check, ignore_errors=True)
def recorded(result: Result) -> dict:
"""A run as results.json holds it."""
found = result.figures or Figures("", "", Tokens(), 0, 0, None)
measured = {"claude_code": found.version, "model": found.model, "tokens": dataclasses.asdict(found.tokens), "turns": found.turns, "seconds": found.seconds, "reads": found.reads}
return {"bug": result.task, "side": result.side, "run": result.run, "passed": result.passed, "finished": result.figures is not None, **(measured if result.figures else {})}
def counted(count: int, noun: str) -> str:
return f"{count} {noun}{'' if count == 1 else 's'}"