308 lines
16 KiB
Python
308 lines
16 KiB
Python
r"""Measure how hard a project's code is to fix a bug in, at a base commit and at a head: plant bugs its tests catch, have Haiku fix
|
|
each several times at both, and print how often the fix passes and what it took.
|
|
|
|
usage: task-bench --image <image> --test <shell text> [--base <ref>] [--head <ref>] [--prefix <shell text>] [--seed <n>] [--bugs <n>] [--runs <n>]
|
|
|
|
Run it in a checkout, under ~/.claude/bin/run-heavy, with a long-lived token in CLAUDE_CODE_OAUTH_TOKEN, which `claude setup-token`
|
|
prints. The head is --head, by default HEAD. The base is the merge base of the head and --base, by default the default branch:
|
|
origin's as last fetched, else main or master. `--base HEAD` benches the head against itself. No model judges anything: the bench
|
|
runs programs, and reads Haiku's own stream.
|
|
|
|
- Plant. It copies the tree at the base and runs Gremlins there, as `gremlins unleash`, over the Go files that hold the places
|
|
`larv-review places` prints, the ones the comprehension seats found hardest. Of the mutants the tests kill, it draws --bugs, 5 by
|
|
default, in the order --seed sets, 1 by default. A drawn mutant becomes a bug where its line stands unchanged at the head, and
|
|
where, planted at each side, the tests fail on an assertion whose message says something. Others are passed over, so a change to
|
|
the very line of a bug is not benched. Go is the one language so far.
|
|
- Run. Per bug and side, --runs times, 5 by default, sides in turn: a copy of that side's tree with the bug in it, committed once,
|
|
so git shows nothing of the bug. In it, in a container, `claude -p --strict-mcp-config --disable-slash-commands
|
|
--no-session-persistence --model claude-haiku-5-5 --effort low` works with every tool, told the failing assertion's message at that
|
|
side, without the test's name, file or line. The container's Claude home is new and empty, so the project's own CLAUDE.md, or
|
|
AGENTS.md, in the copy is all the setup that loads. Its network reaches api.anthropic.com and nothing else. A fix ends after 30
|
|
minutes.
|
|
- Check. The test command runs over the planted tree with each file that is no test as Haiku left it; a fix passes where it exits 0.
|
|
Tests Haiku changed, removed or added count for nothing.
|
|
|
|
--image is the project's development image, such as golang:1.27.4. The bench adds Claude Code to it, as its installer installs it,
|
|
so the image needs curl and bash; the result names the version. Haiku works in that image, and --test runs in it, as sh runs it at
|
|
the copy's top, with no network: the image must hold what the tests download otherwise.
|
|
--prefix is shell text that runs Gremlins, on the host, as mutate-changes' --prefix does: sh runs `<prefix> "$@"` in the copy of the
|
|
base, which $PWD names, with `gremlins unleash ...` as "$@". For example:
|
|
|
|
docker run --rm --memory 8g --memory-swap 8g -v "$PWD:$PWD" -w "$PWD" golang:1.27.4 \
|
|
sh -c 'shift; exec go run github.com/go-gremlins/gremlins/cmd/gremlins@v0.6.0 "$@"' sh
|
|
|
|
It prints the base and the head, then a table: a row per bug and side with how many fixes passed and the medians, over the runs
|
|
that finished, of the input, cache-write, cache-read and output tokens, the turns, the wall time, and the reads and searches before
|
|
the run first opened the file the bug is in; then a row per side over every bug, and the head's change against the base. Each
|
|
run's result and stream are in the folder its last line names, `task-bench` in the chunk's review folder, which each bench empties
|
|
first.
|
|
|
|
Exit 1, saying why, on an unknown or repeated argument, a missing --image or --test, a count that is no whole number from 1, or
|
|
no token; and where git, docker or Gremlins fails, the seats' places name no Go file, the tests fail at a side without a bug, no
|
|
mutant becomes a bug, or the API refuses a run.
|
|
"""
|
|
import dataclasses
|
|
import functools
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import sys
|
|
import tarfile
|
|
import typing
|
|
import uuid
|
|
|
|
from claude_code.tokens import Tokens
|
|
from failure import Failure
|
|
from git_cli import git_repo
|
|
from git_cli.git_repo import default_ref, Git, top
|
|
from larv_review.history_commands import read_places
|
|
from machine_load.slots import CPUS_PER_SLOT
|
|
from mutate_changes.tool_process import failed, Ran, run_tool
|
|
from review_rules.review_folder import made_review_folder
|
|
from run_heavy.capped_docker import SHARE_VARIABLE
|
|
from task_bench.figures import Figures, figures, Result, SIDES, table, WORK
|
|
from task_bench.plant import drawn, killed_mutants, line_at_head, Mutant, mutated, place_files, task_text
|
|
from task_bench.runs import checked_tree, closed, Containers, docker_cli, fix_attempted, MEMORY_MIB, opened, tests_run, TOKEN
|
|
|
|
USAGE = "usage: task-bench --image <image> --test <shell text> [--base <ref>] [--head <ref>] [--prefix <shell text>] [--seed <n>] [--bugs <n>] [--runs <n>]"
|
|
# A panel of three readers, seated as the README's audience, picked these options on 2026-11-21, each 3 to 0.
|
|
OPTIONS = ("--base", "--bugs", "--head", "--image", "--prefix", "--runs", "--seed", "--test")
|
|
COUNTS = {"--bugs": "5", "--runs": "5", "--seed": "1"}
|
|
# The planted commit names no one, and no hook or signing key of the caller's reaches it.
|
|
COMMIT = ["-c", "user.name=task-bench", "-c", "user.email=task-bench@localhost", "-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", "commit", "--quiet", "--message", "Import"]
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Task:
|
|
"""A bug to fix: its number from 1, the mutant at the base, and per side the planted tree and what Haiku is told there."""
|
|
number: int
|
|
mutant: Mutant
|
|
trees: dict[str, str]
|
|
messages: dict[str, str]
|
|
|
|
def named(self) -> str:
|
|
return f"{self.mutant.path}:{self.mutant.line}:{self.mutant.column} {self.mutant.mutator}"
|
|
|
|
|
|
def main(argv: list[str], env: typing.Mapping[str, str]) -> int:
|
|
"""Runs task-bench with the arguments argv; git, docker and Gremlins run in the environment env."""
|
|
if argv in (["-h"], ["--help"]):
|
|
print(__doc__.strip())
|
|
return 0
|
|
given = options(argv, env)
|
|
if isinstance(given, Failure):
|
|
print(f"{given.reason}; {USAGE}", file=sys.stderr)
|
|
return 1
|
|
home = pathlib.Path(env.get("HOME") or pathlib.Path.home())
|
|
places = read_places(home)
|
|
report = places if isinstance(places, Failure) else bench(given, env, places, home)
|
|
if isinstance(report, Failure):
|
|
print(report.reason, file=sys.stderr)
|
|
return 1
|
|
print("\n".join(report))
|
|
return 0
|
|
|
|
|
|
def options(argv: list[str], env: typing.Mapping[str, str]) -> dict[str, str] | Failure:
|
|
given: dict[str, str] = {}
|
|
rest = list(argv)
|
|
while rest:
|
|
name, equals, value = rest.pop(0).partition("=")
|
|
if name not in OPTIONS or name in given:
|
|
return Failure(f"unknown or repeated argument {name}")
|
|
if not equals and not rest:
|
|
return Failure(f"{name} needs a value")
|
|
given[name] = value if equals else rest.pop(0)
|
|
for name in ("--image", "--test"):
|
|
if not given.get(name):
|
|
return Failure(f"{name} is missing")
|
|
for name, default in COUNTS.items():
|
|
if not given.setdefault(name, default).isdecimal() or int(given[name]) < 1:
|
|
return Failure(f"{name} takes a whole number from 1, not {given[name]}")
|
|
if not env.get(TOKEN):
|
|
return Failure(f"{TOKEN} holds no token")
|
|
return given
|
|
|
|
|
|
def bench(given: dict[str, str], env: typing.Mapping[str, str], places: str, home: pathlib.Path) -> list[str] | Failure:
|
|
"""The report of one bench of the checkout the process runs in, with `places` what `larv-review places` prints there."""
|
|
git = functools.partial(git_repo.git, env=env)
|
|
checkout = top(git)
|
|
if isinstance(checkout, Failure):
|
|
return checkout
|
|
sides = commits(given, checkout, git)
|
|
if isinstance(sides, Failure):
|
|
return sides
|
|
print(f"task-bench: base {sides['base'][:12]}, head {sides['head'][:12]}.", file=sys.stderr)
|
|
review = made_review_folder(checkout, home)
|
|
if isinstance(review, Failure):
|
|
return review
|
|
folder = review / "task-bench"
|
|
containers = Containers(docker_cli(env), f"task-bench-{uuid.uuid4().hex[:12]}", given["--test"], int(env.get(SHARE_VARIABLE) or MEMORY_MIB), f"{os.getuid()}:{os.getgid()}")
|
|
try:
|
|
shutil.rmtree(folder, ignore_errors=True)
|
|
trees: dict[str, str] = {}
|
|
for side, commit in sides.items():
|
|
tree = archived(commit, folder / "trees" / side, checkout, git)
|
|
if isinstance(tree, Failure):
|
|
return tree
|
|
trees[side] = tree
|
|
files = [path for path in place_files(places) if pathlib.Path(trees["base"], path).is_file()]
|
|
if not files:
|
|
return Failure("The places the seats found hard name no Go file that the base holds, so there is nowhere to plant a bug.")
|
|
run = functools.partial(run_tool, prefix=given.get("--prefix"), env=env, killpg=os.killpg)
|
|
mutants = killed_mutants(trees["base"], files, run, CPUS_PER_SLOT)
|
|
if isinstance(mutants, Failure):
|
|
return mutants
|
|
try:
|
|
found = opened(containers, given["--image"]) or benched(given, checkout, sides, trees, mutants, containers, folder, git)
|
|
finally:
|
|
closed(containers)
|
|
except OSError as failure:
|
|
return Failure(f"could not write under {folder}: {failure}")
|
|
return found if isinstance(found, Failure) else [*found, f"Each run's result and stream: {folder}"]
|
|
|
|
|
|
def commits(given: dict[str, str], checkout: str, git: Git) -> dict[str, str] | Failure:
|
|
"""The base's commit and the head's."""
|
|
head = git("rev-parse", "--verify", f"{given.get('--head', 'HEAD')}^{{commit}}", cwd=checkout)
|
|
against = given.get("--base") or default_ref(cwd=checkout, run=git)
|
|
if isinstance(head, Failure):
|
|
return head
|
|
if isinstance(against, Failure):
|
|
return against
|
|
base = git("merge-base", against, head.strip(), cwd=checkout)
|
|
return base if isinstance(base, Failure) else {"base": base.strip(), "head": head.strip()}
|
|
|
|
|
|
def archived(commit: str, into: pathlib.Path, checkout: str, git: Git) -> str | Failure:
|
|
"""Makes `into` the tree at commit, without git's own data, and returns its path."""
|
|
into.mkdir(parents=True)
|
|
archive = into.with_suffix(".tar")
|
|
wrote = git("archive", "--format=tar", "--output", str(archive), commit, cwd=checkout)
|
|
if isinstance(wrote, Failure):
|
|
return wrote
|
|
try:
|
|
with tarfile.open(archive) as held:
|
|
held.extractall(into, filter="tar")
|
|
except tarfile.TarError as failure:
|
|
return Failure(f"could not unpack the tree at {commit}: {failure}")
|
|
archive.unlink()
|
|
return str(into)
|
|
|
|
|
|
def benched(given: dict[str, str], checkout: str, sides: dict[str, str], trees: dict[str, str], mutants: list[Mutant], containers: Containers, folder: pathlib.Path, git: Git) -> list[str] | Failure:
|
|
"""Plants the bugs, runs each fix, writes the results into folder and returns the report."""
|
|
for side, tree in trees.items():
|
|
ran = tests_in_copy(containers, tree, folder)
|
|
if isinstance(ran, Failure) or ran.code:
|
|
return ran if isinstance(ran, Failure) else failed(f"The tests fail at the {side} without a planted bug: the test command", ran)
|
|
tasks: list[Task] = []
|
|
for mutant in drawn(mutants, int(given["--seed"])):
|
|
if len(tasks) == int(given["--bugs"]):
|
|
break
|
|
task = planted(len(tasks) + 1, mutant, checkout, sides, trees, containers, folder, git)
|
|
if isinstance(task, Failure):
|
|
return task
|
|
tasks += [task] if task else []
|
|
if not tasks:
|
|
return Failure(f"None of the {len(mutants)} mutants the tests kill becomes a bug: each is on a line the head changed, or fails no assertion with a message.")
|
|
results: list[Result] = []
|
|
total = int(given["--runs"]) * len(tasks) * len(SIDES)
|
|
for number in range(1, int(given["--runs"]) + 1):
|
|
for task in tasks:
|
|
for side in SIDES:
|
|
result = fixed(task, side, number, containers, folder)
|
|
if isinstance(result, Failure):
|
|
return result
|
|
results.append(result)
|
|
print(f"task-bench: run {len(results)} of {total}, bug {task.number} at the {side}: {'passed' if result.passed else 'not passed'}.", file=sys.stderr)
|
|
(folder / "results.json").write_text(json.dumps({"sides": sides, "seed": int(given["--seed"]), "bugs": {task.number: task.named() for task in tasks}, "runs": [recorded(result) for result in results]}, indent="\t") + "\n", encoding="utf-8")
|
|
ran_on = sorted({f"{found.model} in Claude Code {found.version}" for result in results if (found := result.figures)})
|
|
short = [] if len(tasks) == int(given["--bugs"]) else [f"Planted {len(tasks)} of the {given['--bugs']} bugs asked for: no other mutant the tests kill becomes one."]
|
|
return [
|
|
f"task-bench: base {sides['base'][:12]}, head {sides['head'][:12]}. {counted(len(tasks), 'bug')}, fixed {counted(int(given['--runs']), 'time')} at each side by {' and '.join(ran_on) or 'no run that finished'}.",
|
|
*short,
|
|
"",
|
|
*table(results, {task.number: task.named() for task in tasks}),
|
|
"",
|
|
]
|
|
|
|
|
|
def planted(number: int, mutant: Mutant, checkout: str, sides: dict[str, str], trees: dict[str, str], containers: Containers, folder: pathlib.Path, git: Git) -> Task | Failure | None:
|
|
"""The task of fixing mutant, with its planted tree at each side; None where the mutant is no bug at both sides.
|
|
|
|
The bug is the same change to the same line at both sides, so only the code around it differs between them."""
|
|
diff = git("diff", "-U0", "--no-color", "--no-ext-diff", "--no-textconv", sides["base"], sides["head"], "--", mutant.path, cwd=checkout, errors="replace")
|
|
if isinstance(diff, Failure):
|
|
return diff
|
|
at_head = line_at_head(diff, mutant.line)
|
|
if at_head is None:
|
|
return None
|
|
planted_trees, messages = {}, {}
|
|
for side, placed in (("base", mutant), ("head", mutant._replace(line=at_head))):
|
|
source = pathlib.Path(trees[side], mutant.path)
|
|
changed = mutated(source.read_bytes(), placed) if source.is_file() else None
|
|
if changed is None:
|
|
return None
|
|
tree = folder / "bugs" / str(number) / side
|
|
shutil.rmtree(tree, ignore_errors=True)
|
|
shutil.copytree(trees[side], tree, symlinks=True)
|
|
(tree / mutant.path).write_bytes(changed)
|
|
ran = tests_in_copy(containers, str(tree), folder)
|
|
if isinstance(ran, Failure):
|
|
return ran
|
|
message = task_text(ran.output) if ran.code else None
|
|
if message is None:
|
|
return None
|
|
for args in (["init", "--quiet"], ["add", "--all", "--force"], COMMIT):
|
|
done = git(*args, cwd=str(tree))
|
|
if isinstance(done, Failure):
|
|
return done
|
|
planted_trees[side], messages[side] = str(tree), message
|
|
return Task(number, mutant, planted_trees, messages)
|
|
|
|
|
|
def tests_in_copy(containers: Containers, tree: str, folder: pathlib.Path) -> Ran | Failure:
|
|
"""The test command's run in a copy of tree that goes afterwards, so what the tests write stays out of tree."""
|
|
copy = folder / "tested"
|
|
shutil.copytree(tree, copy, symlinks=True)
|
|
try:
|
|
return tests_run(containers, str(copy))
|
|
finally:
|
|
shutil.rmtree(copy, ignore_errors=True)
|
|
|
|
|
|
def fixed(task: Task, side: str, number: int, containers: Containers, folder: pathlib.Path) -> Result | Failure:
|
|
"""One run: Haiku's fix of task at side in a copy of its planted tree, then the tests over what it changed. The stream stays in
|
|
folder; the copy goes."""
|
|
stream = folder / "runs" / f"bug-{task.number}-{side}-{number}.jsonl"
|
|
stream.parent.mkdir(exist_ok=True)
|
|
copy, check = folder / "copy", folder / "check"
|
|
try:
|
|
shutil.copytree(task.trees[side], copy, symlinks=True)
|
|
attempted = fix_attempted(containers, str(copy), task.messages[side], stream)
|
|
if isinstance(attempted, Failure):
|
|
return attempted
|
|
found = figures(stream.read_bytes(), os.path.join(WORK, task.mutant.path))
|
|
if isinstance(found, Failure):
|
|
return found
|
|
checked_tree(task.trees[side], str(copy), str(check))
|
|
ran = tests_run(containers, str(check))
|
|
return Result(task.number, side, number, not isinstance(ran, Failure) and ran.code == 0, found)
|
|
finally:
|
|
shutil.rmtree(copy, ignore_errors=True)
|
|
shutil.rmtree(check, ignore_errors=True)
|
|
|
|
|
|
def recorded(result: Result) -> dict:
|
|
"""A run as results.json holds it."""
|
|
found = result.figures or Figures("", "", Tokens(), 0, 0, None)
|
|
measured = {"claude_code": found.version, "model": found.model, "tokens": dataclasses.asdict(found.tokens), "turns": found.turns, "seconds": found.seconds, "reads": found.reads}
|
|
return {"bug": result.task, "side": result.side, "run": result.run, "passed": result.passed, "finished": result.figures is not None, **(measured if result.figures else {})}
|
|
|
|
|
|
def counted(count: int, noun: str) -> str:
|
|
return f"{count} {noun}{'' if count == 1 else 's'}"
|