Files
claude-config/bin/task_bench/runs.py
T

195 lines
10 KiB
Python

"""Run task-bench's containers: the image that adds Claude Code to the project's, the network that reaches only the API, one fix
by Haiku in a copy of the code, and the project's tests over what it changed."""
import contextlib
import dataclasses
import os
import pathlib
import shutil
import subprocess
import typing
from mutate_changes.tool_process import failed, Ran
from failure import Failure
from task_bench.figures import HOME, WORK
from mutate_changes.changed_lines import is_test
from task_bench.api_proxy import PORT
MODEL = "claude-haiku-5-5"
# No MCP server, skill, command or stored session, so a run starts from the code alone; -p needs --verbose with stream-json, and no
# one answers a permission prompt. Not --bare, which can't sign in by a long-lived token: docs/decisions.md → `Claude Code in a home
# of its own`.
CLAUDE = ["claude", "-p", "--strict-mcp-config", "--disable-slash-commands", "--no-session-persistence", "--model", MODEL, "--effort", "low", "--output-format", "stream-json", "--verbose", "--permission-mode", "bypassPermissions"]
PROMPT = "A test of this project fails with the message below. Find the bug in the code and fix it. Change no test.\n\n{message}"
TOKEN = "CLAUDE_CODE_OAUTH_TOKEN"
API = "api.anthropic.com:443"
# Holds only the proxy, which needs Python's standard library and nothing else.
PROXY_IMAGE = "python:3.15.2-slim"
# The installer puts claude in root's home, which no other user reads; each run works as the caller's user, so files it writes
# in the copy are the caller's.
DOCKERFILE = 'FROM {image}\nUSER root\nRUN curl -fsSL https://claude.ai/install.sh | bash && cp -L /root/.local/bin/claude /usr/local/bin/claude\n'
BUILD_SECONDS = 20 * 60
# How long one fix, or one run of the tests, may take before its container is removed; the run then counts as not passed.
RUN_SECONDS = 30 * 60
# What `docker run` exits with where the container, or its command, never started.
NOT_STARTED = (125, 126, 127)
# The memory each container gets without run-heavy, which hands its slot's share in RUN_HEAVY_MEMORY, in MiB.
MEMORY_MIB = 4096
class TimedOut(Failure):
"""docker still ran when its time was up."""
class Docker(typing.Protocol):
"""Runs `docker <args>` and waits for it, at most `seconds`, and is TimedOut after that. Its output goes to `stdout` and its
errors to `stderr` where the call gives them, and into the Ran otherwise."""
def __call__(self, args: list[str], *, seconds: float, stdin: str | None = None, stdout: typing.IO[bytes] | None = None, stderr: typing.IO[bytes] | None = None) -> Ran | Failure: ...
def docker_cli(env: typing.Mapping[str, str], run: typing.Callable[..., subprocess.CompletedProcess[bytes]] = subprocess.run) -> Docker:
def docker(args: list[str], *, seconds: float, stdin: str | None = None, stdout: typing.IO[bytes] | None = None, stderr: typing.IO[bytes] | None = None) -> Ran | Failure:
try:
ran = run(["docker", *args], env=env, input=None if stdin is None else stdin.encode(), stdin=subprocess.DEVNULL if stdin is None else None, stdout=stdout or subprocess.PIPE, stderr=stderr or subprocess.STDOUT, timeout=seconds, check=False)
except OSError as failure:
return Failure(f"docker could not start: {failure.strerror}")
except subprocess.TimeoutExpired:
return TimedOut(f"docker {args[0]} still ran after {round(seconds / 60)} min")
return Ran(ran.returncode, (ran.stdout or b"").decode("utf-8", errors="replace"))
return docker
@dataclasses.dataclass(frozen=True)
class Containers:
"""What one bench holds in docker, all named after `name`, which no other bench shares."""
docker: Docker
name: str
test: str
memory_mib: int
user: str
@property
def image(self) -> str:
return f"{self.name}:throwaway"
@property
def proxy(self) -> str:
return f"{self.name}-proxy"
@property
def running(self) -> str:
"""The one container that fixes or tests at a time."""
return f"{self.name}-run"
def limits(self) -> list[str]:
return ["--memory", f"{self.memory_mib}m", "--memory-swap", f"{self.memory_mib}m"]
def opened(containers: Containers, image: str) -> Failure | None:
"""Builds the bench's image, the project's with Claude Code added, and starts the network and its proxy.
The network has no route out. The proxy sits on it and on docker's default network, and tunnels to the API alone, so a run
reaches the API and nothing else: not a package index, and not the repository the code came from."""
docker, name = containers.docker, containers.name
steps: list[tuple[list[str], float, str | None]] = [
(["build", "--quiet", *containers.limits(), "--tag", containers.image, "-"], BUILD_SECONDS, DOCKERFILE.format(image=image)),
(["network", "create", "--internal", name], 60, None),
(["run", "--detach", "--rm", "--name", containers.proxy, "--memory", "256m", "--memory-swap", "256m", "--volume", f"{pathlib.Path(__file__).with_name('api_proxy.py')}:/api_proxy.py:ro", PROXY_IMAGE, "python", "/api_proxy.py", API], 300, None),
(["network", "connect", name, containers.proxy], 60, None),
]
for args, seconds, stdin in steps:
ran = docker(args, seconds=seconds, stdin=stdin)
if isinstance(ran, Failure) or ran.code:
return ran if isinstance(ran, Failure) else failed(f"docker {args[0]}", ran)
return None
def closed(containers: Containers) -> None:
"""Removes what `opened` made and any container still running. A bench that broke half-way made only some of them, and removing
one that is missing doesn't stop the rest."""
for args in (["rm", "--force", containers.running], ["rm", "--force", containers.proxy], ["network", "rm", containers.name], ["image", "rm", containers.image]):
containers.docker(args, seconds=120)
def ran_tests(containers: Containers, tree: str) -> Ran | Failure:
"""The project's test command, run at the top of tree with no network; the tests pass where it exits 0. TimedOut where it still
ran after RUN_SECONDS, and a Failure where docker could not run it."""
return run_in_image(containers, ["--network", "none", *in_tree(containers, tree), containers.image, "sh", "-c", containers.test])
def fix_attempted(containers: Containers, copy: str, message: str, stream: pathlib.Path) -> Failure | None:
"""Has Haiku fix the bug in copy, told only the failing assertion's message, and writes what its claude prints to `stream`, and its
errors beside it. A fix that still runs after RUN_SECONDS ends there, with the stream as far as it came.
The container's home is new and empty, so no instruction, memory, hook, skill or plugin of the caller loads; and docker hands the
token on by its name, so its value is on no command line. A Failure where docker could not run claude."""
args = ["--network", containers.name, *in_tree(containers, copy), "--env", TOKEN, "--env", f"HTTPS_PROXY=http://{containers.proxy}:{PORT}", "--env", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1", containers.image, *CLAUDE, PROMPT.format(message=message)]
errors_file = stream.with_suffix(".stderr")
try:
with stream.open("wb") as out, errors_file.open("wb") as errors:
ran = run_in_image(containers, args, stdout=out, stderr=errors)
except OSError as failure:
return Failure(f"could not write {stream}: {failure}")
# Out of time is the run's outcome, not the bench's failure.
if isinstance(ran, TimedOut) or not isinstance(ran, Failure):
return None
return Failure(f"{ran.reason}; its errors are in {errors_file}")
def run_in_image(containers: Containers, args: list[str], stdout: typing.IO[bytes] | None = None, stderr: typing.IO[bytes] | None = None) -> Ran | Failure:
"""One `docker run` as the bench's one running container, for at most RUN_SECONDS, after which the container is removed.
A run whose command never started is a Failure, since the next would end the same: counted as a fix that failed, it would fill
the table with runs that measured docker."""
ran = containers.docker(["run", "--rm", "--name", containers.running, *args], seconds=RUN_SECONDS, stdout=stdout, stderr=stderr)
if isinstance(ran, Failure):
containers.docker(["rm", "--force", containers.running], seconds=120)
return ran
if ran.code in NOT_STARTED:
return Failure(f"docker run could not start its command, and exited {ran.code}: {ran.output.strip()}".removesuffix(": "))
return ran
def in_tree(containers: Containers, tree: str) -> list[str]:
"""The options that run a container in tree, mounted at WORK on every run so no path tells the sides apart, as the caller's user."""
return [*containers.limits(), "--user", containers.user, "--env", f"HOME={HOME}", "--volume", f"{tree}:{WORK}", "--workdir", WORK]
def checked_tree(planted: str, copy: str, into: str) -> None:
"""Makes at `into` the tree the tests check a fix in: `planted`, the code with the bug as Haiku got it, with each file that is no
test as `copy` holds it after the fix. So a test Haiku changed, removed or added doesn't count, and neither does a file under
a `testdata` folder, where Go's tests keep the output they expect.
Raises OSError where a file cannot be copied."""
shutil.copytree(planted, into, symlinks=True)
before, after = set(entries(planted)), set(entries(copy))
for path in before - after:
if not of_tests(path):
with contextlib.suppress(FileNotFoundError):
os.unlink(os.path.join(into, path))
for path in sorted(after):
target = os.path.join(into, path)
# A link is copied as a link, never followed; and one that would stand in for a folder of the planted tree is left out.
if of_tests(path) or (os.path.isdir(target) and not os.path.islink(target)):
continue
os.makedirs(os.path.dirname(target), exist_ok=True)
with contextlib.suppress(FileNotFoundError):
os.unlink(target)
shutil.copy2(os.path.join(copy, path), target, follow_symlinks=False)
def of_tests(path: str) -> bool:
return is_test(path) or "testdata" in pathlib.PurePath(path).parts
def entries(top: str) -> list[str]:
"""Each file and link under top, by its path from top, without git's own under `.git`."""
found = []
for folder, folders, names in os.walk(top):
if folder == top and ".git" in folders:
folders.remove(".git")
links = [name for name in folders if os.path.islink(os.path.join(folder, name))]
found += [os.path.relpath(os.path.join(folder, name), top) for name in (*names, *links)]
return found