Files
claude-config/bin/task_bench/runs.py
T
2026-11-21 01:39:12 +01:00

172 lines
9.0 KiB
Python

"""Run task-bench's containers: the image that adds Claude Code to the project's, the network that reaches only the API, one fix
by Haiku in a copy of the code, and the project's tests over what it changed."""
import contextlib
import dataclasses
import os
import pathlib
import shutil
import subprocess
import typing
from mutate_changes.tool_process import failed, Ran
from failure import Failure
from task_bench.figures import HOME, WORK
from mutate_changes.changed_lines import is_test
from task_bench.api_proxy import PORT
MODEL = "claude-haiku-5-5"
# No MCP server, skill, command or stored session, so a run starts from the code alone; -p needs --verbose with stream-json, and no
# one answers a permission prompt. Not --bare, which can't sign in by a long-lived token: docs/decisions.md → `Claude Code in a home
# of its own`.
CLAUDE = ["claude", "-p", "--strict-mcp-config", "--disable-slash-commands", "--no-session-persistence", "--model", MODEL, "--effort", "low", "--output-format", "stream-json", "--verbose", "--permission-mode", "bypassPermissions"]
PROMPT = "A test of this project fails with the message below. Find the bug in the code and fix it. Change no test.\n\n{message}"
TOKEN = "CLAUDE_CODE_OAUTH_TOKEN"
API = "api.anthropic.com:443"
# Holds only the proxy, which needs Python's standard library and nothing else.
PROXY_IMAGE = "python:3.15.2-slim"
# The installer puts claude in root's home, which no other user reads; each run works as the caller's user, so files it writes
# in the copy are the caller's.
DOCKERFILE = 'FROM {image}\nUSER root\nRUN curl -fsSL https://claude.ai/install.sh | bash && cp -L /root/.local/bin/claude /usr/local/bin/claude\n'
BUILD_SECONDS = 20 * 60
# How long one fix, or one run of the tests, may take before its container is removed; the run then counts as not passed.
RUN_SECONDS = 30 * 60
# The memory each container gets without run-heavy, which hands its slot's share in RUN_HEAVY_MEMORY, in MiB.
MEMORY_MIB = 4096
class Docker(typing.Protocol):
"""Runs `docker <args>` and waits for it, at most `seconds`. Its output goes to `stdout` and its errors to `stderr` where the
call gives them, and into the Ran otherwise."""
def __call__(self, args: list[str], *, seconds: float, stdin: str | None = None, stdout: typing.IO[bytes] | None = None, stderr: typing.IO[bytes] | None = None) -> Ran | Failure: ...
def docker_cli(env: typing.Mapping[str, str], run: typing.Callable[..., subprocess.CompletedProcess[bytes]] = subprocess.run) -> Docker:
def docker(args: list[str], *, seconds: float, stdin: str | None = None, stdout: typing.IO[bytes] | None = None, stderr: typing.IO[bytes] | None = None) -> Ran | Failure:
try:
ran = run(["docker", *args], env=env, input=None if stdin is None else stdin.encode(), stdin=subprocess.DEVNULL if stdin is None else None, stdout=stdout or subprocess.PIPE, stderr=stderr or subprocess.STDOUT, timeout=seconds, check=False)
except OSError as failure:
return Failure(f"docker could not start: {failure.strerror}")
except subprocess.TimeoutExpired:
return Failure(f"docker {args[0]} still ran after {round(seconds / 60)} min")
return Ran(ran.returncode, (ran.stdout or b"").decode("utf-8", errors="replace"))
return docker
@dataclasses.dataclass(frozen=True)
class Containers:
"""What one bench holds in docker, all named after `name`, which no other bench shares."""
docker: Docker
name: str
test: str
memory_mib: int
user: str
@property
def image(self) -> str:
return f"{self.name}:throwaway"
@property
def proxy(self) -> str:
return f"{self.name}-proxy"
@property
def running(self) -> str:
"""The one container that fixes or tests at a time."""
return f"{self.name}-run"
def limits(self) -> list[str]:
return ["--memory", f"{self.memory_mib}m", "--memory-swap", f"{self.memory_mib}m"]
def opened(containers: Containers, image: str) -> Failure | None:
"""Builds the bench's image, the project's with Claude Code added, and starts the network and its proxy.
The network has no route out. The proxy sits on it and on docker's default network, and tunnels to the API alone, so a run
reaches the API and nothing else: not a package index, and not the repository the code came from."""
docker, name = containers.docker, containers.name
steps: list[tuple[list[str], float, str | None]] = [
(["build", "--quiet", *containers.limits(), "--tag", containers.image, "-"], BUILD_SECONDS, DOCKERFILE.format(image=image)),
(["network", "create", "--internal", name], 60, None),
(["run", "--detach", "--rm", "--name", containers.proxy, "--memory", "256m", "--memory-swap", "256m", "--volume", f"{pathlib.Path(__file__).with_name('api_proxy.py')}:/api_proxy.py:ro", PROXY_IMAGE, "python", "/api_proxy.py", API], 300, None),
(["network", "connect", name, containers.proxy], 60, None),
]
for args, seconds, stdin in steps:
ran = docker(args, seconds=seconds, stdin=stdin)
if isinstance(ran, Failure) or ran.code:
return ran if isinstance(ran, Failure) else failed(f"docker {args[0]}", ran)
return None
def closed(containers: Containers) -> None:
"""Removes what `opened` made and any container still running, where each exists; a bench that broke half-way leaves the rest."""
for args in (["rm", "--force", containers.running], ["rm", "--force", containers.proxy], ["network", "rm", containers.name], ["image", "rm", containers.image]):
containers.docker(args, seconds=120)
def ran_tests(containers: Containers, tree: str) -> Ran | Failure:
"""The project's test command, run at the top of tree with no network; the tests pass where it exits 0. A Failure where it still
ran after RUN_SECONDS."""
ran = containers.docker(["run", "--rm", "--name", containers.running, "--network", "none", *in_tree(containers, tree), containers.image, "sh", "-c", containers.test], seconds=RUN_SECONDS)
if isinstance(ran, Failure):
containers.docker(["rm", "--force", containers.running], seconds=120)
return ran
def fix_attempted(containers: Containers, copy: str, message: str, stream: pathlib.Path) -> Failure | None:
"""Has Haiku fix the bug in copy, told only the failing assertion's message, and writes what its claude prints to `stream`, and its
errors beside it. A fix that still runs after RUN_SECONDS ends there, with the stream as far as it came.
The container's home is new and empty, so no instruction, memory, hook, skill or plugin of the caller loads; and docker hands the
token on by its name, so its value is on no command line. A Failure where docker itself could not run."""
args = ["run", "--rm", "--name", containers.running, "--network", containers.name, *in_tree(containers, copy), "--env", TOKEN, "--env", f"HTTPS_PROXY=http://{containers.proxy}:{PORT}", "--env", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1", containers.image, *CLAUDE, PROMPT.format(message=message)]
try:
with stream.open("wb") as out, stream.with_suffix(".stderr").open("wb") as errors:
ran = containers.docker(args, seconds=RUN_SECONDS, stdout=out, stderr=errors)
except OSError as failure:
return Failure(f"could not write {stream}: {failure}")
if isinstance(ran, Failure):
containers.docker(["rm", "--force", containers.running], seconds=120)
# Out of time is the run's outcome, not the bench's failure.
return None if "still ran" in ran.reason else ran
return None
def in_tree(containers: Containers, tree: str) -> list[str]:
"""The options that run a container in tree, mounted at WORK on every run so no path tells the sides apart, as the caller's user."""
return [*containers.limits(), "--user", containers.user, "--env", f"HOME={HOME}", "--volume", f"{tree}:{WORK}", "--workdir", WORK]
def checked_tree(planted: str, copy: str, into: str) -> None:
"""Makes at `into` the tree the tests check a fix in: `planted`, the code with the bug as Haiku got it, with each file that is no
test as `copy` holds it after the fix. So a test Haiku changed, removed or added counts for nothing.
Raises OSError where a file cannot be copied."""
shutil.copytree(planted, into, symlinks=True)
before, after = set(entries(planted)), set(entries(copy))
for path in before - after:
if not is_test(path):
with contextlib.suppress(FileNotFoundError):
os.unlink(os.path.join(into, path))
for path in sorted(after):
target = os.path.join(into, path)
# A link is copied as a link, never followed; and one that would stand in for a folder of the planted tree is left out.
if is_test(path) or (os.path.isdir(target) and not os.path.islink(target)):
continue
os.makedirs(os.path.dirname(target), exist_ok=True)
with contextlib.suppress(FileNotFoundError):
os.unlink(target)
shutil.copy2(os.path.join(copy, path), target, follow_symlinks=False)
def entries(top: str) -> list[str]:
"""Each file and link under top, by its path from top, without git's own under `.git`."""
found = []
for folder, folders, names in os.walk(top):
if folder == top and ".git" in folders:
folders.remove(".git")
links = [name for name in folders if os.path.islink(os.path.join(folder, name))]
found += [os.path.relpath(os.path.join(folder, name), top) for name in (*names, *links)]
return found