195 lines
10 KiB
Python
195 lines
10 KiB
Python
"""Run task-bench's containers: the image that adds Claude Code to the project's, the network that reaches only the API, one fix
|
|
by Haiku in a copy of the code, and the project's tests over what it changed."""
|
|
import contextlib
|
|
import dataclasses
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import subprocess
|
|
import typing
|
|
|
|
from mutate_changes.tool_process import failed, Ran
|
|
from failure import Failure
|
|
from task_bench.figures import HOME, WORK
|
|
from mutate_changes.changed_lines import is_test
|
|
from task_bench.api_proxy import PORT
|
|
|
|
MODEL = "claude-haiku-5-5"
|
|
# No MCP server, skill, command or stored session, so a run starts from the code alone; -p needs --verbose with stream-json, and no
|
|
# one answers a permission prompt. Not --bare, which can't sign in by a long-lived token: docs/decisions.md → `Claude Code in a home
|
|
# of its own`.
|
|
CLAUDE = ["claude", "-p", "--strict-mcp-config", "--disable-slash-commands", "--no-session-persistence", "--model", MODEL, "--effort", "low", "--output-format", "stream-json", "--verbose", "--permission-mode", "bypassPermissions"]
|
|
PROMPT = "A test of this project fails with the message below. Find the bug in the code and fix it. Change no test.\n\n{message}"
|
|
TOKEN = "CLAUDE_CODE_OAUTH_TOKEN"
|
|
API = "api.anthropic.com:443"
|
|
# Holds only the proxy, which needs Python's standard library and nothing else.
|
|
PROXY_IMAGE = "python:3.15.2-slim"
|
|
# The installer puts claude in root's home, which no other user reads; each run works as the caller's user, so files it writes
|
|
# in the copy are the caller's.
|
|
DOCKERFILE = 'FROM {image}\nUSER root\nRUN curl -fsSL https://claude.ai/install.sh | bash && cp -L /root/.local/bin/claude /usr/local/bin/claude\n'
|
|
BUILD_SECONDS = 20 * 60
|
|
# How long one fix, or one run of the tests, may take before its container is removed; the run then counts as not passed.
|
|
RUN_SECONDS = 30 * 60
|
|
# What `docker run` exits with where the container, or its command, never started.
|
|
NOT_STARTED = (125, 126, 127)
|
|
# The memory each container gets without run-heavy, which hands its slot's share in RUN_HEAVY_MEMORY, in MiB.
|
|
MEMORY_MIB = 4096
|
|
|
|
|
|
class TimedOut(Failure):
|
|
"""docker still ran when its time was up."""
|
|
|
|
|
|
class Docker(typing.Protocol):
|
|
"""Runs `docker <args>` and waits for it, at most `seconds`, and is TimedOut after that. Its output goes to `stdout` and its
|
|
errors to `stderr` where the call gives them, and into the Ran otherwise."""
|
|
|
|
def __call__(self, args: list[str], *, seconds: float, stdin: str | None = None, stdout: typing.IO[bytes] | None = None, stderr: typing.IO[bytes] | None = None) -> Ran | Failure: ...
|
|
|
|
|
|
def docker_cli(env: typing.Mapping[str, str], run: typing.Callable[..., subprocess.CompletedProcess[bytes]] = subprocess.run) -> Docker:
|
|
def docker(args: list[str], *, seconds: float, stdin: str | None = None, stdout: typing.IO[bytes] | None = None, stderr: typing.IO[bytes] | None = None) -> Ran | Failure:
|
|
try:
|
|
ran = run(["docker", *args], env=env, input=None if stdin is None else stdin.encode(), stdin=subprocess.DEVNULL if stdin is None else None, stdout=stdout or subprocess.PIPE, stderr=stderr or subprocess.STDOUT, timeout=seconds, check=False)
|
|
except OSError as failure:
|
|
return Failure(f"docker could not start: {failure.strerror}")
|
|
except subprocess.TimeoutExpired:
|
|
return TimedOut(f"docker {args[0]} still ran after {round(seconds / 60)} min")
|
|
return Ran(ran.returncode, (ran.stdout or b"").decode("utf-8", errors="replace"))
|
|
|
|
return docker
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Containers:
|
|
"""What one bench holds in docker, all named after `name`, which no other bench shares."""
|
|
docker: Docker
|
|
name: str
|
|
test: str
|
|
memory_mib: int
|
|
user: str
|
|
|
|
@property
|
|
def image(self) -> str:
|
|
return f"{self.name}:throwaway"
|
|
|
|
@property
|
|
def proxy(self) -> str:
|
|
return f"{self.name}-proxy"
|
|
|
|
@property
|
|
def running(self) -> str:
|
|
"""The one container that fixes or tests at a time."""
|
|
return f"{self.name}-run"
|
|
|
|
def limits(self) -> list[str]:
|
|
return ["--memory", f"{self.memory_mib}m", "--memory-swap", f"{self.memory_mib}m"]
|
|
|
|
|
|
def opened(containers: Containers, image: str) -> Failure | None:
|
|
"""Builds the bench's image, the project's with Claude Code added, and starts the network and its proxy.
|
|
|
|
The network has no route out. The proxy sits on it and on docker's default network, and tunnels to the API alone, so a run
|
|
reaches the API and nothing else: not a package index, and not the repository the code came from."""
|
|
docker, name = containers.docker, containers.name
|
|
steps: list[tuple[list[str], float, str | None]] = [
|
|
(["build", "--quiet", *containers.limits(), "--tag", containers.image, "-"], BUILD_SECONDS, DOCKERFILE.format(image=image)),
|
|
(["network", "create", "--internal", name], 60, None),
|
|
(["run", "--detach", "--rm", "--name", containers.proxy, "--memory", "256m", "--memory-swap", "256m", "--volume", f"{pathlib.Path(__file__).with_name('api_proxy.py')}:/api_proxy.py:ro", PROXY_IMAGE, "python", "/api_proxy.py", API], 300, None),
|
|
(["network", "connect", name, containers.proxy], 60, None),
|
|
]
|
|
for args, seconds, stdin in steps:
|
|
ran = docker(args, seconds=seconds, stdin=stdin)
|
|
if isinstance(ran, Failure) or ran.code:
|
|
return ran if isinstance(ran, Failure) else failed(f"docker {args[0]}", ran)
|
|
return None
|
|
|
|
|
|
def closed(containers: Containers) -> None:
|
|
"""Removes what `opened` made and any container still running. A bench that broke half-way made only some of them, and removing
|
|
one that is missing doesn't stop the rest."""
|
|
for args in (["rm", "--force", containers.running], ["rm", "--force", containers.proxy], ["network", "rm", containers.name], ["image", "rm", containers.image]):
|
|
containers.docker(args, seconds=120)
|
|
|
|
|
|
def ran_tests(containers: Containers, tree: str) -> Ran | Failure:
|
|
"""The project's test command, run at the top of tree with no network; the tests pass where it exits 0. TimedOut where it still
|
|
ran after RUN_SECONDS, and a Failure where docker could not run it."""
|
|
return run_in_image(containers, ["--network", "none", *in_tree(containers, tree), containers.image, "sh", "-c", containers.test])
|
|
|
|
|
|
def fix_attempted(containers: Containers, copy: str, message: str, stream: pathlib.Path) -> Failure | None:
|
|
"""Has Haiku fix the bug in copy, told only the failing assertion's message, and writes what its claude prints to `stream`, and its
|
|
errors beside it. A fix that still runs after RUN_SECONDS ends there, with the stream as far as it came.
|
|
|
|
The container's home is new and empty, so no instruction, memory, hook, skill or plugin of the caller loads; and docker hands the
|
|
token on by its name, so its value is on no command line. A Failure where docker could not run claude."""
|
|
args = ["--network", containers.name, *in_tree(containers, copy), "--env", TOKEN, "--env", f"HTTPS_PROXY=http://{containers.proxy}:{PORT}", "--env", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1", containers.image, *CLAUDE, PROMPT.format(message=message)]
|
|
errors_file = stream.with_suffix(".stderr")
|
|
try:
|
|
with stream.open("wb") as out, errors_file.open("wb") as errors:
|
|
ran = run_in_image(containers, args, stdout=out, stderr=errors)
|
|
except OSError as failure:
|
|
return Failure(f"could not write {stream}: {failure}")
|
|
# Out of time is the run's outcome, not the bench's failure.
|
|
if isinstance(ran, TimedOut) or not isinstance(ran, Failure):
|
|
return None
|
|
return Failure(f"{ran.reason}; its errors are in {errors_file}")
|
|
|
|
|
|
def run_in_image(containers: Containers, args: list[str], stdout: typing.IO[bytes] | None = None, stderr: typing.IO[bytes] | None = None) -> Ran | Failure:
|
|
"""One `docker run` as the bench's one running container, for at most RUN_SECONDS, after which the container is removed.
|
|
|
|
A run whose command never started is a Failure, since the next would end the same: counted as a fix that failed, it would fill
|
|
the table with runs that measured docker."""
|
|
ran = containers.docker(["run", "--rm", "--name", containers.running, *args], seconds=RUN_SECONDS, stdout=stdout, stderr=stderr)
|
|
if isinstance(ran, Failure):
|
|
containers.docker(["rm", "--force", containers.running], seconds=120)
|
|
return ran
|
|
if ran.code in NOT_STARTED:
|
|
return Failure(f"docker run could not start its command, and exited {ran.code}: {ran.output.strip()}".removesuffix(": "))
|
|
return ran
|
|
|
|
|
|
def in_tree(containers: Containers, tree: str) -> list[str]:
|
|
"""The options that run a container in tree, mounted at WORK on every run so no path tells the sides apart, as the caller's user."""
|
|
return [*containers.limits(), "--user", containers.user, "--env", f"HOME={HOME}", "--volume", f"{tree}:{WORK}", "--workdir", WORK]
|
|
|
|
|
|
def checked_tree(planted: str, copy: str, into: str) -> None:
|
|
"""Makes at `into` the tree the tests check a fix in: `planted`, the code with the bug as Haiku got it, with each file that is no
|
|
test as `copy` holds it after the fix. So a test Haiku changed, removed or added doesn't count, and neither does a file under
|
|
a `testdata` folder, where Go's tests keep the output they expect.
|
|
|
|
Raises OSError where a file cannot be copied."""
|
|
shutil.copytree(planted, into, symlinks=True)
|
|
before, after = set(entries(planted)), set(entries(copy))
|
|
for path in before - after:
|
|
if not of_tests(path):
|
|
with contextlib.suppress(FileNotFoundError):
|
|
os.unlink(os.path.join(into, path))
|
|
for path in sorted(after):
|
|
target = os.path.join(into, path)
|
|
# A link is copied as a link, never followed; and one that would stand in for a folder of the planted tree is left out.
|
|
if of_tests(path) or (os.path.isdir(target) and not os.path.islink(target)):
|
|
continue
|
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
with contextlib.suppress(FileNotFoundError):
|
|
os.unlink(target)
|
|
shutil.copy2(os.path.join(copy, path), target, follow_symlinks=False)
|
|
|
|
|
|
def of_tests(path: str) -> bool:
|
|
return is_test(path) or "testdata" in pathlib.PurePath(path).parts
|
|
|
|
|
|
def entries(top: str) -> list[str]:
|
|
"""Each file and link under top, by its path from top, without git's own under `.git`."""
|
|
found = []
|
|
for folder, folders, names in os.walk(top):
|
|
if folder == top and ".git" in folders:
|
|
folders.remove(".git")
|
|
links = [name for name in folders if os.path.islink(os.path.join(folder, name))]
|
|
found += [os.path.relpath(os.path.join(folder, name), top) for name in (*names, *links)]
|
|
return found
|