Files
claude-config/bin/task_bench/test_task_bench.py
T

592 lines
35 KiB
Python

"""task-bench from its command line down: a whole bench over a small repository with docker, Gremlins and claude faked, the tree a
fix is checked in, and the proxy that reaches only the API."""
import asyncio
import contextlib
import io
import json
import os
import pathlib
import subprocess
import sys
import tempfile
import textwrap
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from task_bench import api_proxy, main, runs
from task_bench.runs import checked_tree, closed, Containers, docker_cli, fix_attempted, opened, ran_tests, TimedOut
from task_bench.plant import drawn, Mutant
from failure import Failure
from test_env import isolated_env
from mutate_changes.tool_process import Ran
from task_bench.figures import Result
BASE_CODE = "package calc\n\nfunc Add(a, b int) int {\n\treturn a + b\n}\n\nfunc Twice(a int) int {\n\treturn a * 2\n}\n"
# The head adds a line right above Add, which moves it down, and rewrites Twice's line.
HEAD_CODE = BASE_CODE.replace("func Add", "// Add adds.\nfunc Add").replace("a * 2", "2 * a")
# Stands in for `go test`: a shell text in a file named as a Go test, which fails as go test reports a failed assertion.
TESTS = textwrap.dedent("""\
grep -q 'a + b' calc.go && exit 0
printf -- '--- FAIL: TestAdd (0.00s)\\n calc_test.go:9: Add(1, 2) = -1, want 3\\nFAIL\\n'
exit 1
""")
# Reports `+` in Add, which the head keeps, and the `*` in Twice, which the head rewrites, as killed; and a mutant no test caught.
GREMLINS = textwrap.dedent("""\
import json, os, sys
lines = open("calc.go").read().split("\\n")
found = [{"line": at, "column": 11, "status": "KILLED", "type": "ARITHMETIC_BASE"} for at, line in enumerate(lines, 1) if " + " in line or " * " in line]
report = {"files": [{"file_name": "calc.go", "mutations": [*found, {**found[0], "status": "LIVED", "type": "INVERT_NEGATIVES"}]}]}
json.dump(report, open(sys.argv[sys.argv.index("--output") + 1], "w"))
if os.environ.get("FAKE_GREMLINS_LOG"):
json.dump([os.getcwd(), *sys.argv[1:]], open(os.environ["FAKE_GREMLINS_LOG"], "w"))
""")
# Logs each call. A `run` of the bench's image runs its command in the mounted tree: sh as it is, and claude as the fake below.
DOCKER = textwrap.dedent("""\
#!/usr/bin/env python3
import json, os, subprocess, sys
args = sys.argv[1:]
with open(os.environ["FAKE_LOG"], "a") as log:
log.write(json.dumps(args) + "\\n")
if args[0] == "build":
held = sys.stdin.read()
if os.environ.get("FAKE_STDIN"):
open(os.environ["FAKE_STDIN"], "w").write(held)
if args[0] != "run" or "--detach" in args:
sys.exit(int(os.environ.get("FAKE_FAILS") == args[0]))
tree = args[args.index("--volume") + 1].rpartition(":")[0]
command = args[next(at for at, arg in enumerate(args) if arg.endswith(":throwaway")) + 1:]
if command[0] == "claude":
command = [sys.executable, os.environ["FAKE_CLAUDE"], command[-1]]
sys.exit(subprocess.run(command, cwd=tree).returncode)
""")
# Reads the README, then the file with the bug. At the head it fixes the bug; at the base it rewrites the test to pass instead.
# From its FAKE_REFUSED_FROM-th run on, the API refuses it.
CLAUDE = textwrap.dedent("""\
import json, os, pathlib, sys
def call(name, path):
return {"type": "assistant", "message": {"id": "msg_" + path, "content": [{"type": "tool_use", "id": "toolu_" + path, "name": name, "input": {"file_path": path}}], "usage": {}}}
count = pathlib.Path(os.environ["FAKE_LOG"] + ".fixes")
count.write_text(count.read_text() + "." if count.exists() else ".")
if len(count.read_text()) >= int(os.environ.get("FAKE_REFUSED_FROM") or 10 ** 9):
print(json.dumps({"type": "result", "is_error": True, "api_error_status": 401, "result": "API Error: 401"}))
sys.exit(1)
print(json.dumps({"type": "system", "subtype": "init", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330"}))
print(json.dumps(call("Read", "/work/README.md")))
print(json.dumps(call("Read", "/work/calc.go")))
code = pathlib.Path("calc.go")
if "// Add adds." in code.read_text():
code.write_text(code.read_text().replace("a - b", "a + b"))
else:
pathlib.Path("calc_test.go").write_text("exit 0\\n")
pathlib.Path("prompt").write_text(sys.argv[1])
usage = {"input_tokens": 40, "cache_creation_input_tokens": 9000, "cache_read_input_tokens": 52000, "output_tokens": 800}
print(json.dumps({"type": "result", "subtype": "success", "is_error": False, "num_turns": 3, "duration_ms": 61500, "usage": usage, "result": "Fixed."}))
""")
PLACES = "## 2026-10-08T12:00:00Z, PR #1 at aaaaaaa\n\nJunior A, hardest first:\n\n- `calc.go:3` `Add`: long\n- `gone.go:1` `Gone`: removed since\n- `README.md:1` `Calc`: thin\n"
class BenchTest(unittest.TestCase):
def setUp(self):
folder = tempfile.TemporaryDirectory()
self.addCleanup(folder.cleanup)
self.root = pathlib.Path(folder.name).resolve()
self.home, self.repo, fakes = self.root / "home", self.root / "repo", self.root / "fakes"
for made in (self.home, self.repo, fakes):
made.mkdir()
(fakes / "docker").write_text(DOCKER)
(fakes / "docker").chmod(0o755)
(fakes / "claude.py").write_text(CLAUDE)
(fakes / "gremlins.py").write_text(GREMLINS)
self.log = self.root / "docker.log"
self.env = {**isolated_env(), "HOME": str(self.home), "PATH": f"{fakes}{os.pathsep}{os.environ['PATH']}", "FAKE_LOG": str(self.log), "FAKE_CLAUDE": str(fakes / "claude.py"), "CLAUDE_CODE_OAUTH_TOKEN": "sk-secret-token", "RUN_HEAVY_MEMORY": "2048"}
self.given = {"--image": "golang:1.27.4", "--test": "sh calc_test.go", "--prefix": f"{sys.executable} {fakes / 'gremlins.py'}", "--seed": "1", "--bugs": "2", "--runs": "2"}
self.git("init", "--quiet", "--initial-branch", "main")
for name, text in (("calc.go", BASE_CODE), ("calc_test.go", TESTS), ("README.md", "# Calc\n")):
(self.repo / name).write_text(text)
# A link to nothing, which a copy that followed links would fail on.
os.symlink("missing", self.repo / "dangling")
self.base = self.commit("Base")
self.git("checkout", "--quiet", "-b", "item-1")
(self.repo / "calc.go").write_text(HEAD_CODE)
self.head = self.commit("Head")
# The bench runs from a folder below the checkout's top.
(self.repo / "below").mkdir()
self.enterContext(contextlib.chdir(self.repo / "below"))
def git(self, *args: str) -> str:
return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@localhost", "-c", "commit.gpgsign=false", *args], cwd=self.repo, env=self.env, check=True, capture_output=True, text=True).stdout.strip()
def commit(self, title: str) -> str:
self.git("add", "--all")
self.git("commit", "--quiet", "--message", title)
return self.git("rev-parse", "HEAD")
def bench(self, **given: str):
with contextlib.redirect_stderr(io.StringIO()) as self.progress:
return main.bench({**self.given, "--base": "main", **given}, self.env, PLACES, self.home)
def calls(self) -> list[list[str]]:
return [json.loads(line) for line in self.log.read_text().splitlines()]
def test_plants_the_killed_mutant_the_head_left_alone_and_prints_how_each_side_fixed_it(self):
report = self.bench()
self.assertNotIsInstance(report, Failure, report)
folder = pathlib.Path(report[-1].removeprefix("Each run's result and stream: "))
self.assertEqual(report[:-1], [
f"task-bench: base {self.base[:12]}, head {self.head[:12]}. 1 bug, each run 2 times at each side by claude-haiku-5-5 in Claude Code 2.1.330.",
"Planted 1 of the 2 bugs asked for: no other mutant the tests kill becomes one.",
"",
"| Bug | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |",
"|---|---|---|---|---|---|---|---|---|---|",
"| 1: calc.go:4:11 ARITHMETIC_BASE | base | 0/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
"| 1: calc.go:4:11 ARITHMETIC_BASE | head | 2/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
"| All | base | 0/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
"| All | head | 2/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
"| All | change | +2 | 0 % | 0 % | 0 % | 0 % | 0 | 0 % | 0 |",
"",
])
self.assertTrue(folder.is_relative_to(self.home / ".local/state/larv-review") and folder.name == "task-bench", folder)
results = json.loads((folder / "results.json").read_text())
self.assertEqual((results["sides"], results["bugs"], len(results["runs"])), ({"base": self.base, "head": self.head}, {"1": "calc.go:4:11 ARITHMETIC_BASE"}, 4))
self.assertEqual(results["runs"][0], {"bug": 1, "side": "base", "run": 1, "passed": False, "finished": True, "claude_code": "2.1.330", "model": "claude-haiku-5-5", "tokens": {"input": 40, "write_5m": 9000, "write_1h": 0, "read": 52000, "output": 800}, "turns": 3, "seconds": 61.5, "reads": 1})
self.assertEqual(sorted(path.name for path in (folder / "runs").iterdir() if path.suffix == ".jsonl"), ["bug-1-base-1.jsonl", "bug-1-base-2.jsonl", "bug-1-head-1.jsonl", "bug-1-head-2.jsonl"])
self.assertEqual(results["seed"], 1)
self.assertEqual(self.progress.getvalue().splitlines(), [
f"task-bench: base {self.base[:12]}, head {self.head[:12]}.",
"task-bench: run 1 of 4, bug 1 at the base: not passed.",
"task-bench: run 2 of 4, bug 1 at the head: passed.",
"task-bench: run 3 of 4, bug 1 at the base: not passed.",
"task-bench: run 4 of 4, bug 1 at the head: passed.",
])
self.assertEqual(sorted(path.name for path in folder.iterdir()), ["bugs", "results.json", "runs", "trees"])
self.assertEqual((folder / "runs/bug-1-base-1.stderr").read_bytes(), b"")
self.assertEqual(os.readlink(folder / "bugs/1/head/dangling"), "missing")
def test_the_command_line_runs_the_bench_on_the_seats_places_and_prints_its_report(self):
argv = [part for name, value in self.given.items() for part in (name, value)]
homes = []
with contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(main.main(argv, self.env, lambda home: homes.append(home) or PLACES), 0)
self.assertEqual(homes, [self.home])
# Without --base, the base is the merge base with the default branch.
self.assertEqual(out.getvalue().splitlines()[0], f"task-bench: base {self.base[:12]}, head {self.head[:12]}. 1 bug, each run 2 times at each side by claude-haiku-5-5 in Claude Code 2.1.330.")
self.assertEqual(out.getvalue().splitlines()[3], "| Bug | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |")
self.assertTrue(out.getvalue().endswith("task-bench\n"), out.getvalue()[-80:])
def test_the_command_line_fails_with_the_reason_where_the_places_or_the_bench_fail(self):
argv = [part for name, value in self.given.items() for part in (name, value)]
for places, said in ((Failure("no history"), "no history\n"), ("- `gone.go:1` `Gone`: removed\n", "The places the seats found hard name no Go file that the base holds, so there is nowhere to plant a bug.\n")):
with contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()) as errors:
self.assertEqual(main.main(argv, self.env, lambda home: places), 1)
self.assertEqual((out.getvalue(), errors.getvalue().splitlines(keepends=True)[-1]), ("", said))
def test_the_head_is_the_commit_asked_for(self):
report = self.bench(**{"--head": "main", "--runs": "1"})
self.assertEqual(report[0], f"task-bench: base {self.base[:12]}, head {self.base[:12]}. 1 bug, each run 1 time at each side by claude-haiku-5-5 in Claude Code 2.1.330.")
def test_fails_on_a_head_or_a_base_git_does_not_know(self):
for name in ("--head", "--base"):
found = self.bench(**{name: "no-such-ref"})
self.assertIsInstance(found, Failure)
self.assertIn("no-such-ref", found.reason)
def test_the_image_is_built_from_the_one_asked_for_with_the_caps_memory(self):
self.env["FAKE_STDIN"] = str(self.root / "dockerfile")
self.env.pop("RUN_HEAVY_MEMORY")
self.bench(**{"--runs": "1"})
self.assertEqual((self.root / "dockerfile").read_text().splitlines()[0], "FROM golang:1.27.4")
self.assertEqual(self.calls()[0][1:6], ["--quiet", "--memory", "4096m", "--memory-swap", "4096m"])
def test_gremlins_runs_through_the_prefix_in_the_copy_of_the_base_with_a_slots_workers(self):
self.env["FAKE_GREMLINS_LOG"] = str(self.root / "gremlins.log")
folder = pathlib.Path(self.bench(**{"--runs": "1"})[-1].removeprefix("Each run's result and stream: "))
cwd, *argv = json.loads((self.root / "gremlins.log").read_text())
self.assertEqual((cwd, argv[:2], argv[-2:]), (str(folder / "trees/base"), ["gremlins", "unleash"], ["--workers", str(main.CPUS_PER_SLOT)]))
def test_each_side_gets_its_own_tree_with_the_bug_committed_once_and_no_trace_of_the_checkout(self):
folder = pathlib.Path(self.bench()[-1].removeprefix("Each run's result and stream: "))
for side, code in (("base", BASE_CODE), ("head", HEAD_CODE)):
tree = folder / "bugs/1" / side
self.assertEqual((tree / "calc.go").read_text(), code.replace("a + b", "a - b"))
log = subprocess.run(["git", "log", "--format=%an %s"], cwd=tree, env=self.env, check=True, capture_output=True, text=True).stdout
self.assertEqual(log, "task-bench Import\n")
self.assertEqual(subprocess.run(["git", "status", "--porcelain"], cwd=tree, env=self.env, check=True, capture_output=True, text=True).stdout, "")
def test_claude_runs_in_the_benchs_image_on_its_network_with_the_message_alone_and_the_token_by_name(self):
self.bench(**{"--runs": "1"})
calls = self.calls()
fixes = [call for call in calls if "claude" in call]
self.assertEqual(len(fixes), 2)
name = fixes[0][fixes[0].index("--network") + 1]
self.assertRegex(name, r"^task-bench-[0-9a-f]{12}$")
image = fixes[0].index(f"{name}:throwaway")
self.assertEqual(fixes[0][image + 1:], ["claude", "-p", "--strict-mcp-config", "--disable-slash-commands", "--no-session-persistence", "--model", "claude-haiku-5-5", "--effort", "low", "--output-format", "stream-json", "--verbose", "--permission-mode", "bypassPermissions", "A test of this project fails with the message below. Find the bug in the code and fix it. Change no test.\n\nAdd(1, 2) = -1, want 3"])
options = fixes[0][:image]
for expected in (["--memory", "2048m"], ["--memory-swap", "2048m"], ["--user", f"{os.getuid()}:{os.getgid()}"], ["--env", "HOME=/tmp"], ["--env", "CLAUDE_CODE_OAUTH_TOKEN"], ["--env", f"HTTPS_PROXY=http://{name}-proxy:8080"], ["--workdir", "/work"]):
self.assertIn(expected, [options[at:at + 2] for at in range(len(options))])
self.assertNotIn("sk-secret-token", self.log.read_text())
self.assertEqual(calls[0][:1] + calls[0][-3:], ["build", "--tag", f"{name}:throwaway", "-"])
self.assertEqual(calls[1], ["network", "create", "--internal", name])
self.assertEqual([call[:2] for call in calls[-4:]], [["rm", "--force"], ["rm", "--force"], ["network", "rm"], ["image", "rm"]])
def test_the_tests_run_without_a_network(self):
self.bench(**{"--runs": "1"})
tests = [call for call in self.calls() if call[-3:] == ["sh", "-c", "sh calc_test.go"]]
# Each side without a bug, each side with it, and each side's fix.
self.assertEqual(len(tests), 6)
for call in tests:
self.assertEqual(call[call.index("--network") + 1], "none")
def test_benches_the_head_against_itself_where_the_base_is_the_head(self):
report = self.bench(**{"--base": "HEAD", "--runs": "1", "--bugs": "1"})
self.assertEqual(report[0], f"task-bench: base {self.head[:12]}, head {self.head[:12]}. 1 bug, each run 1 time at each side by claude-haiku-5-5 in Claude Code 2.1.330.")
self.assertEqual(report[4:6], ["| 1: calc.go:5:11 ARITHMETIC_BASE | base | 1/1 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |", "| 1: calc.go:5:11 ARITHMETIC_BASE | head | 1/1 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |"])
def test_fails_where_the_places_name_no_go_file_of_the_base(self):
with contextlib.redirect_stderr(io.StringIO()):
found = main.bench({**self.given, "--base": "main"}, self.env, "- `gone.go:1` `Gone`: removed\n- `README.md:1` `Calc`: thin\n", self.home)
self.assertEqual(found, Failure("The places the seats found hard name no Go file that the base holds, so there is nowhere to plant a bug."))
self.assertFalse(self.log.exists())
def test_fails_where_the_tests_fail_without_a_bug(self):
found = self.bench(**{"--test": "echo broken; exit 2"})
self.assertIsInstance(found, Failure)
self.assertIn("The tests fail at the base without a planted bug", found.reason)
self.assertIn("broken", found.reason)
def test_fails_where_no_mutant_becomes_a_bug(self):
# Passing tests with the bug planted: nothing to tell Haiku.
(self.repo / "calc_test.go").write_text("exit 0\n")
self.commit("Tests that catch nothing")
self.assertEqual(self.bench(), Failure("None of the 2 mutants the tests kill becomes a bug: each is on a line the head changed, or the test it fails doesn't print a message."))
def test_a_run_the_api_refuses_fails_the_bench_and_still_removes_what_it_made(self):
self.env["FAKE_REFUSED_FROM"] = "1"
self.assertEqual(self.bench(), Failure("claude could not use the API: API Error: 401"))
self.assertEqual(self.calls()[-1][:2], ["image", "rm"])
def test_a_run_the_api_refuses_later_fails_the_bench_after_the_table_of_the_runs_that_finished(self):
self.env["FAKE_REFUSED_FROM"] = "3"
found = self.bench()
self.assertIsInstance(found, Failure)
lines = found.reason.splitlines()
folder = next(self.home.glob(".local/state/larv-review/**/task-bench"))
self.assertEqual(lines[:3], ["claude could not use the API: API Error: 401", f"task-bench stopped after 2 of 4 runs, which results.json holds in {folder}:", ""])
self.assertEqual(lines[-3:-1], ["| All | base | 0/1 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |", "| All | head | 1/1 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |"])
self.assertEqual([(run["side"], run["run"]) for run in json.loads((folder / "results.json").read_text())["runs"]], [("base", 1), ("head", 1)])
def test_a_change_to_another_file_neither_moves_nor_drops_the_bug(self):
(self.repo / "notes.txt").write_text("one\ntwo\nthree\nfour\nfive\nsix\n")
self.commit("Notes")
report = self.bench(**{"--bugs": "1", "--runs": "1"})
self.assertNotIsInstance(report, Failure, report)
self.assertEqual(report[4].partition(" | ")[0], "| 1: calc.go:4:11 ARITHMETIC_BASE")
def test_the_seed_sets_which_mutant_becomes_the_first_bug(self):
# Tests that catch both mutants, at a head benched against itself, where each mutant's line stands.
(self.repo / "calc_test.go").write_text(TESTS.replace("calc.go &&", "calc.go && grep -q '2 \\* a' calc.go &&"))
self.commit("Tests that catch Twice")
mutants = [Mutant("calc.go", 5, 11, "ARITHMETIC_BASE"), Mutant("calc.go", 9, 11, "ARITHMETIC_BASE")]
for seed in (1, 2, 3, 4):
report = self.bench(**{"--base": "HEAD", "--bugs": "1", "--runs": "1", "--seed": str(seed)})
self.assertNotIsInstance(report, Failure, report)
first = drawn(mutants, seed)[0]
self.assertEqual(report[4].partition(" | ")[0], f"| 1: calc.go:{first.line}:11 ARITHMETIC_BASE", seed)
def test_a_docker_that_fails_to_build_fails_the_bench(self):
self.env["FAKE_FAILS"] = "build"
found = self.bench()
self.assertIsInstance(found, Failure)
self.assertIn("docker build", found.reason)
class MainTest(unittest.TestCase):
def run_main(self, *argv: str, token: str = "t") -> tuple[int, str]:
with contextlib.redirect_stderr(io.StringIO()) as errors, contextlib.redirect_stdout(io.StringIO()):
return main.main(list(argv), {"CLAUDE_CODE_OAUTH_TOKEN": token, "HOME": "/nonexistent"}), errors.getvalue()
def test_refuses_an_unknown_repeated_or_valueless_argument(self):
for argv, said in ((["--imgae", "x"], "unknown argument --imgae"), (["--image", "x", "--image=y"], "--image is given twice"), (["--image"], "--image needs a value")):
code, errors = self.run_main(*argv)
self.assertEqual((code, errors.startswith(f"{said}; usage: task-bench --image")), (1, True), errors)
def test_takes_a_value_after_an_equals_sign_too(self):
with contextlib.redirect_stderr(io.StringIO()) as errors:
self.assertEqual(main.main(["--image=golang:1.27.4", "--test=go test ./...", "--bugs=2"], {"CLAUDE_CODE_OAUTH_TOKEN": "t", "HOME": "/nonexistent"}, lambda home: Failure("read the places")), 1)
self.assertEqual(errors.getvalue(), "read the places\n")
def test_refuses_a_missing_image_or_test_command(self):
self.assertEqual(self.run_main("--test", "go test ./...")[1].partition(";")[0], "--image is missing")
self.assertEqual(self.run_main("--image", "golang:1.27.4")[1].partition(";")[0], "--test is missing")
def test_refuses_a_count_that_is_no_whole_number_from_1(self):
for name, value in (("--bugs", "0"), ("--runs", "-1"), ("--seed", "x")):
self.assertEqual(self.run_main("--image", "i", "--test", "t", name, value)[1].partition(";")[0], f"{name} takes a whole number from 1, not {value}")
def test_refuses_to_start_without_a_token(self):
self.assertEqual(self.run_main("--image", "i", "--test", "t", token="")[1].partition(";")[0], "CLAUDE_CODE_OAUTH_TOKEN holds no token")
def test_prints_its_help(self):
with contextlib.redirect_stdout(io.StringIO()) as out:
self.assertEqual((main.main(["--help"], {}), main.main(["-h"], {})), (0, 0))
self.assertEqual(out.getvalue().count("usage: task-bench --image <image> --test <shell text>"), 2)
def test_records_a_run_that_did_not_finish_without_figures(self):
self.assertEqual(main.recorded(Result(1, "base", 2, False, None)), {"bug": 1, "side": "base", "run": 2, "passed": False, "finished": False})
class Recorded:
"""A docker that records each call and answers by the call's first word, else that it ran."""
def __init__(self, **answers: Ran | Failure):
self.calls: list[tuple[list[str], float, str | None]] = []
self.answers = answers
def __call__(self, args, *, seconds, stdin=None, stdout=None, stderr=None):
self.calls.append((args, seconds, stdin))
if stdout:
stdout.write(b"stream")
stderr.write(b"errors")
return self.answers.get(args[0], Ran(0, "ran"))
class InTurn(Recorded):
"""A docker that answers each `run` with the next of `runs`."""
def __init__(self, *runs: Ran | Failure):
super().__init__()
self.runs = list(runs)
def __call__(self, args, *, seconds, stdin=None, stdout=None, stderr=None):
answer = super().__call__(args, seconds=seconds, stdin=stdin, stdout=stdout, stderr=stderr)
return self.runs.pop(0) if args[0] == "run" else answer
class RunOutcomesTest(unittest.TestCase):
MUTANT = Mutant("calc.go", 4, 11, "ARITHMETIC_BASE")
def setUp(self):
folder = tempfile.TemporaryDirectory()
self.addCleanup(folder.cleanup)
self.folder = pathlib.Path(folder.name)
self.tree = self.folder / "tree"
self.tree.mkdir()
(self.tree / "calc.go").write_text(BASE_CODE)
self.trees = {"base": str(self.tree), "head": str(self.tree)}
self.task = main.Task(1, self.MUTANT, self.trees, {"base": "got 3", "head": "got 3"})
def containers(self, *runs: Ran | Failure) -> Containers:
return Containers(InTurn(*runs), "bench", "go test ./...", 512, "1000:1000")
def test_a_fix_whose_tests_run_out_of_time_does_not_pass(self):
late = TimedOut("docker run still ran after 30 min")
self.assertEqual(main.fixed(self.task, "base", 1, self.containers(Ran(0, ""), late), self.folder), Result(1, "base", 1, False, None))
def test_a_fix_whose_tests_docker_could_not_run_fails_the_bench(self):
broke = Failure("docker could not start: No such file or directory")
self.assertEqual(main.fixed(self.task, "base", 1, self.containers(Ran(0, ""), broke), self.folder), broke)
def test_tests_docker_could_not_run_without_a_bug_fail_the_bench(self):
broke = Failure("docker could not start: No such file or directory")
self.assertEqual(main.benched({}, "", {}, self.trees, [], self.containers(broke), self.folder, None), broke)
def test_a_mutant_whose_tests_run_out_of_time_is_passed_over(self):
late = TimedOut("docker run still ran after 30 min")
self.assertIsNone(main.planted(1, self.MUTANT, "", {"base": "b", "head": "h"}, self.trees, self.containers(late), self.folder, lambda *args, **given: ""))
def test_a_mutant_whose_tests_docker_could_not_run_fails_the_bench(self):
broke = Failure("docker could not start: No such file or directory")
self.assertEqual(main.planted(1, self.MUTANT, "", {"base": "b", "head": "h"}, self.trees, self.containers(broke), self.folder, lambda *args, **given: ""), broke)
class ContainersTest(unittest.TestCase):
IN_TREE = ["--memory", "512m", "--memory-swap", "512m", "--user", "1000:1000", "--env", "HOME=/tmp", "--volume", "/tree:/work", "--workdir", "/work"]
REMOVED = (["rm", "--force", "bench-run"], 120, None)
def setUp(self):
folder = tempfile.TemporaryDirectory()
self.addCleanup(folder.cleanup)
self.stream = pathlib.Path(folder.name, "run.jsonl")
def containers(self, **answers: Ran | Failure) -> Containers:
self.docker = Recorded(**answers)
return Containers(self.docker, "bench", "go test ./...", 512, "1000:1000")
def test_opens_the_image_the_network_without_a_route_out_and_the_proxy_on_both_networks(self):
self.assertIsNone(opened(self.containers(), "golang:1.27.4"))
proxy = str(pathlib.Path(runs.__file__).with_name("api_proxy.py"))
self.assertEqual(self.docker.calls, [
(["build", "--quiet", "--memory", "512m", "--memory-swap", "512m", "--tag", "bench:throwaway", "-"], 1200, "FROM golang:1.27.4\nUSER root\nRUN curl -fsSL https://claude.ai/install.sh | bash && cp -L /root/.local/bin/claude /usr/local/bin/claude\n"),
(["network", "create", "--internal", "bench"], 60, None),
(["run", "--detach", "--rm", "--name", "bench-proxy", "--memory", "256m", "--memory-swap", "256m", "--volume", f"{proxy}:/api_proxy.py:ro", "python:3.15.2-slim", "python", "/api_proxy.py", "api.anthropic.com:443"], 300, None),
(["network", "connect", "bench", "bench-proxy"], 60, None),
])
def test_opening_stops_at_the_first_step_that_fails_and_says_which(self):
found = opened(self.containers(network=Ran(1, "pool overlaps")), "golang:1.27.4")
self.assertEqual((found.reason.startswith("docker network"), "pool overlaps" in found.reason, len(self.docker.calls)), (True, True, 2))
self.assertEqual(opened(self.containers(build=Failure("docker could not start")), "golang:1.27.4"), Failure("docker could not start"))
def test_closing_removes_the_running_container_the_proxy_the_network_and_the_image(self):
closed(self.containers())
self.assertEqual(self.docker.calls, [self.REMOVED, (["rm", "--force", "bench-proxy"], 120, None), (["network", "rm", "bench"], 120, None), (["image", "rm", "bench:throwaway"], 120, None)])
def test_the_tests_run_in_the_tree_without_a_network_for_at_most_half_an_hour(self):
self.assertEqual(ran_tests(self.containers(), "/tree"), Ran(0, "ran"))
self.assertEqual(self.docker.calls, [(["run", "--rm", "--name", "bench-run", "--network", "none", *self.IN_TREE, "bench:throwaway", "sh", "-c", "go test ./..."], 1800, None)])
def test_tests_that_run_out_of_time_are_a_failure_and_their_container_goes(self):
late = TimedOut("docker run still ran after 30 min")
self.assertEqual(ran_tests(self.containers(run=late), "/tree"), late)
self.assertEqual(self.docker.calls[1:], [self.REMOVED])
def test_a_fix_writes_claudes_stream_and_its_errors_beside_it(self):
self.assertIsNone(fix_attempted(self.containers(), "/tree", "got 3, want 4", self.stream))
self.assertEqual((self.stream.read_bytes(), self.stream.with_suffix(".stderr").read_bytes()), (b"stream", b"errors"))
(args, seconds, stdin), = self.docker.calls
self.assertEqual((args[:6], seconds, stdin), (["run", "--rm", "--name", "bench-run", "--network", "bench"], 1800, None))
self.assertEqual(args[6:18], self.IN_TREE)
self.assertEqual(args[18:25], ["--env", "CLAUDE_CODE_OAUTH_TOKEN", "--env", "HTTPS_PROXY=http://bench-proxy:8080", "--env", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1", "bench:throwaway"])
def test_a_fix_that_runs_out_of_time_is_no_failure_and_its_container_goes(self):
self.assertIsNone(fix_attempted(self.containers(run=TimedOut("docker run still ran after 30 min")), "/tree", "got 3", self.stream))
self.assertEqual(self.docker.calls[1:], [self.REMOVED])
def test_a_run_whose_command_never_started_is_a_failure_that_says_what_docker_printed(self):
for code in (125, 126, 127):
self.assertEqual(ran_tests(self.containers(run=Ran(code, "sh: gotestsum: not found\n")), "/tree"), Failure(f"docker run could not start its command, and exited {code}: sh: gotestsum: not found"))
self.assertEqual(ran_tests(self.containers(run=Ran(124, "late")), "/tree"), Ran(124, "late"))
self.assertEqual(ran_tests(self.containers(run=Ran(128, "killed")), "/tree"), Ran(128, "killed"))
self.assertEqual(fix_attempted(self.containers(run=Ran(125, "")), "/tree", "got 3", self.stream), Failure(f"docker run could not start its command, and exited 125; its errors are in {self.stream.with_suffix('.stderr')}"))
def test_a_fix_docker_could_not_start_is_a_failure(self):
broke = Failure("docker could not start: No such file or directory")
self.assertEqual(fix_attempted(self.containers(run=broke), "/tree", "got 3", self.stream), Failure(f"{broke.reason}; its errors are in {self.stream.with_suffix('.stderr')}"))
self.assertEqual(self.docker.calls[1:], [self.REMOVED])
def test_a_stream_that_cannot_be_written_is_a_failure(self):
found = fix_attempted(self.containers(), "/tree", "got 3", self.stream / "below" / "run.jsonl")
self.assertTrue(found.reason.startswith(f"could not write {self.stream}/below/run.jsonl: "), found.reason)
self.assertEqual(self.docker.calls, [])
class DockerCliTest(unittest.TestCase):
def setUp(self):
folder = tempfile.TemporaryDirectory()
self.addCleanup(folder.cleanup)
self.root = pathlib.Path(folder.name)
(self.root / "docker").write_text('#!/bin/sh\necho "$@ $SAID"\ncat\necho broke >&2\nexit 3\n')
(self.root / "docker").chmod(0o755)
self.docker = docker_cli({"PATH": f"{self.root}{os.pathsep}{os.environ['PATH']}", "SAID": "caf\xe9"})
def test_runs_docker_in_the_environment_with_the_text_on_its_input_and_returns_its_code_and_all_it_printed(self):
self.assertEqual(self.docker(["build", "-"], seconds=30, stdin="FROM x\n"), Ran(3, "build - caf\xe9\nFROM x\nbroke\n"))
def test_gives_docker_no_input_where_the_call_has_none(self):
self.assertEqual(self.docker(["ps"], seconds=30), Ran(3, "ps caf\xe9\nbroke\n"))
def test_writes_the_output_and_the_errors_to_the_files_the_call_gives(self):
with (self.root / "out").open("wb") as out, (self.root / "errors").open("wb") as errors:
self.assertEqual(self.docker(["ps"], seconds=30, stdout=out, stderr=errors), Ran(3, ""))
self.assertEqual(((self.root / "out").read_text(), (self.root / "errors").read_text()), ("ps caf\xe9\n", "broke\n"))
def test_output_that_is_no_utf_8_still_reads(self):
(self.root / "docker").write_text("#!/bin/sh\nprintf 'a\\377b'\n")
self.assertEqual(self.docker(["ps"], seconds=30), Ran(0, "a\ufffdb"))
def test_a_docker_that_is_not_there_is_a_failure(self):
self.assertEqual(docker_cli({"PATH": str(self.root / "none")})(["ps"], seconds=30), Failure("docker could not start: No such file or directory"))
def test_a_docker_that_runs_too_long_is_a_failure_that_names_the_command_and_the_minutes(self):
waited = []
def late(argv, timeout, **given):
waited.append(timeout)
raise subprocess.TimeoutExpired(argv, timeout)
self.assertEqual(docker_cli({}, late)(["run", "--rm"], seconds=90), TimedOut("docker run still ran after 2 min"))
self.assertEqual(waited, [90])
class CheckedTreeTest(unittest.TestCase):
def setUp(self):
folder = tempfile.TemporaryDirectory()
self.addCleanup(folder.cleanup)
self.root = pathlib.Path(folder.name)
def tree(self, name: str, files: dict[str, str]) -> str:
for path, text in files.items():
(self.root / name / path).parent.mkdir(parents=True, exist_ok=True)
(self.root / name / path).write_text(text)
return str(self.root / name)
def held(self, name: str) -> dict[str, str]:
top = self.root / name
return {str(path.relative_to(top)): path.read_text() for path in sorted(top.rglob("*")) if path.is_file()}
def test_takes_the_code_as_fixed_and_the_tests_as_planted(self):
planted = self.tree("planted", {"calc.go": "bug", "calc_test.go": "tests", "sub/old.go": "old", "sub/old_test.go": "old tests", ".git/HEAD": "ref"})
copy = self.tree("copy", {"calc.go": "fixed", "calc_test.go": "rewritten", "extra_test.go": "added", "sub/new.go": "new", ".git/HEAD": "moved"})
checked_tree(planted, copy, str(self.root / "check"))
self.assertEqual(self.held("check"), {".git/HEAD": "ref", "calc.go": "fixed", "calc_test.go": "tests", "sub/new.go": "new", "sub/old_test.go": "old tests"})
def test_takes_the_files_under_testdata_as_planted_since_a_test_reads_what_it_expects_there(self):
planted = self.tree("planted", {"calc.go": "bug", "testdata/want.golden": "3", "sub/testdata/gone.golden": "4", "testdata.go": "code"})
copy = self.tree("copy", {"calc.go": "fixed", "testdata/want.golden": "-1", "testdata/added.golden": "5", "testdata.go": "fixed"})
checked_tree(planted, copy, str(self.root / "check"))
self.assertEqual(self.held("check"), {"calc.go": "fixed", "sub/testdata/gone.golden": "4", "testdata.go": "fixed", "testdata/want.golden": "3"})
def test_copies_a_link_as_a_link(self):
planted = self.tree("planted", {"calc.go": "bug"})
copy = self.tree("copy", {"calc.go": "fixed"})
os.symlink("/etc/hostname", os.path.join(copy, "host.go"))
checked_tree(planted, copy, str(self.root / "check"))
self.assertEqual(os.readlink(self.root / "check/host.go"), "/etc/hostname")
def test_a_link_never_stands_in_for_a_folder_of_the_planted_tree_and_a_linked_folder_is_copied_as_a_link(self):
planted = self.tree("planted", {"calc.go": "bug", "sub/old.go": "old", "gone/a.go": "a"})
copy = self.tree("copy", {"calc.go": "fixed"})
os.symlink("/etc", os.path.join(copy, "sub"))
os.symlink("/etc", os.path.join(copy, "etc"))
checked_tree(planted, copy, str(self.root / "check"))
check = self.root / "check"
self.assertEqual((check.joinpath("sub").is_symlink(), os.readlink(check / "etc")), (False, "/etc"))
self.assertEqual(self.held("check"), {"calc.go": "fixed"})
class ApiProxyTest(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self):
async def echoed(reader, writer):
writer.write(b"api:" + await reader.read(5))
await writer.drain()
writer.close()
# On an address of its own, so a tunnel to any other host finds nothing listening.
api = await asyncio.start_server(echoed, "127.0.0.2", 0)
self.allowed = f"127.0.0.2:{api.sockets[0].getsockname()[1]}"
proxy = await asyncio.start_server(lambda reader, writer: api_proxy.served(self.allowed, reader, writer), "127.0.0.1", 0)
self.port = proxy.sockets[0].getsockname()[1]
for server in (api, proxy):
self.addAsyncCleanup(server.wait_closed)
self.addCleanup(server.close)
async def answered(self, request: bytes) -> bytes:
reader, writer = await asyncio.open_connection("127.0.0.1", self.port)
writer.write(request)
answer = await asyncio.wait_for(reader.read(), 5)
writer.close()
return answer
async def test_tunnels_a_connect_to_the_one_allowed_target(self):
self.assertEqual(await self.answered(f"CONNECT {self.allowed} HTTP/1.1\r\n\r\nhello".encode()), b"HTTP/1.1 200 Connection established\r\n\r\napi:hello")
async def test_answers_403_to_any_other_target_or_method(self):
for request in (b"CONNECT example.com:443 HTTP/1.1\r\n\r\n", f"GET http://{self.allowed}/ HTTP/1.1\r\n\r\n".encode(), f"GET {self.allowed} HTTP/1.1\r\n\r\n".encode(), f"CONNECT {self.allowed}\r\n\r\n".encode(), b"junk\r\n\r\n"):
self.assertEqual(await self.answered(request), b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n")
if __name__ == "__main__":
unittest.main()