492 lines
28 KiB
Python
492 lines
28 KiB
Python
"""task-bench from its command line down: a whole bench over a small repository with docker, Gremlins and claude faked, the tree a
|
|
fix is checked in, and the proxy that reaches only the API."""
|
|
import asyncio
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import textwrap
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from task_bench import api_proxy, main, runs
|
|
from task_bench.runs import checked_tree, closed, Containers, docker_cli, fix_attempted, opened, ran_tests
|
|
from failure import Failure
|
|
from test_env import isolated_env
|
|
from mutate_changes.tool_process import Ran
|
|
from task_bench.figures import Result
|
|
|
|
BASE_CODE = "package calc\n\nfunc Add(a, b int) int {\n\treturn a + b\n}\n\nfunc Twice(a int) int {\n\treturn a * 2\n}\n"
|
|
# The head moves Add down a line and rewrites Twice's line.
|
|
HEAD_CODE = "// Package calc adds.\n" + BASE_CODE.replace("a * 2", "2 * a")
|
|
# Stands in for `go test`: a shell text in a file named as a Go test, which fails as go test reports a failed assertion.
|
|
TESTS = textwrap.dedent("""\
|
|
grep -q 'a + b' calc.go && exit 0
|
|
printf -- '--- FAIL: TestAdd (0.00s)\\n calc_test.go:9: Add(1, 2) = -1, want 3\\nFAIL\\n'
|
|
exit 1
|
|
""")
|
|
# Reports `+` in Add, which the head keeps, and the `*` in Twice, which the head rewrites, as killed; and a mutant no test caught.
|
|
GREMLINS = textwrap.dedent("""\
|
|
import json, os, sys
|
|
lines = open("calc.go").read().split("\\n")
|
|
found = [{"line": at, "column": 11, "status": "KILLED", "type": "ARITHMETIC_BASE"} for at, line in enumerate(lines, 1) if " + " in line or " * " in line]
|
|
report = {"files": [{"file_name": "calc.go", "mutations": [*found, {**found[0], "status": "LIVED", "type": "INVERT_NEGATIVES"}]}]}
|
|
json.dump(report, open(sys.argv[sys.argv.index("--output") + 1], "w"))
|
|
if os.environ.get("FAKE_GREMLINS_LOG"):
|
|
json.dump([os.getcwd(), *sys.argv[1:]], open(os.environ["FAKE_GREMLINS_LOG"], "w"))
|
|
""")
|
|
# Logs each call. A `run` of the bench's image runs its command in the mounted tree: sh as it is, and claude as the fake below.
|
|
DOCKER = textwrap.dedent("""\
|
|
#!/usr/bin/env python3
|
|
import json, os, subprocess, sys
|
|
args = sys.argv[1:]
|
|
with open(os.environ["FAKE_LOG"], "a") as log:
|
|
log.write(json.dumps(args) + "\\n")
|
|
if args[0] == "build":
|
|
held = sys.stdin.read()
|
|
if os.environ.get("FAKE_STDIN"):
|
|
open(os.environ["FAKE_STDIN"], "w").write(held)
|
|
if args[0] != "run" or "--detach" in args:
|
|
sys.exit(int(os.environ.get("FAKE_FAILS") == args[0]))
|
|
tree = args[args.index("--volume") + 1].rpartition(":")[0]
|
|
command = args[next(at for at, arg in enumerate(args) if arg.endswith(":throwaway")) + 1:]
|
|
if command[0] == "claude":
|
|
command = [sys.executable, os.environ["FAKE_CLAUDE"], command[-1]]
|
|
sys.exit(subprocess.run(command, cwd=tree).returncode)
|
|
""")
|
|
# Reads the README, then the file with the bug. At the head it fixes the bug; at the base it rewrites the test to pass instead.
|
|
CLAUDE = textwrap.dedent("""\
|
|
import json, os, pathlib, sys
|
|
def call(name, path):
|
|
return {"type": "assistant", "message": {"id": "msg_" + path, "content": [{"type": "tool_use", "id": "toolu_" + path, "name": name, "input": {"file_path": path}}], "usage": {}}}
|
|
if os.environ.get("FAKE_REFUSED"):
|
|
print(json.dumps({"type": "result", "is_error": True, "api_error_status": 401, "result": "API Error: 401"}))
|
|
sys.exit(1)
|
|
print(json.dumps({"type": "system", "subtype": "init", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330"}))
|
|
print(json.dumps(call("Read", "/work/README.md")))
|
|
print(json.dumps(call("Read", "/work/calc.go")))
|
|
code = pathlib.Path("calc.go")
|
|
if code.read_text().startswith("//"):
|
|
code.write_text(code.read_text().replace("a - b", "a + b"))
|
|
else:
|
|
pathlib.Path("calc_test.go").write_text("exit 0\\n")
|
|
pathlib.Path("prompt").write_text(sys.argv[1])
|
|
usage = {"input_tokens": 40, "cache_creation_input_tokens": 9000, "cache_read_input_tokens": 52000, "output_tokens": 800}
|
|
print(json.dumps({"type": "result", "subtype": "success", "is_error": False, "num_turns": 3, "duration_ms": 61500, "usage": usage, "result": "Fixed."}))
|
|
""")
|
|
PLACES = "## 2026-10-08T12:00:00Z, PR #1 at aaaaaaa\n\nJunior A, hardest first:\n\n- `calc.go:3` `Add`: long\n- `gone.go:1` `Gone`: removed since\n- `README.md:1` `Calc`: thin\n"
|
|
|
|
|
|
class BenchTest(unittest.TestCase):
|
|
def setUp(self):
|
|
folder = tempfile.TemporaryDirectory()
|
|
self.addCleanup(folder.cleanup)
|
|
self.root = pathlib.Path(folder.name).resolve()
|
|
self.home, self.repo, fakes = self.root / "home", self.root / "repo", self.root / "fakes"
|
|
for made in (self.home, self.repo, fakes):
|
|
made.mkdir()
|
|
(fakes / "docker").write_text(DOCKER)
|
|
(fakes / "docker").chmod(0o755)
|
|
(fakes / "claude.py").write_text(CLAUDE)
|
|
(fakes / "gremlins.py").write_text(GREMLINS)
|
|
self.log = self.root / "docker.log"
|
|
self.env = {**isolated_env(), "HOME": str(self.home), "PATH": f"{fakes}{os.pathsep}{os.environ['PATH']}", "FAKE_LOG": str(self.log), "FAKE_CLAUDE": str(fakes / "claude.py"), "CLAUDE_CODE_OAUTH_TOKEN": "sk-secret-token", "RUN_HEAVY_MEMORY": "2048"}
|
|
self.given = {"--image": "golang:1.27.4", "--test": "sh calc_test.go", "--prefix": f"{sys.executable} {fakes / 'gremlins.py'}", "--seed": "1", "--bugs": "2", "--runs": "2"}
|
|
self.git("init", "--quiet", "--initial-branch", "main")
|
|
for name, text in (("calc.go", BASE_CODE), ("calc_test.go", TESTS), ("README.md", "# Calc\n")):
|
|
(self.repo / name).write_text(text)
|
|
# A link to nothing, which a copy that followed links would fail on.
|
|
os.symlink("missing", self.repo / "dangling")
|
|
self.base = self.commit("Base")
|
|
self.git("checkout", "--quiet", "-b", "item-1")
|
|
(self.repo / "calc.go").write_text(HEAD_CODE)
|
|
self.head = self.commit("Head")
|
|
# The bench runs from a folder below the checkout's top.
|
|
(self.repo / "below").mkdir()
|
|
self.enterContext(contextlib.chdir(self.repo / "below"))
|
|
|
|
def git(self, *args: str) -> str:
|
|
return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@localhost", "-c", "commit.gpgsign=false", *args], cwd=self.repo, env=self.env, check=True, capture_output=True, text=True).stdout.strip()
|
|
|
|
def commit(self, title: str) -> str:
|
|
self.git("add", "--all")
|
|
self.git("commit", "--quiet", "--message", title)
|
|
return self.git("rev-parse", "HEAD")
|
|
|
|
def bench(self, **given: str):
|
|
with contextlib.redirect_stderr(io.StringIO()) as self.progress:
|
|
return main.bench({**self.given, "--base": "main", **given}, self.env, PLACES, self.home)
|
|
|
|
def calls(self) -> list[list[str]]:
|
|
return [json.loads(line) for line in self.log.read_text().splitlines()]
|
|
|
|
def test_plants_the_killed_mutant_the_head_left_alone_and_prints_how_each_side_fixed_it(self):
|
|
report = self.bench()
|
|
self.assertNotIsInstance(report, Failure, report)
|
|
folder = pathlib.Path(report[-1].removeprefix("Each run's result and stream: "))
|
|
self.assertEqual(report[:-1], [
|
|
f"task-bench: base {self.base[:12]}, head {self.head[:12]}. 1 bug, fixed 2 times at each side by claude-haiku-5-5 in Claude Code 2.1.330.",
|
|
"Planted 1 of the 2 bugs asked for: no other mutant the tests kill becomes one.",
|
|
"",
|
|
"| Task | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |",
|
|
"|---|---|---|---|---|---|---|---|---|---|",
|
|
"| 1: calc.go:4:11 ARITHMETIC_BASE | base | 0/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
|
|
"| 1: calc.go:4:11 ARITHMETIC_BASE | head | 2/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
|
|
"| All | base | 0/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
|
|
"| All | head | 2/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
|
|
"| All | change | +2 | 0 % | 0 % | 0 % | 0 % | 0 | 0 % | 0 |",
|
|
"",
|
|
])
|
|
self.assertTrue(folder.is_relative_to(self.home / ".local/state/larv-review") and folder.name == "task-bench", folder)
|
|
results = json.loads((folder / "results.json").read_text())
|
|
self.assertEqual((results["sides"], results["bugs"], len(results["runs"])), ({"base": self.base, "head": self.head}, {"1": "calc.go:4:11 ARITHMETIC_BASE"}, 4))
|
|
self.assertEqual(results["runs"][0], {"bug": 1, "side": "base", "run": 1, "passed": False, "finished": True, "claude_code": "2.1.330", "model": "claude-haiku-5-5", "tokens": {"input": 40, "write_5m": 9000, "write_1h": 0, "read": 52000, "output": 800}, "turns": 3, "seconds": 61.5, "reads": 1})
|
|
self.assertEqual(sorted(path.name for path in (folder / "runs").iterdir() if path.suffix == ".jsonl"), ["bug-1-base-1.jsonl", "bug-1-base-2.jsonl", "bug-1-head-1.jsonl", "bug-1-head-2.jsonl"])
|
|
self.assertEqual(results["seed"], 1)
|
|
self.assertEqual(self.progress.getvalue().splitlines(), [
|
|
f"task-bench: base {self.base[:12]}, head {self.head[:12]}.",
|
|
"task-bench: run 1 of 4, bug 1 at the base: not passed.",
|
|
"task-bench: run 2 of 4, bug 1 at the head: passed.",
|
|
"task-bench: run 3 of 4, bug 1 at the base: not passed.",
|
|
"task-bench: run 4 of 4, bug 1 at the head: passed.",
|
|
])
|
|
self.assertEqual(sorted(path.name for path in folder.iterdir()), ["bugs", "results.json", "runs", "trees"])
|
|
self.assertEqual((folder / "runs/bug-1-base-1.stderr").read_bytes(), b"")
|
|
self.assertEqual(os.readlink(folder / "bugs/1/head/dangling"), "missing")
|
|
|
|
def test_the_command_line_runs_the_bench_on_the_seats_places_and_prints_its_report(self):
|
|
argv = [part for name, value in self.given.items() for part in (name, value)]
|
|
homes = []
|
|
with contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(main.main(argv, self.env, lambda home: homes.append(home) or PLACES), 0)
|
|
self.assertEqual(homes, [self.home])
|
|
# Without --base, the base is the merge base with the default branch.
|
|
self.assertEqual(out.getvalue().splitlines()[0], f"task-bench: base {self.base[:12]}, head {self.head[:12]}. 1 bug, fixed 2 times at each side by claude-haiku-5-5 in Claude Code 2.1.330.")
|
|
self.assertEqual(out.getvalue().splitlines()[3], "| Task | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |")
|
|
self.assertTrue(out.getvalue().endswith("task-bench\n"), out.getvalue()[-80:])
|
|
|
|
def test_the_command_line_fails_with_the_reason_where_the_places_or_the_bench_fail(self):
|
|
argv = [part for name, value in self.given.items() for part in (name, value)]
|
|
for places, said in ((Failure("no history"), "no history\n"), ("- `gone.go:1` `Gone`: removed\n", "The places the seats found hard name no Go file that the base holds, so there is nowhere to plant a bug.\n")):
|
|
with contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()) as errors:
|
|
self.assertEqual(main.main(argv, self.env, lambda home: places), 1)
|
|
self.assertEqual((out.getvalue(), errors.getvalue().splitlines(keepends=True)[-1]), ("", said))
|
|
|
|
def test_the_head_is_the_commit_asked_for(self):
|
|
report = self.bench(**{"--head": "main", "--runs": "1"})
|
|
self.assertEqual(report[0], f"task-bench: base {self.base[:12]}, head {self.base[:12]}. 1 bug, fixed 1 time at each side by claude-haiku-5-5 in Claude Code 2.1.330.")
|
|
|
|
def test_fails_on_a_head_or_a_base_git_does_not_know(self):
|
|
for name in ("--head", "--base"):
|
|
found = self.bench(**{name: "no-such-ref"})
|
|
self.assertIsInstance(found, Failure)
|
|
self.assertIn("no-such-ref", found.reason)
|
|
|
|
def test_the_image_is_built_from_the_one_asked_for_with_the_caps_memory(self):
|
|
self.env["FAKE_STDIN"] = str(self.root / "dockerfile")
|
|
self.env.pop("RUN_HEAVY_MEMORY")
|
|
self.bench(**{"--runs": "1"})
|
|
self.assertEqual((self.root / "dockerfile").read_text().splitlines()[0], "FROM golang:1.27.4")
|
|
self.assertEqual(self.calls()[0][1:6], ["--quiet", "--memory", "4096m", "--memory-swap", "4096m"])
|
|
|
|
def test_gremlins_runs_through_the_prefix_in_the_copy_of_the_base_with_a_slots_workers(self):
|
|
self.env["FAKE_GREMLINS_LOG"] = str(self.root / "gremlins.log")
|
|
folder = pathlib.Path(self.bench(**{"--runs": "1"})[-1].removeprefix("Each run's result and stream: "))
|
|
cwd, *argv = json.loads((self.root / "gremlins.log").read_text())
|
|
self.assertEqual((cwd, argv[:2], argv[-2:]), (str(folder / "trees/base"), ["gremlins", "unleash"], ["--workers", str(main.CPUS_PER_SLOT)]))
|
|
|
|
def test_each_side_gets_its_own_tree_with_the_bug_committed_once_and_no_trace_of_the_checkout(self):
|
|
folder = pathlib.Path(self.bench()[-1].removeprefix("Each run's result and stream: "))
|
|
for side, code in (("base", BASE_CODE), ("head", HEAD_CODE)):
|
|
tree = folder / "bugs/1" / side
|
|
self.assertEqual((tree / "calc.go").read_text(), code.replace("a + b", "a - b"))
|
|
log = subprocess.run(["git", "log", "--format=%an %s"], cwd=tree, env=self.env, check=True, capture_output=True, text=True).stdout
|
|
self.assertEqual(log, "task-bench Import\n")
|
|
self.assertEqual(subprocess.run(["git", "status", "--porcelain"], cwd=tree, env=self.env, check=True, capture_output=True, text=True).stdout, "")
|
|
|
|
def test_claude_runs_in_the_benchs_image_on_its_network_with_the_message_alone_and_the_token_by_name(self):
|
|
self.bench(**{"--runs": "1"})
|
|
calls = self.calls()
|
|
fixes = [call for call in calls if "claude" in call]
|
|
self.assertEqual(len(fixes), 2)
|
|
name = fixes[0][fixes[0].index("--network") + 1]
|
|
self.assertRegex(name, r"^task-bench-[0-9a-f]{12}$")
|
|
image = fixes[0].index(f"{name}:throwaway")
|
|
self.assertEqual(fixes[0][image + 1:], ["claude", "-p", "--strict-mcp-config", "--disable-slash-commands", "--no-session-persistence", "--model", "claude-haiku-5-5", "--effort", "low", "--output-format", "stream-json", "--verbose", "--permission-mode", "bypassPermissions", "A test of this project fails with the message below. Find the bug in the code and fix it. Change no test.\n\nAdd(1, 2) = -1, want 3"])
|
|
options = fixes[0][:image]
|
|
for expected in (["--memory", "2048m"], ["--memory-swap", "2048m"], ["--user", f"{os.getuid()}:{os.getgid()}"], ["--env", "HOME=/tmp"], ["--env", "CLAUDE_CODE_OAUTH_TOKEN"], ["--env", f"HTTPS_PROXY=http://{name}-proxy:8080"], ["--workdir", "/work"]):
|
|
self.assertIn(expected, [options[at:at + 2] for at in range(len(options))])
|
|
self.assertNotIn("sk-secret-token", self.log.read_text())
|
|
self.assertEqual(calls[0][:1] + calls[0][-3:], ["build", "--tag", f"{name}:throwaway", "-"])
|
|
self.assertEqual(calls[1], ["network", "create", "--internal", name])
|
|
self.assertEqual([call[:2] for call in calls[-4:]], [["rm", "--force"], ["rm", "--force"], ["network", "rm"], ["image", "rm"]])
|
|
|
|
def test_the_tests_run_without_a_network(self):
|
|
self.bench(**{"--runs": "1"})
|
|
tests = [call for call in self.calls() if call[-3:] == ["sh", "-c", "sh calc_test.go"]]
|
|
# Each side without a bug, each side with it, and each side's fix.
|
|
self.assertEqual(len(tests), 6)
|
|
for call in tests:
|
|
self.assertEqual(call[call.index("--network") + 1], "none")
|
|
|
|
def test_benches_the_head_against_itself_where_the_base_is_the_head(self):
|
|
report = self.bench(**{"--base": "HEAD", "--runs": "1", "--bugs": "1"})
|
|
self.assertEqual(report[0], f"task-bench: base {self.head[:12]}, head {self.head[:12]}. 1 bug, fixed 1 time at each side by claude-haiku-5-5 in Claude Code 2.1.330.")
|
|
self.assertEqual(report[4:6], ["| 1: calc.go:5:11 ARITHMETIC_BASE | base | 1/1 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |", "| 1: calc.go:5:11 ARITHMETIC_BASE | head | 1/1 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |"])
|
|
|
|
def test_fails_where_the_places_name_no_go_file_of_the_base(self):
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
found = main.bench({**self.given, "--base": "main"}, self.env, "- `gone.go:1` `Gone`: removed\n- `README.md:1` `Calc`: thin\n", self.home)
|
|
self.assertEqual(found, Failure("The places the seats found hard name no Go file that the base holds, so there is nowhere to plant a bug."))
|
|
self.assertFalse(self.log.exists())
|
|
|
|
def test_fails_where_the_tests_fail_without_a_bug(self):
|
|
found = self.bench(**{"--test": "echo broken; exit 2"})
|
|
self.assertIsInstance(found, Failure)
|
|
self.assertIn("The tests fail at the base without a planted bug", found.reason)
|
|
self.assertIn("broken", found.reason)
|
|
|
|
def test_fails_where_no_mutant_becomes_a_bug(self):
|
|
# Passing tests with the bug planted: nothing to tell Haiku.
|
|
(self.repo / "calc_test.go").write_text("exit 0\n")
|
|
self.commit("Tests that catch nothing")
|
|
self.assertEqual(self.bench(), Failure("None of the 2 mutants the tests kill becomes a bug: each is on a line the head changed, or fails no assertion with a message."))
|
|
|
|
def test_a_run_the_api_refuses_fails_the_bench_and_still_removes_what_it_made(self):
|
|
self.env["FAKE_REFUSED"] = "1"
|
|
self.assertEqual(self.bench(), Failure("claude could not use the API: API Error: 401"))
|
|
self.assertEqual(self.calls()[-1][:2], ["image", "rm"])
|
|
|
|
def test_a_docker_that_fails_to_build_fails_the_bench(self):
|
|
self.env["FAKE_FAILS"] = "build"
|
|
found = self.bench()
|
|
self.assertIsInstance(found, Failure)
|
|
self.assertIn("docker build", found.reason)
|
|
|
|
|
|
class MainTest(unittest.TestCase):
|
|
def run_main(self, *argv: str, token: str = "t") -> tuple[int, str]:
|
|
with contextlib.redirect_stderr(io.StringIO()) as errors, contextlib.redirect_stdout(io.StringIO()):
|
|
return main.main(list(argv), {"CLAUDE_CODE_OAUTH_TOKEN": token, "HOME": "/nonexistent"}), errors.getvalue()
|
|
|
|
def test_refuses_an_unknown_repeated_or_valueless_argument(self):
|
|
for argv, said in ((["--imgae", "x"], "unknown or repeated argument --imgae"), (["--image", "x", "--image=y"], "unknown or repeated argument --image"), (["--image"], "--image needs a value")):
|
|
code, errors = self.run_main(*argv)
|
|
self.assertEqual((code, errors.startswith(f"{said}; usage: task-bench --image")), (1, True), errors)
|
|
|
|
def test_refuses_a_missing_image_or_test_command(self):
|
|
self.assertEqual(self.run_main("--test", "go test ./...")[1].partition(";")[0], "--image is missing")
|
|
self.assertEqual(self.run_main("--image", "golang:1.27.4")[1].partition(";")[0], "--test is missing")
|
|
|
|
def test_refuses_a_count_that_is_no_whole_number_from_1(self):
|
|
for name, value in (("--bugs", "0"), ("--runs", "-1"), ("--seed", "x")):
|
|
self.assertEqual(self.run_main("--image", "i", "--test", "t", name, value)[1].partition(";")[0], f"{name} takes a whole number from 1, not {value}")
|
|
|
|
def test_refuses_to_start_without_a_token(self):
|
|
self.assertEqual(self.run_main("--image", "i", "--test", "t", token="")[1].partition(";")[0], "CLAUDE_CODE_OAUTH_TOKEN holds no token")
|
|
|
|
def test_prints_its_help(self):
|
|
with contextlib.redirect_stdout(io.StringIO()) as out:
|
|
self.assertEqual(main.main(["--help"], {}), 0)
|
|
self.assertIn("usage: task-bench --image <image> --test <shell text>", out.getvalue())
|
|
|
|
|
|
def test_records_a_run_that_did_not_finish_without_figures(self):
|
|
self.assertEqual(main.recorded(Result(1, "base", 2, False, None)), {"bug": 1, "side": "base", "run": 2, "passed": False, "finished": False})
|
|
|
|
|
|
class Recorded:
|
|
"""A docker that records each call and answers by the call's first word, else that it ran."""
|
|
|
|
def __init__(self, **answers: Ran | Failure):
|
|
self.calls: list[tuple[list[str], float, str | None]] = []
|
|
self.answers = answers
|
|
|
|
def __call__(self, args, *, seconds, stdin=None, stdout=None, stderr=None):
|
|
self.calls.append((args, seconds, stdin))
|
|
if stdout:
|
|
stdout.write(b"stream")
|
|
stderr.write(b"errors")
|
|
return self.answers.get(args[0], Ran(0, "ran"))
|
|
|
|
|
|
class ContainersTest(unittest.TestCase):
|
|
IN_TREE = ["--memory", "512m", "--memory-swap", "512m", "--user", "1000:1000", "--env", "HOME=/tmp", "--volume", "/tree:/work", "--workdir", "/work"]
|
|
REMOVED = (["rm", "--force", "bench-run"], 120, None)
|
|
|
|
def setUp(self):
|
|
folder = tempfile.TemporaryDirectory()
|
|
self.addCleanup(folder.cleanup)
|
|
self.stream = pathlib.Path(folder.name, "run.jsonl")
|
|
|
|
def containers(self, **answers: Ran | Failure) -> Containers:
|
|
self.docker = Recorded(**answers)
|
|
return Containers(self.docker, "bench", "go test ./...", 512, "1000:1000")
|
|
|
|
def test_opens_the_image_the_network_without_a_route_out_and_the_proxy_on_both_networks(self):
|
|
self.assertIsNone(opened(self.containers(), "golang:1.27.4"))
|
|
proxy = str(pathlib.Path(runs.__file__).with_name("api_proxy.py"))
|
|
self.assertEqual(self.docker.calls, [
|
|
(["build", "--quiet", "--memory", "512m", "--memory-swap", "512m", "--tag", "bench:throwaway", "-"], 1200, "FROM golang:1.27.4\nUSER root\nRUN curl -fsSL https://claude.ai/install.sh | bash && cp -L /root/.local/bin/claude /usr/local/bin/claude\n"),
|
|
(["network", "create", "--internal", "bench"], 60, None),
|
|
(["run", "--detach", "--rm", "--name", "bench-proxy", "--memory", "256m", "--memory-swap", "256m", "--volume", f"{proxy}:/api_proxy.py:ro", "python:3.15.2-slim", "python", "/api_proxy.py", "api.anthropic.com:443"], 300, None),
|
|
(["network", "connect", "bench", "bench-proxy"], 60, None),
|
|
])
|
|
|
|
def test_opening_stops_at_the_first_step_that_fails_and_says_which(self):
|
|
found = opened(self.containers(network=Ran(1, "pool overlaps")), "golang:1.27.4")
|
|
self.assertEqual((found.reason.startswith("docker network"), "pool overlaps" in found.reason, len(self.docker.calls)), (True, True, 2))
|
|
self.assertEqual(opened(self.containers(build=Failure("docker could not start")), "golang:1.27.4"), Failure("docker could not start"))
|
|
|
|
def test_closing_removes_the_running_container_the_proxy_the_network_and_the_image(self):
|
|
closed(self.containers())
|
|
self.assertEqual(self.docker.calls, [self.REMOVED, (["rm", "--force", "bench-proxy"], 120, None), (["network", "rm", "bench"], 120, None), (["image", "rm", "bench:throwaway"], 120, None)])
|
|
|
|
def test_the_tests_run_in_the_tree_without_a_network_for_at_most_half_an_hour(self):
|
|
self.assertEqual(ran_tests(self.containers(), "/tree"), Ran(0, "ran"))
|
|
self.assertEqual(self.docker.calls, [(["run", "--rm", "--name", "bench-run", "--network", "none", *self.IN_TREE, "bench:throwaway", "sh", "-c", "go test ./..."], 1800, None)])
|
|
|
|
def test_tests_that_run_out_of_time_are_a_failure_and_their_container_goes(self):
|
|
late = Failure("docker run still ran after 30 min")
|
|
self.assertEqual(ran_tests(self.containers(run=late), "/tree"), late)
|
|
self.assertEqual(self.docker.calls[1:], [self.REMOVED])
|
|
|
|
def test_a_fix_writes_claudes_stream_and_its_errors_beside_it(self):
|
|
self.assertIsNone(fix_attempted(self.containers(), "/tree", "got 3, want 4", self.stream))
|
|
self.assertEqual((self.stream.read_bytes(), self.stream.with_suffix(".stderr").read_bytes()), (b"stream", b"errors"))
|
|
(args, seconds, stdin), = self.docker.calls
|
|
self.assertEqual((args[:6], seconds, stdin), (["run", "--rm", "--name", "bench-run", "--network", "bench"], 1800, None))
|
|
self.assertEqual(args[6:18], self.IN_TREE)
|
|
self.assertEqual(args[18:25], ["--env", "CLAUDE_CODE_OAUTH_TOKEN", "--env", "HTTPS_PROXY=http://bench-proxy:8080", "--env", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1", "bench:throwaway"])
|
|
|
|
def test_a_fix_that_runs_out_of_time_is_no_failure_and_its_container_goes(self):
|
|
self.assertIsNone(fix_attempted(self.containers(run=Failure("docker run still ran after 30 min")), "/tree", "got 3", self.stream))
|
|
self.assertEqual(self.docker.calls[1:], [self.REMOVED])
|
|
|
|
def test_a_fix_docker_could_not_start_is_a_failure(self):
|
|
broke = Failure("docker could not start: No such file or directory")
|
|
self.assertEqual(fix_attempted(self.containers(run=broke), "/tree", "got 3", self.stream), broke)
|
|
self.assertEqual(self.docker.calls[1:], [self.REMOVED])
|
|
|
|
def test_a_stream_that_cannot_be_written_is_a_failure(self):
|
|
found = fix_attempted(self.containers(), "/tree", "got 3", self.stream / "below" / "run.jsonl")
|
|
self.assertTrue(found.reason.startswith(f"could not write {self.stream}/below/run.jsonl: "), found.reason)
|
|
self.assertEqual(self.docker.calls, [])
|
|
|
|
|
|
class DockerCliTest(unittest.TestCase):
|
|
def setUp(self):
|
|
folder = tempfile.TemporaryDirectory()
|
|
self.addCleanup(folder.cleanup)
|
|
self.root = pathlib.Path(folder.name)
|
|
(self.root / "docker").write_text('#!/bin/sh\necho "$@ $SAID"\ncat\necho broke >&2\nexit 3\n')
|
|
(self.root / "docker").chmod(0o755)
|
|
self.docker = docker_cli({"PATH": f"{self.root}{os.pathsep}{os.environ['PATH']}", "SAID": "caf\xe9"})
|
|
|
|
def test_runs_docker_in_the_environment_with_the_text_on_its_input_and_returns_its_code_and_all_it_printed(self):
|
|
self.assertEqual(self.docker(["build", "-"], seconds=30, stdin="FROM x\n"), Ran(3, "build - caf\xe9\nFROM x\nbroke\n"))
|
|
|
|
def test_gives_docker_no_input_where_the_call_has_none(self):
|
|
self.assertEqual(self.docker(["ps"], seconds=30), Ran(3, "ps caf\xe9\nbroke\n"))
|
|
|
|
def test_writes_the_output_and_the_errors_to_the_files_the_call_gives(self):
|
|
with (self.root / "out").open("wb") as out, (self.root / "errors").open("wb") as errors:
|
|
self.assertEqual(self.docker(["ps"], seconds=30, stdout=out, stderr=errors), Ran(3, ""))
|
|
self.assertEqual(((self.root / "out").read_text(), (self.root / "errors").read_text()), ("ps caf\xe9\n", "broke\n"))
|
|
|
|
def test_output_that_is_no_utf_8_still_reads(self):
|
|
(self.root / "docker").write_text("#!/bin/sh\nprintf 'a\\377b'\n")
|
|
self.assertEqual(self.docker(["ps"], seconds=30), Ran(0, "a\ufffdb"))
|
|
|
|
def test_a_docker_that_is_not_there_is_a_failure(self):
|
|
self.assertEqual(docker_cli({"PATH": str(self.root / "none")})(["ps"], seconds=30), Failure("docker could not start: No such file or directory"))
|
|
|
|
def test_a_docker_that_runs_too_long_is_a_failure_that_names_the_command_and_the_minutes(self):
|
|
waited = []
|
|
|
|
def late(argv, timeout, **given):
|
|
waited.append(timeout)
|
|
raise subprocess.TimeoutExpired(argv, timeout)
|
|
|
|
self.assertEqual(docker_cli({}, late)(["run", "--rm"], seconds=90), Failure("docker run still ran after 2 min"))
|
|
self.assertEqual(waited, [90])
|
|
|
|
|
|
class CheckedTreeTest(unittest.TestCase):
|
|
def setUp(self):
|
|
folder = tempfile.TemporaryDirectory()
|
|
self.addCleanup(folder.cleanup)
|
|
self.root = pathlib.Path(folder.name)
|
|
|
|
def tree(self, name: str, files: dict[str, str]) -> str:
|
|
for path, text in files.items():
|
|
(self.root / name / path).parent.mkdir(parents=True, exist_ok=True)
|
|
(self.root / name / path).write_text(text)
|
|
return str(self.root / name)
|
|
|
|
def held(self, name: str) -> dict[str, str]:
|
|
top = self.root / name
|
|
return {str(path.relative_to(top)): path.read_text() for path in sorted(top.rglob("*")) if path.is_file()}
|
|
|
|
def test_takes_the_code_as_fixed_and_the_tests_as_planted(self):
|
|
planted = self.tree("planted", {"calc.go": "bug", "calc_test.go": "tests", "sub/old.go": "old", "sub/old_test.go": "old tests", ".git/HEAD": "ref"})
|
|
copy = self.tree("copy", {"calc.go": "fixed", "calc_test.go": "rewritten", "extra_test.go": "added", "sub/new.go": "new", ".git/HEAD": "moved"})
|
|
checked_tree(planted, copy, str(self.root / "check"))
|
|
self.assertEqual(self.held("check"), {".git/HEAD": "ref", "calc.go": "fixed", "calc_test.go": "tests", "sub/new.go": "new", "sub/old_test.go": "old tests"})
|
|
|
|
def test_copies_a_link_as_a_link(self):
|
|
planted = self.tree("planted", {"calc.go": "bug"})
|
|
copy = self.tree("copy", {"calc.go": "fixed"})
|
|
os.symlink("/etc/hostname", os.path.join(copy, "host.go"))
|
|
checked_tree(planted, copy, str(self.root / "check"))
|
|
self.assertEqual(os.readlink(self.root / "check/host.go"), "/etc/hostname")
|
|
|
|
def test_a_link_never_stands_in_for_a_folder_of_the_planted_tree_and_a_linked_folder_is_copied_as_a_link(self):
|
|
planted = self.tree("planted", {"calc.go": "bug", "sub/old.go": "old", "gone/a.go": "a"})
|
|
copy = self.tree("copy", {"calc.go": "fixed"})
|
|
os.symlink("/etc", os.path.join(copy, "sub"))
|
|
os.symlink("/etc", os.path.join(copy, "etc"))
|
|
checked_tree(planted, copy, str(self.root / "check"))
|
|
check = self.root / "check"
|
|
self.assertEqual((check.joinpath("sub").is_symlink(), os.readlink(check / "etc")), (False, "/etc"))
|
|
self.assertEqual(self.held("check"), {"calc.go": "fixed"})
|
|
|
|
|
|
class ApiProxyTest(unittest.IsolatedAsyncioTestCase):
|
|
async def asyncSetUp(self):
|
|
async def echoed(reader, writer):
|
|
writer.write(b"api:" + await reader.read(5))
|
|
await writer.drain()
|
|
writer.close()
|
|
|
|
api = await asyncio.start_server(echoed, "127.0.0.1", 0)
|
|
self.allowed = f"127.0.0.1:{api.sockets[0].getsockname()[1]}"
|
|
proxy = await asyncio.start_server(lambda reader, writer: api_proxy.served(self.allowed, reader, writer), "127.0.0.1", 0)
|
|
self.port = proxy.sockets[0].getsockname()[1]
|
|
for server in (api, proxy):
|
|
self.addAsyncCleanup(server.wait_closed)
|
|
self.addCleanup(server.close)
|
|
|
|
async def answered(self, request: bytes) -> bytes:
|
|
reader, writer = await asyncio.open_connection("127.0.0.1", self.port)
|
|
writer.write(request)
|
|
answer = await asyncio.wait_for(reader.read(), 5)
|
|
writer.close()
|
|
return answer
|
|
|
|
async def test_tunnels_a_connect_to_the_one_allowed_target(self):
|
|
self.assertEqual(await self.answered(f"CONNECT {self.allowed} HTTP/1.1\r\n\r\nhello".encode()), b"HTTP/1.1 200 Connection established\r\n\r\napi:hello")
|
|
|
|
async def test_answers_403_to_any_other_target_or_method(self):
|
|
for request in (b"CONNECT example.com:443 HTTP/1.1\r\n\r\n", f"GET http://{self.allowed}/ HTTP/1.1\r\n\r\n".encode(), b"junk\r\n\r\n"):
|
|
self.assertEqual(await self.answered(request), b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|