300 lines
17 KiB
Python
300 lines
17 KiB
Python
"""task-bench from its command line down: a whole bench over a small repository with docker, Gremlins and claude faked, the tree a
|
|
fix is checked in, and the proxy that reaches only the API."""
|
|
import asyncio
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import textwrap
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from failure import Failure
|
|
from task_bench import api_proxy, main
|
|
from task_bench.runs import checked_tree
|
|
from test_env import isolated_env
|
|
|
|
BASE_CODE = "package calc\n\nfunc Add(a, b int) int {\n\treturn a + b\n}\n\nfunc Twice(a int) int {\n\treturn a * 2\n}\n"
|
|
# The head moves Add down a line and rewrites Twice's line.
|
|
HEAD_CODE = "// Package calc adds.\n" + BASE_CODE.replace("a * 2", "2 * a")
|
|
# Stands in for `go test`: a shell text in a file named as a Go test, which fails as go test reports a failed assertion.
|
|
TESTS = textwrap.dedent("""\
|
|
grep -q 'a + b' calc.go && exit 0
|
|
printf -- '--- FAIL: TestAdd (0.00s)\\n calc_test.go:9: Add(1, 2) = -1, want 3\\nFAIL\\n'
|
|
exit 1
|
|
""")
|
|
# Reports `+` in Add, which the head keeps, and the `*` in Twice, which the head rewrites, as killed; and a mutant no test caught.
|
|
GREMLINS = textwrap.dedent("""\
|
|
import json, sys
|
|
lines = open("calc.go").read().split("\\n")
|
|
found = [{"line": at, "column": 11, "status": "KILLED", "type": "ARITHMETIC_BASE"} for at, line in enumerate(lines, 1) if " + " in line or " * " in line]
|
|
report = {"files": [{"file_name": "calc.go", "mutations": [*found, {**found[0], "status": "LIVED", "type": "INVERT_NEGATIVES"}]}]}
|
|
json.dump(report, open(sys.argv[sys.argv.index("--output") + 1], "w"))
|
|
""")
|
|
# Logs each call. A `run` of the bench's image runs its command in the mounted tree: sh as it is, and claude as the fake below.
|
|
DOCKER = textwrap.dedent("""\
|
|
#!/usr/bin/env python3
|
|
import json, os, subprocess, sys
|
|
args = sys.argv[1:]
|
|
with open(os.environ["FAKE_LOG"], "a") as log:
|
|
log.write(json.dumps(args) + "\\n")
|
|
if args[0] == "build":
|
|
sys.stdin.read()
|
|
if args[0] != "run" or "--detach" in args:
|
|
sys.exit(int(os.environ.get("FAKE_FAILS") == args[0]))
|
|
tree = args[args.index("--volume") + 1].rpartition(":")[0]
|
|
command = args[next(at for at, arg in enumerate(args) if arg.endswith(":throwaway")) + 1:]
|
|
if command[0] == "claude":
|
|
command = [sys.executable, os.environ["FAKE_CLAUDE"], command[-1]]
|
|
sys.exit(subprocess.run(command, cwd=tree).returncode)
|
|
""")
|
|
# Reads the README, then the file with the bug. At the head it fixes the bug; at the base it rewrites the test to pass instead.
|
|
CLAUDE = textwrap.dedent("""\
|
|
import json, os, pathlib, sys
|
|
def call(name, path):
|
|
return {"type": "assistant", "message": {"id": "msg_" + path, "content": [{"type": "tool_use", "id": "toolu_" + path, "name": name, "input": {"file_path": path}}], "usage": {}}}
|
|
if os.environ.get("FAKE_REFUSED"):
|
|
print(json.dumps({"type": "result", "is_error": True, "api_error_status": 401, "result": "API Error: 401"}))
|
|
sys.exit(1)
|
|
print(json.dumps({"type": "system", "subtype": "init", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330"}))
|
|
print(json.dumps(call("Read", "/work/README.md")))
|
|
print(json.dumps(call("Read", "/work/calc.go")))
|
|
code = pathlib.Path("calc.go")
|
|
if code.read_text().startswith("//"):
|
|
code.write_text(code.read_text().replace("a - b", "a + b"))
|
|
else:
|
|
pathlib.Path("calc_test.go").write_text("exit 0\\n")
|
|
pathlib.Path("prompt").write_text(sys.argv[1])
|
|
usage = {"input_tokens": 40, "cache_creation_input_tokens": 9000, "cache_read_input_tokens": 52000, "output_tokens": 800}
|
|
print(json.dumps({"type": "result", "subtype": "success", "is_error": False, "num_turns": 3, "duration_ms": 61500, "usage": usage, "result": "Fixed."}))
|
|
""")
|
|
PLACES = "## 2026-10-08T12:00:00Z, PR #1 at aaaaaaa\n\nJunior A, hardest first:\n\n- `calc.go:3` `Add`: long\n- `gone.go:1` `Gone`: removed since\n- `README.md:1` `Calc`: thin\n"
|
|
|
|
|
|
class BenchTest(unittest.TestCase):
|
|
def setUp(self):
|
|
folder = tempfile.TemporaryDirectory()
|
|
self.addCleanup(folder.cleanup)
|
|
self.root = pathlib.Path(folder.name).resolve()
|
|
self.home, self.repo, fakes = self.root / "home", self.root / "repo", self.root / "fakes"
|
|
for made in (self.home, self.repo, fakes):
|
|
made.mkdir()
|
|
(fakes / "docker").write_text(DOCKER)
|
|
(fakes / "docker").chmod(0o755)
|
|
(fakes / "claude.py").write_text(CLAUDE)
|
|
(fakes / "gremlins.py").write_text(GREMLINS)
|
|
self.log = self.root / "docker.log"
|
|
self.env = {**isolated_env(), "HOME": str(self.home), "PATH": f"{fakes}{os.pathsep}{os.environ['PATH']}", "FAKE_LOG": str(self.log), "FAKE_CLAUDE": str(fakes / "claude.py"), "CLAUDE_CODE_OAUTH_TOKEN": "sk-secret-token", "RUN_HEAVY_MEMORY": "2048"}
|
|
self.given = {"--image": "golang:1.27.4", "--test": "sh calc_test.go", "--prefix": f"{sys.executable} {fakes / 'gremlins.py'}", "--seed": "1", "--bugs": "2", "--runs": "2"}
|
|
self.git("init", "--quiet", "--initial-branch", "main")
|
|
for name, text in (("calc.go", BASE_CODE), ("calc_test.go", TESTS), ("README.md", "# Calc\n")):
|
|
(self.repo / name).write_text(text)
|
|
self.base = self.commit("Base")
|
|
self.git("checkout", "--quiet", "-b", "item-1")
|
|
(self.repo / "calc.go").write_text(HEAD_CODE)
|
|
self.head = self.commit("Head")
|
|
self.enterContext(contextlib.chdir(self.repo))
|
|
|
|
def git(self, *args: str) -> str:
|
|
return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@localhost", "-c", "commit.gpgsign=false", *args], cwd=self.repo, env=self.env, check=True, capture_output=True, text=True).stdout.strip()
|
|
|
|
def commit(self, title: str) -> str:
|
|
self.git("add", "--all")
|
|
self.git("commit", "--quiet", "--message", title)
|
|
return self.git("rev-parse", "HEAD")
|
|
|
|
def bench(self, **given: str):
|
|
with contextlib.redirect_stderr(io.StringIO()) as self.progress:
|
|
return main.bench({**self.given, "--base": "main", **given}, self.env, PLACES, self.home)
|
|
|
|
def calls(self) -> list[list[str]]:
|
|
return [json.loads(line) for line in self.log.read_text().splitlines()]
|
|
|
|
def test_plants_the_killed_mutant_the_head_left_alone_and_prints_how_each_side_fixed_it(self):
|
|
report = self.bench()
|
|
self.assertNotIsInstance(report, Failure, report)
|
|
folder = pathlib.Path(report[-1].removeprefix("Each run's result and stream: "))
|
|
self.assertEqual(report[:-1], [
|
|
f"task-bench: base {self.base[:12]}, head {self.head[:12]}. 1 bug, fixed 2 times at each side by claude-haiku-5-5 in Claude Code 2.1.330.",
|
|
"Planted 1 of the 2 bugs asked for: no other mutant the tests kill becomes one.",
|
|
"",
|
|
"| Task | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |",
|
|
"|---|---|---|---|---|---|---|---|---|---|",
|
|
"| 1: calc.go:4:11 ARITHMETIC_BASE | base | 0/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
|
|
"| 1: calc.go:4:11 ARITHMETIC_BASE | head | 2/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
|
|
"| All | base | 0/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
|
|
"| All | head | 2/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
|
|
"| All | change | +2 | 0 % | 0 % | 0 % | 0 % | 0 | 0 % | 0 |",
|
|
"",
|
|
])
|
|
self.assertTrue(folder.is_relative_to(self.home / ".local/state/larv-review") and folder.name == "task-bench", folder)
|
|
results = json.loads((folder / "results.json").read_text())
|
|
self.assertEqual((results["sides"], results["bugs"], len(results["runs"])), ({"base": self.base, "head": self.head}, {"1": "calc.go:4:11 ARITHMETIC_BASE"}, 4))
|
|
self.assertEqual(results["runs"][0], {"bug": 1, "side": "base", "run": 1, "passed": False, "finished": True, "claude_code": "2.1.330", "model": "claude-haiku-5-5", "tokens": {"input": 40, "write_5m": 9000, "write_1h": 0, "read": 52000, "output": 800}, "turns": 3, "seconds": 61.5, "reads": 1})
|
|
self.assertEqual(sorted(path.name for path in (folder / "runs").iterdir() if path.suffix == ".jsonl"), ["bug-1-base-1.jsonl", "bug-1-base-2.jsonl", "bug-1-head-1.jsonl", "bug-1-head-2.jsonl"])
|
|
self.assertIn("run 4 of 4, bug 1 at the head: passed.", self.progress.getvalue())
|
|
|
|
def test_each_side_gets_its_own_tree_with_the_bug_committed_once_and_no_trace_of_the_checkout(self):
|
|
folder = pathlib.Path(self.bench()[-1].removeprefix("Each run's result and stream: "))
|
|
for side, code in (("base", BASE_CODE), ("head", HEAD_CODE)):
|
|
tree = folder / "bugs/1" / side
|
|
self.assertEqual((tree / "calc.go").read_text(), code.replace("a + b", "a - b"))
|
|
log = subprocess.run(["git", "log", "--format=%an %s"], cwd=tree, env=self.env, check=True, capture_output=True, text=True).stdout
|
|
self.assertEqual(log, "task-bench Import\n")
|
|
self.assertEqual(subprocess.run(["git", "status", "--porcelain"], cwd=tree, env=self.env, check=True, capture_output=True, text=True).stdout, "")
|
|
|
|
def test_claude_runs_in_the_benchs_image_on_its_network_with_the_message_alone_and_the_token_by_name(self):
|
|
self.bench(**{"--runs": "1"})
|
|
calls = self.calls()
|
|
fixes = [call for call in calls if "claude" in call]
|
|
self.assertEqual(len(fixes), 2)
|
|
name = fixes[0][fixes[0].index("--network") + 1]
|
|
self.assertRegex(name, r"^task-bench-[0-9a-f]{12}$")
|
|
image = fixes[0].index(f"{name}:throwaway")
|
|
self.assertEqual(fixes[0][image + 1:], ["claude", "-p", "--strict-mcp-config", "--disable-slash-commands", "--no-session-persistence", "--model", "claude-haiku-5-5", "--effort", "low", "--output-format", "stream-json", "--verbose", "--permission-mode", "bypassPermissions", "A test of this project fails with the message below. Find the bug in the code and fix it. Change no test.\n\nAdd(1, 2) = -1, want 3"])
|
|
options = fixes[0][:image]
|
|
for expected in (["--memory", "2048m"], ["--memory-swap", "2048m"], ["--user", f"{os.getuid()}:{os.getgid()}"], ["--env", "HOME=/tmp"], ["--env", "CLAUDE_CODE_OAUTH_TOKEN"], ["--env", f"HTTPS_PROXY=http://{name}-proxy:8080"], ["--workdir", "/work"]):
|
|
self.assertIn(expected, [options[at:at + 2] for at in range(len(options))])
|
|
self.assertNotIn("sk-secret-token", self.log.read_text())
|
|
self.assertEqual(calls[0][:1] + calls[0][-3:], ["build", "--tag", f"{name}:throwaway", "-"])
|
|
self.assertEqual(calls[1], ["network", "create", "--internal", name])
|
|
self.assertEqual([call[:2] for call in calls[-4:]], [["rm", "--force"], ["rm", "--force"], ["network", "rm"], ["image", "rm"]])
|
|
|
|
def test_the_tests_run_without_a_network(self):
|
|
self.bench(**{"--runs": "1"})
|
|
tests = [call for call in self.calls() if call[-3:] == ["sh", "-c", "sh calc_test.go"]]
|
|
# Each side without a bug, each side with it, and each side's fix.
|
|
self.assertEqual(len(tests), 6)
|
|
for call in tests:
|
|
self.assertEqual(call[call.index("--network") + 1], "none")
|
|
|
|
def test_benches_the_head_against_itself_where_the_base_is_the_head(self):
|
|
report = self.bench(**{"--base": "HEAD", "--runs": "1", "--bugs": "1"})
|
|
self.assertEqual(report[0], f"task-bench: base {self.head[:12]}, head {self.head[:12]}. 1 bug, fixed 1 time at each side by claude-haiku-5-5 in Claude Code 2.1.330.")
|
|
self.assertEqual(report[4:6], ["| 1: calc.go:5:11 ARITHMETIC_BASE | base | 1/1 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |", "| 1: calc.go:5:11 ARITHMETIC_BASE | head | 1/1 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |"])
|
|
|
|
def test_fails_where_the_places_name_no_go_file_of_the_base(self):
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
found = main.bench({**self.given, "--base": "main"}, self.env, "- `gone.go:1` `Gone`: removed\n- `README.md:1` `Calc`: thin\n", self.home)
|
|
self.assertEqual(found, Failure("The places the seats found hard name no Go file that the base holds, so there is nowhere to plant a bug."))
|
|
self.assertFalse(self.log.exists())
|
|
|
|
def test_fails_where_the_tests_fail_without_a_bug(self):
|
|
found = self.bench(**{"--test": "echo broken; exit 2"})
|
|
self.assertIsInstance(found, Failure)
|
|
self.assertIn("The tests fail at the base without a planted bug", found.reason)
|
|
self.assertIn("broken", found.reason)
|
|
|
|
def test_fails_where_no_mutant_becomes_a_bug(self):
|
|
# Passing tests with the bug planted: nothing to tell Haiku.
|
|
(self.repo / "calc_test.go").write_text("exit 0\n")
|
|
self.commit("Tests that catch nothing")
|
|
self.assertEqual(self.bench(), Failure("None of the 2 mutants the tests kill becomes a bug: each is on a line the head changed, or fails no assertion with a message."))
|
|
|
|
def test_a_run_the_api_refuses_fails_the_bench_and_still_removes_what_it_made(self):
|
|
self.env["FAKE_REFUSED"] = "1"
|
|
self.assertEqual(self.bench(), Failure("claude could not use the API: API Error: 401"))
|
|
self.assertEqual(self.calls()[-1][:2], ["image", "rm"])
|
|
|
|
def test_a_docker_that_fails_to_build_fails_the_bench(self):
|
|
self.env["FAKE_FAILS"] = "build"
|
|
found = self.bench()
|
|
self.assertIsInstance(found, Failure)
|
|
self.assertIn("docker build", found.reason)
|
|
|
|
|
|
class MainTest(unittest.TestCase):
|
|
def run_main(self, *argv: str, token: str = "t") -> tuple[int, str]:
|
|
with contextlib.redirect_stderr(io.StringIO()) as errors, contextlib.redirect_stdout(io.StringIO()):
|
|
return main.main(list(argv), {"CLAUDE_CODE_OAUTH_TOKEN": token, "HOME": "/nonexistent"}), errors.getvalue()
|
|
|
|
def test_refuses_an_unknown_repeated_or_valueless_argument(self):
|
|
for argv, said in ((["--imgae", "x"], "unknown or repeated argument --imgae"), (["--image", "x", "--image=y"], "unknown or repeated argument --image"), (["--image"], "--image needs a value")):
|
|
code, errors = self.run_main(*argv)
|
|
self.assertEqual((code, errors.startswith(f"{said}; usage: task-bench --image")), (1, True), errors)
|
|
|
|
def test_refuses_a_missing_image_or_test_command(self):
|
|
self.assertEqual(self.run_main("--test", "go test ./...")[1].partition(";")[0], "--image is missing")
|
|
self.assertEqual(self.run_main("--image", "golang:1.27.4")[1].partition(";")[0], "--test is missing")
|
|
|
|
def test_refuses_a_count_that_is_no_whole_number_from_1(self):
|
|
for name, value in (("--bugs", "0"), ("--runs", "-1"), ("--seed", "x")):
|
|
self.assertEqual(self.run_main("--image", "i", "--test", "t", name, value)[1].partition(";")[0], f"{name} takes a whole number from 1, not {value}")
|
|
|
|
def test_refuses_to_start_without_a_token(self):
|
|
self.assertEqual(self.run_main("--image", "i", "--test", "t", token="")[1].partition(";")[0], "CLAUDE_CODE_OAUTH_TOKEN holds no token")
|
|
|
|
def test_prints_its_help(self):
|
|
with contextlib.redirect_stdout(io.StringIO()) as out:
|
|
self.assertEqual(main.main(["--help"], {}), 0)
|
|
self.assertIn("usage: task-bench --image <image> --test <shell text>", out.getvalue())
|
|
|
|
|
|
class CheckedTreeTest(unittest.TestCase):
|
|
def setUp(self):
|
|
folder = tempfile.TemporaryDirectory()
|
|
self.addCleanup(folder.cleanup)
|
|
self.root = pathlib.Path(folder.name)
|
|
|
|
def tree(self, name: str, files: dict[str, str]) -> str:
|
|
for path, text in files.items():
|
|
(self.root / name / path).parent.mkdir(parents=True, exist_ok=True)
|
|
(self.root / name / path).write_text(text)
|
|
return str(self.root / name)
|
|
|
|
def held(self, name: str) -> dict[str, str]:
|
|
top = self.root / name
|
|
return {str(path.relative_to(top)): path.read_text() for path in sorted(top.rglob("*")) if path.is_file()}
|
|
|
|
def test_takes_the_code_as_fixed_and_the_tests_as_planted(self):
|
|
planted = self.tree("planted", {"calc.go": "bug", "calc_test.go": "tests", "sub/old.go": "old", "sub/old_test.go": "old tests", ".git/HEAD": "ref"})
|
|
copy = self.tree("copy", {"calc.go": "fixed", "calc_test.go": "rewritten", "extra_test.go": "added", "sub/new.go": "new", ".git/HEAD": "moved"})
|
|
checked_tree(planted, copy, str(self.root / "check"))
|
|
self.assertEqual(self.held("check"), {".git/HEAD": "ref", "calc.go": "fixed", "calc_test.go": "tests", "sub/new.go": "new", "sub/old_test.go": "old tests"})
|
|
|
|
def test_copies_a_link_as_a_link(self):
|
|
planted = self.tree("planted", {"calc.go": "bug"})
|
|
copy = self.tree("copy", {"calc.go": "fixed"})
|
|
os.symlink("/etc/hostname", os.path.join(copy, "host.go"))
|
|
checked_tree(planted, copy, str(self.root / "check"))
|
|
self.assertEqual(os.readlink(self.root / "check/host.go"), "/etc/hostname")
|
|
|
|
|
|
class ApiProxyTest(unittest.IsolatedAsyncioTestCase):
|
|
async def asyncSetUp(self):
|
|
async def echoed(reader, writer):
|
|
writer.write(b"api:" + await reader.read(5))
|
|
await writer.drain()
|
|
writer.close()
|
|
|
|
api = await asyncio.start_server(echoed, "127.0.0.1", 0)
|
|
self.allowed = f"127.0.0.1:{api.sockets[0].getsockname()[1]}"
|
|
proxy = await asyncio.start_server(lambda reader, writer: api_proxy.served(self.allowed, reader, writer), "127.0.0.1", 0)
|
|
self.port = proxy.sockets[0].getsockname()[1]
|
|
for server in (api, proxy):
|
|
self.addAsyncCleanup(server.wait_closed)
|
|
self.addCleanup(server.close)
|
|
|
|
async def answered(self, request: bytes) -> bytes:
|
|
reader, writer = await asyncio.open_connection("127.0.0.1", self.port)
|
|
writer.write(request)
|
|
answer = await asyncio.wait_for(reader.read(), 5)
|
|
writer.close()
|
|
return answer
|
|
|
|
async def test_tunnels_a_connect_to_the_one_allowed_target(self):
|
|
self.assertEqual(await self.answered(f"CONNECT {self.allowed} HTTP/1.1\r\n\r\nhello".encode()), b"HTTP/1.1 200 Connection established\r\n\r\napi:hello")
|
|
|
|
async def test_answers_403_to_any_other_target_or_method(self):
|
|
for request in (b"CONNECT example.com:443 HTTP/1.1\r\n\r\n", f"GET http://{self.allowed}/ HTTP/1.1\r\n\r\n".encode(), b"junk\r\n\r\n"):
|
|
self.assertEqual(await self.answered(request), b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|