Files
claude-config/bin/task_bench/test_task_bench.py
T

300 lines
17 KiB
Python

"""task-bench from its command line down: a whole bench over a small repository with docker, Gremlins and claude faked, the tree a
fix is checked in, and the proxy that reaches only the API."""
import asyncio
import contextlib
import io
import json
import os
import pathlib
import subprocess
import sys
import tempfile
import textwrap
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from failure import Failure
from task_bench import api_proxy, main
from task_bench.runs import checked_tree
from test_env import isolated_env
BASE_CODE = "package calc\n\nfunc Add(a, b int) int {\n\treturn a + b\n}\n\nfunc Twice(a int) int {\n\treturn a * 2\n}\n"
# The head moves Add down a line and rewrites Twice's line.
HEAD_CODE = "// Package calc adds.\n" + BASE_CODE.replace("a * 2", "2 * a")
# Stands in for `go test`: a shell text in a file named as a Go test, which fails as go test reports a failed assertion.
TESTS = textwrap.dedent("""\
grep -q 'a + b' calc.go && exit 0
printf -- '--- FAIL: TestAdd (0.00s)\\n calc_test.go:9: Add(1, 2) = -1, want 3\\nFAIL\\n'
exit 1
""")
# Reports `+` in Add, which the head keeps, and the `*` in Twice, which the head rewrites, as killed; and a mutant no test caught.
GREMLINS = textwrap.dedent("""\
import json, sys
lines = open("calc.go").read().split("\\n")
found = [{"line": at, "column": 11, "status": "KILLED", "type": "ARITHMETIC_BASE"} for at, line in enumerate(lines, 1) if " + " in line or " * " in line]
report = {"files": [{"file_name": "calc.go", "mutations": [*found, {**found[0], "status": "LIVED", "type": "INVERT_NEGATIVES"}]}]}
json.dump(report, open(sys.argv[sys.argv.index("--output") + 1], "w"))
""")
# Logs each call. A `run` of the bench's image runs its command in the mounted tree: sh as it is, and claude as the fake below.
DOCKER = textwrap.dedent("""\
#!/usr/bin/env python3
import json, os, subprocess, sys
args = sys.argv[1:]
with open(os.environ["FAKE_LOG"], "a") as log:
log.write(json.dumps(args) + "\\n")
if args[0] == "build":
sys.stdin.read()
if args[0] != "run" or "--detach" in args:
sys.exit(int(os.environ.get("FAKE_FAILS") == args[0]))
tree = args[args.index("--volume") + 1].rpartition(":")[0]
command = args[next(at for at, arg in enumerate(args) if arg.endswith(":throwaway")) + 1:]
if command[0] == "claude":
command = [sys.executable, os.environ["FAKE_CLAUDE"], command[-1]]
sys.exit(subprocess.run(command, cwd=tree).returncode)
""")
# Reads the README, then the file with the bug. At the head it fixes the bug; at the base it rewrites the test to pass instead.
CLAUDE = textwrap.dedent("""\
import json, os, pathlib, sys
def call(name, path):
return {"type": "assistant", "message": {"id": "msg_" + path, "content": [{"type": "tool_use", "id": "toolu_" + path, "name": name, "input": {"file_path": path}}], "usage": {}}}
if os.environ.get("FAKE_REFUSED"):
print(json.dumps({"type": "result", "is_error": True, "api_error_status": 401, "result": "API Error: 401"}))
sys.exit(1)
print(json.dumps({"type": "system", "subtype": "init", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330"}))
print(json.dumps(call("Read", "/work/README.md")))
print(json.dumps(call("Read", "/work/calc.go")))
code = pathlib.Path("calc.go")
if code.read_text().startswith("//"):
code.write_text(code.read_text().replace("a - b", "a + b"))
else:
pathlib.Path("calc_test.go").write_text("exit 0\\n")
pathlib.Path("prompt").write_text(sys.argv[1])
usage = {"input_tokens": 40, "cache_creation_input_tokens": 9000, "cache_read_input_tokens": 52000, "output_tokens": 800}
print(json.dumps({"type": "result", "subtype": "success", "is_error": False, "num_turns": 3, "duration_ms": 61500, "usage": usage, "result": "Fixed."}))
""")
PLACES = "## 2026-10-08T12:00:00Z, PR #1 at aaaaaaa\n\nJunior A, hardest first:\n\n- `calc.go:3` `Add`: long\n- `gone.go:1` `Gone`: removed since\n- `README.md:1` `Calc`: thin\n"
class BenchTest(unittest.TestCase):
def setUp(self):
folder = tempfile.TemporaryDirectory()
self.addCleanup(folder.cleanup)
self.root = pathlib.Path(folder.name).resolve()
self.home, self.repo, fakes = self.root / "home", self.root / "repo", self.root / "fakes"
for made in (self.home, self.repo, fakes):
made.mkdir()
(fakes / "docker").write_text(DOCKER)
(fakes / "docker").chmod(0o755)
(fakes / "claude.py").write_text(CLAUDE)
(fakes / "gremlins.py").write_text(GREMLINS)
self.log = self.root / "docker.log"
self.env = {**isolated_env(), "HOME": str(self.home), "PATH": f"{fakes}{os.pathsep}{os.environ['PATH']}", "FAKE_LOG": str(self.log), "FAKE_CLAUDE": str(fakes / "claude.py"), "CLAUDE_CODE_OAUTH_TOKEN": "sk-secret-token", "RUN_HEAVY_MEMORY": "2048"}
self.given = {"--image": "golang:1.27.4", "--test": "sh calc_test.go", "--prefix": f"{sys.executable} {fakes / 'gremlins.py'}", "--seed": "1", "--bugs": "2", "--runs": "2"}
self.git("init", "--quiet", "--initial-branch", "main")
for name, text in (("calc.go", BASE_CODE), ("calc_test.go", TESTS), ("README.md", "# Calc\n")):
(self.repo / name).write_text(text)
self.base = self.commit("Base")
self.git("checkout", "--quiet", "-b", "item-1")
(self.repo / "calc.go").write_text(HEAD_CODE)
self.head = self.commit("Head")
self.enterContext(contextlib.chdir(self.repo))
def git(self, *args: str) -> str:
return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@localhost", "-c", "commit.gpgsign=false", *args], cwd=self.repo, env=self.env, check=True, capture_output=True, text=True).stdout.strip()
def commit(self, title: str) -> str:
self.git("add", "--all")
self.git("commit", "--quiet", "--message", title)
return self.git("rev-parse", "HEAD")
def bench(self, **given: str):
with contextlib.redirect_stderr(io.StringIO()) as self.progress:
return main.bench({**self.given, "--base": "main", **given}, self.env, PLACES, self.home)
def calls(self) -> list[list[str]]:
return [json.loads(line) for line in self.log.read_text().splitlines()]
def test_plants_the_killed_mutant_the_head_left_alone_and_prints_how_each_side_fixed_it(self):
report = self.bench()
self.assertNotIsInstance(report, Failure, report)
folder = pathlib.Path(report[-1].removeprefix("Each run's result and stream: "))
self.assertEqual(report[:-1], [
f"task-bench: base {self.base[:12]}, head {self.head[:12]}. 1 bug, fixed 2 times at each side by claude-haiku-5-5 in Claude Code 2.1.330.",
"Planted 1 of the 2 bugs asked for: no other mutant the tests kill becomes one.",
"",
"| Task | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |",
"|---|---|---|---|---|---|---|---|---|---|",
"| 1: calc.go:4:11 ARITHMETIC_BASE | base | 0/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
"| 1: calc.go:4:11 ARITHMETIC_BASE | head | 2/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
"| All | base | 0/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
"| All | head | 2/2 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |",
"| All | change | +2 | 0 % | 0 % | 0 % | 0 % | 0 | 0 % | 0 |",
"",
])
self.assertTrue(folder.is_relative_to(self.home / ".local/state/larv-review") and folder.name == "task-bench", folder)
results = json.loads((folder / "results.json").read_text())
self.assertEqual((results["sides"], results["bugs"], len(results["runs"])), ({"base": self.base, "head": self.head}, {"1": "calc.go:4:11 ARITHMETIC_BASE"}, 4))
self.assertEqual(results["runs"][0], {"bug": 1, "side": "base", "run": 1, "passed": False, "finished": True, "claude_code": "2.1.330", "model": "claude-haiku-5-5", "tokens": {"input": 40, "write_5m": 9000, "write_1h": 0, "read": 52000, "output": 800}, "turns": 3, "seconds": 61.5, "reads": 1})
self.assertEqual(sorted(path.name for path in (folder / "runs").iterdir() if path.suffix == ".jsonl"), ["bug-1-base-1.jsonl", "bug-1-base-2.jsonl", "bug-1-head-1.jsonl", "bug-1-head-2.jsonl"])
self.assertIn("run 4 of 4, bug 1 at the head: passed.", self.progress.getvalue())
def test_each_side_gets_its_own_tree_with_the_bug_committed_once_and_no_trace_of_the_checkout(self):
folder = pathlib.Path(self.bench()[-1].removeprefix("Each run's result and stream: "))
for side, code in (("base", BASE_CODE), ("head", HEAD_CODE)):
tree = folder / "bugs/1" / side
self.assertEqual((tree / "calc.go").read_text(), code.replace("a + b", "a - b"))
log = subprocess.run(["git", "log", "--format=%an %s"], cwd=tree, env=self.env, check=True, capture_output=True, text=True).stdout
self.assertEqual(log, "task-bench Import\n")
self.assertEqual(subprocess.run(["git", "status", "--porcelain"], cwd=tree, env=self.env, check=True, capture_output=True, text=True).stdout, "")
def test_claude_runs_in_the_benchs_image_on_its_network_with_the_message_alone_and_the_token_by_name(self):
self.bench(**{"--runs": "1"})
calls = self.calls()
fixes = [call for call in calls if "claude" in call]
self.assertEqual(len(fixes), 2)
name = fixes[0][fixes[0].index("--network") + 1]
self.assertRegex(name, r"^task-bench-[0-9a-f]{12}$")
image = fixes[0].index(f"{name}:throwaway")
self.assertEqual(fixes[0][image + 1:], ["claude", "-p", "--strict-mcp-config", "--disable-slash-commands", "--no-session-persistence", "--model", "claude-haiku-5-5", "--effort", "low", "--output-format", "stream-json", "--verbose", "--permission-mode", "bypassPermissions", "A test of this project fails with the message below. Find the bug in the code and fix it. Change no test.\n\nAdd(1, 2) = -1, want 3"])
options = fixes[0][:image]
for expected in (["--memory", "2048m"], ["--memory-swap", "2048m"], ["--user", f"{os.getuid()}:{os.getgid()}"], ["--env", "HOME=/tmp"], ["--env", "CLAUDE_CODE_OAUTH_TOKEN"], ["--env", f"HTTPS_PROXY=http://{name}-proxy:8080"], ["--workdir", "/work"]):
self.assertIn(expected, [options[at:at + 2] for at in range(len(options))])
self.assertNotIn("sk-secret-token", self.log.read_text())
self.assertEqual(calls[0][:1] + calls[0][-3:], ["build", "--tag", f"{name}:throwaway", "-"])
self.assertEqual(calls[1], ["network", "create", "--internal", name])
self.assertEqual([call[:2] for call in calls[-4:]], [["rm", "--force"], ["rm", "--force"], ["network", "rm"], ["image", "rm"]])
def test_the_tests_run_without_a_network(self):
self.bench(**{"--runs": "1"})
tests = [call for call in self.calls() if call[-3:] == ["sh", "-c", "sh calc_test.go"]]
# Each side without a bug, each side with it, and each side's fix.
self.assertEqual(len(tests), 6)
for call in tests:
self.assertEqual(call[call.index("--network") + 1], "none")
def test_benches_the_head_against_itself_where_the_base_is_the_head(self):
report = self.bench(**{"--base": "HEAD", "--runs": "1", "--bugs": "1"})
self.assertEqual(report[0], f"task-bench: base {self.head[:12]}, head {self.head[:12]}. 1 bug, fixed 1 time at each side by claude-haiku-5-5 in Claude Code 2.1.330.")
self.assertEqual(report[4:6], ["| 1: calc.go:5:11 ARITHMETIC_BASE | base | 1/1 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |", "| 1: calc.go:5:11 ARITHMETIC_BASE | head | 1/1 | 40 | 9000 | 52000 | 800 | 3 | 61.5 | 1 |"])
def test_fails_where_the_places_name_no_go_file_of_the_base(self):
with contextlib.redirect_stderr(io.StringIO()):
found = main.bench({**self.given, "--base": "main"}, self.env, "- `gone.go:1` `Gone`: removed\n- `README.md:1` `Calc`: thin\n", self.home)
self.assertEqual(found, Failure("The places the seats found hard name no Go file that the base holds, so there is nowhere to plant a bug."))
self.assertFalse(self.log.exists())
def test_fails_where_the_tests_fail_without_a_bug(self):
found = self.bench(**{"--test": "echo broken; exit 2"})
self.assertIsInstance(found, Failure)
self.assertIn("The tests fail at the base without a planted bug", found.reason)
self.assertIn("broken", found.reason)
def test_fails_where_no_mutant_becomes_a_bug(self):
# Passing tests with the bug planted: nothing to tell Haiku.
(self.repo / "calc_test.go").write_text("exit 0\n")
self.commit("Tests that catch nothing")
self.assertEqual(self.bench(), Failure("None of the 2 mutants the tests kill becomes a bug: each is on a line the head changed, or fails no assertion with a message."))
def test_a_run_the_api_refuses_fails_the_bench_and_still_removes_what_it_made(self):
self.env["FAKE_REFUSED"] = "1"
self.assertEqual(self.bench(), Failure("claude could not use the API: API Error: 401"))
self.assertEqual(self.calls()[-1][:2], ["image", "rm"])
def test_a_docker_that_fails_to_build_fails_the_bench(self):
self.env["FAKE_FAILS"] = "build"
found = self.bench()
self.assertIsInstance(found, Failure)
self.assertIn("docker build", found.reason)
class MainTest(unittest.TestCase):
def run_main(self, *argv: str, token: str = "t") -> tuple[int, str]:
with contextlib.redirect_stderr(io.StringIO()) as errors, contextlib.redirect_stdout(io.StringIO()):
return main.main(list(argv), {"CLAUDE_CODE_OAUTH_TOKEN": token, "HOME": "/nonexistent"}), errors.getvalue()
def test_refuses_an_unknown_repeated_or_valueless_argument(self):
for argv, said in ((["--imgae", "x"], "unknown or repeated argument --imgae"), (["--image", "x", "--image=y"], "unknown or repeated argument --image"), (["--image"], "--image needs a value")):
code, errors = self.run_main(*argv)
self.assertEqual((code, errors.startswith(f"{said}; usage: task-bench --image")), (1, True), errors)
def test_refuses_a_missing_image_or_test_command(self):
self.assertEqual(self.run_main("--test", "go test ./...")[1].partition(";")[0], "--image is missing")
self.assertEqual(self.run_main("--image", "golang:1.27.4")[1].partition(";")[0], "--test is missing")
def test_refuses_a_count_that_is_no_whole_number_from_1(self):
for name, value in (("--bugs", "0"), ("--runs", "-1"), ("--seed", "x")):
self.assertEqual(self.run_main("--image", "i", "--test", "t", name, value)[1].partition(";")[0], f"{name} takes a whole number from 1, not {value}")
def test_refuses_to_start_without_a_token(self):
self.assertEqual(self.run_main("--image", "i", "--test", "t", token="")[1].partition(";")[0], "CLAUDE_CODE_OAUTH_TOKEN holds no token")
def test_prints_its_help(self):
with contextlib.redirect_stdout(io.StringIO()) as out:
self.assertEqual(main.main(["--help"], {}), 0)
self.assertIn("usage: task-bench --image <image> --test <shell text>", out.getvalue())
class CheckedTreeTest(unittest.TestCase):
def setUp(self):
folder = tempfile.TemporaryDirectory()
self.addCleanup(folder.cleanup)
self.root = pathlib.Path(folder.name)
def tree(self, name: str, files: dict[str, str]) -> str:
for path, text in files.items():
(self.root / name / path).parent.mkdir(parents=True, exist_ok=True)
(self.root / name / path).write_text(text)
return str(self.root / name)
def held(self, name: str) -> dict[str, str]:
top = self.root / name
return {str(path.relative_to(top)): path.read_text() for path in sorted(top.rglob("*")) if path.is_file()}
def test_takes_the_code_as_fixed_and_the_tests_as_planted(self):
planted = self.tree("planted", {"calc.go": "bug", "calc_test.go": "tests", "sub/old.go": "old", "sub/old_test.go": "old tests", ".git/HEAD": "ref"})
copy = self.tree("copy", {"calc.go": "fixed", "calc_test.go": "rewritten", "extra_test.go": "added", "sub/new.go": "new", ".git/HEAD": "moved"})
checked_tree(planted, copy, str(self.root / "check"))
self.assertEqual(self.held("check"), {".git/HEAD": "ref", "calc.go": "fixed", "calc_test.go": "tests", "sub/new.go": "new", "sub/old_test.go": "old tests"})
def test_copies_a_link_as_a_link(self):
planted = self.tree("planted", {"calc.go": "bug"})
copy = self.tree("copy", {"calc.go": "fixed"})
os.symlink("/etc/hostname", os.path.join(copy, "host.go"))
checked_tree(planted, copy, str(self.root / "check"))
self.assertEqual(os.readlink(self.root / "check/host.go"), "/etc/hostname")
class ApiProxyTest(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self):
async def echoed(reader, writer):
writer.write(b"api:" + await reader.read(5))
await writer.drain()
writer.close()
api = await asyncio.start_server(echoed, "127.0.0.1", 0)
self.allowed = f"127.0.0.1:{api.sockets[0].getsockname()[1]}"
proxy = await asyncio.start_server(lambda reader, writer: api_proxy.served(self.allowed, reader, writer), "127.0.0.1", 0)
self.port = proxy.sockets[0].getsockname()[1]
for server in (api, proxy):
self.addAsyncCleanup(server.wait_closed)
self.addCleanup(server.close)
async def answered(self, request: bytes) -> bytes:
reader, writer = await asyncio.open_connection("127.0.0.1", self.port)
writer.write(request)
answer = await asyncio.wait_for(reader.read(), 5)
writer.close()
return answer
async def test_tunnels_a_connect_to_the_one_allowed_target(self):
self.assertEqual(await self.answered(f"CONNECT {self.allowed} HTTP/1.1\r\n\r\nhello".encode()), b"HTTP/1.1 200 Connection established\r\n\r\napi:hello")
async def test_answers_403_to_any_other_target_or_method(self):
for request in (b"CONNECT example.com:443 HTTP/1.1\r\n\r\n", f"GET http://{self.allowed}/ HTTP/1.1\r\n\r\n".encode(), b"junk\r\n\r\n"):
self.assertEqual(await self.answered(request), b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n")
if __name__ == "__main__":
unittest.main()