Test task-bench's docker calls, command line and planter where mutants survived
CI / tests (pull_request) Successful in 4m38s

This commit is contained in:
2026-11-21 01:39:12 +01:00
parent fed84cc510
commit ebbe926ab7
7 changed files with 316 additions and 35 deletions
+2 -2
View File
@@ -3,10 +3,10 @@ import dataclasses
import statistics
import typing
from claude_code.entries import json_object, parse_entry, tool_calls, ToolCall
from claude_code.tokens import Tokens, tokens_of
from failure import Failure
from claude_code.entries import json_object, parse_entry, tool_calls, ToolCall
from reading_cost.measure import reads_of
from claude_code.tokens import Tokens, tokens_of
# Where each run's container mounts its copy of the code, and its home folder: the paths in a run's stream start there.
WORK = "/work"
+19 -18
View File
@@ -52,18 +52,17 @@ import tarfile
import typing
import uuid
from claude_code.tokens import Tokens
from failure import Failure
from git_cli import git_repo
from git_cli.git_repo import default_ref, Git, top
from larv_review.history_commands import read_places
from task_bench.runs import checked_tree, closed, Containers, docker_cli, fix_attempted, MEMORY_MIB, opened, ran_tests, TOKEN
from machine_load.slots import CPUS_PER_SLOT
from mutate_changes.tool_process import failed, Ran, run_tool
from review_rules.review_folder import made_review_folder
from run_heavy.capped_docker import SHARE_VARIABLE
from task_bench.figures import Figures, figures, Result, SIDES, table, WORK
from git_cli.git_repo import default_ref, Git, top
from task_bench.plant import drawn, killed_mutants, line_at_head, Mutant, mutated, place_files, task_text
from task_bench.runs import checked_tree, closed, Containers, docker_cli, fix_attempted, MEMORY_MIB, opened, tests_run, TOKEN
from mutate_changes.tool_process import failed, Ran, run_tool
from failure import Failure
from task_bench.figures import figures, Result, SIDES, table, WORK
from git_cli import git_repo
from review_rules.review_folder import made_review_folder
from larv_review.history_commands import read_places
from run_heavy.capped_docker import SHARE_VARIABLE
USAGE = "usage: task-bench --image <image> --test <shell text> [--base <ref>] [--head <ref>] [--prefix <shell text>] [--seed <n>] [--bugs <n>] [--runs <n>]"
# A panel of three readers, seated as the README's audience, picked these options on 2026-11-21, each 3 to 0.
@@ -85,8 +84,9 @@ class Task:
return f"{self.mutant.path}:{self.mutant.line}:{self.mutant.column} {self.mutant.mutator}"
def main(argv: list[str], env: typing.Mapping[str, str]) -> int:
"""Runs task-bench with the arguments argv; git, docker and Gremlins run in the environment env."""
def main(argv: list[str], env: typing.Mapping[str, str], read: typing.Callable[[pathlib.Path], str | Failure] = read_places) -> int:
"""Runs task-bench with the arguments argv; git, docker and Gremlins run in the environment env, and `read` reads what
`larv-review places` prints for the checkout."""
if argv in (["-h"], ["--help"]):
print(__doc__.strip())
return 0
@@ -95,7 +95,7 @@ def main(argv: list[str], env: typing.Mapping[str, str]) -> int:
print(f"{given.reason}; {USAGE}", file=sys.stderr)
return 1
home = pathlib.Path(env.get("HOME") or pathlib.Path.home())
places = read_places(home)
places = read(home)
report = places if isinstance(places, Failure) else bench(given, env, places, home)
if isinstance(report, Failure):
print(report.reason, file=sys.stderr)
@@ -269,7 +269,7 @@ def tests_in_copy(containers: Containers, tree: str, folder: pathlib.Path) -> Ra
copy = folder / "tested"
shutil.copytree(tree, copy, symlinks=True)
try:
return tests_run(containers, str(copy))
return ran_tests(containers, str(copy))
finally:
shutil.rmtree(copy, ignore_errors=True)
@@ -289,7 +289,7 @@ def fixed(task: Task, side: str, number: int, containers: Containers, folder: pa
if isinstance(found, Failure):
return found
checked_tree(task.trees[side], str(copy), str(check))
ran = tests_run(containers, str(check))
ran = ran_tests(containers, str(check))
return Result(task.number, side, number, not isinstance(ran, Failure) and ran.code == 0, found)
finally:
shutil.rmtree(copy, ignore_errors=True)
@@ -298,9 +298,10 @@ def fixed(task: Task, side: str, number: int, containers: Containers, folder: pa
def recorded(result: Result) -> dict:
"""A run as results.json holds it."""
found = result.figures or Figures("", "", Tokens(), 0, 0, None)
measured = {"claude_code": found.version, "model": found.model, "tokens": dataclasses.asdict(found.tokens), "turns": found.turns, "seconds": found.seconds, "reads": found.reads}
return {"bug": result.task, "side": result.side, "run": result.run, "passed": result.passed, "finished": result.figures is not None, **(measured if result.figures else {})}
run = {"bug": result.task, "side": result.side, "run": result.run, "passed": result.passed, "finished": result.figures is not None}
if not (found := result.figures):
return run
return {**run, "claude_code": found.version, "model": found.model, "tokens": dataclasses.asdict(found.tokens), "turns": found.turns, "seconds": found.seconds, "reads": found.reads}
def counted(count: int, noun: str) -> str:
+1 -1
View File
@@ -6,8 +6,8 @@ import re
import typing
from failure import Failure
from mutate_changes.changed_lines import is_test
from mutate_changes.in_diff_tools import gremlins_mutants
from mutate_changes.changed_lines import is_test
from mutate_changes.tool_process import Run
# A place as `larv-review places` prints it: `<path>:<line>` in backticks.
+6 -6
View File
@@ -8,11 +8,11 @@ import shutil
import subprocess
import typing
from failure import Failure
from mutate_changes.changed_lines import is_test
from mutate_changes.tool_process import failed, Ran
from task_bench.api_proxy import PORT
from failure import Failure
from task_bench.figures import HOME, WORK
from mutate_changes.changed_lines import is_test
from task_bench.api_proxy import PORT
MODEL = "claude-haiku-5-5"
# No MCP server, skill, command or stored session, so a run starts from the code alone; -p needs --verbose with stream-json, and no
@@ -41,10 +41,10 @@ class Docker(typing.Protocol):
def __call__(self, args: list[str], *, seconds: float, stdin: str | None = None, stdout: typing.IO[bytes] | None = None, stderr: typing.IO[bytes] | None = None) -> Ran | Failure: ...
def docker_cli(env: typing.Mapping[str, str]) -> Docker:
def docker_cli(env: typing.Mapping[str, str], run: typing.Callable[..., subprocess.CompletedProcess[bytes]] = subprocess.run) -> Docker:
def docker(args: list[str], *, seconds: float, stdin: str | None = None, stdout: typing.IO[bytes] | None = None, stderr: typing.IO[bytes] | None = None) -> Ran | Failure:
try:
ran = subprocess.run(["docker", *args], env=env, input=None if stdin is None else stdin.encode(), stdin=subprocess.DEVNULL if stdin is None else None, stdout=stdout or subprocess.PIPE, stderr=stderr or subprocess.STDOUT, timeout=seconds, check=False)
ran = run(["docker", *args], env=env, input=None if stdin is None else stdin.encode(), stdin=subprocess.DEVNULL if stdin is None else None, stdout=stdout or subprocess.PIPE, stderr=stderr or subprocess.STDOUT, timeout=seconds, check=False)
except OSError as failure:
return Failure(f"docker could not start: {failure.strerror}")
except subprocess.TimeoutExpired:
@@ -105,7 +105,7 @@ def closed(containers: Containers) -> None:
containers.docker(args, seconds=120)
def tests_run(containers: Containers, tree: str) -> Ran | Failure:
def ran_tests(containers: Containers, tree: str) -> Ran | Failure:
"""The project's test command, run at the top of tree with no network; the tests pass where it exits 0. A Failure where it still
ran after RUN_SECONDS."""
ran = containers.docker(["run", "--rm", "--name", containers.running, "--network", "none", *in_tree(containers, tree), containers.image, "sh", "-c", containers.test], seconds=RUN_SECONDS)
+17 -1
View File
@@ -6,9 +6,9 @@ import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from claude_code.tokens import Tokens
from failure import Failure
from task_bench.figures import Figures, figures, Result, table
from claude_code.tokens import Tokens
INIT = {"type": "system", "subtype": "init", "cwd": "/work", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330"}
USAGE = {"input_tokens": 40, "cache_creation_input_tokens": 9000, "cache_read_input_tokens": 52000, "output_tokens": 800, "cache_creation": {"ephemeral_1h_input_tokens": 9000, "ephemeral_5m_input_tokens": 0}}
@@ -53,6 +53,18 @@ class FiguresTest(unittest.TestCase):
self.assertIsNone(figures(stream(INIT, call("Read", file_path="/work/calc.go")), "/work/calc.go"))
self.assertIsNone(figures(b"", "/work/calc.go"))
def test_a_result_without_turns_or_a_duration_counts_none(self):
found = figures(stream(INIT, {"type": "result", "usage": USAGE}), "/work/calc.go")
self.assertEqual((found.turns, found.seconds), (0, 0))
def test_a_stream_without_an_init_names_no_version_or_model(self):
found = figures(stream(RESULT), "/work/calc.go")
self.assertEqual((found.version, found.model), ("", ""))
def test_a_shell_command_that_reads_under_the_runs_home_reads_no_file_of_the_code(self):
found = figures(stream(INIT, call("Bash", command="cat ~/calc.go"), call("Read", file_path="/work/calc.go"), RESULT), "/work/calc.go")
self.assertEqual(found.reads, 1)
def test_a_run_the_api_refused_is_a_failure_that_quotes_claude(self):
refused = {**RESULT, "is_error": True, "api_error_status": 401, "result": "Failed to authenticate. API Error: 401 OAuth access token is invalid."}
self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: Failed to authenticate. API Error: 401 OAuth access token is invalid."))
@@ -86,6 +98,10 @@ class TableTest(unittest.TestCase):
"| All | change | +2 | -50 % | 0 % | 0 % | 0 % | -3.5 | -50 % | -2.5 |",
])
def test_the_change_is_a_dash_where_the_base_measured_zero_and_signed_where_it_rose(self):
results = [run(1, "base", 1, True, input_=0, turns=10, seconds=60, reads=0), run(1, "head", 1, False, input_=50, turns=12, seconds=90, reads=2)]
self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[-1], "| All | change | -1 | - | 0 % | 0 % | 0 % | +2 | +50 % | +2 |")
def test_a_run_without_figures_counts_as_not_passed_and_a_side_without_any_shows_a_dash(self):
results = [Result(1, "base", 1, False, None), run(1, "head", 1, True)]
self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[2:], [
+73 -1
View File
@@ -1,11 +1,15 @@
"""task-bench's planter: which files it mutates, which mutants it draws, how it re-applies one and what it tells Haiku."""
import json
import pathlib
import sys
import tempfile
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from task_bench.plant import drawn, line_at_head, Mutant, mutated, place_files, task_text
from task_bench.plant import drawn, go_files, killed_mutants, line_at_head, Mutant, mutated, place_files, task_text
from failure import Failure
from mutate_changes.tool_process import Ran
PLACES = """## 2026-11-19T22:33:15Z, PR #646 at a357cc7b
@@ -128,3 +132,71 @@ class LineAtHeadTest(unittest.TestCase):
def test_no_diff_keeps_every_line(self):
self.assertEqual(line_at_head("", 12), 12)
class KilledMutantsTest(unittest.TestCase):
def setUp(self):
folder = tempfile.TemporaryDirectory()
self.addCleanup(folder.cleanup)
self.top = pathlib.Path(folder.name)
for path in ("calc.go", "calc_test.go", "render/pick.go", "render/a+b.go", "render/notes.md", "cli/main.go"):
(self.top / path).parent.mkdir(parents=True, exist_ok=True)
(self.top / path).write_text("package x\n")
self.ran: list[tuple[list[str], str]] = []
def gremlins(self, report: dict | None, code: int = 0):
def run(argv: list[str], cwd: str) -> Ran:
self.ran.append((argv, cwd))
if report is not None:
pathlib.Path(cwd, argv[argv.index("--output") + 1]).write_text(json.dumps(report))
return Ran(code, "gremlins said so")
return run
def test_names_each_go_file_gremlins_would_mutate_by_its_path_from_the_top(self):
self.assertEqual(sorted(go_files(str(self.top))), ["calc.go", "cli/main.go", "render/a+b.go", "render/pick.go"])
def test_has_gremlins_leave_out_every_other_go_file_by_its_whole_path(self):
killed_mutants(str(self.top), ["render/pick.go"], self.gremlins({"files": []}), 3)
argv, cwd = self.ran[0]
self.assertEqual(cwd, str(self.top))
self.assertEqual(argv[:8], ["gremlins", "unleash", "--exclude-files", r"^calc\.go$", "--exclude-files", r"^cli/main\.go$", "--exclude-files", r"^render/a\+b\.go$"])
self.assertEqual((argv[8], argv[10:]), ("--output", ["--workers", "3"]))
def test_keeps_the_killed_mutants_of_the_files_asked_for(self):
report = {"files": [
{"file_name": "render/pick.go", "mutations": [{"line": 4, "column": 7, "status": "KILLED", "type": "ARITHMETIC_BASE"}, {"line": 5, "column": 2, "status": "LIVED", "type": "INVERT_LOGICAL"}, {"line": 6, "column": 2, "status": "NOT COVERED", "type": "INVERT_LOGICAL"}]},
{"file_name": "calc.go", "mutations": [{"line": 1, "column": 1, "status": "KILLED", "type": "ARITHMETIC_BASE"}]},
]}
self.assertEqual(killed_mutants(str(self.top), ["render/pick.go"], self.gremlins(report), 3), [Mutant("render/pick.go", 4, 7, "ARITHMETIC_BASE")])
def test_a_gremlins_that_fails_is_a_failure_with_what_it_printed(self):
found = killed_mutants(str(self.top), ["render/pick.go"], self.gremlins(None, code=1), 3)
self.assertIsInstance(found, Failure)
self.assertIn("gremlins said so", found.reason)
class EdgesTest(unittest.TestCase):
def test_mutates_the_last_line_of_a_file(self):
self.assertEqual(mutated(b"a\nb + c", Mutant("calc.go", 2, 3, "ARITHMETIC_BASE")), b"a\nb - c")
self.assertIsNone(mutated(b"a\nb + c", Mutant("calc.go", 3, 3, "ARITHMETIC_BASE")))
self.assertIsNone(mutated(b"a\nb + c", Mutant("calc.go", 0, 3, "ARITHMETIC_BASE")))
def test_takes_the_longest_token_at_the_column(self):
self.assertEqual(mutated(b"a >= b", Mutant("calc.go", 1, 3, "CONDITIONALS_BOUNDARY")), b"a > b")
self.assertEqual(mutated(b"a &^ b", Mutant("calc.go", 1, 3, "INVERT_BITWISE")), b"a & b")
def test_a_message_ends_at_the_next_line_indented_as_its_first(self):
output = "--- FAIL: TestPick/empty_input (0.00s)\n pick_test.go:42: got 3\n want 4\n pick_test.go:50: got 5\n"
self.assertEqual(task_text(output), "got 3\nwant 4")
def test_a_message_runs_to_the_end_of_the_output(self):
self.assertEqual(task_text("--- FAIL: TestPick (0.00s)\n pick_test.go:42: got 3\n want 4"), "got 3\nwant 4")
def test_a_message_may_name_a_part_of_a_subtests_name_that_is_no_test(self):
output = "--- FAIL: TestPick/empty (0.00s)\n pick_test.go:42: empty input: got 3\n in TestPick\n in TestPick/empty\n"
self.assertEqual(task_text(output), "empty input: got 3")
if __name__ == "__main__":
unittest.main()
+198 -6
View File
@@ -14,10 +14,12 @@ import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from task_bench import api_proxy, main, runs
from task_bench.runs import checked_tree, closed, Containers, docker_cli, fix_attempted, opened, ran_tests
from failure import Failure
from task_bench import api_proxy, main
from task_bench.runs import checked_tree
from test_env import isolated_env
from mutate_changes.tool_process import Ran
from task_bench.figures import Result
BASE_CODE = "package calc\n\nfunc Add(a, b int) int {\n\treturn a + b\n}\n\nfunc Twice(a int) int {\n\treturn a * 2\n}\n"
# The head moves Add down a line and rewrites Twice's line.
@@ -30,11 +32,13 @@ TESTS = textwrap.dedent("""\
""")
# Reports `+` in Add, which the head keeps, and the `*` in Twice, which the head rewrites, as killed; and a mutant no test caught.
GREMLINS = textwrap.dedent("""\
import json, sys
import json, os, sys
lines = open("calc.go").read().split("\\n")
found = [{"line": at, "column": 11, "status": "KILLED", "type": "ARITHMETIC_BASE"} for at, line in enumerate(lines, 1) if " + " in line or " * " in line]
report = {"files": [{"file_name": "calc.go", "mutations": [*found, {**found[0], "status": "LIVED", "type": "INVERT_NEGATIVES"}]}]}
json.dump(report, open(sys.argv[sys.argv.index("--output") + 1], "w"))
if os.environ.get("FAKE_GREMLINS_LOG"):
json.dump([os.getcwd(), *sys.argv[1:]], open(os.environ["FAKE_GREMLINS_LOG"], "w"))
""")
# Logs each call. A `run` of the bench's image runs its command in the mounted tree: sh as it is, and claude as the fake below.
DOCKER = textwrap.dedent("""\
@@ -44,7 +48,9 @@ DOCKER = textwrap.dedent("""\
with open(os.environ["FAKE_LOG"], "a") as log:
log.write(json.dumps(args) + "\\n")
if args[0] == "build":
sys.stdin.read()
held = sys.stdin.read()
if os.environ.get("FAKE_STDIN"):
open(os.environ["FAKE_STDIN"], "w").write(held)
if args[0] != "run" or "--detach" in args:
sys.exit(int(os.environ.get("FAKE_FAILS") == args[0]))
tree = args[args.index("--volume") + 1].rpartition(":")[0]
@@ -94,11 +100,15 @@ class BenchTest(unittest.TestCase):
self.git("init", "--quiet", "--initial-branch", "main")
for name, text in (("calc.go", BASE_CODE), ("calc_test.go", TESTS), ("README.md", "# Calc\n")):
(self.repo / name).write_text(text)
# A link to nothing, which a copy that followed links would fail on.
os.symlink("missing", self.repo / "dangling")
self.base = self.commit("Base")
self.git("checkout", "--quiet", "-b", "item-1")
(self.repo / "calc.go").write_text(HEAD_CODE)
self.head = self.commit("Head")
self.enterContext(contextlib.chdir(self.repo))
# The bench runs from a folder below the checkout's top.
(self.repo / "below").mkdir()
self.enterContext(contextlib.chdir(self.repo / "below"))
def git(self, *args: str) -> str:
return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@localhost", "-c", "commit.gpgsign=false", *args], cwd=self.repo, env=self.env, check=True, capture_output=True, text=True).stdout.strip()
@@ -137,7 +147,58 @@ class BenchTest(unittest.TestCase):
self.assertEqual((results["sides"], results["bugs"], len(results["runs"])), ({"base": self.base, "head": self.head}, {"1": "calc.go:4:11 ARITHMETIC_BASE"}, 4))
self.assertEqual(results["runs"][0], {"bug": 1, "side": "base", "run": 1, "passed": False, "finished": True, "claude_code": "2.1.330", "model": "claude-haiku-5-5", "tokens": {"input": 40, "write_5m": 9000, "write_1h": 0, "read": 52000, "output": 800}, "turns": 3, "seconds": 61.5, "reads": 1})
self.assertEqual(sorted(path.name for path in (folder / "runs").iterdir() if path.suffix == ".jsonl"), ["bug-1-base-1.jsonl", "bug-1-base-2.jsonl", "bug-1-head-1.jsonl", "bug-1-head-2.jsonl"])
self.assertIn("run 4 of 4, bug 1 at the head: passed.", self.progress.getvalue())
self.assertEqual(results["seed"], 1)
self.assertEqual(self.progress.getvalue().splitlines(), [
f"task-bench: base {self.base[:12]}, head {self.head[:12]}.",
"task-bench: run 1 of 4, bug 1 at the base: not passed.",
"task-bench: run 2 of 4, bug 1 at the head: passed.",
"task-bench: run 3 of 4, bug 1 at the base: not passed.",
"task-bench: run 4 of 4, bug 1 at the head: passed.",
])
self.assertEqual(sorted(path.name for path in folder.iterdir()), ["bugs", "results.json", "runs", "trees"])
self.assertEqual((folder / "runs/bug-1-base-1.stderr").read_bytes(), b"")
self.assertEqual(os.readlink(folder / "bugs/1/head/dangling"), "missing")
def test_the_command_line_runs_the_bench_on_the_seats_places_and_prints_its_report(self):
argv = [part for name, value in self.given.items() for part in (name, value)]
homes = []
with contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(main.main(argv, self.env, lambda home: homes.append(home) or PLACES), 0)
self.assertEqual(homes, [self.home])
# Without --base, the base is the merge base with the default branch.
self.assertEqual(out.getvalue().splitlines()[0], f"task-bench: base {self.base[:12]}, head {self.head[:12]}. 1 bug, fixed 2 times at each side by claude-haiku-5-5 in Claude Code 2.1.330.")
self.assertEqual(out.getvalue().splitlines()[3], "| Task | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |")
self.assertTrue(out.getvalue().endswith("task-bench\n"), out.getvalue()[-80:])
def test_the_command_line_fails_with_the_reason_where_the_places_or_the_bench_fail(self):
argv = [part for name, value in self.given.items() for part in (name, value)]
for places, said in ((Failure("no history"), "no history\n"), ("- `gone.go:1` `Gone`: removed\n", "The places the seats found hard name no Go file that the base holds, so there is nowhere to plant a bug.\n")):
with contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()) as errors:
self.assertEqual(main.main(argv, self.env, lambda home: places), 1)
self.assertEqual((out.getvalue(), errors.getvalue().splitlines(keepends=True)[-1]), ("", said))
def test_the_head_is_the_commit_asked_for(self):
report = self.bench(**{"--head": "main", "--runs": "1"})
self.assertEqual(report[0], f"task-bench: base {self.base[:12]}, head {self.base[:12]}. 1 bug, fixed 1 time at each side by claude-haiku-5-5 in Claude Code 2.1.330.")
def test_fails_on_a_head_or_a_base_git_does_not_know(self):
for name in ("--head", "--base"):
found = self.bench(**{name: "no-such-ref"})
self.assertIsInstance(found, Failure)
self.assertIn("no-such-ref", found.reason)
def test_the_image_is_built_from_the_one_asked_for_with_the_caps_memory(self):
self.env["FAKE_STDIN"] = str(self.root / "dockerfile")
self.env.pop("RUN_HEAVY_MEMORY")
self.bench(**{"--runs": "1"})
self.assertEqual((self.root / "dockerfile").read_text().splitlines()[0], "FROM golang:1.27.4")
self.assertEqual(self.calls()[0][1:6], ["--quiet", "--memory", "4096m", "--memory-swap", "4096m"])
def test_gremlins_runs_through_the_prefix_in_the_copy_of_the_base_with_a_slots_workers(self):
self.env["FAKE_GREMLINS_LOG"] = str(self.root / "gremlins.log")
folder = pathlib.Path(self.bench(**{"--runs": "1"})[-1].removeprefix("Each run's result and stream: "))
cwd, *argv = json.loads((self.root / "gremlins.log").read_text())
self.assertEqual((cwd, argv[:2], argv[-2:]), (str(folder / "trees/base"), ["gremlins", "unleash"], ["--workers", str(main.CPUS_PER_SLOT)]))
def test_each_side_gets_its_own_tree_with_the_bug_committed_once_and_no_trace_of_the_checkout(self):
folder = pathlib.Path(self.bench()[-1].removeprefix("Each run's result and stream: "))
@@ -235,6 +296,127 @@ class MainTest(unittest.TestCase):
self.assertIn("usage: task-bench --image <image> --test <shell text>", out.getvalue())
def test_records_a_run_that_did_not_finish_without_figures(self):
self.assertEqual(main.recorded(Result(1, "base", 2, False, None)), {"bug": 1, "side": "base", "run": 2, "passed": False, "finished": False})
class Recorded:
"""A docker that records each call and answers by the call's first word, else that it ran."""
def __init__(self, **answers: Ran | Failure):
self.calls: list[tuple[list[str], float, str | None]] = []
self.answers = answers
def __call__(self, args, *, seconds, stdin=None, stdout=None, stderr=None):
self.calls.append((args, seconds, stdin))
if stdout:
stdout.write(b"stream")
stderr.write(b"errors")
return self.answers.get(args[0], Ran(0, "ran"))
class ContainersTest(unittest.TestCase):
IN_TREE = ["--memory", "512m", "--memory-swap", "512m", "--user", "1000:1000", "--env", "HOME=/tmp", "--volume", "/tree:/work", "--workdir", "/work"]
REMOVED = (["rm", "--force", "bench-run"], 120, None)
def setUp(self):
folder = tempfile.TemporaryDirectory()
self.addCleanup(folder.cleanup)
self.stream = pathlib.Path(folder.name, "run.jsonl")
def containers(self, **answers: Ran | Failure) -> Containers:
self.docker = Recorded(**answers)
return Containers(self.docker, "bench", "go test ./...", 512, "1000:1000")
def test_opens_the_image_the_network_without_a_route_out_and_the_proxy_on_both_networks(self):
self.assertIsNone(opened(self.containers(), "golang:1.27.4"))
proxy = str(pathlib.Path(runs.__file__).with_name("api_proxy.py"))
self.assertEqual(self.docker.calls, [
(["build", "--quiet", "--memory", "512m", "--memory-swap", "512m", "--tag", "bench:throwaway", "-"], 1200, "FROM golang:1.27.4\nUSER root\nRUN curl -fsSL https://claude.ai/install.sh | bash && cp -L /root/.local/bin/claude /usr/local/bin/claude\n"),
(["network", "create", "--internal", "bench"], 60, None),
(["run", "--detach", "--rm", "--name", "bench-proxy", "--memory", "256m", "--memory-swap", "256m", "--volume", f"{proxy}:/api_proxy.py:ro", "python:3.15.2-slim", "python", "/api_proxy.py", "api.anthropic.com:443"], 300, None),
(["network", "connect", "bench", "bench-proxy"], 60, None),
])
def test_opening_stops_at_the_first_step_that_fails_and_says_which(self):
found = opened(self.containers(network=Ran(1, "pool overlaps")), "golang:1.27.4")
self.assertEqual((found.reason.startswith("docker network"), "pool overlaps" in found.reason, len(self.docker.calls)), (True, True, 2))
self.assertEqual(opened(self.containers(build=Failure("docker could not start")), "golang:1.27.4"), Failure("docker could not start"))
def test_closing_removes_the_running_container_the_proxy_the_network_and_the_image(self):
closed(self.containers())
self.assertEqual(self.docker.calls, [self.REMOVED, (["rm", "--force", "bench-proxy"], 120, None), (["network", "rm", "bench"], 120, None), (["image", "rm", "bench:throwaway"], 120, None)])
def test_the_tests_run_in_the_tree_without_a_network_for_at_most_half_an_hour(self):
self.assertEqual(ran_tests(self.containers(), "/tree"), Ran(0, "ran"))
self.assertEqual(self.docker.calls, [(["run", "--rm", "--name", "bench-run", "--network", "none", *self.IN_TREE, "bench:throwaway", "sh", "-c", "go test ./..."], 1800, None)])
def test_tests_that_run_out_of_time_are_a_failure_and_their_container_goes(self):
late = Failure("docker run still ran after 30 min")
self.assertEqual(ran_tests(self.containers(run=late), "/tree"), late)
self.assertEqual(self.docker.calls[1:], [self.REMOVED])
def test_a_fix_writes_claudes_stream_and_its_errors_beside_it(self):
self.assertIsNone(fix_attempted(self.containers(), "/tree", "got 3, want 4", self.stream))
self.assertEqual((self.stream.read_bytes(), self.stream.with_suffix(".stderr").read_bytes()), (b"stream", b"errors"))
(args, seconds, stdin), = self.docker.calls
self.assertEqual((args[:6], seconds, stdin), (["run", "--rm", "--name", "bench-run", "--network", "bench"], 1800, None))
self.assertEqual(args[6:18], self.IN_TREE)
self.assertEqual(args[18:25], ["--env", "CLAUDE_CODE_OAUTH_TOKEN", "--env", "HTTPS_PROXY=http://bench-proxy:8080", "--env", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1", "bench:throwaway"])
def test_a_fix_that_runs_out_of_time_is_no_failure_and_its_container_goes(self):
self.assertIsNone(fix_attempted(self.containers(run=Failure("docker run still ran after 30 min")), "/tree", "got 3", self.stream))
self.assertEqual(self.docker.calls[1:], [self.REMOVED])
def test_a_fix_docker_could_not_start_is_a_failure(self):
broke = Failure("docker could not start: No such file or directory")
self.assertEqual(fix_attempted(self.containers(run=broke), "/tree", "got 3", self.stream), broke)
self.assertEqual(self.docker.calls[1:], [self.REMOVED])
def test_a_stream_that_cannot_be_written_is_a_failure(self):
found = fix_attempted(self.containers(), "/tree", "got 3", self.stream / "below" / "run.jsonl")
self.assertTrue(found.reason.startswith(f"could not write {self.stream}/below/run.jsonl: "), found.reason)
self.assertEqual(self.docker.calls, [])
class DockerCliTest(unittest.TestCase):
def setUp(self):
folder = tempfile.TemporaryDirectory()
self.addCleanup(folder.cleanup)
self.root = pathlib.Path(folder.name)
(self.root / "docker").write_text('#!/bin/sh\necho "$@ $SAID"\ncat\necho broke >&2\nexit 3\n')
(self.root / "docker").chmod(0o755)
self.docker = docker_cli({"PATH": f"{self.root}{os.pathsep}{os.environ['PATH']}", "SAID": "caf\xe9"})
def test_runs_docker_in_the_environment_with_the_text_on_its_input_and_returns_its_code_and_all_it_printed(self):
self.assertEqual(self.docker(["build", "-"], seconds=30, stdin="FROM x\n"), Ran(3, "build - caf\xe9\nFROM x\nbroke\n"))
def test_gives_docker_no_input_where_the_call_has_none(self):
self.assertEqual(self.docker(["ps"], seconds=30), Ran(3, "ps caf\xe9\nbroke\n"))
def test_writes_the_output_and_the_errors_to_the_files_the_call_gives(self):
with (self.root / "out").open("wb") as out, (self.root / "errors").open("wb") as errors:
self.assertEqual(self.docker(["ps"], seconds=30, stdout=out, stderr=errors), Ran(3, ""))
self.assertEqual(((self.root / "out").read_text(), (self.root / "errors").read_text()), ("ps caf\xe9\n", "broke\n"))
def test_output_that_is_no_utf_8_still_reads(self):
(self.root / "docker").write_text("#!/bin/sh\nprintf 'a\\377b'\n")
self.assertEqual(self.docker(["ps"], seconds=30), Ran(0, "a\ufffdb"))
def test_a_docker_that_is_not_there_is_a_failure(self):
self.assertEqual(docker_cli({"PATH": str(self.root / "none")})(["ps"], seconds=30), Failure("docker could not start: No such file or directory"))
def test_a_docker_that_runs_too_long_is_a_failure_that_names_the_command_and_the_minutes(self):
waited = []
def late(argv, timeout, **given):
waited.append(timeout)
raise subprocess.TimeoutExpired(argv, timeout)
self.assertEqual(docker_cli({}, late)(["run", "--rm"], seconds=90), Failure("docker run still ran after 2 min"))
self.assertEqual(waited, [90])
class CheckedTreeTest(unittest.TestCase):
def setUp(self):
folder = tempfile.TemporaryDirectory()
@@ -264,6 +446,16 @@ class CheckedTreeTest(unittest.TestCase):
checked_tree(planted, copy, str(self.root / "check"))
self.assertEqual(os.readlink(self.root / "check/host.go"), "/etc/hostname")
def test_a_link_never_stands_in_for_a_folder_of_the_planted_tree_and_a_linked_folder_is_copied_as_a_link(self):
planted = self.tree("planted", {"calc.go": "bug", "sub/old.go": "old", "gone/a.go": "a"})
copy = self.tree("copy", {"calc.go": "fixed"})
os.symlink("/etc", os.path.join(copy, "sub"))
os.symlink("/etc", os.path.join(copy, "etc"))
checked_tree(planted, copy, str(self.root / "check"))
check = self.root / "check"
self.assertEqual((check.joinpath("sub").is_symlink(), os.readlink(check / "etc")), (False, "/etc"))
self.assertEqual(self.held("check"), {"calc.go": "fixed"})
class ApiProxyTest(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self):