From ebbe926ab7db47faad6e08d82784bfc9159d187b Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sat, 21 Nov 2026 01:39:12 +0100 Subject: [PATCH] Test task-bench's docker calls, command line and planter where mutants survived --- bin/task_bench/figures.py | 4 +- bin/task_bench/main.py | 37 +++--- bin/task_bench/plant.py | 2 +- bin/task_bench/runs.py | 12 +- bin/task_bench/test_figures.py | 18 ++- bin/task_bench/test_plant.py | 74 ++++++++++- bin/task_bench/test_task_bench.py | 204 +++++++++++++++++++++++++++++- 7 files changed, 316 insertions(+), 35 deletions(-) diff --git a/bin/task_bench/figures.py b/bin/task_bench/figures.py index fbea4fdb..f9d955eb 100644 --- a/bin/task_bench/figures.py +++ b/bin/task_bench/figures.py @@ -3,10 +3,10 @@ import dataclasses import statistics import typing -from claude_code.entries import json_object, parse_entry, tool_calls, ToolCall -from claude_code.tokens import Tokens, tokens_of from failure import Failure +from claude_code.entries import json_object, parse_entry, tool_calls, ToolCall from reading_cost.measure import reads_of +from claude_code.tokens import Tokens, tokens_of # Where each run's container mounts its copy of the code, and its home folder: the paths in a run's stream start there. WORK = "/work" diff --git a/bin/task_bench/main.py b/bin/task_bench/main.py index db6c0926..7da856fc 100644 --- a/bin/task_bench/main.py +++ b/bin/task_bench/main.py @@ -52,18 +52,17 @@ import tarfile import typing import uuid -from claude_code.tokens import Tokens -from failure import Failure -from git_cli import git_repo -from git_cli.git_repo import default_ref, Git, top -from larv_review.history_commands import read_places +from task_bench.runs import checked_tree, closed, Containers, docker_cli, fix_attempted, MEMORY_MIB, opened, ran_tests, TOKEN from machine_load.slots import CPUS_PER_SLOT -from mutate_changes.tool_process import failed, Ran, run_tool -from review_rules.review_folder import made_review_folder -from run_heavy.capped_docker import SHARE_VARIABLE -from task_bench.figures import Figures, figures, Result, SIDES, table, WORK +from git_cli.git_repo import default_ref, Git, top from task_bench.plant import drawn, killed_mutants, line_at_head, Mutant, mutated, place_files, task_text -from task_bench.runs import checked_tree, closed, Containers, docker_cli, fix_attempted, MEMORY_MIB, opened, tests_run, TOKEN +from mutate_changes.tool_process import failed, Ran, run_tool +from failure import Failure +from task_bench.figures import figures, Result, SIDES, table, WORK +from git_cli import git_repo +from review_rules.review_folder import made_review_folder +from larv_review.history_commands import read_places +from run_heavy.capped_docker import SHARE_VARIABLE USAGE = "usage: task-bench --image --test [--base ] [--head ] [--prefix ] [--seed ] [--bugs ] [--runs ]" # A panel of three readers, seated as the README's audience, picked these options on 2026-11-21, each 3 to 0. @@ -85,8 +84,9 @@ class Task: return f"{self.mutant.path}:{self.mutant.line}:{self.mutant.column} {self.mutant.mutator}" -def main(argv: list[str], env: typing.Mapping[str, str]) -> int: - """Runs task-bench with the arguments argv; git, docker and Gremlins run in the environment env.""" +def main(argv: list[str], env: typing.Mapping[str, str], read: typing.Callable[[pathlib.Path], str | Failure] = read_places) -> int: + """Runs task-bench with the arguments argv; git, docker and Gremlins run in the environment env, and `read` reads what + `larv-review places` prints for the checkout.""" if argv in (["-h"], ["--help"]): print(__doc__.strip()) return 0 @@ -95,7 +95,7 @@ def main(argv: list[str], env: typing.Mapping[str, str]) -> int: print(f"{given.reason}; {USAGE}", file=sys.stderr) return 1 home = pathlib.Path(env.get("HOME") or pathlib.Path.home()) - places = read_places(home) + places = read(home) report = places if isinstance(places, Failure) else bench(given, env, places, home) if isinstance(report, Failure): print(report.reason, file=sys.stderr) @@ -269,7 +269,7 @@ def tests_in_copy(containers: Containers, tree: str, folder: pathlib.Path) -> Ra copy = folder / "tested" shutil.copytree(tree, copy, symlinks=True) try: - return tests_run(containers, str(copy)) + return ran_tests(containers, str(copy)) finally: shutil.rmtree(copy, ignore_errors=True) @@ -289,7 +289,7 @@ def fixed(task: Task, side: str, number: int, containers: Containers, folder: pa if isinstance(found, Failure): return found checked_tree(task.trees[side], str(copy), str(check)) - ran = tests_run(containers, str(check)) + ran = ran_tests(containers, str(check)) return Result(task.number, side, number, not isinstance(ran, Failure) and ran.code == 0, found) finally: shutil.rmtree(copy, ignore_errors=True) @@ -298,9 +298,10 @@ def fixed(task: Task, side: str, number: int, containers: Containers, folder: pa def recorded(result: Result) -> dict: """A run as results.json holds it.""" - found = result.figures or Figures("", "", Tokens(), 0, 0, None) - measured = {"claude_code": found.version, "model": found.model, "tokens": dataclasses.asdict(found.tokens), "turns": found.turns, "seconds": found.seconds, "reads": found.reads} - return {"bug": result.task, "side": result.side, "run": result.run, "passed": result.passed, "finished": result.figures is not None, **(measured if result.figures else {})} + run = {"bug": result.task, "side": result.side, "run": result.run, "passed": result.passed, "finished": result.figures is not None} + if not (found := result.figures): + return run + return {**run, "claude_code": found.version, "model": found.model, "tokens": dataclasses.asdict(found.tokens), "turns": found.turns, "seconds": found.seconds, "reads": found.reads} def counted(count: int, noun: str) -> str: diff --git a/bin/task_bench/plant.py b/bin/task_bench/plant.py index a561171a..dcf50ddf 100644 --- a/bin/task_bench/plant.py +++ b/bin/task_bench/plant.py @@ -6,8 +6,8 @@ import re import typing from failure import Failure -from mutate_changes.changed_lines import is_test from mutate_changes.in_diff_tools import gremlins_mutants +from mutate_changes.changed_lines import is_test from mutate_changes.tool_process import Run # A place as `larv-review places` prints it: `:` in backticks. diff --git a/bin/task_bench/runs.py b/bin/task_bench/runs.py index ba5c02a6..a35f1625 100644 --- a/bin/task_bench/runs.py +++ b/bin/task_bench/runs.py @@ -8,11 +8,11 @@ import shutil import subprocess import typing -from failure import Failure -from mutate_changes.changed_lines import is_test from mutate_changes.tool_process import failed, Ran -from task_bench.api_proxy import PORT +from failure import Failure from task_bench.figures import HOME, WORK +from mutate_changes.changed_lines import is_test +from task_bench.api_proxy import PORT MODEL = "claude-haiku-5-5" # No MCP server, skill, command or stored session, so a run starts from the code alone; -p needs --verbose with stream-json, and no @@ -41,10 +41,10 @@ class Docker(typing.Protocol): def __call__(self, args: list[str], *, seconds: float, stdin: str | None = None, stdout: typing.IO[bytes] | None = None, stderr: typing.IO[bytes] | None = None) -> Ran | Failure: ... -def docker_cli(env: typing.Mapping[str, str]) -> Docker: +def docker_cli(env: typing.Mapping[str, str], run: typing.Callable[..., subprocess.CompletedProcess[bytes]] = subprocess.run) -> Docker: def docker(args: list[str], *, seconds: float, stdin: str | None = None, stdout: typing.IO[bytes] | None = None, stderr: typing.IO[bytes] | None = None) -> Ran | Failure: try: - ran = subprocess.run(["docker", *args], env=env, input=None if stdin is None else stdin.encode(), stdin=subprocess.DEVNULL if stdin is None else None, stdout=stdout or subprocess.PIPE, stderr=stderr or subprocess.STDOUT, timeout=seconds, check=False) + ran = run(["docker", *args], env=env, input=None if stdin is None else stdin.encode(), stdin=subprocess.DEVNULL if stdin is None else None, stdout=stdout or subprocess.PIPE, stderr=stderr or subprocess.STDOUT, timeout=seconds, check=False) except OSError as failure: return Failure(f"docker could not start: {failure.strerror}") except subprocess.TimeoutExpired: @@ -105,7 +105,7 @@ def closed(containers: Containers) -> None: containers.docker(args, seconds=120) -def tests_run(containers: Containers, tree: str) -> Ran | Failure: +def ran_tests(containers: Containers, tree: str) -> Ran | Failure: """The project's test command, run at the top of tree with no network; the tests pass where it exits 0. A Failure where it still ran after RUN_SECONDS.""" ran = containers.docker(["run", "--rm", "--name", containers.running, "--network", "none", *in_tree(containers, tree), containers.image, "sh", "-c", containers.test], seconds=RUN_SECONDS) diff --git a/bin/task_bench/test_figures.py b/bin/task_bench/test_figures.py index 757e3667..770dbc33 100644 --- a/bin/task_bench/test_figures.py +++ b/bin/task_bench/test_figures.py @@ -6,9 +6,9 @@ import unittest sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) -from claude_code.tokens import Tokens from failure import Failure from task_bench.figures import Figures, figures, Result, table +from claude_code.tokens import Tokens INIT = {"type": "system", "subtype": "init", "cwd": "/work", "model": "claude-haiku-5-5", "claude_code_version": "2.1.330"} USAGE = {"input_tokens": 40, "cache_creation_input_tokens": 9000, "cache_read_input_tokens": 52000, "output_tokens": 800, "cache_creation": {"ephemeral_1h_input_tokens": 9000, "ephemeral_5m_input_tokens": 0}} @@ -53,6 +53,18 @@ class FiguresTest(unittest.TestCase): self.assertIsNone(figures(stream(INIT, call("Read", file_path="/work/calc.go")), "/work/calc.go")) self.assertIsNone(figures(b"", "/work/calc.go")) + def test_a_result_without_turns_or_a_duration_counts_none(self): + found = figures(stream(INIT, {"type": "result", "usage": USAGE}), "/work/calc.go") + self.assertEqual((found.turns, found.seconds), (0, 0)) + + def test_a_stream_without_an_init_names_no_version_or_model(self): + found = figures(stream(RESULT), "/work/calc.go") + self.assertEqual((found.version, found.model), ("", "")) + + def test_a_shell_command_that_reads_under_the_runs_home_reads_no_file_of_the_code(self): + found = figures(stream(INIT, call("Bash", command="cat ~/calc.go"), call("Read", file_path="/work/calc.go"), RESULT), "/work/calc.go") + self.assertEqual(found.reads, 1) + def test_a_run_the_api_refused_is_a_failure_that_quotes_claude(self): refused = {**RESULT, "is_error": True, "api_error_status": 401, "result": "Failed to authenticate. API Error: 401 OAuth access token is invalid."} self.assertEqual(figures(stream(INIT, refused), "/work/calc.go"), Failure("claude could not use the API: Failed to authenticate. API Error: 401 OAuth access token is invalid.")) @@ -86,6 +98,10 @@ class TableTest(unittest.TestCase): "| All | change | +2 | -50 % | 0 % | 0 % | 0 % | -3.5 | -50 % | -2.5 |", ]) + def test_the_change_is_a_dash_where_the_base_measured_zero_and_signed_where_it_rose(self): + results = [run(1, "base", 1, True, input_=0, turns=10, seconds=60, reads=0), run(1, "head", 1, False, input_=50, turns=12, seconds=90, reads=2)] + self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[-1], "| All | change | -1 | - | 0 % | 0 % | 0 % | +2 | +50 % | +2 |") + def test_a_run_without_figures_counts_as_not_passed_and_a_side_without_any_shows_a_dash(self): results = [Result(1, "base", 1, False, None), run(1, "head", 1, True)] self.assertEqual(table(results, {1: "calc.go:4:7 ARITHMETIC_BASE"})[2:], [ diff --git a/bin/task_bench/test_plant.py b/bin/task_bench/test_plant.py index 059876b2..6c3b8eae 100644 --- a/bin/task_bench/test_plant.py +++ b/bin/task_bench/test_plant.py @@ -1,11 +1,15 @@ """task-bench's planter: which files it mutates, which mutants it draws, how it re-applies one and what it tells Haiku.""" +import json import pathlib import sys +import tempfile import unittest sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) -from task_bench.plant import drawn, line_at_head, Mutant, mutated, place_files, task_text +from task_bench.plant import drawn, go_files, killed_mutants, line_at_head, Mutant, mutated, place_files, task_text +from failure import Failure +from mutate_changes.tool_process import Ran PLACES = """## 2026-11-19T22:33:15Z, PR #646 at a357cc7b @@ -128,3 +132,71 @@ class LineAtHeadTest(unittest.TestCase): def test_no_diff_keeps_every_line(self): self.assertEqual(line_at_head("", 12), 12) + + +class KilledMutantsTest(unittest.TestCase): + def setUp(self): + folder = tempfile.TemporaryDirectory() + self.addCleanup(folder.cleanup) + self.top = pathlib.Path(folder.name) + for path in ("calc.go", "calc_test.go", "render/pick.go", "render/a+b.go", "render/notes.md", "cli/main.go"): + (self.top / path).parent.mkdir(parents=True, exist_ok=True) + (self.top / path).write_text("package x\n") + self.ran: list[tuple[list[str], str]] = [] + + def gremlins(self, report: dict | None, code: int = 0): + def run(argv: list[str], cwd: str) -> Ran: + self.ran.append((argv, cwd)) + if report is not None: + pathlib.Path(cwd, argv[argv.index("--output") + 1]).write_text(json.dumps(report)) + return Ran(code, "gremlins said so") + + return run + + def test_names_each_go_file_gremlins_would_mutate_by_its_path_from_the_top(self): + self.assertEqual(sorted(go_files(str(self.top))), ["calc.go", "cli/main.go", "render/a+b.go", "render/pick.go"]) + + def test_has_gremlins_leave_out_every_other_go_file_by_its_whole_path(self): + killed_mutants(str(self.top), ["render/pick.go"], self.gremlins({"files": []}), 3) + argv, cwd = self.ran[0] + self.assertEqual(cwd, str(self.top)) + self.assertEqual(argv[:8], ["gremlins", "unleash", "--exclude-files", r"^calc\.go$", "--exclude-files", r"^cli/main\.go$", "--exclude-files", r"^render/a\+b\.go$"]) + self.assertEqual((argv[8], argv[10:]), ("--output", ["--workers", "3"])) + + def test_keeps_the_killed_mutants_of_the_files_asked_for(self): + report = {"files": [ + {"file_name": "render/pick.go", "mutations": [{"line": 4, "column": 7, "status": "KILLED", "type": "ARITHMETIC_BASE"}, {"line": 5, "column": 2, "status": "LIVED", "type": "INVERT_LOGICAL"}, {"line": 6, "column": 2, "status": "NOT COVERED", "type": "INVERT_LOGICAL"}]}, + {"file_name": "calc.go", "mutations": [{"line": 1, "column": 1, "status": "KILLED", "type": "ARITHMETIC_BASE"}]}, + ]} + self.assertEqual(killed_mutants(str(self.top), ["render/pick.go"], self.gremlins(report), 3), [Mutant("render/pick.go", 4, 7, "ARITHMETIC_BASE")]) + + def test_a_gremlins_that_fails_is_a_failure_with_what_it_printed(self): + found = killed_mutants(str(self.top), ["render/pick.go"], self.gremlins(None, code=1), 3) + self.assertIsInstance(found, Failure) + self.assertIn("gremlins said so", found.reason) + + +class EdgesTest(unittest.TestCase): + def test_mutates_the_last_line_of_a_file(self): + self.assertEqual(mutated(b"a\nb + c", Mutant("calc.go", 2, 3, "ARITHMETIC_BASE")), b"a\nb - c") + self.assertIsNone(mutated(b"a\nb + c", Mutant("calc.go", 3, 3, "ARITHMETIC_BASE"))) + self.assertIsNone(mutated(b"a\nb + c", Mutant("calc.go", 0, 3, "ARITHMETIC_BASE"))) + + def test_takes_the_longest_token_at_the_column(self): + self.assertEqual(mutated(b"a >= b", Mutant("calc.go", 1, 3, "CONDITIONALS_BOUNDARY")), b"a > b") + self.assertEqual(mutated(b"a &^ b", Mutant("calc.go", 1, 3, "INVERT_BITWISE")), b"a & b") + + def test_a_message_ends_at_the_next_line_indented_as_its_first(self): + output = "--- FAIL: TestPick/empty_input (0.00s)\n pick_test.go:42: got 3\n want 4\n pick_test.go:50: got 5\n" + self.assertEqual(task_text(output), "got 3\nwant 4") + + def test_a_message_runs_to_the_end_of_the_output(self): + self.assertEqual(task_text("--- FAIL: TestPick (0.00s)\n pick_test.go:42: got 3\n want 4"), "got 3\nwant 4") + + def test_a_message_may_name_a_part_of_a_subtests_name_that_is_no_test(self): + output = "--- FAIL: TestPick/empty (0.00s)\n pick_test.go:42: empty input: got 3\n in TestPick\n in TestPick/empty\n" + self.assertEqual(task_text(output), "empty input: got 3") + + +if __name__ == "__main__": + unittest.main() diff --git a/bin/task_bench/test_task_bench.py b/bin/task_bench/test_task_bench.py index 91ffd05b..47d339ed 100644 --- a/bin/task_bench/test_task_bench.py +++ b/bin/task_bench/test_task_bench.py @@ -14,10 +14,12 @@ import unittest sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) +from task_bench import api_proxy, main, runs +from task_bench.runs import checked_tree, closed, Containers, docker_cli, fix_attempted, opened, ran_tests from failure import Failure -from task_bench import api_proxy, main -from task_bench.runs import checked_tree from test_env import isolated_env +from mutate_changes.tool_process import Ran +from task_bench.figures import Result BASE_CODE = "package calc\n\nfunc Add(a, b int) int {\n\treturn a + b\n}\n\nfunc Twice(a int) int {\n\treturn a * 2\n}\n" # The head moves Add down a line and rewrites Twice's line. @@ -30,11 +32,13 @@ TESTS = textwrap.dedent("""\ """) # Reports `+` in Add, which the head keeps, and the `*` in Twice, which the head rewrites, as killed; and a mutant no test caught. GREMLINS = textwrap.dedent("""\ - import json, sys + import json, os, sys lines = open("calc.go").read().split("\\n") found = [{"line": at, "column": 11, "status": "KILLED", "type": "ARITHMETIC_BASE"} for at, line in enumerate(lines, 1) if " + " in line or " * " in line] report = {"files": [{"file_name": "calc.go", "mutations": [*found, {**found[0], "status": "LIVED", "type": "INVERT_NEGATIVES"}]}]} json.dump(report, open(sys.argv[sys.argv.index("--output") + 1], "w")) + if os.environ.get("FAKE_GREMLINS_LOG"): + json.dump([os.getcwd(), *sys.argv[1:]], open(os.environ["FAKE_GREMLINS_LOG"], "w")) """) # Logs each call. A `run` of the bench's image runs its command in the mounted tree: sh as it is, and claude as the fake below. DOCKER = textwrap.dedent("""\ @@ -44,7 +48,9 @@ DOCKER = textwrap.dedent("""\ with open(os.environ["FAKE_LOG"], "a") as log: log.write(json.dumps(args) + "\\n") if args[0] == "build": - sys.stdin.read() + held = sys.stdin.read() + if os.environ.get("FAKE_STDIN"): + open(os.environ["FAKE_STDIN"], "w").write(held) if args[0] != "run" or "--detach" in args: sys.exit(int(os.environ.get("FAKE_FAILS") == args[0])) tree = args[args.index("--volume") + 1].rpartition(":")[0] @@ -94,11 +100,15 @@ class BenchTest(unittest.TestCase): self.git("init", "--quiet", "--initial-branch", "main") for name, text in (("calc.go", BASE_CODE), ("calc_test.go", TESTS), ("README.md", "# Calc\n")): (self.repo / name).write_text(text) + # A link to nothing, which a copy that followed links would fail on. + os.symlink("missing", self.repo / "dangling") self.base = self.commit("Base") self.git("checkout", "--quiet", "-b", "item-1") (self.repo / "calc.go").write_text(HEAD_CODE) self.head = self.commit("Head") - self.enterContext(contextlib.chdir(self.repo)) + # The bench runs from a folder below the checkout's top. + (self.repo / "below").mkdir() + self.enterContext(contextlib.chdir(self.repo / "below")) def git(self, *args: str) -> str: return subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@localhost", "-c", "commit.gpgsign=false", *args], cwd=self.repo, env=self.env, check=True, capture_output=True, text=True).stdout.strip() @@ -137,7 +147,58 @@ class BenchTest(unittest.TestCase): self.assertEqual((results["sides"], results["bugs"], len(results["runs"])), ({"base": self.base, "head": self.head}, {"1": "calc.go:4:11 ARITHMETIC_BASE"}, 4)) self.assertEqual(results["runs"][0], {"bug": 1, "side": "base", "run": 1, "passed": False, "finished": True, "claude_code": "2.1.330", "model": "claude-haiku-5-5", "tokens": {"input": 40, "write_5m": 9000, "write_1h": 0, "read": 52000, "output": 800}, "turns": 3, "seconds": 61.5, "reads": 1}) self.assertEqual(sorted(path.name for path in (folder / "runs").iterdir() if path.suffix == ".jsonl"), ["bug-1-base-1.jsonl", "bug-1-base-2.jsonl", "bug-1-head-1.jsonl", "bug-1-head-2.jsonl"]) - self.assertIn("run 4 of 4, bug 1 at the head: passed.", self.progress.getvalue()) + self.assertEqual(results["seed"], 1) + self.assertEqual(self.progress.getvalue().splitlines(), [ + f"task-bench: base {self.base[:12]}, head {self.head[:12]}.", + "task-bench: run 1 of 4, bug 1 at the base: not passed.", + "task-bench: run 2 of 4, bug 1 at the head: passed.", + "task-bench: run 3 of 4, bug 1 at the base: not passed.", + "task-bench: run 4 of 4, bug 1 at the head: passed.", + ]) + self.assertEqual(sorted(path.name for path in folder.iterdir()), ["bugs", "results.json", "runs", "trees"]) + self.assertEqual((folder / "runs/bug-1-base-1.stderr").read_bytes(), b"") + self.assertEqual(os.readlink(folder / "bugs/1/head/dangling"), "missing") + + def test_the_command_line_runs_the_bench_on_the_seats_places_and_prints_its_report(self): + argv = [part for name, value in self.given.items() for part in (name, value)] + homes = [] + with contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(main.main(argv, self.env, lambda home: homes.append(home) or PLACES), 0) + self.assertEqual(homes, [self.home]) + # Without --base, the base is the merge base with the default branch. + self.assertEqual(out.getvalue().splitlines()[0], f"task-bench: base {self.base[:12]}, head {self.head[:12]}. 1 bug, fixed 2 times at each side by claude-haiku-5-5 in Claude Code 2.1.330.") + self.assertEqual(out.getvalue().splitlines()[3], "| Task | Side | Passed | Input | Cache write | Cache read | Output | Turns | Wall, s | Reads |") + self.assertTrue(out.getvalue().endswith("task-bench\n"), out.getvalue()[-80:]) + + def test_the_command_line_fails_with_the_reason_where_the_places_or_the_bench_fail(self): + argv = [part for name, value in self.given.items() for part in (name, value)] + for places, said in ((Failure("no history"), "no history\n"), ("- `gone.go:1` `Gone`: removed\n", "The places the seats found hard name no Go file that the base holds, so there is nowhere to plant a bug.\n")): + with contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()) as errors: + self.assertEqual(main.main(argv, self.env, lambda home: places), 1) + self.assertEqual((out.getvalue(), errors.getvalue().splitlines(keepends=True)[-1]), ("", said)) + + def test_the_head_is_the_commit_asked_for(self): + report = self.bench(**{"--head": "main", "--runs": "1"}) + self.assertEqual(report[0], f"task-bench: base {self.base[:12]}, head {self.base[:12]}. 1 bug, fixed 1 time at each side by claude-haiku-5-5 in Claude Code 2.1.330.") + + def test_fails_on_a_head_or_a_base_git_does_not_know(self): + for name in ("--head", "--base"): + found = self.bench(**{name: "no-such-ref"}) + self.assertIsInstance(found, Failure) + self.assertIn("no-such-ref", found.reason) + + def test_the_image_is_built_from_the_one_asked_for_with_the_caps_memory(self): + self.env["FAKE_STDIN"] = str(self.root / "dockerfile") + self.env.pop("RUN_HEAVY_MEMORY") + self.bench(**{"--runs": "1"}) + self.assertEqual((self.root / "dockerfile").read_text().splitlines()[0], "FROM golang:1.27.4") + self.assertEqual(self.calls()[0][1:6], ["--quiet", "--memory", "4096m", "--memory-swap", "4096m"]) + + def test_gremlins_runs_through_the_prefix_in_the_copy_of_the_base_with_a_slots_workers(self): + self.env["FAKE_GREMLINS_LOG"] = str(self.root / "gremlins.log") + folder = pathlib.Path(self.bench(**{"--runs": "1"})[-1].removeprefix("Each run's result and stream: ")) + cwd, *argv = json.loads((self.root / "gremlins.log").read_text()) + self.assertEqual((cwd, argv[:2], argv[-2:]), (str(folder / "trees/base"), ["gremlins", "unleash"], ["--workers", str(main.CPUS_PER_SLOT)])) def test_each_side_gets_its_own_tree_with_the_bug_committed_once_and_no_trace_of_the_checkout(self): folder = pathlib.Path(self.bench()[-1].removeprefix("Each run's result and stream: ")) @@ -235,6 +296,127 @@ class MainTest(unittest.TestCase): self.assertIn("usage: task-bench --image --test ", out.getvalue()) + def test_records_a_run_that_did_not_finish_without_figures(self): + self.assertEqual(main.recorded(Result(1, "base", 2, False, None)), {"bug": 1, "side": "base", "run": 2, "passed": False, "finished": False}) + + +class Recorded: + """A docker that records each call and answers by the call's first word, else that it ran.""" + + def __init__(self, **answers: Ran | Failure): + self.calls: list[tuple[list[str], float, str | None]] = [] + self.answers = answers + + def __call__(self, args, *, seconds, stdin=None, stdout=None, stderr=None): + self.calls.append((args, seconds, stdin)) + if stdout: + stdout.write(b"stream") + stderr.write(b"errors") + return self.answers.get(args[0], Ran(0, "ran")) + + +class ContainersTest(unittest.TestCase): + IN_TREE = ["--memory", "512m", "--memory-swap", "512m", "--user", "1000:1000", "--env", "HOME=/tmp", "--volume", "/tree:/work", "--workdir", "/work"] + REMOVED = (["rm", "--force", "bench-run"], 120, None) + + def setUp(self): + folder = tempfile.TemporaryDirectory() + self.addCleanup(folder.cleanup) + self.stream = pathlib.Path(folder.name, "run.jsonl") + + def containers(self, **answers: Ran | Failure) -> Containers: + self.docker = Recorded(**answers) + return Containers(self.docker, "bench", "go test ./...", 512, "1000:1000") + + def test_opens_the_image_the_network_without_a_route_out_and_the_proxy_on_both_networks(self): + self.assertIsNone(opened(self.containers(), "golang:1.27.4")) + proxy = str(pathlib.Path(runs.__file__).with_name("api_proxy.py")) + self.assertEqual(self.docker.calls, [ + (["build", "--quiet", "--memory", "512m", "--memory-swap", "512m", "--tag", "bench:throwaway", "-"], 1200, "FROM golang:1.27.4\nUSER root\nRUN curl -fsSL https://claude.ai/install.sh | bash && cp -L /root/.local/bin/claude /usr/local/bin/claude\n"), + (["network", "create", "--internal", "bench"], 60, None), + (["run", "--detach", "--rm", "--name", "bench-proxy", "--memory", "256m", "--memory-swap", "256m", "--volume", f"{proxy}:/api_proxy.py:ro", "python:3.15.2-slim", "python", "/api_proxy.py", "api.anthropic.com:443"], 300, None), + (["network", "connect", "bench", "bench-proxy"], 60, None), + ]) + + def test_opening_stops_at_the_first_step_that_fails_and_says_which(self): + found = opened(self.containers(network=Ran(1, "pool overlaps")), "golang:1.27.4") + self.assertEqual((found.reason.startswith("docker network"), "pool overlaps" in found.reason, len(self.docker.calls)), (True, True, 2)) + self.assertEqual(opened(self.containers(build=Failure("docker could not start")), "golang:1.27.4"), Failure("docker could not start")) + + def test_closing_removes_the_running_container_the_proxy_the_network_and_the_image(self): + closed(self.containers()) + self.assertEqual(self.docker.calls, [self.REMOVED, (["rm", "--force", "bench-proxy"], 120, None), (["network", "rm", "bench"], 120, None), (["image", "rm", "bench:throwaway"], 120, None)]) + + def test_the_tests_run_in_the_tree_without_a_network_for_at_most_half_an_hour(self): + self.assertEqual(ran_tests(self.containers(), "/tree"), Ran(0, "ran")) + self.assertEqual(self.docker.calls, [(["run", "--rm", "--name", "bench-run", "--network", "none", *self.IN_TREE, "bench:throwaway", "sh", "-c", "go test ./..."], 1800, None)]) + + def test_tests_that_run_out_of_time_are_a_failure_and_their_container_goes(self): + late = Failure("docker run still ran after 30 min") + self.assertEqual(ran_tests(self.containers(run=late), "/tree"), late) + self.assertEqual(self.docker.calls[1:], [self.REMOVED]) + + def test_a_fix_writes_claudes_stream_and_its_errors_beside_it(self): + self.assertIsNone(fix_attempted(self.containers(), "/tree", "got 3, want 4", self.stream)) + self.assertEqual((self.stream.read_bytes(), self.stream.with_suffix(".stderr").read_bytes()), (b"stream", b"errors")) + (args, seconds, stdin), = self.docker.calls + self.assertEqual((args[:6], seconds, stdin), (["run", "--rm", "--name", "bench-run", "--network", "bench"], 1800, None)) + self.assertEqual(args[6:18], self.IN_TREE) + self.assertEqual(args[18:25], ["--env", "CLAUDE_CODE_OAUTH_TOKEN", "--env", "HTTPS_PROXY=http://bench-proxy:8080", "--env", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1", "bench:throwaway"]) + + def test_a_fix_that_runs_out_of_time_is_no_failure_and_its_container_goes(self): + self.assertIsNone(fix_attempted(self.containers(run=Failure("docker run still ran after 30 min")), "/tree", "got 3", self.stream)) + self.assertEqual(self.docker.calls[1:], [self.REMOVED]) + + def test_a_fix_docker_could_not_start_is_a_failure(self): + broke = Failure("docker could not start: No such file or directory") + self.assertEqual(fix_attempted(self.containers(run=broke), "/tree", "got 3", self.stream), broke) + self.assertEqual(self.docker.calls[1:], [self.REMOVED]) + + def test_a_stream_that_cannot_be_written_is_a_failure(self): + found = fix_attempted(self.containers(), "/tree", "got 3", self.stream / "below" / "run.jsonl") + self.assertTrue(found.reason.startswith(f"could not write {self.stream}/below/run.jsonl: "), found.reason) + self.assertEqual(self.docker.calls, []) + + +class DockerCliTest(unittest.TestCase): + def setUp(self): + folder = tempfile.TemporaryDirectory() + self.addCleanup(folder.cleanup) + self.root = pathlib.Path(folder.name) + (self.root / "docker").write_text('#!/bin/sh\necho "$@ $SAID"\ncat\necho broke >&2\nexit 3\n') + (self.root / "docker").chmod(0o755) + self.docker = docker_cli({"PATH": f"{self.root}{os.pathsep}{os.environ['PATH']}", "SAID": "caf\xe9"}) + + def test_runs_docker_in_the_environment_with_the_text_on_its_input_and_returns_its_code_and_all_it_printed(self): + self.assertEqual(self.docker(["build", "-"], seconds=30, stdin="FROM x\n"), Ran(3, "build - caf\xe9\nFROM x\nbroke\n")) + + def test_gives_docker_no_input_where_the_call_has_none(self): + self.assertEqual(self.docker(["ps"], seconds=30), Ran(3, "ps caf\xe9\nbroke\n")) + + def test_writes_the_output_and_the_errors_to_the_files_the_call_gives(self): + with (self.root / "out").open("wb") as out, (self.root / "errors").open("wb") as errors: + self.assertEqual(self.docker(["ps"], seconds=30, stdout=out, stderr=errors), Ran(3, "")) + self.assertEqual(((self.root / "out").read_text(), (self.root / "errors").read_text()), ("ps caf\xe9\n", "broke\n")) + + def test_output_that_is_no_utf_8_still_reads(self): + (self.root / "docker").write_text("#!/bin/sh\nprintf 'a\\377b'\n") + self.assertEqual(self.docker(["ps"], seconds=30), Ran(0, "a\ufffdb")) + + def test_a_docker_that_is_not_there_is_a_failure(self): + self.assertEqual(docker_cli({"PATH": str(self.root / "none")})(["ps"], seconds=30), Failure("docker could not start: No such file or directory")) + + def test_a_docker_that_runs_too_long_is_a_failure_that_names_the_command_and_the_minutes(self): + waited = [] + + def late(argv, timeout, **given): + waited.append(timeout) + raise subprocess.TimeoutExpired(argv, timeout) + + self.assertEqual(docker_cli({}, late)(["run", "--rm"], seconds=90), Failure("docker run still ran after 2 min")) + self.assertEqual(waited, [90]) + + class CheckedTreeTest(unittest.TestCase): def setUp(self): folder = tempfile.TemporaryDirectory() @@ -264,6 +446,16 @@ class CheckedTreeTest(unittest.TestCase): checked_tree(planted, copy, str(self.root / "check")) self.assertEqual(os.readlink(self.root / "check/host.go"), "/etc/hostname") + def test_a_link_never_stands_in_for_a_folder_of_the_planted_tree_and_a_linked_folder_is_copied_as_a_link(self): + planted = self.tree("planted", {"calc.go": "bug", "sub/old.go": "old", "gone/a.go": "a"}) + copy = self.tree("copy", {"calc.go": "fixed"}) + os.symlink("/etc", os.path.join(copy, "sub")) + os.symlink("/etc", os.path.join(copy, "etc")) + checked_tree(planted, copy, str(self.root / "check")) + check = self.root / "check" + self.assertEqual((check.joinpath("sub").is_symlink(), os.readlink(check / "etc")), (False, "/etc")) + self.assertEqual(self.held("check"), {"calc.go": "fixed"}) + class ApiProxyTest(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self):