Answer stability round 1: 1 🔴, 2 🟡 (0 misread); pin the git diff settings Gremlins parses, show the mutant total, and a container prefix that runs as root

This commit is contained in:
2026-11-21 00:02:27 +01:00
parent 89f5fcbc16
commit 846d6c6e36
4 changed files with 87 additions and 59 deletions
+36 -21
View File
@@ -1,4 +1,4 @@
"""Mutate a branch's changed lines with cargo-mutants, Gremlins, Stryker and Infection, each of which takes the changed lines itself."""
"""Mutate a branch's changed lines with cargo-mutants, Gremlins, Infection and Stryker, each of which takes the changed lines itself."""
import json
import os
import pathlib
@@ -12,8 +12,10 @@ from git_cli.git_repo import Git
# cargo-mutants' exit codes for a run that finished: every mutant caught, some missed, and some timed out.
CARGO_FINISHED = frozenset({0, 2, 3})
CARGO_OUTCOMES = ("caught", "missed", "timeout", "unviable")
# The statuses in Gremlins' report of a mutant that its tests don't kill.
GREMLINS_UNKILLED = frozenset({"LIVED", "NOT COVERED"})
# What Gremlins v0.6.0 needs of the diff it reads from git: one run of lines per hunk is all it mutates, so a hunk holds the changed
# lines and none around them; it takes each file's path after a b/; and it reads no colour codes. A user's git settings change each.
GREMLINS_GIT_SETTINGS = {"diff.context": "0", "diff.noprefix": "false", "diff.mnemonicPrefix": "false", "diff.srcPrefix": "a/", "diff.dstPrefix": "b/", "color.diff": "never"}
def run_cargo_mutants(top: str, base: str, paths: list[str], run: Run, git: Git, jobs: int) -> list[str] | Failure:
@@ -42,37 +44,50 @@ def run_cargo_mutants(top: str, base: str, paths: list[str], run: Run, git: Git,
return [f"cargo-mutants: {sum(map(len, outcomes.values()))} mutants on changed lines, {len(uncaught)} not caught.", *uncaught]
def run_gremlins(top: str, base: str, run: Run, jobs: int) -> list[str] | Failure:
"""A line with the number of mutants Gremlins reports as lived or not covered, then a line per mutant, as
`<path>:<line>:<column>: <status>: <mutator>`.
class Mutant(typing.NamedTuple):
"""A mutant in Gremlins' report, in the order the report prints them."""
Gremlins runs in top and mutates the Go lines the branch changes since base. It skips test files itself. Its git needs `run` to
hold the settings that `with_zero_diff_context` adds to an environment."""
path: str
line: int
column: int
status: str
mutator: str
def run_gremlins(top: str, base: str, run: Run, jobs: int) -> list[str] | Failure:
"""A line with the number of mutants on the changed lines and how many of them Gremlins reports as lived or not covered, then a
line per such mutant, as `<path>:<line>:<column>: <status>: <mutator>`.
Gremlins runs in top and mutates the Go lines the branch changes since base. It skips `*_test.go` itself. It reads the diff from
git, so `run` must hold the environment that `with_plain_git_diff` makes. Gremlins exits nonzero on a mutation score below a
threshold the project sets, which its exit code doesn't tell from a run that broke, so any nonzero exit fails the run."""
scratch = scratch_folder(top)
if isinstance(scratch, Failure):
return scratch
try:
report = os.path.join(os.path.relpath(scratch, top), "report.json")
ran = run(["gremlins", "unleash", "--diff", base, "--output", report, "--workers", str(jobs)], top)
report = pathlib.Path(scratch, "report.json")
ran = run(["gremlins", "unleash", "--diff", base, "--output", os.path.join(os.path.relpath(scratch, top), report.name), "--workers", str(jobs)], top)
if isinstance(ran, Failure) or ran.code:
return ran if isinstance(ran, Failure) else failed("gremlins", ran)
# Gremlins writes `null` for a list with no entry.
files = json.loads(pathlib.Path(scratch, "report.json").read_text(encoding="utf-8"))["files"] or []
unkilled = sorted((file["file_name"], mutant["line"], mutant["column"], mutant["status"], mutant["type"]) for file in files for mutant in file["mutations"] or [] if mutant["status"] in GREMLINS_UNKILLED)
return ran if isinstance(ran, Failure) else whole_output("Gremlins", ran)
# Gremlins writes no report where it finds no mutant, and `null` for a list with no entry.
files = (json.loads(report.read_text(encoding="utf-8"))["files"] or []) if report.is_file() else []
mutants = [Mutant(file["file_name"], mutant["line"], mutant["column"], mutant["status"], mutant["type"]) for file in files for mutant in file["mutations"] or []]
except (OSError, ValueError, KeyError, TypeError) as failure:
return Failure(f"could not read Gremlins' report in {scratch}: {failure}")
finally:
remove(scratch)
return [f"gremlins: {len(unkilled)} mutants not killed.", *(f"{path}:{line}:{column}: {status}: {mutator}" for path, line, column, status, mutator in unkilled)]
# Gremlins lists the mutants on the unchanged lines of a hunk too, as SKIPPED. It lists the files in no fixed order.
unkilled = sorted(mutant for mutant in mutants if mutant.status in GREMLINS_UNKILLED)
on_changed_lines = sum(mutant.status != "SKIPPED" for mutant in mutants)
return [f"gremlins: {on_changed_lines} mutants on changed lines, {len(unkilled)} not killed.", *(f"{m.path}:{m.line}:{m.column}: {m.status}: {m.mutator}" for m in unkilled)]
def with_zero_diff_context(env: typing.Mapping[str, str]) -> dict[str, str]:
"""env with git set, through its GIT_CONFIG_* variables, to print no context lines around a change, after any setting env holds.
Gremlins v0.6.0 mutates one run of lines per hunk, so a hunk must hold only the lines the branch changes."""
held = env.get("GIT_CONFIG_COUNT", "0")
count = int(held) if held.isdecimal() else 0
return {**env, "GIT_CONFIG_COUNT": str(count + 1), f"GIT_CONFIG_KEY_{count}": "diff.context", f"GIT_CONFIG_VALUE_{count}": "0"}
def with_plain_git_diff(env: typing.Mapping[str, str]) -> dict[str, str]:
"""env with the settings of GREMLINS_GIT_SETTINGS added to the ones it holds, as git reads them from its GIT_CONFIG_* variables."""
held = env.get("GIT_CONFIG_COUNT", "")
first = int(held) if held.isdecimal() else 0
added = {f"GIT_CONFIG_{part}_{first + index}": text for index, (key, value) in enumerate(GREMLINS_GIT_SETTINGS.items()) for part, text in (("KEY", key), ("VALUE", value))}
return {**env, **added, "GIT_CONFIG_COUNT": str(first + len(GREMLINS_GIT_SETTINGS))}
def outcome_lines(path: pathlib.Path) -> list[str]:
+17 -14
View File
@@ -1,10 +1,10 @@
"""Mutation-test the lines a branch adds or changes, and print each mutant its tests don't kill.
r"""Mutation-test the lines a branch adds or changes, and print each mutant its tests don't kill.
usage: mutate-changes [--base <ref>] [--prefix <shell text>] [--jobs <n>]
Run it in a checkout, under ~/.claude/bin/run-heavy. It diffs the working tree's tracked files against the merge base of HEAD and
--base, by default the default branch: origin's as last fetched, else main or master. It picks the tool by each changed file's
extension, and leaves out test files: test_*.py, *_test.py, conftest.py, *.test.* and *.spec.*, and any file under a tests/, test/
extension, and leaves out test files: test_*.py, *_test.py, *_test.go, conftest.py, *.test.* and *.spec.*, and any file under a tests/, test/
or __tests__/ folder.
- .py: mutmut 3, in a scratch copy of the nearest folder above the file that holds a setup.cfg or a pyproject.toml, else of the
@@ -16,8 +16,9 @@ or __tests__/ folder.
in its own process, so a mutant that only a child process reaches, such as a tool a test starts, shows as `no tests`. A
pyproject.toml with a [tool.mutmut] section fails the run. A crash inside mutmut, such as a KeyError in its worker pool,
becomes one line for its folder, naming the error and telling you to mutate that folder's changed lines by hand.
- .go: Gremlins, as `gremlins unleash --diff`, in the checkout's top. It skips *_test.go itself. Its git runs with diff.context=0,
set in GIT_CONFIG_* variables after any the environment holds, since Gremlins v0.6.0 mutates one run of lines per hunk.
- .go: Gremlins, as `gremlins unleash --diff`, in the checkout's top, where its go.mod must be. Gremlins reads the diff from git, so
the run adds settings to the GIT_CONFIG_* variables, after any the environment holds: no context lines, since Gremlins mutates
every line of a hunk; a/ and b/ as the file prefixes; and no colour.
- .rs: cargo-mutants, as `cargo mutants --in-diff`, in the checkout's top.
- .cjs, .cts, .js, .jsx, .mjs, .mts, .ts, .tsx: Stryker, as `npx --no --package=@stryker-mutator/core stryker run --mutate` on the
changed lines, in the nearest folder above the file that holds a package.json. A file with no package.json above it isn't
@@ -29,10 +30,12 @@ as "$@". For example, 'uvx --from "mutmut>=3,<4"'. Without --prefix, the command
run there, as 'docker compose -f "$(git rev-parse --show-toplevel)/compose.yaml" run --rm -v "$PWD:$PWD" -w "$PWD" app' does. "$1"
is the command's first word, mutmut, cargo, gremlins, npx or vendor/bin/infection, so a prefix can pick what to run per tool:
'case "$1" in mutmut) set -- uvx --from "mutmut>=3,<4" "$@";; esac;'.
Gremlins' git runs in the container too. In a git worktree, git keeps its data in the main checkout, outside $PWD, so the container
mounts that folder as well, and it passes on the GIT_CONFIG_* variables that set diff.context, as `-e GIT_CONFIG_COUNT -e
GIT_CONFIG_KEY_0 -e GIT_CONFIG_VALUE_0` where the environment holds no setting of its own, and one more KEY and VALUE for each it holds:
'common="$(git rev-parse --path-format=absolute --git-common-dir)"; docker run --rm -v "$PWD:$PWD" -v "$common:$common" -w "$PWD" -e GIT_CONFIG_COUNT -e GIT_CONFIG_KEY_0 -e GIT_CONFIG_VALUE_0 <image> "$@"'.
Gremlins' git runs in the container too, so a Go prefix mounts git's data and passes on the GIT_CONFIG_* variables, and allows git
to read a checkout that the container's root user doesn't own. In a git worktree, git's data is in the main checkout, outside $PWD:
common="$(git rev-parse --path-format=absolute --git-common-dir)"
docker run --rm -v "$PWD:$PWD" -v "$common:$common" -w "$PWD" $(printenv | sed -n 's/^\(GIT_CONFIG_[A-Z0-9_]*\)=.*/-e \1/p') \
<image holding gremlins> sh -c 'git config --global --add safe.directory "*" && exec "$@"' sh
--jobs sets how many tests each tool runs at once, 4 by default: one heavy slot's CPUs. Under run-heavy, a container that sets
neither memory limit gets the slot's share, in MiB in RUN_HEAVY_MEMORY; set --jobs so that that many test runs fit in it.
@@ -40,7 +43,7 @@ neither memory limit gets the slot's share, in MiB in RUN_HEAVY_MEMORY; set --jo
It prints:
- for mutmut, per folder, a line with the mutant count, then a line per mutant on a changed line that is not killed, as
`<path>:<line>: <status>: <the line as mutated>`, or `<path>: <status>: <name>, on a line mutate-changes cannot find`;
- for Gremlins, a line with the number of mutants not killed, then a line per mutant that lived or is not covered, as
- for Gremlins, a line with the mutant count and how many it did not kill, then a line per mutant that lived or is not covered, as
`<path>:<line>:<column>: <status>: <mutator>`;
- for cargo-mutants, a line with the mutant count, then a line per mutant it missed or that timed out, as
`<path>:<line>:<column>: <outcome>: <mutation>`;
@@ -50,9 +53,9 @@ It prints:
- last, the wall time.
Exit 1, saying why, on an unknown or repeated argument, an option without its value, or a --jobs that is no whole number from 1.
Exit 1 too when git fails, a file cannot be read or copied, a tool cannot start, or a tool fails before it reports. Stryker and
Infection also exit nonzero on a mutation score below a threshold the project sets, which fails the run as well and prints their
whole report.
Exit 1 too when git fails, a file cannot be read or copied, a tool cannot start, or a tool fails before it reports. Stryker, Infection
and Gremlins also exit nonzero on a mutation score below a threshold the project sets, which fails the run as well and prints their
whole output.
"""
import functools
import os
@@ -65,7 +68,7 @@ from machine_load.slots import CPUS_PER_SLOT
from git_cli.git_repo import default_ref, Git, top
from failure import Failure
from git_cli import git_repo
from mutate_changes.in_diff_tools import run_cargo_mutants, run_gremlins, run_infection, run_stryker, with_zero_diff_context
from mutate_changes.in_diff_tools import run_cargo_mutants, run_gremlins, run_infection, run_stryker, with_plain_git_diff
from mutate_changes.mutmut_tool import run_mutmut
from mutate_changes.tool_process import run_tool
@@ -134,7 +137,7 @@ def mutate(given: dict[str, str], env: typing.Mapping[str, str]) -> list[str] |
if tool == "cargo-mutants":
lines = run_cargo_mutants(checkout, merge_base, sorted(files), run, git, jobs)
elif tool == "gremlins":
lines = run_gremlins(checkout, merge_base, functools.partial(run, env=with_zero_diff_context(env)), jobs)
lines = run_gremlins(checkout, merge_base, functools.partial(run, env=with_plain_git_diff(env)), jobs)
elif tool == "infection":
lines = run_infection(checkout, merge_base, run, jobs)
elif tool == "stryker":
+11 -7
View File
@@ -14,7 +14,7 @@ import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from mutate_changes.changed_lines import added_lines, by_folder, not_mutated, runs, tool_for
from mutate_changes.in_diff_tools import with_zero_diff_context
from mutate_changes.in_diff_tools import with_plain_git_diff
from mutate_changes.mutmut_tool import copy_files
from mutate_changes.mutmut_mutants import crash_in_mutmut, function_keys, mutant_globs, mutated_line, named, statuses
from failure import Failure
@@ -148,15 +148,19 @@ class ChangedLines(unittest.TestCase):
class InDiffTools(unittest.TestCase):
def test_zero_diff_context_is_added_after_the_settings_the_environment_holds(self):
setting = {"GIT_CONFIG_KEY_0": "safe.directory", "GIT_CONFIG_VALUE_0": "*"}
self.assertEqual(with_zero_diff_context({"HOME": "/h", "GIT_CONFIG_COUNT": "1", **setting}), {"HOME": "/h", "GIT_CONFIG_COUNT": "2", **setting, "GIT_CONFIG_KEY_1": "diff.context", "GIT_CONFIG_VALUE_1": "0"})
def test_the_git_settings_gremlins_needs_are_added_after_the_ones_the_environment_holds(self):
held = {"HOME": "/h", "GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "safe.directory", "GIT_CONFIG_VALUE_0": "*"}
env = with_plain_git_diff(held)
self.assertEqual(env["GIT_CONFIG_COUNT"], "7")
self.assertEqual({name: value for name, value in env.items() if name in held}, held | {"GIT_CONFIG_COUNT": "7"})
settings = [(env[f"GIT_CONFIG_KEY_{index}"], env[f"GIT_CONFIG_VALUE_{index}"]) for index in range(1, 7)]
self.assertEqual(settings, [("diff.context", "0"), ("diff.noprefix", "false"), ("diff.mnemonicPrefix", "false"), ("diff.srcPrefix", "a/"), ("diff.dstPrefix", "b/"), ("color.diff", "never")])
def test_zero_diff_context_is_the_first_setting_where_the_environment_holds_none_or_a_count_that_is_no_number(self):
first = {"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "diff.context", "GIT_CONFIG_VALUE_0": "0"}
def test_the_git_settings_start_at_the_first_for_an_environment_without_a_count_or_with_one_that_is_not_a_number(self):
for held in ({}, {"GIT_CONFIG_COUNT": "many"}, {"GIT_CONFIG_COUNT": ""}):
with self.subTest(held):
self.assertEqual(with_zero_diff_context(held), first)
env = with_plain_git_diff(held)
self.assertEqual((env["GIT_CONFIG_COUNT"], env["GIT_CONFIG_KEY_0"], env["GIT_CONFIG_KEY_5"], "GIT_CONFIG_KEY_6" in env), ("6", "diff.context", "color.diff", False))
class MutmutSetup(unittest.TestCase):
+23 -17
View File
@@ -65,7 +65,8 @@ else:
# cargo-mutants, Gremlins, Infection and Stryker, which log the folder they run in and how they were called, and exit with the code the test
# gives. cargo-mutants also logs the diff it is given and writes its outcome files, all but unviable.txt, as a run without an
# unviable mutant does; with CAUGHT_UNREADABLE set, nobody may read caught.txt, and with OUTPUT_LOCKED, nobody may delete from mutants.out.
# Gremlins also logs the git settings it finds in its environment and writes a JSON report in Gremlins' shape, with BAD_REPORT set not JSON.
# Gremlins also logs the git settings it finds in its environment and writes a JSON report in Gremlins' shape. With REPORT set, it
# writes that text in its place, or with REPORT=none, no report, as Gremlins does where it finds no mutant.
FAKE_TOOLS = r"""import json, os, pathlib, sys
log, code, args = pathlib.Path(sys.argv[1]), int(sys.argv[2]), sys.argv[3:]
with log.open("a") as written:
@@ -86,8 +87,9 @@ with log.open("a") as written:
written.write("".join(f"{name}={os.environ[name]}\n" for name in git_config))
killed, skipped, uncovered, lived = (dict(type=kind, status=status, line=line, column=column) for kind, status, line, column in (("ARITHMETIC_BASE", "KILLED", 4, 11), ("CONDITIONALS_BOUNDARY", "SKIPPED", 8, 11), ("CONDITIONALS_BOUNDARY", "NOT COVERED", 12, 11), ("ARITHMETIC_BASE", "LIVED", 4, 15)))
files = [dict(file_name="calc.go", mutations=[killed, skipped, uncovered, lived]), dict(file_name="sub/other.go", mutations=[dict(type="INVERT_NEGATIVES", status="LIVED", line=2, column=9)])]
report = "not json" if os.environ.get("BAD_REPORT") else json.dumps(dict(go_module="x", files=files))
pathlib.Path(args[args.index("--output") + 1]).write_text(report)
report = os.environ.get("REPORT") or json.dumps(dict(go_module="x", files=files))
if report != "none":
pathlib.Path(args[args.index("--output") + 1]).write_text(report)
print(f"{args[0]} reports")
sys.exit(code)
"""
@@ -424,32 +426,36 @@ class MutateChangesRuns(unittest.TestCase):
self.change_go()
fake = f"python3 {self.temp / 'fake_tools.py'} {self.log} 0"
ran = self.run_tool("--base", "HEAD", "--jobs=3", "--prefix", fake, cwd=self.checkout / "pkg")
report = ["gremlins: 3 mutants not killed.", "calc.go:4:15: LIVED: ARITHMETIC_BASE", "calc.go:12:11: NOT COVERED: CONDITIONALS_BOUNDARY", "sub/other.go:2:9: LIVED: INVERT_NEGATIVES"]
report = ["gremlins: 4 mutants on changed lines, 3 not killed.", "calc.go:4:15: LIVED: ARITHMETIC_BASE", "calc.go:12:11: NOT COVERED: CONDITIONALS_BOUNDARY", "sub/other.go:2:9: LIVED: INVERT_NEGATIVES"]
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-2], ran.stderr), (0, report, ""))
self.assertEqual(ran.stdout.splitlines()[-2], "Not mutated, as test files: calc_test.go.")
folder, command, config = self.log.read_text().split("\n", 2)
self.assertEqual(folder, str(self.checkout.resolve()))
base = self.git("rev-parse", "HEAD").strip()
self.assertRegex(command, rf"^gremlins unleash --diff {base} --output (\.mutate-changes-\w+)/report\.json --workers 3$")
# Gremlins mutates one run of lines per hunk, so the hunks hold the changed lines and no lines around them.
self.assertEqual(config, "GIT_CONFIG_COUNT=1\nGIT_CONFIG_KEY_0=diff.context\nGIT_CONFIG_VALUE_0=0\n")
keys = ["diff.context", "diff.noprefix", "diff.mnemonicPrefix", "diff.srcPrefix", "diff.dstPrefix", "color.diff"]
values = ["0", "false", "false", "a/", "b/", "never"]
self.assertEqual(config, "GIT_CONFIG_COUNT=6\n" + "".join(f"GIT_CONFIG_KEY_{n}={key}\n" for n, key in enumerate(keys)) + "".join(f"GIT_CONFIG_VALUE_{n}={value}\n" for n, value in enumerate(values)))
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "calc.go", "calc_test.go", "pkg"])
def test_gremlins_git_settings_come_after_those_the_environment_holds(self):
def test_a_gremlins_run_that_finds_no_mutant_reports_none(self):
self.change_go()
env = {"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "safe.directory", "GIT_CONFIG_VALUE_0": "*"}
self.run_tool("--base", "HEAD", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", env=env)
config = self.log.read_text().split("\n", 2)[2]
self.assertEqual(config, "GIT_CONFIG_COUNT=2\nGIT_CONFIG_KEY_0=safe.directory\nGIT_CONFIG_KEY_1=diff.context\nGIT_CONFIG_VALUE_0=*\nGIT_CONFIG_VALUE_1=0\n")
for report in ("none", '{"files": null}', '{"files": [{"file_name": "calc.go", "mutations": null}]}'):
with self.subTest(report):
ran = self.run_tool("--base", "HEAD", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", env={"REPORT": report})
self.assertEqual((ran.returncode, ran.stdout.splitlines()[0], ran.stderr), (0, "gremlins: 0 mutants on changed lines, 0 not killed.", ""))
def test_a_gremlins_run_that_fails_or_leaves_no_report_fails_the_run(self):
def test_a_gremlins_run_that_fails_or_leaves_a_report_it_cannot_read_fails_the_run(self):
self.change_go()
fake = f"python3 {self.temp / 'fake_tools.py'} {self.log}"
ran = self.run_tool("--base", "HEAD", "--prefix", f"{fake} 1")
self.assertEqual((ran.returncode, ran.stderr), (1, "gremlins exited 1:\ngremlins reports\n"))
ran = self.run_tool("--base", "HEAD", "--prefix", f"{fake} 0", env={"BAD_REPORT": "1"})
self.assertEqual(ran.returncode, 1)
self.assertRegex(ran.stderr, rf"^could not read Gremlins' report in {re.escape(str(self.checkout.resolve()))}/\.mutate-changes-\w+: ")
# Gremlins also exits nonzero on a mutation score below the project's threshold, where its output lists what survived.
ran = self.run_tool("--base", "HEAD", "--prefix", f"{fake} 10")
self.assertEqual((ran.returncode, ran.stderr), (1, "Gremlins exited 10:\ngremlins reports\n"))
for report in ("not json", "[]", '{"files": 3}'):
with self.subTest(report):
ran = self.run_tool("--base", "HEAD", "--prefix", f"{fake} 0", env={"REPORT": report})
self.assertEqual(ran.returncode, 1)
self.assertRegex(ran.stderr, rf"^could not read Gremlins' report in {re.escape(str(self.checkout.resolve()))}/\.mutate-changes-\w+: ")
ran = self.run_tool("--base", "HEAD", env={"PATH": str(self.path_with_git())})
self.assertEqual(ran.returncode, 1)
self.assertRegex(ran.stderr, "^gremlins unleash .* could not start: ")