Answer stability round 1: 1 🔴, 2 🟡 (0 misread); pin the git diff settings Gremlins parses, show the mutant total, and a container prefix that runs as root
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
"""Mutate a branch's changed lines with cargo-mutants, Gremlins, Stryker and Infection, each of which takes the changed lines itself."""
|
||||
"""Mutate a branch's changed lines with cargo-mutants, Gremlins, Infection and Stryker, each of which takes the changed lines itself."""
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
@@ -12,8 +12,10 @@ from git_cli.git_repo import Git
|
||||
# cargo-mutants' exit codes for a run that finished: every mutant caught, some missed, and some timed out.
|
||||
CARGO_FINISHED = frozenset({0, 2, 3})
|
||||
CARGO_OUTCOMES = ("caught", "missed", "timeout", "unviable")
|
||||
# The statuses in Gremlins' report of a mutant that its tests don't kill.
|
||||
GREMLINS_UNKILLED = frozenset({"LIVED", "NOT COVERED"})
|
||||
# What Gremlins v0.6.0 needs of the diff it reads from git: one run of lines per hunk is all it mutates, so a hunk holds the changed
|
||||
# lines and none around them; it takes each file's path after a b/; and it reads no colour codes. A user's git settings change each.
|
||||
GREMLINS_GIT_SETTINGS = {"diff.context": "0", "diff.noprefix": "false", "diff.mnemonicPrefix": "false", "diff.srcPrefix": "a/", "diff.dstPrefix": "b/", "color.diff": "never"}
|
||||
|
||||
|
||||
def run_cargo_mutants(top: str, base: str, paths: list[str], run: Run, git: Git, jobs: int) -> list[str] | Failure:
|
||||
@@ -42,37 +44,50 @@ def run_cargo_mutants(top: str, base: str, paths: list[str], run: Run, git: Git,
|
||||
return [f"cargo-mutants: {sum(map(len, outcomes.values()))} mutants on changed lines, {len(uncaught)} not caught.", *uncaught]
|
||||
|
||||
|
||||
def run_gremlins(top: str, base: str, run: Run, jobs: int) -> list[str] | Failure:
|
||||
"""A line with the number of mutants Gremlins reports as lived or not covered, then a line per mutant, as
|
||||
`<path>:<line>:<column>: <status>: <mutator>`.
|
||||
class Mutant(typing.NamedTuple):
|
||||
"""A mutant in Gremlins' report, in the order the report prints them."""
|
||||
|
||||
Gremlins runs in top and mutates the Go lines the branch changes since base. It skips test files itself. Its git needs `run` to
|
||||
hold the settings that `with_zero_diff_context` adds to an environment."""
|
||||
path: str
|
||||
line: int
|
||||
column: int
|
||||
status: str
|
||||
mutator: str
|
||||
|
||||
|
||||
def run_gremlins(top: str, base: str, run: Run, jobs: int) -> list[str] | Failure:
|
||||
"""A line with the number of mutants on the changed lines and how many of them Gremlins reports as lived or not covered, then a
|
||||
line per such mutant, as `<path>:<line>:<column>: <status>: <mutator>`.
|
||||
|
||||
Gremlins runs in top and mutates the Go lines the branch changes since base. It skips `*_test.go` itself. It reads the diff from
|
||||
git, so `run` must hold the environment that `with_plain_git_diff` makes. Gremlins exits nonzero on a mutation score below a
|
||||
threshold the project sets, which its exit code doesn't tell from a run that broke, so any nonzero exit fails the run."""
|
||||
scratch = scratch_folder(top)
|
||||
if isinstance(scratch, Failure):
|
||||
return scratch
|
||||
try:
|
||||
report = os.path.join(os.path.relpath(scratch, top), "report.json")
|
||||
ran = run(["gremlins", "unleash", "--diff", base, "--output", report, "--workers", str(jobs)], top)
|
||||
report = pathlib.Path(scratch, "report.json")
|
||||
ran = run(["gremlins", "unleash", "--diff", base, "--output", os.path.join(os.path.relpath(scratch, top), report.name), "--workers", str(jobs)], top)
|
||||
if isinstance(ran, Failure) or ran.code:
|
||||
return ran if isinstance(ran, Failure) else failed("gremlins", ran)
|
||||
# Gremlins writes `null` for a list with no entry.
|
||||
files = json.loads(pathlib.Path(scratch, "report.json").read_text(encoding="utf-8"))["files"] or []
|
||||
unkilled = sorted((file["file_name"], mutant["line"], mutant["column"], mutant["status"], mutant["type"]) for file in files for mutant in file["mutations"] or [] if mutant["status"] in GREMLINS_UNKILLED)
|
||||
return ran if isinstance(ran, Failure) else whole_output("Gremlins", ran)
|
||||
# Gremlins writes no report where it finds no mutant, and `null` for a list with no entry.
|
||||
files = (json.loads(report.read_text(encoding="utf-8"))["files"] or []) if report.is_file() else []
|
||||
mutants = [Mutant(file["file_name"], mutant["line"], mutant["column"], mutant["status"], mutant["type"]) for file in files for mutant in file["mutations"] or []]
|
||||
except (OSError, ValueError, KeyError, TypeError) as failure:
|
||||
return Failure(f"could not read Gremlins' report in {scratch}: {failure}")
|
||||
finally:
|
||||
remove(scratch)
|
||||
return [f"gremlins: {len(unkilled)} mutants not killed.", *(f"{path}:{line}:{column}: {status}: {mutator}" for path, line, column, status, mutator in unkilled)]
|
||||
# Gremlins lists the mutants on the unchanged lines of a hunk too, as SKIPPED. It lists the files in no fixed order.
|
||||
unkilled = sorted(mutant for mutant in mutants if mutant.status in GREMLINS_UNKILLED)
|
||||
on_changed_lines = sum(mutant.status != "SKIPPED" for mutant in mutants)
|
||||
return [f"gremlins: {on_changed_lines} mutants on changed lines, {len(unkilled)} not killed.", *(f"{m.path}:{m.line}:{m.column}: {m.status}: {m.mutator}" for m in unkilled)]
|
||||
|
||||
|
||||
def with_zero_diff_context(env: typing.Mapping[str, str]) -> dict[str, str]:
|
||||
"""env with git set, through its GIT_CONFIG_* variables, to print no context lines around a change, after any setting env holds.
|
||||
|
||||
Gremlins v0.6.0 mutates one run of lines per hunk, so a hunk must hold only the lines the branch changes."""
|
||||
held = env.get("GIT_CONFIG_COUNT", "0")
|
||||
count = int(held) if held.isdecimal() else 0
|
||||
return {**env, "GIT_CONFIG_COUNT": str(count + 1), f"GIT_CONFIG_KEY_{count}": "diff.context", f"GIT_CONFIG_VALUE_{count}": "0"}
|
||||
def with_plain_git_diff(env: typing.Mapping[str, str]) -> dict[str, str]:
|
||||
"""env with the settings of GREMLINS_GIT_SETTINGS added to the ones it holds, as git reads them from its GIT_CONFIG_* variables."""
|
||||
held = env.get("GIT_CONFIG_COUNT", "")
|
||||
first = int(held) if held.isdecimal() else 0
|
||||
added = {f"GIT_CONFIG_{part}_{first + index}": text for index, (key, value) in enumerate(GREMLINS_GIT_SETTINGS.items()) for part, text in (("KEY", key), ("VALUE", value))}
|
||||
return {**env, **added, "GIT_CONFIG_COUNT": str(first + len(GREMLINS_GIT_SETTINGS))}
|
||||
|
||||
|
||||
def outcome_lines(path: pathlib.Path) -> list[str]:
|
||||
|
||||
+17
-14
@@ -1,10 +1,10 @@
|
||||
"""Mutation-test the lines a branch adds or changes, and print each mutant its tests don't kill.
|
||||
r"""Mutation-test the lines a branch adds or changes, and print each mutant its tests don't kill.
|
||||
|
||||
usage: mutate-changes [--base <ref>] [--prefix <shell text>] [--jobs <n>]
|
||||
|
||||
Run it in a checkout, under ~/.claude/bin/run-heavy. It diffs the working tree's tracked files against the merge base of HEAD and
|
||||
--base, by default the default branch: origin's as last fetched, else main or master. It picks the tool by each changed file's
|
||||
extension, and leaves out test files: test_*.py, *_test.py, conftest.py, *.test.* and *.spec.*, and any file under a tests/, test/
|
||||
extension, and leaves out test files: test_*.py, *_test.py, *_test.go, conftest.py, *.test.* and *.spec.*, and any file under a tests/, test/
|
||||
or __tests__/ folder.
|
||||
|
||||
- .py: mutmut 3, in a scratch copy of the nearest folder above the file that holds a setup.cfg or a pyproject.toml, else of the
|
||||
@@ -16,8 +16,9 @@ or __tests__/ folder.
|
||||
in its own process, so a mutant that only a child process reaches, such as a tool a test starts, shows as `no tests`. A
|
||||
pyproject.toml with a [tool.mutmut] section fails the run. A crash inside mutmut, such as a KeyError in its worker pool,
|
||||
becomes one line for its folder, naming the error and telling you to mutate that folder's changed lines by hand.
|
||||
- .go: Gremlins, as `gremlins unleash --diff`, in the checkout's top. It skips *_test.go itself. Its git runs with diff.context=0,
|
||||
set in GIT_CONFIG_* variables after any the environment holds, since Gremlins v0.6.0 mutates one run of lines per hunk.
|
||||
- .go: Gremlins, as `gremlins unleash --diff`, in the checkout's top, where its go.mod must be. Gremlins reads the diff from git, so
|
||||
the run adds settings to the GIT_CONFIG_* variables, after any the environment holds: no context lines, since Gremlins mutates
|
||||
every line of a hunk; a/ and b/ as the file prefixes; and no colour.
|
||||
- .rs: cargo-mutants, as `cargo mutants --in-diff`, in the checkout's top.
|
||||
- .cjs, .cts, .js, .jsx, .mjs, .mts, .ts, .tsx: Stryker, as `npx --no --package=@stryker-mutator/core stryker run --mutate` on the
|
||||
changed lines, in the nearest folder above the file that holds a package.json. A file with no package.json above it isn't
|
||||
@@ -29,10 +30,12 @@ as "$@". For example, 'uvx --from "mutmut>=3,<4"'. Without --prefix, the command
|
||||
run there, as 'docker compose -f "$(git rev-parse --show-toplevel)/compose.yaml" run --rm -v "$PWD:$PWD" -w "$PWD" app' does. "$1"
|
||||
is the command's first word, mutmut, cargo, gremlins, npx or vendor/bin/infection, so a prefix can pick what to run per tool:
|
||||
'case "$1" in mutmut) set -- uvx --from "mutmut>=3,<4" "$@";; esac;'.
|
||||
Gremlins' git runs in the container too. In a git worktree, git keeps its data in the main checkout, outside $PWD, so the container
|
||||
mounts that folder as well, and it passes on the GIT_CONFIG_* variables that set diff.context, as `-e GIT_CONFIG_COUNT -e
|
||||
GIT_CONFIG_KEY_0 -e GIT_CONFIG_VALUE_0` where the environment holds no setting of its own, and one more KEY and VALUE for each it holds:
|
||||
'common="$(git rev-parse --path-format=absolute --git-common-dir)"; docker run --rm -v "$PWD:$PWD" -v "$common:$common" -w "$PWD" -e GIT_CONFIG_COUNT -e GIT_CONFIG_KEY_0 -e GIT_CONFIG_VALUE_0 <image> "$@"'.
|
||||
Gremlins' git runs in the container too, so a Go prefix mounts git's data and passes on the GIT_CONFIG_* variables, and allows git
|
||||
to read a checkout that the container's root user doesn't own. In a git worktree, git's data is in the main checkout, outside $PWD:
|
||||
|
||||
common="$(git rev-parse --path-format=absolute --git-common-dir)"
|
||||
docker run --rm -v "$PWD:$PWD" -v "$common:$common" -w "$PWD" $(printenv | sed -n 's/^\(GIT_CONFIG_[A-Z0-9_]*\)=.*/-e \1/p') \
|
||||
<image holding gremlins> sh -c 'git config --global --add safe.directory "*" && exec "$@"' sh
|
||||
|
||||
--jobs sets how many tests each tool runs at once, 4 by default: one heavy slot's CPUs. Under run-heavy, a container that sets
|
||||
neither memory limit gets the slot's share, in MiB in RUN_HEAVY_MEMORY; set --jobs so that that many test runs fit in it.
|
||||
@@ -40,7 +43,7 @@ neither memory limit gets the slot's share, in MiB in RUN_HEAVY_MEMORY; set --jo
|
||||
It prints:
|
||||
- for mutmut, per folder, a line with the mutant count, then a line per mutant on a changed line that is not killed, as
|
||||
`<path>:<line>: <status>: <the line as mutated>`, or `<path>: <status>: <name>, on a line mutate-changes cannot find`;
|
||||
- for Gremlins, a line with the number of mutants not killed, then a line per mutant that lived or is not covered, as
|
||||
- for Gremlins, a line with the mutant count and how many it did not kill, then a line per mutant that lived or is not covered, as
|
||||
`<path>:<line>:<column>: <status>: <mutator>`;
|
||||
- for cargo-mutants, a line with the mutant count, then a line per mutant it missed or that timed out, as
|
||||
`<path>:<line>:<column>: <outcome>: <mutation>`;
|
||||
@@ -50,9 +53,9 @@ It prints:
|
||||
- last, the wall time.
|
||||
|
||||
Exit 1, saying why, on an unknown or repeated argument, an option without its value, or a --jobs that is no whole number from 1.
|
||||
Exit 1 too when git fails, a file cannot be read or copied, a tool cannot start, or a tool fails before it reports. Stryker and
|
||||
Infection also exit nonzero on a mutation score below a threshold the project sets, which fails the run as well and prints their
|
||||
whole report.
|
||||
Exit 1 too when git fails, a file cannot be read or copied, a tool cannot start, or a tool fails before it reports. Stryker, Infection
|
||||
and Gremlins also exit nonzero on a mutation score below a threshold the project sets, which fails the run as well and prints their
|
||||
whole output.
|
||||
"""
|
||||
import functools
|
||||
import os
|
||||
@@ -65,7 +68,7 @@ from machine_load.slots import CPUS_PER_SLOT
|
||||
from git_cli.git_repo import default_ref, Git, top
|
||||
from failure import Failure
|
||||
from git_cli import git_repo
|
||||
from mutate_changes.in_diff_tools import run_cargo_mutants, run_gremlins, run_infection, run_stryker, with_zero_diff_context
|
||||
from mutate_changes.in_diff_tools import run_cargo_mutants, run_gremlins, run_infection, run_stryker, with_plain_git_diff
|
||||
from mutate_changes.mutmut_tool import run_mutmut
|
||||
from mutate_changes.tool_process import run_tool
|
||||
|
||||
@@ -134,7 +137,7 @@ def mutate(given: dict[str, str], env: typing.Mapping[str, str]) -> list[str] |
|
||||
if tool == "cargo-mutants":
|
||||
lines = run_cargo_mutants(checkout, merge_base, sorted(files), run, git, jobs)
|
||||
elif tool == "gremlins":
|
||||
lines = run_gremlins(checkout, merge_base, functools.partial(run, env=with_zero_diff_context(env)), jobs)
|
||||
lines = run_gremlins(checkout, merge_base, functools.partial(run, env=with_plain_git_diff(env)), jobs)
|
||||
elif tool == "infection":
|
||||
lines = run_infection(checkout, merge_base, run, jobs)
|
||||
elif tool == "stryker":
|
||||
|
||||
@@ -14,7 +14,7 @@ import unittest
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from mutate_changes.changed_lines import added_lines, by_folder, not_mutated, runs, tool_for
|
||||
from mutate_changes.in_diff_tools import with_zero_diff_context
|
||||
from mutate_changes.in_diff_tools import with_plain_git_diff
|
||||
from mutate_changes.mutmut_tool import copy_files
|
||||
from mutate_changes.mutmut_mutants import crash_in_mutmut, function_keys, mutant_globs, mutated_line, named, statuses
|
||||
from failure import Failure
|
||||
@@ -148,15 +148,19 @@ class ChangedLines(unittest.TestCase):
|
||||
|
||||
|
||||
class InDiffTools(unittest.TestCase):
|
||||
def test_zero_diff_context_is_added_after_the_settings_the_environment_holds(self):
|
||||
setting = {"GIT_CONFIG_KEY_0": "safe.directory", "GIT_CONFIG_VALUE_0": "*"}
|
||||
self.assertEqual(with_zero_diff_context({"HOME": "/h", "GIT_CONFIG_COUNT": "1", **setting}), {"HOME": "/h", "GIT_CONFIG_COUNT": "2", **setting, "GIT_CONFIG_KEY_1": "diff.context", "GIT_CONFIG_VALUE_1": "0"})
|
||||
def test_the_git_settings_gremlins_needs_are_added_after_the_ones_the_environment_holds(self):
|
||||
held = {"HOME": "/h", "GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "safe.directory", "GIT_CONFIG_VALUE_0": "*"}
|
||||
env = with_plain_git_diff(held)
|
||||
self.assertEqual(env["GIT_CONFIG_COUNT"], "7")
|
||||
self.assertEqual({name: value for name, value in env.items() if name in held}, held | {"GIT_CONFIG_COUNT": "7"})
|
||||
settings = [(env[f"GIT_CONFIG_KEY_{index}"], env[f"GIT_CONFIG_VALUE_{index}"]) for index in range(1, 7)]
|
||||
self.assertEqual(settings, [("diff.context", "0"), ("diff.noprefix", "false"), ("diff.mnemonicPrefix", "false"), ("diff.srcPrefix", "a/"), ("diff.dstPrefix", "b/"), ("color.diff", "never")])
|
||||
|
||||
def test_zero_diff_context_is_the_first_setting_where_the_environment_holds_none_or_a_count_that_is_no_number(self):
|
||||
first = {"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "diff.context", "GIT_CONFIG_VALUE_0": "0"}
|
||||
def test_the_git_settings_start_at_the_first_for_an_environment_without_a_count_or_with_one_that_is_not_a_number(self):
|
||||
for held in ({}, {"GIT_CONFIG_COUNT": "many"}, {"GIT_CONFIG_COUNT": ""}):
|
||||
with self.subTest(held):
|
||||
self.assertEqual(with_zero_diff_context(held), first)
|
||||
env = with_plain_git_diff(held)
|
||||
self.assertEqual((env["GIT_CONFIG_COUNT"], env["GIT_CONFIG_KEY_0"], env["GIT_CONFIG_KEY_5"], "GIT_CONFIG_KEY_6" in env), ("6", "diff.context", "color.diff", False))
|
||||
|
||||
|
||||
class MutmutSetup(unittest.TestCase):
|
||||
|
||||
@@ -65,7 +65,8 @@ else:
|
||||
# cargo-mutants, Gremlins, Infection and Stryker, which log the folder they run in and how they were called, and exit with the code the test
|
||||
# gives. cargo-mutants also logs the diff it is given and writes its outcome files, all but unviable.txt, as a run without an
|
||||
# unviable mutant does; with CAUGHT_UNREADABLE set, nobody may read caught.txt, and with OUTPUT_LOCKED, nobody may delete from mutants.out.
|
||||
# Gremlins also logs the git settings it finds in its environment and writes a JSON report in Gremlins' shape, with BAD_REPORT set not JSON.
|
||||
# Gremlins also logs the git settings it finds in its environment and writes a JSON report in Gremlins' shape. With REPORT set, it
|
||||
# writes that text in its place, or with REPORT=none, no report, as Gremlins does where it finds no mutant.
|
||||
FAKE_TOOLS = r"""import json, os, pathlib, sys
|
||||
log, code, args = pathlib.Path(sys.argv[1]), int(sys.argv[2]), sys.argv[3:]
|
||||
with log.open("a") as written:
|
||||
@@ -86,8 +87,9 @@ with log.open("a") as written:
|
||||
written.write("".join(f"{name}={os.environ[name]}\n" for name in git_config))
|
||||
killed, skipped, uncovered, lived = (dict(type=kind, status=status, line=line, column=column) for kind, status, line, column in (("ARITHMETIC_BASE", "KILLED", 4, 11), ("CONDITIONALS_BOUNDARY", "SKIPPED", 8, 11), ("CONDITIONALS_BOUNDARY", "NOT COVERED", 12, 11), ("ARITHMETIC_BASE", "LIVED", 4, 15)))
|
||||
files = [dict(file_name="calc.go", mutations=[killed, skipped, uncovered, lived]), dict(file_name="sub/other.go", mutations=[dict(type="INVERT_NEGATIVES", status="LIVED", line=2, column=9)])]
|
||||
report = "not json" if os.environ.get("BAD_REPORT") else json.dumps(dict(go_module="x", files=files))
|
||||
pathlib.Path(args[args.index("--output") + 1]).write_text(report)
|
||||
report = os.environ.get("REPORT") or json.dumps(dict(go_module="x", files=files))
|
||||
if report != "none":
|
||||
pathlib.Path(args[args.index("--output") + 1]).write_text(report)
|
||||
print(f"{args[0]} reports")
|
||||
sys.exit(code)
|
||||
"""
|
||||
@@ -424,32 +426,36 @@ class MutateChangesRuns(unittest.TestCase):
|
||||
self.change_go()
|
||||
fake = f"python3 {self.temp / 'fake_tools.py'} {self.log} 0"
|
||||
ran = self.run_tool("--base", "HEAD", "--jobs=3", "--prefix", fake, cwd=self.checkout / "pkg")
|
||||
report = ["gremlins: 3 mutants not killed.", "calc.go:4:15: LIVED: ARITHMETIC_BASE", "calc.go:12:11: NOT COVERED: CONDITIONALS_BOUNDARY", "sub/other.go:2:9: LIVED: INVERT_NEGATIVES"]
|
||||
report = ["gremlins: 4 mutants on changed lines, 3 not killed.", "calc.go:4:15: LIVED: ARITHMETIC_BASE", "calc.go:12:11: NOT COVERED: CONDITIONALS_BOUNDARY", "sub/other.go:2:9: LIVED: INVERT_NEGATIVES"]
|
||||
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-2], ran.stderr), (0, report, ""))
|
||||
self.assertEqual(ran.stdout.splitlines()[-2], "Not mutated, as test files: calc_test.go.")
|
||||
folder, command, config = self.log.read_text().split("\n", 2)
|
||||
self.assertEqual(folder, str(self.checkout.resolve()))
|
||||
base = self.git("rev-parse", "HEAD").strip()
|
||||
self.assertRegex(command, rf"^gremlins unleash --diff {base} --output (\.mutate-changes-\w+)/report\.json --workers 3$")
|
||||
# Gremlins mutates one run of lines per hunk, so the hunks hold the changed lines and no lines around them.
|
||||
self.assertEqual(config, "GIT_CONFIG_COUNT=1\nGIT_CONFIG_KEY_0=diff.context\nGIT_CONFIG_VALUE_0=0\n")
|
||||
keys = ["diff.context", "diff.noprefix", "diff.mnemonicPrefix", "diff.srcPrefix", "diff.dstPrefix", "color.diff"]
|
||||
values = ["0", "false", "false", "a/", "b/", "never"]
|
||||
self.assertEqual(config, "GIT_CONFIG_COUNT=6\n" + "".join(f"GIT_CONFIG_KEY_{n}={key}\n" for n, key in enumerate(keys)) + "".join(f"GIT_CONFIG_VALUE_{n}={value}\n" for n, value in enumerate(values)))
|
||||
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "calc.go", "calc_test.go", "pkg"])
|
||||
|
||||
def test_gremlins_git_settings_come_after_those_the_environment_holds(self):
|
||||
def test_a_gremlins_run_that_finds_no_mutant_reports_none(self):
|
||||
self.change_go()
|
||||
env = {"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "safe.directory", "GIT_CONFIG_VALUE_0": "*"}
|
||||
self.run_tool("--base", "HEAD", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", env=env)
|
||||
config = self.log.read_text().split("\n", 2)[2]
|
||||
self.assertEqual(config, "GIT_CONFIG_COUNT=2\nGIT_CONFIG_KEY_0=safe.directory\nGIT_CONFIG_KEY_1=diff.context\nGIT_CONFIG_VALUE_0=*\nGIT_CONFIG_VALUE_1=0\n")
|
||||
for report in ("none", '{"files": null}', '{"files": [{"file_name": "calc.go", "mutations": null}]}'):
|
||||
with self.subTest(report):
|
||||
ran = self.run_tool("--base", "HEAD", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", env={"REPORT": report})
|
||||
self.assertEqual((ran.returncode, ran.stdout.splitlines()[0], ran.stderr), (0, "gremlins: 0 mutants on changed lines, 0 not killed.", ""))
|
||||
|
||||
def test_a_gremlins_run_that_fails_or_leaves_no_report_fails_the_run(self):
|
||||
def test_a_gremlins_run_that_fails_or_leaves_a_report_it_cannot_read_fails_the_run(self):
|
||||
self.change_go()
|
||||
fake = f"python3 {self.temp / 'fake_tools.py'} {self.log}"
|
||||
ran = self.run_tool("--base", "HEAD", "--prefix", f"{fake} 1")
|
||||
self.assertEqual((ran.returncode, ran.stderr), (1, "gremlins exited 1:\ngremlins reports\n"))
|
||||
ran = self.run_tool("--base", "HEAD", "--prefix", f"{fake} 0", env={"BAD_REPORT": "1"})
|
||||
self.assertEqual(ran.returncode, 1)
|
||||
self.assertRegex(ran.stderr, rf"^could not read Gremlins' report in {re.escape(str(self.checkout.resolve()))}/\.mutate-changes-\w+: ")
|
||||
# Gremlins also exits nonzero on a mutation score below the project's threshold, where its output lists what survived.
|
||||
ran = self.run_tool("--base", "HEAD", "--prefix", f"{fake} 10")
|
||||
self.assertEqual((ran.returncode, ran.stderr), (1, "Gremlins exited 10:\ngremlins reports\n"))
|
||||
for report in ("not json", "[]", '{"files": 3}'):
|
||||
with self.subTest(report):
|
||||
ran = self.run_tool("--base", "HEAD", "--prefix", f"{fake} 0", env={"REPORT": report})
|
||||
self.assertEqual(ran.returncode, 1)
|
||||
self.assertRegex(ran.stderr, rf"^could not read Gremlins' report in {re.escape(str(self.checkout.resolve()))}/\.mutate-changes-\w+: ")
|
||||
ran = self.run_tool("--base", "HEAD", env={"PATH": str(self.path_with_git())})
|
||||
self.assertEqual(ran.returncode, 1)
|
||||
self.assertRegex(ran.stderr, "^gremlins unleash .* could not start: ")
|
||||
|
||||
Reference in New Issue
Block a user