Files
claude-config/bin/mutate_changes/test_mutate_changes_runs.py
T

443 lines
26 KiB
Python

"""mutate-changes run on a branch, against fake mutation tools that log how they were called."""
import contextlib
import functools
import io
import os
import pathlib
import re
import shutil
import signal
import subprocess
import sys
import tempfile
import time
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from mutate_changes.test_mutate_changes import ended
from git_cli import git_repo
from test_env import isolated_env
from mutate_changes import main
from mutate_changes.in_diff_tools import run_cargo_mutants
from mutate_changes.tool_process import run_tool
TOOL = pathlib.Path(__file__).resolve().parents[1] / "mutate-changes"
SETUP = "[mutmut]\ndo_not_mutate = *_old.py\n"
PASSED = ["mutmut in pkg: 2 mutants on changed lines, 1 not killed.", "pkg/calc.py:6: survived: return n / 3"]
NOT_MUTATED = ["Not mutated, as no tool covers their language: README.md.", "Not mutated, as test files: pkg/test_calc.py."]
# Stands in for mutmut, run through --prefix as `python3 <this> <log> <mode> mutmut ...` in the scratch copy. It logs each call with
# the folder it runs in and the setup.cfg it finds there, then writes a mutants/ copy and prints results, both shaped like mutmut 3's.
# In mode locks, it also leaves a folder in the copy that nobody may delete from.
FAKE_MUTMUT = r"""import os, pathlib, sys
log, mode, args = pathlib.Path(sys.argv[1]), sys.argv[2], sys.argv[3:]
config = pathlib.Path("setup.cfg").read_text()
with log.open("a") as written:
written.write(f"{os.getcwd()}\n{' '.join(args)}\n{config}---\n")
if args[:2] == ["mutmut", "results"]:
if mode == "results-fail" or args[2:] != ["--all", "true"]:
sys.exit("mutmut results broke")
double, triple_1, triple_2 = ("no tests",) * 3 if mode == "no-test" else ("survived", "survived", "killed")
# The branch doesn't change aside.py, so mutate-changes skips its mutant, which comes first in name order.
print(f" aside.x_f__mutmut_1: survived\n calc.x_double__mutmut_1: {double}\n calc.x_triple__mutmut_1: {triple_1}\n calc.x_triple__mutmut_2: {triple_2}")
elif mode == "fails-once" and not ("--deselect=test_calc.py::test_slow[n=1]" in config and "--ignore=test_io.py" in config):
print("==== short test summary info ====\nFAILED test_calc.py::test_slow[n=1] - boom\nERROR test_io.py - FileNotFoundError\nfailed to collect stats. runner returned 1")
sys.exit(1)
else:
if mode != "elsewhere":
pathlib.Path("mutants").mkdir(exist_ok=True)
if mode not in ("elsewhere", "no-copy"):
functions = ["x_double__mutmut_orig(n): # doubles\n\treturn n * 2", "x_double__mutmut_1(n): # doubles\n\treturn n / 2", "x_triple__mutmut_orig(n):\n\treturn n * 3", "x_triple__mutmut_1(n):\n\treturn n / 3", "x_triple__mutmut_2(n):\n\treturn n * 4"]
pathlib.Path("mutants/calc.py").write_bytes(b"# \xff is not UTF-8\n" + "".join(f"def {function}\n" for function in functions).encode())
if mode == "locks":
pathlib.Path("locked/inner").mkdir(parents=True)
pathlib.Path("locked").chmod(0o555)
if mode == "no-test":
print("Stopping early, because we could not find any test case for any mutant. It seems that the selected tests do not cover any code that we mutated.")
sys.exit(1)
if mode == "setup-fault":
print("Stopping early, because tests recorded trampoline hits but none match any mutant key. This usually means imports bypass the mutants.")
sys.exit(1)
"""
# cargo-mutants, Infection and Stryker, which log the folder they run in and how they were called, and exit with the code the test
# gives. cargo-mutants also logs the diff it is given and writes its outcome files, all but unviable.txt, as a run without an
# unviable mutant does; with CAUGHT_UNREADABLE set, nobody may read caught.txt, and with OUTPUT_LOCKED, nobody may delete from mutants.out.
FAKE_TOOLS = r"""import os, pathlib, sys
log, code, args = pathlib.Path(sys.argv[1]), int(sys.argv[2]), sys.argv[3:]
with log.open("a") as written:
written.write(f"{os.getcwd()}\n{' '.join(args)}\n")
if args[0] == "cargo":
written.write(pathlib.Path(args[args.index("--in-diff") + 1]).read_text())
outcomes = pathlib.Path(args[args.index("--output") + 1], "mutants.out")
outcomes.mkdir()
(outcomes / "caught.txt").write_bytes(b"src/lib.rs:2:7: replace + with - in add, \xff not being UTF-8\n")
(outcomes / "missed.txt").write_text("src/lib.rs:2:5: replace add(a: i32, b: i32) -> i32 with 0\n")
(outcomes / "timeout.txt").write_text("src/lib.rs:2:7: replace + with * in add\n")
if os.environ.get("CAUGHT_UNREADABLE"):
(outcomes / "caught.txt").chmod(0)
if os.environ.get("OUTPUT_LOCKED"):
outcomes.chmod(0o555)
print(f"{args[0]} reports")
sys.exit(code)
"""
@contextlib.contextmanager
def stdin_holding(text: str):
"""This process's stdin, the one a child inherits, as a pipe holding text."""
read, write = os.pipe()
os.write(write, text.encode())
os.close(write)
saved = os.dup(0)
os.dup2(read, 0)
os.close(read)
try:
yield
finally:
os.dup2(saved, 0)
os.close(saved)
class MutateChangesRuns(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
self.temp = pathlib.Path(temp.name)
self.checkout = self.temp / "proj"
(self.checkout / "pkg").mkdir(parents=True)
(self.checkout / "pkg" / "setup.cfg").write_text(SETUP)
(self.checkout / "pkg" / "calc.py").write_text("def double(n):\n\treturn n * 2\n")
(self.checkout / "pkg" / "test_calc.py").write_text("")
self.git("init", "-q", "-b", "main")
self.commit()
self.git("checkout", "-q", "-b", "work")
# git lists the README first, and mutate-changes mutates no Markdown.
(self.checkout / "README.md").write_text("Doubles and triples.\n")
(self.checkout / "pkg" / "calc.py").write_text("def double(n): # doubles\n\treturn n * 2\n\n\ndef triple(n):\n\treturn n * 3\n")
(self.checkout / "pkg" / "test_calc.py").write_text("# changed, but a test file is not mutated\n")
self.commit()
# Settings a user may have that change what git diff prints, and a file git lists as untracked and one it ignores.
for name, value in (("color.diff", "always"), ("diff.mnemonicPrefix", "true"), ("diff.relative", "true"), ("diff.external", "true"), ("diff.blank.textconv", "true")):
self.git("config", name, value)
(self.checkout / ".git" / "info").mkdir(exist_ok=True)
(self.checkout / ".git" / "info" / "attributes").write_text("* diff=blank\n")
(self.checkout / ".git" / "info" / "exclude").write_text("*.log\n")
(self.checkout / "pkg" / "fixture.txt").write_text("")
(self.checkout / "pkg" / "run.log").write_text("")
(self.temp / "fake_mutmut.py").write_text(FAKE_MUTMUT)
(self.temp / "fake_tools.py").write_text(FAKE_TOOLS)
self.log = self.temp / "calls.log"
def git(self, *args):
return subprocess.run(["git", *args], check=True, capture_output=True, cwd=self.checkout, encoding="utf-8", env={**isolated_env(), "GIT_CONFIG_GLOBAL": "/dev/null"}).stdout
def commit(self):
self.git("add", ".")
self.git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "x")
def run_tool(self, *args, cwd=None, env=None):
"""Run mutate-changes in this process, from cwd, else the checkout: mutmut counts only the code a test reaches in its own process."""
before = os.getcwd()
os.chdir(cwd or self.checkout)
try:
with contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()) as err:
code = main.main(list(args), {**isolated_env(), **(env or {})})
finally:
os.chdir(before)
return subprocess.CompletedProcess(args, code, out.getvalue(), err.getvalue())
def faked(self, mode):
return ["--prefix", f"python3 {self.temp / 'fake_mutmut.py'} {self.log} {mode}"]
def change_rust_php_and_typescript(self):
"""Commit a Rust, a PHP and a TypeScript file, then change them: Rust's line 2 in the index, PHP's line 2 with a byte that is
not UTF-8, and TypeScript's lines 1 and 4. Also change line 1 of a TypeScript file in web/. The top and web/ each hold a package.json."""
(self.checkout / "src").mkdir()
(self.checkout / "web" / "src").mkdir(parents=True)
(self.checkout / "package.json").write_text("{}\n")
(self.checkout / "web" / "package.json").write_text("{}\n")
(self.checkout / "web" / "src" / "app.ts").write_text("export const two = 2;\n")
(self.checkout / "src" / "lib.rs").write_text("pub fn add(a: i32, b: i32) -> i32 {\n\ta + b\n}\n")
(self.checkout / "src" / "Calc.php").write_text("<?php\nfunction add($a, $b) { return $a + $b; }\n")
(self.checkout / "src" / "calc.ts").write_text("export function add(a: number, b: number) {\n\treturn a + b;\n}\nexport const one = 1;\n")
self.commit()
(self.checkout / "src" / "lib.rs").write_text("pub fn add(a: i32, b: i32) -> i32 {\n\tb + a\n}\n")
self.git("add", "src/lib.rs")
(self.checkout / "src" / "Calc.php").write_bytes(b"<?php\nfunction add($a, $b) { return $b + $a; } // caf\xe9\n")
(self.checkout / "src" / "calc.ts").write_text("export function add(b: number, a: number) {\n\treturn a + b;\n}\nexport const one = 2;\n")
(self.checkout / "web" / "src" / "app.ts").write_text("export const two = 3;\n")
def path_with_git(self):
"""A PATH that finds git and no mutation tool."""
tools = self.temp / "tools"
tools.mkdir()
(tools / "git").symlink_to(shutil.which("git"))
return tools
def test_prints_each_mutant_on_a_changed_line_that_its_tests_do_not_kill(self):
ran = self.run_tool(*self.faked("passes"), "--jobs", "2")
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-1], ran.stderr), (0, [*PASSED, *NOT_MUTATED], ""))
self.assertRegex(ran.stdout.splitlines()[-1], r"^mutate-changes took \d+ s\.$")
calls = self.log.read_text()
# mutmut's stats and mutant runs stop at the first test that fails, unless an option tells pytest otherwise.
self.assertNotIn("--maxfail=0", calls)
self.assertRegex(calls, rf"(?m)^{re.escape(str(self.checkout.resolve()))}/\.mutate-changes-\w+/copy$")
self.assertIn("mutmut run --max-children 2 calc.x_double* calc.x_triple*\n", calls)
self.assertIn("do_not_mutate = *_old.py\n", calls)
self.assertIn("source_paths = calc.py\n", calls)
self.assertIn("also_copy = calc.py\n\tfixture.txt\n\tsetup.cfg\n\ttest_calc.py\n", calls)
self.assertEqual((self.checkout / "pkg" / "setup.cfg").read_text(), SETUP)
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "pkg"])
def test_runs_alike_from_a_folder_in_the_checkout(self):
ran = self.run_tool(*self.faked("passes"), cwd=self.checkout / "pkg")
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:2]), (0, PASSED), ran.stderr)
def test_leaves_out_the_tests_that_fail_before_any_mutant_and_names_them(self):
ran = self.run_tool(*self.faked("fails-once"))
self.assertEqual(ran.returncode, 0, ran.stderr)
self.assertEqual(ran.stdout.splitlines()[:2], ["mutmut in pkg: 2 mutants on changed lines, 1 not killed. Left out the tests that fail without a mutant, in test_calc.py, test_io.py.", "pkg/calc.py:6: survived: return n / 3"])
runs = self.log.read_text().split("---\n")[:3]
self.assertEqual(["--maxfail=0" in run for run in runs], [False, True, False])
self.assertIn("--deselect=test_calc.py::test_slow[n=1]", runs[2])
def test_says_where_no_test_reaches_the_mutants(self):
ran = self.run_tool(*self.faked("no-test"))
self.assertEqual(ran.stdout.splitlines()[:3], ["mutmut in pkg: 2 mutants on changed lines, 2 not killed. No test reaches them.", "pkg/calc.py:6: no tests: return n * 4", "pkg/calc.py:6: no tests: return n / 3"], ran.stderr)
def test_a_run_that_stops_early_for_another_reason_fails_the_run(self):
ran = self.run_tool(*self.faked("setup-fault"))
self.assertEqual((ran.returncode, ran.stderr.splitlines()[0]), (1, "mutmut run in pkg exited 1:"))
def test_a_prefix_that_runs_mutmut_elsewhere_fails_the_run(self):
ran = self.run_tool(*self.faked("elsewhere"))
self.assertEqual(ran.returncode, 1)
self.assertRegex(ran.stderr, rf"^mutmut in pkg wrote no mutants/ in {re.escape(str(self.checkout.resolve()))}/\.mutate-changes-\w+/copy; --prefix must run it in \$PWD\n$")
def test_names_a_mutant_whose_line_it_cannot_find(self):
ran = self.run_tool(*self.faked("no-copy"))
self.assertEqual(ran.stdout.splitlines()[:3], ["mutmut in pkg: 3 mutants on changed lines, 2 not killed.", "pkg/calc.py: survived: calc.x_double__mutmut_1, on a line mutate-changes cannot find", "pkg/calc.py: survived: calc.x_triple__mutmut_1, on a line mutate-changes cannot find"], ran.stderr)
def test_a_pyproject_toml_that_sets_mutmut_up_fails_the_run(self):
(self.checkout / "pkg" / "setup.cfg").unlink()
(self.checkout / "pkg" / "pyproject.toml").write_text("[tool.mutmut]\nmax_stack_depth = 8\n")
ran = self.run_tool(*self.faked("passes"))
self.assertEqual(ran.returncode, 1)
self.assertIn("[tool.mutmut]", ran.stderr)
(self.checkout / "pkg" / "pyproject.toml").write_text("[tool.pytest.ini_options]\naddopts = \"-q\"\n")
ran = self.run_tool(*self.faked("passes"))
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:2]), (0, PASSED), ran.stderr)
def test_a_changed_file_or_setting_it_cannot_read_fails_the_run(self):
pkg = self.checkout.resolve() / "pkg"
for name, text, reason in (
("calc.py", "def double(n:\n", f"could not read {pkg}/calc.py: "),
("pyproject.toml", "[tool\n", f"could not read the mutmut settings in {pkg}: "),
("setup.cfg", "[mutmut\n", "pkg: could not read setup.cfg: "),
):
with self.subTest(name):
(pkg / name).write_text(text)
ran = self.run_tool(*self.faked("passes"))
self.assertEqual(ran.returncode, 1)
self.assertRegex(ran.stderr, f"^{re.escape(reason)}")
self.git("checkout", "HEAD", "--", "pkg")
(pkg / "pyproject.toml").unlink(missing_ok=True)
def test_a_scratch_folder_it_cannot_make_fails_the_run(self):
top = self.checkout.resolve()
self.addCleanup(top.chmod, 0o755)
failed = f"^could not make a scratch folder in {re.escape(str(top))}: "
top.chmod(0o555)
self.assertRegex(self.run_tool(*self.faked("passes")).stderr, failed)
top.chmod(0o755)
self.change_rust_php_and_typescript()
top.chmod(0o555)
self.assertRegex(self.run_tool("--base", "HEAD").stderr, failed)
def test_a_file_it_cannot_copy_fails_the_run(self):
(self.checkout / "pkg" / "fixture.txt").chmod(0)
ran = self.run_tool(*self.faked("passes"))
top = re.escape(str(self.checkout.resolve()))
self.assertRegex(ran.stderr, rf"^could not set mutmut up in {top}/\.mutate-changes-\w+, a copy of {top}/pkg: ")
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "pkg"])
def test_a_scratch_folder_it_cannot_remove_is_named_and_the_report_stands(self):
top = re.escape(str(self.checkout.resolve()))
self.addCleanup(lambda: [path.chmod(0o755) for path in self.checkout.glob(".mutate-changes-*/**/*") if path.is_dir()])
ran = self.run_tool(*self.faked("locks"))
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:2]), (0, PASSED))
self.assertRegex(ran.stderr, rf"^could not remove {top}/\.mutate-changes-\w+: .*\n$")
self.change_rust_php_and_typescript()
ran = self.run_tool("--base", "HEAD", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", env={"OUTPUT_LOCKED": "1"})
self.assertEqual((ran.returncode, ran.stdout.splitlines()[0]), (0, "cargo-mutants: 3 mutants on changed lines, 2 not caught."))
self.assertRegex(ran.stderr, rf"^could not remove {top}/\.mutate-changes-\w+: .*\n$")
def test_a_signal_that_ends_the_run_ends_the_tool_and_removes_the_scratch_copy(self):
started = self.temp / "sleep.pid"
for number in (signal.SIGHUP, signal.SIGTERM):
with self.subTest(number.name):
started.unlink(missing_ok=True)
launched = subprocess.Popen([str(TOOL), "--prefix", f"sleep 60 & echo $! > {started}; wait;"], cwd=self.checkout, env=isolated_env(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
while launched.poll() is None and not (started.is_file() and started.read_text().strip()):
time.sleep(0.05)
launched.send_signal(number)
self.assertEqual(launched.wait(timeout=30), 128 + number)
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "pkg"])
for _ in range(200):
if ended(int(started.read_text())):
break
time.sleep(0.05)
else:
self.fail("the sleep the tool started still runs")
def test_a_tool_that_fails_fails_the_run_with_the_end_of_its_output(self):
ran = self.run_tool("--prefix", "seq 45; echo broken >&2; printf '\\377'; false")
self.assertEqual((ran.returncode, ran.stderr), (1, "\n".join(["mutmut run in pkg exited 1:", *map(str, range(8, 46)), "broken", "�"]) + "\n"))
def test_mutmut_results_that_fails_fails_the_run(self):
ran = self.run_tool(*self.faked("results-fail"))
self.assertEqual((ran.returncode, ran.stderr), (1, "mutmut results in pkg exited 1:\nmutmut results broke\n"))
def test_a_tool_gets_an_empty_input_in_place_of_the_terminal(self):
# A tool that reads its input, such as a test that asks a question, gets an end of file instead of waiting on the terminal.
with stdin_holding("y\n"):
ran = self.run_tool("--prefix", "cat; false")
self.assertEqual(ran.stderr, "mutmut run in pkg exited 1:\n")
def test_without_a_prefix_each_tool_runs_from_path(self):
tools = self.path_with_git()
ran = self.run_tool(env={"PATH": str(tools)})
self.assertEqual(ran.returncode, 1)
self.assertRegex(ran.stderr, "^mutmut run .* could not start: ")
(tools / "mutmut").write_text(f'#!/bin/sh\nexec {sys.executable} {self.temp / "fake_mutmut.py"} {self.log} passes mutmut "$@"\n')
(tools / "mutmut").chmod(0o755)
ran = self.run_tool(env={"PATH": str(tools)})
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:2]), (0, PASSED), ran.stderr)
def test_a_mutmut_results_that_cannot_start_fails_the_run(self):
tools = self.path_with_git()
(tools / "mutmut").write_text(f'#!/bin/sh\n[ "$1" = run ] && {shutil.which("rm")} "$0"\nexec {sys.executable} {self.temp / "fake_mutmut.py"} {self.log} passes mutmut "$@"\n')
(tools / "mutmut").chmod(0o755)
ran = self.run_tool(env={"PATH": str(tools)})
self.assertEqual(ran.returncode, 1)
self.assertRegex(ran.stderr, "^mutmut results --all true could not start: ")
def test_rust_php_and_typescript_go_to_cargo_mutants_and_infection_at_the_top_and_to_stryker_by_package(self):
self.change_rust_php_and_typescript()
ran = self.run_tool("--base", "HEAD", "--jobs=3", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", cwd=self.checkout / "src")
report = ["cargo-mutants: 3 mutants on changed lines, 2 not caught.", "src/lib.rs:2:5: missed: replace add(a: i32, b: i32) -> i32 with 0", "src/lib.rs:2:7: timeout: replace + with * in add"]
stryker_reports = ["Stryker in the checkout's top:", "npx reports", "Stryker in web:", "npx reports"]
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-1]), (0, [*report, "Infection in the checkout's top:", "vendor/bin/infection reports", *stryker_reports]), ran.stderr)
cargo, infection, stryker, web = self.log.read_text().split(f"{self.checkout.resolve()}")[1:]
command, diff = cargo.removeprefix("\n").split("\n", 1)
self.assertRegex(command, r"^cargo mutants --in-diff (\.mutate-changes-\w+)/branch\.diff --output \1 --jobs 3$")
self.assertEqual([line for line in diff.splitlines() if line[:1] in "+-"], ["--- a/src/lib.rs", "+++ b/src/lib.rs", "-\ta + b", "+\tb + a"])
self.assertEqual(infection, f"\nvendor/bin/infection --git-diff-lines --git-diff-base={self.git('rev-parse', 'HEAD').strip()} --threads=3 --show-mutations=max --no-progress\n")
self.assertEqual(stryker, "\nnpx --no --package=@stryker-mutator/core stryker run --mutate src/calc.ts:1-1,src/calc.ts:4-4 --concurrency 3 --reporters clear-text\n")
self.assertEqual(web, "/web\nnpx --no --package=@stryker-mutator/core stryker run --mutate src/app.ts:1-1 --concurrency 3 --reporters clear-text\n")
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "package.json", "pkg", "src", "web"])
def test_a_javascript_file_with_no_package_json_above_it_is_named_and_not_mutated(self):
(self.checkout / "presentation").mkdir()
(self.checkout / "presentation" / "render.js").write_text("export const one = 1;\n")
self.git("add", "presentation/render.js")
ran = self.run_tool(*self.faked("passes"))
skipped = "Not mutated, as no package.json is above them: presentation/render.js."
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-1], ran.stderr), (0, [*PASSED, skipped, *NOT_MUTATED], ""))
self.assertNotIn("npx", self.log.read_text())
def test_a_stryker_or_infection_run_that_exits_nonzero_fails_the_run_with_its_whole_output(self):
self.change_rust_php_and_typescript()
self.git("checkout", "HEAD", "--", "src/lib.rs")
# Either exits nonzero on a mutation score below the project's threshold, where its output is the report of what survived.
fails = ["--base", "HEAD", "--prefix", "seq 45; exit 1"]
whole = "".join(f"{n}\n" for n in range(1, 46))
self.assertEqual(self.run_tool(*fails).stderr, f"Infection exited 1:\n{whole}")
self.git("checkout", "HEAD", "--", "src/Calc.php")
self.assertEqual(self.run_tool(*fails).stderr, f"Stryker in the checkout's top exited 1:\n{whole}")
def test_a_changed_file_named_like_an_option_is_mutated(self):
for name in ("-x.py", "-x.rs"):
(self.checkout / name).write_text("")
self.commit()
(self.checkout / "-x.py").write_text("def f():\n\treturn 1\n")
(self.checkout / "-x.rs").write_text("fn f() {}\n")
base = self.git("rev-parse", "HEAD").strip()
git = functools.partial(git_repo.git, env=isolated_env())
self.assertEqual(main.changed_by_tool(str(self.checkout), base, ["-x.py", "-x.rs"], git), {"cargo-mutants": {"-x.rs": frozenset({1})}, "mutmut": {"-x.py": frozenset({1, 2})}})
run = functools.partial(run_tool, prefix=f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", env=isolated_env(), killpg=os.killpg)
ran = run_cargo_mutants(str(self.checkout), base, ["-x.rs"], run, git, 1)
self.assertIsInstance(ran, list)
self.assertIn("+++ b/-x.rs\n", self.log.read_text())
def test_a_tool_that_cannot_start_fails_the_run(self):
self.change_rust_php_and_typescript()
path = str(self.path_with_git())
# The tools run in the order of their names, so undoing the change of the one that failed lets the next one run.
for tool, undone in (("cargo mutants", "src/lib.rs"), ("vendor/bin/infection", "src/Calc.php"), ("npx --no --package=@stryker-mutator/core stryker", "src/calc.ts")):
with self.subTest(tool):
ran = self.run_tool("--base", "HEAD", env={"PATH": path})
self.assertEqual(ran.returncode, 1)
self.assertRegex(ran.stderr, f"^{re.escape(tool)} .* could not start: ")
self.git("checkout", "HEAD", "--", undone)
def test_cargo_mutants_fails_the_run_unless_it_finished(self):
self.change_rust_php_and_typescript()
# The other tools fail on any code but 0.
self.git("checkout", "HEAD", "--", "src/Calc.php", "src/calc.ts", "web/src/app.ts")
for code, failed in ((0, False), (1, True), (2, False), (3, False), (4, True)):
with self.subTest(code):
ran = self.run_tool("--base", "HEAD", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} {code}")
self.assertEqual((ran.returncode, ran.stderr), (1, f"cargo mutants exited {code}:\ncargo reports\n") if failed else (0, ""))
def test_cargo_mutants_outcomes_it_cannot_read_fail_the_run(self):
self.change_rust_php_and_typescript()
ran = self.run_tool("--base", "HEAD", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", env={"CAUGHT_UNREADABLE": "1"})
self.assertEqual(ran.returncode, 1)
self.assertRegex(ran.stderr, rf"^could not hand cargo-mutants the diff or read its outcomes in {re.escape(str(self.checkout.resolve()))}/\.mutate-changes-\w+: ")
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "package.json", "pkg", "src", "web"])
def test_a_branch_that_adds_no_line_a_tool_mutates_says_so_and_names_what_it_changes(self):
# A file the branch deletes is no longer there to mutate, so the report doesn't name it.
(self.checkout / "pkg" / "old.md").write_text("Gone.\n")
self.commit()
(self.checkout / "pkg" / "old.md").unlink()
ran = self.run_tool(*self.faked("passes"), "--base", "HEAD")
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-1]), (0, ["No tool ran: the branch adds no line to a file that a tool mutates."]))
(self.checkout / "README.md").write_text("Doubles.\n")
(self.checkout / "pkg" / "test_calc.py").write_text("")
(self.checkout / "pkg" / "calc.py").write_text("def double(n): # doubles\n\treturn n * 2\n")
ran = self.run_tool(*self.faked("passes"), "--base", "HEAD")
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-1]), (0, ["No tool ran: the branch adds no line to a file that a tool mutates.", *NOT_MUTATED]))
self.assertRegex(ran.stdout.splitlines()[-1], r"^mutate-changes took \d+ s\.$")
self.assertFalse(self.log.exists())
def test_help_prints_the_usage(self):
for flag in ("-h", "--help"):
with self.subTest(flag):
ran = self.run_tool(flag)
self.assertEqual((ran.returncode, ran.stdout.splitlines()[0]), (0, "Mutation-test the lines a branch adds or changes, and print each mutant its tests don't kill."))
def test_unknown_arguments_fail_saying_why(self):
usage = "usage: mutate-changes [--base <ref>] [--prefix <shell text>] [--jobs <n>]"
for args, reason in (
(["--jobs", "0"], "--jobs takes a whole number from 1, not 0"),
(["--jobs", "x"], "--jobs takes a whole number from 1, not x"),
(["--jobs", "²"], "--jobs takes a whole number from 1, not ²"),
(["--jobs"], "--jobs needs a value"),
(["--colour"], "unknown or repeated argument --colour"),
(["--base", "a", "--base", "b"], "unknown or repeated argument --base"),
):
with self.subTest(args):
ran = self.run_tool(*args)
self.assertEqual((ran.returncode, ran.stderr), (1, f"{reason}; {usage}\n"))
def test_the_launcher_runs_the_tool(self):
# Run from the launcher's folder: there, under mutmut, the child finds the setup.cfg that mutmut's mutated code reads.
ran = subprocess.run([str(TOOL), "--help"], capture_output=True, cwd=TOOL.parent, encoding="utf-8", env=isolated_env())
self.assertEqual((ran.returncode, ran.stdout.splitlines()[0]), (0, "Mutation-test the lines a branch adds or changes, and print each mutant its tests don't kill."))
if __name__ == "__main__":
unittest.main()