443 lines
26 KiB
Python
443 lines
26 KiB
Python
"""mutate-changes run on a branch, against fake mutation tools that log how they were called."""
|
|
import contextlib
|
|
import functools
|
|
import io
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from mutate_changes.test_mutate_changes import ended
|
|
from git_cli import git_repo
|
|
from test_env import isolated_env
|
|
from mutate_changes import main
|
|
from mutate_changes.in_diff_tools import run_cargo_mutants
|
|
from mutate_changes.tool_process import run_tool
|
|
|
|
TOOL = pathlib.Path(__file__).resolve().parents[1] / "mutate-changes"
|
|
SETUP = "[mutmut]\ndo_not_mutate = *_old.py\n"
|
|
PASSED = ["mutmut in pkg: 2 mutants on changed lines, 1 not killed.", "pkg/calc.py:6: survived: return n / 3"]
|
|
NOT_MUTATED = ["Not mutated, as no tool covers their language: README.md.", "Not mutated, as test files: pkg/test_calc.py."]
|
|
# Stands in for mutmut, run through --prefix as `python3 <this> <log> <mode> mutmut ...` in the scratch copy. It logs each call with
|
|
# the folder it runs in and the setup.cfg it finds there, then writes a mutants/ copy and prints results, both shaped like mutmut 3's.
|
|
# In mode locks, it also leaves a folder in the copy that nobody may delete from.
|
|
FAKE_MUTMUT = r"""import os, pathlib, sys
|
|
log, mode, args = pathlib.Path(sys.argv[1]), sys.argv[2], sys.argv[3:]
|
|
config = pathlib.Path("setup.cfg").read_text()
|
|
with log.open("a") as written:
|
|
written.write(f"{os.getcwd()}\n{' '.join(args)}\n{config}---\n")
|
|
if args[:2] == ["mutmut", "results"]:
|
|
if mode == "results-fail" or args[2:] != ["--all", "true"]:
|
|
sys.exit("mutmut results broke")
|
|
double, triple_1, triple_2 = ("no tests",) * 3 if mode == "no-test" else ("survived", "survived", "killed")
|
|
# The branch doesn't change aside.py, so mutate-changes skips its mutant, which comes first in name order.
|
|
print(f" aside.x_f__mutmut_1: survived\n calc.x_double__mutmut_1: {double}\n calc.x_triple__mutmut_1: {triple_1}\n calc.x_triple__mutmut_2: {triple_2}")
|
|
elif mode == "fails-once" and not ("--deselect=test_calc.py::test_slow[n=1]" in config and "--ignore=test_io.py" in config):
|
|
print("==== short test summary info ====\nFAILED test_calc.py::test_slow[n=1] - boom\nERROR test_io.py - FileNotFoundError\nfailed to collect stats. runner returned 1")
|
|
sys.exit(1)
|
|
else:
|
|
if mode != "elsewhere":
|
|
pathlib.Path("mutants").mkdir(exist_ok=True)
|
|
if mode not in ("elsewhere", "no-copy"):
|
|
functions = ["x_double__mutmut_orig(n): # doubles\n\treturn n * 2", "x_double__mutmut_1(n): # doubles\n\treturn n / 2", "x_triple__mutmut_orig(n):\n\treturn n * 3", "x_triple__mutmut_1(n):\n\treturn n / 3", "x_triple__mutmut_2(n):\n\treturn n * 4"]
|
|
pathlib.Path("mutants/calc.py").write_bytes(b"# \xff is not UTF-8\n" + "".join(f"def {function}\n" for function in functions).encode())
|
|
if mode == "locks":
|
|
pathlib.Path("locked/inner").mkdir(parents=True)
|
|
pathlib.Path("locked").chmod(0o555)
|
|
if mode == "no-test":
|
|
print("Stopping early, because we could not find any test case for any mutant. It seems that the selected tests do not cover any code that we mutated.")
|
|
sys.exit(1)
|
|
if mode == "setup-fault":
|
|
print("Stopping early, because tests recorded trampoline hits but none match any mutant key. This usually means imports bypass the mutants.")
|
|
sys.exit(1)
|
|
"""
|
|
# cargo-mutants, Infection and Stryker, which log the folder they run in and how they were called, and exit with the code the test
|
|
# gives. cargo-mutants also logs the diff it is given and writes its outcome files, all but unviable.txt, as a run without an
|
|
# unviable mutant does; with CAUGHT_UNREADABLE set, nobody may read caught.txt, and with OUTPUT_LOCKED, nobody may delete from mutants.out.
|
|
FAKE_TOOLS = r"""import os, pathlib, sys
|
|
log, code, args = pathlib.Path(sys.argv[1]), int(sys.argv[2]), sys.argv[3:]
|
|
with log.open("a") as written:
|
|
written.write(f"{os.getcwd()}\n{' '.join(args)}\n")
|
|
if args[0] == "cargo":
|
|
written.write(pathlib.Path(args[args.index("--in-diff") + 1]).read_text())
|
|
outcomes = pathlib.Path(args[args.index("--output") + 1], "mutants.out")
|
|
outcomes.mkdir()
|
|
(outcomes / "caught.txt").write_bytes(b"src/lib.rs:2:7: replace + with - in add, \xff not being UTF-8\n")
|
|
(outcomes / "missed.txt").write_text("src/lib.rs:2:5: replace add(a: i32, b: i32) -> i32 with 0\n")
|
|
(outcomes / "timeout.txt").write_text("src/lib.rs:2:7: replace + with * in add\n")
|
|
if os.environ.get("CAUGHT_UNREADABLE"):
|
|
(outcomes / "caught.txt").chmod(0)
|
|
if os.environ.get("OUTPUT_LOCKED"):
|
|
outcomes.chmod(0o555)
|
|
print(f"{args[0]} reports")
|
|
sys.exit(code)
|
|
"""
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def stdin_holding(text: str):
|
|
"""This process's stdin, the one a child inherits, as a pipe holding text."""
|
|
read, write = os.pipe()
|
|
os.write(write, text.encode())
|
|
os.close(write)
|
|
saved = os.dup(0)
|
|
os.dup2(read, 0)
|
|
os.close(read)
|
|
try:
|
|
yield
|
|
finally:
|
|
os.dup2(saved, 0)
|
|
os.close(saved)
|
|
|
|
|
|
class MutateChangesRuns(unittest.TestCase):
|
|
def setUp(self):
|
|
temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temp.cleanup)
|
|
self.temp = pathlib.Path(temp.name)
|
|
self.checkout = self.temp / "proj"
|
|
(self.checkout / "pkg").mkdir(parents=True)
|
|
(self.checkout / "pkg" / "setup.cfg").write_text(SETUP)
|
|
(self.checkout / "pkg" / "calc.py").write_text("def double(n):\n\treturn n * 2\n")
|
|
(self.checkout / "pkg" / "test_calc.py").write_text("")
|
|
self.git("init", "-q", "-b", "main")
|
|
self.commit()
|
|
self.git("checkout", "-q", "-b", "work")
|
|
# git lists the README first, and mutate-changes mutates no Markdown.
|
|
(self.checkout / "README.md").write_text("Doubles and triples.\n")
|
|
(self.checkout / "pkg" / "calc.py").write_text("def double(n): # doubles\n\treturn n * 2\n\n\ndef triple(n):\n\treturn n * 3\n")
|
|
(self.checkout / "pkg" / "test_calc.py").write_text("# changed, but a test file is not mutated\n")
|
|
self.commit()
|
|
# Settings a user may have that change what git diff prints, and a file git lists as untracked and one it ignores.
|
|
for name, value in (("color.diff", "always"), ("diff.mnemonicPrefix", "true"), ("diff.relative", "true"), ("diff.external", "true"), ("diff.blank.textconv", "true")):
|
|
self.git("config", name, value)
|
|
(self.checkout / ".git" / "info").mkdir(exist_ok=True)
|
|
(self.checkout / ".git" / "info" / "attributes").write_text("* diff=blank\n")
|
|
(self.checkout / ".git" / "info" / "exclude").write_text("*.log\n")
|
|
(self.checkout / "pkg" / "fixture.txt").write_text("")
|
|
(self.checkout / "pkg" / "run.log").write_text("")
|
|
(self.temp / "fake_mutmut.py").write_text(FAKE_MUTMUT)
|
|
(self.temp / "fake_tools.py").write_text(FAKE_TOOLS)
|
|
self.log = self.temp / "calls.log"
|
|
|
|
def git(self, *args):
|
|
return subprocess.run(["git", *args], check=True, capture_output=True, cwd=self.checkout, encoding="utf-8", env={**isolated_env(), "GIT_CONFIG_GLOBAL": "/dev/null"}).stdout
|
|
|
|
def commit(self):
|
|
self.git("add", ".")
|
|
self.git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "x")
|
|
|
|
def run_tool(self, *args, cwd=None, env=None):
|
|
"""Run mutate-changes in this process, from cwd, else the checkout: mutmut counts only the code a test reaches in its own process."""
|
|
before = os.getcwd()
|
|
os.chdir(cwd or self.checkout)
|
|
try:
|
|
with contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()) as err:
|
|
code = main.main(list(args), {**isolated_env(), **(env or {})})
|
|
finally:
|
|
os.chdir(before)
|
|
return subprocess.CompletedProcess(args, code, out.getvalue(), err.getvalue())
|
|
|
|
def faked(self, mode):
|
|
return ["--prefix", f"python3 {self.temp / 'fake_mutmut.py'} {self.log} {mode}"]
|
|
|
|
def change_rust_php_and_typescript(self):
|
|
"""Commit a Rust, a PHP and a TypeScript file, then change them: Rust's line 2 in the index, PHP's line 2 with a byte that is
|
|
not UTF-8, and TypeScript's lines 1 and 4. Also change line 1 of a TypeScript file in web/. The top and web/ each hold a package.json."""
|
|
(self.checkout / "src").mkdir()
|
|
(self.checkout / "web" / "src").mkdir(parents=True)
|
|
(self.checkout / "package.json").write_text("{}\n")
|
|
(self.checkout / "web" / "package.json").write_text("{}\n")
|
|
(self.checkout / "web" / "src" / "app.ts").write_text("export const two = 2;\n")
|
|
(self.checkout / "src" / "lib.rs").write_text("pub fn add(a: i32, b: i32) -> i32 {\n\ta + b\n}\n")
|
|
(self.checkout / "src" / "Calc.php").write_text("<?php\nfunction add($a, $b) { return $a + $b; }\n")
|
|
(self.checkout / "src" / "calc.ts").write_text("export function add(a: number, b: number) {\n\treturn a + b;\n}\nexport const one = 1;\n")
|
|
self.commit()
|
|
(self.checkout / "src" / "lib.rs").write_text("pub fn add(a: i32, b: i32) -> i32 {\n\tb + a\n}\n")
|
|
self.git("add", "src/lib.rs")
|
|
(self.checkout / "src" / "Calc.php").write_bytes(b"<?php\nfunction add($a, $b) { return $b + $a; } // caf\xe9\n")
|
|
(self.checkout / "src" / "calc.ts").write_text("export function add(b: number, a: number) {\n\treturn a + b;\n}\nexport const one = 2;\n")
|
|
(self.checkout / "web" / "src" / "app.ts").write_text("export const two = 3;\n")
|
|
|
|
def path_with_git(self):
|
|
"""A PATH that finds git and no mutation tool."""
|
|
tools = self.temp / "tools"
|
|
tools.mkdir()
|
|
(tools / "git").symlink_to(shutil.which("git"))
|
|
return tools
|
|
|
|
def test_prints_each_mutant_on_a_changed_line_that_its_tests_do_not_kill(self):
|
|
ran = self.run_tool(*self.faked("passes"), "--jobs", "2")
|
|
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-1], ran.stderr), (0, [*PASSED, *NOT_MUTATED], ""))
|
|
self.assertRegex(ran.stdout.splitlines()[-1], r"^mutate-changes took \d+ s\.$")
|
|
calls = self.log.read_text()
|
|
# mutmut's stats and mutant runs stop at the first test that fails, unless an option tells pytest otherwise.
|
|
self.assertNotIn("--maxfail=0", calls)
|
|
self.assertRegex(calls, rf"(?m)^{re.escape(str(self.checkout.resolve()))}/\.mutate-changes-\w+/copy$")
|
|
self.assertIn("mutmut run --max-children 2 calc.x_double* calc.x_triple*\n", calls)
|
|
self.assertIn("do_not_mutate = *_old.py\n", calls)
|
|
self.assertIn("source_paths = calc.py\n", calls)
|
|
self.assertIn("also_copy = calc.py\n\tfixture.txt\n\tsetup.cfg\n\ttest_calc.py\n", calls)
|
|
self.assertEqual((self.checkout / "pkg" / "setup.cfg").read_text(), SETUP)
|
|
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "pkg"])
|
|
|
|
def test_runs_alike_from_a_folder_in_the_checkout(self):
|
|
ran = self.run_tool(*self.faked("passes"), cwd=self.checkout / "pkg")
|
|
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:2]), (0, PASSED), ran.stderr)
|
|
|
|
def test_leaves_out_the_tests_that_fail_before_any_mutant_and_names_them(self):
|
|
ran = self.run_tool(*self.faked("fails-once"))
|
|
self.assertEqual(ran.returncode, 0, ran.stderr)
|
|
self.assertEqual(ran.stdout.splitlines()[:2], ["mutmut in pkg: 2 mutants on changed lines, 1 not killed. Left out the tests that fail without a mutant, in test_calc.py, test_io.py.", "pkg/calc.py:6: survived: return n / 3"])
|
|
runs = self.log.read_text().split("---\n")[:3]
|
|
self.assertEqual(["--maxfail=0" in run for run in runs], [False, True, False])
|
|
self.assertIn("--deselect=test_calc.py::test_slow[n=1]", runs[2])
|
|
|
|
def test_says_where_no_test_reaches_the_mutants(self):
|
|
ran = self.run_tool(*self.faked("no-test"))
|
|
self.assertEqual(ran.stdout.splitlines()[:3], ["mutmut in pkg: 2 mutants on changed lines, 2 not killed. No test reaches them.", "pkg/calc.py:6: no tests: return n * 4", "pkg/calc.py:6: no tests: return n / 3"], ran.stderr)
|
|
|
|
def test_a_run_that_stops_early_for_another_reason_fails_the_run(self):
|
|
ran = self.run_tool(*self.faked("setup-fault"))
|
|
self.assertEqual((ran.returncode, ran.stderr.splitlines()[0]), (1, "mutmut run in pkg exited 1:"))
|
|
|
|
def test_a_prefix_that_runs_mutmut_elsewhere_fails_the_run(self):
|
|
ran = self.run_tool(*self.faked("elsewhere"))
|
|
self.assertEqual(ran.returncode, 1)
|
|
self.assertRegex(ran.stderr, rf"^mutmut in pkg wrote no mutants/ in {re.escape(str(self.checkout.resolve()))}/\.mutate-changes-\w+/copy; --prefix must run it in \$PWD\n$")
|
|
|
|
def test_names_a_mutant_whose_line_it_cannot_find(self):
|
|
ran = self.run_tool(*self.faked("no-copy"))
|
|
self.assertEqual(ran.stdout.splitlines()[:3], ["mutmut in pkg: 3 mutants on changed lines, 2 not killed.", "pkg/calc.py: survived: calc.x_double__mutmut_1, on a line mutate-changes cannot find", "pkg/calc.py: survived: calc.x_triple__mutmut_1, on a line mutate-changes cannot find"], ran.stderr)
|
|
|
|
def test_a_pyproject_toml_that_sets_mutmut_up_fails_the_run(self):
|
|
(self.checkout / "pkg" / "setup.cfg").unlink()
|
|
(self.checkout / "pkg" / "pyproject.toml").write_text("[tool.mutmut]\nmax_stack_depth = 8\n")
|
|
ran = self.run_tool(*self.faked("passes"))
|
|
self.assertEqual(ran.returncode, 1)
|
|
self.assertIn("[tool.mutmut]", ran.stderr)
|
|
(self.checkout / "pkg" / "pyproject.toml").write_text("[tool.pytest.ini_options]\naddopts = \"-q\"\n")
|
|
ran = self.run_tool(*self.faked("passes"))
|
|
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:2]), (0, PASSED), ran.stderr)
|
|
|
|
def test_a_changed_file_or_setting_it_cannot_read_fails_the_run(self):
|
|
pkg = self.checkout.resolve() / "pkg"
|
|
for name, text, reason in (
|
|
("calc.py", "def double(n:\n", f"could not read {pkg}/calc.py: "),
|
|
("pyproject.toml", "[tool\n", f"could not read the mutmut settings in {pkg}: "),
|
|
("setup.cfg", "[mutmut\n", "pkg: could not read setup.cfg: "),
|
|
):
|
|
with self.subTest(name):
|
|
(pkg / name).write_text(text)
|
|
ran = self.run_tool(*self.faked("passes"))
|
|
self.assertEqual(ran.returncode, 1)
|
|
self.assertRegex(ran.stderr, f"^{re.escape(reason)}")
|
|
self.git("checkout", "HEAD", "--", "pkg")
|
|
(pkg / "pyproject.toml").unlink(missing_ok=True)
|
|
|
|
def test_a_scratch_folder_it_cannot_make_fails_the_run(self):
|
|
top = self.checkout.resolve()
|
|
self.addCleanup(top.chmod, 0o755)
|
|
failed = f"^could not make a scratch folder in {re.escape(str(top))}: "
|
|
top.chmod(0o555)
|
|
self.assertRegex(self.run_tool(*self.faked("passes")).stderr, failed)
|
|
top.chmod(0o755)
|
|
self.change_rust_php_and_typescript()
|
|
top.chmod(0o555)
|
|
self.assertRegex(self.run_tool("--base", "HEAD").stderr, failed)
|
|
|
|
def test_a_file_it_cannot_copy_fails_the_run(self):
|
|
(self.checkout / "pkg" / "fixture.txt").chmod(0)
|
|
ran = self.run_tool(*self.faked("passes"))
|
|
top = re.escape(str(self.checkout.resolve()))
|
|
self.assertRegex(ran.stderr, rf"^could not set mutmut up in {top}/\.mutate-changes-\w+, a copy of {top}/pkg: ")
|
|
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "pkg"])
|
|
|
|
def test_a_scratch_folder_it_cannot_remove_is_named_and_the_report_stands(self):
|
|
top = re.escape(str(self.checkout.resolve()))
|
|
self.addCleanup(lambda: [path.chmod(0o755) for path in self.checkout.glob(".mutate-changes-*/**/*") if path.is_dir()])
|
|
ran = self.run_tool(*self.faked("locks"))
|
|
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:2]), (0, PASSED))
|
|
self.assertRegex(ran.stderr, rf"^could not remove {top}/\.mutate-changes-\w+: .*\n$")
|
|
self.change_rust_php_and_typescript()
|
|
ran = self.run_tool("--base", "HEAD", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", env={"OUTPUT_LOCKED": "1"})
|
|
self.assertEqual((ran.returncode, ran.stdout.splitlines()[0]), (0, "cargo-mutants: 3 mutants on changed lines, 2 not caught."))
|
|
self.assertRegex(ran.stderr, rf"^could not remove {top}/\.mutate-changes-\w+: .*\n$")
|
|
|
|
def test_a_signal_that_ends_the_run_ends_the_tool_and_removes_the_scratch_copy(self):
|
|
started = self.temp / "sleep.pid"
|
|
for number in (signal.SIGHUP, signal.SIGTERM):
|
|
with self.subTest(number.name):
|
|
started.unlink(missing_ok=True)
|
|
launched = subprocess.Popen([str(TOOL), "--prefix", f"sleep 60 & echo $! > {started}; wait;"], cwd=self.checkout, env=isolated_env(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
while launched.poll() is None and not (started.is_file() and started.read_text().strip()):
|
|
time.sleep(0.05)
|
|
launched.send_signal(number)
|
|
self.assertEqual(launched.wait(timeout=30), 128 + number)
|
|
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "pkg"])
|
|
for _ in range(200):
|
|
if ended(int(started.read_text())):
|
|
break
|
|
time.sleep(0.05)
|
|
else:
|
|
self.fail("the sleep the tool started still runs")
|
|
|
|
def test_a_tool_that_fails_fails_the_run_with_the_end_of_its_output(self):
|
|
ran = self.run_tool("--prefix", "seq 45; echo broken >&2; printf '\\377'; false")
|
|
self.assertEqual((ran.returncode, ran.stderr), (1, "\n".join(["mutmut run in pkg exited 1:", *map(str, range(8, 46)), "broken", "�"]) + "\n"))
|
|
|
|
def test_mutmut_results_that_fails_fails_the_run(self):
|
|
ran = self.run_tool(*self.faked("results-fail"))
|
|
self.assertEqual((ran.returncode, ran.stderr), (1, "mutmut results in pkg exited 1:\nmutmut results broke\n"))
|
|
|
|
def test_a_tool_gets_an_empty_input_in_place_of_the_terminal(self):
|
|
# A tool that reads its input, such as a test that asks a question, gets an end of file instead of waiting on the terminal.
|
|
with stdin_holding("y\n"):
|
|
ran = self.run_tool("--prefix", "cat; false")
|
|
self.assertEqual(ran.stderr, "mutmut run in pkg exited 1:\n")
|
|
|
|
def test_without_a_prefix_each_tool_runs_from_path(self):
|
|
tools = self.path_with_git()
|
|
ran = self.run_tool(env={"PATH": str(tools)})
|
|
self.assertEqual(ran.returncode, 1)
|
|
self.assertRegex(ran.stderr, "^mutmut run .* could not start: ")
|
|
(tools / "mutmut").write_text(f'#!/bin/sh\nexec {sys.executable} {self.temp / "fake_mutmut.py"} {self.log} passes mutmut "$@"\n')
|
|
(tools / "mutmut").chmod(0o755)
|
|
ran = self.run_tool(env={"PATH": str(tools)})
|
|
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:2]), (0, PASSED), ran.stderr)
|
|
|
|
def test_a_mutmut_results_that_cannot_start_fails_the_run(self):
|
|
tools = self.path_with_git()
|
|
(tools / "mutmut").write_text(f'#!/bin/sh\n[ "$1" = run ] && {shutil.which("rm")} "$0"\nexec {sys.executable} {self.temp / "fake_mutmut.py"} {self.log} passes mutmut "$@"\n')
|
|
(tools / "mutmut").chmod(0o755)
|
|
ran = self.run_tool(env={"PATH": str(tools)})
|
|
self.assertEqual(ran.returncode, 1)
|
|
self.assertRegex(ran.stderr, "^mutmut results --all true could not start: ")
|
|
|
|
def test_rust_php_and_typescript_go_to_cargo_mutants_and_infection_at_the_top_and_to_stryker_by_package(self):
|
|
self.change_rust_php_and_typescript()
|
|
ran = self.run_tool("--base", "HEAD", "--jobs=3", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", cwd=self.checkout / "src")
|
|
report = ["cargo-mutants: 3 mutants on changed lines, 2 not caught.", "src/lib.rs:2:5: missed: replace add(a: i32, b: i32) -> i32 with 0", "src/lib.rs:2:7: timeout: replace + with * in add"]
|
|
stryker_reports = ["Stryker in the checkout's top:", "npx reports", "Stryker in web:", "npx reports"]
|
|
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-1]), (0, [*report, "Infection in the checkout's top:", "vendor/bin/infection reports", *stryker_reports]), ran.stderr)
|
|
cargo, infection, stryker, web = self.log.read_text().split(f"{self.checkout.resolve()}")[1:]
|
|
command, diff = cargo.removeprefix("\n").split("\n", 1)
|
|
self.assertRegex(command, r"^cargo mutants --in-diff (\.mutate-changes-\w+)/branch\.diff --output \1 --jobs 3$")
|
|
self.assertEqual([line for line in diff.splitlines() if line[:1] in "+-"], ["--- a/src/lib.rs", "+++ b/src/lib.rs", "-\ta + b", "+\tb + a"])
|
|
self.assertEqual(infection, f"\nvendor/bin/infection --git-diff-lines --git-diff-base={self.git('rev-parse', 'HEAD').strip()} --threads=3 --show-mutations=max --no-progress\n")
|
|
self.assertEqual(stryker, "\nnpx --no --package=@stryker-mutator/core stryker run --mutate src/calc.ts:1-1,src/calc.ts:4-4 --concurrency 3 --reporters clear-text\n")
|
|
self.assertEqual(web, "/web\nnpx --no --package=@stryker-mutator/core stryker run --mutate src/app.ts:1-1 --concurrency 3 --reporters clear-text\n")
|
|
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "package.json", "pkg", "src", "web"])
|
|
|
|
def test_a_javascript_file_with_no_package_json_above_it_is_named_and_not_mutated(self):
|
|
(self.checkout / "presentation").mkdir()
|
|
(self.checkout / "presentation" / "render.js").write_text("export const one = 1;\n")
|
|
self.git("add", "presentation/render.js")
|
|
ran = self.run_tool(*self.faked("passes"))
|
|
skipped = "Not mutated, as no package.json is above them: presentation/render.js."
|
|
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-1], ran.stderr), (0, [*PASSED, skipped, *NOT_MUTATED], ""))
|
|
self.assertNotIn("npx", self.log.read_text())
|
|
|
|
def test_a_stryker_or_infection_run_that_exits_nonzero_fails_the_run_with_its_whole_output(self):
|
|
self.change_rust_php_and_typescript()
|
|
self.git("checkout", "HEAD", "--", "src/lib.rs")
|
|
# Either exits nonzero on a mutation score below the project's threshold, where its output is the report of what survived.
|
|
fails = ["--base", "HEAD", "--prefix", "seq 45; exit 1"]
|
|
whole = "".join(f"{n}\n" for n in range(1, 46))
|
|
self.assertEqual(self.run_tool(*fails).stderr, f"Infection exited 1:\n{whole}")
|
|
self.git("checkout", "HEAD", "--", "src/Calc.php")
|
|
self.assertEqual(self.run_tool(*fails).stderr, f"Stryker in the checkout's top exited 1:\n{whole}")
|
|
|
|
def test_a_changed_file_named_like_an_option_is_mutated(self):
|
|
for name in ("-x.py", "-x.rs"):
|
|
(self.checkout / name).write_text("")
|
|
self.commit()
|
|
(self.checkout / "-x.py").write_text("def f():\n\treturn 1\n")
|
|
(self.checkout / "-x.rs").write_text("fn f() {}\n")
|
|
base = self.git("rev-parse", "HEAD").strip()
|
|
git = functools.partial(git_repo.git, env=isolated_env())
|
|
self.assertEqual(main.changed_by_tool(str(self.checkout), base, ["-x.py", "-x.rs"], git), {"cargo-mutants": {"-x.rs": frozenset({1})}, "mutmut": {"-x.py": frozenset({1, 2})}})
|
|
run = functools.partial(run_tool, prefix=f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", env=isolated_env(), killpg=os.killpg)
|
|
ran = run_cargo_mutants(str(self.checkout), base, ["-x.rs"], run, git, 1)
|
|
self.assertIsInstance(ran, list)
|
|
self.assertIn("+++ b/-x.rs\n", self.log.read_text())
|
|
|
|
def test_a_tool_that_cannot_start_fails_the_run(self):
|
|
self.change_rust_php_and_typescript()
|
|
path = str(self.path_with_git())
|
|
# The tools run in the order of their names, so undoing the change of the one that failed lets the next one run.
|
|
for tool, undone in (("cargo mutants", "src/lib.rs"), ("vendor/bin/infection", "src/Calc.php"), ("npx --no --package=@stryker-mutator/core stryker", "src/calc.ts")):
|
|
with self.subTest(tool):
|
|
ran = self.run_tool("--base", "HEAD", env={"PATH": path})
|
|
self.assertEqual(ran.returncode, 1)
|
|
self.assertRegex(ran.stderr, f"^{re.escape(tool)} .* could not start: ")
|
|
self.git("checkout", "HEAD", "--", undone)
|
|
|
|
def test_cargo_mutants_fails_the_run_unless_it_finished(self):
|
|
self.change_rust_php_and_typescript()
|
|
# The other tools fail on any code but 0.
|
|
self.git("checkout", "HEAD", "--", "src/Calc.php", "src/calc.ts", "web/src/app.ts")
|
|
for code, failed in ((0, False), (1, True), (2, False), (3, False), (4, True)):
|
|
with self.subTest(code):
|
|
ran = self.run_tool("--base", "HEAD", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} {code}")
|
|
self.assertEqual((ran.returncode, ran.stderr), (1, f"cargo mutants exited {code}:\ncargo reports\n") if failed else (0, ""))
|
|
|
|
def test_cargo_mutants_outcomes_it_cannot_read_fail_the_run(self):
|
|
self.change_rust_php_and_typescript()
|
|
ran = self.run_tool("--base", "HEAD", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", env={"CAUGHT_UNREADABLE": "1"})
|
|
self.assertEqual(ran.returncode, 1)
|
|
self.assertRegex(ran.stderr, rf"^could not hand cargo-mutants the diff or read its outcomes in {re.escape(str(self.checkout.resolve()))}/\.mutate-changes-\w+: ")
|
|
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "package.json", "pkg", "src", "web"])
|
|
|
|
def test_a_branch_that_adds_no_line_a_tool_mutates_says_so_and_names_what_it_changes(self):
|
|
# A file the branch deletes is no longer there to mutate, so the report doesn't name it.
|
|
(self.checkout / "pkg" / "old.md").write_text("Gone.\n")
|
|
self.commit()
|
|
(self.checkout / "pkg" / "old.md").unlink()
|
|
ran = self.run_tool(*self.faked("passes"), "--base", "HEAD")
|
|
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-1]), (0, ["No tool ran: the branch adds no line to a file that a tool mutates."]))
|
|
(self.checkout / "README.md").write_text("Doubles.\n")
|
|
(self.checkout / "pkg" / "test_calc.py").write_text("")
|
|
(self.checkout / "pkg" / "calc.py").write_text("def double(n): # doubles\n\treturn n * 2\n")
|
|
ran = self.run_tool(*self.faked("passes"), "--base", "HEAD")
|
|
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-1]), (0, ["No tool ran: the branch adds no line to a file that a tool mutates.", *NOT_MUTATED]))
|
|
self.assertRegex(ran.stdout.splitlines()[-1], r"^mutate-changes took \d+ s\.$")
|
|
self.assertFalse(self.log.exists())
|
|
|
|
def test_help_prints_the_usage(self):
|
|
for flag in ("-h", "--help"):
|
|
with self.subTest(flag):
|
|
ran = self.run_tool(flag)
|
|
self.assertEqual((ran.returncode, ran.stdout.splitlines()[0]), (0, "Mutation-test the lines a branch adds or changes, and print each mutant its tests don't kill."))
|
|
|
|
def test_unknown_arguments_fail_saying_why(self):
|
|
usage = "usage: mutate-changes [--base <ref>] [--prefix <shell text>] [--jobs <n>]"
|
|
for args, reason in (
|
|
(["--jobs", "0"], "--jobs takes a whole number from 1, not 0"),
|
|
(["--jobs", "x"], "--jobs takes a whole number from 1, not x"),
|
|
(["--jobs", "²"], "--jobs takes a whole number from 1, not ²"),
|
|
(["--jobs"], "--jobs needs a value"),
|
|
(["--colour"], "unknown or repeated argument --colour"),
|
|
(["--base", "a", "--base", "b"], "unknown or repeated argument --base"),
|
|
):
|
|
with self.subTest(args):
|
|
ran = self.run_tool(*args)
|
|
self.assertEqual((ran.returncode, ran.stderr), (1, f"{reason}; {usage}\n"))
|
|
|
|
def test_the_launcher_runs_the_tool(self):
|
|
# Run from the launcher's folder: there, under mutmut, the child finds the setup.cfg that mutmut's mutated code reads.
|
|
ran = subprocess.run([str(TOOL), "--help"], capture_output=True, cwd=TOOL.parent, encoding="utf-8", env=isolated_env())
|
|
self.assertEqual((ran.returncode, ran.stdout.splitlines()[0]), (0, "Mutation-test the lines a branch adds or changes, and print each mutant its tests don't kill."))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|