232 lines
11 KiB
Python
232 lines
11 KiB
Python
import configparser
|
|
import pathlib
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import textwrap
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from mutate_changes.changed_lines import added_lines, runs, tool_for
|
|
from failure import Failure
|
|
from mutate_changes.mutmut_mutants import function_keys, mutant_globs, mutated_line, named, statuses
|
|
from test_env import isolated_env
|
|
from mutate_changes.mutmut_setup import left_out, mutmut_config
|
|
|
|
TOOL = pathlib.Path(__file__).resolve().parents[1] / "mutate-changes"
|
|
SOURCE = "def double(n):\n\treturn n * 2\n\n\nclass Pt:\n\tdef moved(self, by=1):\n\t\treturn self.x + by\n"
|
|
# The shape of the file mutmut 3 writes under mutants/: the original of each function and its mutants, with the original's lines.
|
|
COPY = """from inspect import signature as _mutmut_signature
|
|
|
|
|
|
def x_double__mutmut_orig(n):
|
|
return n * 2
|
|
def x_double__mutmut_1(n):
|
|
return n / 2
|
|
|
|
|
|
class Pt:
|
|
def xǁPtǁmoved__mutmut_orig(self, by=1):
|
|
return self.x + by
|
|
def xǁPtǁmoved__mutmut_1(self, by=2):
|
|
return self.x + by
|
|
def xǁPtǁmoved__mutmut_2(self, by=1):
|
|
return self.x - by
|
|
"""
|
|
|
|
|
|
def read_as_mutmut(text: str, key: str) -> list[str]:
|
|
"""A [mutmut] list as mutmut 3 reads it from setup.cfg: with configparser's default interpolation, one item per line."""
|
|
config = configparser.ConfigParser()
|
|
config.read_string(text)
|
|
return [item for item in config.get("mutmut", key).split("\n") if item]
|
|
|
|
|
|
class ChangedLines(unittest.TestCase):
|
|
def test_added_lines_are_the_new_side_of_each_hunk_and_a_deleting_hunk_adds_none(self):
|
|
diff = "diff --git a/x.py b/x.py\n--- a/x.py\n+++ b/x.py\n@@ -3,0 +4,2 @@ def f():\n+a\n+b\n@@ -10 +12 @@\n-c\n+d\n@@ -20,3 +23,0 @@\n-e\n-f\n-g\n"
|
|
self.assertEqual(added_lines(diff), {4, 5, 12})
|
|
|
|
def test_runs_join_consecutive_lines(self):
|
|
self.assertEqual(runs({10, 1, 2, 3, 7, 9}), [(1, 3), (7, 7), (9, 10)])
|
|
|
|
def test_the_tool_follows_the_extension_and_test_files_get_none(self):
|
|
expected = {
|
|
"bin/x/main.py": "mutmut",
|
|
"src/lib.rs": "cargo-mutants",
|
|
"src/Order.php": "infection",
|
|
"web/app.tsx": "stryker",
|
|
"web/app.mjs": "stryker",
|
|
"bin/x/test_main.py": None,
|
|
"bin/x/main_test.py": None,
|
|
"pkg/conftest.py": None,
|
|
"tests/helpers.py": None,
|
|
"web/app.test.ts": None,
|
|
"web/app.spec.js": None,
|
|
"web/__tests__/a.js": None,
|
|
"README.md": None,
|
|
}
|
|
for path, tool in expected.items():
|
|
with self.subTest(path):
|
|
self.assertEqual(tool_for(path), tool)
|
|
|
|
|
|
class MutmutSetup(unittest.TestCase):
|
|
def test_the_config_mutates_the_sources_and_copies_the_whole_folder(self):
|
|
text = mutmut_config("", ["x/a.py", "b.py"], [])
|
|
self.assertEqual(read_as_mutmut(text, "source_paths"), ["x/a.py", "b.py"])
|
|
self.assertEqual(read_as_mutmut(text, "also_copy"), ["."])
|
|
|
|
def test_the_config_keeps_the_projects_settings_and_appends_the_tests_left_out(self):
|
|
existing = "[tool:pytest]\naddopts = -q\n\n[mutmut]\npytest_add_cli_args = -k\n not Slow\ndebug = true\n"
|
|
text = mutmut_config(existing, ["a.py"], ["--deselect=t/test_a.py::T::test_b[50%]"])
|
|
self.assertEqual(read_as_mutmut(text, "pytest_add_cli_args"), ["-k", "not Slow", "--deselect=t/test_a.py::T::test_b[50%]"])
|
|
self.assertEqual(read_as_mutmut(text, "debug"), ["true"])
|
|
config = configparser.ConfigParser()
|
|
config.read_string(text)
|
|
self.assertEqual(config.get("tool:pytest", "addopts"), "-q")
|
|
|
|
def test_a_setup_cfg_that_cannot_be_read_fails(self):
|
|
self.assertIsInstance(mutmut_config("[mutmut]\na = 1\na = 2\n", ["a.py"], []), Failure)
|
|
|
|
def test_left_out_names_each_test_the_summary_reports_failing(self):
|
|
output = textwrap.dedent("""\
|
|
FAILED fake.py::t - this line is a test's own output, above the summary
|
|
ERROR root:x.py:3 a log line
|
|
=========================== short test summary info ============================
|
|
FAILED repo_checks/test_repo_checks.py::Repo::test_a - AssertionError: x - y
|
|
ERROR larv_review/test_larv_review.py - FileNotFoundError: SKILL.md
|
|
FAILED a/test_b.py::T::test_c
|
|
2 failed, 1 error in 3.10s
|
|
failed to collect stats. runner returned 1
|
|
""")
|
|
self.assertEqual(left_out(output), ["--deselect=repo_checks/test_repo_checks.py::Repo::test_a", "--ignore=larv_review/test_larv_review.py", "--deselect=a/test_b.py::T::test_c"])
|
|
|
|
def test_left_out_is_empty_without_a_summary(self):
|
|
self.assertEqual(left_out("failed to collect stats. runner returned 1\n"), [])
|
|
|
|
|
|
class MutmutMutants(unittest.TestCase):
|
|
def test_function_keys_name_the_functions_and_methods_a_changed_line_falls_in(self):
|
|
source = SOURCE + "\n\n@cached\ndef later():\n\treturn 1\n"
|
|
self.assertEqual(function_keys(source, frozenset({2, 7})), ["x_double", "xǁPtǁmoved"])
|
|
self.assertEqual(function_keys(source, frozenset({10})), ["x_later"])
|
|
self.assertEqual(function_keys(source, frozenset({3, 5})), [])
|
|
|
|
def test_mutant_globs_follow_mutmuts_module_names(self):
|
|
self.assertEqual(mutant_globs("pkg/__init__.py", ["x_f"]), ["pkg.x_f*"])
|
|
self.assertEqual(mutant_globs("src/pkg/mod.py", ["x_f", "xǁPtǁmoved"]), ["pkg.mod.x_f*", "pkg.mod.xǁPtǁmoved*"])
|
|
|
|
def test_named_finds_a_mutants_file_function_and_number(self):
|
|
paths = ["pkg/__init__.py", "src/pkg/mod.py"]
|
|
self.assertEqual(named("pkg.x_f__mutmut_3", paths), ("pkg/__init__.py", "x_f", 3))
|
|
self.assertEqual(named("pkg.mod.xǁPtǁmoved__mutmut_12", paths), ("src/pkg/mod.py", "xǁPtǁmoved", 12))
|
|
self.assertIsNone(named("other.x_f__mutmut_1", paths))
|
|
|
|
def test_statuses_read_mutmut_results(self):
|
|
results = " pkg.x_f__mutmut_1: survived\n pkg.x_f__mutmut_2: no tests\n\n pkg.x_f__mutmut_3: killed\n"
|
|
self.assertEqual(statuses(results), {"pkg.x_f__mutmut_1": "survived", "pkg.x_f__mutmut_2": "no tests", "pkg.x_f__mutmut_3": "killed"})
|
|
|
|
def test_a_mutated_line_is_the_first_line_the_mutant_changes_at_its_place_in_the_source(self):
|
|
self.assertEqual(mutated_line(SOURCE, COPY, "x_double", 1), (2, "return n / 2"))
|
|
self.assertEqual(mutated_line(SOURCE, COPY, "xǁPtǁmoved", 1), (6, "def moved(self, by=2):"))
|
|
self.assertEqual(mutated_line(SOURCE, COPY, "xǁPtǁmoved", 2), (7, "return self.x - by"))
|
|
|
|
def test_a_mutant_the_copy_lacks_or_a_copy_unlike_the_source_has_no_line(self):
|
|
self.assertIsNone(mutated_line(SOURCE, COPY, "x_double", 9))
|
|
self.assertIsNone(mutated_line(SOURCE.replace("n * 2", "n * 2 + 1"), COPY, "x_double", 1))
|
|
self.assertIsNone(mutated_line(SOURCE, "def broken(:\n", "x_double", 1))
|
|
|
|
|
|
# Stands in for mutmut, run through --prefix as `python3 <this> <log> <mode> mutmut ...` in the scratch copy. It logs each call with
|
|
# the setup.cfg it finds there, then writes a mutants/ copy and prints results, both shaped like mutmut 3's.
|
|
FAKE_MUTMUT = r"""import pathlib, sys
|
|
log, mode, args = pathlib.Path(sys.argv[1]), sys.argv[2], sys.argv[3:]
|
|
config = pathlib.Path("setup.cfg").read_text()
|
|
with log.open("a") as written:
|
|
written.write(" ".join(args) + "\n" + config + "---\n")
|
|
if args[:2] == ["mutmut", "results"]:
|
|
print(" calc.x_triple__mutmut_1: survived\n calc.x_triple__mutmut_2: killed\n calc.x_double__mutmut_1: survived")
|
|
elif mode == "fails-once" and "--deselect=test_calc.py::Calc::test_slow" not in config:
|
|
print("==== short test summary info ====\nFAILED test_calc.py::Calc::test_slow - boom\nfailed to collect stats. runner returned 1")
|
|
sys.exit(1)
|
|
else:
|
|
pathlib.Path("mutants").mkdir(exist_ok=True)
|
|
functions = ["x_double__mutmut_orig(n):\n\treturn n * 2", "x_double__mutmut_1(n):\n\treturn n / 2", "x_triple__mutmut_orig(n):\n\treturn n * 3", "x_triple__mutmut_1(n):\n\treturn n / 3", "x_triple__mutmut_2(n):\n\treturn n * 4"]
|
|
pathlib.Path("mutants/calc.py").write_text("".join(f"def {function}\n" for function in functions))
|
|
"""
|
|
|
|
|
|
class MutateChangesTool(unittest.TestCase):
|
|
def setUp(self):
|
|
temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temp.cleanup)
|
|
self.temp = pathlib.Path(temp.name)
|
|
self.checkout = self.temp / "proj"
|
|
(self.checkout / "pkg").mkdir(parents=True)
|
|
(self.checkout / "pkg" / "setup.cfg").write_text("[mutmut]\n")
|
|
(self.checkout / "pkg" / "calc.py").write_text("def double(n):\n\treturn n * 2\n")
|
|
(self.checkout / "pkg" / "test_calc.py").write_text("")
|
|
self.git("init", "-q", "-b", "main")
|
|
self.commit()
|
|
self.git("checkout", "-q", "-b", "work")
|
|
(self.checkout / "pkg" / "calc.py").write_text("def double(n):\n\treturn n * 2\n\n\ndef triple(n):\n\treturn n * 3\n")
|
|
(self.checkout / "pkg" / "test_calc.py").write_text("# changed, but a test file is not mutated\n")
|
|
self.commit()
|
|
(self.temp / "fake_mutmut.py").write_text(FAKE_MUTMUT)
|
|
self.log = self.temp / "calls.log"
|
|
|
|
def git(self, *args):
|
|
subprocess.run(["git", *args], check=True, cwd=self.checkout, env={**isolated_env(), "GIT_CONFIG_GLOBAL": "/dev/null"})
|
|
|
|
def commit(self):
|
|
self.git("add", ".")
|
|
self.git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "x")
|
|
|
|
def run_tool(self, *args):
|
|
return subprocess.run([str(TOOL), *args], capture_output=True, cwd=self.checkout, encoding="utf-8", env=isolated_env())
|
|
|
|
def faked(self, mode):
|
|
return ["--prefix", f"python3 {self.temp / 'fake_mutmut.py'} {self.log} {mode}"]
|
|
|
|
def test_prints_each_mutant_on_a_changed_line_that_its_tests_do_not_kill(self):
|
|
ran = self.run_tool(*self.faked("passes"), "--jobs", "2")
|
|
self.assertEqual(ran.returncode, 0, ran.stderr)
|
|
self.assertEqual(ran.stdout.splitlines()[:2], ["mutmut in pkg: 2 mutants on changed lines, 1 not killed.", "pkg/calc.py:6: survived: return n / 3"])
|
|
calls = self.log.read_text()
|
|
self.assertIn("mutmut run --max-children 2 calc.x_triple*\n", calls)
|
|
self.assertIn("source_paths = calc.py\n", calls)
|
|
self.assertEqual((self.checkout / "pkg" / "setup.cfg").read_text(), "[mutmut]\n")
|
|
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "pkg"])
|
|
|
|
def test_leaves_out_the_tests_that_fail_before_any_mutant_and_names_them(self):
|
|
ran = self.run_tool(*self.faked("fails-once"))
|
|
self.assertEqual(ran.returncode, 0, ran.stderr)
|
|
self.assertEqual(ran.stdout.splitlines()[:2], ["mutmut in pkg: 2 mutants on changed lines, 1 not killed. Left out tests in test_calc.py that fail without a mutant.", "pkg/calc.py:6: survived: return n / 3"])
|
|
first, second = self.log.read_text().split("---\n")[:2]
|
|
self.assertIn("--maxfail=0", first)
|
|
self.assertNotIn("--maxfail=0", second)
|
|
|
|
def test_a_tool_that_fails_fails_the_run_with_its_output(self):
|
|
ran = self.run_tool("--prefix", "echo broken; false")
|
|
self.assertEqual(ran.returncode, 1)
|
|
self.assertIn("mutmut run in pkg exited 1", ran.stderr)
|
|
self.assertIn("broken", ran.stderr)
|
|
|
|
def test_a_branch_that_changes_no_mutated_language_says_so(self):
|
|
ran = self.run_tool(*self.faked("passes"), "--base", "HEAD")
|
|
self.assertEqual((ran.returncode, ran.stdout), (0, "No line the branch changes is in a language mutate-changes mutates.\n"))
|
|
self.assertFalse(self.log.exists())
|
|
|
|
def test_unknown_arguments_fail(self):
|
|
for args in (["--jobs", "0"], ["--jobs"], ["--colour"], ["--base", "a", "--base", "b"]):
|
|
with self.subTest(args):
|
|
ran = self.run_tool(*args)
|
|
self.assertEqual(ran.returncode, 1)
|
|
self.assertIn("usage: mutate-changes", ran.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|