Files
claude-config/bin/mutate_changes/test_mutate_changes.py
T

232 lines
11 KiB
Python

import configparser
import pathlib
import subprocess
import sys
import tempfile
import textwrap
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from mutate_changes.changed_lines import added_lines, runs, tool_for
from failure import Failure
from mutate_changes.mutmut_mutants import function_keys, mutant_globs, mutated_line, named, statuses
from test_env import isolated_env
from mutate_changes.mutmut_setup import left_out, mutmut_config
TOOL = pathlib.Path(__file__).resolve().parents[1] / "mutate-changes"
SOURCE = "def double(n):\n\treturn n * 2\n\n\nclass Pt:\n\tdef moved(self, by=1):\n\t\treturn self.x + by\n"
# The shape of the file mutmut 3 writes under mutants/: the original of each function and its mutants, with the original's lines.
COPY = """from inspect import signature as _mutmut_signature
def x_double__mutmut_orig(n):
return n * 2
def x_double__mutmut_1(n):
return n / 2
class Pt:
def xǁPtǁmoved__mutmut_orig(self, by=1):
return self.x + by
def xǁPtǁmoved__mutmut_1(self, by=2):
return self.x + by
def xǁPtǁmoved__mutmut_2(self, by=1):
return self.x - by
"""
def read_as_mutmut(text: str, key: str) -> list[str]:
"""A [mutmut] list as mutmut 3 reads it from setup.cfg: with configparser's default interpolation, one item per line."""
config = configparser.ConfigParser()
config.read_string(text)
return [item for item in config.get("mutmut", key).split("\n") if item]
class ChangedLines(unittest.TestCase):
def test_added_lines_are_the_new_side_of_each_hunk_and_a_deleting_hunk_adds_none(self):
diff = "diff --git a/x.py b/x.py\n--- a/x.py\n+++ b/x.py\n@@ -3,0 +4,2 @@ def f():\n+a\n+b\n@@ -10 +12 @@\n-c\n+d\n@@ -20,3 +23,0 @@\n-e\n-f\n-g\n"
self.assertEqual(added_lines(diff), {4, 5, 12})
def test_runs_join_consecutive_lines(self):
self.assertEqual(runs({10, 1, 2, 3, 7, 9}), [(1, 3), (7, 7), (9, 10)])
def test_the_tool_follows_the_extension_and_test_files_get_none(self):
expected = {
"bin/x/main.py": "mutmut",
"src/lib.rs": "cargo-mutants",
"src/Order.php": "infection",
"web/app.tsx": "stryker",
"web/app.mjs": "stryker",
"bin/x/test_main.py": None,
"bin/x/main_test.py": None,
"pkg/conftest.py": None,
"tests/helpers.py": None,
"web/app.test.ts": None,
"web/app.spec.js": None,
"web/__tests__/a.js": None,
"README.md": None,
}
for path, tool in expected.items():
with self.subTest(path):
self.assertEqual(tool_for(path), tool)
class MutmutSetup(unittest.TestCase):
def test_the_config_mutates_the_sources_and_copies_the_whole_folder(self):
text = mutmut_config("", ["x/a.py", "b.py"], [])
self.assertEqual(read_as_mutmut(text, "source_paths"), ["x/a.py", "b.py"])
self.assertEqual(read_as_mutmut(text, "also_copy"), ["."])
def test_the_config_keeps_the_projects_settings_and_appends_the_tests_left_out(self):
existing = "[tool:pytest]\naddopts = -q\n\n[mutmut]\npytest_add_cli_args = -k\n not Slow\ndebug = true\n"
text = mutmut_config(existing, ["a.py"], ["--deselect=t/test_a.py::T::test_b[50%]"])
self.assertEqual(read_as_mutmut(text, "pytest_add_cli_args"), ["-k", "not Slow", "--deselect=t/test_a.py::T::test_b[50%]"])
self.assertEqual(read_as_mutmut(text, "debug"), ["true"])
config = configparser.ConfigParser()
config.read_string(text)
self.assertEqual(config.get("tool:pytest", "addopts"), "-q")
def test_a_setup_cfg_that_cannot_be_read_fails(self):
self.assertIsInstance(mutmut_config("[mutmut]\na = 1\na = 2\n", ["a.py"], []), Failure)
def test_left_out_names_each_test_the_summary_reports_failing(self):
output = textwrap.dedent("""\
FAILED fake.py::t - this line is a test's own output, above the summary
ERROR root:x.py:3 a log line
=========================== short test summary info ============================
FAILED repo_checks/test_repo_checks.py::Repo::test_a - AssertionError: x - y
ERROR larv_review/test_larv_review.py - FileNotFoundError: SKILL.md
FAILED a/test_b.py::T::test_c
2 failed, 1 error in 3.10s
failed to collect stats. runner returned 1
""")
self.assertEqual(left_out(output), ["--deselect=repo_checks/test_repo_checks.py::Repo::test_a", "--ignore=larv_review/test_larv_review.py", "--deselect=a/test_b.py::T::test_c"])
def test_left_out_is_empty_without_a_summary(self):
self.assertEqual(left_out("failed to collect stats. runner returned 1\n"), [])
class MutmutMutants(unittest.TestCase):
def test_function_keys_name_the_functions_and_methods_a_changed_line_falls_in(self):
source = SOURCE + "\n\n@cached\ndef later():\n\treturn 1\n"
self.assertEqual(function_keys(source, frozenset({2, 7})), ["x_double", "xǁPtǁmoved"])
self.assertEqual(function_keys(source, frozenset({10})), ["x_later"])
self.assertEqual(function_keys(source, frozenset({3, 5})), [])
def test_mutant_globs_follow_mutmuts_module_names(self):
self.assertEqual(mutant_globs("pkg/__init__.py", ["x_f"]), ["pkg.x_f*"])
self.assertEqual(mutant_globs("src/pkg/mod.py", ["x_f", "xǁPtǁmoved"]), ["pkg.mod.x_f*", "pkg.mod.xǁPtǁmoved*"])
def test_named_finds_a_mutants_file_function_and_number(self):
paths = ["pkg/__init__.py", "src/pkg/mod.py"]
self.assertEqual(named("pkg.x_f__mutmut_3", paths), ("pkg/__init__.py", "x_f", 3))
self.assertEqual(named("pkg.mod.xǁPtǁmoved__mutmut_12", paths), ("src/pkg/mod.py", "xǁPtǁmoved", 12))
self.assertIsNone(named("other.x_f__mutmut_1", paths))
def test_statuses_read_mutmut_results(self):
results = " pkg.x_f__mutmut_1: survived\n pkg.x_f__mutmut_2: no tests\n\n pkg.x_f__mutmut_3: killed\n"
self.assertEqual(statuses(results), {"pkg.x_f__mutmut_1": "survived", "pkg.x_f__mutmut_2": "no tests", "pkg.x_f__mutmut_3": "killed"})
def test_a_mutated_line_is_the_first_line_the_mutant_changes_at_its_place_in_the_source(self):
self.assertEqual(mutated_line(SOURCE, COPY, "x_double", 1), (2, "return n / 2"))
self.assertEqual(mutated_line(SOURCE, COPY, "xǁPtǁmoved", 1), (6, "def moved(self, by=2):"))
self.assertEqual(mutated_line(SOURCE, COPY, "xǁPtǁmoved", 2), (7, "return self.x - by"))
def test_a_mutant_the_copy_lacks_or_a_copy_unlike_the_source_has_no_line(self):
self.assertIsNone(mutated_line(SOURCE, COPY, "x_double", 9))
self.assertIsNone(mutated_line(SOURCE.replace("n * 2", "n * 2 + 1"), COPY, "x_double", 1))
self.assertIsNone(mutated_line(SOURCE, "def broken(:\n", "x_double", 1))
# Stands in for mutmut, run through --prefix as `python3 <this> <log> <mode> mutmut ...` in the scratch copy. It logs each call with
# the setup.cfg it finds there, then writes a mutants/ copy and prints results, both shaped like mutmut 3's.
FAKE_MUTMUT = r"""import pathlib, sys
log, mode, args = pathlib.Path(sys.argv[1]), sys.argv[2], sys.argv[3:]
config = pathlib.Path("setup.cfg").read_text()
with log.open("a") as written:
written.write(" ".join(args) + "\n" + config + "---\n")
if args[:2] == ["mutmut", "results"]:
print(" calc.x_triple__mutmut_1: survived\n calc.x_triple__mutmut_2: killed\n calc.x_double__mutmut_1: survived")
elif mode == "fails-once" and "--deselect=test_calc.py::Calc::test_slow" not in config:
print("==== short test summary info ====\nFAILED test_calc.py::Calc::test_slow - boom\nfailed to collect stats. runner returned 1")
sys.exit(1)
else:
pathlib.Path("mutants").mkdir(exist_ok=True)
functions = ["x_double__mutmut_orig(n):\n\treturn n * 2", "x_double__mutmut_1(n):\n\treturn n / 2", "x_triple__mutmut_orig(n):\n\treturn n * 3", "x_triple__mutmut_1(n):\n\treturn n / 3", "x_triple__mutmut_2(n):\n\treturn n * 4"]
pathlib.Path("mutants/calc.py").write_text("".join(f"def {function}\n" for function in functions))
"""
class MutateChangesTool(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
self.temp = pathlib.Path(temp.name)
self.checkout = self.temp / "proj"
(self.checkout / "pkg").mkdir(parents=True)
(self.checkout / "pkg" / "setup.cfg").write_text("[mutmut]\n")
(self.checkout / "pkg" / "calc.py").write_text("def double(n):\n\treturn n * 2\n")
(self.checkout / "pkg" / "test_calc.py").write_text("")
self.git("init", "-q", "-b", "main")
self.commit()
self.git("checkout", "-q", "-b", "work")
(self.checkout / "pkg" / "calc.py").write_text("def double(n):\n\treturn n * 2\n\n\ndef triple(n):\n\treturn n * 3\n")
(self.checkout / "pkg" / "test_calc.py").write_text("# changed, but a test file is not mutated\n")
self.commit()
(self.temp / "fake_mutmut.py").write_text(FAKE_MUTMUT)
self.log = self.temp / "calls.log"
def git(self, *args):
subprocess.run(["git", *args], check=True, cwd=self.checkout, env={**isolated_env(), "GIT_CONFIG_GLOBAL": "/dev/null"})
def commit(self):
self.git("add", ".")
self.git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "x")
def run_tool(self, *args):
return subprocess.run([str(TOOL), *args], capture_output=True, cwd=self.checkout, encoding="utf-8", env=isolated_env())
def faked(self, mode):
return ["--prefix", f"python3 {self.temp / 'fake_mutmut.py'} {self.log} {mode}"]
def test_prints_each_mutant_on_a_changed_line_that_its_tests_do_not_kill(self):
ran = self.run_tool(*self.faked("passes"), "--jobs", "2")
self.assertEqual(ran.returncode, 0, ran.stderr)
self.assertEqual(ran.stdout.splitlines()[:2], ["mutmut in pkg: 2 mutants on changed lines, 1 not killed.", "pkg/calc.py:6: survived: return n / 3"])
calls = self.log.read_text()
self.assertIn("mutmut run --max-children 2 calc.x_triple*\n", calls)
self.assertIn("source_paths = calc.py\n", calls)
self.assertEqual((self.checkout / "pkg" / "setup.cfg").read_text(), "[mutmut]\n")
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "pkg"])
def test_leaves_out_the_tests_that_fail_before_any_mutant_and_names_them(self):
ran = self.run_tool(*self.faked("fails-once"))
self.assertEqual(ran.returncode, 0, ran.stderr)
self.assertEqual(ran.stdout.splitlines()[:2], ["mutmut in pkg: 2 mutants on changed lines, 1 not killed. Left out tests in test_calc.py that fail without a mutant.", "pkg/calc.py:6: survived: return n / 3"])
first, second = self.log.read_text().split("---\n")[:2]
self.assertIn("--maxfail=0", first)
self.assertNotIn("--maxfail=0", second)
def test_a_tool_that_fails_fails_the_run_with_its_output(self):
ran = self.run_tool("--prefix", "echo broken; false")
self.assertEqual(ran.returncode, 1)
self.assertIn("mutmut run in pkg exited 1", ran.stderr)
self.assertIn("broken", ran.stderr)
def test_a_branch_that_changes_no_mutated_language_says_so(self):
ran = self.run_tool(*self.faked("passes"), "--base", "HEAD")
self.assertEqual((ran.returncode, ran.stdout), (0, "No line the branch changes is in a language mutate-changes mutates.\n"))
self.assertFalse(self.log.exists())
def test_unknown_arguments_fail(self):
for args in (["--jobs", "0"], ["--jobs"], ["--colour"], ["--base", "a", "--base", "b"]):
with self.subTest(args):
ran = self.run_tool(*args)
self.assertEqual(ran.returncode, 1)
self.assertIn("usage: mutate-changes", ran.stderr)
if __name__ == "__main__":
unittest.main()