import configparser import os import pathlib import signal import subprocess import sys import tempfile import textwrap import threading import time import typing import unittest sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) from mutate_changes.changed_lines import added_lines, by_folder, not_mutated, runs, tool_for from mutate_changes.mutmut_tool import copy_files from failure import Failure from mutate_changes.mutmut_mutants import function_keys, mutant_globs, mutated_line, named, statuses from test_env import isolated_env from mutate_changes.mutmut_setup import left_out, mutmut_config from mutate_changes import main from mutate_changes.tool_process import run_tool, scratch_folder SOURCE = "def double(n):\n\treturn n * 2\n\n\nclass Pt:\n\tdef moved(self, by=1):\n\t\treturn self.x + by\n" # The shape of the file mutmut 3 writes under mutants/: the original of each function and its mutants, with the original's lines. COPY = """from inspect import signature as _mutmut_signature def x_double__mutmut_orig(n): return n * 2 def x_double__mutmut_1(n): return n / 2 class Pt: def xǁPtǁmoved__mutmut_orig(self, by=1): return self.x + by def xǁPtǁmoved__mutmut_1(self, by=2): return self.x + by def xǁPtǁmoved__mutmut_2(self, by=1): return self.x - by """ def read_as_mutmut(text: str, key: str) -> list[str]: """A [mutmut] list as mutmut 3 reads it from setup.cfg: with configparser's default interpolation, one item per line.""" config = configparser.ConfigParser() config.read_string(text) return [item for item in config.get("mutmut", key).split("\n") if item] class ChangedLines(unittest.TestCase): def test_added_lines_are_the_new_side_of_each_hunk_and_a_deleting_hunk_adds_none(self): diff = "diff --git a/x.py b/x.py\n--- a/x.py\n+++ b/x.py\n@@ -3,0 +4,2 @@ def f():\n+a\n+b\n@@ -10 +12 @@\n-c\n+d\n@@ -20,3 +23,0 @@\n-e\n-f\n-g\n" self.assertEqual(added_lines(diff), {4, 5, 12}) def test_runs_join_consecutive_lines(self): self.assertEqual(runs({10, 1, 2, 3, 7, 9}), [(1, 3), (7, 7), (9, 10)]) def test_the_tool_follows_the_extension_and_test_files_get_none(self): expected = { "bin/x/main.py": "mutmut", "src/lib.rs": "cargo-mutants", "src/Order.php": "infection", "web/app.tsx": "stryker", "web/app.mjs": "stryker", "bin/x/test_main.py": None, "bin/x/main_test.py": None, "pkg/conftest.py": None, "tests/helpers.py": None, "web/app.test.ts": None, "web/app.spec.js": None, "web/__tests__/a.js": None, "README.md": None, } for path, tool in expected.items(): with self.subTest(path): self.assertEqual(tool_for(path), tool) def test_each_file_goes_to_the_nearest_folder_above_it_that_holds_one_of_the_configs(self): with tempfile.TemporaryDirectory() as top: for config in ("pkg/setup.cfg", "pkg/inner/lib/pyproject.toml", "web/package.json"): pathlib.Path(top, config).parent.mkdir(parents=True, exist_ok=True) pathlib.Path(top, config).write_text("") lines = frozenset({1}) grouped = by_folder(top, {"pkg/inner/deep.py": lines, "pkg/inner/lib/x.py": lines, "top.py": lines}, ("pyproject.toml", "setup.cfg")) self.assertEqual(grouped, {"pkg": {"inner/deep.py": lines}, "pkg/inner/lib": {"x.py": lines}, ".": {"top.py": lines}}) self.assertEqual(by_folder(top, {"web/src/a.ts": lines, "pkg/b.ts": lines}, ("package.json",)), {"web": {"src/a.ts": lines}, ".": {"pkg/b.ts": lines}}) def test_not_mutated_names_the_files_no_tool_covers_then_the_test_files(self): self.assertEqual(not_mutated(["README.md", "a.py", "b.go", "test_a.py", "web/a.test.ts"]), ["Not mutated, as no tool covers their language: README.md, b.go.", "Not mutated, as test files: test_a.py, web/a.test.ts."]) self.assertEqual((not_mutated(["a.py", "test_a.py"]), not_mutated(["a.py", "b.go"]), not_mutated([])), (["Not mutated, as test files: test_a.py."], ["Not mutated, as no tool covers their language: b.go."], [])) class MutmutSetup(unittest.TestCase): def test_the_config_mutates_the_sources_and_copies_each_entry_of_the_folder(self): for existing in ("", "[mutmut]\nsource_paths = old.py\nalso_copy = old\n"): with self.subTest(existing): text = mutmut_config(existing, ["x/a.py", "b.py"], ["b.py", "x"], []) self.assertEqual(read_as_mutmut(text, "source_paths"), ["x/a.py", "b.py"]) self.assertEqual(read_as_mutmut(text, "also_copy"), ["b.py", "x"]) def test_the_config_keeps_the_projects_settings_and_appends_the_tests_left_out(self): existing = "[tool:pytest]\naddopts = -q\n\n[mutmut]\npytest_add_cli_args = -k\n not Slow\ndebug = true\n" text = mutmut_config(existing, ["a.py"], ["a.py", "t"], ["--deselect=t/test_a.py::T::test_b[50%]"]) self.assertEqual(read_as_mutmut(text, "pytest_add_cli_args"), ["-k", "not Slow", "--deselect=t/test_a.py::T::test_b[50%]"]) self.assertEqual(read_as_mutmut(text, "debug"), ["true"]) config = configparser.ConfigParser() config.read_string(text) self.assertEqual(config.get("tool:pytest", "addopts"), "-q") def test_the_tests_left_out_are_the_only_pytest_options_where_the_project_sets_none(self): self.assertEqual(read_as_mutmut(mutmut_config("", ["a.py"], ["a.py"], ["--ignore=t.py"]), "pytest_add_cli_args"), ["--ignore=t.py"]) def test_a_setup_cfg_that_cannot_be_read_fails(self): failed = mutmut_config("[mutmut]\na = 1\na = 2\n", ["a.py"], ["a.py"], []) assert isinstance(failed, Failure) self.assertIn("could not read setup.cfg", failed.reason) def test_left_out_names_each_test_the_summary_reports_failing(self): output = textwrap.dedent("""\ ==== short test summary info ==== FAILED fake.py::t - these two lines are a test's own output, above the summary ERROR root:x.py:3 a log line =========================== short test summary info ============================ FAILED repo_checks/test_repo_checks.py::Repo::test_a - AssertionError: x - y ERROR larv_review/test_larv_review.py - FileNotFoundError: SKILL.md FAILED a/test_b.py::T::test_c SUBFAILED[['--jobs', '0']] a/test_d.py::T::test_e - AssertionError: 1 != 0 SUBFAILED[['--jobs']] a/test_d.py::T::test_e - AssertionError: 1 != 0 SUBFAILED[m] (path='/dev/zero', kind='a b') a/test_d.py::T::test_f 2 failed, 1 error in 3.10s failed to collect stats. runner returned 1 """) self.assertEqual(left_out(output), ["--deselect=repo_checks/test_repo_checks.py::Repo::test_a", "--ignore=larv_review/test_larv_review.py", "--deselect=a/test_b.py::T::test_c", "--deselect=a/test_d.py::T::test_e", "--deselect=a/test_d.py::T::test_f"]) def test_left_out_is_empty_without_a_summary(self): self.assertEqual(left_out("failed to collect stats. runner returned 1\n"), []) class MutmutMutants(unittest.TestCase): def test_function_keys_name_the_functions_and_methods_a_changed_line_falls_in(self): source = SOURCE + "\n\n@cached\ndef later():\n\treturn 1\n" self.assertEqual(function_keys(source, frozenset({2, 7})), ["x_double", "xǁPtǁmoved"]) self.assertEqual(function_keys(source, frozenset({10})), ["x_later"]) self.assertEqual(function_keys(source, frozenset({3, 5})), []) def test_mutant_globs_follow_mutmuts_module_names(self): self.assertEqual(mutant_globs("pkg/__init__.py", ["x_f"]), ["pkg.x_f*"]) self.assertEqual(mutant_globs("src/pkg/mod.py", ["x_f", "xǁPtǁmoved"]), ["pkg.mod.x_f*", "pkg.mod.xǁPtǁmoved*"]) def test_named_finds_a_mutants_file_function_and_number(self): paths = ["pkg/__init__.py", "src/pkg/mod.py"] self.assertEqual(named("pkg.x_f__mutmut_3", paths), ("pkg/__init__.py", "x_f", 3)) self.assertEqual(named("pkg.mod.xǁPtǁmoved__mutmut_12", paths), ("src/pkg/mod.py", "xǁPtǁmoved", 12)) self.assertIsNone(named("other.x_f__mutmut_1", paths)) def test_statuses_read_mutmut_results(self): results = " pkg.x_f__mutmut_1: survived\n pkg.x_f__mutmut_2: no tests\n\n pkg.x_f__mutmut_3: killed\n pkg.x_f__mutmut_4:\n mutants: 4\n" self.assertEqual(statuses(results), {"pkg.x_f__mutmut_1": "survived", "pkg.x_f__mutmut_2": "no tests", "pkg.x_f__mutmut_3": "killed"}) def test_a_mutated_line_is_the_first_line_the_mutant_changes_at_its_place_in_the_source(self): self.assertEqual(mutated_line(SOURCE, COPY, "x_double", 1), (2, "return n / 2")) self.assertEqual(mutated_line(SOURCE, COPY, "xǁPtǁmoved", 1), (6, "def moved(self, by=2):")) self.assertEqual(mutated_line(SOURCE, COPY, "xǁPtǁmoved", 2), (7, "return self.x - by")) def test_a_mutant_the_copy_lacks_or_a_copy_unlike_the_source_has_no_line(self): self.assertIsNone(mutated_line(SOURCE, COPY, "x_double", 9)) self.assertIsNone(mutated_line(SOURCE.replace("n * 2", "n * 2 + 1"), COPY, "x_double", 1)) self.assertIsNone(mutated_line(SOURCE, COPY + "def x_double__mutmut_2(n):\n\treturn n * 2\n", "x_double", 2)) def test_a_key_the_source_does_not_define_has_no_line(self): for key in ("x_double", "x_nope", "xǁNopeǁmoved", "xǁPtǁnope"): with self.subTest(key): self.assertIsNone(mutated_line("import os\n\n\nclass Pt:\n\tpass\n", COPY, key, 1)) def test_a_mutated_line_is_found_in_the_function_or_method_its_key_names_among_others(self): others = "import os\n\n\ndef triple(n):\n\treturn n * 3\n\n\nclass Q:\n\tdef moved(self, by=1):\n\t\treturn 0\n\n\n" self.assertEqual(mutated_line(others + SOURCE, COPY, "x_double", 1), (14, "return n / 2")) self.assertEqual(mutated_line(others + SOURCE, COPY, "xǁPtǁmoved", 2), (19, "return self.x - by")) self.assertIsNone(mutated_line(SOURCE, "def broken(:\n", "x_double", 1)) class MutmutTool(unittest.TestCase): def test_copy_files_copies_what_is_listed_but_mutmuts_own_folder_and_names_the_entries_at_the_top(self): with tempfile.TemporaryDirectory() as temp: root, copy = pathlib.Path(temp, "root"), pathlib.Path(temp, "copy") for path in ("a.py", "d/b.py", "mutants/x.py", "sub/c.py"): (root / path).parent.mkdir(parents=True, exist_ok=True) (root / path).write_text("") (root / "link").symlink_to("a.py") (root / "dirlink").symlink_to("d") # The list names a submodule as one path, a folder, and ends with an empty path. listed = ["", "mutants/x.py", "gone.py", "sub", "a.py", "d/b.py", "dirlink", "link"] self.assertEqual(copy_files(str(root), listed, str(copy)), ["a.py", "d", "dirlink", "link"]) walked = sorted(os.path.relpath(os.path.join(folder, name), copy) for folder, folders, files in os.walk(copy) for name in folders + files) self.assertEqual(walked, ["a.py", "d", "d/b.py", "dirlink", "link"]) self.assertEqual((os.readlink(copy / "link"), os.readlink(copy / "dirlink")), ("a.py", "d")) class ToolProcess(unittest.TestCase): def test_git_lists_nothing_in_a_scratch_folder(self): with tempfile.TemporaryDirectory() as top: env = {**isolated_env(), "GIT_CONFIG_GLOBAL": "/dev/null"} subprocess.run(["git", "init", "-q", top], check=True, env=env) scratch = scratch_folder(top) pathlib.Path(scratch, "copy.py").write_text("") status = subprocess.run(["git", "status", "--porcelain", "--untracked-files=all"], capture_output=True, check=True, cwd=top, encoding="utf-8", env=env).stdout self.assertEqual((os.path.dirname(scratch), os.path.basename(scratch)[:16], status), (top, ".mutate-changes-", "")) def test_a_scratch_folder_it_cannot_make_is_a_failure(self): self.assertRegex(scratch_folder("/nonexistent/top").reason, "^could not make a scratch folder in /nonexistent/top: ") def test_an_interrupted_run_ends_at_once_with_each_process_the_tool_started(self): with tempfile.TemporaryDirectory() as folder: # sh starts the sleep in the background, where it ignores SIGINT, so only the kill after the tool ends stops it. self.assertLess(self.interrupted(folder, "sleep 60 & echo $! > started; wait;"), 30) self.assert_ends(int(pathlib.Path(folder, "started").read_text())) def test_an_interrupt_reaches_the_tool_before_the_run_kills_it(self): # A container's CLI, such as docker compose, stops its container on the SIGINT it is passed, and can't once killed. # The loop ends by itself after about 40 seconds: under mutmut, the interrupt doesn't reach this shell, and a shell that looped for ever hung the run. with tempfile.TemporaryDirectory() as folder: self.interrupted(folder, "trap 'echo > stopped; exit 130' INT; echo $$ > started; n=0; while [ $n -lt 400 ]; do sleep 0.1; n=$((n+1)); done;") self.assertTrue(pathlib.Path(folder, "stopped").is_file()) def test_an_interrupted_tool_that_ignores_sigint_is_killed_after_the_grace_period(self): with tempfile.TemporaryDirectory() as folder: self.assertLess(self.interrupted(folder, "trap '' INT; echo $$ > started; sleep 60;", grace=0.5), 30) self.assert_ends(int(pathlib.Path(folder, "started").read_text())) def test_an_interrupt_ends_the_run_where_the_tool_s_processes_end_first(self): killpg = os.killpg def then_gone(group: int, number: int) -> None: killpg(group, number) raise ProcessLookupError with tempfile.TemporaryDirectory() as folder: self.interrupted(folder, "sleep 60 & echo $! > started; wait;", killpg=then_gone) def interrupted(self, folder: str, script: str, **seams: typing.Any) -> float: """Run script as the prefix of a tool in folder, interrupt run_tool once script writes folder/started, and return how long run_tool took.""" started = pathlib.Path(folder, "started") def interrupt(): while not started.is_file() or not started.read_text().strip(): time.sleep(0.05) os.kill(os.getpid(), signal.SIGINT) threading.Thread(target=interrupt, daemon=True).start() began = time.monotonic() with self.assertRaises(KeyboardInterrupt): run_tool(["true"], folder, **{"prefix": script, "env": os.environ, "killpg": os.killpg, **seams}) return time.monotonic() - began def assert_ends(self, pid: int) -> None: for _ in range(200): if ended(pid): return time.sleep(0.05) self.fail(f"process {pid}, which the tool started, still runs") def ended(pid: int) -> bool: """Whether a process has ended: gone, or a zombie its parent has yet to reap.""" try: return pathlib.Path(f"/proc/{pid}/stat").read_text().split(") ")[1][0] == "Z" except (FileNotFoundError, ProcessLookupError): return True class Took(unittest.TestCase): def test_took_rounds_to_the_second_and_counts_minutes_from_one(self): self.assertEqual([main.took(seconds) for seconds in (59.4, 59.6, 61, 3725.4)], ["59 s", "1 min 0 s", "1 min 1 s", "62 min 5 s"]) if __name__ == "__main__": unittest.main()