Files
2026-11-21 00:42:27 +01:00

351 lines
19 KiB
Python

import configparser
import os
import pathlib
import signal
import subprocess
import sys
import tempfile
import textwrap
import threading
import time
import typing
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from mutate_changes.changed_lines import added_lines, by_folder, not_mutated, runs, tool_for
from mutate_changes.mutmut_tool import copy_files
from mutate_changes.mutmut_mutants import crash_in_mutmut, function_keys, mutant_globs, mutated_line, named, statuses
from failure import Failure
from mutate_changes.bare_imports import imported_by_file_name
from test_env import isolated_env
from mutate_changes.mutmut_setup import left_out, mutmut_config
from mutate_changes import main
from mutate_changes.tool_process import run_tool, scratch_folder
from mutate_changes.in_diff_tools import with_plain_git_diff
SOURCE = "def double(n):\n\treturn n * 2\n\n\nclass Pt:\n\tdef moved(self, by=1):\n\t\treturn self.x + by\n"
# The shape of the file mutmut 3 writes under mutants/: the original of each function and its mutants, with the original's lines.
COPY = """from inspect import signature as _mutmut_signature
def x_double__mutmut_orig(n):
return n * 2
def x_double__mutmut_1(n):
return n / 2
class Pt:
def xǁPtǁmoved__mutmut_orig(self, by=1):
return self.x + by
def xǁPtǁmoved__mutmut_1(self, by=2):
return self.x + by
def xǁPtǁmoved__mutmut_2(self, by=1):
return self.x - by
"""
def read_as_mutmut(text: str, key: str) -> list[str]:
"""A [mutmut] list as mutmut 3 reads it from setup.cfg: with configparser's default interpolation, one item per line."""
config = configparser.ConfigParser()
config.read_string(text)
return [item for item in config.get("mutmut", key).split("\n") if item]
class ChangedLines(unittest.TestCase):
def test_added_lines_are_the_new_side_of_each_hunk_and_a_deleting_hunk_adds_none(self):
diff = "diff --git a/x.py b/x.py\n--- a/x.py\n+++ b/x.py\n@@ -3,0 +4,2 @@ def f():\n+a\n+b\n@@ -10 +12 @@\n-c\n+d\n@@ -20,3 +23,0 @@\n-e\n-f\n-g\n"
self.assertEqual(added_lines(diff), {4, 5, 12})
def test_runs_join_consecutive_lines(self):
self.assertEqual(runs({10, 1, 2, 3, 7, 9}), [(1, 3), (7, 7), (9, 10)])
def test_the_tool_follows_the_extension_and_test_files_get_none(self):
expected = {
"bin/x/main.py": "mutmut",
"src/lib.rs": "cargo-mutants",
"src/Order.php": "infection",
"web/app.tsx": "stryker",
"web/app.mjs": "stryker",
"cmd/serve/main.go": "gremlins",
"bin/x/test_main.py": None,
"bin/x/main_test.py": None,
"pkg/conftest.py": None,
"tests/helpers.py": None,
"web/app.test.ts": None,
"web/app.spec.js": None,
"cmd/serve/main_test.go": None,
"web/__tests__/a.js": None,
"README.md": None,
}
for path, tool in expected.items():
with self.subTest(path):
self.assertEqual(tool_for(path), tool)
def test_each_file_goes_to_the_nearest_folder_above_it_that_holds_one_of_the_configs(self):
with tempfile.TemporaryDirectory() as top:
for config in ("pkg/setup.cfg", "pkg/inner/lib/pyproject.toml", "web/package.json"):
pathlib.Path(top, config).parent.mkdir(parents=True, exist_ok=True)
pathlib.Path(top, config).write_text("")
lines = frozenset({1})
grouped = by_folder(top, {"pkg/inner/deep.py": lines, "pkg/inner/lib/x.py": lines, "top.py": lines}, ("pyproject.toml", "setup.cfg"))
self.assertEqual(grouped, {"pkg": {"inner/deep.py": lines}, "pkg/inner/lib": {"x.py": lines}, ".": {"top.py": lines}})
self.assertEqual(by_folder(top, {"web/src/a.ts": lines, "pkg/b.ts": lines}, ("package.json",)), {"web": {"src/a.ts": lines}, ".": {"pkg/b.ts": lines}})
def test_a_file_its_sibling_test_imports_by_file_name_moves_to_its_own_folder(self):
with tempfile.TemporaryDirectory() as top:
tests = {
"docker/scripts/a_helper.py": "import not_a_test\n",
"docker/scripts/test_a_bad.py": "def (:\n",
"docker/scripts/test_b_run.py": "import unittest\nimport run_harness as rh\nimport dotted.sub\nfrom other import thing\nfrom dotted2.sub import z\nfrom .rel import y\n",
"docker/scripts/test_c_odd.py": "import \u00e9\n",
"pkg/test_pkg.py": "from pkg.mod import x\nimport mod\n",
"pkg/__init__.py": "",
"lib/test_lib.py": "import lib_tool\n",
"lib/__init__.py": "",
"test_top.py": "import top_tool\n",
}
for path, text in tests.items():
pathlib.Path(top, path).parent.mkdir(parents=True, exist_ok=True)
pathlib.Path(top, path).write_text(text, encoding="utf-8")
lines = frozenset({1})
names = ("docker/scripts/run_harness.py", "docker/scripts/dotted.py", "docker/scripts/other.py", "docker/scripts/dotted2.py", "docker/scripts/rel.py",
"docker/scripts/solo.py", "docker/scripts/not_a_test.py", "pkg/mod.py", "lib/lib_tool.py", "top_tool.py", "solo_top.py")
grouped = {".": {name: lines for name in names}, "web": {"app.py": lines}}
moved = ("run_harness.py", "dotted.py", "other.py", "dotted2.py")
self.assertEqual(imported_by_file_name(top, grouped, ("setup.cfg",)), {
".": {"docker/scripts/rel.py": lines, "docker/scripts/solo.py": lines, "docker/scripts/not_a_test.py": lines, "pkg/mod.py": lines, "lib/lib_tool.py": lines, "top_tool.py": lines, "solo_top.py": lines},
"docker/scripts": {name: lines for name in moved},
"web": {"app.py": lines},
})
self.assertEqual(imported_by_file_name(top, {"web": {"app.py": lines}}, ("setup.cfg",)), {"web": {"app.py": lines}})
pathlib.Path(top, "setup.cfg").write_text("")
self.assertEqual(imported_by_file_name(top, grouped, ("setup.cfg",)), grouped)
def test_a_traceback_that_ends_inside_mutmut_is_a_crash_of_mutmut(self):
def trace(*frames, error="KeyError: 974598", after=""):
shown = "".join(f' File "{path}", line {number}, in f\n code()\n ~~^^\n' for path, number in frames)
return f"Traceback (most recent call last):\n{shown}{error}\n{after}"
lib = "/uv/lib/python3.14/site-packages/mutmut"
inside = trace((f"{lib}/__main__.py", 9), (f"{lib}/workers/isolation.py", 889), (f"{lib}/workers/isolation.py", 12))
outside = trace((f"{lib}/__main__.py", 9), ("/repo/calc.py", 3), error="KeyError: 1")
place = "KeyError: 974598, in mutmut/workers/isolation.py line 12"
self.assertEqual((crash_in_mutmut(inside), crash_in_mutmut("Running stats\n" + inside + "\n")), (place, place))
self.assertEqual(crash_in_mutmut(inside + "later output\n"), place)
self.assertEqual(crash_in_mutmut(outside + trace((f"{lib}/x.py", 1), error="E: 2")), "E: 2, in mutmut/x.py line 1")
self.assertEqual(crash_in_mutmut(trace((f"{lib}/x.py", 1), error="E: 2") + outside), None)
self.assertEqual(crash_in_mutmut(trace(("C:\\py\\Lib\\site-packages\\mutmut\\workers\\x.py", 5))), "KeyError: 974598, in mutmut/workers/x.py line 5")
for path in ("/mutmut/repo/calc.py", "mutmut_tool.py", "<string>", "/a/site-packages/mutmut_tool.py"):
with self.subTest(path):
self.assertEqual(crash_in_mutmut(trace((path, 1))), None)
self.assertEqual(crash_in_mutmut(trace((f"{lib}/x.py", 1)).replace("Traceback (most recent call last):", "")), None)
self.assertEqual((crash_in_mutmut("all fine"), crash_in_mutmut(""), crash_in_mutmut("Traceback (most recent call last):\nKeyError: 1")), (None,) * 3)
self.assertEqual(crash_in_mutmut(trace((f"{lib}/x.py", 1), error="")), None)
def test_not_mutated_names_the_files_no_tool_covers_then_the_test_files(self):
self.assertEqual(not_mutated(["README.md", "a.py", "b.rb", "test_a.py", "web/a.test.ts"]), ["Not mutated, as no tool covers their language: README.md, b.rb.", "Not mutated, as test files: test_a.py, web/a.test.ts."])
self.assertEqual((not_mutated(["a.py", "test_a.py"]), not_mutated(["a.py", "b.rb"]), not_mutated([])), (["Not mutated, as test files: test_a.py."], ["Not mutated, as no tool covers their language: b.rb."], []))
class InDiffTools(unittest.TestCase):
def test_the_git_settings_gremlins_needs_are_added_after_the_ones_the_environment_holds(self):
held = {"HOME": "/h", "GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "safe.directory", "GIT_CONFIG_VALUE_0": "*"}
env = with_plain_git_diff(held)
self.assertEqual(env["GIT_CONFIG_COUNT"], "8")
self.assertEqual({name: value for name, value in env.items() if name in held}, held | {"GIT_CONFIG_COUNT": "8"})
settings = [(env[f"GIT_CONFIG_KEY_{index}"], env[f"GIT_CONFIG_VALUE_{index}"]) for index in range(1, 8)]
self.assertEqual(settings, [("diff.context", "0"), ("diff.noprefix", "false"), ("diff.mnemonicPrefix", "false"), ("diff.srcPrefix", "a/"), ("diff.dstPrefix", "b/"), ("color.diff", "never"), ("diff.interHunkContext", "0")])
def test_the_git_settings_start_at_the_first_for_an_environment_without_a_count_or_with_one_that_is_not_a_number(self):
for held in ({}, {"GIT_CONFIG_COUNT": "many"}, {"GIT_CONFIG_COUNT": ""}):
with self.subTest(held):
env = with_plain_git_diff(held)
self.assertEqual((env["GIT_CONFIG_COUNT"], env["GIT_CONFIG_KEY_0"], env["GIT_CONFIG_KEY_6"], "GIT_CONFIG_KEY_7" in env), ("7", "diff.context", "diff.interHunkContext", False))
class MutmutSetup(unittest.TestCase):
def test_the_config_mutates_the_sources_and_copies_each_entry_of_the_folder(self):
for existing in ("", "[mutmut]\nsource_paths = old.py\nalso_copy = old\n"):
with self.subTest(existing):
text = mutmut_config(existing, ["x/a.py", "b.py"], ["b.py", "x"], [])
self.assertEqual(read_as_mutmut(text, "source_paths"), ["x/a.py", "b.py"])
self.assertEqual(read_as_mutmut(text, "also_copy"), ["b.py", "x"])
def test_the_config_keeps_the_projects_settings_and_appends_the_tests_left_out(self):
existing = "[tool:pytest]\naddopts = -q\n\n[mutmut]\npytest_add_cli_args = -k\n not Slow\ndebug = true\n"
text = mutmut_config(existing, ["a.py"], ["a.py", "t"], ["--deselect=t/test_a.py::T::test_b[50%]"])
self.assertEqual(read_as_mutmut(text, "pytest_add_cli_args"), ["-k", "not Slow", "--deselect=t/test_a.py::T::test_b[50%]"])
self.assertEqual(read_as_mutmut(text, "debug"), ["true"])
config = configparser.ConfigParser()
config.read_string(text)
self.assertEqual(config.get("tool:pytest", "addopts"), "-q")
def test_the_tests_left_out_are_the_only_pytest_options_where_the_project_sets_none(self):
self.assertEqual(read_as_mutmut(mutmut_config("", ["a.py"], ["a.py"], ["--ignore=t.py"]), "pytest_add_cli_args"), ["--ignore=t.py"])
def test_a_setup_cfg_that_cannot_be_read_fails(self):
failed = mutmut_config("[mutmut]\na = 1\na = 2\n", ["a.py"], ["a.py"], [])
assert isinstance(failed, Failure)
self.assertIn("could not read setup.cfg", failed.reason)
def test_left_out_names_each_test_the_summary_reports_failing(self):
output = textwrap.dedent("""\
==== short test summary info ====
FAILED fake.py::t - these two lines are a test's own output, above the summary
ERROR root:x.py:3 a log line
=========================== short test summary info ============================
FAILED repo_checks/test_repo_checks.py::Repo::test_a - AssertionError: x - y
ERROR larv_review/test_larv_review.py - FileNotFoundError: SKILL.md
FAILED a/test_b.py::T::test_c
SUBFAILED[['--jobs', '0']] a/test_d.py::T::test_e - AssertionError: 1 != 0
SUBFAILED[['--jobs']] a/test_d.py::T::test_e - AssertionError: 1 != 0
SUBFAILED[m] (path='/dev/zero', kind='a b') a/test_d.py::T::test_f
2 failed, 1 error in 3.10s
failed to collect stats. runner returned 1
""")
self.assertEqual(left_out(output), ["--deselect=repo_checks/test_repo_checks.py::Repo::test_a", "--ignore=larv_review/test_larv_review.py", "--deselect=a/test_b.py::T::test_c", "--deselect=a/test_d.py::T::test_e", "--deselect=a/test_d.py::T::test_f"])
def test_left_out_is_empty_without_a_summary(self):
self.assertEqual(left_out("failed to collect stats. runner returned 1\n"), [])
class MutmutMutants(unittest.TestCase):
def test_function_keys_name_the_functions_and_methods_a_changed_line_falls_in(self):
source = SOURCE + "\n\n@cached\ndef later():\n\treturn 1\n"
self.assertEqual(function_keys(source, frozenset({2, 7})), ["x_double", "xǁPtǁmoved"])
self.assertEqual(function_keys(source, frozenset({10})), ["x_later"])
self.assertEqual(function_keys(source, frozenset({3, 5})), [])
def test_mutant_globs_follow_mutmuts_module_names(self):
self.assertEqual(mutant_globs("pkg/__init__.py", ["x_f"]), ["pkg.x_f*"])
self.assertEqual(mutant_globs("src/pkg/mod.py", ["x_f", "xǁPtǁmoved"]), ["pkg.mod.x_f*", "pkg.mod.xǁPtǁmoved*"])
def test_named_finds_a_mutants_file_function_and_number(self):
paths = ["pkg/__init__.py", "src/pkg/mod.py"]
self.assertEqual(named("pkg.x_f__mutmut_3", paths), ("pkg/__init__.py", "x_f", 3))
self.assertEqual(named("pkg.mod.xǁPtǁmoved__mutmut_12", paths), ("src/pkg/mod.py", "xǁPtǁmoved", 12))
self.assertIsNone(named("other.x_f__mutmut_1", paths))
def test_statuses_read_mutmut_results(self):
results = " pkg.x_f__mutmut_1: survived\n pkg.x_f__mutmut_2: no tests\n\n pkg.x_f__mutmut_3: killed\n pkg.x_f__mutmut_4:\n mutants: 4\n"
self.assertEqual(statuses(results), {"pkg.x_f__mutmut_1": "survived", "pkg.x_f__mutmut_2": "no tests", "pkg.x_f__mutmut_3": "killed"})
def test_a_mutated_line_is_the_first_line_the_mutant_changes_at_its_place_in_the_source(self):
self.assertEqual(mutated_line(SOURCE, COPY, "x_double", 1), (2, "return n / 2"))
self.assertEqual(mutated_line(SOURCE, COPY, "xǁPtǁmoved", 1), (6, "def moved(self, by=2):"))
self.assertEqual(mutated_line(SOURCE, COPY, "xǁPtǁmoved", 2), (7, "return self.x - by"))
def test_a_mutant_the_copy_lacks_or_a_copy_unlike_the_source_has_no_line(self):
self.assertIsNone(mutated_line(SOURCE, COPY, "x_double", 9))
self.assertIsNone(mutated_line(SOURCE.replace("n * 2", "n * 2 + 1"), COPY, "x_double", 1))
self.assertIsNone(mutated_line(SOURCE, COPY + "def x_double__mutmut_2(n):\n\treturn n * 2\n", "x_double", 2))
def test_a_key_the_source_does_not_define_has_no_line(self):
for key in ("x_double", "x_nope", "xǁNopeǁmoved", "xǁPtǁnope"):
with self.subTest(key):
self.assertIsNone(mutated_line("import os\n\n\nclass Pt:\n\tpass\n", COPY, key, 1))
def test_a_mutated_line_is_found_in_the_function_or_method_its_key_names_among_others(self):
others = "import os\n\n\ndef triple(n):\n\treturn n * 3\n\n\nclass Q:\n\tdef moved(self, by=1):\n\t\treturn 0\n\n\n"
self.assertEqual(mutated_line(others + SOURCE, COPY, "x_double", 1), (14, "return n / 2"))
self.assertEqual(mutated_line(others + SOURCE, COPY, "xǁPtǁmoved", 2), (19, "return self.x - by"))
self.assertIsNone(mutated_line(SOURCE, "def broken(:\n", "x_double", 1))
class MutmutTool(unittest.TestCase):
def test_copy_files_copies_what_is_listed_but_mutmuts_own_folder_and_names_the_entries_at_the_top(self):
with tempfile.TemporaryDirectory() as temp:
root, copy = pathlib.Path(temp, "root"), pathlib.Path(temp, "copy")
for path in ("a.py", "d/b.py", "mutants/x.py", "sub/c.py"):
(root / path).parent.mkdir(parents=True, exist_ok=True)
(root / path).write_text("")
(root / "link").symlink_to("a.py")
(root / "dirlink").symlink_to("d")
# The list names a submodule as one path, a folder, and ends with an empty path.
listed = ["", "mutants/x.py", "gone.py", "sub", "a.py", "d/b.py", "dirlink", "link"]
self.assertEqual(copy_files(str(root), listed, str(copy)), ["a.py", "d", "dirlink", "link"])
walked = sorted(os.path.relpath(os.path.join(folder, name), copy) for folder, folders, files in os.walk(copy) for name in folders + files)
self.assertEqual(walked, ["a.py", "d", "d/b.py", "dirlink", "link"])
self.assertEqual((os.readlink(copy / "link"), os.readlink(copy / "dirlink")), ("a.py", "d"))
class ToolProcess(unittest.TestCase):
def test_git_lists_nothing_in_a_scratch_folder(self):
with tempfile.TemporaryDirectory() as top:
env = {**isolated_env(), "GIT_CONFIG_GLOBAL": "/dev/null"}
subprocess.run(["git", "init", "-q", top], check=True, env=env)
scratch = scratch_folder(top)
pathlib.Path(scratch, "copy.py").write_text("")
status = subprocess.run(["git", "status", "--porcelain", "--untracked-files=all"], capture_output=True, check=True, cwd=top, encoding="utf-8", env=env).stdout
self.assertEqual((os.path.dirname(scratch), os.path.basename(scratch)[:16], status), (top, ".mutate-changes-", ""))
def test_a_scratch_folder_it_cannot_make_is_a_failure(self):
self.assertRegex(scratch_folder("/nonexistent/top").reason, "^could not make a scratch folder in /nonexistent/top: ")
def test_an_interrupted_run_ends_at_once_with_each_process_the_tool_started(self):
with tempfile.TemporaryDirectory() as folder:
# sh starts the sleep in the background, where it ignores SIGINT, so only the kill after the tool ends stops it.
self.assertLess(self.interrupted(folder, "sleep 60 & echo $! > started; wait;"), 30)
self.assert_ends(int(pathlib.Path(folder, "started").read_text()))
def test_an_interrupt_reaches_the_tool_before_the_run_kills_it(self):
# A container's CLI, such as docker compose, stops its container on the SIGINT it is passed, and can't once killed.
# The loop ends by itself after about 40 seconds: under mutmut, the interrupt doesn't reach this shell, and a shell that looped for ever hung the run.
with tempfile.TemporaryDirectory() as folder:
self.interrupted(folder, "trap 'echo > stopped; exit 130' INT; echo $$ > started; n=0; while [ $n -lt 400 ]; do sleep 0.1; n=$((n+1)); done;")
self.assertTrue(pathlib.Path(folder, "stopped").is_file())
def test_an_interrupted_tool_that_ignores_sigint_is_killed_after_the_grace_period(self):
with tempfile.TemporaryDirectory() as folder:
self.assertLess(self.interrupted(folder, "trap '' INT; echo $$ > started; sleep 60;", grace=0.5), 30)
self.assert_ends(int(pathlib.Path(folder, "started").read_text()))
def test_an_interrupt_ends_the_run_where_the_tool_s_processes_end_first(self):
killpg = os.killpg
def then_gone(group: int, number: int) -> None:
killpg(group, number)
raise ProcessLookupError
with tempfile.TemporaryDirectory() as folder:
self.interrupted(folder, "sleep 60 & echo $! > started; wait;", killpg=then_gone)
def interrupted(self, folder: str, script: str, **seams: typing.Any) -> float:
"""Run script as the prefix of a tool in folder, interrupt run_tool once script writes folder/started, and return how long run_tool took."""
started = pathlib.Path(folder, "started")
def interrupt():
while not started.is_file() or not started.read_text().strip():
time.sleep(0.05)
os.kill(os.getpid(), signal.SIGINT)
threading.Thread(target=interrupt, daemon=True).start()
began = time.monotonic()
with self.assertRaises(KeyboardInterrupt):
run_tool(["true"], folder, **{"prefix": script, "env": os.environ, "killpg": os.killpg, **seams})
return time.monotonic() - began
def assert_ends(self, pid: int) -> None:
for _ in range(200):
if ended(pid):
return
time.sleep(0.05)
self.fail(f"process {pid}, which the tool started, still runs")
def ended(pid: int) -> bool:
"""Whether a process has ended: gone, or a zombie its parent has yet to reap."""
try:
return pathlib.Path(f"/proc/{pid}/stat").read_text().split(") ")[1][0] == "Z"
except (FileNotFoundError, ProcessLookupError):
return True
class Took(unittest.TestCase):
def test_took_rounds_to_the_second_and_counts_minutes_from_one(self):
self.assertEqual([main.took(seconds) for seconds in (59.4, 59.6, 61, 3725.4)], ["59 s", "1 min 0 s", "1 min 1 s", "62 min 5 s"])
if __name__ == "__main__":
unittest.main()