281 lines
14 KiB
Python
281 lines
14 KiB
Python
import configparser
|
|
import os
|
|
import pathlib
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import textwrap
|
|
import threading
|
|
import time
|
|
import typing
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from mutate_changes.changed_lines import added_lines, by_folder, not_mutated, runs, tool_for
|
|
from mutate_changes.mutmut_tool import copy_files
|
|
from failure import Failure
|
|
from mutate_changes.mutmut_mutants import function_keys, mutant_globs, mutated_line, named, statuses
|
|
from test_env import isolated_env
|
|
from mutate_changes.mutmut_setup import left_out, mutmut_config
|
|
from mutate_changes import main
|
|
from mutate_changes.tool_process import run_tool, scratch_folder
|
|
|
|
SOURCE = "def double(n):\n\treturn n * 2\n\n\nclass Pt:\n\tdef moved(self, by=1):\n\t\treturn self.x + by\n"
|
|
# The shape of the file mutmut 3 writes under mutants/: the original of each function and its mutants, with the original's lines.
|
|
COPY = """from inspect import signature as _mutmut_signature
|
|
|
|
|
|
def x_double__mutmut_orig(n):
|
|
return n * 2
|
|
def x_double__mutmut_1(n):
|
|
return n / 2
|
|
|
|
|
|
class Pt:
|
|
def xǁPtǁmoved__mutmut_orig(self, by=1):
|
|
return self.x + by
|
|
def xǁPtǁmoved__mutmut_1(self, by=2):
|
|
return self.x + by
|
|
def xǁPtǁmoved__mutmut_2(self, by=1):
|
|
return self.x - by
|
|
"""
|
|
|
|
|
|
def read_as_mutmut(text: str, key: str) -> list[str]:
|
|
"""A [mutmut] list as mutmut 3 reads it from setup.cfg: with configparser's default interpolation, one item per line."""
|
|
config = configparser.ConfigParser()
|
|
config.read_string(text)
|
|
return [item for item in config.get("mutmut", key).split("\n") if item]
|
|
|
|
|
|
class ChangedLines(unittest.TestCase):
|
|
def test_added_lines_are_the_new_side_of_each_hunk_and_a_deleting_hunk_adds_none(self):
|
|
diff = "diff --git a/x.py b/x.py\n--- a/x.py\n+++ b/x.py\n@@ -3,0 +4,2 @@ def f():\n+a\n+b\n@@ -10 +12 @@\n-c\n+d\n@@ -20,3 +23,0 @@\n-e\n-f\n-g\n"
|
|
self.assertEqual(added_lines(diff), {4, 5, 12})
|
|
|
|
def test_runs_join_consecutive_lines(self):
|
|
self.assertEqual(runs({10, 1, 2, 3, 7, 9}), [(1, 3), (7, 7), (9, 10)])
|
|
|
|
def test_the_tool_follows_the_extension_and_test_files_get_none(self):
|
|
expected = {
|
|
"bin/x/main.py": "mutmut",
|
|
"src/lib.rs": "cargo-mutants",
|
|
"src/Order.php": "infection",
|
|
"web/app.tsx": "stryker",
|
|
"web/app.mjs": "stryker",
|
|
"bin/x/test_main.py": None,
|
|
"bin/x/main_test.py": None,
|
|
"pkg/conftest.py": None,
|
|
"tests/helpers.py": None,
|
|
"web/app.test.ts": None,
|
|
"web/app.spec.js": None,
|
|
"web/__tests__/a.js": None,
|
|
"README.md": None,
|
|
}
|
|
for path, tool in expected.items():
|
|
with self.subTest(path):
|
|
self.assertEqual(tool_for(path), tool)
|
|
|
|
def test_each_file_goes_to_the_nearest_folder_above_it_that_holds_one_of_the_configs(self):
|
|
with tempfile.TemporaryDirectory() as top:
|
|
for config in ("pkg/setup.cfg", "pkg/inner/lib/pyproject.toml", "web/package.json"):
|
|
pathlib.Path(top, config).parent.mkdir(parents=True, exist_ok=True)
|
|
pathlib.Path(top, config).write_text("")
|
|
lines = frozenset({1})
|
|
grouped = by_folder(top, {"pkg/inner/deep.py": lines, "pkg/inner/lib/x.py": lines, "top.py": lines}, ("pyproject.toml", "setup.cfg"))
|
|
self.assertEqual(grouped, {"pkg": {"inner/deep.py": lines}, "pkg/inner/lib": {"x.py": lines}, ".": {"top.py": lines}})
|
|
self.assertEqual(by_folder(top, {"web/src/a.ts": lines, "pkg/b.ts": lines}, ("package.json",)), {"web": {"src/a.ts": lines}, ".": {"pkg/b.ts": lines}})
|
|
|
|
def test_not_mutated_names_the_files_no_tool_covers_then_the_test_files(self):
|
|
self.assertEqual(not_mutated(["README.md", "a.py", "b.go", "test_a.py", "web/a.test.ts"]), ["Not mutated, as no tool covers their language: README.md, b.go.", "Not mutated, as test files: test_a.py, web/a.test.ts."])
|
|
self.assertEqual((not_mutated(["a.py", "test_a.py"]), not_mutated(["a.py", "b.go"]), not_mutated([])), (["Not mutated, as test files: test_a.py."], ["Not mutated, as no tool covers their language: b.go."], []))
|
|
|
|
|
|
class MutmutSetup(unittest.TestCase):
|
|
def test_the_config_mutates_the_sources_and_copies_each_entry_of_the_folder(self):
|
|
for existing in ("", "[mutmut]\nsource_paths = old.py\nalso_copy = old\n"):
|
|
with self.subTest(existing):
|
|
text = mutmut_config(existing, ["x/a.py", "b.py"], ["b.py", "x"], [])
|
|
self.assertEqual(read_as_mutmut(text, "source_paths"), ["x/a.py", "b.py"])
|
|
self.assertEqual(read_as_mutmut(text, "also_copy"), ["b.py", "x"])
|
|
|
|
def test_the_config_keeps_the_projects_settings_and_appends_the_tests_left_out(self):
|
|
existing = "[tool:pytest]\naddopts = -q\n\n[mutmut]\npytest_add_cli_args = -k\n not Slow\ndebug = true\n"
|
|
text = mutmut_config(existing, ["a.py"], ["a.py", "t"], ["--deselect=t/test_a.py::T::test_b[50%]"])
|
|
self.assertEqual(read_as_mutmut(text, "pytest_add_cli_args"), ["-k", "not Slow", "--deselect=t/test_a.py::T::test_b[50%]"])
|
|
self.assertEqual(read_as_mutmut(text, "debug"), ["true"])
|
|
config = configparser.ConfigParser()
|
|
config.read_string(text)
|
|
self.assertEqual(config.get("tool:pytest", "addopts"), "-q")
|
|
|
|
def test_the_tests_left_out_are_the_only_pytest_options_where_the_project_sets_none(self):
|
|
self.assertEqual(read_as_mutmut(mutmut_config("", ["a.py"], ["a.py"], ["--ignore=t.py"]), "pytest_add_cli_args"), ["--ignore=t.py"])
|
|
|
|
def test_a_setup_cfg_that_cannot_be_read_fails(self):
|
|
failed = mutmut_config("[mutmut]\na = 1\na = 2\n", ["a.py"], ["a.py"], [])
|
|
assert isinstance(failed, Failure)
|
|
self.assertIn("could not read setup.cfg", failed.reason)
|
|
|
|
def test_left_out_names_each_test_the_summary_reports_failing(self):
|
|
output = textwrap.dedent("""\
|
|
==== short test summary info ====
|
|
FAILED fake.py::t - these two lines are a test's own output, above the summary
|
|
ERROR root:x.py:3 a log line
|
|
=========================== short test summary info ============================
|
|
FAILED repo_checks/test_repo_checks.py::Repo::test_a - AssertionError: x - y
|
|
ERROR larv_review/test_larv_review.py - FileNotFoundError: SKILL.md
|
|
FAILED a/test_b.py::T::test_c
|
|
SUBFAILED[['--jobs', '0']] a/test_d.py::T::test_e - AssertionError: 1 != 0
|
|
SUBFAILED[['--jobs']] a/test_d.py::T::test_e - AssertionError: 1 != 0
|
|
SUBFAILED[m] (path='/dev/zero', kind='a b') a/test_d.py::T::test_f
|
|
2 failed, 1 error in 3.10s
|
|
failed to collect stats. runner returned 1
|
|
""")
|
|
self.assertEqual(left_out(output), ["--deselect=repo_checks/test_repo_checks.py::Repo::test_a", "--ignore=larv_review/test_larv_review.py", "--deselect=a/test_b.py::T::test_c", "--deselect=a/test_d.py::T::test_e", "--deselect=a/test_d.py::T::test_f"])
|
|
|
|
def test_left_out_is_empty_without_a_summary(self):
|
|
self.assertEqual(left_out("failed to collect stats. runner returned 1\n"), [])
|
|
|
|
|
|
class MutmutMutants(unittest.TestCase):
|
|
def test_function_keys_name_the_functions_and_methods_a_changed_line_falls_in(self):
|
|
source = SOURCE + "\n\n@cached\ndef later():\n\treturn 1\n"
|
|
self.assertEqual(function_keys(source, frozenset({2, 7})), ["x_double", "xǁPtǁmoved"])
|
|
self.assertEqual(function_keys(source, frozenset({10})), ["x_later"])
|
|
self.assertEqual(function_keys(source, frozenset({3, 5})), [])
|
|
|
|
def test_mutant_globs_follow_mutmuts_module_names(self):
|
|
self.assertEqual(mutant_globs("pkg/__init__.py", ["x_f"]), ["pkg.x_f*"])
|
|
self.assertEqual(mutant_globs("src/pkg/mod.py", ["x_f", "xǁPtǁmoved"]), ["pkg.mod.x_f*", "pkg.mod.xǁPtǁmoved*"])
|
|
|
|
def test_named_finds_a_mutants_file_function_and_number(self):
|
|
paths = ["pkg/__init__.py", "src/pkg/mod.py"]
|
|
self.assertEqual(named("pkg.x_f__mutmut_3", paths), ("pkg/__init__.py", "x_f", 3))
|
|
self.assertEqual(named("pkg.mod.xǁPtǁmoved__mutmut_12", paths), ("src/pkg/mod.py", "xǁPtǁmoved", 12))
|
|
self.assertIsNone(named("other.x_f__mutmut_1", paths))
|
|
|
|
def test_statuses_read_mutmut_results(self):
|
|
results = " pkg.x_f__mutmut_1: survived\n pkg.x_f__mutmut_2: no tests\n\n pkg.x_f__mutmut_3: killed\n pkg.x_f__mutmut_4:\n mutants: 4\n"
|
|
self.assertEqual(statuses(results), {"pkg.x_f__mutmut_1": "survived", "pkg.x_f__mutmut_2": "no tests", "pkg.x_f__mutmut_3": "killed"})
|
|
|
|
def test_a_mutated_line_is_the_first_line_the_mutant_changes_at_its_place_in_the_source(self):
|
|
self.assertEqual(mutated_line(SOURCE, COPY, "x_double", 1), (2, "return n / 2"))
|
|
self.assertEqual(mutated_line(SOURCE, COPY, "xǁPtǁmoved", 1), (6, "def moved(self, by=2):"))
|
|
self.assertEqual(mutated_line(SOURCE, COPY, "xǁPtǁmoved", 2), (7, "return self.x - by"))
|
|
|
|
def test_a_mutant_the_copy_lacks_or_a_copy_unlike_the_source_has_no_line(self):
|
|
self.assertIsNone(mutated_line(SOURCE, COPY, "x_double", 9))
|
|
self.assertIsNone(mutated_line(SOURCE.replace("n * 2", "n * 2 + 1"), COPY, "x_double", 1))
|
|
self.assertIsNone(mutated_line(SOURCE, COPY + "def x_double__mutmut_2(n):\n\treturn n * 2\n", "x_double", 2))
|
|
|
|
def test_a_key_the_source_does_not_define_has_no_line(self):
|
|
for key in ("x_double", "x_nope", "xǁNopeǁmoved", "xǁPtǁnope"):
|
|
with self.subTest(key):
|
|
self.assertIsNone(mutated_line("import os\n\n\nclass Pt:\n\tpass\n", COPY, key, 1))
|
|
|
|
def test_a_mutated_line_is_found_in_the_function_or_method_its_key_names_among_others(self):
|
|
others = "import os\n\n\ndef triple(n):\n\treturn n * 3\n\n\nclass Q:\n\tdef moved(self, by=1):\n\t\treturn 0\n\n\n"
|
|
self.assertEqual(mutated_line(others + SOURCE, COPY, "x_double", 1), (14, "return n / 2"))
|
|
self.assertEqual(mutated_line(others + SOURCE, COPY, "xǁPtǁmoved", 2), (19, "return self.x - by"))
|
|
self.assertIsNone(mutated_line(SOURCE, "def broken(:\n", "x_double", 1))
|
|
|
|
|
|
class MutmutTool(unittest.TestCase):
|
|
def test_copy_files_copies_what_is_listed_but_mutmuts_own_folder_and_names_the_entries_at_the_top(self):
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
root, copy = pathlib.Path(temp, "root"), pathlib.Path(temp, "copy")
|
|
for path in ("a.py", "d/b.py", "mutants/x.py", "sub/c.py"):
|
|
(root / path).parent.mkdir(parents=True, exist_ok=True)
|
|
(root / path).write_text("")
|
|
(root / "link").symlink_to("a.py")
|
|
(root / "dirlink").symlink_to("d")
|
|
# The list names a submodule as one path, a folder, and ends with an empty path.
|
|
listed = ["", "mutants/x.py", "gone.py", "sub", "a.py", "d/b.py", "dirlink", "link"]
|
|
self.assertEqual(copy_files(str(root), listed, str(copy)), ["a.py", "d", "dirlink", "link"])
|
|
walked = sorted(os.path.relpath(os.path.join(folder, name), copy) for folder, folders, files in os.walk(copy) for name in folders + files)
|
|
self.assertEqual(walked, ["a.py", "d", "d/b.py", "dirlink", "link"])
|
|
self.assertEqual((os.readlink(copy / "link"), os.readlink(copy / "dirlink")), ("a.py", "d"))
|
|
|
|
|
|
class ToolProcess(unittest.TestCase):
|
|
def test_git_lists_nothing_in_a_scratch_folder(self):
|
|
with tempfile.TemporaryDirectory() as top:
|
|
env = {**isolated_env(), "GIT_CONFIG_GLOBAL": "/dev/null"}
|
|
subprocess.run(["git", "init", "-q", top], check=True, env=env)
|
|
scratch = scratch_folder(top)
|
|
pathlib.Path(scratch, "copy.py").write_text("")
|
|
status = subprocess.run(["git", "status", "--porcelain", "--untracked-files=all"], capture_output=True, check=True, cwd=top, encoding="utf-8", env=env).stdout
|
|
self.assertEqual((os.path.dirname(scratch), os.path.basename(scratch)[:16], status), (top, ".mutate-changes-", ""))
|
|
|
|
def test_a_scratch_folder_it_cannot_make_is_a_failure(self):
|
|
self.assertRegex(scratch_folder("/nonexistent/top").reason, "^could not make a scratch folder in /nonexistent/top: ")
|
|
|
|
def test_an_interrupted_run_ends_at_once_with_each_process_the_tool_started(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
# sh starts the sleep in the background, where it ignores SIGINT, so only the kill after the tool ends stops it.
|
|
self.assertLess(self.interrupted(folder, "sleep 60 & echo $! > started; wait;"), 30)
|
|
self.assert_ends(int(pathlib.Path(folder, "started").read_text()))
|
|
|
|
def test_an_interrupt_reaches_the_tool_before_the_run_kills_it(self):
|
|
# A container's CLI, such as docker compose, stops its container on the SIGINT it is passed, and can't once killed.
|
|
# The loop ends by itself after about 40 seconds: under mutmut, the interrupt doesn't reach this shell, and a shell that looped for ever hung the run.
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
self.interrupted(folder, "trap 'echo > stopped; exit 130' INT; echo $$ > started; n=0; while [ $n -lt 400 ]; do sleep 0.1; n=$((n+1)); done;")
|
|
self.assertTrue(pathlib.Path(folder, "stopped").is_file())
|
|
|
|
def test_an_interrupted_tool_that_ignores_sigint_is_killed_after_the_grace_period(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
self.assertLess(self.interrupted(folder, "trap '' INT; echo $$ > started; sleep 60;", grace=0.5), 30)
|
|
self.assert_ends(int(pathlib.Path(folder, "started").read_text()))
|
|
|
|
def test_an_interrupt_ends_the_run_where_the_tool_s_processes_end_first(self):
|
|
killpg = os.killpg
|
|
|
|
def then_gone(group: int, number: int) -> None:
|
|
killpg(group, number)
|
|
raise ProcessLookupError
|
|
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
self.interrupted(folder, "sleep 60 & echo $! > started; wait;", killpg=then_gone)
|
|
|
|
def interrupted(self, folder: str, script: str, **seams: typing.Any) -> float:
|
|
"""Run script as the prefix of a tool in folder, interrupt run_tool once script writes folder/started, and return how long run_tool took."""
|
|
started = pathlib.Path(folder, "started")
|
|
|
|
def interrupt():
|
|
while not started.is_file() or not started.read_text().strip():
|
|
time.sleep(0.05)
|
|
os.kill(os.getpid(), signal.SIGINT)
|
|
|
|
threading.Thread(target=interrupt, daemon=True).start()
|
|
began = time.monotonic()
|
|
with self.assertRaises(KeyboardInterrupt):
|
|
run_tool(["true"], folder, **{"prefix": script, "env": os.environ, "killpg": os.killpg, **seams})
|
|
return time.monotonic() - began
|
|
|
|
def assert_ends(self, pid: int) -> None:
|
|
for _ in range(200):
|
|
if ended(pid):
|
|
return
|
|
time.sleep(0.05)
|
|
self.fail(f"process {pid}, which the tool started, still runs")
|
|
|
|
|
|
def ended(pid: int) -> bool:
|
|
"""Whether a process has ended: gone, or a zombie its parent has yet to reap."""
|
|
try:
|
|
return pathlib.Path(f"/proc/{pid}/stat").read_text().split(") ")[1][0] == "Z"
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
return True
|
|
|
|
|
|
class Took(unittest.TestCase):
|
|
def test_took_rounds_to_the_second_and_counts_minutes_from_one(self):
|
|
self.assertEqual([main.took(seconds) for seconds in (59.4, 59.6, 61, 3725.4)], ["59 s", "1 min 0 s", "1 min 1 s", "62 min 5 s"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|