Run mutate-changes on a script its tests import by file name, and name a crash inside mutmut in one line

This commit is contained in:
Lilleman
2026-11-20 19:12:10 +01:00
parent cd8e335c3e
commit 91cf86e0cd
6 changed files with 106 additions and 5 deletions
+43
View File
@@ -0,0 +1,43 @@
"""Files whose tests import them by file name, which mutmut names right only from the file's own folder.
A test in docker/scripts/ that runs `import run_harness` gets it because pytest, or unittest's discover, puts that folder on sys.path. mutmut run from the
checkout's top names the same file docker.scripts.run_harness, and finds none of the tests' calls to it."""
import ast
import os
import pathlib
import posixpath
from mutate_changes.changed_lines import is_test
def imported_by_file_name(top: str, grouped: dict[str, dict[str, frozenset[int]]], configs: tuple[str, ...]) -> dict[str, dict[str, frozenset[int]]]:
"""grouped, with each file that a test beside it imports by its file name moved to its own folder, named from the checkout's top.
A folder that holds one of configs keeps its files: the settings there decide how its tests import them."""
regrouped: dict[str, dict[str, frozenset[int]]] = {}
for folder, files in grouped.items():
configured = any(os.path.isfile(os.path.join(top, folder, name)) for name in configs)
for path, lines in files.items():
own = posixpath.join(folder, posixpath.dirname(path))
moved = not configured and posixpath.basename(path)[:-3] in top_level_imports(os.path.join(top, own)) and not os.path.isfile(os.path.join(top, own, "__init__.py"))
target, name = (posixpath.normpath(own), posixpath.basename(path)) if moved else (folder, path)
regrouped.setdefault(target, {})[name] = lines
return regrouped
def top_level_imports(folder: str) -> set[str]:
"""The modules the test files in folder import by their first name. A test file that doesn't parse names none."""
names: set[str] = set()
for test in pathlib.Path(folder).glob("*.py"):
if not is_test(test.name):
continue
try:
tree = ast.parse(test.read_text(encoding="utf-8"))
except (OSError, SyntaxError, UnicodeDecodeError, ValueError):
continue
for node in ast.walk(tree):
if isinstance(node, ast.Import):
names.update(alias.name.partition(".")[0] for alias in node.names)
elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
names.add(node.module.partition(".")[0])
return names
+3 -2
View File
@@ -8,10 +8,11 @@ extension, and leaves out test files: test_*.py, *_test.py, conftest.py, *.test.
or __tests__/ folder.
- .py: mutmut 3, in a scratch copy of the nearest folder above the file that holds a setup.cfg or a pyproject.toml, else of the
checkout's top. It mutates each function or method a changed line falls in, and counts the mutants on changed lines. The
checkout's top, but a file that a test beside it imports by its file name, as `import run_harness`, from its own folder, which
has no setup.cfg or pyproject.toml. It mutates each function or method a changed line falls in, and counts the mutants on changed lines. The
folder's setup.cfg settings stay, but source_paths and also_copy are set, and pytest leaves out each test that fails without a
mutant; the report names their files. mutmut sees only the code a test runs in its own process, so a mutant that only a child
process reaches, such as a tool a test starts, shows as `no tests`. A pyproject.toml with a [tool.mutmut] section fails the run.
process reaches, such as a tool a test starts, shows as `no tests`. A pyproject.toml with a [tool.mutmut] section fails the run, and so does a crash inside mutmut, in one line that names the error.
- .rs: cargo-mutants, as `cargo mutants --in-diff`, in the checkout's top.
- .cjs, .cts, .js, .jsx, .mjs, .mts, .ts, .tsx: Stryker, as `npx --no --package=@stryker-mutator/core stryker run --mutate` on the
changed lines, in the nearest folder above the file that holds a package.json. A file with no package.json above it isn't
+15
View File
@@ -3,6 +3,7 @@
mutmut names a mutant `<module>.<key>__mutmut_<number>`. The key is x_<function> for a function at a module's top, and xǁ<class>ǁ<method> for a method of a class there; mutmut mutates no other code. Its mutants/ copy of a file holds the original of each function mutmut mutates, as `<key>__mutmut_orig`, and each mutant as a changed copy of that original."""
import ast
import functools
import re
from typing import NamedTuple
from collections.abc import Sequence
@@ -40,6 +41,20 @@ def named(name: str, paths: list[str]) -> tuple[str, str, int] | None:
return (found[0], key, int(number)) if found and number.isdigit() else None
TRACEBACK = "Traceback (most recent call last):"
FRAME = re.compile(r'^\s*File ".*?[/\\](mutmut[/\\].*?|[^/\\]*?)", line (\d+)', re.MULTILINE)
def crash_in_mutmut(output: str) -> str | None:
"""The error and place of the last traceback in output, where it ends in a frame inside mutmut's own code, else None."""
block = output.rpartition(TRACEBACK)[2] if TRACEBACK in output else ""
frames = FRAME.findall(block)
lines = block.strip().splitlines()
if not frames or not lines or not frames[-1][0].startswith("mutmut"):
return None
return f"{lines[-1]}, in {frames[-1][0].replace(chr(92), '/')} line {frames[-1][1]}"
def statuses(results: str) -> dict[str, str]:
"""Each mutant's status by name, from the ` <name>: <status>` lines `mutmut results --all true` prints."""
return {name: status for name, _, status in (line.strip().partition(": ") for line in results.splitlines()) if "__mutmut_" in name and status}
+6 -2
View File
@@ -10,10 +10,11 @@ import posixpath
import shutil
import tomllib
from mutate_changes.bare_imports import imported_by_file_name
from mutate_changes.changed_lines import by_folder, label
from mutate_changes.tool_process import failed, Ran, remove, Run, scratch_folder
from failure import Failure
from mutate_changes.mutmut_mutants import function_keys, mutant_globs, mutated_line, named, statuses
from mutate_changes.mutmut_mutants import crash_in_mutmut, function_keys, mutant_globs, mutated_line, named, statuses
from git_cli.git_repo import Git
from mutate_changes.mutmut_setup import left_out, mutmut_config
@@ -47,7 +48,7 @@ def run_mutmut(top: str, changed: dict[str, frozenset[int]], run: Run, git: Git,
changed maps each file, by its path from top, to its changed lines."""
report: list[str] = []
for path, files in sorted(by_folder(top, changed, CONFIGS).items()):
for path, files in sorted(imported_by_file_name(top, by_folder(top, changed, CONFIGS), CONFIGS).items()):
lines = mutate_folder(top, path, files, run, git, jobs)
if isinstance(lines, Failure):
return lines
@@ -152,6 +153,9 @@ def run_until_stats_pass(copy: str, copied: list[str], folder: Folder, argv: lis
if isinstance(ran, Failure):
return ran
if ran.code and NO_MUTANT not in ran.output and NO_TEST not in ran.output:
crash = crash_in_mutmut(ran.output)
if crash:
return Failure(f"mutmut crashed in {folder.label} with {crash}: mutate the changed lines of {folder.label} by hand.")
return failed(f"mutmut run in {folder.label}", ran)
return ran.output, left
+32 -1
View File
@@ -13,10 +13,11 @@ import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from mutate_changes.bare_imports import imported_by_file_name
from mutate_changes.changed_lines import added_lines, by_folder, not_mutated, runs, tool_for
from mutate_changes.mutmut_tool import copy_files
from failure import Failure
from mutate_changes.mutmut_mutants import function_keys, mutant_globs, mutated_line, named, statuses
from mutate_changes.mutmut_mutants import crash_in_mutmut, function_keys, mutant_globs, mutated_line, named, statuses
from test_env import isolated_env
from mutate_changes.mutmut_setup import left_out, mutmut_config
from mutate_changes import main
@@ -88,6 +89,36 @@ class ChangedLines(unittest.TestCase):
self.assertEqual(grouped, {"pkg": {"inner/deep.py": lines}, "pkg/inner/lib": {"x.py": lines}, ".": {"top.py": lines}})
self.assertEqual(by_folder(top, {"web/src/a.ts": lines, "pkg/b.ts": lines}, ("package.json",)), {"web": {"src/a.ts": lines}, ".": {"pkg/b.ts": lines}})
def test_a_file_its_sibling_test_imports_by_file_name_moves_to_its_own_folder(self):
with tempfile.TemporaryDirectory() as top:
tests = {
"docker/scripts/test_run.py": "import unittest\nimport run_harness as rh\n",
"docker/scripts/test_other.py": "from other import thing\n",
"pkg/test_pkg.py": "from pkg.mod import x\n",
"pkg/__init__.py": "",
"lib/test_lib.py": "import lib_tool\n",
"lib/__init__.py": "",
"broken/test_broken.py": "def (:\n",
}
for path, text in tests.items():
pathlib.Path(top, path).parent.mkdir(parents=True, exist_ok=True)
pathlib.Path(top, path).write_text(text)
lines = frozenset({1})
grouped = {".": {name: lines for name in ("docker/scripts/run_harness.py", "docker/scripts/solo.py", "pkg/mod.py", "lib/lib_tool.py", "broken/broken.py")}}
self.assertEqual(imported_by_file_name(top, grouped, ("setup.cfg",)), {
".": {"docker/scripts/solo.py": lines, "pkg/mod.py": lines, "lib/lib_tool.py": lines, "broken/broken.py": lines},
"docker/scripts": {"run_harness.py": lines},
})
pathlib.Path(top, "setup.cfg").write_text("")
self.assertEqual(imported_by_file_name(top, grouped, ("setup.cfg",)), grouped)
def test_a_traceback_that_ends_inside_mutmut_is_a_crash_of_mutmut(self):
inside = 'Traceback (most recent call last):\n File "/a/mutmut/__main__.py", line 9, in run\n File "/a/mutmut/workers/isolation.py", line 889, in wait\nKeyError: 974598\n'
outside = 'Traceback (most recent call last):\n File "/repo/calc.py", line 3, in add\nKeyError: 1\n'
self.assertEqual(crash_in_mutmut(inside), "KeyError: 974598, in mutmut/workers/isolation.py line 889")
self.assertEqual(crash_in_mutmut("Running stats\n" + inside + "\n"), "KeyError: 974598, in mutmut/workers/isolation.py line 889")
self.assertEqual((crash_in_mutmut(outside), crash_in_mutmut("all fine"), crash_in_mutmut("")), (None, None, None))
def test_not_mutated_names_the_files_no_tool_covers_then_the_test_files(self):
self.assertEqual(not_mutated(["README.md", "a.py", "b.go", "test_a.py", "web/a.test.ts"]), ["Not mutated, as no tool covers their language: README.md, b.go.", "Not mutated, as test files: test_a.py, web/a.test.ts."])
self.assertEqual((not_mutated(["a.py", "test_a.py"]), not_mutated(["a.py", "b.go"]), not_mutated([])), (["Not mutated, as test files: test_a.py."], ["Not mutated, as no tool covers their language: b.go."], []))
@@ -58,6 +58,9 @@ else:
if mode == "setup-fault":
print("Stopping early, because tests recorded trampoline hits but none match any mutant key. This usually means imports bypass the mutants.")
sys.exit(1)
if mode == "crash":
print("Traceback (most recent call last):\n File \"/uv/site-packages/mutmut/__main__.py\", line 1050, in _run\n File \"/uv/site-packages/mutmut/workers/isolation.py\", line 889, in wait_for_result\nKeyError: 974598")
sys.exit(1)
"""
# cargo-mutants, Infection and Stryker, which log the folder they run in and how they were called, and exit with the code the test
# gives. cargo-mutants also logs the diff it is given and writes its outcome files, all but unviable.txt, as a run without an
@@ -205,6 +208,10 @@ class MutateChangesRuns(unittest.TestCase):
ran = self.run_tool(*self.faked("no-test"))
self.assertEqual(ran.stdout.splitlines()[:3], ["mutmut in pkg: 2 mutants on changed lines, 2 not killed. No test reaches them.", "pkg/calc.py:6: no tests: return n * 4", "pkg/calc.py:6: no tests: return n / 3"], ran.stderr)
def test_a_crash_inside_mutmut_fails_the_run_in_one_line_naming_mutmut_and_the_error(self):
ran = self.run_tool(*self.faked("crash"))
self.assertEqual((ran.returncode, ran.stderr), (1, "mutmut crashed in pkg with KeyError: 974598, in mutmut/workers/isolation.py line 889: mutate the changed lines of pkg by hand.\n"))
def test_a_run_that_stops_early_for_another_reason_fails_the_run(self):
ran = self.run_tool(*self.faked("setup-fault"))
self.assertEqual((ran.returncode, ran.stderr.splitlines()[0]), (1, "mutmut run in pkg exited 1:"))