Run mutate-changes on a script its tests import by file name, and name a crash inside mutmut in one line
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
"""Files whose tests import them by file name, which mutmut names right only from the file's own folder.
|
||||
|
||||
A test in docker/scripts/ that runs `import run_harness` gets it because pytest, or unittest's discover, puts that folder on sys.path. mutmut run from the
|
||||
checkout's top names the same file docker.scripts.run_harness, and finds none of the tests' calls to it."""
|
||||
import ast
|
||||
import os
|
||||
import pathlib
|
||||
import posixpath
|
||||
|
||||
from mutate_changes.changed_lines import is_test
|
||||
|
||||
|
||||
def imported_by_file_name(top: str, grouped: dict[str, dict[str, frozenset[int]]], configs: tuple[str, ...]) -> dict[str, dict[str, frozenset[int]]]:
|
||||
"""grouped, with each file that a test beside it imports by its file name moved to its own folder, named from the checkout's top.
|
||||
|
||||
A folder that holds one of configs keeps its files: the settings there decide how its tests import them."""
|
||||
regrouped: dict[str, dict[str, frozenset[int]]] = {}
|
||||
for folder, files in grouped.items():
|
||||
configured = any(os.path.isfile(os.path.join(top, folder, name)) for name in configs)
|
||||
for path, lines in files.items():
|
||||
own = posixpath.join(folder, posixpath.dirname(path))
|
||||
moved = not configured and posixpath.basename(path)[:-3] in top_level_imports(os.path.join(top, own)) and not os.path.isfile(os.path.join(top, own, "__init__.py"))
|
||||
target, name = (posixpath.normpath(own), posixpath.basename(path)) if moved else (folder, path)
|
||||
regrouped.setdefault(target, {})[name] = lines
|
||||
return regrouped
|
||||
|
||||
|
||||
def top_level_imports(folder: str) -> set[str]:
|
||||
"""The modules the test files in folder import by their first name. A test file that doesn't parse names none."""
|
||||
names: set[str] = set()
|
||||
for test in pathlib.Path(folder).glob("*.py"):
|
||||
if not is_test(test.name):
|
||||
continue
|
||||
try:
|
||||
tree = ast.parse(test.read_text(encoding="utf-8"))
|
||||
except (OSError, SyntaxError, UnicodeDecodeError, ValueError):
|
||||
continue
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
names.update(alias.name.partition(".")[0] for alias in node.names)
|
||||
elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
|
||||
names.add(node.module.partition(".")[0])
|
||||
return names
|
||||
@@ -8,10 +8,11 @@ extension, and leaves out test files: test_*.py, *_test.py, conftest.py, *.test.
|
||||
or __tests__/ folder.
|
||||
|
||||
- .py: mutmut 3, in a scratch copy of the nearest folder above the file that holds a setup.cfg or a pyproject.toml, else of the
|
||||
checkout's top. It mutates each function or method a changed line falls in, and counts the mutants on changed lines. The
|
||||
checkout's top, but a file that a test beside it imports by its file name, as `import run_harness`, from its own folder, which
|
||||
has no setup.cfg or pyproject.toml. It mutates each function or method a changed line falls in, and counts the mutants on changed lines. The
|
||||
folder's setup.cfg settings stay, but source_paths and also_copy are set, and pytest leaves out each test that fails without a
|
||||
mutant; the report names their files. mutmut sees only the code a test runs in its own process, so a mutant that only a child
|
||||
process reaches, such as a tool a test starts, shows as `no tests`. A pyproject.toml with a [tool.mutmut] section fails the run.
|
||||
process reaches, such as a tool a test starts, shows as `no tests`. A pyproject.toml with a [tool.mutmut] section fails the run, and so does a crash inside mutmut, in one line that names the error.
|
||||
- .rs: cargo-mutants, as `cargo mutants --in-diff`, in the checkout's top.
|
||||
- .cjs, .cts, .js, .jsx, .mjs, .mts, .ts, .tsx: Stryker, as `npx --no --package=@stryker-mutator/core stryker run --mutate` on the
|
||||
changed lines, in the nearest folder above the file that holds a package.json. A file with no package.json above it isn't
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
mutmut names a mutant `<module>.<key>__mutmut_<number>`. The key is x_<function> for a function at a module's top, and xǁ<class>ǁ<method> for a method of a class there; mutmut mutates no other code. Its mutants/ copy of a file holds the original of each function mutmut mutates, as `<key>__mutmut_orig`, and each mutant as a changed copy of that original."""
|
||||
import ast
|
||||
import functools
|
||||
import re
|
||||
from typing import NamedTuple
|
||||
from collections.abc import Sequence
|
||||
|
||||
@@ -40,6 +41,20 @@ def named(name: str, paths: list[str]) -> tuple[str, str, int] | None:
|
||||
return (found[0], key, int(number)) if found and number.isdigit() else None
|
||||
|
||||
|
||||
TRACEBACK = "Traceback (most recent call last):"
|
||||
FRAME = re.compile(r'^\s*File ".*?[/\\](mutmut[/\\].*?|[^/\\]*?)", line (\d+)', re.MULTILINE)
|
||||
|
||||
|
||||
def crash_in_mutmut(output: str) -> str | None:
|
||||
"""The error and place of the last traceback in output, where it ends in a frame inside mutmut's own code, else None."""
|
||||
block = output.rpartition(TRACEBACK)[2] if TRACEBACK in output else ""
|
||||
frames = FRAME.findall(block)
|
||||
lines = block.strip().splitlines()
|
||||
if not frames or not lines or not frames[-1][0].startswith("mutmut"):
|
||||
return None
|
||||
return f"{lines[-1]}, in {frames[-1][0].replace(chr(92), '/')} line {frames[-1][1]}"
|
||||
|
||||
|
||||
def statuses(results: str) -> dict[str, str]:
|
||||
"""Each mutant's status by name, from the ` <name>: <status>` lines `mutmut results --all true` prints."""
|
||||
return {name: status for name, _, status in (line.strip().partition(": ") for line in results.splitlines()) if "__mutmut_" in name and status}
|
||||
|
||||
@@ -10,10 +10,11 @@ import posixpath
|
||||
import shutil
|
||||
import tomllib
|
||||
|
||||
from mutate_changes.bare_imports import imported_by_file_name
|
||||
from mutate_changes.changed_lines import by_folder, label
|
||||
from mutate_changes.tool_process import failed, Ran, remove, Run, scratch_folder
|
||||
from failure import Failure
|
||||
from mutate_changes.mutmut_mutants import function_keys, mutant_globs, mutated_line, named, statuses
|
||||
from mutate_changes.mutmut_mutants import crash_in_mutmut, function_keys, mutant_globs, mutated_line, named, statuses
|
||||
from git_cli.git_repo import Git
|
||||
from mutate_changes.mutmut_setup import left_out, mutmut_config
|
||||
|
||||
@@ -47,7 +48,7 @@ def run_mutmut(top: str, changed: dict[str, frozenset[int]], run: Run, git: Git,
|
||||
|
||||
changed maps each file, by its path from top, to its changed lines."""
|
||||
report: list[str] = []
|
||||
for path, files in sorted(by_folder(top, changed, CONFIGS).items()):
|
||||
for path, files in sorted(imported_by_file_name(top, by_folder(top, changed, CONFIGS), CONFIGS).items()):
|
||||
lines = mutate_folder(top, path, files, run, git, jobs)
|
||||
if isinstance(lines, Failure):
|
||||
return lines
|
||||
@@ -152,6 +153,9 @@ def run_until_stats_pass(copy: str, copied: list[str], folder: Folder, argv: lis
|
||||
if isinstance(ran, Failure):
|
||||
return ran
|
||||
if ran.code and NO_MUTANT not in ran.output and NO_TEST not in ran.output:
|
||||
crash = crash_in_mutmut(ran.output)
|
||||
if crash:
|
||||
return Failure(f"mutmut crashed in {folder.label} with {crash}: mutate the changed lines of {folder.label} by hand.")
|
||||
return failed(f"mutmut run in {folder.label}", ran)
|
||||
return ran.output, left
|
||||
|
||||
|
||||
@@ -13,10 +13,11 @@ import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from mutate_changes.bare_imports import imported_by_file_name
|
||||
from mutate_changes.changed_lines import added_lines, by_folder, not_mutated, runs, tool_for
|
||||
from mutate_changes.mutmut_tool import copy_files
|
||||
from failure import Failure
|
||||
from mutate_changes.mutmut_mutants import function_keys, mutant_globs, mutated_line, named, statuses
|
||||
from mutate_changes.mutmut_mutants import crash_in_mutmut, function_keys, mutant_globs, mutated_line, named, statuses
|
||||
from test_env import isolated_env
|
||||
from mutate_changes.mutmut_setup import left_out, mutmut_config
|
||||
from mutate_changes import main
|
||||
@@ -88,6 +89,36 @@ class ChangedLines(unittest.TestCase):
|
||||
self.assertEqual(grouped, {"pkg": {"inner/deep.py": lines}, "pkg/inner/lib": {"x.py": lines}, ".": {"top.py": lines}})
|
||||
self.assertEqual(by_folder(top, {"web/src/a.ts": lines, "pkg/b.ts": lines}, ("package.json",)), {"web": {"src/a.ts": lines}, ".": {"pkg/b.ts": lines}})
|
||||
|
||||
def test_a_file_its_sibling_test_imports_by_file_name_moves_to_its_own_folder(self):
|
||||
with tempfile.TemporaryDirectory() as top:
|
||||
tests = {
|
||||
"docker/scripts/test_run.py": "import unittest\nimport run_harness as rh\n",
|
||||
"docker/scripts/test_other.py": "from other import thing\n",
|
||||
"pkg/test_pkg.py": "from pkg.mod import x\n",
|
||||
"pkg/__init__.py": "",
|
||||
"lib/test_lib.py": "import lib_tool\n",
|
||||
"lib/__init__.py": "",
|
||||
"broken/test_broken.py": "def (:\n",
|
||||
}
|
||||
for path, text in tests.items():
|
||||
pathlib.Path(top, path).parent.mkdir(parents=True, exist_ok=True)
|
||||
pathlib.Path(top, path).write_text(text)
|
||||
lines = frozenset({1})
|
||||
grouped = {".": {name: lines for name in ("docker/scripts/run_harness.py", "docker/scripts/solo.py", "pkg/mod.py", "lib/lib_tool.py", "broken/broken.py")}}
|
||||
self.assertEqual(imported_by_file_name(top, grouped, ("setup.cfg",)), {
|
||||
".": {"docker/scripts/solo.py": lines, "pkg/mod.py": lines, "lib/lib_tool.py": lines, "broken/broken.py": lines},
|
||||
"docker/scripts": {"run_harness.py": lines},
|
||||
})
|
||||
pathlib.Path(top, "setup.cfg").write_text("")
|
||||
self.assertEqual(imported_by_file_name(top, grouped, ("setup.cfg",)), grouped)
|
||||
|
||||
def test_a_traceback_that_ends_inside_mutmut_is_a_crash_of_mutmut(self):
|
||||
inside = 'Traceback (most recent call last):\n File "/a/mutmut/__main__.py", line 9, in run\n File "/a/mutmut/workers/isolation.py", line 889, in wait\nKeyError: 974598\n'
|
||||
outside = 'Traceback (most recent call last):\n File "/repo/calc.py", line 3, in add\nKeyError: 1\n'
|
||||
self.assertEqual(crash_in_mutmut(inside), "KeyError: 974598, in mutmut/workers/isolation.py line 889")
|
||||
self.assertEqual(crash_in_mutmut("Running stats\n" + inside + "\n"), "KeyError: 974598, in mutmut/workers/isolation.py line 889")
|
||||
self.assertEqual((crash_in_mutmut(outside), crash_in_mutmut("all fine"), crash_in_mutmut("")), (None, None, None))
|
||||
|
||||
def test_not_mutated_names_the_files_no_tool_covers_then_the_test_files(self):
|
||||
self.assertEqual(not_mutated(["README.md", "a.py", "b.go", "test_a.py", "web/a.test.ts"]), ["Not mutated, as no tool covers their language: README.md, b.go.", "Not mutated, as test files: test_a.py, web/a.test.ts."])
|
||||
self.assertEqual((not_mutated(["a.py", "test_a.py"]), not_mutated(["a.py", "b.go"]), not_mutated([])), (["Not mutated, as test files: test_a.py."], ["Not mutated, as no tool covers their language: b.go."], []))
|
||||
|
||||
@@ -58,6 +58,9 @@ else:
|
||||
if mode == "setup-fault":
|
||||
print("Stopping early, because tests recorded trampoline hits but none match any mutant key. This usually means imports bypass the mutants.")
|
||||
sys.exit(1)
|
||||
if mode == "crash":
|
||||
print("Traceback (most recent call last):\n File \"/uv/site-packages/mutmut/__main__.py\", line 1050, in _run\n File \"/uv/site-packages/mutmut/workers/isolation.py\", line 889, in wait_for_result\nKeyError: 974598")
|
||||
sys.exit(1)
|
||||
"""
|
||||
# cargo-mutants, Infection and Stryker, which log the folder they run in and how they were called, and exit with the code the test
|
||||
# gives. cargo-mutants also logs the diff it is given and writes its outcome files, all but unviable.txt, as a run without an
|
||||
@@ -205,6 +208,10 @@ class MutateChangesRuns(unittest.TestCase):
|
||||
ran = self.run_tool(*self.faked("no-test"))
|
||||
self.assertEqual(ran.stdout.splitlines()[:3], ["mutmut in pkg: 2 mutants on changed lines, 2 not killed. No test reaches them.", "pkg/calc.py:6: no tests: return n * 4", "pkg/calc.py:6: no tests: return n / 3"], ran.stderr)
|
||||
|
||||
def test_a_crash_inside_mutmut_fails_the_run_in_one_line_naming_mutmut_and_the_error(self):
|
||||
ran = self.run_tool(*self.faked("crash"))
|
||||
self.assertEqual((ran.returncode, ran.stderr), (1, "mutmut crashed in pkg with KeyError: 974598, in mutmut/workers/isolation.py line 889: mutate the changed lines of pkg by hand.\n"))
|
||||
|
||||
def test_a_run_that_stops_early_for_another_reason_fails_the_run(self):
|
||||
ran = self.run_tool(*self.faked("setup-fault"))
|
||||
self.assertEqual((ran.returncode, ran.stderr.splitlines()[0]), (1, "mutmut run in pkg exited 1:"))
|
||||
|
||||
Reference in New Issue
Block a user