Answer stability round 1: 0 🔴, 2 🟡 (0 misread); tell mutmut's own frames by their site-packages path, take the error from after the last frame, and fix the module docstring and help

This commit is contained in:
Lilleman
2026-11-20 21:26:22 +01:00
parent 8d896d772d
commit ce0344f703
6 changed files with 55 additions and 43 deletions
+5 -5
View File
@@ -1,7 +1,7 @@
"""Files whose tests import them by file name, which mutmut names right only from the file's own folder.
"""Files that tests import by their file name, which mutmut finds only when it runs in the file's own folder.
A test in docker/scripts/ that runs `import run_harness` gets it because pytest, or unittest's discover, puts that folder on sys.path. mutmut run from the
checkout's top names the same file docker.scripts.run_harness, and finds none of the tests' calls to it."""
checkout's top names the same file docker.scripts.run_harness, so it finds none of the tests' calls to it."""
import ast
import os
import pathlib
@@ -11,9 +11,9 @@ from mutate_changes.changed_lines import is_test
def imported_by_file_name(top: str, grouped: dict[str, dict[str, frozenset[int]]], configs: tuple[str, ...]) -> dict[str, dict[str, frozenset[int]]]:
"""grouped, with each file of the checkout's top group that a test beside it imports by its file name moved to its own folder.
"""grouped, with each file in its "." group moved to its own folder where a test beside it imports it by its file name and the folder has no __init__.py.
A file in a folder that holds one of configs stays: the settings there decide how its tests import it. So do all files where the top holds one."""
A file in a folder that holds one of configs stays, as does every file where the top holds one: the settings there decide how its tests import it."""
if "." not in grouped or any(os.path.isfile(os.path.join(top, name)) for name in configs):
return grouped
regrouped: dict[str, dict[str, frozenset[int]]] = {name: files for name, files in grouped.items() if name != "."}
@@ -25,7 +25,7 @@ def imported_by_file_name(top: str, grouped: dict[str, dict[str, frozenset[int]]
def top_level_imports(folder: str) -> set[str]:
"""The modules the test files in folder import by their first name. A test file that doesn't parse names none."""
"""The top-level module names that the test files in folder import, as `a` for `import a.b`. A test file that can't be read or parsed adds none."""
names: set[str] = set()
for test in sorted(pathlib.Path(folder).glob("*.py")):
if not is_test(test.name):
+8 -5
View File
@@ -8,11 +8,14 @@ extension, and leaves out test files: test_*.py, *_test.py, conftest.py, *.test.
or __tests__/ folder.
- .py: mutmut 3, in a scratch copy of the nearest folder above the file that holds a setup.cfg or a pyproject.toml, else of the
checkout's top, but a file that a test beside it imports by its file name, as `import run_harness`, from its own folder, which
has no setup.cfg or pyproject.toml. It mutates each function or method a changed line falls in, and counts the mutants on changed lines. The
folder's setup.cfg settings stay, but source_paths and also_copy are set, and pytest leaves out each test that fails without a
mutant; the report names their files. mutmut sees only the code a test runs in its own process, so a mutant that only a child
process reaches, such as a tool a test starts, shows as `no tests`. A pyproject.toml with a [tool.mutmut] section fails the run, and so does a crash inside mutmut, in one line that names the error.
checkout's top. The exception is a file that a test beside it imports by its file name, as `import run_harness`, where the
folder has no __init__.py and neither it nor the checkout's top holds a setup.cfg or pyproject.toml: mutmut runs in that file's
own folder, as there the tests find the file under the same name. It mutates each function or method a changed line falls in,
and counts the mutants on changed lines. The folder's setup.cfg settings stay, but source_paths and also_copy are set, and
pytest leaves out each test that fails without a mutant; the report names their files. mutmut sees only the code a test runs
in its own process, so a mutant that only a child process reaches, such as a tool a test starts, shows as `no tests`. A
pyproject.toml with a [tool.mutmut] section fails the run. A crash inside mutmut, such as a KeyError in its worker pool,
becomes one line for its folder, naming the error and telling you to mutate that folder's changed lines by hand.
- .rs: cargo-mutants, as `cargo mutants --in-diff`, in the checkout's top.
- .cjs, .cts, .js, .jsx, .mjs, .mts, .ts, .tsx: Stryker, as `npx --no --package=@stryker-mutator/core stryker run --mutate` on the
changed lines, in the nearest folder above the file that holds a package.json. A file with no package.json above it isn't
+9 -7
View File
@@ -3,8 +3,8 @@
mutmut names a mutant `<module>.<key>__mutmut_<number>`. The key is x_<function> for a function at a module's top, and xǁ<class>ǁ<method> for a method of a class there; mutmut mutates no other code. Its mutants/ copy of a file holds the original of each function mutmut mutates, as `<key>__mutmut_orig`, and each mutant as a changed copy of that original."""
import ast
import functools
import re
from typing import NamedTuple
import re
from collections.abc import Sequence
@@ -42,17 +42,19 @@ def named(name: str, paths: list[str]) -> tuple[str, str, int] | None:
TRACEBACK = "Traceback (most recent call last):"
FRAME = re.compile(r'^\s*File ".*?[/\\](mutmut[/\\].*?|[^/\\]*?)", line (\d+)', re.MULTILINE)
FRAME = re.compile(r'^\s*File "(.*?)", line (\d+)', re.MULTILINE)
MUTMUT_FILE = re.compile(r"[/\\]site-packages[/\\](mutmut[/\\].*)")
def crash_in_mutmut(output: str) -> str | None:
"""The error and place of the last traceback in output, where it ends in a frame inside mutmut's own code, else None."""
"""The error and place of the last traceback in output, where its last frame is in the mutmut package, else None."""
block = output.rpartition(TRACEBACK)[2] if TRACEBACK in output else ""
frames = FRAME.findall(block)
lines = block.strip().splitlines()
if not frames or not lines or not frames[-1][0].startswith("mutmut"):
frames = list(FRAME.finditer(block))
if not frames:
return None
return f"{lines[-1]}, in {frames[-1][0].replace(chr(92), '/')} line {frames[-1][1]}"
installed = MUTMUT_FILE.search(frames[-1][1])
error = next((line for line in block[frames[-1].end():].splitlines()[1:] if line and not line[0].isspace()), "")
return f"{error}, in {installed[1].replace(chr(92), '/')} line {frames[-1][2]}" if installed and error else None
def statuses(results: str) -> dict[str, str]:
+12 -12
View File
@@ -1,8 +1,8 @@
"""Mutate the changed lines of a branch's Python files with mutmut 3, in a scratch copy of the folder each file's tests run from.
mutmut reads its settings from the folder it runs in, writes beside them, and names each module by its path from there. So each file is
mutated from the nearest folder above it, up to the checkout's top, that holds a setup.cfg or a pyproject.toml: the folder its tests
import it from."""
mutated from the folder its tests import it from: the nearest folder above it, up to the checkout's top, that holds a setup.cfg or a
pyproject.toml, or the file's own folder where a test beside it imports it by its file name (bare_imports.py)."""
import dataclasses
import os
import pathlib
@@ -10,12 +10,12 @@ import posixpath
import shutil
import tomllib
from mutate_changes.bare_imports import imported_by_file_name
from mutate_changes.changed_lines import by_folder, label
from mutate_changes.mutmut_mutants import crash_in_mutmut, function_keys, mutant_globs, mutated_line, named, statuses
from mutate_changes.tool_process import failed, Ran, remove, Run, scratch_folder
from failure import Failure
from mutate_changes.mutmut_mutants import crash_in_mutmut, function_keys, mutant_globs, mutated_line, named, statuses
from git_cli.git_repo import Git
from mutate_changes.bare_imports import imported_by_file_name
from mutate_changes.mutmut_setup import left_out, mutmut_config
CAUGHT = frozenset({"caught by type check", "killed"})
@@ -127,7 +127,10 @@ def mutate_copy(copy: str, copied: list[str], folder: Folder, argv: list[str], r
ran = run_until_stats_pass(copy, copied, folder, argv, run)
if isinstance(ran, Failure):
return ran
output, left = ran
(output, code), left = ran
crash = crash_in_mutmut(output)
if code and crash:
return [f"mutmut in {folder.label} crashed with {crash}: mutate its changed lines by hand."]
if not os.path.isdir(os.path.join(copy, "mutants")):
return Failure(f"mutmut in {folder.label} wrote no mutants/ in {copy}; --prefix must run it in $PWD")
if NO_MUTANT in output:
@@ -142,8 +145,8 @@ def mutate_copy(copy: str, copied: list[str], folder: Folder, argv: list[str], r
return [f"mutmut in {folder.label}: {counted} mutants on changed lines, {len(shown)} not killed.{unreached}{left_note}", *shown]
def run_until_stats_pass(copy: str, copied: list[str], folder: Folder, argv: list[str], run: Run) -> tuple[str, list[str]] | Failure:
"""mutmut's output, and the options that left out each test that fails before any mutant: mutmut runs no mutant while one fails."""
def run_until_stats_pass(copy: str, copied: list[str], folder: Folder, argv: list[str], run: Run) -> tuple[tuple[str, int], list[str]] | Failure:
"""mutmut's output and exit code, and the options that left out each test that fails before any mutant: mutmut runs no mutant while one fails."""
ran = configured_run(copy, copied, folder, [], argv, run)
left: list[str] = []
if isinstance(ran, Ran) and STATS_FAILED in ran.output:
@@ -152,12 +155,9 @@ def run_until_stats_pass(copy: str, copied: list[str], folder: Folder, argv: lis
ran = configured_run(copy, copied, folder, left, argv, run)
if isinstance(ran, Failure):
return ran
if ran.code and NO_MUTANT not in ran.output and NO_TEST not in ran.output:
crash = crash_in_mutmut(ran.output)
if crash:
return Failure(f"mutmut crashed in {folder.label} with {crash}: mutate the changed lines of {folder.label} by hand.")
if ran.code and NO_MUTANT not in ran.output and NO_TEST not in ran.output and not crash_in_mutmut(ran.output):
return failed(f"mutmut run in {folder.label}", ran)
return ran.output, left
return (ran.output, ran.code), left
def configured_run(copy: str, copied: list[str], folder: Folder, options: list[str], argv: list[str], run: Run) -> Ran | Failure:
+18 -11
View File
@@ -13,11 +13,11 @@ import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from mutate_changes.bare_imports import imported_by_file_name
from mutate_changes.changed_lines import added_lines, by_folder, not_mutated, runs, tool_for
from mutate_changes.mutmut_tool import copy_files
from failure import Failure
from mutate_changes.mutmut_mutants import crash_in_mutmut, function_keys, mutant_globs, mutated_line, named, statuses
from failure import Failure
from mutate_changes.bare_imports import imported_by_file_name
from test_env import isolated_env
from mutate_changes.mutmut_setup import left_out, mutmut_config
from mutate_changes import main
@@ -120,17 +120,24 @@ class ChangedLines(unittest.TestCase):
self.assertEqual(imported_by_file_name(top, grouped, ("setup.cfg",)), grouped)
def test_a_traceback_that_ends_inside_mutmut_is_a_crash_of_mutmut(self):
def trace(*frames, error="KeyError: 974598"):
return "Traceback (most recent call last):\n" + "".join(f' File "{path}", line {number}, in f\n' for path, number in frames) + error + "\n"
inside = trace(("/a/mutmut/__main__.py", 9), ("/a/mutmut/workers/isolation.py", 889), ("/a/mutmut/workers/isolation.py", 12))
outside = trace(("/a/mutmut/__main__.py", 9), ("/repo/calc.py", 3), error="KeyError: 1")
def trace(*frames, error="KeyError: 974598", after=""):
shown = "".join(f' File "{path}", line {number}, in f\n code()\n ~~^^\n' for path, number in frames)
return f"Traceback (most recent call last):\n{shown}{error}\n{after}"
lib = "/uv/lib/python3.14/site-packages/mutmut"
inside = trace((f"{lib}/__main__.py", 9), (f"{lib}/workers/isolation.py", 889), (f"{lib}/workers/isolation.py", 12))
outside = trace((f"{lib}/__main__.py", 9), ("/repo/calc.py", 3), error="KeyError: 1")
place = "KeyError: 974598, in mutmut/workers/isolation.py line 12"
self.assertEqual((crash_in_mutmut(inside), crash_in_mutmut("Running stats\n" + inside + "\n")), (place, place))
self.assertEqual(crash_in_mutmut(outside + trace(("/a/mutmut/x.py", 1), error="E: 2")), "E: 2, in mutmut/x.py line 1")
self.assertEqual(crash_in_mutmut(trace(("/a/mutmut/x.py", 1), error="E: 2") + outside), None)
self.assertEqual(crash_in_mutmut(trace(("C:\\py\\mutmut\\workers\\x.py", 5))), "KeyError: 974598, in mutmut/workers/x.py line 5")
self.assertEqual(crash_in_mutmut(trace(("/a/mutmut/x.py", 1), error="").replace("Traceback (most recent call last):", "")), None)
self.assertEqual((crash_in_mutmut(outside), crash_in_mutmut("all fine"), crash_in_mutmut(""), crash_in_mutmut("Traceback (most recent call last):\nKeyError: 1")), (None,) * 4)
self.assertEqual(crash_in_mutmut(inside + "later output\n"), place)
self.assertEqual(crash_in_mutmut(outside + trace((f"{lib}/x.py", 1), error="E: 2")), "E: 2, in mutmut/x.py line 1")
self.assertEqual(crash_in_mutmut(trace((f"{lib}/x.py", 1), error="E: 2") + outside), None)
self.assertEqual(crash_in_mutmut(trace(("C:\\py\\Lib\\site-packages\\mutmut\\workers\\x.py", 5))), "KeyError: 974598, in mutmut/workers/x.py line 5")
for path in ("/mutmut/repo/calc.py", "mutmut_tool.py", "<string>", "/a/site-packages/mutmut_tool.py"):
with self.subTest(path):
self.assertEqual(crash_in_mutmut(trace((path, 1))), None)
self.assertEqual(crash_in_mutmut(trace((f"{lib}/x.py", 1)).replace("Traceback (most recent call last):", "")), None)
self.assertEqual((crash_in_mutmut("all fine"), crash_in_mutmut(""), crash_in_mutmut("Traceback (most recent call last):\nKeyError: 1")), (None,) * 3)
self.assertEqual(crash_in_mutmut(trace((f"{lib}/x.py", 1), error="")), None)
def test_not_mutated_names_the_files_no_tool_covers_then_the_test_files(self):
self.assertEqual(not_mutated(["README.md", "a.py", "b.go", "test_a.py", "web/a.test.ts"]), ["Not mutated, as no tool covers their language: README.md, b.go.", "Not mutated, as test files: test_a.py, web/a.test.ts."])
@@ -59,7 +59,7 @@ else:
print("Stopping early, because tests recorded trampoline hits but none match any mutant key. This usually means imports bypass the mutants.")
sys.exit(1)
if mode == "crash":
print("Traceback (most recent call last):\n File \"/uv/site-packages/mutmut/__main__.py\", line 1050, in _run\n File \"/uv/site-packages/mutmut/workers/isolation.py\", line 889, in wait_for_result\nKeyError: 974598")
print("Traceback (most recent call last):\n File \"/uv/site-packages/mutmut/__main__.py\", line 1050, in _run\n File \"/uv/site-packages/mutmut/workers/isolation.py\", line 889, in wait_for_result\n worker = pop(pid)\nKeyError: 974598")
sys.exit(1)
"""
# cargo-mutants, Infection and Stryker, which log the folder they run in and how they were called, and exit with the code the test
@@ -208,9 +208,9 @@ class MutateChangesRuns(unittest.TestCase):
ran = self.run_tool(*self.faked("no-test"))
self.assertEqual(ran.stdout.splitlines()[:3], ["mutmut in pkg: 2 mutants on changed lines, 2 not killed. No test reaches them.", "pkg/calc.py:6: no tests: return n * 4", "pkg/calc.py:6: no tests: return n / 3"], ran.stderr)
def test_a_crash_inside_mutmut_fails_the_run_in_one_line_naming_mutmut_and_the_error(self):
def test_a_crash_inside_mutmut_is_one_line_naming_mutmut_and_the_error_and_the_run_goes_on(self):
ran = self.run_tool(*self.faked("crash"))
self.assertEqual((ran.returncode, ran.stderr), (1, "mutmut crashed in pkg with KeyError: 974598, in mutmut/workers/isolation.py line 889: mutate the changed lines of pkg by hand.\n"))
self.assertEqual((ran.returncode, ran.stderr, ran.stdout.splitlines()[0]), (0, "", "mutmut in pkg crashed with KeyError: 974598, in mutmut/workers/isolation.py line 889: mutate its changed lines by hand."))
def test_a_run_that_stops_early_for_another_reason_fails_the_run(self):
ran = self.run_tool(*self.faked("setup-fault"))