Answer stability round 1: 0 🔴, 2 🟡 (0 misread); tell mutmut's own frames by their site-packages path, take the error from after the last frame, and fix the module docstring and help
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
"""Files whose tests import them by file name, which mutmut names right only from the file's own folder.
|
||||
"""Files that tests import by their file name, which mutmut finds only when it runs in the file's own folder.
|
||||
|
||||
A test in docker/scripts/ that runs `import run_harness` gets it because pytest, or unittest's discover, puts that folder on sys.path. mutmut run from the
|
||||
checkout's top names the same file docker.scripts.run_harness, and finds none of the tests' calls to it."""
|
||||
checkout's top names the same file docker.scripts.run_harness, so it finds none of the tests' calls to it."""
|
||||
import ast
|
||||
import os
|
||||
import pathlib
|
||||
@@ -11,9 +11,9 @@ from mutate_changes.changed_lines import is_test
|
||||
|
||||
|
||||
def imported_by_file_name(top: str, grouped: dict[str, dict[str, frozenset[int]]], configs: tuple[str, ...]) -> dict[str, dict[str, frozenset[int]]]:
|
||||
"""grouped, with each file of the checkout's top group that a test beside it imports by its file name moved to its own folder.
|
||||
"""grouped, with each file in its "." group moved to its own folder where a test beside it imports it by its file name and the folder has no __init__.py.
|
||||
|
||||
A file in a folder that holds one of configs stays: the settings there decide how its tests import it. So do all files where the top holds one."""
|
||||
A file in a folder that holds one of configs stays, as does every file where the top holds one: the settings there decide how its tests import it."""
|
||||
if "." not in grouped or any(os.path.isfile(os.path.join(top, name)) for name in configs):
|
||||
return grouped
|
||||
regrouped: dict[str, dict[str, frozenset[int]]] = {name: files for name, files in grouped.items() if name != "."}
|
||||
@@ -25,7 +25,7 @@ def imported_by_file_name(top: str, grouped: dict[str, dict[str, frozenset[int]]
|
||||
|
||||
|
||||
def top_level_imports(folder: str) -> set[str]:
|
||||
"""The modules the test files in folder import by their first name. A test file that doesn't parse names none."""
|
||||
"""The top-level module names that the test files in folder import, as `a` for `import a.b`. A test file that can't be read or parsed adds none."""
|
||||
names: set[str] = set()
|
||||
for test in sorted(pathlib.Path(folder).glob("*.py")):
|
||||
if not is_test(test.name):
|
||||
|
||||
@@ -8,11 +8,14 @@ extension, and leaves out test files: test_*.py, *_test.py, conftest.py, *.test.
|
||||
or __tests__/ folder.
|
||||
|
||||
- .py: mutmut 3, in a scratch copy of the nearest folder above the file that holds a setup.cfg or a pyproject.toml, else of the
|
||||
checkout's top, but a file that a test beside it imports by its file name, as `import run_harness`, from its own folder, which
|
||||
has no setup.cfg or pyproject.toml. It mutates each function or method a changed line falls in, and counts the mutants on changed lines. The
|
||||
folder's setup.cfg settings stay, but source_paths and also_copy are set, and pytest leaves out each test that fails without a
|
||||
mutant; the report names their files. mutmut sees only the code a test runs in its own process, so a mutant that only a child
|
||||
process reaches, such as a tool a test starts, shows as `no tests`. A pyproject.toml with a [tool.mutmut] section fails the run, and so does a crash inside mutmut, in one line that names the error.
|
||||
checkout's top. The exception is a file that a test beside it imports by its file name, as `import run_harness`, where the
|
||||
folder has no __init__.py and neither it nor the checkout's top holds a setup.cfg or pyproject.toml: mutmut runs in that file's
|
||||
own folder, as there the tests find the file under the same name. It mutates each function or method a changed line falls in,
|
||||
and counts the mutants on changed lines. The folder's setup.cfg settings stay, but source_paths and also_copy are set, and
|
||||
pytest leaves out each test that fails without a mutant; the report names their files. mutmut sees only the code a test runs
|
||||
in its own process, so a mutant that only a child process reaches, such as a tool a test starts, shows as `no tests`. A
|
||||
pyproject.toml with a [tool.mutmut] section fails the run. A crash inside mutmut, such as a KeyError in its worker pool,
|
||||
becomes one line for its folder, naming the error and telling you to mutate that folder's changed lines by hand.
|
||||
- .rs: cargo-mutants, as `cargo mutants --in-diff`, in the checkout's top.
|
||||
- .cjs, .cts, .js, .jsx, .mjs, .mts, .ts, .tsx: Stryker, as `npx --no --package=@stryker-mutator/core stryker run --mutate` on the
|
||||
changed lines, in the nearest folder above the file that holds a package.json. A file with no package.json above it isn't
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
mutmut names a mutant `<module>.<key>__mutmut_<number>`. The key is x_<function> for a function at a module's top, and xǁ<class>ǁ<method> for a method of a class there; mutmut mutates no other code. Its mutants/ copy of a file holds the original of each function mutmut mutates, as `<key>__mutmut_orig`, and each mutant as a changed copy of that original."""
|
||||
import ast
|
||||
import functools
|
||||
import re
|
||||
from typing import NamedTuple
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
|
||||
|
||||
@@ -42,17 +42,19 @@ def named(name: str, paths: list[str]) -> tuple[str, str, int] | None:
|
||||
|
||||
|
||||
TRACEBACK = "Traceback (most recent call last):"
|
||||
FRAME = re.compile(r'^\s*File ".*?[/\\](mutmut[/\\].*?|[^/\\]*?)", line (\d+)', re.MULTILINE)
|
||||
FRAME = re.compile(r'^\s*File "(.*?)", line (\d+)', re.MULTILINE)
|
||||
MUTMUT_FILE = re.compile(r"[/\\]site-packages[/\\](mutmut[/\\].*)")
|
||||
|
||||
|
||||
def crash_in_mutmut(output: str) -> str | None:
|
||||
"""The error and place of the last traceback in output, where it ends in a frame inside mutmut's own code, else None."""
|
||||
"""The error and place of the last traceback in output, where its last frame is in the mutmut package, else None."""
|
||||
block = output.rpartition(TRACEBACK)[2] if TRACEBACK in output else ""
|
||||
frames = FRAME.findall(block)
|
||||
lines = block.strip().splitlines()
|
||||
if not frames or not lines or not frames[-1][0].startswith("mutmut"):
|
||||
frames = list(FRAME.finditer(block))
|
||||
if not frames:
|
||||
return None
|
||||
return f"{lines[-1]}, in {frames[-1][0].replace(chr(92), '/')} line {frames[-1][1]}"
|
||||
installed = MUTMUT_FILE.search(frames[-1][1])
|
||||
error = next((line for line in block[frames[-1].end():].splitlines()[1:] if line and not line[0].isspace()), "")
|
||||
return f"{error}, in {installed[1].replace(chr(92), '/')} line {frames[-1][2]}" if installed and error else None
|
||||
|
||||
|
||||
def statuses(results: str) -> dict[str, str]:
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
"""Mutate the changed lines of a branch's Python files with mutmut 3, in a scratch copy of the folder each file's tests run from.
|
||||
|
||||
mutmut reads its settings from the folder it runs in, writes beside them, and names each module by its path from there. So each file is
|
||||
mutated from the nearest folder above it, up to the checkout's top, that holds a setup.cfg or a pyproject.toml: the folder its tests
|
||||
import it from."""
|
||||
mutated from the folder its tests import it from: the nearest folder above it, up to the checkout's top, that holds a setup.cfg or a
|
||||
pyproject.toml, or the file's own folder where a test beside it imports it by its file name (bare_imports.py)."""
|
||||
import dataclasses
|
||||
import os
|
||||
import pathlib
|
||||
@@ -10,12 +10,12 @@ import posixpath
|
||||
import shutil
|
||||
import tomllib
|
||||
|
||||
from mutate_changes.bare_imports import imported_by_file_name
|
||||
from mutate_changes.changed_lines import by_folder, label
|
||||
from mutate_changes.mutmut_mutants import crash_in_mutmut, function_keys, mutant_globs, mutated_line, named, statuses
|
||||
from mutate_changes.tool_process import failed, Ran, remove, Run, scratch_folder
|
||||
from failure import Failure
|
||||
from mutate_changes.mutmut_mutants import crash_in_mutmut, function_keys, mutant_globs, mutated_line, named, statuses
|
||||
from git_cli.git_repo import Git
|
||||
from mutate_changes.bare_imports import imported_by_file_name
|
||||
from mutate_changes.mutmut_setup import left_out, mutmut_config
|
||||
|
||||
CAUGHT = frozenset({"caught by type check", "killed"})
|
||||
@@ -127,7 +127,10 @@ def mutate_copy(copy: str, copied: list[str], folder: Folder, argv: list[str], r
|
||||
ran = run_until_stats_pass(copy, copied, folder, argv, run)
|
||||
if isinstance(ran, Failure):
|
||||
return ran
|
||||
output, left = ran
|
||||
(output, code), left = ran
|
||||
crash = crash_in_mutmut(output)
|
||||
if code and crash:
|
||||
return [f"mutmut in {folder.label} crashed with {crash}: mutate its changed lines by hand."]
|
||||
if not os.path.isdir(os.path.join(copy, "mutants")):
|
||||
return Failure(f"mutmut in {folder.label} wrote no mutants/ in {copy}; --prefix must run it in $PWD")
|
||||
if NO_MUTANT in output:
|
||||
@@ -142,8 +145,8 @@ def mutate_copy(copy: str, copied: list[str], folder: Folder, argv: list[str], r
|
||||
return [f"mutmut in {folder.label}: {counted} mutants on changed lines, {len(shown)} not killed.{unreached}{left_note}", *shown]
|
||||
|
||||
|
||||
def run_until_stats_pass(copy: str, copied: list[str], folder: Folder, argv: list[str], run: Run) -> tuple[str, list[str]] | Failure:
|
||||
"""mutmut's output, and the options that left out each test that fails before any mutant: mutmut runs no mutant while one fails."""
|
||||
def run_until_stats_pass(copy: str, copied: list[str], folder: Folder, argv: list[str], run: Run) -> tuple[tuple[str, int], list[str]] | Failure:
|
||||
"""mutmut's output and exit code, and the options that left out each test that fails before any mutant: mutmut runs no mutant while one fails."""
|
||||
ran = configured_run(copy, copied, folder, [], argv, run)
|
||||
left: list[str] = []
|
||||
if isinstance(ran, Ran) and STATS_FAILED in ran.output:
|
||||
@@ -152,12 +155,9 @@ def run_until_stats_pass(copy: str, copied: list[str], folder: Folder, argv: lis
|
||||
ran = configured_run(copy, copied, folder, left, argv, run)
|
||||
if isinstance(ran, Failure):
|
||||
return ran
|
||||
if ran.code and NO_MUTANT not in ran.output and NO_TEST not in ran.output:
|
||||
crash = crash_in_mutmut(ran.output)
|
||||
if crash:
|
||||
return Failure(f"mutmut crashed in {folder.label} with {crash}: mutate the changed lines of {folder.label} by hand.")
|
||||
if ran.code and NO_MUTANT not in ran.output and NO_TEST not in ran.output and not crash_in_mutmut(ran.output):
|
||||
return failed(f"mutmut run in {folder.label}", ran)
|
||||
return ran.output, left
|
||||
return (ran.output, ran.code), left
|
||||
|
||||
|
||||
def configured_run(copy: str, copied: list[str], folder: Folder, options: list[str], argv: list[str], run: Run) -> Ran | Failure:
|
||||
|
||||
@@ -13,11 +13,11 @@ import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from mutate_changes.bare_imports import imported_by_file_name
|
||||
from mutate_changes.changed_lines import added_lines, by_folder, not_mutated, runs, tool_for
|
||||
from mutate_changes.mutmut_tool import copy_files
|
||||
from failure import Failure
|
||||
from mutate_changes.mutmut_mutants import crash_in_mutmut, function_keys, mutant_globs, mutated_line, named, statuses
|
||||
from failure import Failure
|
||||
from mutate_changes.bare_imports import imported_by_file_name
|
||||
from test_env import isolated_env
|
||||
from mutate_changes.mutmut_setup import left_out, mutmut_config
|
||||
from mutate_changes import main
|
||||
@@ -120,17 +120,24 @@ class ChangedLines(unittest.TestCase):
|
||||
self.assertEqual(imported_by_file_name(top, grouped, ("setup.cfg",)), grouped)
|
||||
|
||||
def test_a_traceback_that_ends_inside_mutmut_is_a_crash_of_mutmut(self):
|
||||
def trace(*frames, error="KeyError: 974598"):
|
||||
return "Traceback (most recent call last):\n" + "".join(f' File "{path}", line {number}, in f\n' for path, number in frames) + error + "\n"
|
||||
inside = trace(("/a/mutmut/__main__.py", 9), ("/a/mutmut/workers/isolation.py", 889), ("/a/mutmut/workers/isolation.py", 12))
|
||||
outside = trace(("/a/mutmut/__main__.py", 9), ("/repo/calc.py", 3), error="KeyError: 1")
|
||||
def trace(*frames, error="KeyError: 974598", after=""):
|
||||
shown = "".join(f' File "{path}", line {number}, in f\n code()\n ~~^^\n' for path, number in frames)
|
||||
return f"Traceback (most recent call last):\n{shown}{error}\n{after}"
|
||||
lib = "/uv/lib/python3.14/site-packages/mutmut"
|
||||
inside = trace((f"{lib}/__main__.py", 9), (f"{lib}/workers/isolation.py", 889), (f"{lib}/workers/isolation.py", 12))
|
||||
outside = trace((f"{lib}/__main__.py", 9), ("/repo/calc.py", 3), error="KeyError: 1")
|
||||
place = "KeyError: 974598, in mutmut/workers/isolation.py line 12"
|
||||
self.assertEqual((crash_in_mutmut(inside), crash_in_mutmut("Running stats\n" + inside + "\n")), (place, place))
|
||||
self.assertEqual(crash_in_mutmut(outside + trace(("/a/mutmut/x.py", 1), error="E: 2")), "E: 2, in mutmut/x.py line 1")
|
||||
self.assertEqual(crash_in_mutmut(trace(("/a/mutmut/x.py", 1), error="E: 2") + outside), None)
|
||||
self.assertEqual(crash_in_mutmut(trace(("C:\\py\\mutmut\\workers\\x.py", 5))), "KeyError: 974598, in mutmut/workers/x.py line 5")
|
||||
self.assertEqual(crash_in_mutmut(trace(("/a/mutmut/x.py", 1), error="").replace("Traceback (most recent call last):", "")), None)
|
||||
self.assertEqual((crash_in_mutmut(outside), crash_in_mutmut("all fine"), crash_in_mutmut(""), crash_in_mutmut("Traceback (most recent call last):\nKeyError: 1")), (None,) * 4)
|
||||
self.assertEqual(crash_in_mutmut(inside + "later output\n"), place)
|
||||
self.assertEqual(crash_in_mutmut(outside + trace((f"{lib}/x.py", 1), error="E: 2")), "E: 2, in mutmut/x.py line 1")
|
||||
self.assertEqual(crash_in_mutmut(trace((f"{lib}/x.py", 1), error="E: 2") + outside), None)
|
||||
self.assertEqual(crash_in_mutmut(trace(("C:\\py\\Lib\\site-packages\\mutmut\\workers\\x.py", 5))), "KeyError: 974598, in mutmut/workers/x.py line 5")
|
||||
for path in ("/mutmut/repo/calc.py", "mutmut_tool.py", "<string>", "/a/site-packages/mutmut_tool.py"):
|
||||
with self.subTest(path):
|
||||
self.assertEqual(crash_in_mutmut(trace((path, 1))), None)
|
||||
self.assertEqual(crash_in_mutmut(trace((f"{lib}/x.py", 1)).replace("Traceback (most recent call last):", "")), None)
|
||||
self.assertEqual((crash_in_mutmut("all fine"), crash_in_mutmut(""), crash_in_mutmut("Traceback (most recent call last):\nKeyError: 1")), (None,) * 3)
|
||||
self.assertEqual(crash_in_mutmut(trace((f"{lib}/x.py", 1), error="")), None)
|
||||
|
||||
def test_not_mutated_names_the_files_no_tool_covers_then_the_test_files(self):
|
||||
self.assertEqual(not_mutated(["README.md", "a.py", "b.go", "test_a.py", "web/a.test.ts"]), ["Not mutated, as no tool covers their language: README.md, b.go.", "Not mutated, as test files: test_a.py, web/a.test.ts."])
|
||||
|
||||
@@ -59,7 +59,7 @@ else:
|
||||
print("Stopping early, because tests recorded trampoline hits but none match any mutant key. This usually means imports bypass the mutants.")
|
||||
sys.exit(1)
|
||||
if mode == "crash":
|
||||
print("Traceback (most recent call last):\n File \"/uv/site-packages/mutmut/__main__.py\", line 1050, in _run\n File \"/uv/site-packages/mutmut/workers/isolation.py\", line 889, in wait_for_result\nKeyError: 974598")
|
||||
print("Traceback (most recent call last):\n File \"/uv/site-packages/mutmut/__main__.py\", line 1050, in _run\n File \"/uv/site-packages/mutmut/workers/isolation.py\", line 889, in wait_for_result\n worker = pop(pid)\nKeyError: 974598")
|
||||
sys.exit(1)
|
||||
"""
|
||||
# cargo-mutants, Infection and Stryker, which log the folder they run in and how they were called, and exit with the code the test
|
||||
@@ -208,9 +208,9 @@ class MutateChangesRuns(unittest.TestCase):
|
||||
ran = self.run_tool(*self.faked("no-test"))
|
||||
self.assertEqual(ran.stdout.splitlines()[:3], ["mutmut in pkg: 2 mutants on changed lines, 2 not killed. No test reaches them.", "pkg/calc.py:6: no tests: return n * 4", "pkg/calc.py:6: no tests: return n / 3"], ran.stderr)
|
||||
|
||||
def test_a_crash_inside_mutmut_fails_the_run_in_one_line_naming_mutmut_and_the_error(self):
|
||||
def test_a_crash_inside_mutmut_is_one_line_naming_mutmut_and_the_error_and_the_run_goes_on(self):
|
||||
ran = self.run_tool(*self.faked("crash"))
|
||||
self.assertEqual((ran.returncode, ran.stderr), (1, "mutmut crashed in pkg with KeyError: 974598, in mutmut/workers/isolation.py line 889: mutate the changed lines of pkg by hand.\n"))
|
||||
self.assertEqual((ran.returncode, ran.stderr, ran.stdout.splitlines()[0]), (0, "", "mutmut in pkg crashed with KeyError: 974598, in mutmut/workers/isolation.py line 889: mutate its changed lines by hand."))
|
||||
|
||||
def test_a_run_that_stops_early_for_another_reason_fails_the_run(self):
|
||||
ran = self.run_tool(*self.faked("setup-fault"))
|
||||
|
||||
Reference in New Issue
Block a user