Answer stability round 1: 3 🔴, 5 🟡 (0 misread); run only a Stryker the project installed, tell mutmut's setup fault from no test, pin cargo-mutants' diff prefixes, keep -x, end the tool and report its scratch folder on SIGTERM, and test option-like names; with product-owner's and prose's code fixes
This commit is contained in:
@@ -1,9 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
import pathlib
|
||||
import signal
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
||||
|
||||
from mutate_changes import main
|
||||
|
||||
# Python ends on SIGTERM or SIGHUP without running a finally, which would leave the scratch copy and the tool running. As
|
||||
# SystemExit, the run unwinds as on Ctrl-C.
|
||||
for number in (signal.SIGHUP, signal.SIGTERM):
|
||||
signal.signal(number, lambda number, frame: sys.exit(128 + number))
|
||||
sys.exit(main.main(sys.argv[1:]))
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Which lines of a file a branch adds or changes, and which mutation tool mutates the file."""
|
||||
"""Which lines of a file a branch adds or changes, which mutation tool mutates the file, and in which folder."""
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import typing
|
||||
@@ -29,7 +30,34 @@ def runs(lines: typing.Iterable[int]) -> list[tuple[int, int]]:
|
||||
|
||||
def tool_for(path: str) -> str | None:
|
||||
"""The tool that mutates a file, by its extension; None for a test file, or a language none of the tools covers."""
|
||||
return None if is_test(path) else TOOLS.get(posixpath.splitext(path)[1])
|
||||
|
||||
|
||||
def is_test(path: str) -> bool:
|
||||
folders, name = posixpath.split(path)
|
||||
if TEST_FOLDERS & set(folders.split("/")) or TEST_NAME.fullmatch(name):
|
||||
return None
|
||||
return TOOLS.get(posixpath.splitext(name)[1])
|
||||
return bool(TEST_FOLDERS & set(folders.split("/")) or TEST_NAME.fullmatch(name))
|
||||
|
||||
|
||||
def not_mutated(paths: list[str]) -> list[str]:
|
||||
"""A line naming the paths in a language no tool covers, then one naming the test files, leaving out a line that would name none."""
|
||||
uncovered = [path for path in paths if tool_for(path) is None and not is_test(path)]
|
||||
tests = [path for path in paths if is_test(path)]
|
||||
lines = (("Not mutated, as no tool covers their language", uncovered), ("Not mutated, as test files", tests))
|
||||
return [f"{text}: {', '.join(found)}." for text, found in lines if found]
|
||||
|
||||
|
||||
def by_folder(top: str, changed: dict[str, frozenset[int]], configs: tuple[str, ...]) -> dict[str, dict[str, frozenset[int]]]:
|
||||
"""The changed files by the nearest folder above each that holds one of configs, "." for top where none does, each file by its
|
||||
path from that folder."""
|
||||
grouped: dict[str, dict[str, frozenset[int]]] = {}
|
||||
for path, lines in changed.items():
|
||||
folder = posixpath.dirname(path)
|
||||
while folder and not any(os.path.isfile(os.path.join(top, folder, name)) for name in configs):
|
||||
folder = posixpath.dirname(folder)
|
||||
grouped.setdefault(folder or ".", {})[posixpath.relpath(path, folder or ".")] = lines
|
||||
return grouped
|
||||
|
||||
|
||||
def label(folder: str) -> str:
|
||||
"""A folder from the checkout's top, as a report names it."""
|
||||
return "the checkout's top" if folder == "." else folder
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
"""Mutate a branch's changed lines with the tools that limit a run to them themselves: cargo-mutants, Stryker and Infection. Each runs at the checkout's top."""
|
||||
"""Mutate a branch's changed lines with cargo-mutants, Stryker and Infection, each of which takes the changed lines itself."""
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
from mutate_changes.tool_process import failed, run_tool
|
||||
from mutate_changes.changed_lines import by_folder, label, runs
|
||||
from mutate_changes.tool_process import failed, remove, run_tool, scratch_folder
|
||||
from failure import Failure
|
||||
from git_cli.git_repo import git
|
||||
from mutate_changes.changed_lines import runs
|
||||
|
||||
# cargo-mutants' exit codes for a run that finished: every mutant caught, some missed, and some timed out.
|
||||
CARGO_FINISHED = frozenset({0, 2, 3})
|
||||
@@ -15,14 +13,16 @@ CARGO_OUTCOMES = ("caught", "missed", "timeout", "unviable")
|
||||
|
||||
|
||||
def run_cargo_mutants(top: str, base: str, paths: list[str], prefix: str | None, jobs: int) -> list[str] | Failure:
|
||||
"""A line with cargo-mutants' mutant count, then a line per mutant it missed or that timed out, as `<path>:<line>:<column>: <outcome>: <mutation>`."""
|
||||
diff = git("diff", "--no-color", "--no-ext-diff", "--no-textconv", base, "--", *paths, cwd=top)
|
||||
"""A line with cargo-mutants' mutant count, then a line per mutant it missed or that timed out, as `<path>:<line>:<column>: <outcome>: <mutation>`.
|
||||
|
||||
cargo-mutants runs in top."""
|
||||
# cargo-mutants finds each file's path after a b/, a prefix that settings such as diff.mnemonicPrefix change.
|
||||
diff = git("diff", "--no-color", "--no-ext-diff", "--no-textconv", "--src-prefix=a/", "--dst-prefix=b/", base, "--", *paths, cwd=top)
|
||||
if isinstance(diff, Failure):
|
||||
return diff
|
||||
try:
|
||||
scratch = tempfile.mkdtemp(prefix=".mutate-changes-", dir=top)
|
||||
except OSError as failure:
|
||||
return Failure(f"could not make a scratch folder in {top}: {failure}")
|
||||
scratch = scratch_folder(top)
|
||||
if isinstance(scratch, Failure):
|
||||
return scratch
|
||||
try:
|
||||
pathlib.Path(scratch, "branch.diff").write_text(diff, encoding="utf-8")
|
||||
relative = os.path.relpath(scratch, top)
|
||||
@@ -31,9 +31,9 @@ def run_cargo_mutants(top: str, base: str, paths: list[str], prefix: str | None,
|
||||
return ran if isinstance(ran, Failure) else failed("cargo mutants", ran)
|
||||
outcomes = {outcome: outcome_lines(pathlib.Path(scratch, "mutants.out", f"{outcome}.txt")) for outcome in CARGO_OUTCOMES}
|
||||
except OSError as failure:
|
||||
return Failure(f"could not read cargo-mutants' outcomes in {scratch}: {failure}")
|
||||
return Failure(f"could not hand cargo-mutants the diff or read its outcomes in {scratch}: {failure}")
|
||||
finally:
|
||||
shutil.rmtree(scratch, ignore_errors=True)
|
||||
remove(scratch)
|
||||
uncaught = [f"{location}: {outcome}: {mutation}" for outcome in ("missed", "timeout") for location, _, mutation in (line.partition(": ") for line in outcomes[outcome])]
|
||||
return [f"cargo-mutants: {sum(map(len, outcomes.values()))} mutants on changed lines, {len(uncaught)} not caught.", *uncaught]
|
||||
|
||||
@@ -42,14 +42,26 @@ def outcome_lines(path: pathlib.Path) -> list[str]:
|
||||
return path.read_text(encoding="utf-8", errors="replace").splitlines() if path.is_file() else []
|
||||
|
||||
|
||||
# Stryker and Infection also exit nonzero on a mutation score below a threshold the project sets, which their exit code doesn't
|
||||
# tell from a run that broke, so either fails the run.
|
||||
def run_stryker(top: str, changed: dict[str, frozenset[int]], prefix: str | None, jobs: int) -> list[str] | Failure:
|
||||
"""Stryker's exit code, then its own report of the mutants on the changed lines."""
|
||||
mutate = ",".join(f"{path}:{first}-{last}" for path, lines in sorted(changed.items()) for first, last in runs(lines))
|
||||
ran = run_tool(["npx", "stryker", "run", "--mutate", mutate, "--concurrency", str(jobs), "--reporters", "clear-text"], top, prefix)
|
||||
return ran if isinstance(ran, Failure) else [f"Stryker exited {ran.code}; its report:", *ran.output.splitlines()]
|
||||
"""For each folder Stryker runs in, a line naming it, then Stryker's own report of the mutants on the changed lines.
|
||||
|
||||
Stryker runs in the nearest folder above each file that holds a package.json, else in top. Through `npx --no`, it runs only where
|
||||
the project installed it."""
|
||||
report: list[str] = []
|
||||
for folder, files in sorted(by_folder(top, changed, ("package.json",)).items()):
|
||||
mutate = ",".join(f"{path}:{first}-{last}" for path, lines in sorted(files.items()) for first, last in runs(lines))
|
||||
ran = run_tool(["npx", "--no", "stryker", "run", "--mutate", mutate, "--concurrency", str(jobs), "--reporters", "clear-text"], os.path.join(top, folder), prefix)
|
||||
if isinstance(ran, Failure) or ran.code:
|
||||
return ran if isinstance(ran, Failure) else failed(f"Stryker in {label(folder)}", ran)
|
||||
report.extend([f"Stryker in {label(folder)}:", *ran.output.splitlines()])
|
||||
return report
|
||||
|
||||
|
||||
def run_infection(top: str, base: str, prefix: str | None, jobs: int) -> list[str] | Failure:
|
||||
"""Infection's exit code, then its own report of the mutants on the lines the branch changes since base."""
|
||||
"""A line naming where Infection ran, top, then its own report of the mutants on the lines the branch changes since base."""
|
||||
ran = run_tool(["vendor/bin/infection", "--git-diff-lines", f"--git-diff-base={base}", f"--threads={jobs}", "--show-mutations=max", "--no-progress"], top, prefix)
|
||||
return ran if isinstance(ran, Failure) else [f"Infection exited {ran.code}; its report:", *ran.output.splitlines()]
|
||||
if isinstance(ran, Failure) or ran.code:
|
||||
return ran if isinstance(ran, Failure) else failed("Infection", ran)
|
||||
return ["Infection in the checkout's top:", *ran.output.splitlines()]
|
||||
|
||||
+44
-33
@@ -12,25 +12,38 @@ or __tests__/ folder.
|
||||
folder's setup.cfg settings stay, but source_paths and also_copy are set, and pytest leaves out each test that fails without a
|
||||
mutant; the report names their files. mutmut sees only the code a test runs in its own process, so a mutant that only a child
|
||||
process reaches, such as a tool a test starts, shows as `no tests`. A pyproject.toml with a [tool.mutmut] section fails the run.
|
||||
- .rs: cargo-mutants, with --in-diff.
|
||||
- .cjs, .cts, .js, .jsx, .mjs, .mts, .ts, .tsx: Stryker, with --mutate on the changed lines.
|
||||
- .php: Infection, with --git-diff-lines.
|
||||
- .rs: cargo-mutants, as `cargo mutants --in-diff`, in the checkout's top.
|
||||
- .cjs, .cts, .js, .jsx, .mjs, .mts, .ts, .tsx: Stryker, as `npx --no stryker run --mutate` on the changed lines, in the nearest
|
||||
folder above the file that holds a package.json, else in the checkout's top. Through --no, npx runs only a Stryker the project
|
||||
installed.
|
||||
- .php: Infection, as `vendor/bin/infection --git-diff-lines`, in the checkout's top.
|
||||
|
||||
--prefix is shell text put before each tool's command, such as 'uvx --from "mutmut>=3,<4"' or 'docker compose run --rm app'.
|
||||
Without it, each tool must be on PATH. mutmut runs in its scratch copy, a folder in the checkout's top, and every other tool in
|
||||
the checkout's top; in the prefix, $PWD names that folder. --jobs sets how many tests each tool runs at once, 4 by default: one
|
||||
heavy slot's CPUs.
|
||||
--prefix is shell text that runs each tool: sh runs `<prefix> "$@"` in the tool's folder, which $PWD names, with the tool's command
|
||||
as "$@". For example, 'uvx --from "mutmut>=3,<4"'. Without --prefix, the command runs as it is. A container must mount $PWD and
|
||||
run there, as 'docker compose -f "$(git rev-parse --show-toplevel)/compose.yaml" run --rm -v "$PWD:$PWD" -w "$PWD" app' does. "$1"
|
||||
is the command's first word, mutmut, cargo, npx or vendor/bin/infection, so a prefix can pick what to run per tool:
|
||||
'case "$1" in mutmut) set -- uvx --from "mutmut>=3,<4" "$@";; esac;'.
|
||||
|
||||
For mutmut and cargo-mutants, it prints a line with the mutant count, then a line per mutant on a changed line that is not
|
||||
killed: `<path>:<line>: <status>: <the line as mutated>`. Stryker and Infection print their own report, after their exit code.
|
||||
The last line gives the wall time.
|
||||
--jobs sets how many tests each tool runs at once, 4 by default: one heavy slot's CPUs. Under run-heavy, a container that sets
|
||||
neither memory limit gets the slot's share, in MiB in RUN_HEAVY_MEMORY; set --jobs so that that many test runs fit in it.
|
||||
|
||||
Exit 1, saying why, on unknown arguments, or when git fails, a tool cannot start, or mutmut or cargo-mutants fails before it reports.
|
||||
It prints:
|
||||
- for mutmut, per folder, a line with the mutant count, then a line per mutant on a changed line that is not killed, as
|
||||
`<path>:<line>: <status>: <the line as mutated>`, or `<path>: <status>: <name>, on a line mutate-changes cannot find`;
|
||||
- for cargo-mutants, a line with the mutant count, then a line per mutant it missed or that timed out, as
|
||||
`<path>:<line>:<column>: <outcome>: <mutation>`;
|
||||
- for Stryker and Infection, a line naming the folder the tool ran in, then the tool's own report;
|
||||
- a line naming the changed files in a language no tool covers, then one naming the changed test files, none of which it mutates;
|
||||
- last, the wall time.
|
||||
|
||||
Exit 1, saying why, on an unknown or repeated argument, an option without its value, or a --jobs that is no whole number from 1.
|
||||
Exit 1 too when git fails, a file cannot be read or copied, a tool cannot start, or a tool fails before it reports. Stryker and
|
||||
Infection also exit nonzero on a mutation score below a threshold the project sets, which fails the run as well.
|
||||
"""
|
||||
import sys
|
||||
import time
|
||||
|
||||
from mutate_changes.changed_lines import added_lines, tool_for
|
||||
from mutate_changes.changed_lines import added_lines, not_mutated, tool_for
|
||||
from machine_load.slots import CPUS_PER_SLOT
|
||||
from git_cli.git_repo import default_ref, git, top
|
||||
from failure import Failure
|
||||
@@ -55,11 +68,7 @@ def main(argv: list[str]) -> int:
|
||||
if isinstance(report, Failure):
|
||||
print(report.reason, file=sys.stderr)
|
||||
return 1
|
||||
if not report:
|
||||
print("No line the branch changes is in a language mutate-changes mutates.")
|
||||
return 0
|
||||
print("\n".join(report))
|
||||
print(f"mutate-changes took {took(time.monotonic() - started)}.")
|
||||
print("\n".join([*report, f"mutate-changes took {took(time.monotonic() - started)}."]))
|
||||
return 0
|
||||
|
||||
|
||||
@@ -73,32 +82,37 @@ def options(argv: list[str]) -> dict[str, str] | Failure:
|
||||
if not equals and not rest:
|
||||
return Failure(f"{name} needs a value")
|
||||
given[name] = value if equals else rest.pop(0)
|
||||
if not given.get("--jobs", "1").isdigit() or int(given.get("--jobs", "1")) < 1:
|
||||
if not given.get("--jobs", "1").isdecimal() or int(given.get("--jobs", "1")) < 1:
|
||||
return Failure(f"--jobs takes a whole number from 1, not {given['--jobs']}")
|
||||
return given
|
||||
|
||||
|
||||
def mutate(given: dict[str, str]) -> list[str] | Failure:
|
||||
"""Each tool's report on the branch's changed lines; empty where none of them is in a language a tool covers."""
|
||||
"""Each tool's report on the lines the branch adds, then the lines naming the changed files no tool mutates."""
|
||||
checkout = top()
|
||||
if isinstance(checkout, Failure):
|
||||
return checkout
|
||||
base = given.get("--base") or default_ref(cwd=checkout)
|
||||
if isinstance(base, Failure):
|
||||
return base
|
||||
merge_base = git("merge-base", base, "HEAD", cwd=checkout)
|
||||
if isinstance(merge_base, Failure):
|
||||
return merge_base
|
||||
changed = changed_by_tool(checkout, merge_base.strip())
|
||||
found = git("merge-base", base, "HEAD", cwd=checkout)
|
||||
if isinstance(found, Failure):
|
||||
return found
|
||||
merge_base = found.strip()
|
||||
names = git("diff", "--name-only", "-z", "--diff-filter=d", merge_base, cwd=checkout)
|
||||
if isinstance(names, Failure):
|
||||
return names
|
||||
paths = list(filter(None, names.split("\0")))
|
||||
changed = changed_by_tool(checkout, merge_base, paths)
|
||||
if isinstance(changed, Failure):
|
||||
return changed
|
||||
prefix, jobs = given.get("--prefix"), int(given.get("--jobs", CPUS_PER_SLOT))
|
||||
report: list[str] = []
|
||||
report = [] if changed else ["No tool ran: the branch adds no line to a file that a tool mutates."]
|
||||
for tool, files in sorted(changed.items()):
|
||||
if tool == "cargo-mutants":
|
||||
lines = run_cargo_mutants(checkout, merge_base.strip(), sorted(files), prefix, jobs)
|
||||
lines = run_cargo_mutants(checkout, merge_base, sorted(files), prefix, jobs)
|
||||
elif tool == "infection":
|
||||
lines = run_infection(checkout, merge_base.strip(), prefix, jobs)
|
||||
lines = run_infection(checkout, merge_base, prefix, jobs)
|
||||
elif tool == "stryker":
|
||||
lines = run_stryker(checkout, files, prefix, jobs)
|
||||
else:
|
||||
@@ -106,16 +120,13 @@ def mutate(given: dict[str, str]) -> list[str] | Failure:
|
||||
if isinstance(lines, Failure):
|
||||
return lines
|
||||
report.extend(lines)
|
||||
return report
|
||||
return [*report, *not_mutated(paths)]
|
||||
|
||||
|
||||
def changed_by_tool(checkout: str, merge_base: str) -> dict[str, dict[str, frozenset[int]]] | Failure:
|
||||
"""Each tool's files that the working tree changes since merge_base, by path from the checkout's top, with their changed lines."""
|
||||
names = git("diff", "--name-only", "-z", "--diff-filter=d", merge_base, cwd=checkout)
|
||||
if isinstance(names, Failure):
|
||||
return names
|
||||
def changed_by_tool(checkout: str, merge_base: str, paths: list[str]) -> dict[str, dict[str, frozenset[int]]] | Failure:
|
||||
"""Each tool's files among paths, by path from the checkout's top, with the lines the working tree adds since merge_base."""
|
||||
changed: dict[str, dict[str, frozenset[int]]] = {}
|
||||
for path in filter(None, names.split("\0")):
|
||||
for path in paths:
|
||||
if (tool := tool_for(path)) is None:
|
||||
continue
|
||||
diff = git("diff", "-U0", "--no-color", "--no-ext-diff", "--no-textconv", merge_base, "--", path, cwd=checkout, errors="replace")
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Which of mutmut 3's mutants the changed lines hold, from their names, `mutmut results` and the copy of each file mutmut writes under mutants/.
|
||||
|
||||
mutmut names a mutant `<module>.<key>__mutmut_<number>`. The key is x_<function> for a function at a module's top, and xǁ<class>ǁ<method> for a method of a class there; mutmut mutates no other code. Its mutants/ copy of a file holds the original of each function it mutates, as `<key>__mutmut_orig`, and each mutant as a copy of it."""
|
||||
mutmut names a mutant `<module>.<key>__mutmut_<number>`. The key is x_<function> for a function at a module's top, and xǁ<class>ǁ<method> for a method of a class there; mutmut mutates no other code. Its mutants/ copy of a file holds the original of each function mutmut mutates, as `<key>__mutmut_orig`, and each mutant as a changed copy of that original."""
|
||||
import ast
|
||||
import functools
|
||||
from typing import NamedTuple
|
||||
@@ -28,12 +28,12 @@ def module(path: str) -> str:
|
||||
|
||||
|
||||
def mutant_globs(path: str, keys: list[str]) -> list[str]:
|
||||
"""The `mutmut run` patterns that pick each function's mutants. A pattern without __mutmut_ also matches the function, so mutmut runs only the tests that reach one of them."""
|
||||
"""The `mutmut run` patterns that pick each function's mutants. Each pattern has no __mutmut_, so it also matches the function's own name, and mutmut runs only the tests that reach one of the functions."""
|
||||
return [f"{module(path)}.{key}*".replace(".__init__.", ".") for key in keys]
|
||||
|
||||
|
||||
def named(name: str, paths: list[str]) -> tuple[str, str, int] | None:
|
||||
"""The path, of those given, holding the mutant mutmut names, with its function's key and its number; None where none holds it."""
|
||||
"""Of the paths given, the one holding the mutant called name, with the mutant's function key and number; None where none holds it."""
|
||||
prefix, _, number = name.rpartition("__mutmut_")
|
||||
_, _, key = prefix.rpartition(".")
|
||||
found = [path for path in paths if f"{module(path)}.{key}".replace(".__init__.", ".") == prefix]
|
||||
@@ -97,6 +97,6 @@ def defined(tree: ast.Module, key: str) -> ast.AsyncFunctionDef | ast.FunctionDe
|
||||
|
||||
|
||||
def as_named(lines: Sequence[str], function: ast.AsyncFunctionDef | ast.FunctionDef, name: str) -> list[str]:
|
||||
"""A function's lines from its def, decorators left out, with the def naming it name."""
|
||||
"""A function's lines from its def line on, decorators left out, with name in place of the function's own name on the def line."""
|
||||
spanned = lines[function.lineno - 1:function.end_lineno]
|
||||
return [spanned[0].replace(function.name, name), *spanned[1:]]
|
||||
|
||||
@@ -6,7 +6,7 @@ import re
|
||||
from failure import Failure
|
||||
|
||||
SUMMARY = re.compile(r"^=+ short test summary info =+$", re.MULTILINE)
|
||||
# A test pytest's short summary reports as failed or erroring; its message, where one fits, follows " - ". A failed subtest's line
|
||||
# A line of pytest's short summary for a test that failed or errored; the test's message, where it fits, follows " - ". A failed subtest's line
|
||||
# names its test after a description in [] or (), which may hold spaces, and pytest reports no line for the test itself.
|
||||
REPORTED = re.compile(r"^(?:(?:ERROR|FAILED) (\S.*?)|SUBFAILED.*?[\])] (\S+::\S+))(?: - .*)?$", re.MULTILINE)
|
||||
|
||||
@@ -19,7 +19,7 @@ class CaseKept(configparser.ConfigParser):
|
||||
|
||||
|
||||
def mutmut_config(existing: str, sources: list[str], copied: list[str], left: list[str]) -> str | Failure:
|
||||
"""The setup.cfg that has mutmut mutate the sources, copy the folder's other entries, which their tests import, and pass pytest the options that leave tests out.
|
||||
"""The setup.cfg that has mutmut mutate the sources, copy every entry of the folder, which the sources' tests import, and pass pytest the options that leave tests out.
|
||||
|
||||
copied names each file and folder at the folder's top. Not ".": mutmut makes mutants/ before it copies, so "." would copy mutants/ into itself and
|
||||
pytest would collect the tests twice. existing is the project's own setup.cfg, empty where it has none. Its settings stay, though its comments go, and its [mutmut] pytest_add_cli_args come before the options that leave tests out."""
|
||||
|
||||
@@ -8,10 +8,10 @@ import os
|
||||
import pathlib
|
||||
import posixpath
|
||||
import shutil
|
||||
import tempfile
|
||||
import tomllib
|
||||
|
||||
from mutate_changes.tool_process import failed, Ran, run_tool
|
||||
from mutate_changes.changed_lines import by_folder, label
|
||||
from mutate_changes.tool_process import failed, Ran, remove, run_tool, scratch_folder
|
||||
from failure import Failure
|
||||
from mutate_changes.mutmut_mutants import function_keys, mutant_globs, mutated_line, named, statuses
|
||||
from git_cli.git_repo import git
|
||||
@@ -19,11 +19,12 @@ from mutate_changes.mutmut_setup import left_out, mutmut_config
|
||||
|
||||
CAUGHT = frozenset({"caught by type check", "killed"})
|
||||
CONFIGS = ("pyproject.toml", "setup.cfg")
|
||||
# The first run reports every test that fails before any mutant, not only the first, so that the next run can leave them all out.
|
||||
# Where a test fails before any mutant, a run with these lists every such test, not only the first, so that the next run can leave
|
||||
# them all out. Each mutant's run would take them too, and run every test where the first that fails is enough.
|
||||
LIST_FAILURES = ["--maxfail=0", "--continue-on-collection-errors"]
|
||||
# What mutmut 3.8 prints where no mutant matches the patterns, where no test reaches a mutated function, and where a test fails unmutated.
|
||||
NO_MUTANT = "Filtered for specific mutants, but nothing matches"
|
||||
NO_TEST = "Stopping early, because"
|
||||
NO_TEST = "Stopping early, because we could not find any test case for any mutant"
|
||||
STATS_FAILED = "failed to collect stats. runner returned"
|
||||
|
||||
|
||||
@@ -38,7 +39,7 @@ class Folder:
|
||||
|
||||
@property
|
||||
def label(self) -> str:
|
||||
return "the checkout's top" if self.path == "." else self.path
|
||||
return label(self.path)
|
||||
|
||||
|
||||
def run_mutmut(top: str, changed: dict[str, frozenset[int]], prefix: str | None, jobs: int) -> list[str] | Failure:
|
||||
@@ -46,7 +47,7 @@ def run_mutmut(top: str, changed: dict[str, frozenset[int]], prefix: str | None,
|
||||
|
||||
changed maps each file, by its path from top, to its changed lines."""
|
||||
report: list[str] = []
|
||||
for path, files in sorted(by_folder(top, changed).items()):
|
||||
for path, files in sorted(by_folder(top, changed, CONFIGS).items()):
|
||||
lines = mutate_folder(top, path, files, prefix, jobs)
|
||||
if isinstance(lines, Failure):
|
||||
return lines
|
||||
@@ -54,17 +55,6 @@ def run_mutmut(top: str, changed: dict[str, frozenset[int]], prefix: str | None,
|
||||
return report
|
||||
|
||||
|
||||
def by_folder(top: str, changed: dict[str, frozenset[int]]) -> dict[str, dict[str, frozenset[int]]]:
|
||||
"""The changed files by the folder mutmut runs in for each, "." for top, each file by its path from that folder."""
|
||||
grouped: dict[str, dict[str, frozenset[int]]] = {}
|
||||
for path, lines in changed.items():
|
||||
folder = posixpath.dirname(path)
|
||||
while folder and not any(os.path.isfile(os.path.join(top, folder, name)) for name in CONFIGS):
|
||||
folder = posixpath.dirname(folder)
|
||||
grouped.setdefault(folder or ".", {})[posixpath.relpath(path, folder or ".")] = lines
|
||||
return grouped
|
||||
|
||||
|
||||
def mutate_folder(top: str, path: str, files: dict[str, frozenset[int]], prefix: str | None, jobs: int) -> list[str] | Failure:
|
||||
root = os.path.join(top, path)
|
||||
folder = read_folder(root, path, files)
|
||||
@@ -76,10 +66,9 @@ def mutate_folder(top: str, path: str, files: dict[str, frozenset[int]], prefix:
|
||||
listed = git("ls-files", "-z", "--cached", "--others", "--exclude-standard", cwd=root)
|
||||
if isinstance(listed, Failure):
|
||||
return listed
|
||||
try:
|
||||
scratch = tempfile.mkdtemp(prefix=".mutate-changes-", dir=top)
|
||||
except OSError as failure:
|
||||
return Failure(f"could not make a scratch folder in {top}: {failure}")
|
||||
scratch = scratch_folder(top)
|
||||
if isinstance(scratch, Failure):
|
||||
return scratch
|
||||
try:
|
||||
copy = os.path.join(scratch, "copy")
|
||||
copied = copy_files(root, listed.split("\0"), copy)
|
||||
@@ -87,7 +76,7 @@ def mutate_folder(top: str, path: str, files: dict[str, frozenset[int]], prefix:
|
||||
except OSError as failure:
|
||||
return Failure(f"could not set mutmut up in {scratch}, a copy of {root}: {failure}")
|
||||
finally:
|
||||
shutil.rmtree(scratch, ignore_errors=True)
|
||||
remove(scratch)
|
||||
|
||||
|
||||
def read_folder(root: str, path: str, files: dict[str, frozenset[int]]) -> Folder | Failure:
|
||||
@@ -110,7 +99,7 @@ def project_setup(root: str) -> str | Failure:
|
||||
pyproject = pathlib.Path(root, "pyproject.toml")
|
||||
tool = tomllib.loads(pyproject.read_text(encoding="utf-8")).get("tool") if pyproject.is_file() else None
|
||||
if isinstance(tool, dict) and "mutmut" in tool:
|
||||
return Failure(f"{pyproject} sets mutmut up under [tool.mutmut]; move that to setup.cfg's [mutmut], the only place mutate-changes sets mutmut up through")
|
||||
return Failure(f"{pyproject} sets mutmut up under [tool.mutmut]; move that to setup.cfg's [mutmut], since mutate-changes sets mutmut up only there")
|
||||
setup = pathlib.Path(root, "setup.cfg")
|
||||
return setup.read_text(encoding="utf-8") if setup.is_file() else ""
|
||||
except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as failure:
|
||||
@@ -118,14 +107,14 @@ def project_setup(root: str) -> str | Failure:
|
||||
|
||||
|
||||
def copy_files(root: str, paths: list[str], copy: str) -> list[str]:
|
||||
"""Copy the files git lists in root, tracked or not ignored, to copy, symlinks as symlinks, and name each file and folder copied to copy's top.
|
||||
"""Copy each file git lists in root, tracked or untracked but not ignored, into copy, keeping symlinks as symlinks, and return the names of the files and folders at copy's top.
|
||||
|
||||
Leaves out mutmut's own mutants/ and the scratch folders of runs that never cleaned up."""
|
||||
Leaves out mutmut's own mutants/."""
|
||||
copied = set()
|
||||
for path in paths:
|
||||
source, target = os.path.join(root, path), os.path.join(copy, path)
|
||||
gone_or_submodule = not os.path.lexists(source) or os.path.isdir(source) and not os.path.islink(source)
|
||||
if not path or path.split("/")[0] == "mutants" or path.startswith(".mutate-changes-") or gone_or_submodule:
|
||||
if not path or path.split("/")[0] == "mutants" or gone_or_submodule:
|
||||
continue
|
||||
os.makedirs(os.path.dirname(target), exist_ok=True)
|
||||
shutil.copy2(source, target, follow_symlinks=False)
|
||||
@@ -138,6 +127,8 @@ def mutate_copy(copy: str, copied: list[str], folder: Folder, argv: list[str], p
|
||||
if isinstance(ran, Failure):
|
||||
return ran
|
||||
output, left = ran
|
||||
if not os.path.isdir(os.path.join(copy, "mutants")):
|
||||
return Failure(f"mutmut in {folder.label} wrote no mutants/ in {copy}; --prefix must run it in $PWD")
|
||||
if NO_MUTANT in output:
|
||||
return [f"mutmut in {folder.label}: none of the functions the changed lines fall in has a mutant."]
|
||||
results = run_tool(["mutmut", "results", "--all", "true"], copy, prefix)
|
||||
@@ -146,16 +137,18 @@ def mutate_copy(copy: str, copied: list[str], folder: Folder, argv: list[str], p
|
||||
shown, counted = not_killed(folder, statuses(results.output), copy)
|
||||
unreached = " No test reaches them." if NO_TEST in output else ""
|
||||
left_files = sorted({option.partition("=")[2].partition("::")[0] for option in left})
|
||||
left_note = f" Left out tests in {', '.join(left_files)} that fail without a mutant." if left else ""
|
||||
left_note = f" Left out the tests that fail without a mutant, in {', '.join(left_files)}." if left else ""
|
||||
return [f"mutmut in {folder.label}: {counted} mutants on changed lines, {len(shown)} not killed.{unreached}{left_note}", *shown]
|
||||
|
||||
|
||||
def run_until_stats_pass(copy: str, copied: list[str], folder: Folder, argv: list[str], prefix: str | None) -> tuple[str, list[str]] | Failure:
|
||||
"""mutmut's output, and the options that left out each test that fails before any mutant: mutmut runs no mutant while one fails."""
|
||||
ran = configured_run(copy, copied, folder, LIST_FAILURES, argv, prefix)
|
||||
ran = configured_run(copy, copied, folder, [], argv, prefix)
|
||||
left: list[str] = []
|
||||
if isinstance(ran, Ran) and STATS_FAILED in ran.output and (left := left_out(ran.output)):
|
||||
ran = configured_run(copy, copied, folder, left, argv, prefix)
|
||||
if isinstance(ran, Ran) and STATS_FAILED in ran.output:
|
||||
ran = configured_run(copy, copied, folder, LIST_FAILURES, argv, prefix)
|
||||
if isinstance(ran, Ran) and STATS_FAILED in ran.output and (left := left_out(ran.output)):
|
||||
ran = configured_run(copy, copied, folder, left, argv, prefix)
|
||||
if isinstance(ran, Failure):
|
||||
return ran
|
||||
if ran.code and NO_MUTANT not in ran.output and NO_TEST not in ran.output:
|
||||
|
||||
@@ -1,19 +1,25 @@
|
||||
import configparser
|
||||
import os
|
||||
import pathlib
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from mutate_changes.changed_lines import added_lines, runs, tool_for
|
||||
from mutate_changes.mutmut_tool import by_folder, copy_files
|
||||
from mutate_changes.changed_lines import added_lines, by_folder, not_mutated, runs, tool_for
|
||||
from mutate_changes.mutmut_tool import copy_files
|
||||
from failure import Failure
|
||||
from mutate_changes.mutmut_mutants import function_keys, mutant_globs, mutated_line, named, statuses
|
||||
from test_env import isolated_env
|
||||
from mutate_changes.mutmut_setup import left_out, mutmut_config
|
||||
from mutate_changes import main
|
||||
from mutate_changes.tool_process import run_tool, scratch_folder
|
||||
|
||||
SOURCE = "def double(n):\n\treturn n * 2\n\n\nclass Pt:\n\tdef moved(self, by=1):\n\t\treturn self.x + by\n"
|
||||
# The shape of the file mutmut 3 writes under mutants/: the original of each function and its mutants, with the original's lines.
|
||||
@@ -71,6 +77,20 @@ class ChangedLines(unittest.TestCase):
|
||||
with self.subTest(path):
|
||||
self.assertEqual(tool_for(path), tool)
|
||||
|
||||
def test_each_file_goes_to_the_nearest_folder_above_it_that_holds_one_of_the_configs(self):
|
||||
with tempfile.TemporaryDirectory() as top:
|
||||
for config in ("pkg/setup.cfg", "pkg/inner/lib/pyproject.toml", "web/package.json"):
|
||||
pathlib.Path(top, config).parent.mkdir(parents=True, exist_ok=True)
|
||||
pathlib.Path(top, config).write_text("")
|
||||
lines = frozenset({1})
|
||||
grouped = by_folder(top, {"pkg/inner/deep.py": lines, "pkg/inner/lib/x.py": lines, "top.py": lines}, ("pyproject.toml", "setup.cfg"))
|
||||
self.assertEqual(grouped, {"pkg": {"inner/deep.py": lines}, "pkg/inner/lib": {"x.py": lines}, ".": {"top.py": lines}})
|
||||
self.assertEqual(by_folder(top, {"web/src/a.ts": lines, "pkg/b.ts": lines}, ("package.json",)), {"web": {"src/a.ts": lines}, ".": {"pkg/b.ts": lines}})
|
||||
|
||||
def test_not_mutated_names_the_files_no_tool_covers_then_the_test_files(self):
|
||||
self.assertEqual(not_mutated(["README.md", "a.py", "b.go", "test_a.py", "web/a.test.ts"]), ["Not mutated, as no tool covers their language: README.md, b.go.", "Not mutated, as test files: test_a.py, web/a.test.ts."])
|
||||
self.assertEqual((not_mutated(["a.py", "test_a.py"]), not_mutated(["a.py", "b.go"]), not_mutated([])), (["Not mutated, as test files: test_a.py."], ["Not mutated, as no tool covers their language: b.go."], []))
|
||||
|
||||
|
||||
class MutmutSetup(unittest.TestCase):
|
||||
def test_the_config_mutates_the_sources_and_copies_each_entry_of_the_folder(self):
|
||||
@@ -162,31 +182,64 @@ class MutmutMutants(unittest.TestCase):
|
||||
|
||||
|
||||
class MutmutTool(unittest.TestCase):
|
||||
def test_each_file_goes_to_the_nearest_folder_above_it_that_sets_python_up(self):
|
||||
with tempfile.TemporaryDirectory() as top:
|
||||
for config in ("pkg/setup.cfg", "pkg/inner/lib/pyproject.toml"):
|
||||
pathlib.Path(top, config).parent.mkdir(parents=True)
|
||||
pathlib.Path(top, config).write_text("")
|
||||
lines = frozenset({1})
|
||||
grouped = by_folder(top, {"pkg/inner/deep.py": lines, "pkg/inner/lib/x.py": lines, "top.py": lines})
|
||||
self.assertEqual(grouped, {"pkg": {"inner/deep.py": lines}, "pkg/inner/lib": {"x.py": lines}, ".": {"top.py": lines}})
|
||||
|
||||
def test_copy_files_copies_what_is_listed_but_mutmuts_own_folders_and_names_the_entries_at_the_top(self):
|
||||
def test_copy_files_copies_what_is_listed_but_mutmuts_own_folder_and_names_the_entries_at_the_top(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root, copy = pathlib.Path(temp, "root"), pathlib.Path(temp, "copy")
|
||||
for path in ("a.py", "d/b.py", "mutants/x.py", ".mutate-changes-old/copy/y.py", "sub/c.py"):
|
||||
for path in ("a.py", "d/b.py", "mutants/x.py", "sub/c.py"):
|
||||
(root / path).parent.mkdir(parents=True, exist_ok=True)
|
||||
(root / path).write_text("")
|
||||
(root / "link").symlink_to("a.py")
|
||||
(root / "dirlink").symlink_to("d")
|
||||
# The list names a submodule as one path, a folder, and ends with an empty path.
|
||||
listed = ["", "mutants/x.py", ".mutate-changes-old/copy/y.py", "gone.py", "sub", "a.py", "d/b.py", "dirlink", "link"]
|
||||
listed = ["", "mutants/x.py", "gone.py", "sub", "a.py", "d/b.py", "dirlink", "link"]
|
||||
self.assertEqual(copy_files(str(root), listed, str(copy)), ["a.py", "d", "dirlink", "link"])
|
||||
walked = sorted(os.path.relpath(os.path.join(folder, name), copy) for folder, folders, files in os.walk(copy) for name in folders + files)
|
||||
self.assertEqual(walked, ["a.py", "d", "d/b.py", "dirlink", "link"])
|
||||
self.assertEqual((os.readlink(copy / "link"), os.readlink(copy / "dirlink")), ("a.py", "d"))
|
||||
|
||||
|
||||
class ToolProcess(unittest.TestCase):
|
||||
def test_git_lists_nothing_in_a_scratch_folder(self):
|
||||
with tempfile.TemporaryDirectory() as top:
|
||||
env = {**isolated_env(), "GIT_CONFIG_GLOBAL": "/dev/null"}
|
||||
subprocess.run(["git", "init", "-q", top], check=True, env=env)
|
||||
scratch = scratch_folder(top)
|
||||
pathlib.Path(scratch, "copy.py").write_text("")
|
||||
status = subprocess.run(["git", "status", "--porcelain", "--untracked-files=all"], capture_output=True, check=True, cwd=top, encoding="utf-8", env=env).stdout
|
||||
self.assertEqual((os.path.dirname(scratch), os.path.basename(scratch)[:16], status), (top, ".mutate-changes-", ""))
|
||||
|
||||
def test_a_scratch_folder_it_cannot_make_is_a_failure(self):
|
||||
self.assertRegex(scratch_folder("/nonexistent/top").reason, "^could not make a scratch folder in /nonexistent/top: ")
|
||||
|
||||
def test_an_interrupted_run_ends_each_process_the_tool_started(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
started = pathlib.Path(temp, "sleep.pid")
|
||||
|
||||
def interrupt():
|
||||
while not started.is_file() or not started.read_text().strip():
|
||||
time.sleep(0.05)
|
||||
os.kill(os.getpid(), signal.SIGINT)
|
||||
|
||||
threading.Thread(target=interrupt, daemon=True).start()
|
||||
with self.assertRaises(KeyboardInterrupt):
|
||||
run_tool(["true"], temp, f"sleep 60 & echo $! > {started}; wait;")
|
||||
sleeper = int(started.read_text())
|
||||
for _ in range(200):
|
||||
if ended(sleeper):
|
||||
break
|
||||
time.sleep(0.05)
|
||||
else:
|
||||
self.fail(f"sleep {sleeper}, which the tool started, still runs")
|
||||
|
||||
|
||||
def ended(pid: int) -> bool:
|
||||
"""Whether a process has ended: gone, or a zombie its parent has yet to reap."""
|
||||
try:
|
||||
return pathlib.Path(f"/proc/{pid}/stat").read_text().split(") ")[1][0] == "Z"
|
||||
except (FileNotFoundError, ProcessLookupError):
|
||||
return True
|
||||
|
||||
|
||||
class Took(unittest.TestCase):
|
||||
def test_took_rounds_to_the_second_and_counts_minutes_from_one(self):
|
||||
self.assertEqual([main.took(seconds) for seconds in (59.4, 59.6, 61, 3725.4)], ["59 s", "1 min 0 s", "1 min 1 s", "62 min 5 s"])
|
||||
|
||||
@@ -5,9 +5,11 @@ import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
import unittest.mock
|
||||
|
||||
@@ -15,12 +17,15 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from test_env import isolated_env
|
||||
from mutate_changes import main
|
||||
from mutate_changes.in_diff_tools import run_cargo_mutants
|
||||
|
||||
TOOL = pathlib.Path(__file__).resolve().parents[1] / "mutate-changes"
|
||||
SETUP = "[mutmut]\ndo_not_mutate = *_old.py\n"
|
||||
PASSED = ["mutmut in pkg: 2 mutants on changed lines, 1 not killed.", "pkg/calc.py:6: survived: return n / 3"]
|
||||
NOT_MUTATED = ["Not mutated, as no tool covers their language: README.md.", "Not mutated, as test files: pkg/test_calc.py."]
|
||||
# Stands in for mutmut, run through --prefix as `python3 <this> <log> <mode> mutmut ...` in the scratch copy. It logs each call with
|
||||
# the folder it runs in and the setup.cfg it finds there, then writes a mutants/ copy and prints results, both shaped like mutmut 3's.
|
||||
# In mode locks, it also leaves a folder in the copy that nobody may delete from.
|
||||
FAKE_MUTMUT = r"""import os, pathlib, sys
|
||||
log, mode, args = pathlib.Path(sys.argv[1]), sys.argv[2], sys.argv[3:]
|
||||
config = pathlib.Path("setup.cfg").read_text()
|
||||
@@ -36,17 +41,24 @@ elif mode == "fails-once" and not ("--deselect=test_calc.py::test_slow[n=1]" in
|
||||
print("==== short test summary info ====\nFAILED test_calc.py::test_slow[n=1] - boom\nERROR test_io.py - FileNotFoundError\nfailed to collect stats. runner returned 1")
|
||||
sys.exit(1)
|
||||
else:
|
||||
if mode != "no-copy":
|
||||
if mode != "elsewhere":
|
||||
pathlib.Path("mutants").mkdir(exist_ok=True)
|
||||
if mode not in ("elsewhere", "no-copy"):
|
||||
functions = ["x_double__mutmut_orig(n): # doubles\n\treturn n * 2", "x_double__mutmut_1(n): # doubles\n\treturn n / 2", "x_triple__mutmut_orig(n):\n\treturn n * 3", "x_triple__mutmut_1(n):\n\treturn n / 3", "x_triple__mutmut_2(n):\n\treturn n * 4"]
|
||||
pathlib.Path("mutants/calc.py").write_bytes(b"# \xff is not UTF-8\n" + "".join(f"def {function}\n" for function in functions).encode())
|
||||
if mode == "locks":
|
||||
pathlib.Path("locked/inner").mkdir(parents=True)
|
||||
pathlib.Path("locked").chmod(0o555)
|
||||
if mode == "no-test":
|
||||
print("Stopping early, because we could not find any test case for any mutant.")
|
||||
print("Stopping early, because we could not find any test case for any mutant. It seems that the selected tests do not cover any code that we mutated.")
|
||||
sys.exit(1)
|
||||
if mode == "setup-fault":
|
||||
print("Stopping early, because tests recorded trampoline hits but none match any mutant key. This usually means imports bypass the mutants.")
|
||||
sys.exit(1)
|
||||
"""
|
||||
# cargo-mutants, Infection and Stryker, which log the folder they run in and how they were called, and exit with the code the test
|
||||
# gives. cargo-mutants also logs the diff it is given and writes its outcome files, all but unviable.txt, as a run without an
|
||||
# unviable mutant does; with CAUGHT_UNREADABLE set, nobody may read caught.txt.
|
||||
# unviable mutant does; with CAUGHT_UNREADABLE set, nobody may read caught.txt, and with OUTPUT_LOCKED, nobody may delete from mutants.out.
|
||||
FAKE_TOOLS = r"""import os, pathlib, sys
|
||||
log, code, args = pathlib.Path(sys.argv[1]), int(sys.argv[2]), sys.argv[3:]
|
||||
with log.open("a") as written:
|
||||
@@ -60,6 +72,8 @@ with log.open("a") as written:
|
||||
(outcomes / "timeout.txt").write_text("src/lib.rs:2:7: replace + with * in add\n")
|
||||
if os.environ.get("CAUGHT_UNREADABLE"):
|
||||
(outcomes / "caught.txt").chmod(0)
|
||||
if os.environ.get("OUTPUT_LOCKED"):
|
||||
outcomes.chmod(0o555)
|
||||
print(f"{args[0]} reports")
|
||||
sys.exit(code)
|
||||
"""
|
||||
@@ -100,7 +114,7 @@ class MutateChangesRuns(unittest.TestCase):
|
||||
(self.checkout / "pkg" / "test_calc.py").write_text("# changed, but a test file is not mutated\n")
|
||||
self.commit()
|
||||
# Settings a user may have that change what git diff prints, and a file git lists as untracked and one it ignores.
|
||||
for name, value in (("color.diff", "always"), ("diff.relative", "true"), ("diff.external", "true"), ("diff.blank.textconv", "true")):
|
||||
for name, value in (("color.diff", "always"), ("diff.mnemonicPrefix", "true"), ("diff.relative", "true"), ("diff.external", "true"), ("diff.blank.textconv", "true")):
|
||||
self.git("config", name, value)
|
||||
(self.checkout / ".git" / "info").mkdir(exist_ok=True)
|
||||
(self.checkout / ".git" / "info" / "attributes").write_text("* diff=blank\n")
|
||||
@@ -134,8 +148,11 @@ class MutateChangesRuns(unittest.TestCase):
|
||||
|
||||
def change_rust_php_and_typescript(self):
|
||||
"""Commit a Rust, a PHP and a TypeScript file, then change them: Rust's line 2 in the index, PHP's line 2 with a byte that is
|
||||
not UTF-8, and TypeScript's lines 1 and 4."""
|
||||
not UTF-8, and TypeScript's lines 1 and 4. Also change line 1 of a TypeScript file in web/, which holds a package.json."""
|
||||
(self.checkout / "src").mkdir()
|
||||
(self.checkout / "web" / "src").mkdir(parents=True)
|
||||
(self.checkout / "web" / "package.json").write_text("{}\n")
|
||||
(self.checkout / "web" / "src" / "app.ts").write_text("export const two = 2;\n")
|
||||
(self.checkout / "src" / "lib.rs").write_text("pub fn add(a: i32, b: i32) -> i32 {\n\ta + b\n}\n")
|
||||
(self.checkout / "src" / "Calc.php").write_text("<?php\nfunction add($a, $b) { return $a + $b; }\n")
|
||||
(self.checkout / "src" / "calc.ts").write_text("export function add(a: number, b: number) {\n\treturn a + b;\n}\nexport const one = 1;\n")
|
||||
@@ -144,6 +161,7 @@ class MutateChangesRuns(unittest.TestCase):
|
||||
self.git("add", "src/lib.rs")
|
||||
(self.checkout / "src" / "Calc.php").write_bytes(b"<?php\nfunction add($a, $b) { return $b + $a; } // caf\xe9\n")
|
||||
(self.checkout / "src" / "calc.ts").write_text("export function add(b: number, a: number) {\n\treturn a + b;\n}\nexport const one = 2;\n")
|
||||
(self.checkout / "web" / "src" / "app.ts").write_text("export const two = 3;\n")
|
||||
|
||||
def path_with_git(self):
|
||||
"""A PATH that finds git and no mutation tool."""
|
||||
@@ -154,9 +172,11 @@ class MutateChangesRuns(unittest.TestCase):
|
||||
|
||||
def test_prints_each_mutant_on_a_changed_line_that_its_tests_do_not_kill(self):
|
||||
ran = self.run_tool(*self.faked("passes"), "--jobs", "2")
|
||||
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:2]), (0, PASSED), ran.stderr)
|
||||
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-1], ran.stderr), (0, [*PASSED, *NOT_MUTATED], ""))
|
||||
self.assertRegex(ran.stdout.splitlines()[-1], r"^mutate-changes took \d+ s\.$")
|
||||
calls = self.log.read_text()
|
||||
# mutmut's stats and mutant runs stop at the first test that fails, unless an option tells pytest otherwise.
|
||||
self.assertNotIn("--maxfail=0", calls)
|
||||
self.assertRegex(calls, rf"(?m)^{re.escape(str(self.checkout.resolve()))}/\.mutate-changes-\w+/copy$")
|
||||
self.assertIn("mutmut run --max-children 2 calc.x_double* calc.x_triple*\n", calls)
|
||||
self.assertIn("do_not_mutate = *_old.py\n", calls)
|
||||
@@ -172,15 +192,24 @@ class MutateChangesRuns(unittest.TestCase):
|
||||
def test_leaves_out_the_tests_that_fail_before_any_mutant_and_names_them(self):
|
||||
ran = self.run_tool(*self.faked("fails-once"))
|
||||
self.assertEqual(ran.returncode, 0, ran.stderr)
|
||||
self.assertEqual(ran.stdout.splitlines()[:2], ["mutmut in pkg: 2 mutants on changed lines, 1 not killed. Left out tests in test_calc.py, test_io.py that fail without a mutant.", "pkg/calc.py:6: survived: return n / 3"])
|
||||
first, second = self.log.read_text().split("---\n")[:2]
|
||||
self.assertIn("--maxfail=0", first)
|
||||
self.assertNotIn("--maxfail=0", second)
|
||||
self.assertEqual(ran.stdout.splitlines()[:2], ["mutmut in pkg: 2 mutants on changed lines, 1 not killed. Left out the tests that fail without a mutant, in test_calc.py, test_io.py.", "pkg/calc.py:6: survived: return n / 3"])
|
||||
runs = self.log.read_text().split("---\n")[:3]
|
||||
self.assertEqual(["--maxfail=0" in run for run in runs], [False, True, False])
|
||||
self.assertIn("--deselect=test_calc.py::test_slow[n=1]", runs[2])
|
||||
|
||||
def test_says_where_no_test_reaches_the_mutants(self):
|
||||
ran = self.run_tool(*self.faked("no-test"))
|
||||
self.assertEqual(ran.stdout.splitlines()[:3], ["mutmut in pkg: 2 mutants on changed lines, 2 not killed. No test reaches them.", "pkg/calc.py:6: no tests: return n * 4", "pkg/calc.py:6: no tests: return n / 3"], ran.stderr)
|
||||
|
||||
def test_a_run_that_stops_early_for_another_reason_fails_the_run(self):
|
||||
ran = self.run_tool(*self.faked("setup-fault"))
|
||||
self.assertEqual((ran.returncode, ran.stderr.splitlines()[0]), (1, "mutmut run in pkg exited 1:"))
|
||||
|
||||
def test_a_prefix_that_runs_mutmut_elsewhere_fails_the_run(self):
|
||||
ran = self.run_tool(*self.faked("elsewhere"))
|
||||
self.assertEqual(ran.returncode, 1)
|
||||
self.assertRegex(ran.stderr, rf"^mutmut in pkg wrote no mutants/ in {re.escape(str(self.checkout.resolve()))}/\.mutate-changes-\w+/copy; --prefix must run it in \$PWD\n$")
|
||||
|
||||
def test_names_a_mutant_whose_line_it_cannot_find(self):
|
||||
ran = self.run_tool(*self.faked("no-copy"))
|
||||
self.assertEqual(ran.stdout.splitlines()[:3], ["mutmut in pkg: 3 mutants on changed lines, 2 not killed.", "pkg/calc.py: survived: calc.x_double__mutmut_1, on a line mutate-changes cannot find", "pkg/calc.py: survived: calc.x_triple__mutmut_1, on a line mutate-changes cannot find"], ran.stderr)
|
||||
@@ -228,6 +257,33 @@ class MutateChangesRuns(unittest.TestCase):
|
||||
self.assertRegex(ran.stderr, rf"^could not set mutmut up in {top}/\.mutate-changes-\w+, a copy of {top}/pkg: ")
|
||||
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "pkg"])
|
||||
|
||||
def test_a_scratch_folder_it_cannot_remove_is_named_and_the_report_stands(self):
|
||||
top = re.escape(str(self.checkout.resolve()))
|
||||
self.addCleanup(lambda: [path.chmod(0o755) for path in self.checkout.glob(".mutate-changes-*/**/*") if path.is_dir()])
|
||||
ran = self.run_tool(*self.faked("locks"))
|
||||
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:2]), (0, PASSED))
|
||||
self.assertRegex(ran.stderr, rf"^could not remove {top}/\.mutate-changes-\w+: .*\n$")
|
||||
self.change_rust_php_and_typescript()
|
||||
ran = self.run_tool("--base", "HEAD", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", env={"OUTPUT_LOCKED": "1"})
|
||||
self.assertEqual((ran.returncode, ran.stdout.splitlines()[0]), (0, "cargo-mutants: 3 mutants on changed lines, 2 not caught."))
|
||||
self.assertRegex(ran.stderr, rf"^could not remove {top}/\.mutate-changes-\w+: .*\n$")
|
||||
|
||||
def test_a_signal_that_ends_the_run_ends_the_tool_and_removes_the_scratch_copy(self):
|
||||
started = self.temp / "sleep.pid"
|
||||
launched = subprocess.Popen([str(TOOL), "--prefix", f"sleep 60 & echo $! > {started}; wait;"], cwd=self.checkout, env=isolated_env(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
while launched.poll() is None and not (started.is_file() and started.read_text().strip()):
|
||||
time.sleep(0.05)
|
||||
launched.send_signal(signal.SIGTERM)
|
||||
self.assertEqual(launched.wait(timeout=30), 128 + signal.SIGTERM)
|
||||
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "pkg"])
|
||||
sleeper = pathlib.Path(f"/proc/{started.read_text().strip()}/stat")
|
||||
for _ in range(200):
|
||||
if not sleeper.exists() or sleeper.read_text().split(") ")[1][0] == "Z":
|
||||
break
|
||||
time.sleep(0.05)
|
||||
else:
|
||||
self.fail("the sleep the tool started still runs")
|
||||
|
||||
def test_a_tool_that_fails_fails_the_run_with_the_end_of_its_output(self):
|
||||
ran = self.run_tool("--prefix", "seq 45; echo broken >&2; printf '\\377'; false")
|
||||
self.assertEqual((ran.returncode, ran.stderr), (1, "\n".join(["mutmut run in pkg exited 1:", *map(str, range(8, 46)), "broken", "�"]) + "\n"))
|
||||
@@ -237,7 +293,7 @@ class MutateChangesRuns(unittest.TestCase):
|
||||
self.assertEqual((ran.returncode, ran.stderr), (1, "mutmut results in pkg exited 1:\nmutmut results broke\n"))
|
||||
|
||||
def test_a_tool_gets_an_empty_input_in_place_of_the_terminal(self):
|
||||
# npx, for one, asks before it installs a package, and would wait for an answer.
|
||||
# A tool that reads its input, such as a test that asks a question, gets an end of file instead of waiting on the terminal.
|
||||
with stdin_holding("y\n"):
|
||||
ran = self.run_tool("--prefix", "cat; false")
|
||||
self.assertEqual(ran.stderr, "mutmut run in pkg exited 1:\n")
|
||||
@@ -252,24 +308,46 @@ class MutateChangesRuns(unittest.TestCase):
|
||||
ran = self.run_tool(env={"PATH": str(tools)})
|
||||
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:2]), (0, PASSED), ran.stderr)
|
||||
|
||||
def test_rust_php_and_typescript_go_to_cargo_mutants_infection_and_stryker_at_the_checkouts_top(self):
|
||||
def test_rust_php_and_typescript_go_to_cargo_mutants_and_infection_at_the_top_and_to_stryker_by_package(self):
|
||||
self.change_rust_php_and_typescript()
|
||||
ran = self.run_tool("--base", "HEAD", "--jobs=3", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} 2", cwd=self.checkout / "src")
|
||||
ran = self.run_tool("--base", "HEAD", "--jobs=3", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", cwd=self.checkout / "src")
|
||||
report = ["cargo-mutants: 3 mutants on changed lines, 2 not caught.", "src/lib.rs:2:5: missed: replace add(a: i32, b: i32) -> i32 with 0", "src/lib.rs:2:7: timeout: replace + with * in add"]
|
||||
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-1]), (0, [*report, "Infection exited 2; its report:", "vendor/bin/infection reports", "Stryker exited 2; its report:", "npx reports"]), ran.stderr)
|
||||
cargo, infection, stryker = self.log.read_text().split(f"{self.checkout.resolve()}\n")[1:]
|
||||
command, diff = cargo.split("\n", 1)
|
||||
stryker_reports = ["Stryker in the checkout's top:", "npx reports", "Stryker in web:", "npx reports"]
|
||||
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-1]), (0, [*report, "Infection in the checkout's top:", "vendor/bin/infection reports", *stryker_reports]), ran.stderr)
|
||||
cargo, infection, stryker, web = self.log.read_text().split(f"{self.checkout.resolve()}")[1:]
|
||||
command, diff = cargo.removeprefix("\n").split("\n", 1)
|
||||
self.assertRegex(command, r"^cargo mutants --in-diff (\.mutate-changes-\w+)/branch\.diff --output \1 --jobs 3$")
|
||||
self.assertEqual([line for line in diff.splitlines() if line[:1] in "+-"], ["--- a/src/lib.rs", "+++ b/src/lib.rs", "-\ta + b", "+\tb + a"])
|
||||
self.assertEqual(infection, f"vendor/bin/infection --git-diff-lines --git-diff-base={self.git('rev-parse', 'HEAD').strip()} --threads=3 --show-mutations=max --no-progress\n")
|
||||
self.assertEqual(stryker, "npx stryker run --mutate src/calc.ts:1-1,src/calc.ts:4-4 --concurrency 3 --reporters clear-text\n")
|
||||
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "pkg", "src"])
|
||||
self.assertEqual(infection, f"\nvendor/bin/infection --git-diff-lines --git-diff-base={self.git('rev-parse', 'HEAD').strip()} --threads=3 --show-mutations=max --no-progress\n")
|
||||
self.assertEqual(stryker, "\nnpx --no stryker run --mutate src/calc.ts:1-1,src/calc.ts:4-4 --concurrency 3 --reporters clear-text\n")
|
||||
self.assertEqual(web, "/web\nnpx --no stryker run --mutate src/app.ts:1-1 --concurrency 3 --reporters clear-text\n")
|
||||
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "pkg", "src", "web"])
|
||||
|
||||
def test_a_stryker_or_infection_run_that_exits_nonzero_fails_the_run(self):
|
||||
self.change_rust_php_and_typescript()
|
||||
self.git("checkout", "HEAD", "--", "src/lib.rs")
|
||||
fails = ["--base", "HEAD", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} 1"]
|
||||
self.assertEqual(self.run_tool(*fails).stderr, "Infection exited 1:\nvendor/bin/infection reports\n")
|
||||
self.git("checkout", "HEAD", "--", "src/Calc.php")
|
||||
self.assertEqual(self.run_tool(*fails).stderr, "Stryker in the checkout's top exited 1:\nnpx reports\n")
|
||||
|
||||
def test_a_changed_file_named_like_an_option_is_mutated(self):
|
||||
for name in ("-x.py", "-x.rs"):
|
||||
(self.checkout / name).write_text("")
|
||||
self.commit()
|
||||
(self.checkout / "-x.py").write_text("def f():\n\treturn 1\n")
|
||||
(self.checkout / "-x.rs").write_text("fn f() {}\n")
|
||||
base = self.git("rev-parse", "HEAD").strip()
|
||||
self.assertEqual(main.changed_by_tool(str(self.checkout), base, ["-x.py", "-x.rs"]), {"cargo-mutants": {"-x.rs": frozenset({1})}, "mutmut": {"-x.py": frozenset({1, 2})}})
|
||||
ran = run_cargo_mutants(str(self.checkout), base, ["-x.rs"], f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", 1)
|
||||
self.assertIsInstance(ran, list)
|
||||
self.assertIn("+++ b/-x.rs\n", self.log.read_text())
|
||||
|
||||
def test_a_tool_that_cannot_start_fails_the_run(self):
|
||||
self.change_rust_php_and_typescript()
|
||||
path = str(self.path_with_git())
|
||||
# The tools run in the order of their names, so undoing the change of the one that failed lets the next one run.
|
||||
for tool, undone in (("cargo mutants", "src/lib.rs"), ("vendor/bin/infection", "src/Calc.php"), ("npx stryker", "src/calc.ts")):
|
||||
for tool, undone in (("cargo mutants", "src/lib.rs"), ("vendor/bin/infection", "src/Calc.php"), ("npx --no stryker", "src/calc.ts")):
|
||||
with self.subTest(tool):
|
||||
ran = self.run_tool("--base", "HEAD", env={"PATH": path})
|
||||
self.assertEqual(ran.returncode, 1)
|
||||
@@ -278,7 +356,9 @@ class MutateChangesRuns(unittest.TestCase):
|
||||
|
||||
def test_cargo_mutants_fails_the_run_unless_it_finished(self):
|
||||
self.change_rust_php_and_typescript()
|
||||
for code, failed in ((0, False), (1, True), (3, False), (4, True)):
|
||||
# The other tools fail on any code but 0.
|
||||
self.git("checkout", "HEAD", "--", "src/Calc.php", "src/calc.ts", "web/src/app.ts")
|
||||
for code, failed in ((0, False), (1, True), (2, False), (3, False), (4, True)):
|
||||
with self.subTest(code):
|
||||
ran = self.run_tool("--base", "HEAD", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} {code}")
|
||||
self.assertEqual((ran.returncode, ran.stderr), (1, f"cargo mutants exited {code}:\ncargo reports\n") if failed else (0, ""))
|
||||
@@ -287,12 +367,18 @@ class MutateChangesRuns(unittest.TestCase):
|
||||
self.change_rust_php_and_typescript()
|
||||
ran = self.run_tool("--base", "HEAD", "--prefix", f"python3 {self.temp / 'fake_tools.py'} {self.log} 0", env={"CAUGHT_UNREADABLE": "1"})
|
||||
self.assertEqual(ran.returncode, 1)
|
||||
self.assertRegex(ran.stderr, rf"^could not read cargo-mutants' outcomes in {re.escape(str(self.checkout.resolve()))}/\.mutate-changes-\w+: ")
|
||||
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "pkg", "src"])
|
||||
self.assertRegex(ran.stderr, rf"^could not hand cargo-mutants the diff or read its outcomes in {re.escape(str(self.checkout.resolve()))}/\.mutate-changes-\w+: ")
|
||||
self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "README.md", "pkg", "src", "web"])
|
||||
|
||||
def test_a_branch_that_changes_no_mutated_language_says_so(self):
|
||||
def test_a_branch_that_adds_no_line_a_tool_mutates_says_so_and_names_what_it_changes(self):
|
||||
ran = self.run_tool(*self.faked("passes"), "--base", "HEAD")
|
||||
self.assertEqual((ran.returncode, ran.stdout), (0, "No line the branch changes is in a language mutate-changes mutates.\n"))
|
||||
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-1]), (0, ["No tool ran: the branch adds no line to a file that a tool mutates."]))
|
||||
(self.checkout / "README.md").write_text("Doubles.\n")
|
||||
(self.checkout / "pkg" / "test_calc.py").write_text("")
|
||||
(self.checkout / "pkg" / "calc.py").write_text("def double(n): # doubles\n\treturn n * 2\n")
|
||||
ran = self.run_tool(*self.faked("passes"), "--base", "HEAD")
|
||||
self.assertEqual((ran.returncode, ran.stdout.splitlines()[:-1]), (0, ["No tool ran: the branch adds no line to a file that a tool mutates.", *NOT_MUTATED]))
|
||||
self.assertRegex(ran.stdout.splitlines()[-1], r"^mutate-changes took \d+ s\.$")
|
||||
self.assertFalse(self.log.exists())
|
||||
|
||||
def test_help_prints_the_usage(self):
|
||||
@@ -306,6 +392,7 @@ class MutateChangesRuns(unittest.TestCase):
|
||||
for args, reason in (
|
||||
(["--jobs", "0"], "--jobs takes a whole number from 1, not 0"),
|
||||
(["--jobs", "x"], "--jobs takes a whole number from 1, not x"),
|
||||
(["--jobs", "²"], "--jobs takes a whole number from 1, not ²"),
|
||||
(["--jobs"], "--jobs needs a value"),
|
||||
(["--colour"], "unknown or repeated argument --colour"),
|
||||
(["--base", "a", "--base", "b"], "unknown or repeated argument --base"),
|
||||
|
||||
@@ -1,7 +1,14 @@
|
||||
"""Start a mutation tool, through --prefix's shell text where the run gives one, and wait for it."""
|
||||
"""Start a mutation tool, through --prefix's shell text where the run gives one, and wait for it; and make and remove its scratch folder."""
|
||||
import contextlib
|
||||
import dataclasses
|
||||
import os
|
||||
import pathlib
|
||||
import shlex
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from failure import Failure
|
||||
|
||||
@@ -18,12 +25,39 @@ def run_tool(argv: list[str], cwd: str, prefix: str | None) -> Ran | Failure:
|
||||
With a prefix, sh runs `<prefix> "$@"` with argv as its arguments, so the prefix keeps its own quoting and each of argv stays one word."""
|
||||
command = ["sh", "-c", f'{prefix} "$@"', "sh", *argv] if prefix else argv
|
||||
try:
|
||||
ran = subprocess.run(command, cwd=cwd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8", errors="replace")
|
||||
# In a session of its own, the tool and each process it starts share one process group, which ends whole below.
|
||||
process = subprocess.Popen(command, cwd=cwd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8", errors="replace", start_new_session=True)
|
||||
except OSError as failure:
|
||||
return Failure(f"{shlex.join(argv)} could not start: {failure.strerror}")
|
||||
return Ran(ran.returncode, ran.stdout)
|
||||
try:
|
||||
output = process.communicate()[0]
|
||||
except BaseException:
|
||||
# The run is ending, as on Ctrl-C or the SIGTERM the launcher turns into SystemExit, so end what the tool started too.
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait()
|
||||
raise
|
||||
return Ran(process.returncode, output)
|
||||
|
||||
|
||||
def failed(name: str, ran: Ran) -> Failure:
|
||||
"""A tool's exit code and the last 40 lines of its output, which hold its error."""
|
||||
return Failure("\n".join([f"{name} exited {ran.code}:", *ran.output.splitlines()[-40:]]))
|
||||
|
||||
|
||||
def scratch_folder(top: str) -> str | Failure:
|
||||
"""A new folder in top for one tool's files. It holds a .gitignore of `*`, as pytest's cache folder does, so git lists none of them."""
|
||||
try:
|
||||
scratch = tempfile.mkdtemp(prefix=".mutate-changes-", dir=top)
|
||||
pathlib.Path(scratch, ".gitignore").write_text("*\n", encoding="utf-8")
|
||||
except OSError as failure:
|
||||
return Failure(f"could not make a scratch folder in {top}: {failure}")
|
||||
return scratch
|
||||
|
||||
|
||||
def remove(scratch: str) -> None:
|
||||
"""Remove a scratch folder, saying on stderr where it can't: the run's report holds, so the run doesn't fail."""
|
||||
try:
|
||||
shutil.rmtree(scratch)
|
||||
except OSError as failure:
|
||||
print(f"could not remove {scratch}: {failure}", file=sys.stderr)
|
||||
|
||||
Reference in New Issue
Block a user