From a52ebb94bf849a3f82580c02e51f968da94be52d Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Wed, 11 Nov 2026 18:58:31 +0100 Subject: [PATCH] Run mutation tests on a chunk's changed lines before review with bin/mutate-changes, and hand stability-reviewer the survivors --- .gitignore | 5 + AGENTS.md | 2 +- README.md | 18 +- agents/stability-reviewer.md | 2 +- bin/mutate-changes | 9 + bin/mutate_changes/__init__.py | 0 bin/mutate_changes/changed_lines.py | 35 ++++ bin/mutate_changes/in_diff_tools.py | 55 ++++++ bin/mutate_changes/main.py | 130 ++++++++++++ bin/mutate_changes/mutmut_mutants.py | 81 ++++++++ bin/mutate_changes/mutmut_setup.py | 51 +++++ bin/mutate_changes/mutmut_tool.py | 189 ++++++++++++++++++ bin/mutate_changes/test_mutate_changes.py | 231 ++++++++++++++++++++++ bin/mutate_changes/tool_process.py | 29 +++ bin/rule_hook/heavy_command.py | 2 +- bin/rule_hook/rules.py | 2 +- bin/rule_hook/test_rule_hook.py | 4 +- bin/setup.cfg | 2 + docs/decisions.md | 8 +- skills/larv-review/SKILL.md | 4 +- todo.md | 6 +- workflow-bench/setup.cfg | 2 + 22 files changed, 840 insertions(+), 27 deletions(-) create mode 100755 bin/mutate-changes create mode 100644 bin/mutate_changes/__init__.py create mode 100644 bin/mutate_changes/changed_lines.py create mode 100644 bin/mutate_changes/in_diff_tools.py create mode 100644 bin/mutate_changes/main.py create mode 100644 bin/mutate_changes/mutmut_mutants.py create mode 100644 bin/mutate_changes/mutmut_setup.py create mode 100644 bin/mutate_changes/mutmut_tool.py create mode 100644 bin/mutate_changes/test_mutate_changes.py create mode 100644 bin/mutate_changes/tool_process.py create mode 100644 bin/setup.cfg create mode 100644 workflow-bench/setup.cfg diff --git a/.gitignore b/.gitignore index cc723115..e3688003 100644 --- a/.gitignore +++ b/.gitignore @@ -68,6 +68,9 @@ !bin/metric_bench/*.py !bin/model_routing/ !bin/model_routing/*.py +!bin/mutate-changes +!bin/mutate_changes/ +!bin/mutate_changes/*.py !bin/rank_todo.py !bin/reading-cost !bin/reading_cost/ @@ -99,6 +102,7 @@ !bin/score_outcomes/ !bin/score_outcomes/*.py !bin/settings_filter.py +!bin/setup.cfg !bin/statusline.py !bin/subagent-rung !bin/subagent_rung/ @@ -138,5 +142,6 @@ !workflow-bench/compare.py !workflow-bench/review_git.py !workflow-bench/run-prompt.md +!workflow-bench/setup.cfg !workflow-bench/test_bench.py !workflow-bench/test_review_git.py diff --git a/AGENTS.md b/AGENTS.md index b90d9360..7d4cf60d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,7 +43,7 @@ Decisions live in `docs/decisions.md`: - review: the comprehension check records how hard a project is to read and blocks no merge: a paired ruling against the last measured commit in the release review, and the four-seat scoring run, once when an item names the floor and twice on one head when a release's items run out - review: the premises chunk's whole review is the human's approval - review: the principles audit runs only in the release review, and reads the project whole with one auditor checking the goals and every principles file that applies -- review: the stability pass mutation-tests the lines a chunk changes, in place of measuring test coverage +- review: mutate-changes mutation-tests the lines a chunk changes before review, in place of measuring test coverage - routing: a chunk's worker runs on the rung its item's L picks, below the project's rung too; its reviewers run on the highest of the worker's rung, the project's and the one picked by the highest E + I among the chunk's items, unless a replay lowered their pass or the branch changes only Markdown no agent follows, which reviews on the project's rung; the seats and the security pass always on the project's - routing: a review pass runs below the project's rung only on a rung that a replay of its rounds shows finding every 🔴 and 🟡 the project's rung finds, and only where the E + I steps no review up and the worker runs no higher than the project's rung, then even below the worker's; no pass has such a rung - routing: a worker delegates a task only where it needs just the task's conclusion, and each subagent runs on the rung its task's L picks from the project's rung, never below the project's rung when it files or scores todo.md items diff --git a/README.md b/README.md index d989cb35..111d3f0d 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ Install puts only `loop-claude` and `loop-inbox` on `PATH`; call the others by t - `loop-claude ` runs a project's next chunk cycle after cycle; `loop-claude --help` states its contract. - `loop-inbox` shows the interrupts of every loop-claude on the machine, oldest first, takes each one's answer, `/talk` included, and lists every running loop with the `todo.md` items its chunk takes; `loop-inbox --help` states how. - `metric-bench [history]` benches cheap static metrics of a project's Python against the places its comprehension history names as hardest; `metric-bench --help` states the metrics and the scores. +- `mutate-changes [--base ] [--prefix ] [--jobs ]` mutation-tests the lines a branch changes and prints each mutant its tests don't kill; `mutate-changes --help` says which tool it runs for each language. - `rank_todo.py [--check] [todo.md]` scores and ranks a project's `todo.md`; `rank_todo.py --help` states the format. - `reading-cost [--reset] [record]` appends a figure for each chunk `loop-claude` finished to a project's `docs/reading-cost.md`, and says whether the figures rise; `reading-cost --help` states the measure. - `review-replay ` replays a review pass's archived rounds from the projects run on the model, on the code each reviewed, and prints the lowest rung below the project's that, like each rung above it, finds every 🔴 and 🟡 the project's rung finds again; `review-replay --help` states the replay. Each replayed reviewer skips every permission prompt, stability-reviewer has Bash, and an archived prompt may name paths on the host. @@ -113,7 +114,7 @@ Install puts only `loop-claude` and `loop-inbox` on `PATH`; call the others by t - `subagent-rung [--plan] ` prints the Agent tool's model and effort for a subagent's task, picked by its L from the project's model, and with `--plan` never below the project's rung; `subagent-rung --help` states the rule. - `todo-item add …` files an item in a project's `todo.md`, taking `Next ID` as the item's ID, and `todo-item finish ` removes a finished item and adds the bullet passed with `--changelog` to `CHANGELOG.md`; both rank the file, and `todo-item --help` states the flags. -`bin/` is the import root: each launcher puts it on `sys.path`, so a module imports another by its path under `bin/`, such as `from claude_code.entries import parse_entry`. Each tool's package holds its tests. Beside the packages, `bin/`'s root holds only the launchers, the tools of one file with their tests, `test_env.py` and `failure.py`. `failure.py`'s `Failure`, which modules across `bin/` return for a failed step, belongs to no single package. `branch_chunks/`, `claude_code/`, `command_lines/`, `git_cli/`, `loop_effects/`, `loop_state/`, `machine_load/`, `model_routing/`, `records/`, `release_plan/`, `review_rules/`, `rung_replay/` and `transcript_archive/` serve more than one tool. +`bin/` is the import root: each launcher puts it on `sys.path`, so a module imports another by its path under `bin/`, such as `from claude_code.entries import parse_entry`. Each tool's package holds its tests. Beside the packages, `bin/`'s root holds only the launchers, the tools of one file with their tests, `test_env.py`, `failure.py` and `setup.cfg`. `failure.py`'s `Failure`, which modules across `bin/` return for a failed step, belongs to no single package. `branch_chunks/`, `claude_code/`, `command_lines/`, `git_cli/`, `loop_effects/`, `loop_state/`, `machine_load/`, `model_routing/`, `records/`, `release_plan/`, `review_rules/`, `rung_replay/` and `transcript_archive/` serve more than one tool. - `branch_chunks/` cuts a default branch into chunks and finds, for each, the items it shipped and the later chunks that fixed its code, for `score_outcomes` and `routing_figures`. It imports `reading_cost`'s record path. - `command_lines/` reads a shell command: its simple commands, and a docker call's subcommand and memory limits, for `command_patterns`, `reading_cost`, `routing_figures`, `rule_hook` and `run_heavy`. @@ -124,7 +125,7 @@ Install puts only `loop-claude` and `loop-inbox` on `PATH`; call the others by t - `loop_claude` runs `chunk-preflight` as a process of its own. - `loop_effects/` holds `Effects`: the processes, terminal, clock and git a tool reaches, injected so its tests can script them. loop-claude, loop-inbox, both replay tools and `reviewer-merge-trials`' driver use it. - `loop_state/` holds the files loop-claude and loop-inbox share: the question files (`waiting/`), the files a running loop lists itself in (`running_loops.py`, `session_lock.py`), their paths, and `hand_over.py`, which opens claude for `/talk`. A change to any of them changes both tools. -- `machine_load/` takes the machine's heavy slots, its shared lock files, for `run_heavy`, and reads the memory share that loop-claude waits for. +- `machine_load/` takes the machine's heavy slots, its shared lock files, for `run_heavy`, gives `mutate_changes` a slot's CPU count, and reads the memory share that loop-claude waits for. - `model_routing/` holds the routing ladder, which loop-claude, `rule_hook`, both replay tools and `subagent-rung` read. - `rank_todo.py`, `settings_filter.py` and `statusline.py` are tools of one file each, tested beside them. - `records/` reads a Markdown file's `##` sections, quotes a passage and joins names into a list, stamps times in UTC, and replaces a file in one step, private where it is new. @@ -142,18 +143,7 @@ Test on the host with `uvx --python "$(command -v python3)" --with pytest-xdist= CI runs `ci.sh` on every pull request, and `main` merges nothing until it passes. The script runs the same tests on the committed HEAD in a pinned Python container; run it under `run-heavy` to reproduce a CI failure, with Docker on `PATH`. -To mutation-test a changed module, copy `bin/` to a scratch directory (`cp -r bin "$scratch"`): mutmut writes files beside its config. In the copy, name the module and its tests in a `setup.cfg`, as below for `score_outcomes/results.py`. Leave out each test class that starts a tool's launcher through `subprocess`: a mutated module fails to load in a process mutmut did not start. - -```ini -[mutmut] -source_paths=score_outcomes/results.py -also_copy=. -pytest_add_cli_args_test_selection=score_outcomes/ -pytest_add_cli_args=-k - not ScoreOutcomesTool -``` - -Then, in the copy: `uvx --with pytest==9.2.0 'mutmut>=3,<4' run && uvx --with pytest==9.2.0 'mutmut>=3,<4' results`, and `uvx --with pytest==9.2.0 'mutmut>=3,<4' show ` to see a mutant's change. The config keys are mutmut 3's. +To mutation-test the lines a branch changes, run `~/.claude/bin/run-heavy ~/.claude/bin/mutate-changes --prefix 'uvx --python "$(command -v python3)" --with pytest==9.2.0 --from "mutmut>=3,<4"'` in its checkout. `bin/setup.cfg` and `workflow-bench/setup.cfg` make it run mutmut from those folders, since their tests import their modules from there. ## Tracking a new file diff --git a/agents/stability-reviewer.md b/agents/stability-reviewer.md index 84ae9ec3..ce38c8cf 100644 --- a/agents/stability-reviewer.md +++ b/agents/stability-reviewer.md @@ -13,7 +13,7 @@ You are an elite stability-focused code reviewer with deep expertise in defensiv 2. **Read every file under `~/.claude/principles/`**, the shared principles, and `~/.claude/skills/larv-review/standards/quality.md`. These are mandatory org guidelines; all code MUST comply with each that applies. A file's `Applies to` line says what it covers; one without the line covers every change. Re-read each time — they change. Flag violations prominently. 3. **Read README.md, AGENTS.md and `docs/decisions.md`** in the current project for project-specific conventions, build instructions, tooling, and the accepted trade-offs a reviewer should honor rather than re-flag. Note the documented test, lint, and static-analysis commands — you will run them. 4. **Verify, don't just read.** Run the test suite and static analysis/linters through the project's container tooling, unless a decision puts them on the host; tear down test containers afterwards. Run the full suite under `~/.claude/bin/run-heavy`. Run it in the background or with the Bash tool's longest timeout: that timeout includes the wait for a slot. A full-suite run catches broken callers that diff-reading misses. When a suspected bug is cheap to confirm, reproduce it — a scratch script, a targeted test run — before reporting it. Make scratch work with Bash, since Write is for your report alone. Put it in a directory you create for this run, and delete it when done; delete nothing else, since a directory the prompt names may hold other agents' files. Leave the working tree exactly as you found it. Anything you could not run goes in Coverage & Verification, never silently skipped. -5. **Mutation-test the changed lines.** Run the project's documented mutation command, or else a widely deployed mutation tool for the language: cargo-mutants for Rust, Infection for PHP, mutmut for Python, Stryker for JavaScript and TypeScript. Limit it to the lines the change adds or modifies, through the tool's own option where it has one (cargo-mutants `--in-diff `, Infection `--git-diff-lines`, Stryker `--mutate :`); otherwise mutate only the changed files and keep only the mutants on changed lines. Run it through the project's tooling and run-heavy, as in step 4, on a scratch copy: the tools write files. Under run-heavy, a container that sets neither memory limit gets the slot's share, in MiB in `RUN_HEAVY_MEMORY`. Set the tool's concurrency so that its parallel test runs together fit in that share. Report its wall time and mutant count in Coverage & Verification, or why nothing was mutated. Ignore a surviving mutant that leaves behaviour unchanged, or changes only message or log text no caller or user depends on. Report one 🟡 per test to add or strengthen, naming the surviving mutants it would catch. +5. **Judge the surviving mutants.** Your prompt hands you what `~/.claude/bin/mutate-changes` printed on the change, and the commit it ran on: the mutants of changed lines that no test kills. Where it hands none, or names a commit other than the head, run it yourself in the change's checkout, through run-heavy as in step 4: `~/.claude/bin/run-heavy ~/.claude/bin/mutate-changes --prefix ''`. `~/.claude/bin/mutate-changes --help` says which tool it runs for each language and what it prints. Under run-heavy, a container that sets neither memory limit gets the slot's share, in MiB in `RUN_HEAVY_MEMORY`; set `--jobs` so that its parallel test runs together fit in that share. Report the wall time and mutant count in Coverage & Verification, or why nothing was mutated. Ignore a surviving mutant that leaves behaviour unchanged, or changes only message or log text no caller or user depends on. Report one 🟡 per test to add or strengthen, naming the surviving mutants it would catch. 6. **Two passes minimum, then hunt absences.** - *Breadth:* every hunk in every changed file, no skimming. Track which files you have covered; batch large diffs, never truncate the review. - *Depth:* trace the riskiest data paths end-to-end (input → logic → persistence/response), the interactions between changed pieces, and between the change and config (timeout hierarchies, retries × idempotency, queue worker counts, env parity). diff --git a/bin/mutate-changes b/bin/mutate-changes new file mode 100755 index 00000000..bf3d692b --- /dev/null +++ b/bin/mutate-changes @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from mutate_changes import main + +sys.exit(main.main(sys.argv[1:])) diff --git a/bin/mutate_changes/__init__.py b/bin/mutate_changes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bin/mutate_changes/changed_lines.py b/bin/mutate_changes/changed_lines.py new file mode 100644 index 00000000..2ea733d6 --- /dev/null +++ b/bin/mutate_changes/changed_lines.py @@ -0,0 +1,35 @@ +"""Which lines of a file a branch adds or changes, and which mutation tool mutates the file.""" +import posixpath +import re +import typing + +# A hunk header of `git diff -U0`. The new side's count is 0 where the hunk only deletes, and left out where it is 1. +HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@", re.MULTILINE) +STRYKER = (".cjs", ".cts", ".js", ".jsx", ".mjs", ".mts", ".ts", ".tsx") +TOOLS = {".php": "infection", ".py": "mutmut", ".rs": "cargo-mutants", **dict.fromkeys(STRYKER, "stryker")} +TEST_FOLDERS = frozenset({"__tests__", "test", "tests"}) +TEST_NAME = re.compile(r"test_.*\.py|.*_test\.py|conftest\.py|.*\.(?:spec|test)\.[^.]+") + + +def added_lines(diff: str) -> frozenset[int]: + """The line numbers, in the new file, that a `git diff -U0` of one file adds or changes.""" + return frozenset(line for hunk in HUNK.finditer(diff) for line in range(int(hunk[1]), int(hunk[1]) + int(hunk[2] or 1))) + + +def runs(lines: typing.Iterable[int]) -> list[tuple[int, int]]: + """The lines as runs of consecutive numbers, each as its first and last line, in order.""" + found: list[tuple[int, int]] = [] + for line in sorted(lines): + if found and found[-1][1] == line - 1: + found[-1] = (found[-1][0], line) + else: + found.append((line, line)) + return found + + +def tool_for(path: str) -> str | None: + """The tool that mutates a file, by its extension; None for a test file, or a language none of the tools covers.""" + folders, name = posixpath.split(path) + if TEST_FOLDERS & set(folders.split("/")) or TEST_NAME.fullmatch(name): + return None + return TOOLS.get(posixpath.splitext(name)[1]) diff --git a/bin/mutate_changes/in_diff_tools.py b/bin/mutate_changes/in_diff_tools.py new file mode 100644 index 00000000..013e9b0a --- /dev/null +++ b/bin/mutate_changes/in_diff_tools.py @@ -0,0 +1,55 @@ +"""Mutate a branch's changed lines with the tools that limit a run to them themselves: cargo-mutants, Stryker and Infection. Each runs at the checkout's top.""" +import os +import pathlib +import shutil +import tempfile + +from mutate_changes.tool_process import failed, run_tool +from failure import Failure +from git_cli.git_repo import git +from mutate_changes.changed_lines import runs + +# cargo-mutants' exit codes for a run that finished: every mutant caught, some missed, and some timed out. +CARGO_FINISHED = frozenset({0, 2, 3}) +CARGO_OUTCOMES = ("caught", "missed", "timeout", "unviable") + + +def run_cargo_mutants(top: str, base: str, paths: list[str], prefix: str | None, jobs: int) -> list[str] | Failure: + """A line with cargo-mutants' mutant count, then a line per mutant it missed or that timed out, as `::: : `.""" + diff = git("diff", "--no-color", "--no-ext-diff", "--no-textconv", "--no-renames", base, "--", *paths, cwd=top) + if isinstance(diff, Failure): + return diff + try: + scratch = tempfile.mkdtemp(prefix=".mutate-changes-", dir=top) + except OSError as failure: + return Failure(f"could not make a scratch folder in {top}: {failure}") + try: + pathlib.Path(scratch, "branch.diff").write_text(diff, encoding="utf-8") + relative = os.path.relpath(scratch, top) + ran = run_tool(["cargo", "mutants", "--in-diff", os.path.join(relative, "branch.diff"), "--output", relative, "--jobs", str(jobs)], top, prefix) + if isinstance(ran, Failure) or ran.code not in CARGO_FINISHED: + return ran if isinstance(ran, Failure) else failed("cargo mutants", ran) + outcomes = {outcome: outcome_lines(pathlib.Path(scratch, "mutants.out", f"{outcome}.txt")) for outcome in CARGO_OUTCOMES} + except OSError as failure: + return Failure(f"could not read cargo-mutants' outcomes in {scratch}: {failure}") + finally: + shutil.rmtree(scratch, ignore_errors=True) + uncaught = [f"{location}: {outcome}: {mutation}" for outcome in ("missed", "timeout") for location, _, mutation in (line.partition(": ") for line in outcomes[outcome])] + return [f"cargo-mutants: {sum(map(len, outcomes.values()))} mutants on changed lines, {len(uncaught)} not caught.", *uncaught] + + +def outcome_lines(path: pathlib.Path) -> list[str]: + return path.read_text(encoding="utf-8", errors="replace").splitlines() if path.is_file() else [] + + +def run_stryker(top: str, changed: dict[str, frozenset[int]], prefix: str | None, jobs: int) -> list[str] | Failure: + """Stryker's exit code, then its own report of the mutants on the changed lines.""" + mutate = ",".join(f"{path}:{first}-{last}" for path, lines in sorted(changed.items()) for first, last in runs(lines)) + ran = run_tool(["npx", "stryker", "run", "--mutate", mutate, "--concurrency", str(jobs), "--reporters", "clear-text"], top, prefix) + return ran if isinstance(ran, Failure) else [f"Stryker exited {ran.code}; its report:", *ran.output.splitlines()] + + +def run_infection(top: str, base: str, prefix: str | None, jobs: int) -> list[str] | Failure: + """Infection's exit code, then its own report of the mutants on the lines the branch changes since base.""" + ran = run_tool(["vendor/bin/infection", "--git-diff-lines", f"--git-diff-base={base}", f"--threads={jobs}", "--show-mutations=max", "--no-progress"], top, prefix) + return ran if isinstance(ran, Failure) else [f"Infection exited {ran.code}; its report:", *ran.output.splitlines()] diff --git a/bin/mutate_changes/main.py b/bin/mutate_changes/main.py new file mode 100644 index 00000000..1f0f38b9 --- /dev/null +++ b/bin/mutate_changes/main.py @@ -0,0 +1,130 @@ +"""Mutation-test the lines a branch adds or changes, and print each mutant its tests don't kill. + +usage: mutate-changes [--base ] [--prefix ] [--jobs ] + +Run it in a checkout, under ~/.claude/bin/run-heavy. It diffs the working tree's tracked files against the merge base of HEAD and +--base, by default the default branch: origin's as last fetched, else main or master. It picks the tool by each changed file's +extension, and leaves out test files: test_*.py, *_test.py, conftest.py, *.test.* and *.spec.*, and any file under a tests/, test/ +or __tests__/ folder. + +- .py: mutmut 3, in a scratch copy of the nearest folder above the file that holds a setup.cfg or a pyproject.toml, else of the + checkout's top. It mutates each function or method a changed line falls in, and counts the mutants on changed lines. The + folder's setup.cfg settings stay, but source_paths and also_copy are set, and pytest leaves out each test that fails without a + mutant; the report names their files. A pyproject.toml with a [tool.mutmut] section fails the run. +- .rs: cargo-mutants, with --in-diff. +- .cjs, .cts, .js, .jsx, .mjs, .mts, .ts, .tsx: Stryker, with --mutate on the changed lines. +- .php: Infection, with --git-diff-lines. + +--prefix is shell text put before each tool's command, such as 'uvx --from "mutmut>=3,<4"' or 'docker compose run --rm app'. +Without it, each tool must be on PATH. mutmut runs in its scratch copy, a folder in the checkout's top, and every other tool in +the checkout's top; in the prefix, $PWD names that folder. --jobs sets how many tests each tool runs at once, 4 by default: one +heavy slot's CPUs. + +For mutmut and cargo-mutants, it prints a line with the mutant count, then a line per mutant on a changed line that is not +killed: `:: : `. Stryker and Infection print their own report, after their exit code. +The last line gives the wall time. + +Exit 1, saying why, on unknown arguments, or when git fails, a tool cannot start, or mutmut or cargo-mutants fails before it reports. +""" +import sys +import time + +from mutate_changes.changed_lines import added_lines, tool_for +from machine_load.slots import CPUS_PER_SLOT +from git_cli.git_repo import default_ref, git, top +from failure import Failure +from mutate_changes.in_diff_tools import run_cargo_mutants, run_infection, run_stryker +from mutate_changes.mutmut_tool import run_mutmut + +USAGE = "usage: mutate-changes [--base ] [--prefix ] [--jobs ]" +# A panel of three readers, seated as the README's audience, picked the command's name and these options on 2026-11-11. +OPTIONS = ("--base", "--jobs", "--prefix") + + +def main(argv: list[str]) -> int: + if argv in (["-h"], ["--help"]): + print(__doc__.strip()) + return 0 + given = options(argv) + if isinstance(given, Failure): + print(f"{given.reason}; {USAGE}", file=sys.stderr) + return 1 + started = time.monotonic() + report = mutate(given) + if isinstance(report, Failure): + print(report.reason, file=sys.stderr) + return 1 + if not report: + print("No line the branch changes is in a language mutate-changes mutates.") + return 0 + print("\n".join(report)) + print(f"mutate-changes took {took(time.monotonic() - started)}.") + return 0 + + +def options(argv: list[str]) -> dict[str, str] | Failure: + given: dict[str, str] = {} + rest = list(argv) + while rest: + name, equals, value = rest.pop(0).partition("=") + if name not in OPTIONS or name in given: + return Failure(f"unknown or repeated argument {name}") + if not equals and not rest: + return Failure(f"{name} needs a value") + given[name] = value if equals else rest.pop(0) + if not given.get("--jobs", "1").isdigit() or int(given.get("--jobs", "1")) < 1: + return Failure(f"--jobs takes a whole number from 1, not {given['--jobs']}") + return given + + +def mutate(given: dict[str, str]) -> list[str] | Failure: + """Each tool's report on the branch's changed lines; empty where none of them is in a language a tool covers.""" + checkout = top() + if isinstance(checkout, Failure): + return checkout + base = given.get("--base") or default_ref(cwd=checkout) + if isinstance(base, Failure): + return base + merge_base = git("merge-base", base, "HEAD", cwd=checkout) + if isinstance(merge_base, Failure): + return merge_base + changed = changed_by_tool(checkout, merge_base.strip()) + if isinstance(changed, Failure): + return changed + prefix, jobs = given.get("--prefix"), int(given.get("--jobs", CPUS_PER_SLOT)) + report: list[str] = [] + for tool, files in sorted(changed.items()): + if tool == "cargo-mutants": + lines = run_cargo_mutants(checkout, merge_base.strip(), sorted(files), prefix, jobs) + elif tool == "infection": + lines = run_infection(checkout, merge_base.strip(), prefix, jobs) + elif tool == "stryker": + lines = run_stryker(checkout, files, prefix, jobs) + else: + lines = run_mutmut(checkout, files, prefix, jobs) + if isinstance(lines, Failure): + return lines + report.extend(lines) + return report + + +def changed_by_tool(checkout: str, merge_base: str) -> dict[str, dict[str, frozenset[int]]] | Failure: + """Each tool's files that the working tree changes since merge_base, by path from the checkout's top, with their changed lines.""" + names = git("diff", "--name-only", "-z", "--diff-filter=d", "--no-renames", merge_base, cwd=checkout) + if isinstance(names, Failure): + return names + changed: dict[str, dict[str, frozenset[int]]] = {} + for path in filter(None, names.split("\0")): + if (tool := tool_for(path)) is None: + continue + diff = git("diff", "-U0", "--no-color", "--no-ext-diff", "--no-textconv", "--no-renames", merge_base, "--", path, cwd=checkout, errors="replace") + if isinstance(diff, Failure): + return diff + if lines := added_lines(diff): + changed.setdefault(tool, {})[path] = lines + return changed + + +def took(seconds: float) -> str: + whole = round(seconds) + return f"{whole // 60} min {whole % 60} s" if whole >= 60 else f"{whole} s" diff --git a/bin/mutate_changes/mutmut_mutants.py b/bin/mutate_changes/mutmut_mutants.py new file mode 100644 index 00000000..7880bcbb --- /dev/null +++ b/bin/mutate_changes/mutmut_mutants.py @@ -0,0 +1,81 @@ +"""Which of mutmut 3's mutants the changed lines hold, from their names, `mutmut results` and the copy of each file mutmut writes under mutants/. + +mutmut names a mutant `.__mutmut_`. The key is x_ for a function at a module's top, and xǁǁ for a method of a class there; mutmut mutates no other code. Its mutants/ copy of a file holds the original of each function it mutates, as `__mutmut_orig`, and each mutant as a copy of it.""" +import ast + + +def function_keys(source: str, lines: frozenset[int]) -> list[str]: + """mutmut's key of each function or method that one of the lines falls in, decorators included, in source order.""" + keys = [] + for node in ast.parse(source).body: + inner = node.body if isinstance(node, ast.ClassDef) else [node] + for function in inner: + if isinstance(function, (ast.AsyncFunctionDef, ast.FunctionDef)) and lines & set(range(first_line(function), (function.end_lineno or function.lineno) + 1)): + keys.append(f"xǁ{node.name}ǁ{function.name}" if isinstance(node, ast.ClassDef) else f"x_{function.name}") + return keys + + +def first_line(function: ast.AsyncFunctionDef | ast.FunctionDef) -> int: + return min([function.lineno, *(decorator.lineno for decorator in function.decorator_list)]) + + +def module(path: str) -> str: + """The module name mutmut gives a file, from its path relative to the folder mutmut runs in.""" + return path.removesuffix(".py").replace("/", ".").removeprefix("src.") + + +def mutant_globs(path: str, keys: list[str]) -> list[str]: + """The `mutmut run` patterns that pick each function's mutants. A pattern without __mutmut_ also matches the function, so mutmut runs only the tests that reach one of them.""" + return [f"{module(path)}.{key}*".replace(".__init__.", ".") for key in keys] + + +def named(name: str, paths: list[str]) -> tuple[str, str, int] | None: + """The path, of those given, holding the mutant mutmut names, with its function's key and its number; None where none holds it.""" + prefix, _, number = name.rpartition("__mutmut_") + _, _, key = prefix.rpartition(".") + found = [path for path in paths if f"{module(path)}.{key}".replace(".__init__.", ".") == prefix] + return (found[0], key, int(number)) if found and number.isdigit() else None + + +def statuses(results: str) -> dict[str, str]: + """Each mutant's status by name, from the ` : ` lines `mutmut results --all true` prints.""" + return {name: status for name, _, status in (line.strip().partition(": ") for line in results.splitlines()) if "__mutmut_" in name and status} + + +def mutated_line(source: str, copy: str, key: str, number: int) -> tuple[int, str] | None: + """The source line a mutant changes, with that line as the mutant has it; None where the mutants/ copy lacks the mutant, or holds an original unlike the source. + + A mutant keeps the original's lines, so the first line where they differ sits at the same offset from the source function's def.""" + try: + function = defined(ast.parse(source), key) + copied = {node.name: node for node in ast.walk(ast.parse(copy)) if isinstance(node, (ast.AsyncFunctionDef, ast.FunctionDef))} + except SyntaxError: + return None + original, mutant = copied.get(f"{key}__mutmut_orig"), copied.get(f"{key}__mutmut_{number}") + if function is None or original is None or mutant is None: + return None + expected = as_named(source.splitlines(), function, function.name) + kept, changed = (as_named(copy.splitlines(), node, function.name) for node in (original, mutant)) + if kept != expected: + return None + offset = next((index for index, (was, now) in enumerate(zip(kept, changed)) if was != now), None) + return None if offset is None else (function.lineno + offset, changed[offset].strip()) + + +def defined(tree: ast.Module, key: str) -> ast.AsyncFunctionDef | ast.FunctionDef | None: + """The function or method mutmut's key names, from a module's top or a class there.""" + if key.startswith("x_"): + scope, name = tree.body, key.removeprefix("x_") + elif len(parts := key.split("ǁ")) == 3: + _, owner, name = parts + classes = [node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == owner] + scope = classes[0].body if classes else [] + else: + return None + return next((node for node in scope if isinstance(node, (ast.AsyncFunctionDef, ast.FunctionDef)) and node.name == name), None) + + +def as_named(lines: list[str], function: ast.AsyncFunctionDef | ast.FunctionDef, name: str) -> list[str]: + """A function's lines from its def, decorators left out, with the def naming it name.""" + spanned = lines[function.lineno - 1:function.end_lineno] + return [spanned[0].replace(function.name, name, 1), *spanned[1:]] if spanned else [] diff --git a/bin/mutate_changes/mutmut_setup.py b/bin/mutate_changes/mutmut_setup.py new file mode 100644 index 00000000..3ac9bcab --- /dev/null +++ b/bin/mutate_changes/mutmut_setup.py @@ -0,0 +1,51 @@ +"""The setup.cfg that has mutmut mutate a branch's changed files, and the tests left out because they fail before any mutant.""" +import configparser +import io +import re + +from failure import Failure + +SUMMARY = re.compile(r"^=+ short test summary info =+$", re.MULTILINE) +# A test pytest's short summary reports as failed or erroring; its message, where one fits, follows " - ". +REPORTED = re.compile(r"^(?:ERROR|FAILED) (\S.*?)(?: - .*)?$", re.MULTILINE) + + +class CaseKept(configparser.ConfigParser): + """Keeps each key as written, since the project's other sections are copied over unchanged.""" + + def optionxform(self, optionstr: str) -> str: + return optionstr + + +def mutmut_config(existing: str, sources: list[str], left: list[str]) -> str | Failure: + """The setup.cfg that has mutmut mutate the sources, copy the rest of the folder, which their tests import, and pass pytest the options that leave tests out. + + existing is the project's own setup.cfg, empty where it has none. Its settings stay, though its comments go, and its [mutmut] pytest_add_cli_args come before the options that leave tests out.""" + config = CaseKept(interpolation=None) + try: + config.read_string(existing) + except configparser.Error as failure: + return Failure(f"could not read setup.cfg: {failure}") + if not config.has_section("mutmut"): + config.add_section("mutmut") + section = config["mutmut"] + section["source_paths"] = listed(sources) + section["also_copy"] = "." + if left: + section["pytest_add_cli_args"] = "\n".join(filter(None, [section.get("pytest_add_cli_args", ""), listed(left)])) + written = io.StringIO() + config.write(written) + return written.getvalue() + + +def listed(items: list[str]) -> str: + """Items as a [mutmut] list, one per line, each % doubled: mutmut 3 reads setup.cfg with configparser's default interpolation.""" + return "\n".join(item.replace("%", "%%") for item in items) + + +def left_out(output: str) -> list[str]: + """The pytest options that leave out each test a failed run's short summary reports: --deselect for a test, --ignore for a file that failed to collect.""" + summary = SUMMARY.split(output) + if len(summary) < 2: + return [] + return [f"--deselect={nodeid}" if "::" in nodeid else f"--ignore={nodeid}" for nodeid in REPORTED.findall(summary[-1])] diff --git a/bin/mutate_changes/mutmut_tool.py b/bin/mutate_changes/mutmut_tool.py new file mode 100644 index 00000000..333fc41f --- /dev/null +++ b/bin/mutate_changes/mutmut_tool.py @@ -0,0 +1,189 @@ +"""Mutate the changed lines of a branch's Python files with mutmut 3, in a scratch copy of the folder each file's tests run from. + +mutmut reads its settings from the folder it runs in, writes beside them, and names each module by its path from there. So each file is +mutated from the nearest folder above it, up to the checkout's top, that holds a setup.cfg or a pyproject.toml: the folder its tests +import it from.""" +import dataclasses +import os +import pathlib +import posixpath +import shutil +import tempfile +import tomllib + +from mutate_changes.tool_process import failed, Ran, run_tool +from failure import Failure +from mutate_changes.mutmut_mutants import function_keys, mutant_globs, mutated_line, named, statuses +from git_cli.git_repo import git +from mutate_changes.mutmut_setup import left_out, mutmut_config + +CAUGHT = frozenset({"caught by type check", "killed"}) +CONFIGS = ("pyproject.toml", "setup.cfg") +# The first run reports every test that fails before any mutant, not only the first, so that the next run can leave them all out. +LIST_FAILURES = ["--maxfail=0", "--continue-on-collection-errors"] +# What mutmut 3.8 prints where no mutant matches the patterns, where no test reaches a mutated function, and where a test fails unmutated. +NO_MUTANT = "Filtered for specific mutants, but nothing matches" +NO_TEST = "Stopping early, because" +STATS_FAILED = "failed to collect stats. runner returned" + + +@dataclasses.dataclass(frozen=True) +class Folder: + """The changed files mutmut mutates from one folder.""" + path: str # from the checkout's top; "." for the top itself + files: dict[str, frozenset[int]] # each file by its path from the folder, with its changed lines + sources: dict[str, str] + keys: dict[str, list[str]] # mutmut's key of each function or method a changed line falls in + setup: str # the folder's setup.cfg, empty where it has none + + @property + def label(self) -> str: + return "the checkout's top" if self.path == "." else self.path + + +def run_mutmut(top: str, changed: dict[str, frozenset[int]], prefix: str | None, jobs: int) -> list[str] | Failure: + """For each folder mutmut runs in, a line on what it mutated and left out, then a line per mutant on a changed line that its tests don't kill. + + changed maps each file, by its path from top, to its changed lines.""" + report: list[str] = [] + for path, files in sorted(by_folder(top, changed).items()): + lines = mutate_folder(top, path, files, prefix, jobs) + if isinstance(lines, Failure): + return lines + report.extend(lines) + return report + + +def by_folder(top: str, changed: dict[str, frozenset[int]]) -> dict[str, dict[str, frozenset[int]]]: + """The changed files by the folder mutmut runs in for each, "." for top, each file by its path from that folder.""" + grouped: dict[str, dict[str, frozenset[int]]] = {} + for path, lines in changed.items(): + folder = posixpath.dirname(path) + while folder and not any(os.path.isfile(os.path.join(top, folder, name)) for name in CONFIGS): + folder = posixpath.dirname(folder) + grouped.setdefault(folder or ".", {})[posixpath.relpath(path, folder or ".")] = lines + return grouped + + +def mutate_folder(top: str, path: str, files: dict[str, frozenset[int]], prefix: str | None, jobs: int) -> list[str] | Failure: + root = os.path.join(top, path) + folder = read_folder(root, path, files) + if isinstance(folder, Failure): + return folder + globs = [glob for file in sorted(files) for glob in mutant_globs(file, folder.keys[file])] + if not globs: + return [f"mutmut in {folder.label}: no changed line falls in a function or method, the only code mutmut mutates."] + listed = git("ls-files", "-z", "--cached", "--others", "--exclude-standard", cwd=root) + if isinstance(listed, Failure): + return listed + try: + scratch = tempfile.mkdtemp(prefix=".mutate-changes-", dir=top) + except OSError as failure: + return Failure(f"could not make a scratch folder in {top}: {failure}") + try: + copy = os.path.join(scratch, "copy") + copy_files(root, listed.split("\0"), copy) + return mutate_copy(copy, folder, ["mutmut", "run", "--max-children", str(jobs), *globs], prefix) + except OSError as failure: + return Failure(f"could not set mutmut up in {scratch}, a copy of {root}: {failure}") + finally: + shutil.rmtree(scratch, ignore_errors=True) + + +def read_folder(root: str, path: str, files: dict[str, frozenset[int]]) -> Folder | Failure: + setup = project_setup(root) + if isinstance(setup, Failure): + return setup + sources, keys = {}, {} + for file, lines in files.items(): + try: + sources[file] = pathlib.Path(root, file).read_text(encoding="utf-8") + keys[file] = function_keys(sources[file], lines) + except (OSError, SyntaxError, UnicodeDecodeError) as failure: + return Failure(f"could not read {os.path.join(root, file)}: {failure}") + return Folder(path, files, sources, keys, setup) + + +def project_setup(root: str) -> str | Failure: + """The folder's setup.cfg, empty where it has none. A Failure where its pyproject.toml sets mutmut up, since mutmut then reads no setup.cfg.""" + try: + pyproject = pathlib.Path(root, "pyproject.toml") + tool = tomllib.loads(pyproject.read_text(encoding="utf-8")).get("tool") if pyproject.is_file() else None + if isinstance(tool, dict) and "mutmut" in tool: + return Failure(f"{pyproject} sets mutmut up under [tool.mutmut]; move that to setup.cfg's [mutmut], the only place mutate-changes sets mutmut up through") + setup = pathlib.Path(root, "setup.cfg") + return setup.read_text(encoding="utf-8") if setup.is_file() else "" + except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as failure: + return Failure(f"could not read the mutmut settings in {root}: {failure}") + + +def copy_files(root: str, paths: list[str], copy: str) -> None: + """Copy the files git lists in root, tracked or not ignored, to copy, symlinks as symlinks. Leaves out mutmut's own mutants/ and the scratch folders of runs that never cleaned up.""" + for path in paths: + source, target = os.path.join(root, path), os.path.join(copy, path) + gone_or_submodule = not os.path.lexists(source) or os.path.isdir(source) and not os.path.islink(source) + if not path or path.split("/")[0] == "mutants" or path.startswith(".mutate-changes-") or gone_or_submodule: + continue + os.makedirs(os.path.dirname(target), exist_ok=True) + shutil.copy2(source, target, follow_symlinks=False) + + +def mutate_copy(copy: str, folder: Folder, argv: list[str], prefix: str | None) -> list[str] | Failure: + ran = run_until_stats_pass(copy, folder, argv, prefix) + if isinstance(ran, Failure): + return ran + output, left = ran + if NO_MUTANT in output: + return [f"mutmut in {folder.label}: none of the functions the changed lines fall in has a mutant."] + results = run_tool(["mutmut", "results", "--all", "true"], copy, prefix) + if isinstance(results, Failure) or results.code: + return results if isinstance(results, Failure) else failed(f"mutmut results in {folder.label}", results) + shown, counted = not_killed(folder, statuses(results.output), copy) + unreached = " No test reaches them." if NO_TEST in output else "" + left_files = sorted({option.partition("=")[2].partition("::")[0] for option in left}) + left_note = f" Left out tests in {', '.join(left_files)} that fail without a mutant." if left else "" + return [f"mutmut in {folder.label}: {counted} mutants on changed lines, {len(shown)} not killed.{unreached}{left_note}", *shown] + + +def run_until_stats_pass(copy: str, folder: Folder, argv: list[str], prefix: str | None) -> tuple[str, list[str]] | Failure: + """mutmut's output, and the options that left out each test that fails before any mutant: mutmut runs no mutant while one fails.""" + ran = configured_run(copy, folder, LIST_FAILURES, argv, prefix) + left: list[str] = [] + if isinstance(ran, Ran) and STATS_FAILED in ran.output and (left := left_out(ran.output)): + ran = configured_run(copy, folder, left, argv, prefix) + if isinstance(ran, Failure): + return ran + if ran.code and NO_MUTANT not in ran.output and NO_TEST not in ran.output: + return failed(f"mutmut run in {folder.label}", ran) + return ran.output, left + + +def configured_run(copy: str, folder: Folder, options: list[str], argv: list[str], prefix: str | None) -> Ran | Failure: + config = mutmut_config(folder.setup, sorted(folder.files), options) + if isinstance(config, Failure): + return Failure(f"{folder.label}: {config.reason}") + pathlib.Path(copy, "setup.cfg").write_text(config, encoding="utf-8") + return run_tool(argv, copy, prefix) + + +def not_killed(folder: Folder, found: dict[str, str], copy: str) -> tuple[list[str], int]: + """A line per mutant on a changed line that its tests don't kill, in order of path and line, and how many mutants the changed lines hold.""" + copies: dict[str, str] = {} + shown: list[tuple[str, int, str]] = [] + counted = 0 + for name, status in found.items(): + hit = named(name, list(folder.files)) + if hit is None or hit[1] not in folder.keys[hit[0]]: + continue + file, key, number = hit + if file not in copies: + mutated = pathlib.Path(copy, "mutants", file) + copies[file] = mutated.read_text(encoding="utf-8", errors="replace") if mutated.is_file() else "" + where = mutated_line(folder.sources[file], copies[file], key, number) + if where and where[0] not in folder.files[file]: + continue + counted += 1 + if status not in CAUGHT: + line, text = where or (0, f"{name}, on a line mutate-changes cannot find") + shown.append((posixpath.normpath(posixpath.join(folder.path, file)), line, f"{status}: {text}")) + return [f"{file}:{line}: {text}" if line else f"{file}: {text}" for file, line, text in sorted(shown)], counted diff --git a/bin/mutate_changes/test_mutate_changes.py b/bin/mutate_changes/test_mutate_changes.py new file mode 100644 index 00000000..c392510d --- /dev/null +++ b/bin/mutate_changes/test_mutate_changes.py @@ -0,0 +1,231 @@ +import configparser +import pathlib +import subprocess +import sys +import tempfile +import textwrap +import unittest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) + +from mutate_changes.changed_lines import added_lines, runs, tool_for +from failure import Failure +from mutate_changes.mutmut_mutants import function_keys, mutant_globs, mutated_line, named, statuses +from test_env import isolated_env +from mutate_changes.mutmut_setup import left_out, mutmut_config + +TOOL = pathlib.Path(__file__).resolve().parents[1] / "mutate-changes" +SOURCE = "def double(n):\n\treturn n * 2\n\n\nclass Pt:\n\tdef moved(self, by=1):\n\t\treturn self.x + by\n" +# The shape of the file mutmut 3 writes under mutants/: the original of each function and its mutants, with the original's lines. +COPY = """from inspect import signature as _mutmut_signature + + +def x_double__mutmut_orig(n): + return n * 2 +def x_double__mutmut_1(n): + return n / 2 + + +class Pt: + def xǁPtǁmoved__mutmut_orig(self, by=1): + return self.x + by + def xǁPtǁmoved__mutmut_1(self, by=2): + return self.x + by + def xǁPtǁmoved__mutmut_2(self, by=1): + return self.x - by +""" + + +def read_as_mutmut(text: str, key: str) -> list[str]: + """A [mutmut] list as mutmut 3 reads it from setup.cfg: with configparser's default interpolation, one item per line.""" + config = configparser.ConfigParser() + config.read_string(text) + return [item for item in config.get("mutmut", key).split("\n") if item] + + +class ChangedLines(unittest.TestCase): + def test_added_lines_are_the_new_side_of_each_hunk_and_a_deleting_hunk_adds_none(self): + diff = "diff --git a/x.py b/x.py\n--- a/x.py\n+++ b/x.py\n@@ -3,0 +4,2 @@ def f():\n+a\n+b\n@@ -10 +12 @@\n-c\n+d\n@@ -20,3 +23,0 @@\n-e\n-f\n-g\n" + self.assertEqual(added_lines(diff), {4, 5, 12}) + + def test_runs_join_consecutive_lines(self): + self.assertEqual(runs({10, 1, 2, 3, 7, 9}), [(1, 3), (7, 7), (9, 10)]) + + def test_the_tool_follows_the_extension_and_test_files_get_none(self): + expected = { + "bin/x/main.py": "mutmut", + "src/lib.rs": "cargo-mutants", + "src/Order.php": "infection", + "web/app.tsx": "stryker", + "web/app.mjs": "stryker", + "bin/x/test_main.py": None, + "bin/x/main_test.py": None, + "pkg/conftest.py": None, + "tests/helpers.py": None, + "web/app.test.ts": None, + "web/app.spec.js": None, + "web/__tests__/a.js": None, + "README.md": None, + } + for path, tool in expected.items(): + with self.subTest(path): + self.assertEqual(tool_for(path), tool) + + +class MutmutSetup(unittest.TestCase): + def test_the_config_mutates_the_sources_and_copies_the_whole_folder(self): + text = mutmut_config("", ["x/a.py", "b.py"], []) + self.assertEqual(read_as_mutmut(text, "source_paths"), ["x/a.py", "b.py"]) + self.assertEqual(read_as_mutmut(text, "also_copy"), ["."]) + + def test_the_config_keeps_the_projects_settings_and_appends_the_tests_left_out(self): + existing = "[tool:pytest]\naddopts = -q\n\n[mutmut]\npytest_add_cli_args = -k\n not Slow\ndebug = true\n" + text = mutmut_config(existing, ["a.py"], ["--deselect=t/test_a.py::T::test_b[50%]"]) + self.assertEqual(read_as_mutmut(text, "pytest_add_cli_args"), ["-k", "not Slow", "--deselect=t/test_a.py::T::test_b[50%]"]) + self.assertEqual(read_as_mutmut(text, "debug"), ["true"]) + config = configparser.ConfigParser() + config.read_string(text) + self.assertEqual(config.get("tool:pytest", "addopts"), "-q") + + def test_a_setup_cfg_that_cannot_be_read_fails(self): + self.assertIsInstance(mutmut_config("[mutmut]\na = 1\na = 2\n", ["a.py"], []), Failure) + + def test_left_out_names_each_test_the_summary_reports_failing(self): + output = textwrap.dedent("""\ + FAILED fake.py::t - this line is a test's own output, above the summary + ERROR root:x.py:3 a log line + =========================== short test summary info ============================ + FAILED repo_checks/test_repo_checks.py::Repo::test_a - AssertionError: x - y + ERROR larv_review/test_larv_review.py - FileNotFoundError: SKILL.md + FAILED a/test_b.py::T::test_c + 2 failed, 1 error in 3.10s + failed to collect stats. runner returned 1 + """) + self.assertEqual(left_out(output), ["--deselect=repo_checks/test_repo_checks.py::Repo::test_a", "--ignore=larv_review/test_larv_review.py", "--deselect=a/test_b.py::T::test_c"]) + + def test_left_out_is_empty_without_a_summary(self): + self.assertEqual(left_out("failed to collect stats. runner returned 1\n"), []) + + +class MutmutMutants(unittest.TestCase): + def test_function_keys_name_the_functions_and_methods_a_changed_line_falls_in(self): + source = SOURCE + "\n\n@cached\ndef later():\n\treturn 1\n" + self.assertEqual(function_keys(source, frozenset({2, 7})), ["x_double", "xǁPtǁmoved"]) + self.assertEqual(function_keys(source, frozenset({10})), ["x_later"]) + self.assertEqual(function_keys(source, frozenset({3, 5})), []) + + def test_mutant_globs_follow_mutmuts_module_names(self): + self.assertEqual(mutant_globs("pkg/__init__.py", ["x_f"]), ["pkg.x_f*"]) + self.assertEqual(mutant_globs("src/pkg/mod.py", ["x_f", "xǁPtǁmoved"]), ["pkg.mod.x_f*", "pkg.mod.xǁPtǁmoved*"]) + + def test_named_finds_a_mutants_file_function_and_number(self): + paths = ["pkg/__init__.py", "src/pkg/mod.py"] + self.assertEqual(named("pkg.x_f__mutmut_3", paths), ("pkg/__init__.py", "x_f", 3)) + self.assertEqual(named("pkg.mod.xǁPtǁmoved__mutmut_12", paths), ("src/pkg/mod.py", "xǁPtǁmoved", 12)) + self.assertIsNone(named("other.x_f__mutmut_1", paths)) + + def test_statuses_read_mutmut_results(self): + results = " pkg.x_f__mutmut_1: survived\n pkg.x_f__mutmut_2: no tests\n\n pkg.x_f__mutmut_3: killed\n" + self.assertEqual(statuses(results), {"pkg.x_f__mutmut_1": "survived", "pkg.x_f__mutmut_2": "no tests", "pkg.x_f__mutmut_3": "killed"}) + + def test_a_mutated_line_is_the_first_line_the_mutant_changes_at_its_place_in_the_source(self): + self.assertEqual(mutated_line(SOURCE, COPY, "x_double", 1), (2, "return n / 2")) + self.assertEqual(mutated_line(SOURCE, COPY, "xǁPtǁmoved", 1), (6, "def moved(self, by=2):")) + self.assertEqual(mutated_line(SOURCE, COPY, "xǁPtǁmoved", 2), (7, "return self.x - by")) + + def test_a_mutant_the_copy_lacks_or_a_copy_unlike_the_source_has_no_line(self): + self.assertIsNone(mutated_line(SOURCE, COPY, "x_double", 9)) + self.assertIsNone(mutated_line(SOURCE.replace("n * 2", "n * 2 + 1"), COPY, "x_double", 1)) + self.assertIsNone(mutated_line(SOURCE, "def broken(:\n", "x_double", 1)) + + +# Stands in for mutmut, run through --prefix as `python3 mutmut ...` in the scratch copy. It logs each call with +# the setup.cfg it finds there, then writes a mutants/ copy and prints results, both shaped like mutmut 3's. +FAKE_MUTMUT = r"""import pathlib, sys +log, mode, args = pathlib.Path(sys.argv[1]), sys.argv[2], sys.argv[3:] +config = pathlib.Path("setup.cfg").read_text() +with log.open("a") as written: + written.write(" ".join(args) + "\n" + config + "---\n") +if args[:2] == ["mutmut", "results"]: + print(" calc.x_triple__mutmut_1: survived\n calc.x_triple__mutmut_2: killed\n calc.x_double__mutmut_1: survived") +elif mode == "fails-once" and "--deselect=test_calc.py::Calc::test_slow" not in config: + print("==== short test summary info ====\nFAILED test_calc.py::Calc::test_slow - boom\nfailed to collect stats. runner returned 1") + sys.exit(1) +else: + pathlib.Path("mutants").mkdir(exist_ok=True) + functions = ["x_double__mutmut_orig(n):\n\treturn n * 2", "x_double__mutmut_1(n):\n\treturn n / 2", "x_triple__mutmut_orig(n):\n\treturn n * 3", "x_triple__mutmut_1(n):\n\treturn n / 3", "x_triple__mutmut_2(n):\n\treturn n * 4"] + pathlib.Path("mutants/calc.py").write_text("".join(f"def {function}\n" for function in functions)) +""" + + +class MutateChangesTool(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.temp = pathlib.Path(temp.name) + self.checkout = self.temp / "proj" + (self.checkout / "pkg").mkdir(parents=True) + (self.checkout / "pkg" / "setup.cfg").write_text("[mutmut]\n") + (self.checkout / "pkg" / "calc.py").write_text("def double(n):\n\treturn n * 2\n") + (self.checkout / "pkg" / "test_calc.py").write_text("") + self.git("init", "-q", "-b", "main") + self.commit() + self.git("checkout", "-q", "-b", "work") + (self.checkout / "pkg" / "calc.py").write_text("def double(n):\n\treturn n * 2\n\n\ndef triple(n):\n\treturn n * 3\n") + (self.checkout / "pkg" / "test_calc.py").write_text("# changed, but a test file is not mutated\n") + self.commit() + (self.temp / "fake_mutmut.py").write_text(FAKE_MUTMUT) + self.log = self.temp / "calls.log" + + def git(self, *args): + subprocess.run(["git", *args], check=True, cwd=self.checkout, env={**isolated_env(), "GIT_CONFIG_GLOBAL": "/dev/null"}) + + def commit(self): + self.git("add", ".") + self.git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "x") + + def run_tool(self, *args): + return subprocess.run([str(TOOL), *args], capture_output=True, cwd=self.checkout, encoding="utf-8", env=isolated_env()) + + def faked(self, mode): + return ["--prefix", f"python3 {self.temp / 'fake_mutmut.py'} {self.log} {mode}"] + + def test_prints_each_mutant_on_a_changed_line_that_its_tests_do_not_kill(self): + ran = self.run_tool(*self.faked("passes"), "--jobs", "2") + self.assertEqual(ran.returncode, 0, ran.stderr) + self.assertEqual(ran.stdout.splitlines()[:2], ["mutmut in pkg: 2 mutants on changed lines, 1 not killed.", "pkg/calc.py:6: survived: return n / 3"]) + calls = self.log.read_text() + self.assertIn("mutmut run --max-children 2 calc.x_triple*\n", calls) + self.assertIn("source_paths = calc.py\n", calls) + self.assertEqual((self.checkout / "pkg" / "setup.cfg").read_text(), "[mutmut]\n") + self.assertEqual(sorted(path.name for path in self.checkout.iterdir()), [".git", "pkg"]) + + def test_leaves_out_the_tests_that_fail_before_any_mutant_and_names_them(self): + ran = self.run_tool(*self.faked("fails-once")) + self.assertEqual(ran.returncode, 0, ran.stderr) + self.assertEqual(ran.stdout.splitlines()[:2], ["mutmut in pkg: 2 mutants on changed lines, 1 not killed. Left out tests in test_calc.py that fail without a mutant.", "pkg/calc.py:6: survived: return n / 3"]) + first, second = self.log.read_text().split("---\n")[:2] + self.assertIn("--maxfail=0", first) + self.assertNotIn("--maxfail=0", second) + + def test_a_tool_that_fails_fails_the_run_with_its_output(self): + ran = self.run_tool("--prefix", "echo broken; false") + self.assertEqual(ran.returncode, 1) + self.assertIn("mutmut run in pkg exited 1", ran.stderr) + self.assertIn("broken", ran.stderr) + + def test_a_branch_that_changes_no_mutated_language_says_so(self): + ran = self.run_tool(*self.faked("passes"), "--base", "HEAD") + self.assertEqual((ran.returncode, ran.stdout), (0, "No line the branch changes is in a language mutate-changes mutates.\n")) + self.assertFalse(self.log.exists()) + + def test_unknown_arguments_fail(self): + for args in (["--jobs", "0"], ["--jobs"], ["--colour"], ["--base", "a", "--base", "b"]): + with self.subTest(args): + ran = self.run_tool(*args) + self.assertEqual(ran.returncode, 1) + self.assertIn("usage: mutate-changes", ran.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/bin/mutate_changes/tool_process.py b/bin/mutate_changes/tool_process.py new file mode 100644 index 00000000..46ce0d13 --- /dev/null +++ b/bin/mutate_changes/tool_process.py @@ -0,0 +1,29 @@ +"""Start a mutation tool, through --prefix's shell text where the run gives one, and wait for it.""" +import dataclasses +import shlex +import subprocess + +from failure import Failure + + +@dataclasses.dataclass(frozen=True) +class Ran: + code: int + output: str # stdout and stderr, as the tool interleaved them + + +def run_tool(argv: list[str], cwd: str, prefix: str | None) -> Ran | Failure: + """Run a tool in cwd, reading each byte of its output that is not UTF-8 as U+FFFD. + + With a prefix, sh runs ` "$@"` with argv as its arguments, so the prefix keeps its own quoting and each of argv stays one word.""" + command = ["sh", "-c", f'{prefix} "$@"', "sh", *argv] if prefix else argv + try: + ran = subprocess.run(command, cwd=cwd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8", errors="replace") + except OSError as failure: + return Failure(f"{shlex.join(argv)} could not start: {failure.strerror}") + return Ran(ran.returncode, ran.stdout) + + +def failed(name: str, ran: Ran) -> Failure: + """A tool's exit code and the last 40 lines of its output, which hold its error.""" + return Failure("\n".join([f"{name} exited {ran.code}:", *ran.output.splitlines()[-40:]])) diff --git a/bin/rule_hook/heavy_command.py b/bin/rule_hook/heavy_command.py index 83ee29e8..2393c95f 100644 --- a/bin/rule_hook/heavy_command.py +++ b/bin/rule_hook/heavy_command.py @@ -3,7 +3,7 @@ from command_lines.docker_command import docker_subcommand from command_lines.shell_command import launched, tool_name # Each mutation tool, and the words that make its call a mutation run when one of them follows it within two words; where the set is empty, every call is one. -MUTATORS = {"cargo": {"mutants"}, "infection": set(), "mutmut": {"run"}, "stryker": {"run"}} +MUTATORS = {"cargo": {"mutants"}, "infection": set(), "mutate-changes": set(), "mutmut": {"run"}, "stryker": {"run"}} # Flags with which a tool only prints its help, version or list, so a call that carries one is light. Not -h, which docker run reads as the container's hostname. LIGHT_FLAGS = frozenset({"--help", "--list", "--version"}) BUILDS = (["build"], ["buildx", "build"], ["compose", "build"]) diff --git a/bin/rule_hook/rules.py b/bin/rule_hook/rules.py index 9d119eb0..d19d8b0f 100644 --- a/bin/rule_hook/rules.py +++ b/bin/rule_hook/rules.py @@ -77,7 +77,7 @@ RULES: dict[RuleName, Rule] = { ), "run-heavy": Rule( "a heavy command, which loads the machine for minutes, runs under ~/.claude/bin/run-heavy, so sessions running at once cannot starve the machine", - "a docker build, docker buildx build or docker compose build, and a run of cargo mutants, Infection, mutmut or Stryker, the mutation tools stability-reviewer names, whether direct, through a launcher such as uvx, or inside docker run, docker exec or docker compose run or exec. A gate script is not checked, since no program can tell it from a light one", + "a docker build, docker buildx build or docker compose build, and a run of mutate-changes, or of cargo mutants, Infection, mutmut or Stryker, the mutation tools it runs, whether direct, through a launcher such as uvx, or inside docker run, docker exec or docker compose run or exec. A gate script is not checked, since no program can tell it from a light one", ), "todo-rank-check": Rule( "while todo.md has changes, staged or not, a git commit waits until rank_todo.py --check passes on it: the checkout's own bin/rank_todo.py where the checkout is of this setup's repo, else ~/.claude/bin/rank_todo.py", diff --git a/bin/rule_hook/test_rule_hook.py b/bin/rule_hook/test_rule_hook.py index 6adb1e80..4acd7180 100644 --- a/bin/rule_hook/test_rule_hook.py +++ b/bin/rule_hook/test_rule_hook.py @@ -115,12 +115,12 @@ class CommandBreaches(unittest.TestCase): self.assertEqual(checked("git commit -m 'Answer the question in the README'"), []) def test_a_docker_build_or_a_mutation_run_outside_run_heavy_breaks_run_heavy(self): - for command in ["docker build -t x .", "cd /w && docker buildx build .", "docker compose build", "docker compose -f x.yml --verbose build app", "uvx --with pytest 'mutmut>=3,<4' run", "cargo mutants --in-diff d", "npx stryker run", "vendor/bin/infection --git-diff-lines", "docker compose run --rm app mutmut run", "docker run --rm -m 4g --memory-swap 4g -v /w:/w rust:1.90.0 cargo mutants", "docker compose exec app npx stryker run", "docker run -h box -m 4g --memory-swap 4g rust:1.90.0 cargo mutants", "nice -n 19 docker build .", "env DOCKER_BUILDKIT=1 docker build .", "time -p docker compose build", "docker --tlscacert ca.pem build .", "docker --tlscert c.pem --tlskey k.pem build .", "docker buildx --builder b build .", "docker container run -m 4g --memory-swap 4g rust:1.90.0 cargo mutants", "uvx mutmut --verbose run"]: + for command in ["docker build -t x .", "cd /w && docker buildx build .", "docker compose build", "docker compose -f x.yml --verbose build app", "uvx --with pytest 'mutmut>=3,<4' run", "cargo mutants --in-diff d", "npx stryker run", "vendor/bin/infection --git-diff-lines", "docker compose run --rm app mutmut run", "docker run --rm -m 4g --memory-swap 4g -v /w:/w rust:1.90.0 cargo mutants", "docker compose exec app npx stryker run", "docker run -h box -m 4g --memory-swap 4g rust:1.90.0 cargo mutants", "nice -n 19 docker build .", "env DOCKER_BUILDKIT=1 docker build .", "time -p docker compose build", "docker --tlscacert ca.pem build .", "docker --tlscert c.pem --tlskey k.pem build .", "docker buildx --builder b build .", "docker container run -m 4g --memory-swap 4g rust:1.90.0 cargo mutants", "uvx mutmut --verbose run", "~/.claude/bin/mutate-changes", "cd /w && mutate-changes --base main"]: with self.subTest(command): self.assertEqual(checked(command), ["run-heavy"]) def test_a_heavy_command_under_run_heavy_or_a_light_docker_or_mutmut_call_passes(self): - for command in ["~/.claude/bin/run-heavy docker build -t x .", "/home/me/.claude/bin/run-heavy uvx mutmut run", "nice -n 19 ~/.claude/bin/run-heavy docker build .", "env X=1 run-heavy docker compose run app mutmut run", "docker images", "docker compose run --rm app go vet", "docker compose run app make build", "uvx 'mutmut>=3,<4' results", "echo docker build", "npx prettier --check infection.json5", "echo mutmut run", "env", "docker build --help", "cargo mutants --list", "infection --version", "uvx mutmut a b run"]: + for command in ["~/.claude/bin/run-heavy docker build -t x .", "/home/me/.claude/bin/run-heavy uvx mutmut run", "nice -n 19 ~/.claude/bin/run-heavy docker build .", "env X=1 run-heavy docker compose run app mutmut run", "docker images", "docker compose run --rm app go vet", "docker compose run app make build", "uvx 'mutmut>=3,<4' results", "echo docker build", "npx prettier --check infection.json5", "echo mutmut run", "env", "docker build --help", "cargo mutants --list", "infection --version", "uvx mutmut a b run", "~/.claude/bin/run-heavy ~/.claude/bin/mutate-changes --jobs 2", "~/.claude/bin/mutate-changes --help"]: with self.subTest(command): self.assertEqual(checked(command), []) diff --git a/bin/setup.cfg b/bin/setup.cfg new file mode 100644 index 00000000..39a4820f --- /dev/null +++ b/bin/setup.cfg @@ -0,0 +1,2 @@ +# mutate-changes runs mutmut from this folder, since the tests here import their modules from it. +[mutmut] diff --git a/docs/decisions.md b/docs/decisions.md index 6ba13bc3..82539d84 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -284,11 +284,13 @@ A trial checked that it keeps goal 1.1. It replayed the 6 latest first rounds th 2026-10-29, lilleman: it runs only in the release review. `review: the architecture and principles passes run only in the release review…` says why. -## review: the stability pass mutation-tests the lines a chunk changes, in place of measuring test coverage +## review: mutate-changes mutation-tests the lines a chunk changes before review, in place of measuring test coverage -2026-10-13, lilleman through todo item 108, implemented by claude. Serves goals 1.1 and 1.4. Coverage counts the lines a test runs, so a test without a meaningful assertion still raises it. A surviving mutant shows a test that would let behaviour break. Every mutant of an untested changed line survives, so mutation testing of the changed lines also shows what a coverage drop on them would. Coverage would also catch a deleted test; the stability reviewer sees one in the diff without it. Measuring coverage beside it would add cost without raising quality, so todo item 69 was dropped and `principles/technical-principles.md` replaced "Test coverage should not decline" with the mutant rule. +2026-10-13, lilleman through todo item 108, implemented by claude; moved before review on 2026-11-11 through todo item 293. Serves goals 1.1 and 1.4. Coverage counts the lines a test runs, so a test without a meaningful assertion still raises it. A surviving mutant shows a test that would let behaviour break. Every mutant of an untested changed line survives, so mutation testing of the changed lines also shows what a coverage drop on them would. Coverage would also catch a deleted test; the stability reviewer sees one in the diff without it. Measuring coverage beside it would add cost without raising quality, so todo item 69 was dropped and `principles/technical-principles.md` replaced "Test coverage should not decline" with the mutant rule. -The reviewer limits each run to the changed lines, so its cost grows with the chunk. The tools are widely deployed, each with its own way to limit a run to a diff: cargo-mutants `--in-diff` (https://mutants.rs/in-diff.html), Infection `--git-diff-lines` (https://infection.github.io/guide/command-line-options.html), Stryker's mutation ranges (https://stryker-mutator.io/docs/stryker-js/configuration/). mutmut has no such option, so the reviewer mutates the changed files and keeps the mutants on changed lines. On 2026-10-13 mutmut 3 ran on this repo's `larv_review/rounds.py` in under two seconds, set up as README → "Test" says. +Since 2026-11-11 the worker runs `bin/mutate-changes` once before review, kills the survivors that change behaviour, and hands stability-reviewer the rest to judge. Until then, stability-reviewer ran the tools by hand in 133 of its 144 reports, and 52% of its findings were test gaps. One command finds the survivors the same way every time, and the reviewer spends its round on what remains. + +The run is limited to the changed lines, so its cost grows with the chunk. The tools are widely deployed, each with its own way to limit a run to a diff: cargo-mutants `--in-diff` (https://mutants.rs/in-diff.html), Infection `--git-diff-lines` (https://infection.github.io/guide/command-line-options.html), Stryker's mutation ranges (https://stryker-mutator.io/docs/stryker-js/configuration/). mutmut has no such option, so mutate-changes mutates only the functions and methods the changed lines fall in, through mutmut's mutant-name patterns, and counts the mutants on changed lines. The run has no time cap. Limiting it to the changed lines already bounds it by the chunk's size, and no published standard gives a cap to copy. Each run's wall time in the reviewer's report shows when the premise below fails. diff --git a/skills/larv-review/SKILL.md b/skills/larv-review/SKILL.md index c18e27e4..92ea0b35 100644 --- a/skills/larv-review/SKILL.md +++ b/skills/larv-review/SKILL.md @@ -28,6 +28,8 @@ Follow `~/.claude/CLAUDE.md` → "Git, commits, and PRs". Never commit to the de **Reading cost.** Run `~/.claude/bin/reading-cost` in the chunk's worktree, unless loop-claude's prompt shows that `chunk-preflight` already ran it there, and commit what it appends to `docs/reading-cost.md`; the commit re-enters no pass. Resolve a merge conflict in that file by putting the default branch's figures first and dropping a figure for a chunk the default branch already holds. On `Rising:`, file a plan item titled `**Lower the reading cost of the files a chunk reads most before its first commit.**`, scored per `~/.claude/rules/todo-items.md` with the tool's medians and files as its detail; where the plan already holds it, leave it as it is: `docs/reading-cost.md` holds the figures. The item names the next release, or the one after when this chunk ships the next release's last item. On exit 1, repair what its message names and run it again; when the message names no repair, file it as a `question` item, or refresh the question item an earlier run filed, and go on. Leave the plan as it is on any other output. The chunk that ships that item runs the tool with `--reset` instead, so the next rise is measured on chunks that finished after it. +**Mutation tests.** Where the branch changes code, run `~/.claude/bin/run-heavy ~/.claude/bin/mutate-changes --prefix ''` in the chunk's worktree, on a committed head and before the first stability round. It lists each mutant of a changed line that no test kills. Kill each one that changes behaviour with a new or stronger test, commit, and run it again. Hand stability-reviewer what the last run printed, with the commit it ran on. On exit 1, repair what its message names and run it again; when you cannot, hand the reviewer the message. + ## 3. Passes A pass is one kind of review, a row of the table, named by its first column. Its trigger says when it first runs, on the chunk's branch; "Re-entry" says when a fix reruns it. Its reviewer reports the findings. @@ -173,7 +175,7 @@ Launch override-reviewer with each line `rule-hook overrides` prints that no ear ### Stability and product-owner -Launch the reviewer with the change — stability-reviewer also with the places per "Comprehension check" → "Places", product-owner-reviewer with the external surface it touches. Each reports only a defect, a requirement of the plan or the goals the change breaks, a breach of a technical principle, or, for stability-reviewer, a place per "Comprehension check" → "Places"; drop any other finding. Fix along the reviewer's Fix guidance (delete > simplify > modify > add). The defect a persona hits is as often in wording or an upgrade path as in code, so a product-owner fix is as often a message or a doc. +Launch the reviewer with the change — stability-reviewer also with the places per "Comprehension check" → "Places" and the mutants per §2 → "Mutation tests", product-owner-reviewer with the external surface it touches. Each reports only a defect, a requirement of the plan or the goals the change breaks, a breach of a technical principle, or, for stability-reviewer, a place per "Comprehension check" → "Places"; drop any other finding. Fix along the reviewer's Fix guidance (delete > simplify > modify > add). The defect a persona hits is as often in wording or an upgrade path as in code, so a product-owner fix is as often a message or a doc. ### Prose diff --git a/todo.md b/todo.md index a43e4bb2..888b1ca8 100644 --- a/todo.md +++ b/todo.md @@ -23,7 +23,7 @@ | 287 | 0.9.0 | | | **Move the forge helpers `Forge`, `forge_of`, `runner`, `open_pull`, `merged_pull`, `pull_at` and `gitea` into one module that `chunk_preflight`, `chunk_pr`, `deploy_release` and `larv_review` all import from, so no tool imports another tool's internals for them.** | 3 | 1 | 3 | 5 | 5 | code health by 3 | 3.4 | | 284 | 0.9.0 | | owner | **Script each step that 0.7.0's chunks repeated by hand and a program can compute, filing one item per step, and file this audit again for the next release.** | 1 | 2 | 1 | 2 | 2 | 1.4 by 3, 1.5 by 3 | 2.6 | | 288 | 0.9.0 | | | **Point the two reading-cost decisions at code health and goal 1.4's counter-signal, in place of the goal 1.1 they cite.** | 1 | 2 | 1 | 2 | 1 | code health by 1, 1.4 by 1 | 2.6 | -| 270 | 0.9.0 | | | **Move each check a review pass repeats across chunks, other than the mutation tests item 293 moves, into the worker's pre-review gate, starting with the passes most often taken to a second round.** | 2 | 2 | 3 | 5 | 3 | 1.1 by 3, 1.4 by 3 | 2.5 | +| 270 | 0.9.0 | | | **Move each check a review pass repeats across chunks, other than the mutation tests `bin/mutate-changes` runs, into the worker's pre-review gate, starting with the passes most often taken to a second round.** | 2 | 2 | 3 | 5 | 3 | 1.1 by 3, 1.4 by 3 | 2.5 | | 77 | 0.9.0 | | | **Teach `bin/metric-bench` Go and TypeScript, rerun it once two projects' comprehension histories name places, and bring into review only a metric that tracks the panel.** | 4 | 2 | 1 | 5 | 6 | 1.1 by 7, 1.4 by 7 | 2.4 | | 273 | 0.9.0 | | decision | **Re-tune the E + I steps of `REVIEW_STEPS` for 6 to 8 once 10 finished chunks have scored an E + I of 6 or more, from how well E + I orders their review rounds and 🔴 findings.** | 1 | 2 | 2 | 3 | 2 | 1.1 by 2, 1.4 by 3 | 2.3 | | 268 | 0.9.0 | | | **End a run whose release is ready to cut with a notice on loop-claude's terminal, in place of an interrupt that waits for an answer.** | 2 | 2 | 3 | 5 | 3 | 1 by 5 | 2.2 | @@ -115,9 +115,9 @@ The chunk that ships this item files a copy of this audit in the release after t Both `docs/decisions.md` entries under `reading cost: a rise is …` and `reading cost: larv-review measures each finished chunk …` open "Serves goal 1.1: comprehension …". No goal says that any more: code health (`CLAUDE.md` → "Shared goal") holds comprehension, and since PR #455 a rising reading cost is goal 1.4's first counter-signal. Raised by the stability review of PR #455 as a 🟢 outside that change. -### 270. Move each check a review pass repeats across chunks, other than the mutation tests item 293 moves, into the worker's pre-review gate, starting with the passes most often taken to a second round. +### 270. Move each check a review pass repeats across chunks, other than the mutation tests `bin/mutate-changes` runs, into the worker's pre-review gate, starting with the passes most often taken to a second round. -The review folders under `~/.local/state/larv-review/`, review replays left out, hold a stability round 2 beside 137 of 211 round-1 reports (65%), product-owner 73 of 173 (42%) and prose 8 of 277 (3%), on 2026-11-08. The architecture and principles passes run up to five rounds in a release review, so they don't compare. From 2026-11-02 to 2026-11-10, 223 of stability's 428 round-1 findings (52%) were test gaps, and 31 of its 50 round 2s held one; item 293 moves the mutation tests that find most of them. Check which kinds of finding the other round 2s repeat, such as an unhandled error or a product-owner wording fix. Move only a check a program or the worker can run before review; leave judgement with the reviewer. +The review folders under `~/.local/state/larv-review/`, review replays left out, hold a stability round 2 beside 137 of 211 round-1 reports (65%), product-owner 73 of 173 (42%) and prose 8 of 277 (3%), on 2026-11-08. The architecture and principles passes run up to five rounds in a release review, so they don't compare. From 2026-11-02 to 2026-11-10, 223 of stability's 428 round-1 findings (52%) were test gaps, and 31 of its 50 round 2s held one; `bin/mutate-changes` now runs the mutation tests that find most of them before review. Check which kinds of finding the other round 2s repeat, such as an unhandled error or a product-owner wording fix. Move only a check a program or the worker can run before review; leave judgement with the reviewer. ### 77. Teach `bin/metric-bench` Go and TypeScript, rerun it once two projects' comprehension histories name places, and bring into review only a metric that tracks the panel. diff --git a/workflow-bench/setup.cfg b/workflow-bench/setup.cfg new file mode 100644 index 00000000..39a4820f --- /dev/null +++ b/workflow-bench/setup.cfg @@ -0,0 +1,2 @@ +# mutate-changes runs mutmut from this folder, since the tests here import their modules from it. +[mutmut]