diff --git a/.gitignore b/.gitignore index 8dad9699..731fef84 100644 --- a/.gitignore +++ b/.gitignore @@ -142,9 +142,6 @@ !reviewer-merge-trials/cases-*.json !rules/ !rules/*.md -!stability-skip-count/ -!stability-skip-count/count.py -!stability-skip-count/README.md !skills/ !skills/comprehension-panel/ !skills/comprehension-panel/*.md @@ -152,6 +149,9 @@ !skills/larv-review/*.md !skills/larv-review/standards/ !skills/larv-review/standards/*.md +!stability-skip-count/ +!stability-skip-count/README.md +!stability-skip-count/count.py !workflow-bench/ !workflow-bench/advisor-judge-prompt.md !workflow-bench/advisor-run-prompt.md diff --git a/README.md b/README.md index f248abfd..99797330 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Claude Code configuration -My personal `~/.claude`, version controlled: `CLAUDE.md` and `rules/` for instructions, the words under `CLAUDE.md` β†’ "Vocabulary"; `.claude/rules/` for the rules that load in this repo alone; `agents/` and `skills/`; `principles/` and `skills/larv-review/standards/` for what the reviewer agents hold projects to; `docs/` for decisions and the comprehension history; `settings.json` for the harness and `.claude/settings.json` for the harness inside this repo; `CHANGELOG.md` and `todo.md`; the tools in `bin/`; `workflow-bench/`, the scripts and prompts of the 2026-09-30 bench of larv-review and of the 2026-11-12 bench of the advisor tool; `reviewer-merge-trials/`, the 2026-11-09 trials of merging reviewers, with their method, results and scripts; `judge-field-trials/`, the 2026-11-11 trials of which facts code could read in place of loop-claude's judge, with their results and scripts; `stability-skip-count/`, the 2026-11-12 count of archived stability rounds that decided when a stability round 2 is skipped, with its script; and `presentation/`, a page that explains this setup to newcomers, with the script that prints it to PDF. +My personal `~/.claude`, version controlled: `CLAUDE.md` and `rules/` for instructions, the words under `CLAUDE.md` β†’ "Vocabulary"; `.claude/rules/` for the rules that load in this repo alone; `agents/` and `skills/`; `principles/` and `skills/larv-review/standards/` for what the reviewer agents hold projects to; `docs/` for decisions and the comprehension history; `settings.json` for the harness and `.claude/settings.json` for the harness inside this repo; `CHANGELOG.md` and `todo.md`; the tools in `bin/`; `workflow-bench/`, the scripts and prompts of the 2026-09-30 bench of larv-review and of the 2026-11-12 bench of the advisor tool; `reviewer-merge-trials/`, the 2026-11-09 trials of merging reviewers, with their method, results and scripts; `judge-field-trials/`, the 2026-11-11 trials of which facts code could read in place of loop-claude's judge, with their results and scripts; `stability-skip-count/`, the 2026-11-12 count of archived stability rounds, with its script, from which larv-review took when to skip a stability round 2; and `presentation/`, a page that explains this setup to newcomers, with the script that prints it to PDF. ## Goals diff --git a/docs/decisions.md b/docs/decisions.md index e7a1e56d..306036ee 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -245,7 +245,7 @@ Valid while no published standard sets either number, the reading-cost window st 2026-10-29, lilleman. Serves goals 1.4 and 1.5. A 🟑 fix no longer reruns its pass. One project's chunk on 2026-10-29, a README paragraph and plan rows, took 10 review rounds in 45 minutes, and three of its four second rounds answered only 🟑 findings, such as the wording of a question's options. Round 2 found 12 πŸ”΄ in about 200 runs since 2026-10-21, mostly in round 1's fixes of πŸ”΄ findings, so a πŸ”΄ fix still reruns its pass. A 🟑 fix that breaks something merges unreviewed by its pass; another pass whose row it meets for a πŸ”΄ fix in the same round still reviews it. Valid while πŸ”΄ findings in later chunks, fixing code a 🟑 fix wrote, stay rare. -2026-11-12, lilleman through todo item 308. Serves goal 1.4. Since the worker runs the mutation command before each stability round, a finding that asks only that a named mutant be killed or a named failing test pass has an answer a run can check. The archive from 2026-09-30 to 2026-11-12 holds 326 stability reports of round 2 or later, from 13 projects, and 81 of those reports open with BLOCK (`stability-skip-count/`). For each of the 326, the πŸ”΄ and 🟑 lines of the round before were matched on mutant and test-failure wording. In 5 rounds before, every such line matched; read by hand, 1 of the 5 asked only that a failing test pass, and the round 2 after each of the 5 approved. So none of the 81 BLOCKs fell in a round this skips. The skip needs every 🟑 to be of that kind too: in one chunk on 2026-11-12, round 1's only πŸ”΄ was a red suite, and round 2 blocked on a 🟑 whose answer was wrong, which no test or mutant showed. A fix may not edit the named failing test, since a test made to pass that way shows nothing. Fixes from such a round still rerun every other pass whose row they meet. Valid while code that such a skipped round's fixes wrote is rarely what a later chunk's πŸ”΄ findings fix. +2026-11-12, lilleman through todo item 308. Serves goal 1.4. Since the worker runs the mutation command before each stability round, a finding that asks only that a named mutant be killed or a named failing test pass has an answer a run can check. The archive from 2026-09-30 to 2026-11-12 holds 326 stability reports of round 2 or later, from 13 projects, and 81 of those reports open with BLOCK (`stability-skip-count/`). For each of the 326, the πŸ”΄ and 🟑 lines of the report before it were matched on mutant and test-failure wording. Every such line matched in 5 of those earlier reports. Read by hand, 1 of the 5 asked only that a failing test pass, and the round 2 after each of the 5 approved. So none of the 81 BLOCKs fell in a round this skips. The skip needs every 🟑 to be of that kind too: in one chunk on 2026-11-12, round 1's only πŸ”΄ was a red suite, and round 2 blocked on a 🟑 whose answer was wrong, which no test or mutant showed. A fix may not edit the named failing test, since a test made to pass that way shows nothing. Fixes from such a round still rerun every other pass whose row they meet. Valid while code that such a skipped round's fixes wrote is rarely what a later chunk's πŸ”΄ findings fix. ## review: a comment is prose in any file; a branch changing only Markdown no agent follows runs only the prose pass and, for README or docs an outside reader lands on, the product-owner pass; plan rows a chunk files from a finding, re-scores or moves get no review diff --git a/stability-skip-count/README.md b/stability-skip-count/README.md index 29e7226a..72067f61 100644 --- a/stability-skip-count/README.md +++ b/stability-skip-count/README.md @@ -1,21 +1,20 @@ # Stability skip count -This count asks one question: would skipping stability round 2, where round 1's findings had a mechanical answer, have skipped a round that blocked? +This count asks one question: would skipping stability round 2 have skipped a round that blocked, where round 1's findings each asked only that a named mutant be killed or a named failing test pass? It ran on 2026-11-12, over loop-claude's archive in `~/.claude/loop-claude-transcripts/`. `docs/decisions.md` β†’ `review: a fix for a πŸ”΄ reruns the pass that found it…` records what was decided from it, and `skills/larv-review/SKILL.md` Β§3 β†’ "Re-entry" β†’ "Mechanical answers" holds the rule. ## Terms - **Report:** a stability-reviewer run in the archive, with the round number its launch names. -- **Round before:** the previous report in the same archived cycle. +- **Report before:** the previous report in the same archived cycle. - **Mechanical:** a finding that asks only that a named mutant be killed or a named failing test pass. ## Method -`python3 -I stability-skip-count/count.py` prints the totals, then each report of round 2 or later whose round before reported a πŸ”΄ and whose every πŸ”΄ and 🟑 line matches mutant or test-failure wording. The match casts wide; reading each printed round by hand decides which are mechanical. The counts grow as the archive does. +`python3 -I stability-skip-count/count.py` prints the totals, then each report of round 2 or later whose report before held a πŸ”΄ and had every πŸ”΄ and 🟑 line match mutant or test-failure wording, with that report's opening line and the report before's findings. The match casts wide; reading each printed round by hand decides which are mechanical. It reads only reports begun before its `UNTIL`, so a rerun on this machine prints the same counts. It needs the archive, which only the machine that ran the cycles holds. ## Results - 326 reports of round 2 or later, from 13 projects; 81 open with BLOCK. -- 5 rounds before matched; the round 2 after each of the 5 approved, so no BLOCK falls in a round the skip could skip. +- 5 reports before matched, and the round 2 after each of the 5 approved, so no BLOCK falls in a round the skip could skip. - Read by hand, 1 of the 5 is mechanical: fejkdata, 2026-11-04, a pin test that fails. The other 4 matched on wording alone: a test's structure, a workflow condition, a failing type check and a missed timing target. -- A rule counting only the πŸ”΄s would have skipped PR #503's round 2 in this repo, on 2026-11-12. Round 1's only πŸ”΄ was a red suite, and round 2 blocked on a 🟑 whose answer was wrong, which no test or mutant showed. So the rule counts the 🟑s too. diff --git a/stability-skip-count/count.py b/stability-skip-count/count.py index 638b090e..95dcd906 100644 --- a/stability-skip-count/count.py +++ b/stability-skip-count/count.py @@ -3,12 +3,16 @@ import collections import json import pathlib import re +import sys ARCHIVE = pathlib.Path.home() / ".claude/loop-claude-transcripts" +# The count recorded in README.md and docs/decisions.md read the reports that began before this; later ones would change it. +UNTIL = "2026-11-12T10:50:00Z" # A line that opens a πŸ”΄ or 🟑 finding, after any list, number or heading marks. FINDING = re.compile(r"^\s*(?:[-*]\s*|\d+\.\s*|#+\s*)*(?:\*\*)?(πŸ”΄|🟑)") # Wording of a finding that names a surviving mutant or a failing test. It casts wide; the hand reading decides. MECHANICAL = re.compile(r"surviv|mutant|fail(s|ing|ed)? (test|on)|test (fails|failed|failing)|tests? (still )?fail|\bred\b|suite (is red|fails)", re.I) +# A report whose verdict line, at the start of a line, says BLOCK. A verdict stated mid-sentence isn't counted, so each printed round shows its opening line too. BLOCK = re.compile(r"^\W*(Verdict:?\s*)?\**BLOCK", re.M) @@ -48,16 +52,19 @@ def rounds() -> list[dict]: "at": entries[0].get("timestamp", "") if entries else "", "round": int(number.group(1)) if number else 0, "block": bool(BLOCK.search(last)), + "opens": last.strip().split("\n")[0][:120], "findings": [line.strip() for line in last.splitlines() if FINDING.match(line)], }) return found def main() -> None: - """Print the totals, then each round of 2 or later whose round before reported a πŸ”΄ and whose every πŸ”΄ and 🟑 line reads as mechanical.""" - every = rounds() + """Print the totals, then each report of round 2 or later where the report before it, in the same cycle, held a πŸ”΄ and every πŸ”΄ and 🟑 line of that report before reads as mechanical.""" + if not ARCHIVE.is_dir(): + sys.exit(f"No archive at {ARCHIVE}: the count reads loop-claude's transcripts on the machine that ran them.") + every = [run for run in rounds() if run["at"] < UNTIL] later = [run for run in every if run["round"] >= 2] - print(f"{len(later)} reports of round 2 or later, {sum(run['block'] for run in later)} opening with BLOCK, from {len({run['project'] for run in every})} projects") + print(f"{len(later)} reports of round 2 or later, {sum(run['block'] for run in later)} opening with BLOCK, from {len({run['project'] for run in later})} projects") by_cycle = collections.defaultdict(list) for run in every: by_cycle[(run["project"], run["cycle"])].append(run) @@ -66,7 +73,7 @@ def main() -> None: for before, run in zip(runs, runs[1:]): reds = [line for line in before["findings"] if "πŸ”΄" in line[:12]] if run["round"] >= 2 and reds and all(MECHANICAL.search(line) for line in before["findings"]): - print(f"\n{run['project']} {run['at'][:16]} round {run['round']}: {'BLOCK' if run['block'] else 'approved'}") + print(f"\n{run['project']} {run['at'][:16]} round {run['round']}, opening: {run['opens']}") for line in before["findings"]: print(f" before: {line[:300]}")