Answer security round 1: 0 🔴, 0 🟡 (0 misread); un-ignore stability-skip-count/, which the allowlist left out
CI / tests (pull_request) Successful in 2m39s

This commit is contained in:
2026-11-12 11:40:57 +01:00
parent a61afc567d
commit b2ddebce7a
3 changed files with 98 additions and 0 deletions
+21
View File
@@ -0,0 +1,21 @@
# Stability skip count
This count asks one question: would skipping stability round 2, where round 1's findings had a mechanical answer, have skipped a round that blocked?
It ran on 2026-11-12, over loop-claude's archive in `~/.claude/loop-claude-transcripts/`. `docs/decisions.md` → `review: a fix for a 🔴 reruns the pass that found it…` records what was decided from it, and `skills/larv-review/SKILL.md` §3 → "Re-entry" → "Mechanical answers" holds the rule.
## Terms
- **Report:** a stability-reviewer run in the archive, with the round number its launch names.
- **Round before:** the previous report in the same archived cycle.
- **Mechanical:** a finding that asks only that a named mutant be killed or a named failing test pass.
## Method
`python3 -I stability-skip-count/count.py` prints the totals, then each report of round 2 or later whose round before reported a 🔴 and whose every 🔴 and 🟡 line matches mutant or test-failure wording. The match casts wide; reading each printed round by hand decides which are mechanical. The counts grow as the archive does.
## Results
- 326 reports of round 2 or later, from 13 projects; 81 open with BLOCK.
- 5 rounds before matched; the round 2 after each of the 5 approved, so no BLOCK falls in a round the skip could skip.
- Read by hand, 1 of the 5 is mechanical: fejkdata, 2026-11-04, a pin test that fails. The other 4 matched on wording alone: a test's structure, a workflow condition, a failing type check and a missed timing target.
- A rule counting only the 🔴s would have skipped PR #503's round 2 in this repo, on 2026-11-12. Round 1's only 🔴 was a red suite, and round 2 blocked on a 🟡 whose answer was wrong, which no test or mutant showed. So the rule counts the 🟡s too.
+74
View File
@@ -0,0 +1,74 @@
"""Count the archived stability rounds of round 2 or later, their BLOCKs, and the rounds the skip in larv-review §3 → "Re-entry" → "Mechanical answers" could skip, printing each of those for reading by hand."""
import collections
import json
import pathlib
import re
ARCHIVE = pathlib.Path.home() / ".claude/loop-claude-transcripts"
# A line that opens a 🔴 or 🟡 finding, after any list, number or heading marks.
FINDING = re.compile(r"^\s*(?:[-*]\s*|\d+\.\s*|#+\s*)*(?:\*\*)?(🔴|🟡)")
# Wording of a finding that names a surviving mutant or a failing test. It casts wide; the hand reading decides.
MECHANICAL = re.compile(r"surviv|mutant|fail(s|ing|ed)? (test|on)|test (fails|failed|failing)|tests? (still )?fail|\bred\b|suite (is red|fails)", re.I)
BLOCK = re.compile(r"^\W*(Verdict:?\s*)?\**BLOCK", re.M)
def text_of(message: dict) -> str:
"""The text parts of a transcript message."""
content = message.get("content")
if isinstance(content, str):
return content
return "\n".join(part.get("text", "") for part in content or [] if isinstance(part, dict) and part.get("type") == "text")
def parsed(transcript: str) -> list[dict]:
"""A transcript's entries, one JSON object a line. str.splitlines would also split at the U+2028 a message may hold. A transcript of a cycle still running may end in a line half written, which is left out."""
lines = [line for line in transcript.split("\n") if line.strip()]
try:
return [json.loads(line) for line in lines]
except json.JSONDecodeError:
return [json.loads(line) for line in lines[:-1]]
def rounds() -> list[dict]:
"""Each archived stability-reviewer run: its project, its cycle folder, when it began, the round its launch names, whether its last text opens with BLOCK, and its finding lines."""
found = []
for meta_path in ARCHIVE.glob("*/*/session/*/subagents/*.meta.json"):
meta = json.loads(meta_path.read_text())
if meta.get("agentType") != "stability-reviewer":
continue
entries = parsed(meta_path.with_name(meta_path.name.removesuffix(".meta.json") + ".jsonl").read_text())
users = [entry for entry in entries if entry.get("type") == "user"]
prompt = text_of(users[0]["message"]) if users else ""
texts = [text_of(entry["message"]) for entry in entries if entry.get("type") == "assistant"]
last = next((text for text in reversed(texts) if text.strip()), "")
number = re.search(r"[Rr]ound (\d+)", meta.get("description", "") + " " + prompt[:600])
found.append({
"project": meta_path.parts[-6],
"cycle": meta_path.parts[-5],
"at": entries[0].get("timestamp", "") if entries else "",
"round": int(number.group(1)) if number else 0,
"block": bool(BLOCK.search(last)),
"findings": [line.strip() for line in last.splitlines() if FINDING.match(line)],
})
return found
def main() -> None:
"""Print the totals, then each round of 2 or later whose round before reported a 🔴 and whose every 🔴 and 🟡 line reads as mechanical."""
every = rounds()
later = [run for run in every if run["round"] >= 2]
print(f"{len(later)} reports of round 2 or later, {sum(run['block'] for run in later)} opening with BLOCK, from {len({run['project'] for run in every})} projects")
by_cycle = collections.defaultdict(list)
for run in every:
by_cycle[(run["project"], run["cycle"])].append(run)
for runs in by_cycle.values():
runs.sort(key=lambda run: run["at"])
for before, run in zip(runs, runs[1:]):
reds = [line for line in before["findings"] if "🔴" in line[:12]]
if run["round"] >= 2 and reds and all(MECHANICAL.search(line) for line in before["findings"]):
print(f"\n{run['project']} {run['at'][:16]} round {run['round']}: {'BLOCK' if run['block'] else 'approved'}")
for line in before["findings"]:
print(f" before: {line[:300]}")
main()