Answer stability round 1: 1 🔴, 1 🟡 (0 misread); refuse to count a report whose headings hold a mark in another form, and test the hook's report lookup
CI / tests (pull_request) Successful in 3m12s

This commit is contained in:
2026-11-17 08:30:22 +01:00
parent b43f2242c5
commit cb6dd6d3a7
5 changed files with 41 additions and 8 deletions
+2
View File
@@ -52,6 +52,8 @@ def findings_in(report: pathlib.Path) -> Findings | Failure:
except (OSError, UnicodeDecodeError) as failure:
return Failure(f"could not read {report}: {failure}")
findings = reported(text)
if isinstance(findings, Failure):
return Failure(f"{report} cannot be counted: {findings.reason}; count its findings by hand")
if text.lstrip().startswith("BLOCK") and findings.red + findings.yellow == 0:
return Failure(f"{report} says BLOCK but opens no finding with `### 🔴` or `### 🟡`; count its findings by hand")
return findings
+8 -2
View File
@@ -2,6 +2,7 @@
import dataclasses
import re
from failure import Failure
from review_rules.review_passes import PASSES
# The most rounds a pass takes per chunk, per `docs/decisions.md` → `review: a pass gets at most two rounds per chunk, …`. rule-hook's `round-cap` refuses a round commit that `past_cap` finds takes its pass past it, and `larv-review round-count` marks a pass that reached it.
@@ -18,6 +19,7 @@ ROUND = re.compile(r"^Answer (.+?) round ([0-9]+):", re.IGNORECASE)
FINDINGS = re.compile(r" ([0-9]+) 🔴, ([0-9]+) 🟡(?: \(([0-9]+) misread\))?(?:;|$)")
# A 🔴 or 🟡 finding's heading in a round's report, as skills/larv-review/review-findings.md → "Report" pins it: `### `, the severity mark, then ` [misread]` where the reviewer marked it so.
FINDING = re.compile(r"^### (🔴|🟡)( \[misread\])?", re.MULTILINE)
HEADING = re.compile(r"^#{1,6} .*$", re.MULTILINE)
@dataclasses.dataclass(frozen=True)
@@ -97,8 +99,12 @@ def round_findings(titles: list[str]) -> Findings | None:
return Findings(sum(found.red for found in counts), sum(found.yellow for found in counts), sum(misreads) if len(misreads) == len(counts) else None)
def reported(report: str) -> Findings:
"""The 🔴 and 🟡 findings a round's report gives in FINDING headings, and how many of them the reviewer marked `[misread]`."""
def reported(report: str) -> Findings | Failure:
"""The 🔴 and 🟡 findings a round's report gives in FINDING headings, and how many of them the reviewer marked `[misread]`; a Failure naming a heading that holds a 🔴 or 🟡 in another form, which a count would miss."""
for heading in HEADING.findall(report):
pinned = FINDING.match(heading)
if ("🔴" in heading or "🟡" in heading) and (not pinned or "[misread]" in heading and not pinned.group(2)):
return Failure(f"its heading `{heading}` holds a 🔴 or 🟡 but doesn't open `### 🔴` or `### 🟡`, followed by ` [misread]` where marked so")
marks = FINDING.findall(report)
return Findings(sum(mark == "🔴" for mark, _ in marks), sum(mark == "🟡" for mark, _ in marks), sum(bool(misread) for _, misread in marks))
+9 -1
View File
@@ -43,9 +43,17 @@ class RoundReport(unittest.TestCase):
def test_names_a_blocking_report_that_heads_no_finding_as_its_format_asks(self):
with tempfile.TemporaryDirectory() as temp:
(pathlib.Path(temp) / "prose-round-1.md").write_text("BLOCK\n\n## 🟡 Should fix\n\n### 1. `a.md:1`: worse\n")
(pathlib.Path(temp) / "prose-round-1.md").write_text("BLOCK\n\n- 🟡 `a.md:1`: worse\n")
unread = findings_in(standing_report(pathlib.Path(temp), "prose", 1))
self.assertIsInstance(unread, Failure)
self.assertEqual(unread.reason, f"{temp}/prose-round-1.md says BLOCK but opens no finding with `### 🔴` or `### 🟡`; count its findings by hand")
(pathlib.Path(temp) / "prose-round-1.md").write_text("APPROVE\n")
self.assertEqual(findings_in(standing_report(pathlib.Path(temp), "prose", 1)), Findings(0, 0, 0))
def test_names_a_report_whose_headings_it_cannot_count(self):
with tempfile.TemporaryDirectory() as temp:
(pathlib.Path(temp) / "prose-round-1.md").write_text("BLOCK\n\n### 1. 🟡 `a.md:1`: worse\n")
unread = findings_in(standing_report(pathlib.Path(temp), "prose", 1))
self.assertIsInstance(unread, Failure)
self.assertTrue(unread.reason.startswith(f"{temp}/prose-round-1.md cannot be counted: its heading `### 1. 🟡"), unread.reason)
self.assertTrue(unread.reason.endswith("; count its findings by hand"), unread.reason)
+13 -5
View File
@@ -4,6 +4,7 @@ import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from failure import Failure
from review_rules.round_titles import Findings, lacks_counts, past_cap, reported, round_counts, round_findings, round_prefix, rounds_past_first, title_counts
@@ -79,12 +80,8 @@ class RoundFindings(unittest.TestCase):
self.assertIsNone(round_findings(["Answer stability round 1: 1 🔴, 2 🟡 (4 misread); fix"]))
if __name__ == "__main__":
unittest.main()
class Reported(unittest.TestCase):
REPORT = "BLOCK\n\n### 🔴 [misread] `a.py:1` `f`: wrong\n\n### 🟡 `b.py:2`: worse\n\n### 🟡 [misread] `c.py:3`: worse\n\n### 🟢 `d.py:4`: nit\n\n### ❓ `e.py:5`: why\n\n- 🔴 `a.py:1` `f`: wrong, as the reply lists it\n#### 🟡 a sub-heading\nText naming 🔴 and [misread].\n"
REPORT = "BLOCK\n\n### 🔴 [misread] `a.py:1` `f`: wrong\n\n### 🟡 `b.py:2`: worse\n\n### 🟡 [misread] `c.py:3`: worse\n\n### 🟢 `d.py:4`: nit\n\n### ❓ `e.py:5`: why\n\n- 🔴 `a.py:1` `f`: wrong, as the reply lists it\nText naming 🔴 and [misread].\n"
def test_counts_the_findings_whose_heading_opens_with_their_mark(self):
self.assertEqual(reported(self.REPORT), Findings(1, 2, 2))
@@ -95,3 +92,14 @@ class Reported(unittest.TestCase):
def test_spells_the_title_prefix_a_round_commit_opens_with(self):
self.assertEqual(round_prefix("prose", 2, Findings(1, 2, 0)), "Answer prose round 2: 1 🔴, 2 🟡 (0 misread);")
self.assertEqual(title_counts(round_prefix("prose", 2, Findings(1, 2, 1))), Findings(1, 2, 1), "the prefix parses as the counts it spells")
def test_counts_none_of_a_report_whose_headings_hold_a_mark_in_another_form(self):
for heading in ["### 1. 🟡 `a.py:1`: worse", "## 🔴 Must fix", "#### 🟡 a sub-heading", "### 🟡 `a.py:1`: worse [misread]"]:
with self.subTest(heading):
unread = reported(f"BLOCK\n\n### 🔴 `b.py:2`: wrong\n\n{heading}\n")
self.assertIsInstance(unread, Failure)
self.assertEqual(unread.reason, f"its heading `{heading}` holds a 🔴 or 🟡 but doesn't open `### 🔴` or `### 🟡`, followed by ` [misread]` where marked so")
if __name__ == "__main__":
unittest.main()
+9
View File
@@ -1071,6 +1071,15 @@ class Hook(unittest.TestCase):
self.assertEqual(commit("Answer stability round 1: 1 🔴, 1 🟡 (1 misread); fix").returncode, 0)
self.assertEqual(commit("Answer prose round 1: 0 🔴, 3 🟡 (0 misread); fix").returncode, 0, "a round whose report is missing goes through")
self.assertEqual(commit("Answer stability round 1: 1 🔴, 2 🟡 (0 misread); fix # rule-override round-title-counts: the reviewer wrote its findings as a list").returncode, 0)
self.assertEqual(commit("Answer Stability round 1: 1 🔴, 2 🟡 (0 misread); fix").returncode, 2, "a capitalised pass reads its report")
elsewhere = tempfile.mkdtemp()
self.addCleanup(subprocess.run, ["rm", "-rf", elsewhere])
moved = hook({"cwd": elsewhere, "tool_input": {"command": f"git -C {self.repo} commit -m 'Answer stability round 1: 1 🔴, 2 🟡 (0 misread); fix'"}, "tool_name": "Bash"}, elsewhere, {"HOME": home})
self.assertEqual(moved.returncode, 2, "the report is the commit's checkout's, not the hook's working directory's")
outside = hook({"cwd": elsewhere, "tool_input": {"command": "git commit -m 'Answer stability round 1: 1 🔴, 2 🟡 (0 misread); fix'"}, "tool_name": "Bash"}, elsewhere, {"HOME": home})
self.assertEqual((outside.returncode, outside.stderr), (0, ""), "outside a checkout no report is read")
(folder / "stability-round-1-rerun.md").write_text("BLOCK\n\n### 🔴 `a.py:1`: wrong\n\n### 🟡 `b.py:2`: worse\n\n### 🟡 `c.py:3`: worse\n")
self.assertEqual(commit("Answer stability round 1: 1 🔴, 2 🟡 (0 misread); fix").returncode, 0, "the rerun's report stands in place of the round's")
def test_a_round_commit_past_round_2_is_refused_and_offers_no_override(self):
ran = self.round_commit("Answer stability round 3: 0 🔴, 1 🟡 (0 misread); fix # rule-override round-cap: the human said yes")