Answer stability round 1: 0 🔴, 6 🟡 (0 misread); test each failure path the review added, and return a Failure from prepare-scoring's reads
CI / tests (pull_request) Successful in 2m59s

This commit is contained in:
2026-11-13 03:40:18 +01:00
parent 45ecf0798e
commit ade48c57b5
7 changed files with 86 additions and 39 deletions
+29 -20
View File
@@ -94,16 +94,16 @@ def file_globs(paths: list[str], forbidden_files: list[str]) -> str:
return listed(globs + sorted(f"`{path}`" for path in read if "/" not in path), "and") + (f", {TEST_FILES}" if tested & set(folders) else "")
def read(path: pathlib.Path) -> str:
"""path's text. Raises OSError where it cannot be read, and where it is not UTF-8, naming path as that error alone would not."""
def read(path: pathlib.Path) -> str | Failure:
"""path's text, or a Failure naming path where it cannot be read or decoded."""
try:
return path.read_text()
except UnicodeDecodeError as failure:
raise OSError(f"{path}: {failure}") from failure
except (OSError, UnicodeDecodeError) as failure:
return Failure(f"cannot read {path}: {failure}")
def preloaded_files(main: pathlib.Path, home: pathlib.Path) -> str:
"""The files Claude Code loads into a seat started in main before its first tool call: each CLAUDE.md, the files it imports, and each rule file without `paths:`, leaving out those `claudeMdExcludes` names."""
def preloaded_files(main: pathlib.Path, home: pathlib.Path) -> str | Failure:
"""The files Claude Code loads into a seat started in main before its first tool call: each CLAUDE.md, the files it imports, and each rule file without `paths:`, leaving out those `claudeMdExcludes` names. A Failure where one of them cannot be read."""
excludes = claude_md_excludes(main, home)
found = []
@@ -112,9 +112,17 @@ def preloaded_files(main: pathlib.Path, home: pathlib.Path) -> str:
for shown, claude_md, rules in (("~/.claude/", home / ".claude" / CLAUDE_MD, home / ".claude" / "rules"), ("", main / CLAUDE_MD, main / ".claude" / "rules")):
if claude_md.is_file() and kept(claude_md):
text = read(claude_md)
if isinstance(text, Failure):
return text
found.append(f"`{shown}{CLAUDE_MD}`")
found += [f"`{shown}{name}`, which it imports" for name in re.findall(r"^@(\S+)", read(claude_md), re.M)]
found += [f"`{shown}{rule.relative_to(claude_md.parent)}`" for rule in sorted(rules.glob("*.md")) if kept(rule) and not re.search(r"^paths:", read(rule), re.M)]
found += [f"`{shown}{name}`, which it imports" for name in re.findall(r"^@(\S+)", text, re.M)]
for rule in filter(kept, sorted(rules.glob("*.md"))):
text = read(rule)
if isinstance(text, Failure):
return text
if not re.search(r"^paths:", text, re.M):
found.append(f"`{shown}{rule.relative_to(claude_md.parent)}`")
return listed(found, "and") if found else "none"
@@ -136,10 +144,9 @@ def claude_md_excludes(main: pathlib.Path, home: pathlib.Path) -> list[str]:
def prompt_parts() -> dict[str, str] | Failure:
"""seat-prompts.md's opening, shared brief and each report as their fenced text, and each seat's background, keyed by heading or label."""
path = PANEL / "seat-prompts.md"
try:
text = read(path)
except OSError as failure:
return Failure(f"cannot read {path}: {failure}")
text = read(PANEL / "seat-prompts.md")
if isinstance(text, Failure):
return text
parts = {}
for title in ("Opening", "Shared brief", *{report for _, _, report in SEATS}):
lines = section(text, re.escape(title)) or []
@@ -173,10 +180,12 @@ def prepare_scoring(worktree: str, preloaded: str | None, home: pathlib.Path) ->
source = config.paths["working-rules"]
if not source.is_file() and (own := pathlib.Path(worktree) / "skills" / "comprehension-panel" / "seat-prompts.md").is_file():
source = own
try:
rules, skill = read(source) if source.is_file() else "", read(PANEL / "SKILL.md")
except OSError as failure:
return Failure(f"could not read what the seats' prompts quote: {failure}")
rules = read(source) if source.is_file() else ""
if isinstance(rules, Failure):
return rules
skill = read(PANEL / "SKILL.md")
if isinstance(skill, Failure):
return skill
values = project_values(rules, source.name)
if isinstance(values, Failure):
return values
@@ -194,10 +203,10 @@ def prepare_scoring(worktree: str, preloaded: str | None, home: pathlib.Path) ->
if isinstance(head, Failure):
return head
main = listing.splitlines()[0].removeprefix("worktree ")
try:
loaded = preloaded if preloaded is not None else preloaded_files(pathlib.Path(main), home)
except OSError as failure:
return Failure(f"could not read a file Claude Code loads into a seat: {failure}")
# Read before the scoring worktree is made, so a file that cannot be read leaves none behind.
loaded = preloaded if preloaded is not None else preloaded_files(pathlib.Path(main), home)
if isinstance(loaded, Failure):
return loaded
made = made_review_folder(worktree, home)
if isinstance(made, Failure):
return made
+15
View File
@@ -804,6 +804,7 @@ class LarvReviewTool(unittest.TestCase):
ran = self.run_tool("record-ruling", votes_file)
self.assertEqual(ran.returncode, 0, ran.stderr)
self.assertTrue(ran.stdout.startswith("Ruling: better. Overall votes as counted:"))
self.assertTrue(ran.stdout.endswith(f"\nAppended to {self.checkout / 'docs' / 'comprehension-history.md'}.\n"), ran.stdout)
again = self.run_tool("record-ruling", votes_file)
self.assertEqual((again.returncode, again.stdout), (1, ""))
self.assertIn("already holds an entry on", again.stderr)
@@ -818,6 +819,7 @@ class LarvReviewTool(unittest.TestCase):
self.plan()
ran = self.run_tool("record-scoring", self.seats_file(f"PR #2 at {self.start}\n" + seats), "1")
self.assertEqual(ran.returncode, 0, ran.stderr)
self.assertTrue(ran.stdout.startswith(f"Appended to {self.checkout / 'docs' / 'comprehension-history.md'}.\nMeans: "), ran.stdout)
self.assertIn("\nFloor: Locality 6.50, Self-sufficiency 6.50 under the floor of 7.0.\nNext: file one floor item, a refactor lifting Locality and Self-sufficiency, the lowest dimensions,", ran.stdout)
self.assertIn("not /", self.run_tool("record-scoring", "1", self.seats_file(f"PR #2 at {self.start}\n" + seats)).stderr)
text = (self.checkout / "docs" / "comprehension-history.md").read_text()
@@ -844,6 +846,19 @@ class LarvReviewTool(unittest.TestCase):
self.commit("Plan", {"README.md": readme, "todo.md": f"# todo\n\n## Scoring\n\n`Score = 100 * V / 2**S`\n\n`Bar = {bar}`\n\n`Next ID = 3`\n\n## Items\n\n" + header + rows})
self.git("push", "-q", "origin", "HEAD:main")
def test_says_no_ruling_and_no_means_where_the_history_cannot_be_written(self):
docs = self.checkout / "docs"
docs.mkdir(exist_ok=True)
(docs / "comprehension-history.md").unlink(missing_ok=True)
place = "\n\n- Locality, Overall: `bin/a.py:3` `f`: shorter\n\n"
votes_file = self.seats_file(f"PR #2 at {self.start}, against {self.start}\n\nMid A: {said('better', 'worse')}{place}Inherited architect: {said('better', 'worse')}{place}", "votes.md")
self.plan()
docs.chmod(0o500)
self.addCleanup(docs.chmod, 0o700)
for ran in (self.run_tool("record-ruling", votes_file), self.record_sevens()):
self.assertEqual((ran.returncode, ran.stdout), (1, ""), ran.stderr)
self.assertIn("could not write", ran.stderr)
def record_sevens(self):
"""Run record-scoring for item 1 on a scores file in which every seat scores 7 throughout."""
line = "Navigation 7, Locality 7, Shape 7, Self-sufficiency 7, Overall 7"
+17 -3
View File
@@ -185,9 +185,23 @@ class PrepareScoring(unittest.TestCase):
def test_names_a_file_that_is_not_utf_8(self):
(self.root / ".claude").mkdir()
(self.root / ".claude" / "CLAUDE.md").write_bytes(b"\xff")
with self.assertRaises(OSError) as raised:
preloaded_files(self.checkout, self.root)
self.assertEqual(str(raised.exception), f"{self.root}/.claude/CLAUDE.md: 'utf-8' codec can't decode byte 0xff in position 0: invalid start byte")
unread = preloaded_files(self.checkout, self.root)
assert isinstance(unread, Failure), unread
self.assertTrue(unread.reason.startswith(f"cannot read {self.root / '.claude' / CLAUDE_MD}: "), unread.reason)
def test_fails_before_it_makes_the_worktree_where_a_file_it_reads_does_not_decode(self):
values = "## Scoring run\n\n- Language: Python\n- Kind: a CLI\n- Domain: parsing\n- Domain docs: the spec\n- 3am question: the parse hangs\n"
self.commit(values)
(self.checkout / "CLAUDE.md").write_bytes(b"\xff")
unread = self.prepare()
assert isinstance(unread, Failure), unread
self.assertTrue(unread.reason.startswith(f"cannot read {self.checkout / CLAUDE_MD}: "), unread.reason)
self.assertEqual(len(self.git("worktree", "list").splitlines()), 1)
(self.checkout / "CLAUDE.md").unlink()
(self.checkout / "AGENTS.md").write_bytes(b"\xff")
unread = self.prepare()
assert isinstance(unread, Failure), unread
self.assertTrue(unread.reason.startswith(f"cannot read {self.checkout}/AGENTS.md: "), unread.reason)
def test_leaves_out_a_claude_md_the_project_settings_exclude(self):
(self.root / ".claude").mkdir()
+1 -1
View File
@@ -9,7 +9,7 @@ The worker, and the session /talk opens, run on a Route (model_routing/route.py)
A step-up weighs five rungs:
- the project's rung: the cheapest rung of the project's model, which each L counts up from;
- the project's rung: the cheapest rung of the project's model, which each L's rung is counted from, up or down;
- item_rung: the rung the chunk started on, which the routed item's L picked;
- the route's rung: where the worker runs now, at or above item_rung;
- floor_rung: the lowest rung the worker may run on next, which is the route's rung, raised where an item the worker took in has an L that picks a higher one;
-6
View File
@@ -51,12 +51,6 @@ def ranked_goals(goals_text: str, decisions: str) -> list[tuple[str, Mark]] | Fa
return [(CODE_HEALTH, "Achieve"), *goals]
def numbered_goals(goals_text: str) -> list[tuple[str, Mark]] | Failure:
"""The goals file's goals in reading order: each one's number, a sub-goal as 4.1, and its mark, Maintain, Achieve or None."""
body = goals_section(goals_text)
return body if isinstance(body, Failure) else numbered(body)
def goals_section(goals_text: str) -> str | Failure:
body = section(goals_text, "Goals")
return Failure(f"the goals file {body.reason}") if isinstance(body, Failure) else body
+15 -9
View File
@@ -99,6 +99,12 @@ Kept as written.
"""
def numbered_goals(goals_text):
"""The goals ranked_goals numbers in a goals file, before it adds code health."""
body = goals_file.goals_section(goals_text)
return body if isinstance(body, Failure) else goals_file.numbered(body)
class Rank(unittest.TestCase):
def ranked(self, todo=TODO, readme=README, decisions=DECISIONS):
return ranking.rank(todo, readme, decisions, "")
@@ -122,12 +128,12 @@ class Rank(unittest.TestCase):
def test_weighs_each_achieve_goal_97_percent_of_the_one_above_and_no_maintain_goal(self):
readme = "## Goals\n\n1. **Achieve:** A.\n 1. **Maintain:** B.\n 2. **Achieve:** C.\n2. **Achieve:** D.\n"
self.assertEqual({goal: round(weight, 4) for goal, weight in goals_file.goal_weights(goals_file.numbered_goals(readme), None).items()}, {"1": 0.3435, "1.2": 0.3332, "2": 0.3232})
self.assertEqual({goal: round(weight, 4) for goal, weight in goals_file.goal_weights(numbered_goals(readme), None).items()}, {"1": 0.3435, "1.2": 0.3332, "2": 0.3232})
def test_weighs_5_achieve_goals_from_0212_to_0188_and_15_from_0082_to_0053(self):
for count, first, last in ((5, 0.212, 0.188), (15, 0.082, 0.053)):
with self.subTest(count=count):
goals = goals_file.numbered_goals("## Goals\n\n" + "".join(f"{number}. **Achieve:** G.\n" for number in range(1, count + 1)))
goals = numbered_goals("## Goals\n\n" + "".join(f"{number}. **Achieve:** G.\n" for number in range(1, count + 1)))
weights = list(goals_file.goal_weights(goals, None).values())
self.assertEqual((round(weights[0], 3), round(weights[-1], 3), round(sum(weights), 6)), (first, last, 1))
@@ -158,15 +164,15 @@ class Rank(unittest.TestCase):
self.assertEqual(self.ranked(ranked.text.replace("| 2 | 0.1.0 | | |", "| 2 | 0.1.0 | | owner |")).under_bar, [])
def test_reads_goals_and_sub_goals_in_reading_order(self):
self.assertEqual(goals_file.numbered_goals(README), [("1", "Achieve"), ("2", "Maintain"), ("2.1", "Achieve")])
self.assertEqual(goals_file.numbered_goals(README.replace(" 1.", "\t1.")), [("1", "Achieve"), ("2", "Maintain"), ("2.1", "Achieve")])
self.assertEqual(goals_file.numbered_goals(README.replace("2. **Maintain:** Correct.", "```\n## Goals\n```\n2. Correct.")), [("1", "Achieve"), ("2", None), ("2.1", "Achieve")])
self.assertEqual(goals_file.numbered_goals(README.replace("2. **Maintain:**", "1. **Maintain:**")), Failure("the goals file's ## Goals numbers 1, 1, 1.1; number each goal by its position"))
self.assertEqual(numbered_goals(README), [("1", "Achieve"), ("2", "Maintain"), ("2.1", "Achieve")])
self.assertEqual(numbered_goals(README.replace(" 1.", "\t1.")), [("1", "Achieve"), ("2", "Maintain"), ("2.1", "Achieve")])
self.assertEqual(numbered_goals(README.replace("2. **Maintain:** Correct.", "```\n## Goals\n```\n2. Correct.")), [("1", "Achieve"), ("2", None), ("2.1", "Achieve")])
self.assertEqual(numbered_goals(README.replace("2. **Maintain:**", "1. **Maintain:**")), Failure("the goals file's ## Goals numbers 1, 1, 1.1; number each goal by its position"))
self.assertEqual(ranking.parse_weights(["| Goal | W |", "|---|---|"], ["1"]), Failure("the goal weights list no goal, but the goals file's goals are 1; by position they weigh:\n| Goal | W |\n|---|---|\n| 1 | 1.00 |"))
self.assertEqual(goals_file.numbered_goals("# P\n"), Failure("the goals file has no ## Goals heading"))
self.assertEqual(goals_file.numbered_goals("## Goals\n\nFast.\n"), Failure("the goals file's ## Goals has no numbered goal"))
self.assertEqual(numbered_goals("# P\n"), Failure("the goals file has no ## Goals heading"))
self.assertEqual(numbered_goals("## Goals\n\nFast.\n"), Failure("the goals file's ## Goals has no numbered goal"))
self.assertEqual(ranking.parse_weights(["| Goal | W |", "|---|---|", "| 1 | 0.5 |", "| 3 | 0.5 |"], ["1", "2"]), Failure("the goal weights list 1, 3, but the goals file's goals are 1, 2; by position they weigh:\n| Goal | W |\n|---|---|\n| 1 | 1.00 |\n| 2 | 0.50 |"))
self.assertEqual(goals_file.numbered_goals("## Goals\n\n1. **Achieve:** A.\n 1. **Achieve:** B.\n\t2. **Achieve:** C.\n"), [("1", "Achieve"), ("1.1", "Achieve"), ("1.2", "Achieve")])
self.assertEqual(numbered_goals("## Goals\n\n1. **Achieve:** A.\n 1. **Achieve:** B.\n\t2. **Achieve:** C.\n"), [("1", "Achieve"), ("1.1", "Achieve"), ("1.2", "Achieve")])
def test_refuses_a_goal_it_cannot_weigh(self):
cases = {
+9
View File
@@ -1022,6 +1022,15 @@ class CommittedFiles(unittest.TestCase):
git(self.repo, "add", "-A")
self.assertEqual(self.commit("git commit -m x").returncode, 2)
def test_a_breach_in_both_copies_of_a_file_is_one_breach_which_the_work_trees_marker_overrides(self):
self.write("Dockerfile", "FROM node:24\n")
git(self.repo, "add", "-A")
self.write("Dockerfile", "FROM node:24\nRUN true\n")
blocked = self.commit("git commit -m x")
self.assertEqual((blocked.returncode, blocked.stderr.count("Blocked by rule image-pin: ")), (2, 1))
self.write("Dockerfile", "FROM node:24\n# rule-override image-pin: the test needs a moving tag\n")
self.assertEqual(self.commit("git commit -m x").returncode, 0)
def test_a_breach_the_head_already_holds_is_left_alone(self):
self.write("Dockerfile", "FROM node:24\n")
self.write(".gitignore", "b\na\n")