Answer stability round 1: 0 🔴, 8 🟡 (1 misread); count every named pass's rounds past the first again, name unreadable tags in larv-review, drop version_of, and test newest_tag, a failed index refresh, the judge cost a ruling archives, a ruling that cannot be archived, and a nested agents folder

This commit is contained in:
2026-11-04 23:48:18 +01:00
parent 9fea1d8286
commit 37a412b80e
9 changed files with 94 additions and 16 deletions
@@ -60,6 +60,15 @@ class DeployRelease(unittest.TestCase):
self.assertNotIn("Later.", ran.stdout)
self.assertNotIn("/model", ran.stdout)
def test_stops_with_gits_error_where_the_index_cannot_be_refreshed(self):
self.release("0.2.0", "- Second.\n", '{"theme": "dark"}\n')
(self.target / ".git" / "index").write_bytes(b"DIRC")
ran = self.deploy()
self.assertNotEqual(ran.returncode, 0)
self.assertIn("index", ran.stderr)
self.assertNotIn("uncommitted", ran.stderr)
self.assertEqual(self.git(self.target, "rev-parse", "HEAD"), self.git(self.source, "rev-parse", "v0.1.0^{commit}"))
def test_says_so_where_the_checkout_is_on_the_newest_tag_already(self):
ran = self.deploy()
self.assertEqual((ran.returncode, ran.stdout), (0, "Already on v0.1.0, the newest release.\n"))
+2
View File
@@ -34,6 +34,8 @@ def work_landed(release: Release, shipped: list[Item], landed_release: Release |
def landed_towards(found: list[Item], cut: LastCut) -> Release | None | Failure:
"""The next release on the default branch, once work has landed towards it: once one or more changes are unreleased, as `release_standing.unreleased_changes` counts them. None where there is no next release, or no change is unreleased. found is the default branch's plan."""
if isinstance(cut.tag, Failure):
return Failure(f"could not tell the next release: the default branch's tags cannot be read: {cut.tag.reason}")
upcoming = next_release(found, cut)
if isinstance(upcoming, Unknown):
return Failure("could not tell the next release: the default branch's todo.md history cannot be read")
+1
View File
@@ -337,6 +337,7 @@ class LandedTowards(unittest.TestCase):
plan = [item(1, (0, 2, 0))]
self.assertIsInstance(landed_towards(plan, LastCut("v0.1.0", Failure("unreadable"), Failure("count unreadable"), [])), Failure)
self.assertIsInstance(landed_towards([], LastCut("v0.1.0", "", 3, Failure("log unreadable"))), Failure)
self.assertEqual(landed_towards(plan, LastCut(Failure("fatal: bad object"), "", 3, [])), Failure("could not tell the next release: the default branch's tags cannot be read: fatal: bad object"))
class LarvReviewTool(unittest.TestCase):
+23
View File
@@ -345,6 +345,29 @@ class Cycles(unittest.TestCase):
f"{stamp} Judge reply rejected: the judge quoted 'never said', which is not in the transcript\n\nReply:\n> {misquoted.stdout}\n\nJudge: judge-model\n\n"
f"{stamp} Verdict: finished\n\nEvidence:\n> working\n\nJudge: judge-model\nWorker: {OFF_LADDER}\n"])
def test_archives_what_each_judge_run_cost_on_an_interrupt_a_quoted_answer_and_a_rejected_misquote(self):
for first, then, inputs in (
(judged("interrupt", cost=0.25), judged("finished"), ["Use B."]),
(judged("quoted", answer="Use A.", cost=0.25), judged("finished"), []),
(judged("finished", evidence="never said", cost=0.25), judged("finished"), []),
):
with self.subTest(first=first.stdout), tempfile.TemporaryDirectory() as archive:
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)] * 2, claude=[first, then], inputs=inputs)
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
run.run_cycle(dataclasses.replace(self.cycle(fake, records="From AGENTS.md:\nUse A."), project_archive=pathlib.Path(archive)))
kept = "".join(path.read_text() for path in pathlib.Path(archive).glob("*/*.outcomes.md"))
self.assertEqual(kept.count("Judge cost: 0.2500\n"), 1, kept)
def test_stops_with_exit_1_asking_nothing_where_an_interrupt_cannot_be_archived(self):
self.archive.parent.mkdir(parents=True)
self.archive.write_text("")
fake = FakeEffects(streams=[Worker(self.paths.claude_code_transcripts.parent)], claude=[judged("interrupt")], inputs=["Use B."])
with contextlib.redirect_stderr(io.StringIO()) as said:
stop = run.run_cycle(self.cycle(fake))
self.assertEqual((stop.code, stop.reason.split(":")[0]), (1, "could not archive the transcript"))
self.assertNotIn("Which one?", said.getvalue())
self.assertEqual(fake.made("claude_on_terminal"), [])
def test_stops_when_a_rejected_reply_cannot_be_archived(self):
self.archive.parent.mkdir(parents=True)
self.archive.write_text("")
+1 -6
View File
@@ -171,7 +171,7 @@ def cut_version(cut: LastCut) -> Release | None:
"""Which version is cut. It is the tag's, or None where the tag names no version. On a branch with no tag, it is the newest version under a dated CHANGELOG.md heading, else BEFORE_FIRST_RELEASE."""
if cut.tag is None:
return dated_cut(cut.tag, cut.changelog) or BEFORE_FIRST_RELEASE
return version_of(cut.tag)
return tagged(cut.tag) if isinstance(cut.tag, str) else None
def dated_cut(tag: str | None | Failure, changelog: str | Failure) -> Release | None:
@@ -181,11 +181,6 @@ def dated_cut(tag: str | None | Failure, changelog: str | Failure) -> Release |
return max((cut for heading in sections(changelog) if (cut := dated(heading))), default=None)
def version_of(tag: str | None | Failure) -> Release | None:
"""The version x.y.z named by tag, the newest v* tag; None where it names none."""
return tagged(tag) if isinstance(tag, str) else None
def has_unreleased(cut: LastCut) -> bool | Failure:
"""Whether work has landed since cut: one or more changes are unreleased."""
changes = unreleased_changes(cut)
+16 -9
View File
@@ -23,14 +23,21 @@ class Findings:
def round_counts(titles: list[str]) -> dict[str, int]:
"""Each pass's round count among the branch's `Answer <pass> round <n>:` commit titles: the highest <n>, or the number of such commits where that is more, so a repeated number still counts. A pass with none has 0. A title naming no pass counts for none; rule-hook's `round-cap` refuses its commit. Case is ignored."""
highest: dict[str, int] = dict.fromkeys(PASSES, 0)
commits: dict[str, int] = dict.fromkeys(PASSES, 0)
"""Each pass's round count among the branch's `Answer <pass> round <n>:` commit titles, as named_counts counts them. A pass with none has 0. A title naming no pass counts for none; rule-hook's `round-cap` refuses its commit. Case is ignored."""
counted = named_counts(titles)
return {name: counted.get(name, 0) for name in PASSES}
def named_counts(titles: list[str]) -> dict[str, int]:
"""The round count of each name the round titles give, in lower case, whether or not larv-review has a pass of that name: the highest <n>, or the number of such titles where that is more, so a repeated number still counts."""
highest: dict[str, int] = {}
commits: dict[str, int] = {}
for title in titles:
if (named := ROUND.match(title)) and (name := named.group(1).lower()) in PASSES:
highest[name] = max(highest[name], int(named.group(2)))
commits[name] += 1
return {name: max(highest[name], commits[name]) for name in PASSES}
if named := ROUND.match(title):
name = named.group(1).lower()
highest[name] = max(highest.get(name, 0), int(named.group(2)))
commits[name] = commits.get(name, 0) + 1
return {name: max(highest[name], commits[name]) for name in highest}
def round_pass(title: str) -> str | None:
@@ -40,8 +47,8 @@ def round_pass(title: str) -> str | None:
def rounds_past_first(titles: list[str]) -> int:
"""Each pass's rounds past its first among the commit titles, as round_counts counts them, summed."""
return sum(max(0, count - 1) for count in round_counts(titles).values())
"""The rounds past the first of each name the commit titles give, summed. It counts names that are no pass too, so an older title naming two passes, such as `Answer stability and prose round 2:`, still counts."""
return sum(max(0, count - 1) for count in named_counts(titles).values())
def lacks_counts(title: str) -> bool:
+38
View File
@@ -0,0 +1,38 @@
import pathlib
import subprocess
import sys
import tempfile
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
from release import newest_tag, tagged
from test_env import isolated_env
class NewestTag(unittest.TestCase):
def setUp(self):
self.repo = tempfile.mkdtemp()
self.addCleanup(subprocess.run, ["rm", "-rf", self.repo])
self.git("init", "-q", "-b", "main")
def git(self, *args: str) -> None:
subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], capture_output=True, check=True, cwd=self.repo, env=isolated_env())
def tag_commit(self, tag: str) -> None:
self.git("commit", "-q", "--allow-empty", "-m", tag)
self.git("tag", tag)
def test_takes_the_highest_version_sorting_a_release_candidate_below_its_release_and_only_tags_merged_into_the_branch_where_one_is_named(self):
self.assertIsNone(newest_tag(self.repo))
for tag in ("v0.9.0", "v0.10.0", "v1.0.0-rc.1"):
self.tag_commit(tag)
self.assertEqual(newest_tag(self.repo), "v1.0.0-rc.1")
self.git("checkout", "-q", "-b", "side")
self.tag_commit("v1.0.0")
self.git("checkout", "-q", "main")
self.assertEqual(newest_tag(self.repo), "v1.0.0")
self.assertEqual(newest_tag(self.repo, merged_into="main"), "v1.0.0-rc.1")
def test_names_a_release_only_for_a_v_tag_of_a_version(self):
self.assertEqual((tagged("v1.2.3"), tagged("1.2.3"), tagged("v1.2.3-rc.1"), tagged(None)), ((1, 2, 3), None, None, None))
+1 -1
View File
@@ -21,6 +21,6 @@ class ReviewPasses(unittest.TestCase):
for path in ("README.md", "docs/decisions.md", "todo.md", "workflow-bench/README.md"):
with self.subTest(path=path):
self.assertTrue(prose(path))
for path in ("CLAUDE.md", "AGENTS.md", "skills/larv-review/SKILL.md", "agents/prose-reviewer.md", "rules/todo-items.md", "principles/technical-principles.md", "workflow-bench/run-prompt.md", "workflow-bench/compare-prompt.md", "bin/rank_todo.py"):
for path in ("CLAUDE.md", "AGENTS.md", "skills/larv-review/SKILL.md", "agents/prose-reviewer.md", "rules/todo-items.md", "principles/technical-principles.md", "workflow-bench/run-prompt.md", "workflow-bench/compare-prompt.md", ".claude/agents/reviewer.md", ".claude/skills/x/SKILL.md", "bin/rank_todo.py"):
with self.subTest(path=path):
self.assertFalse(prose(path))
+3
View File
@@ -12,6 +12,9 @@ class Rounds(unittest.TestCase):
counts = round_counts(["Answer prose round 1: x", "Answer full scan round 2: y", "Answer full scan round 1: z", "Answer stability: no round", "Answer stabilty round 1: typo", "Answer security round 1: rename round 2: x", "Record the comprehension check: w"])
self.assertEqual((counts["prose"], counts["full scan"], counts["stability"], counts["architecture"], counts["security"]), (1, 2, 0, 0, 1))
def test_counts_the_rounds_past_the_first_of_a_title_naming_two_passes(self):
self.assertEqual(rounds_past_first(["Answer stability and prose round 2: x", "Answer stability and prose round 1: y"]), 1)
def test_counts_no_round_for_a_title_naming_no_pass(self):
self.assertNotIn("stabilty", round_counts(["Answer stabilty round 1: typo"]))
self.assertFalse(past_cap("Answer architectur round 9: 0 🔴, 0 🟡 (0 misread)", []))