Pass the judge a Brief, decide read_seats from its inputs and return a failed bench run as a line

This commit is contained in:
2026-11-08 22:51:27 +01:00
parent 2a0eabe8d9
commit 268200de23
12 changed files with 115 additions and 47 deletions
+1
View File
@@ -110,4 +110,5 @@
!workflow-bench/compare.py
!workflow-bench/review_git.py
!workflow-bench/run-prompt.md
!workflow-bench/test_bench.py
!workflow-bench/test_review_git.py
+1
View File
@@ -2,6 +2,7 @@
## Unreleased
- `workflow-bench/bench.py` now prints a failed run, such as a failed clone, as a `<run>: FAILED: …` line beside the other runs' results, in place of stopping at the first failure.
- A scoring run under the comprehension floor of 7.0 now leaves a plan item meant to reach it. `larv-review record-scoring` follows its `Floor:` line with a `Next:` line. It says to file one ambitious refactor aimed at the lowest dimension, or to refresh the one `todo.md` already holds. The item's tagline holds `comprehension floor`, so its own chunk runs a scoring run. It goes into the release after the chunk's, so the floor doesn't hold up any release (larv-review §3 → "Floor item"). An item already in `todo.md` whose tagline holds those words counts as the open one: drop or reword it if it isn't such a refactor.
- `loop-claude` now interrupts you less. A worker takes its own recommendation on a choice no rule reserves for you, and records it, and the judge answers an ask that carries a recommendation by quoting the rule that says so. You are still asked about a goal, a release's planned items and its cut, a change to the routing ladder, and whatever another rule asks your approval for; `CLAUDE.md` → "Working loop", "Ask, don't guess" lists them.
- This setup's own plan now ranks items by `Score = 100 * V / (2**S + 32)`, up from 8: F, a chunk's fixed cost, is 32, about one size-5 change, fitted to ten finished chunks' tokens, so small items rank lower against large ones. `docs/decisions.md` holds the fit.
+1 -1
View File
@@ -110,7 +110,7 @@ Install puts only `loop-claude` and `loop-inbox` on `PATH`; call the others by t
- `statusline.py` renders Claude Code's status line; `settings.json` → `statusLine` runs it.
- `subagent-rung [--plan] <model> <L>` prints the Agent tool's model and effort for a subagent's task, picked by its L from the project's model, and with `--plan` never below the project's rung; `subagent-rung --help` states the rule.
Test on the host with `uvx --python "$(command -v python3)" --with pytest-xdist==3.8.0 pytest==9.2.0 -n 4 bin`. It needs `uv` on `PATH`, which fetches pytest, pytest-xdist and mypy. pytest-xdist runs the tests in four workers, one heavy slot's CPUs (`docs/decisions.md` → `bin/: the tools support only the latest stable Python, …`). `python3 -m unittest discover -s bin -p 'test_*.py'` runs the same tests one at a time. Test the bench's git wrapper with `python3 -m unittest discover -s workflow-bench -p 'test_*.py'`.
Test on the host with `uvx --python "$(command -v python3)" --with pytest-xdist==3.8.0 pytest==9.2.0 -n 4 bin`. It needs `uv` on `PATH`, which fetches pytest, pytest-xdist and mypy. pytest-xdist runs the tests in four workers, one heavy slot's CPUs (`docs/decisions.md` → `bin/: the tools support only the latest stable Python, …`). `python3 -m unittest discover -s bin -p 'test_*.py'` runs the same tests one at a time. Test the bench's scripts with `python3 -m unittest discover -s workflow-bench -p 'test_*.py'`.
To mutation-test a changed module, copy `bin/` to a scratch directory (`cp -r bin "$scratch"`): mutmut writes files beside its config. In the copy, name the module and its tests in a `setup.cfg`, as below for `score_outcomes/results.py`. Leave out each test class that starts a tool's launcher through `subprocess`: a mutated module fails to load in a process mutmut did not start.
+2 -2
View File
@@ -29,12 +29,12 @@ import typing
from transcript_archive.location import archive_root, transcript_of
from transcript_archive.read import archived_outcomes
from rung_replay.command import arguments_or_exit, down_from, failed, limited_exit, projects_on
from loop_claude.cycle.judge import Brief, judge_once
from judge_replay.replay import Case, case_from, Kind, report, reproduced, sample, verdict_kind
from transcript_archive.project_key import checkout_of
from loop_claude.effects import Effects
from failure import Failure
from rung_replay.search import first_reply, replayed
from loop_claude.cycle.judge import judge_once
from claude_code.reply import Limited
from model_routing.ladder import lowest_worker_rung, project_rung, Rung
from loop_claude.project.premises import read_premises
@@ -99,5 +99,5 @@ def rule(case: Case, rung: Rung, claude_dir: pathlib.Path, effects: Effects) ->
if isinstance(entries, Failure):
return verdict_kind(entries)
# 0 for the route's mark, as before any step-up: the replay compares verdict kinds, and a mark changes only which escalation quotes count.
verdict: Verdict | Failure | Limited = first_reply(lambda: judge_once(case.root, case.task, premises, records, entries, 0, rung, effects))
verdict: Verdict | Failure | Limited = first_reply(lambda: judge_once(case.root, Brief(case.task, premises, records, entries, 0), rung, effects))
return verdict if isinstance(verdict, Limited) else verdict_kind(verdict)
+16 -9
View File
@@ -1,5 +1,6 @@
import pathlib
import time
import typing
from larv_review.agreement import agreement_report
from review_rules.comprehension_history import appended, DIMENSIONS, FLOOR, heading, Heading, HISTORY, latest_places, recorded, ruling_entry, scoring_entry, ScoringRun, vote_text
@@ -85,21 +86,27 @@ def scored_seats(given: SeatsFile) -> ScoringRun | Failure:
def read_seats(path: pathlib.Path, paired: bool) -> tuple[Heading, SeatsFile] | Failure:
"""The seats file and the heading its entry goes under. Fails when the file names a commit the checkout lacks, or when the history already holds an entry on what it reads; it runs before anything is printed."""
"""The seats file and the heading its entry goes under, as decided says. It runs before anything is printed."""
try:
given = seats_file(path.read_text(encoding="utf-8"))
text = path.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError) as failure:
return Failure(f"could not read {path}: {failure}")
if isinstance(given, Failure):
return given
named = heading(time.time(), given.read, paired)
if isinstance(named, Failure):
return named
if unknown := [commit for commit in (named.commit, named.against) if commit and not commit_exists(commit)]:
return Failure(f"the seats file names {', '.join(unknown)}, which is no commit in this checkout")
history = history_file()
if isinstance(history, Failure):
return history
return decided(text, time.time(), paired, commit_exists, history)
def decided(text: str, now: float, paired: bool, commit_known: typing.Callable[[str], bool], history: tuple[pathlib.Path, str]) -> tuple[Heading, SeatsFile] | Failure:
"""The seats file's text and the heading its entry goes under, stamped with now. Fails when the text names a commit commit_known denies, or when the history, as history_file returns it, already holds an entry on what the text reads."""
given = seats_file(text)
if isinstance(given, Failure):
return given
named = heading(now, given.read, paired)
if isinstance(named, Failure):
return named
if unknown := [commit for commit in (named.commit, named.against) if commit and not commit_known(commit)]:
return Failure(f"the seats file names {', '.join(unknown)}, which is no commit in this checkout")
if recorded(history[1], given.read):
return Failure(f"{history[0]} already holds an entry on {given.read}; record each ruling or run once")
return named, given
+28 -1
View File
@@ -15,13 +15,14 @@ from review_rules.comprehension_history import appended, entries, HistoryChange,
from larv_review.plan import checkout_plan, landed_towards, plan_reasons, scoring_run_due
from review_rules import comprehension_history
from larv_review.ruling import counted, ruling, split, surplus, votes
from larv_review.history_commands import decided
from failure import Failure
from larv_review.scoring import floor_step, means, under_floor
from test_env import isolated_env
from release_plan.todo_table import Item, items, NoItemsTable
from release_plan.release_standing import LastCut
from larv_review.ruling_base import measured_commit
from larv_review.seats import score_line, scores, seats_file
from larv_review.seats import score_line, scores, seats_file, SeatsFile
TOOL = pathlib.Path(__file__).resolve().parents[1] / "larv-review"
SCORING = "## 2026-10-08T12:00:00Z, PR #1 at aaaaaaa\n\n| Seat | Overall |\n|---|---|\n| Mean | 7.00 |\n"
@@ -312,6 +313,32 @@ class Agreement(unittest.TestCase):
self.assertEqual(agreement_report("\n".join([*pairs_of_runs(8, 2), *rulings])).split(":")[0], "Assumption fails")
class Decided(unittest.TestCase):
AT = 1_700_000_000.0
HISTORY = (pathlib.Path("history.md"), "")
def test_heads_the_entry_with_the_time_it_is_given(self):
decided_on = decided("PR #2 at abcdef1\n", self.AT, False, lambda commit: True, self.HISTORY)
self.assertEqual(decided_on, (comprehension_history.Heading("2023-11-14T22:13:20Z, PR #2 at abcdef1", "PR #2", "abcdef1", None), SeatsFile("PR #2 at abcdef1", [])))
def test_fails_on_a_commit_the_check_denies_and_names_only_the_denied(self):
asked = []
def known(commit):
asked.append(commit)
return commit == "abcdef1"
failed = decided("PR #2 at abcdef1, against 1234567\n", self.AT, True, known, self.HISTORY)
self.assertEqual(failed, Failure("the seats file names 1234567, which is no commit in this checkout"))
self.assertEqual(asked, ["abcdef1", "1234567"])
def test_fails_on_a_file_the_history_already_holds_an_entry_on(self):
held = scored("2023-11-14T22:13:20Z", "PR #2", "abcdef1", "7", "7")
failed = decided("PR #2 at abcdef1\n", self.AT, False, lambda commit: True, (pathlib.Path("history.md"), appended("", held)))
self.assertEqual(failed, Failure("history.md already holds an entry on PR #2 at abcdef1; record each ruling or run once"))
def test_fails_on_a_seats_file_that_does_not_parse_before_asking_for_a_commit(self):
self.assertEqual(decided("", self.AT, False, lambda commit: self.fail("asked"), self.HISTORY), seats_file(""))
class Base(unittest.TestCase):
def test_takes_the_newest_change_adding_an_entry(self):
older = HistoryChange("old", "", SCORING)
+17 -7
View File
@@ -108,6 +108,15 @@ class Grounds(typing.NamedTuple):
quotable: str
class Brief(typing.NamedTuple):
"""What the judge reads: the worker's task, the premises and records it is judged by, and the transcript's entries with the route's escalation mark, as loop_claude/escalation_mark.py says."""
task: str
premises: str
records: str
entries: list[dict]
escalated_at: int
def verdict_from(output: object) -> Verdict | Failure:
if not isinstance(output, dict):
return Failure("the judge returned no reply")
@@ -150,15 +159,15 @@ def taken_in_from(output: dict) -> TakenIn:
)
def judge(root: str, task: str, premises: str, records: str, entries: list[dict], escalated_at: int, rung: Rung, effects: Effects, keep: typing.Callable[[Rejected], Stop | None]) -> Verdict | Stop:
return retried("judge", lambda: judge_once(root, task, premises, records, entries, escalated_at, rung, effects), effects, lambda failure: keep(failure) if isinstance(failure, Rejected) else None)
def judge(root: str, brief: Brief, rung: Rung, effects: Effects, keep: typing.Callable[[Rejected], Stop | None]) -> Verdict | Stop:
return retried("judge", lambda: judge_once(root, brief, rung, effects), effects, lambda failure: keep(failure) if isinstance(failure, Rejected) else None)
def judge_once(root: str, task: str, premises: str, records: str, entries: list[dict], escalated_at: int, rung: Rung, effects: Effects) -> Verdict | Failure | Limited:
"""One judge run on the transcript's entries, as judge retries it; judge-replay replays it. escalated_at is the route's mark, as loop_claude/escalation_mark.py says."""
prompt, grounds = judge_prompt(task, premises, records, entries, escalated_at)
def judge_once(root: str, brief: Brief, rung: Rung, effects: Effects) -> Verdict | Failure | Limited:
"""One judge run on the brief, as judge retries it; judge-replay replays it."""
prompt, grounds = judge_prompt(brief)
verdict = run_judge(root, judge_args(rung), prompt, grounds, effects)
return verdict if isinstance(verdict, Failure | Limited) else escalation_mark.stamped(verdict, len(entries))
return verdict if isinstance(verdict, Failure | Limited) else escalation_mark.stamped(verdict, len(brief.entries))
def judge_args(rung: Rung) -> list[str]:
@@ -211,8 +220,9 @@ def quotes_checked(verdict: Verdict, grounds: Grounds) -> Verdict | Failure:
return escalation_mark.checked(verdict, since=grounds.since_step, shown=grounds.shown)
def judge_prompt(task: str, premises: str, records: str, entries: list[dict], escalated_at: int) -> tuple[str, Grounds]:
def judge_prompt(brief: Brief) -> tuple[str, Grounds]:
"""The prompt, and what the judge's quotes are checked against."""
task, premises, records, entries, escalated_at = brief
transcript = escalation_mark.transcript(entries, escalated_at, typed_answer)
head = (
f"From README.md:\n{premises}\n\n"
+2 -2
View File
@@ -5,12 +5,12 @@ import typing
from loop_claude.cycle.states import Active, Archived, Closing, Ended, Event, HumanTalked, HumanTyped, Interrupting, Judging, QUOTED_ANSWERS_PER_CYCLE, Ruled, State, to_the_human, transition, TypedAnswer, WorkerRan, Working
from transcript_archive.write import archive
from loop_claude.cycle.worker import AUTO_COMPACT_WINDOW, COMPACT_WINDOW, run_worker, worker_args
from loop_claude.cycle.judge import Brief, judge, Rejected
from claude_code.session import delete_session, Session, session_open, transcript_file
from loop_claude.effects import Effects, say
from failure import Failure
from loop_claude.verdicts import Finished, Interrupt, QuotedAnswer, Unfinished, Verdict
from loop_claude.cycle.interrupt import interrupt, Talked, WorkerSession
from loop_claude.cycle.judge import judge, Rejected
from loop_claude.routing import judge_rung, next_route, step_up, StepUp
from transcript_archive import outcomes
from loop_claude.cycle import prompts
@@ -143,7 +143,7 @@ def rule(cycle: Cycle, transcript_path: str, quoted_answers: int) -> Ruled | Sto
entries = read_transcript(transcript_path)
if isinstance(entries, Failure):
return Stop(1, entries.reason)
verdict = judge(cycle.root, cycle.task, cycle.premises, cycle.records, entries, cycle.route.escalated_at, judge_of(cycle), cycle.effects, lambda rejected: keep_rejection(cycle, transcript_path, rejected))
verdict = judge(cycle.root, Brief(cycle.task, cycle.premises, cycle.records, entries, cycle.route.escalated_at), judge_of(cycle), cycle.effects, lambda rejected: keep_rejection(cycle, transcript_path, rejected))
if isinstance(verdict, Stop):
return verdict
if not isinstance(verdict, Interrupt | QuotedAnswer):
+9 -9
View File
@@ -1609,7 +1609,7 @@ class Records(unittest.TestCase):
class JudgePrompt(unittest.TestCase):
def test_shows_the_transcripts_tail_and_returns_what_it_showed(self):
entries = [assistant({"type": "text", "text": "old " + "x" * judge.JUDGE_PROMPT_CHARS}), assistant({"type": "text", "text": "new"})]
prompt, grounds = judge.judge_prompt("Do it.", "## Goals", "", entries, 0)
prompt, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", entries, 0))
self.assertTrue(grounds.shown.endswith("\n\nnew"))
self.assertNotIn("old", grounds.shown)
self.assertIn(grounds.shown, prompt)
@@ -1618,7 +1618,7 @@ class JudgePrompt(unittest.TestCase):
self.assertEqual(grounds.quotable, "## Goals\n\n")
def test_the_judge_prompt_and_schema_call_the_worker_the_worker(self):
prompt, _ = judge.judge_prompt("Do it.", "## Goals", "", [assistant({"type": "text", "text": "done"})], 1)
prompt, _ = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", [assistant({"type": "text", "text": "done"})], 1))
self.assertNotRegex(prompt + json.dumps(judge.JUDGE_SCHEMA), r"\b[Aa]gent")
def test_the_judge_prompt_names_every_rule_the_judge_reads_from_the_human(self):
@@ -1626,24 +1626,24 @@ class JudgePrompt(unittest.TestCase):
def test_shows_the_judge_where_the_worker_last_stepped_up_and_checks_an_escalation_only_after_it(self):
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "round 3: 🔴"})]
prompt, grounds = judge.judge_prompt("Do it.", "## Goals", "", entries, 1)
prompt, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", entries, 1))
self.assertRegex(prompt, "round 2: 🔴\n\n\\[step\\] [^\n]+\n\nround 3: 🔴\n\n")
self.assertEqual(grounds.since_step, "round 3: 🔴")
prompt, grounds = judge.judge_prompt("Do it.", "## Goals", "", entries, 0)
prompt, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", entries, 0))
self.assertNotIn("stepped the worker up", prompt)
self.assertEqual(grounds.shown, "round 2: 🔴\n\nround 3: 🔴")
self.assertEqual(grounds.since_step, "round 2: 🔴\n\nround 3: 🔴")
def test_drops_an_escalation_quote_from_after_the_step_up_that_the_tail_it_shows_cuts_off(self):
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "old " + "x" * judge.JUDGE_PROMPT_CHARS}), assistant({"type": "text", "text": "new"})]
_, grounds = judge.judge_prompt("Do it.", "## Goals", "", entries, 1)
_, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", entries, 1))
self.assertEqual(judge.quotes_checked(Unfinished("e", "old x", entries_read=3), grounds), Unfinished("e", entries_read=3))
self.assertEqual(judge.quotes_checked(Unfinished("e", "new", entries_read=3), grounds), Unfinished("e", "new", entries_read=3))
def test_shows_a_typed_answer_before_the_step_up_as_the_humans(self):
typed = prompts.typed("Which one?", "Use B.", Route(Rung("worker-model")), None, False)
entries = [{**human(typed), "promptSource": "sdk"}, assistant({"type": "text", "text": "round 3: 🔴"})]
_, grounds = judge.judge_prompt("Do it.", "## Goals", "", entries, 1)
_, grounds = judge.judge_prompt(judge.Brief("Do it.", "## Goals", "", entries, 1))
self.assertIn("\n\n[human] Use B.\n\n[step] ", grounds.shown)
@@ -1748,7 +1748,7 @@ class Verdict(unittest.TestCase):
def test_drops_an_escalation_quote_found_only_in_the_transcript_entries_before_the_workers_last_step_up(self):
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "round 3: 🔴"}), assistant({"type": "text", "text": "working"})]
def judged_on(mark, escalation):
return judge.judge_once("/r", "Do it.", "## Goals", "", entries, mark, Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation=escalation)]).effects())
return judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", entries, mark), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation=escalation)]).effects())
self.assertEqual(judged_on(0, "round 2: 🔴"), Unfinished("working", "round 2: 🔴", entries_read=3))
self.assertEqual(judged_on(1, "round 2: 🔴"), Unfinished("working", entries_read=3))
self.assertEqual(judged_on(1, "round 3: 🔴"), Unfinished("working", "round 3: 🔴", entries_read=3))
@@ -1757,12 +1757,12 @@ class Verdict(unittest.TestCase):
def test_drops_an_escalation_quote_that_only_loop_claudes_prompts_or_the_humans_lines_repeat_after_the_step_up(self):
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), {**human("as the work remaining: round 2: 🔴"), "promptSource": "sdk"}, human("round 2: 🔴 again"), assistant({"type": "text", "text": "working"})]
verdict = judge.judge_once("/r", "Do it.", "## Goals", "", entries, 1, Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation="round 2: 🔴")]).effects())
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", entries, 1), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", escalation="round 2: 🔴")]).effects())
self.assertEqual(verdict, Unfinished("working", entries_read=4))
def test_takes_evidence_from_before_the_workers_last_step_up(self):
entries = [assistant({"type": "text", "text": "round 2: 🔴"}), assistant({"type": "text", "text": "working"})]
verdict = judge.judge_once("/r", "Do it.", "## Goals", "", entries, 1, Rung("judge-model"), FakeEffects(claude=[judged("unfinished", evidence="round 2: 🔴")]).effects())
verdict = judge.judge_once("/r", judge.Brief("Do it.", "## Goals", "", entries, 1), Rung("judge-model"), FakeEffects(claude=[judged("unfinished", evidence="round 2: 🔴")]).effects())
self.assertEqual(verdict, Unfinished("round 2: 🔴", entries_read=2))
-15
View File
@@ -13,7 +13,6 @@
| ID | Release | Blocked by | Exempt | Item | L | E | I | R | S | Moves | Score |
|---|---|---|---|---|---|---|---|---|---|---|---|
| 1 | 0.7.0 | | question | **Say in README → Install how the transcript archive is backed up, once bender-infra backs up ai-sandbox's data volume.** | 1 | 2 | 1 | 2 | 1 | | 0.0 |
| 237 | 0.7.0 | | principle | **Give `judge` and `judge_once` in `bin/loop_claude/cycle/judge.py` a record of the inputs they share, in place of nine and eight parameters, or record why they stay.** | 2 | 1 | 2 | 3 | 3 | code health by 2 | 3.6 |
| 242 | 0.7.0 | | principle | **Run each case of a table-driven test in its own `subTest`, and split each test that checks several behaviours, so one failure does not hide the others and names its case.** | 2 | 1 | 1 | 2 | 3 | code health by 2 | 3.6 |
| 113 | 0.7.0 | | | **Compute an item's S from the code it names, where it names any, and show it beside the finished items `score-outcomes` lists.** | 3 | 2 | 2 | 5 | 4 | 1.1 by 5, 1.4 by 5 | 3.4 |
| 263 | 0.7.0 | | | **Have the judge reject an interrupt that re-asks what the human answered earlier in the chunk, unless the ask quotes what that answer left open.** | 2 | 2 | 3 | 5 | 3 | 1.2 by 6, 1.5 by 2 | 3.2 |
@@ -24,9 +23,7 @@
| 112 | 0.7.0 | | | **Count an item as shipped only where its chunk built it, so an item a chunk dropped from the plan gets no outcome.** | 3 | 2 | 2 | 5 | 3 | 1.1 by 6 | 2.5 |
| 276 | 0.7.0 | | | **Let a worker fix on its own rung a `standards/quality.md` finding whose fix changes only prose, so escalation trigger 4 calls for a higher rung only where code or config must change.** | 2 | 2 | 3 | 5 | 3 | 1.2 by 4, 1.4 by 2 | 2.4 |
| 116 | 0.7.0 | | | **Report L by the share of chunks with a 🔴, once 10 chunks that each shipped one item record finding counts and most record no 🔴.** | 1 | 1 | 2 | 2 | 2 | 1.1 by 5 | 2.3 |
| 101 | 0.7.0 | | principle | **Have `larv_review/history_commands.py`'s `read_seats` take the time, the commit check and the history as inputs, so it parses and decides without effects.** | 2 | 1 | 2 | 3 | 3 | 1.1 by 5 | 2.1 |
| 12 | 0.7.0 | | | **Run the test suite in CI on every PR, or record in `docs/decisions.md` why the local run is enough.** | 3 | 2 | 1 | 4 | 4 | 1.1 by 6 | 2.1 |
| 81 | 0.7.0 | | principle | **Have `workflow-bench/bench.py` return each run's failure as a value, so one failed clone does not hide the later runs' results.** | 1 | 2 | 1 | 2 | 2 | 1.1 by 4 | 1.9 |
| 173 | 0.7.0 | | | **Have rule-hook's `todo-rank-check` run a worktree's own `bin/rank_todo.py` when the worktree is of this repo, so a branch that changes `todo.md`'s shape commits without an override.** | 2 | 1 | 3 | 4 | 2 | 1.1 by 2, 1.4 by 2 | 1.8 |
| 272 | 0.7.0 | | owner | **Make `deploy-release --cut` say when each step finishes, end on one line saying the release is cut and `~/.claude` is on it, and delete the merged release branch.** | 1 | 1 | 1 | 1 | 2 | 1.5 by 3 | 1.3 |
| 274 | 0.7.0 | | | **Let an item name a maintain goal it raises confidence in, such as a test leg that guards it, and weigh that move against the achieve goals in `bin/release_plan/ranking.py`.** | 2 | 2 | 3 | 5 | 4 | 1 by 3 | 1.1 |
@@ -75,10 +72,6 @@ It lives under `~/.claude`, which `.gitignore` keeps from the remote, so a lost
The README line covers the bench's transcripts in `workflow-bench/logs/` too (goal 3).
### 237. Give `judge` and `judge_once` in `bin/loop_claude/cycle/judge.py` a record of the inputs they share, in place of nine and eight parameters, or record why they stay.
Found by the principles audit of PR #376. "About seven things per level" in `principles/technical-principles.md`: `judge` takes nine parameters and `judge_once` eight, past the four or five for things that act on each other, with no recorded reason. Both take root, task, premises, records, entries, escalated_at, rung and effects, which one record could carry.
### 242. Run each case of a table-driven test in its own `subTest`, and split each test that checks several behaviours, so one failure does not hide the others and names its case.
"Tests" in `principles/technical-principles.md`. The principles audit of PR #380 found test bodies that loop over their cases without `subTest`, some checking several behaviours at once: `bin/judge_replay/test_judge_replay.py:38`, `:139`, `:176`; `bin/loop_claude/test_loop_claude.py:590`; `bin/machine_load/test_machine_load.py:19`, `:31`; `bin/rule_hook/test_rule_hook.py:480`; `bin/subagent_rung/test_subagent_rung.py:32`, `:43`, `:45`; `bin/test_settings_filter.py:28`; `bin/test_statusline.py:102`. The first failing case stops the loop, and without a message the failure doesn't say which case broke. Split the tests that check several behaviours at once, `test_judge_replay.py:176` and `test_subagent_rung.py:43-46`, into one test per behaviour ("One behaviour per test").
@@ -121,20 +114,12 @@ On 2026-11-08, PR #432's prose round 1 reported two findings against `standards/
`score-outcomes` reports the median 🔴 count at each guessed R, and how well L orders chunks by 🔴 findings; the part of R a 🔴 tests is L. Where most chunks have no 🔴, every median reads 0 and the rank correlation rests on the few that have one, which tells a scorer little. No round title held counts before PR #254, so how often a chunk has a 🔴 is not yet known.
### 101. Have `larv_review/history_commands.py`'s `read_seats` take the time, the commit check and the history as inputs, so it parses and decides without effects.
Found by the principles audit of PR #242, against "Functional core, imperative shell" (`principles/technical-principles.md`). `bin/larv_review/history_commands.py:87` reads `time.time()`, then checks git (`:90`, `commit_exists`) and the history file (`:92`) inside the function that parses the seats file and decides. A test cannot fix the heading's time, and every test of `read_seats` touches the real checkout.
### 12. Run the test suite in CI on every PR, or record in `docs/decisions.md` why the local run is enough.
The repo has no CI, so larv-review's gate is the local run, and larv-review's Gate skips even that when the last stability round left the suite green: an unattended chunk can merge on the worker's own word.
When CI runs a rule check, such as one of `bin/rule-hook`'s, let `<!-- rule-override <rule>: <reason> -->` in the PR body override it.
### 81. Have `workflow-bench/bench.py` return each run's failure as a value, so one failed clone does not hide the later runs' results.
"Errors as values", in `principles/technical-principles.md`, raised by the principles audit of PR #224: `bench.py` raises `KeyError` and `CalledProcessError`, and `pool.map` re-raises the first while the other threads run on unreported.
### 173. Have rule-hook's `todo-rank-check` run a worktree's own `bin/rank_todo.py` when the worktree is of this repo, so a branch that changes `todo.md`'s shape commits without an override.
Raised by the overrides pass of PR #296. The hook imports the main checkout's `release_plan/ranking.py`, so while a branch of this repo changes the table's shape, every commit touching `todo.md` needs a `todo-rank-check` override: 33 log lines across four branches, the score-outcomes, plan-136, plan-148 and item-146 ones. The hook then checks nothing, and the rule rests on the worker chaining the branch's own check. Run another repo's `bin/rank_todo.py` never: that would run its code before every commit.
+9 -1
View File
@@ -113,9 +113,17 @@ def run(name):
return f"{name}: FAILED after {ATTEMPTS} attempts"
def attempted(name):
"""run's result line, or a line naming the failure, so one failed run leaves the others' results to print."""
try:
return run(name)
except (KeyError, ValueError, subprocess.CalledProcessError, OSError) as failure:
return f"{name}: FAILED: {type(failure).__name__}: {failure}"
def main():
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
for result in pool.map(run, sys.argv[1:]):
for result in pool.map(attempted, sys.argv[1:]):
print(result, flush=True)
+29
View File
@@ -0,0 +1,29 @@
import os
import subprocess
import unittest
import unittest.mock
with unittest.mock.patch.dict(os.environ, {"BENCH_SOURCES": "/nonexistent"}):
import bench
class Attempted(unittest.TestCase):
def test_returns_the_run_result_of_a_run_that_finished(self):
with unittest.mock.patch.object(bench, "run", return_value="a-v-1: done"):
self.assertEqual(bench.attempted("a-v-1"), "a-v-1: done")
def test_returns_a_failed_run_as_a_line_naming_it_so_the_later_runs_still_report(self):
failures = [KeyError("nope"), ValueError("not enough values to unpack"), subprocess.CalledProcessError(128, ["git", "clone"]), FileNotFoundError("claude")]
for failure in failures:
with self.subTest(failure=type(failure).__name__), unittest.mock.patch.object(bench, "run", side_effect=[failure, "b-v-1: done"]):
results = [bench.attempted("a-v-1"), bench.attempted("b-v-1")]
self.assertTrue(results[0].startswith("a-v-1: FAILED: "), results[0])
self.assertIn(type(failure).__name__, results[0])
self.assertEqual(results[1], "b-v-1: done")
def test_a_run_of_an_unknown_case_fails_as_a_line(self):
self.assertRegex(bench.attempted("nope-v-1"), r"^nope-v-1: FAILED: KeyError")
if __name__ == "__main__":
unittest.main()