From 7b38aed614044a0ac8b01ea40e079ac4e4eb50d1 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Sat, 3 Oct 2026 18:21:33 +0200 Subject: [PATCH] Answer prose round 1: precedent names what a pass skips on, the bench and floor-decision prose reads on one pass --- AGENTS.md | 2 +- bin/loop_claude/cycle/archive.py | 1 - bin/loop_claude/cycle/judge_reply.py | 3 +-- bin/loop_claude/cycle/states.py | 2 +- docs/decisions.md | 4 ++-- skills/larv-review/SKILL.md | 12 ++++++------ todo.md | 9 +++++++-- workflow-bench/bench.py | 5 +++-- workflow-bench/compare.py | 2 +- 9 files changed, 22 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 69a509ab..119640b8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ Decisions live in `docs/decisions.md`: - loop-claude: each project's model lives in `~/.claude/loop-claude-models.json`, which only loop-claude writes - review: a pass gets at most three rounds per chunk; instead of a fourth, the run asks the human - review: a fix for a ๐ŸŸข or โ“ re-enters review like any fix -- review: a pass skips only on a standing decision, one the repo held before the chunk began +- review: a pass skips only on a precedent, a decision or plan item the repo held before the chunk began - loop-claude: the archive keeps the raw transcript, and a failed archive stops the loop - loop-claude: a `claude` that cannot start is a failed worker run, retried - review: an ask no user answers leaves the PR unmerged diff --git a/bin/loop_claude/cycle/archive.py b/bin/loop_claude/cycle/archive.py index 07e99112..404ce080 100644 --- a/bin/loop_claude/cycle/archive.py +++ b/bin/loop_claude/cycle/archive.py @@ -24,7 +24,6 @@ def archive(project_key: str, path: str, task: str, outcome: str, paths: Paths) return unkept copy = folder / f"{hashlib.sha256(raw).hexdigest()}.jsonl" outcomes = copy.with_suffix(".outcomes.md") - # A copy whose outcomes file is missing is written afresh, outcomes first. if copy.exists() and outcomes.exists(): return add_outcome(outcomes, outcome) # The copy lands last, so a copy on disk always has its outcomes and its session's files beside it. diff --git a/bin/loop_claude/cycle/judge_reply.py b/bin/loop_claude/cycle/judge_reply.py index fd023bf7..c7f8e1db 100644 --- a/bin/loop_claude/cycle/judge_reply.py +++ b/bin/loop_claude/cycle/judge_reply.py @@ -23,7 +23,7 @@ JUDGE_RULES = ( "Where the agent asked or flagged something itself, the question is that one, in the agent's own " "wording; never construct another from facts it stated.\n" "Where README.md or the project's records answer the agent's own question and no [loop] line has " - "quoted that answer yet, set interrupt=false and quote the answer; the agent is " + "quoted that answer yet, set interrupt=false and quote the passage that answers it; the agent is " "then given that passage in place of the human's reply. An interrupt outranks a quoted answer, and either " "outranks finished.\n" "The chunk is the piece of work the agent is on, shipped as one pull request. " @@ -84,7 +84,6 @@ def verdict_from(output: object) -> Verdict | Failure: for name, kind in REPLY_TYPES.items(): if not isinstance(output.get(name), kind): return Failure(f"the judge's reply has no {name} of type {JSON_TYPES[kind]}") - # The human answers before the chunk can close. if output["interrupt"]: if not output["question"].strip(): return Failure("the judge interrupted without a question") diff --git a/bin/loop_claude/cycle/states.py b/bin/loop_claude/cycle/states.py index 81dc3cad..8b64a743 100644 --- a/bin/loop_claude/cycle/states.py +++ b/bin/loop_claude/cycle/states.py @@ -22,7 +22,7 @@ class Judging: @dataclasses.dataclass(frozen=True) class Interrupting: - """The human answers; a QuotedAnswer here is one past the cycle's QUOTED_ANSWERS_PER_CYCLE.""" + """The human answers; a QuotedAnswer reaches here only after the cycle has used up its QUOTED_ANSWERS_PER_CYCLE.""" transcript_path: str ruling: Interrupt | QuotedAnswer quoted_answers: int = 0 diff --git a/docs/decisions.md b/docs/decisions.md index a406039e..215dd474 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -34,7 +34,7 @@ 2026-09-27, lilleman, restating the 2026-09-24 target and extending it to the overall. Serves goal 4. The 2026-09-28 run, Junior A, Mid A, the maintainability senior and the inherited architect at depth 1 over every non-test file, blind to `todo.md` and this log, scored Navigation 7.25, Locality 6.0, Shape 6.5, Self-sufficiency 6.25, overall 6.25; on 2026-09-26 they were 7.75, 6.25, 7.0, 6.75, 6.75. Every seat scored Locality 6, and three of four named `run_cycle`'s three-way return and larv-review's re-entry rules, split across the passes on 2026-09-28. The same seats on `rebuild` scored 7.5, 6.63, 6.88, 6.5, 7.0 at 37edc9a, later that day, 7.5, 6.75, 6.88, 6.75, 7.0 at 7f9de5c, 7.38, 6.25, 6.88, 6.75, 6.88 at 2f9ab2f on 2026-09-29, Locality lowest again and every seat naming the reload state and the two retry counters, and 7.5, 7.0, 7.25, 6.75, 7.13 at 2c74bbc later that day, Self-sufficiency lowest and every seat naming the round's definition split across larv-review's two files, and 7.38, 6.75, 7.25, 6.5, 6.88 at 8585257 on 2026-10-01, lowering every dimension but Shape, and the overall, Self-sufficiency lowest and every seat naming larv-review ยง3's round rules; the places it named under Self-sufficiency and Locality are filed in `todo.md` under 0.1.0, and a later scoring run may not score any dimension or the overall below its best run on `rebuild`, 2026-10-01, lilleman. `rebuild` merged into main under the floor at 8585257's scores, 2026-10-01, lilleman. The 2026-10-01 run on PR #182 (the loop state machine) scored 7.5, 6.5, 6.88, 6.25, 6.38; lilleman shipped that PR under the floor and below `rebuild`'s best run the same day, its places filed in `todo.md` under 0.1.0. The 2026-10-02 run on PR #187 (`loop_claude`'s names, the exit contract) scored 7.75, 7.25, 7.38, 6.88, 7.25 at 70362d0, every seat naming larv-review ยง3 under Self-sufficiency, and 7.63, 7.13, 7.38, 7.0, 7.13 at 1971dba later that day, the first run at the floor, with no dimension or the overall below its best run on `rebuild`. The 2026-10-02 runs on PR #191 (the status line in Python) scored 7.5, 7.0, 7.13, 6.88, 7.13; 7.25, 6.75, 7.25, 6.88, 7.0; 7.63, 6.88, 7.38, 7.0, 7.13; and 7.63, 7.0, 7.25, 6.75, 7.25. Every seat rated the status line easy. The places they named under Locality and Self-sufficiency are filed in `todo.md` under 0.1.0, except `entries.py`'s heuristics over Claude Code's transcript format, which the seats scored as the problem's own difficulty. lilleman decided the same day to ship that PR under the floor. The 2026-10-02 runs on PR #194 (each cycle's transcript archived) scored 8.0, 7.25, 7.38, 7.13, 7.5 at 6ccd5b1 and 7.75, 7.13, 7.25, 7.0, 7.25 at d104ea0, both at the floor, every seat naming `archive` as the unit it would least want to modify. On 2026-10-02, applying `todo.md`'s bar deleted every place these scoring runs filed except larv-review ยง3's round rules and the Claude Code flags no comment explained. The 2026-10-02 runs on PR #196 (`todo.md` as one scored table) scored 7.63, 6.88, 7.25, 6.88, 7.13 at c0dba66; 7.5, 6.88, 7.13, 6.75, 7.0 at 798dfd7; and 7.5, 7.0, 7.0, 6.75, 7.0 at b712076. Under Self-sufficiency, every seat named larv-review ยง3 and the agents, which a reader follows only with `CLAUDE.md` open. lilleman decided the same day to ship that PR under the floor; the place filed was larv-review ยง3's round rules. The 2026-10-02 run on PR #198 (a usage limit waited out) scored 7.63, 6.88, 7.38, 6.75, 7.13 at 8e85033. Under Self-sufficiency, every seat named larv-review ยง3; under Locality, they named `archive`'s invariants spread over `session.py` and `run.py`, `rank_todo.py` reaching into `premises.sections`, and `stopped` comparing reason strings, each under `todo.md`'s bar and not filed. lilleman decided the same day to ship that PR under the floor. -The 2026-10-03 runs on PR #202 (the judge reads the project's records) scored 7.75, 7.13, 7.25, 7.0, 7.25 in the first round and 7.5, 7.0, 7.0, 6.75, 7.0 in the eighth; between them a dimension moved by up to 0.4 on near-identical code. Under Self-sufficiency every seat named larv-review ยง3 and the agents leaning on `CLAUDE.md`, the Claude Code flags left unexplained, and the judge's contract spread over its prompt, schema, parser and `--help`; under Shape, that "record", "verdict" and "judge" each name two things, and that `loop_claude/` holds modules `rank_todo.py` also uses. lilleman decided the same day to ship that PR under the floor and to fix those places next. All were fixed on 2026-10-03: the flags and the words the judge's prompt left undefined (chunk, plan, a release ready to cut) explained, larv-review ยง3 made readable on its own, the judge's contract given one home in `cycle/judge_reply.py`, the three words one meaning each, and the modules `rank_todo.py` shares moved out of `loop_claude/` to `bin/`. Valid until the next scoring run, which uses the same four seats. +The 2026-10-03 runs on PR #202 (the judge reads the project's records) scored 7.75, 7.13, 7.25, 7.0, 7.25 in the first round and 7.5, 7.0, 7.0, 6.75, 7.0 in the eighth; between them a dimension moved by up to 0.4 on near-identical code. Under Self-sufficiency every seat named larv-review ยง3 and the agents leaning on `CLAUDE.md`, the Claude Code flags left unexplained, and the judge's contract spread over its prompt, schema, parser and `--help`; under Shape, that "record", "verdict" and "judge" each name two things, and that `loop_claude/` holds modules `rank_todo.py` also uses. lilleman decided the same day to ship that PR under the floor and to fix those places next; all were fixed on 2026-10-03. Valid until the next scoring run, which uses the same four seats. ## loop-claude: each project's model lives in `~/.claude/loop-claude-models.json`, which only loop-claude writes @@ -48,7 +48,7 @@ The 2026-10-03 runs on PR #202 (the judge reads the project's records) scored 7. 2026-09-30, lilleman. Serves goal 4: no human reads an unattended chunk before it merges, so every change it makes, a nit's wording included, is reviewed. In the 2026-09-30 bench this cost four of nine rounds in one run and drove two passes to the round cap. Valid while a pass is the only reader of a fix before merge. -## review: a pass skips only on a standing decision, one the repo held before the chunk began +## review: a pass skips only on a precedent, a decision or plan item the repo held before the chunk began 2026-09-30, lilleman. Serves goal 4: a decision the chunk writes is the chunk's own and unreviewed, and an unattended run has no human to catch it. Valid while a pass is the only reader of a chunk before merge. diff --git a/skills/larv-review/SKILL.md b/skills/larv-review/SKILL.md index 04d6fec5..0d1c5ee3 100644 --- a/skills/larv-review/SKILL.md +++ b/skills/larv-review/SKILL.md @@ -41,12 +41,12 @@ A comment is prose, not code. Passes run in this order: line-level polish on cod Give a reviewer without Bash the branch diff and `git ls-files`'s output in its prompt: some harnesses grant it Read alone. -**Standing decision, then ask** โ€” for the architecture pass and the full scan: +**Precedent, then ask** โ€” for the architecture pass and the full scan: -- **Skip.** A standing decision is a `docs/decisions.md` entry, or a planning item crediting the pass, that the repo held before the chunk began. Where one holds the decision this change executes, skip the pass and cite it. -- **Departure.** A change that departs from the standing decision is unreviewed, however thorough the decision is. +- **Skip.** A precedent is a `docs/decisions.md` entry, or a planning item crediting the pass, that the repo held before the chunk began. Where one holds the decision this change executes, skip the pass and cite that entry or item. +- **Departure.** A change that departs from the precedent is unreviewed, however thorough the precedent is. - **Ask.** Fired and not skipped โ†’ ask the user whether to run it, with the evidence and your recommendation; no user reachable โ†’ run it. For the full scan, give the evidence and your recommendation, then launch it: the plugin asks the user to confirm its cost, and that prompt is the ask. -- **Audit.** A standing decision or the user's no skips only the architect, never the principles audit or the scoring run: no decision older than the chunk checked the code it writes. +- **Audit.** A precedent or the user's no skips only the architect, never the principles audit or the scoring run: no precedent checked the code this chunk writes. **Round cap.** @@ -67,10 +67,10 @@ Give a reviewer without Bash the branch diff and `git ls-files`'s output in its **Findings.** A reviewer rates each finding ๐Ÿ”ด must fix, ๐ŸŸก should fix, ๐ŸŸข nit, โ“ question, or ๐Ÿ”€ escalation naming the reviewer whose concern it is. - **Outside the chunk.** A finding outside the chunk's change, whatever its pass or severity, is scored and filed in `todo.md` per `~/.claude/CLAUDE.md` โ†’ "Todo items". A ๐Ÿ”ด is marked `defect`. A ๐ŸŸก against a file under `~/.claude/principles/` is marked `principle`, its detail naming the principle, where no other reason fits. One under the bar and not exempt is named in the summary with its score. A verified security HIGH is handled per "Security". -- **Goal or decision.** A fix that would change a goal, or a standing decision, is asked; no user reachable โ†’ filed, marked `question`, the goal or decision left as is. +- **Goal or decision.** A fix that would change a goal, or a `docs/decisions.md` entry the repo held before the chunk began, is asked; no user reachable โ†’ filed, marked `question`, the goal or decision left as is. - **๐Ÿ”ด.** A ๐Ÿ”ด is fixed, or shown factually wrong, and what disproved it recorded per `~/.claude/CLAUDE.md` โ†’ "Decisions: one home, each linked to a goal or a technical principle": in a comment at the code the finding concerns, or in `docs/decisions.md` when it concerns more than one place. Never declined. - **๐ŸŸก.** A ๐ŸŸก is fixed, shown factually wrong like a ๐Ÿ”ด, or declined. Decline one only where the reason is sound, consistent with the goals and the technical principles, and confidence is very high; the decline and its reason are recorded the same way. A reason neither speaks to is a gap in the goals: fix the finding, or ask per `~/.claude/CLAUDE.md` โ†’ "Working loop", "Ask, don't guess"; no user reachable โ†’ fix. Never drop one silently, and never decline one for convenience or the size of the fix. -- **๐ŸŸข and โ“.** A ๐ŸŸข or โ“ is fixed, answered, declined in the summary with its reason, or filed. Answer one where the code, a test or a standing decision settles it, and quote what in the summary. A โ“ only the user can answer is asked; no user reachable โ†’ filed, marked `question`. +- **๐ŸŸข and โ“.** A ๐ŸŸข or โ“ is fixed, answered, declined in the summary with its reason, or filed. Answer one where the code, a test or a precedent settles it, and quote what in the summary. A โ“ only the user can answer is asked; no user reachable โ†’ filed, marked `question`. - **๐Ÿ”€.** A ๐Ÿ”€ runs the named reviewer's pass once the current pass is done. - **Other scales.** Other reviewers map onto this scale: `/security-review`'s HIGH is ๐Ÿ”ด, MEDIUM ๐ŸŸก, LOW ๐ŸŸข; what a forge's auto-reviewer marks blocking is ๐Ÿ”ด. A finding of theirs repeating one declined or filed does not stand. diff --git a/todo.md b/todo.md index 7fbce108..4d0dbd0b 100644 --- a/todo.md +++ b/todo.md @@ -6,7 +6,7 @@ `Bar = 9` -`Next ID = 53` +`Next ID = 54` | Goal | W | |---|---| @@ -31,6 +31,7 @@ | 23 | 0.1.0 | | **Launch a principles auditor only for a principles file the project shows applies, and rerun only the reviewer whose finding a fix answers.** | 3 | 4 | 7 | 8 | 4.4 | 13.5 | | 21 | 0.1.0 | | **Give `settings.json`'s model one writer, and sort its keys.** | 3 | 3 | 6 | 7 | 4.3 | 12.9 | | 49 | 0.1.0 | principle | **Check that `AGENTS.md` lists exactly the titles of `docs/decisions.md`.** | 1 | 2 | 5 | 6 | 4.3 | 12.5 | +| 53 | 0.1.0 | | **Cut the comprehension-floor decision to its standing rule and each call to ship under the floor.** | 1 | 2 | 4 | 5 | 4.1 | 12.1 | | 12 | 0.1.0 | | **Run the test suite in CI on every PR, or record in `docs/decisions.md` why the local run is enough.** | 2 | 4 | 4 | 6 | 4.1 | 11.7 | | 5 | 0.1.0 | decision | **Key a project on its checkout's full path, and refuse to start while another session uses that key.** | 5 | 5 | 5 | 6 | 4.1 | 10.5 | | 4 | 0.1.0 | | **Name each chunk's archive folder by the chunk's start time, so `/` lists chunks in the order they ran.** | 3 | 2 | 4 | 3 | 2 | 9.8 | @@ -54,7 +55,7 @@ The judge runs without session persistence, and `close` and `ask` archive only t ### 46. Let the judge read and quote `~/.claude/CLAUDE.md` beside the project's records. -The goals audit on PR #208 rated it ๐Ÿ”ด against goals 4 and 4.2: the worker follows `CLAUDE.md`, but the judge sees only the premises, `AGENTS.md` and `docs/decisions.md`. A choice `CLAUDE.md` settles therefore reads to the judge as one nothing discriminates, and a question it answers goes to the human. The fix revises the decision "loop-claude: a judge decides each interrupt and each quoted answer from the transcript, the README's premises, `AGENTS.md` and `docs/decisions.md`", so the human decides it. +The architecture pass's principles audit against README.md's goals, on PR #208, rated the gap ๐Ÿ”ด against goals 4 and 4.2: the worker follows `CLAUDE.md`, but the judge sees only the premises, `AGENTS.md` and `docs/decisions.md`. A choice `CLAUDE.md` settles therefore reads to the judge as one nothing discriminates, and a question it answers goes to the human. The fix revises the decision "loop-claude: a judge decides each interrupt and each quoted answer from the transcript, the README's premises, `AGENTS.md` and `docs/decisions.md`", so the human decides it. ### 48. Stamp each archived outcome with its UTC time. @@ -74,6 +75,10 @@ Every write Claude Code makes to the file rewrites `"model"`, `claude plugin ins DRY, in `principles/technical-principles.md`: the list is copied by hand, and sessions work from the copy. +### 53. Cut the comprehension-floor decision to its standing rule and each call to ship under the floor. + +Its 2026-09-27 paragraph lists every scoring run's five scores from PR #182 to #198 and buries the ratchet rule among them; the scores live in the PRs and git history. + ### 12. Run the test suite in CI on every PR, or record in `docs/decisions.md` why the local run is enough. The repo has no CI, so larv-review's gate is the local run, and larv-review's Gate skips even that when the last stability round left the suite green: an unattended chunk can merge on the worker's own word. diff --git a/workflow-bench/bench.py b/workflow-bench/bench.py index e6d2a4d0..25ab0920 100755 --- a/workflow-bench/bench.py +++ b/workflow-bench/bench.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 """Run larv-review bench runs: BENCH_SOURCES= bench.py -- ... -The 2026-09-30 bench had two versions of larv-review, main and rebuild, each review four real chunks, -the cases, three times; compare.py then rules which version reviews better. Its inputs are untracked: +The 2026-09-30 bench ran two versions of larv-review, main and rebuild. Each reviewed four real chunks, +the cases, three times; compare.py then rules whether each version reviews at least as well as the +other. The bench's inputs are untracked: - versions// beside this script: a copy of ~/.claude at that version, which each run reads in place of ~/.claude (run-prompt.md). - $BENCH_SOURCES/-main/: a clone of the case's project; CASES names the chunk's base and head diff --git a/workflow-bench/compare.py b/workflow-bench/compare.py index 2c266b31..98435398 100755 --- a/workflow-bench/compare.py +++ b/workflow-bench/compare.py @@ -29,7 +29,7 @@ def runs(case, version): def blind(text): - """The report with every name that tells which version wrote it replaced.""" + """The report, with VERSION_DIR in place of every name that tells which version wrote it.""" return re.sub(r"\S*versions/(main|rebuild)\S*|\b(main|rebuild)\b(?=[`\s]*(process|version))", "VERSION_DIR", text)