From 5c3702704637fe3dea3ece6891d94af6380613e1 Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Thu, 12 Nov 2026 20:05:52 +0100 Subject: [PATCH] =?UTF-8?q?Answer=20architecture=20round=201:=201=20?= =?UTF-8?q?=F0=9F=94=B4,=208=20=F0=9F=9F=A1=20(0=20misread);=20one=20reade?= =?UTF-8?q?r=20per=20message,=20prices=20and=20Spent=20block=20moved,=20on?= =?UTF-8?q?e=20table=20of=20larv-review=20commands?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/chunk_pr/main.py | 2 +- bin/claude_code/entries.py | 9 + bin/claude_code/test_claude_code.py | 7 + bin/larv_review/main.py | 78 ++--- bin/larv_review/scoring_setup.py | 38 +-- bin/larv_review/test_larv_review.py | 12 + bin/larv_review/test_scoring_setup.py | 20 +- bin/loop_claude/cycle/run.py | 2 +- bin/loop_claude/cycle/spend.py | 26 +- bin/loop_claude/test_loop_claude.py | 6 +- .../prices.py | 0 bin/reading_cost/measure.py | 5 +- bin/records/markdown_text.py | 30 +- bin/records/test_markdown_text.py | 11 + bin/review_replay/main.py | 267 +----------------- bin/review_replay/omit_comparison.py | 56 ++++ bin/review_replay/recall.py | 34 +++ bin/review_replay/replay.py | 179 ++++++++++++ bin/review_replay/rounds.py | 59 ++-- bin/review_replay/test_review_replay.py | 14 +- bin/review_rules/reviewer_context.py | 13 +- bin/review_rules/test_reviewer_context.py | 4 + bin/routing_figures/dollars.py | 15 +- bin/transcript_archive/outcomes.py | 18 +- docs/decisions.md | 2 +- reviewer-merge-trials/analyze_metered.py | 2 +- reviewer-merge-trials/driver.py | 4 +- skills/larv-review/SKILL.md | 2 + 28 files changed, 503 insertions(+), 412 deletions(-) rename bin/{routing_figures => model_routing}/prices.py (100%) create mode 100644 bin/review_replay/omit_comparison.py create mode 100644 bin/review_replay/replay.py diff --git a/bin/chunk_pr/main.py b/bin/chunk_pr/main.py index 5d0c1e53..362db64e 100644 --- a/bin/chunk_pr/main.py +++ b/bin/chunk_pr/main.py @@ -115,7 +115,7 @@ def checkout(run: Run) -> tuple[Forge, str, str] | Failure: def default_branch(run: Run) -> str | Failure: - """origin's default branch, such as main.""" + """origin's default branch, such as main. Not git_cli's default_ref: that falls back to a local main or master and asks origin where nothing is recorded, but a pull request's base is origin's alone, and this tells the user how to record it.""" default = run(["git", "symbolic-ref", "--short", "refs/remotes/origin/HEAD"]) if isinstance(default, Failure): return Failure(f"origin records no default branch; set it with `git remote set-head origin --auto`: {default.reason}") diff --git a/bin/claude_code/entries.py b/bin/claude_code/entries.py index d7c06744..900ddb48 100644 --- a/bin/claude_code/entries.py +++ b/bin/claude_code/entries.py @@ -145,6 +145,15 @@ def tool_calls(entries: typing.Iterable[Entry]) -> list[tuple[AssistantTurn, Too return [(entry, part) for entry in entries if isinstance(entry, AssistantTurn) for part in entry.parts if isinstance(part, ToolCall)] +def messages(entries: typing.Iterable[Entry]) -> list[list[AssistantTurn]]: + """The assistant turns grouped by message, in the order each message first appears. A message's usage repeats in each of its entries, so read it from the first of a group: none of 1,681 multi-entry messages in the archive of 2026-10-28 differed.""" + grouped: dict[str, list[AssistantTurn]] = {} + for entry in entries: + if isinstance(entry, AssistantTurn): + grouped.setdefault(entry.message_id or entry.uuid, []).append(entry) + return list(grouped.values()) + + def field(found: dict, key: str) -> str: return value if isinstance(value := found.get(key), str) else "" diff --git a/bin/claude_code/test_claude_code.py b/bin/claude_code/test_claude_code.py index bb6aea1e..470ded3e 100644 --- a/bin/claude_code/test_claude_code.py +++ b/bin/claude_code/test_claude_code.py @@ -148,6 +148,13 @@ class EntryKinds(unittest.TestCase): uuid="u", parts=(entries.Text("Starting."), entries.ToolCall("toolu_1", "Bash", {"command": "ls"})), message_id="msg_1", model="claude-opus-5-5", usage={"input_tokens": 2}, )) + def test_messages_group_the_turns_that_share_a_message_id_and_fall_back_to_the_uuid_where_there_is_none(self): + first = entries.AssistantTurn(uuid="a", message_id="msg_1", usage={"input_tokens": 1}) + other = entries.AssistantTurn(uuid="b", message_id="msg_2") + again = entries.AssistantTurn(uuid="c", message_id="msg_1", usage={"input_tokens": 2}) + bare = entries.AssistantTurn(uuid="d") + self.assertEqual(entries.messages([first, entries.HumanTurn(text="hi"), other, again, bare]), [[first, again], [other], [bare]]) + def test_parses_tool_results_and_cost_states_and_reads_queued_notifications_and_other_kinds_as_plain_entries(self): self.assertEqual(entries.entry_from(human([{"type": "tool_result", "tool_use_id": "toolu_1", "content": "hi", "is_error": True}])), entries.ToolResults(results=(entries.ToolResult("toolu_1", "hi", True),))) self.assertEqual(entries.entry_from(queued("done", "task-notification")), entries.Entry()) diff --git a/bin/larv_review/main.py b/bin/larv_review/main.py index c523b69f..b933c08e 100644 --- a/bin/larv_review/main.py +++ b/bin/larv_review/main.py @@ -1,6 +1,6 @@ """Hold larv-review's mechanical rules: the chunk's review folder, the paired ruling's base and vote count, each pass's rounds, the scoring run's trigger and setup; the comprehension history's format, its latest places and its count of how often seats agree; and the header of each reviewer's prompt. -usage: larv-review review-folder | ruling-base | round-count | needs-scoring-run [...] | scoring-run-owed | record-ruling | prepare-scoring [--preloaded ] | record-scoring [...] | places | agreement | reviewer-header [--rerun] | reviewer-context +usage: larv-review {commands} The comprehension check records how hard a project is to read, two ways, and no score blocks a merge. In a paired ruling, simulated readers, the seats, read the branch's head beside a base and vote better, @@ -167,45 +167,57 @@ from larv_review.scoring_setup import prepare_scoring from larv_review.history_commands import record_ruling, record_scoring, show_agreement, show_places from larv_review.reviewer_header import reviewer_header -USAGE = "usage: larv-review review-folder | ruling-base | round-count | needs-scoring-run [...] | scoring-run-owed | record-ruling | prepare-scoring [--preloaded ] | record-scoring [...] | places | agreement | reviewer-header [--rerun] | reviewer-context" + +class Command(typing.NamedTuple): + """One subcommand: how the usage line spells it, whether the arguments after its name fit, and what it does with them.""" + shape: str + fits: typing.Callable[[list[str]], bool] + run: typing.Callable[[list[str]], Failure | None] + + +def no_arguments(given: list[str]) -> bool: + return not given + + +def are_ids(given: list[str]) -> bool: + return all(re.fullmatch(r"[0-9]+", id_) for id_ in given) + + +# One row per subcommand, in the order the usage line and the docstring above list them: main, USAGE and `--help` all read this table. +COMMANDS = { + "review-folder": Command("review-folder", no_arguments, lambda given: make_review_folder()), + "ruling-base": Command("ruling-base", no_arguments, lambda given: with_default_branch(show_ruling_base, stale_ok=False)), + "round-count": Command("round-count", no_arguments, lambda given: with_default_branch(round_count, stale_ok=True)), + "needs-scoring-run": Command("needs-scoring-run [...]", are_ids, lambda given: with_default_branch(lambda default: needs_scoring_run(default, [int(id_) for id_ in given]), stale_ok=False)), + "scoring-run-owed": Command("scoring-run-owed", no_arguments, lambda given: with_default_branch(owed, stale_ok=False)), + "record-ruling": Command("record-ruling ", lambda given: len(given) == 1, lambda given: record_ruling(pathlib.Path(given[0]))), + "prepare-scoring": Command("prepare-scoring [--preloaded ]", lambda given: not given or len(given) == 2 and given[0] == "--preloaded", lambda given: prepared_scoring(given[1] if given else None)), + "record-scoring": Command("record-scoring [...]", lambda given: bool(given) and are_ids(given[1:]), lambda given: with_default_branch(lambda default: record_scoring(default, pathlib.Path(given[0]), [int(id_) for id_ in given[1:]]), stale_ok=False)), + "places": Command("places", no_arguments, lambda given: show_places()), + "agreement": Command("agreement", no_arguments, lambda given: show_agreement()), + "reviewer-header": Command("reviewer-header [--rerun]", lambda given: given[:1] != [] and given[0] in REVIEWERS and given[1:] in ([], ["--rerun"]), lambda given: with_default_branch(lambda default: reviewer_header(default, given[0], given[1:] == ["--rerun"]), stale_ok=True)), + "reviewer-context": Command("reviewer-context", no_arguments, lambda given: printed(hook_reply(sys.stdin.read(), pathlib.Path.home()))), +} +# The commands that take item IDs, each with the number of arguments before the IDs. +ID_COMMANDS = {"needs-scoring-run": 0, "record-scoring": 1} +SHAPES = " | ".join(command.shape for command in COMMANDS.values()) +USAGE = f"usage: larv-review {SHAPES}" def main(argv: list[str]) -> int: if argv in (["-h"], ["--help"]): - print(__doc__.strip().format(cap=CAP, release_cap=RELEASE_REVIEW_CAP, floor=FLOOR, pairs=MIN_PAIRS)) + print(__doc__.strip().format(commands=SHAPES, cap=CAP, release_cap=RELEASE_REVIEW_CAP, floor=FLOOR, pairs=MIN_PAIRS)) return 0 - command, *given = argv or [""] - if command == "review-folder" and not given: - done = make_review_folder() - elif command == "ruling-base" and not given: - done = with_default_branch(show_ruling_base, stale_ok=False) - elif command == "round-count" and not given: - done = with_default_branch(round_count, stale_ok=True) - elif command == "needs-scoring-run" and all(re.fullmatch(r"[0-9]+", id_) for id_ in given): - done = with_default_branch(lambda default: needs_scoring_run(default, [int(id_) for id_ in given]), stale_ok=False) - elif command == "scoring-run-owed" and not given: - done = with_default_branch(owed, stale_ok=False) - elif command == "record-ruling" and len(given) == 1: - done = record_ruling(pathlib.Path(given[0])) - elif command == "prepare-scoring" and (not given or len(given) == 2 and given[0] == "--preloaded"): - done = prepared_scoring(given[1] if given else None) - elif command == "record-scoring" and given and all(re.fullmatch(r"[0-9]+", id_) for id_ in given[1:]): - done = with_default_branch(lambda default: record_scoring(default, pathlib.Path(given[0]), [int(id_) for id_ in given[1:]]), stale_ok=False) - elif command == "places" and not given: - done = show_places() - elif command == "agreement" and not given: - done = show_agreement() - elif command == "reviewer-header" and given[:1] and given[0] in REVIEWERS and given[1:] in ([], ["--rerun"]): - done = with_default_branch(lambda default: reviewer_header(default, given[0], given[1:] == ["--rerun"]), stale_ok=True) - elif command == "reviewer-context" and not given: - done = printed(hook_reply(sys.stdin.read(), pathlib.Path.home())) - elif command in ("needs-scoring-run", "record-scoring") and given: - named = given[1:] if command == "record-scoring" else given - print(f"{command} takes item IDs, the numbers in todo.md's ID column, not {' '.join(id_ for id_ in named if not re.fullmatch(r'[0-9]+', id_))}; {USAGE}", file=sys.stderr) - return 1 - elif not argv: + if not argv: print(USAGE, file=sys.stderr) return 1 + name, *given = argv + command = COMMANDS.get(name) + if command and command.fits(given): + done = command.run(given) + elif command and name in ID_COMMANDS and (named := [id_ for id_ in given[ID_COMMANDS[name]:] if not re.fullmatch(r"[0-9]+", id_)]): + print(f"{name} takes item IDs, the numbers in todo.md's ID column, not {' '.join(named)}; {USAGE}", file=sys.stderr) + return 1 else: print(f"unknown arguments {' '.join(argv)}; {USAGE}", file=sys.stderr) return 1 diff --git a/bin/larv_review/scoring_setup.py b/bin/larv_review/scoring_setup.py index efaefc69..130cddca 100644 --- a/bin/larv_review/scoring_setup.py +++ b/bin/larv_review/scoring_setup.py @@ -5,7 +5,7 @@ import re import shutil from failure import Failure -from records.markdown_text import FENCE, listed +from records.markdown_text import FENCE, listed, section from git_cli.git_repo import git from review_rules.review_folder import made_review_folder, review_folder @@ -23,26 +23,6 @@ FORBIDDEN_FILES = ("todo.md", "docs/decisions.md", "docs/comprehension-history.m FORBIDDEN = "todo.md, docs/decisions.md, docs/comprehension-history.md or docs/reading-cost.md (the plan, the decision log, past panels' records and past chunks' reading cost)" TEST_FILE = re.compile(r"(^|/)(tests?/|test_[^/]*$|[^/]*_test\.[^/]*$|[^/]*\.(test|spec)\.[^/]*$)") TEST_FILES = "test files being `test_*`, `*_test.*`, `*.test.*`, `*.spec.*` and those under a `test` or `tests` folder" -HEADING = re.compile(r"^(#+)\s+(.*?)\s*$") - - -def section(text: str, title: str) -> list[str] | None: - """The lines under the first heading whose text fullmatches the pattern title, up to the next heading of its level or above; None where none does.""" - found: list[str] | None = None - level = 0 - fenced = False - for line in text.splitlines(): - if FENCE.match(line): - fenced = not fenced - heading = None if fenced else HEADING.match(line) - if found is None: - if heading and re.fullmatch(title, heading.group(2)): - found, level = [], len(heading.group(1)) - elif heading and len(heading.group(1)) <= level: - break - else: - found.append(line) - return found def bullets(lines: list[str]) -> list[list[str]]: @@ -62,12 +42,18 @@ def bullets(lines: list[str]) -> list[list[str]]: return groups +ANCHOR = re.compile(r"- \*\*\d+ — ") +DIMENSIONS = ("Navigation", "Locality", "Shape", "Self-sufficiency") + + def score_bullets(skill: str) -> tuple[list[str], list[str]] | Failure: - """The six anchor bullets and the four dimension bullets of SKILL.md → "The score", without bold, each on one line.""" - groups = [[line.replace("**", "") for line in group] for group in bullets(section(skill, "The score") or [])] - if [len(group) for group in groups[:2]] != [6, 4]: - return Failure(f"{PANEL / 'SKILL.md'} → \"The score\" no longer opens with six anchor bullets and then four dimension bullets") - return groups[0], groups[1] + """The six anchor bullets and the four dimension bullets of SKILL.md → "The score", without bold, each on one line. A bullet is one by how it opens, `- ** — ` or `- ****`, so other bullets and their order in the section change nothing.""" + lines = [line.replace("**", "") for group in bullets(section(skill, "The score") or []) for line in group if ANCHOR.match(line) or line.startswith(tuple(f"- **{name}**" for name in DIMENSIONS))] + anchors = [line for line in lines if line[2].isdigit()] + dimensions = [line for line in lines if not line[2].isdigit()] + if len(anchors) != 6 or [line.split(" — ")[0] for line in dimensions] != [f"- {name}" for name in DIMENSIONS]: + return Failure(f"{PANEL / 'SKILL.md'} → \"The score\" no longer holds six bullets opening `- ** — ` and four opening `- ****` for Navigation, Locality, Shape and Self-sufficiency, in that order") + return anchors, dimensions def project_values(text: str, source: str) -> dict[str, str] | Failure: diff --git a/bin/larv_review/test_larv_review.py b/bin/larv_review/test_larv_review.py index 3c188c20..2e150352 100644 --- a/bin/larv_review/test_larv_review.py +++ b/bin/larv_review/test_larv_review.py @@ -872,6 +872,18 @@ class LarvReviewTool(unittest.TestCase): def test_review_folder_takes_no_arguments(self): self.assertIn("unknown arguments", self.run_tool("review-folder", "x").stderr) + def test_names_the_arguments_that_are_not_item_ids(self): + for args, named in ((("needs-scoring-run", "3", "x"), "x"), (("record-scoring", "scores.md", "4", "y1"), "y1")): + with self.subTest(args=args): + ran = self.run_tool(*args) + self.assertEqual(ran.returncode, 1) + self.assertIn(f"takes item IDs, the numbers in todo.md's ID column, not {named};", ran.stderr) + + def test_help_and_the_usage_line_list_the_same_commands(self): + listed = self.run_tool("--help").stdout.splitlines()[2].removeprefix("usage: larv-review ").split(" | ") + self.assertEqual([shape.split()[0] for shape in listed], ["review-folder", "ruling-base", "round-count", "needs-scoring-run", "scoring-run-owed", "record-ruling", "prepare-scoring", "record-scoring", "places", "agreement", "reviewer-header", "reviewer-context"]) + self.assertIn(f"usage: larv-review {' | '.join(listed)}", self.run_tool().stderr) + def test_takes_a_local_main_without_origin_and_reads_origin_as_last_fetched_when_the_fetch_fails(self): self.commit("Answer prose round 1: a") self.git("remote", "set-url", "origin", str(self.checkout.parent / "gone.git")) diff --git a/bin/larv_review/test_scoring_setup.py b/bin/larv_review/test_scoring_setup.py index 2e2311c3..eaace866 100644 --- a/bin/larv_review/test_scoring_setup.py +++ b/bin/larv_review/test_scoring_setup.py @@ -9,7 +9,7 @@ import unittest sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) -from larv_review.scoring_setup import bullets, CLAUDE_MD, file_globs, preloaded_files, prepare_scoring, project_values, removed_setup, score_bullets, section +from larv_review.scoring_setup import bullets, CLAUDE_MD, file_globs, preloaded_files, prepare_scoring, project_values, removed_setup, score_bullets from failure import Failure from test_env import isolated_env @@ -27,6 +27,17 @@ class ScoreBullets(unittest.TestCase): self.assertEqual([line.split(" — ")[0] for line in dimensions], ["- Navigation", "- Locality", "- Shape", "- Self-sufficiency"]) self.assertEqual(dimensions[1], "- Locality — understanding and changing one piece without holding the rest: hidden state, order dependence, action at a distance.") + def test_reads_the_bullets_by_how_they_open_whatever_else_the_section_holds(self): + skill = SKILL.read_text() + with_extra = skill.replace("## The score\n\n", "## The score\n\n- A bullet before the anchors.\n\n", 1) + self.assertEqual(score_bullets(with_extra), score_bullets(skill)) + + def test_fails_where_an_anchor_or_a_dimension_is_missing(self): + skill = SKILL.read_text() + for dropped in ("- **10 — Duress-proof.**", "- **Shape** —"): + with self.subTest(dropped=dropped): + self.assertIsInstance(score_bullets(skill.replace(dropped, "- Dropped:", 1)), Failure) + class ProjectValues(unittest.TestCase): def test_reads_each_spelling_the_projects_use(self): @@ -56,13 +67,6 @@ class FileGlobs(unittest.TestCase): self.assertEqual(file_globs(paths), "`bin/**` except test files, `docs/**`, `README.md` and `ci.sh`, test files being `test_*`, `*_test.*`, `*.test.*`, `*.spec.*` and those under a `test` or `tests` folder") -class Section(unittest.TestCase): - def test_holds_deeper_headings_and_fenced_lines_and_ends_at_its_own_level(self): - text = "# P\n\n## Scoring run\n\na\n\n### Sub\n\n```\n## fenced\n```\n\n## Next\n\nb\n" - self.assertEqual(section(text, "Scoring run"), ["", "a", "", "### Sub", "", "```", "## fenced", "```", ""]) - self.assertIsNone(section(text, "Missing")) - - class Bullets(unittest.TestCase): def test_joins_continuations_and_splits_runs_at_other_lines(self): lines = [" stray", "- a ", " more", "Text", "- b", "", "- c"] diff --git a/bin/loop_claude/cycle/run.py b/bin/loop_claude/cycle/run.py index 530084e3..d6dcf4af 100644 --- a/bin/loop_claude/cycle/run.py +++ b/bin/loop_claude/cycle/run.py @@ -181,7 +181,7 @@ def annotation_of(cycle: Cycle, transcript_path: str, entries: list[Entry], judg if isinstance(subagents, Failure): return subagents spent = spend.spent(since, entries, subagents, judge_of(cycle).model, judged.tokens) - return outcomes.Annotation(cycle.config, spend.spent_block(since, spent), judged.reversed) + return outcomes.Annotation(cycle.config, outcomes.spent_block(since, spent), judged.reversed) def ruling_outcome(cycle: Cycle, ruling: Interrupt | QuotedAnswer, quoted_answers: int, annotation: outcomes.Annotation) -> str: diff --git a/bin/loop_claude/cycle/spend.py b/bin/loop_claude/cycle/spend.py index e1db335a..68c125bf 100644 --- a/bin/loop_claude/cycle/spend.py +++ b/bin/loop_claude/cycle/spend.py @@ -7,16 +7,12 @@ import json import pathlib import typing -from claude_code.entries import AssistantTurn, Entry +from claude_code.entries import AssistantTurn, Entry, messages from failure import Failure -from routing_figures.prices import PRICES from claude_code.transcript import read_transcript -from transcript_archive.outcomes import SPENT +from transcript_archive.outcomes import Spender from claude_code.tokens import Tokens, tokens_of -KINDS = "input, cache write 5m, cache write 1h, cache read, output" -# Who spent, such as `worker`, `judge` or `Explore "Search the hooks"`, and the model they ran on. -Spender = tuple[str, str] WORKER = "worker" JUDGE = "judge" @@ -38,14 +34,10 @@ def spent(since: str, worker: list[Entry], subagents: list[Subagent], judge_mode def thread_spend(entries: list[Entry], since: str) -> dict[str, Tokens]: - """Each model's tokens in one transcript's messages written after since. A message's usage repeats in each entry of it, so each message counts once, by its id.""" + """Each model's tokens in one transcript's messages written after since. Each message counts once.""" start = moment(since) if since else None - counted: dict[str, AssistantTurn] = {} - for entry in entries: - if isinstance(entry, AssistantTurn) and entry.model and (start is None or later(entry.timestamp, start)): - counted.setdefault(entry.message_id or entry.uuid, entry) found: dict[str, Tokens] = {} - for turn in counted.values(): + for turn, *_ in messages(entry for entry in entries if isinstance(entry, AssistantTurn) and entry.model and (start is None or later(entry.timestamp, start))): found[turn.model] = found.get(turn.model, Tokens()).plus(tokens_of(turn.usage)) return found @@ -84,13 +76,3 @@ def subagent_name(transcript: pathlib.Path) -> str: # One line for the type, since a line break could forge an archive line such as `Config: `; json.dumps escapes the description's. kind, description = " ".join(str(meta.get("agentType") or "subagent").split()), str(meta.get("description") or "") return f"{kind} {json.dumps(description, ensure_ascii=False)}" if description else kind - - -def spent_block(since: str, found: dict[Spender, Tokens]) -> str: - """The archive's block for what the stretch spent, then the list prices in force for each model in it; empty where nothing was spent.""" - if not found: - return "" - spenders = [f"- {name}, {model}: {', '.join(map(str, (tokens.input, tokens.write_5m, tokens.write_1h, tokens.read, tokens.output)))}" for (name, model), tokens in found.items()] - priced = [f"- {model}: {price.input:g}, {price.write_5m:g}, {price.write_1h:g}, {price.read:g}, {price.output:g}" for model in dict.fromkeys(model for _, model in found) if (price := PRICES.get(model))] - prices = ["", f"List prices in dollars per million tokens of {KINDS}:", *priced] if priced else [] - return "\n".join([f"{SPENT}{since or 'the chunk began'}, in tokens of {KINDS}:", *spenders, *prices]) diff --git a/bin/loop_claude/test_loop_claude.py b/bin/loop_claude/test_loop_claude.py index 1b791e09..7232235b 100644 --- a/bin/loop_claude/test_loop_claude.py +++ b/bin/loop_claude/test_loop_claude.py @@ -1690,14 +1690,14 @@ class Spend(unittest.TestCase): def test_renders_the_spend_then_the_list_prices_of_each_model_it_knows(self): found = {("worker", "claude-opus-5-5"): Tokens(1, 2, 3, 4, 5), ("judge", "unpriced"): Tokens(input=6)} - self.assertEqual(spend.spent_block("2026-11-01T10:00:00Z", found), """Spent since 2026-11-01T10:00:00Z, in tokens of input, cache write 5m, cache write 1h, cache read, output: + self.assertEqual(outcomes.spent_block("2026-11-01T10:00:00Z", found), """Spent since 2026-11-01T10:00:00Z, in tokens of input, cache write 5m, cache write 1h, cache read, output: - worker, claude-opus-5-5: 1, 2, 3, 4, 5 - judge, unpriced: 6, 0, 0, 0, 0 List prices in dollars per million tokens of input, cache write 5m, cache write 1h, cache read, output: - claude-opus-5-5: 4, 5, 8, 0.2, 20""") - self.assertEqual(spend.spent_block("", {("judge", "unpriced"): Tokens(input=6)}), "Spent since the chunk began, in tokens of input, cache write 5m, cache write 1h, cache read, output:\n- judge, unpriced: 6, 0, 0, 0, 0") - self.assertEqual(spend.spent_block("", {}), "") + self.assertEqual(outcomes.spent_block("", {("judge", "unpriced"): Tokens(input=6)}), "Spent since the chunk began, in tokens of input, cache write 5m, cache write 1h, cache read, output:\n- judge, unpriced: 6, 0, 0, 0, 0") + self.assertEqual(outcomes.spent_block("", {}), "") def test_keeps_the_whole_typed_answer_that_holds_the_judges_quote_and_its_tokens(self): shown = "[human] Use A.\n[human] No, keep the old name. Use X.\nworking: keep the new name" diff --git a/bin/routing_figures/prices.py b/bin/model_routing/prices.py similarity index 100% rename from bin/routing_figures/prices.py rename to bin/model_routing/prices.py diff --git a/bin/reading_cost/measure.py b/bin/reading_cost/measure.py index 9bc83e80..346b2cee 100644 --- a/bin/reading_cost/measure.py +++ b/bin/reading_cost/measure.py @@ -2,7 +2,7 @@ import collections import dataclasses import re -from claude_code.entries import AssistantTurn, Entry, parse_entry, tool_calls, ToolCall, usage_count +from claude_code.entries import AssistantTurn, Entry, messages, parse_entry, tool_calls, ToolCall, usage_count from git_cli.git_command import commits from reading_cost.shell_reads import Reads, shell_reads from command_lines.shell_command import resolved, simple_commands @@ -46,8 +46,7 @@ def commit_time(worker: list[Entry], home: str) -> str | None: def tokens_in(turns: list[AssistantTurn]) -> int: - usages = {turn.message_id or turn.uuid: turn.usage for turn in turns} - return sum(usage_count(usage, "input_tokens") + usage_count(usage, "cache_creation_input_tokens") for usage in usages.values()) + return sum(usage_count(first.usage, "input_tokens") + usage_count(first.usage, "cache_creation_input_tokens") for first, *_ in messages(turns)) def reads_of(call: ToolCall, cwd: str, home: str) -> Reads: diff --git a/bin/records/markdown_text.py b/bin/records/markdown_text.py index e9e37733..91fa97cb 100644 --- a/bin/records/markdown_text.py +++ b/bin/records/markdown_text.py @@ -2,10 +2,12 @@ import re import typing FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})", re.M) +HEADING = re.compile(r"^(#+)\s+(.*?)\s*$") -def sections(text: str) -> dict[str, list[str]]: - blocks: list[tuple[str | None, list[str]]] = [] +def fenced_lines(text: str) -> list[tuple[str, bool]]: + """Each line of text with whether a code fence holds it, the fence's own lines included. A fence closes only on a line of its own marker, as long as the opener or longer.""" + found = [] opener = "" for line in text.splitlines(): if fence := FENCE.match(line): @@ -14,7 +16,13 @@ def sections(text: str) -> dict[str, list[str]]: opener = marker elif marker[0] == opener[0] and len(marker) >= len(opener) and not line[fence.end():].strip(): opener = "" - fenced = bool(opener or fence) + found.append((line, bool(opener or fence))) + return found + + +def sections(text: str) -> dict[str, list[str]]: + blocks: list[tuple[str | None, list[str]]] = [] + for line, fenced in fenced_lines(text): if not fenced and line.startswith("## "): blocks.append((line.strip()[3:], [])) elif not fenced and line.startswith("# "): @@ -28,6 +36,22 @@ def sections(text: str) -> dict[str, list[str]]: return found +def section(text: str, title: str) -> list[str] | None: + """The lines under the first heading whose text fullmatches the pattern title, up to the next heading of its level or above, blank lines kept; None where none does.""" + found: list[str] | None = None + level = 0 + for line, fenced in fenced_lines(text): + heading = None if fenced else HEADING.match(line) + if found is None: + if heading and re.fullmatch(title, heading.group(2)): + found, level = [], len(heading.group(1)) + elif heading and len(heading.group(1)) <= level: + break + else: + found.append(line) + return found + + # Blockquoted, so a passage holding blank lines still ends visibly where the text after it starts. def quoted(passage: str) -> str: return "\n".join(f"> {line}" for line in passage.splitlines()) diff --git a/bin/records/test_markdown_text.py b/bin/records/test_markdown_text.py index 92de02c5..d401a2be 100644 --- a/bin/records/test_markdown_text.py +++ b/bin/records/test_markdown_text.py @@ -21,6 +21,17 @@ class Sections(unittest.TestCase): self.assertEqual(markdown_text.quoted("merged\n\nPR #1"), "> merged\n> \n> PR #1") +class Section(unittest.TestCase): + def test_holds_deeper_headings_and_fenced_lines_and_ends_at_its_own_level(self): + text = "# P\n\n## Scoring run\n\na\n\n### Sub\n\n```\n## fenced\n```\n\n## Next\n\nb\n" + self.assertEqual(markdown_text.section(text, "Scoring run"), ["", "a", "", "### Sub", "", "```", "## fenced", "```", ""]) + self.assertIsNone(markdown_text.section(text, "Missing")) + + def test_a_fence_closes_only_on_its_own_marker_here_too(self): + text = "## A\n````md\n```\n## Quoted\n````\n## B\n" + self.assertEqual(markdown_text.section(text, "A"), ["````md", "```", "## Quoted", "````"]) + + class Listed(unittest.TestCase): def test_joins_names_with_commas_and_the_conjunction_before_the_last(self): self.assertEqual(markdown_text.listed(["a"], "and"), "a") diff --git a/bin/review_replay/main.py b/bin/review_replay/main.py index c05f0e16..8e422b75 100644 --- a/bin/review_replay/main.py +++ b/bin/review_replay/main.py @@ -50,47 +50,26 @@ review or match leaves its round out, except on a rung below the project's, wher nothing. Exit 1, saying why, on bad arguments, an unreadable archive or models file, no archived round to replay, or a usage limit. """ -import dataclasses -import json -import os import pathlib -import shutil import sys -import tempfile import typing from transcript_archive.location import archive_root -from rung_replay.command import arguments_or_exit, failed, limited_exit, projects_on -from review_replay.rounds import authored_by, comparison, lost_inputs, reviewed_path, Round, round_from, sample, stamp_seconds -from review_replay.recall import candidates, enough, needed, Recall, recall_report, Reference, RungRecall, tried -from transcript_archive.project_key import checkout_of -from claude_code.reply import claude_reply, Limited +from review_replay.rounds import archived_rounds, Round, sample +from rung_replay.command import arguments_or_exit, failed, limited_exit +from review_replay.replay import codes_found, findings_replayed +from review_replay.omit_comparison import compared from loop_effects.effects import Effects from failure import Failure -from rung_replay.search import first_reply, replayed_on -from claude_code.transcript import first_timestamped_line, read_transcript -from claude_code.entries import parse_entry, structured_output -from review_rules.reviewer_context import principles_line, reviewer_context +from claude_code.reply import Limited from model_routing.ladder import project_rung, Rung -from review_rules.review_folder import REPORTS, review_folder +from review_replay.recall import recall_report, recalled from review_rules.review_passes import ReviewedPass, REVIEWERS TOOL = "review-replay" OMIT = "--omit-claude-md" -# Starts claude without its instruction files, as `omitClaudeMd` starts a subagent. -NO_CLAUDE_MD = ["--settings", json.dumps({"env": {"CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1"}})] USAGE = f"{TOOL} [{OMIT}], one of {', '.join(REVIEWERS)} and a model on the routing ladder" LATEST = 10 -MATCH_SCHEMA = {"type": "object", "properties": {"found": {"type": "array", "items": {"type": "integer"}}}, "required": ["found"], "additionalProperties": False} - - -@dataclasses.dataclass(frozen=True) -class Code: - """Where a round's reviewer read the code, the commit it read, the default branch, and where that branch stood as the chunk began.""" - path: str - reviewed: str - default: str - base: str def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, effects: Effects) -> int: @@ -106,7 +85,11 @@ def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, eff return failed(TOOL, f"no archived {reviewed} round of projects on {project.model} in {archive_root(home, env)} holds a 🔴 or 🟡 on code it can find") chosen = sample(list(codes), LATEST) if without: - return compared(reviewed, project, chosen, lambda found, bare: findings_replayed(found, codes[found], project, project, home, effects, bare)) + report = compared(reviewed, project, chosen, lambda found, bare: findings_replayed(found, codes[found], project, project, home, effects, bare)) + if isinstance(report, Limited): + return limited_exit(TOOL, report) + print(report) + return 0 def replay_one(found: Round, rung: Rung) -> set[int] | Failure | Limited: matched = findings_replayed(found, codes[found], rung, project, home, effects, False) @@ -121,57 +104,6 @@ def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, eff return 0 -def recalled(chosen: list[Round], project: Rung, replay_one: typing.Callable[[Round, Rung], set[int] | Failure | Limited], what: str) -> Recall | Limited: - """Replays chosen twice on the project's rung, then each candidate, highest first, on the rounds neither of those failed on, until one doesn't qualify.""" - runs: list[list[set[int] | Failure]] = [] - for _ in range(2): - found: list[set[int] | Failure] | Limited = replayed_on(chosen, project, replay_one, what) - if isinstance(found, Limited): - return found - runs.append(found) - rounds: list[Round] = [] - reference: Reference = ([], []) - for one, first, second in zip(chosen, *runs): - if not isinstance(first, Failure) and not isinstance(second, Failure): - rounds.append(one) - reference[0].append(first) - reference[1].append(second) - recalls: list[RungRecall] = [] - lowest = None - for rung in candidates(project) if enough(rounds) and needed(rounds, reference) > 0 else []: - on_rung: list[set[int] | Failure] | Limited = replayed_on(rounds, rung, replay_one, what) - if isinstance(on_rung, Limited): - return on_rung - # A failed replay on a candidate found nothing, so it lowers that rung's count and never lifts it to what a rung needs. - one = tried(rung, rounds, reference, [set() if isinstance(found, Failure) else found for found in on_rung]) - recalls.append(one) - if not one.qualifies: - break - lowest = rung - return Recall(rounds, reference, len(chosen) - len(rounds), recalls, lowest) - - -def compared(reviewed: str, project: Rung, chosen: list[Round], found_on: typing.Callable[[Round, bool], set[int] | Failure | Limited]) -> int: - """Replays chosen three times with CLAUDE.md and once without, as found_on(round, bare) finds, and prints the comparison of the rounds no replay failed on.""" - runs: list[list[set[int] | Failure]] = [] - for bare in (False, False, False, True): - def replay_one(one: Round, _: Rung, bare: bool = bare) -> set[int] | Failure | Limited: - found = found_on(one, bare) - if isinstance(found, Failure): - print(f" {one.root} {one.at} {'without' if bare else 'with'} the instruction files failed, so the round is left out: {found.reason}", file=sys.stderr, flush=True) - return found - - found: list[set[int] | Failure] | Limited = replayed_on(chosen, project, replay_one, f"{reviewed} rounds {'without' if bare else 'with'} the instruction files") - if isinstance(found, Limited): - return limited_exit(TOOL, found) - runs.append(found) - # A failed replay found nothing, which would lower the bar where it had CLAUDE.md and count as misses where it had none. - kept = [(one, *found) for one, *found in zip(chosen, *runs) if not any(isinstance(each, Failure) for each in found)] - first, second, third, without = ([typing.cast(set[int], found[run]) for _, *found in kept] for run in range(4)) - print(comparison(reviewed, project, [one for one, *_ in kept], (first, second, third), without, len(chosen) - len(kept))) - return 0 - - def pass_and_model(argv: list[str]) -> tuple[ReviewedPass, Rung, bool] | None: """The pass, the project's rung of the model, and whether OMIT follows, as argv names them in ` [--omit-claude-md]`; None for any other argv.""" if argv[2:] not in ([], [OMIT]) or len(argv) < 2: @@ -179,180 +111,3 @@ def pass_and_model(argv: list[str]) -> tuple[ReviewedPass, Rung, bool] | None: reviewed = next((kind for kind in REVIEWERS if kind == argv[0]), None) project = project_rung(argv[1]) return (reviewed, project, argv[2:] == [OMIT]) if reviewed and project else None - - -def codes_found(rounds: list[Round], effects: Effects) -> dict[Round, Code]: - """The code each round reviewed, as code_of finds it; stderr names each round left out, and why.""" - codes = {} - for found in rounds: - if isinstance(code := code_of(found, effects), Failure): - print(f"{found.root} {found.at} is left out: {code.reason}", file=sys.stderr, flush=True) - else: - codes[found] = code - return codes - - -def archived_rounds(reviewer: str, model: str, root: pathlib.Path, home: pathlib.Path) -> list[Round] | Failure: - projects = projects_on(model, home) - if isinstance(projects, Failure): - return projects - found = [] - try: - for key in projects: - for chunk in sorted(path for path in (root / key).glob("*") if path.is_dir()): - found += chunk_rounds(reviewer, checkout_of(key, home), chunk) - except (OSError, UnicodeDecodeError, ValueError) as failure: - return Failure(f"could not read the archive {root}: {failure}") - return found - - -def chunk_rounds(reviewer: str, checkout: str, chunk: pathlib.Path) -> list[Round]: - copies = sorted(chunk.glob("*.jsonl")) - began = parse_entry(first_timestamped_line(copies[0].read_bytes())) if copies else None - if not began or not began.timestamp: - return [] - found = [] - for meta in sorted(chunk.glob("session/*/subagents/*.meta.json")): - described = json.loads(meta.read_text(encoding="utf-8")) - if not isinstance(described, dict) or described.get("agentType") != reviewer: - continue - entries = read_transcript(str(meta.with_name(meta.name.removesuffix(".meta.json") + ".jsonl"))) - if not isinstance(entries, Failure) and (one := round_from(reviewer, checkout, stamp_seconds(began.timestamp), entries)): - found.append(one) - return found - - -def code_of(found: Round, effects: Effects) -> Code | Failure: - """Where, and at which commits, found's reviewer read the code, as the archive lets it be found again: the folder rounds.reviewed_path guesses from the prompt, and the commits authored by then. - - Recording the reviewed commit at review time would need no such reading, but then no round archived before that change could be replayed. A wrong guess shows every rung the same wrong code: the project's rung finds fewer of that round's archived findings, which lowers the bar, and the rungs below find fewer too. - """ - if not os.path.isdir(found.root): - return Failure("the project has no checkout") - path = reviewed_path(found.prompt, found.root, os.path.exists) - if path is None: - return Failure("the prompt names neither one removed worktree nor the checkout") - default = effects.git("symbolic-ref", "--short", "refs/remotes/origin/HEAD", cwd=found.root) - if isinstance(default, Failure): - return default - default = default.strip().removeprefix("origin/") - log = effects.git("log", "--format=%H %at", f"refs/remotes/origin/{default}", cwd=found.root) - main_line = effects.git("log", "--first-parent", "--format=%H %at", f"refs/remotes/origin/{default}", cwd=found.root) - if isinstance(log, Failure): - return log - if isinstance(main_line, Failure): - return main_line - reviewed, base = authored_by(commit_log(log), found.seconds), authored_by(commit_log(main_line), found.began) - if reviewed is None or base is None: - return Failure("no commit was authored by then") - return Code(path, reviewed, default, base) - - -def commit_log(listed: str) -> list[tuple[str, float]]: - return [(commit, float(authored)) for commit, authored in (line.split() for line in listed.splitlines())] - - -def findings_replayed(found: Round, code: Code, rung: Rung, project: Rung, home: pathlib.Path, effects: Effects, bare: bool) -> set[int] | Failure | Limited: - """The indexes of the round's findings that its replay on rung reports, as claude on the project's rung matches them, or the Failure of the review or the match; where bare, the replay runs without CLAUDE.md, as `launch` says.""" - reviewed: str | Failure | Limited = first_reply(lambda: review(found, code, rung, home, effects, bare)) - if isinstance(reviewed, (Failure, Limited)): - return reviewed - return first_reply(lambda: match(found.findings, reviewed, found.root, project, effects)) - - -@dataclasses.dataclass(frozen=True) -class Copies: - """The throwaway copies a replay hands its reviewer: clone, of the project's checkout, and reports, of the chunk's review folder.""" - clone: str - reports: str - - -# The shell makes and removes the clone itself, and is tested on a temporary home as judge-replay's is; rounds.py's core takes the filesystem as an argument. -def review(found: Round, code: Code, rung: Rung, home: pathlib.Path, effects: Effects, bare: bool) -> str | Failure | Limited: - try: - (home / REPORTS).mkdir(mode=0o700, parents=True, exist_ok=True) - with tempfile.TemporaryDirectory(prefix="review-replay-", ignore_cleanup_errors=True) as clone, tempfile.TemporaryDirectory(prefix="review-replay-", dir=home / REPORTS, ignore_cleanup_errors=True) as reports: - return reviewed_in(found, code, rung, Copies(clone, reports), home, effects, bare) - except OSError as failure: - return Failure(f"could not prepare the replay: {failure}") - - -def reviewed_in(found: Round, code: Code, rung: Rung, copies: Copies, home: pathlib.Path, effects: Effects, bare: bool) -> str | Failure | Limited: - """The report of found's reviewer on rung, replayed on code cloned into copies.clone, with its review folder copied into copies.reports; without CLAUDE.md where bare.""" - folder = pathlib.Path(review_folder(str(home), code.path)) - copy_reports(found, folder, copies.reports) - prompt = redirected(found.prompt, home, folder, code.path, copies) - if isinstance(prompt, Failure): - return prompt - if lost := lost_inputs(prompt, copies.reports, lambda name: (pathlib.Path(copies.reports) / name).exists()): - return Failure(f"the prompt names {', '.join(lost)}, which the review folder did not hold from the chunk's start until the round's") - if isinstance(cloned := clone_reviewed(found, code, copies.clone, effects), Failure): - return cloned - launched = launch(found.reviewer, rung, prompt, copies.clone, home, bare) - if isinstance(launched, Failure): - return launched - reply = claude_reply(effects.claude, launched[0], copies.clone, launched[1], "claude") - if isinstance(reply, Failure | Limited): - return reply - return reply["result"] if isinstance(reply.get("result"), str) else Failure("the reviewer replied with no report") - - -def launch(reviewer: str, rung: Rung, prompt: str, clone: str, home: pathlib.Path, bare: bool) -> tuple[list[str], str] | Failure: - """claude's arguments and prompt for reviewer on rung. Where bare, it starts without CLAUDE.md, and the prompt holds what the SubagentStart hook and the header's last line hand such a reviewer.""" - args = ["--agent", reviewer, *rung.flags()] - if not bare: - return args, prompt - context = reviewer_context(home) - if isinstance(context, Failure): - return context - return [*args, *NO_CLAUDE_MD], "\n".join([f"{context}\n\n{prompt}", *([principles] if (principles := principles_line(clone)) else [])]) - - -def copy_reports(found: Round, folder: pathlib.Path, reports: str) -> None: - """Copies into reports each report written to folder after found's chunk began and before its round did.""" - # The round's own report, any later one, and one from an earlier attempt at the item in the same worktree would each hand the reviewer the findings it is to find. - for written in folder.iterdir() if folder.is_dir() else (): - if written.is_file() and found.began <= written.stat().st_mtime < found.seconds: - shutil.copy2(written, reports) - - -def redirected(prompt: str, home: pathlib.Path, folder: pathlib.Path, worktree: str, copies: Copies) -> str | Failure: - """prompt naming copies.reports in place of the review folder, in either spelling, and copies.clone in place of the worktree; a Failure where it still names the review folder.""" - # Replace the folder's path first: outside home it contains the worktree's path, which replacing the worktree's would otherwise turn into the clone's. - prompt = prompt.replace(str(folder), copies.reports).replace(f"~/{folder.relative_to(home)}", copies.reports).replace(worktree, copies.clone) - # A spelling the rewrite misses, such as $HOME/, would leave the reviewer the live folder and the findings in it. - if REPORTS in prompt.replace(copies.reports, ""): - return Failure("the prompt names the review folder in a spelling the replay does not redirect") - return prompt - - -def clone_reviewed(found: Round, code: Code, clone: str, effects: Effects) -> Failure | None: - """Clones found's checkout into clone, on the commit its reviewer read, with the default branch where it stood as the chunk began.""" - for command in ( - ("clone", "--quiet", "--shared", "--no-checkout", found.root, clone), - ("-C", clone, "update-ref", f"refs/remotes/origin/{code.default}", code.base), - ("-C", clone, "update-ref", f"refs/heads/{code.default}", code.base), - ("-C", clone, "checkout", "--quiet", *(("-B", found.branch) if found.branch and found.branch != code.default else ("--detach",)), code.reviewed), - ): - if isinstance(ran := effects.git(*command, cwd=found.root), Failure): - return ran - return None - - -def match(findings: tuple[str, ...], replay_report: str, root: str, rung: Rung, effects: Effects) -> set[int] | Failure | Limited: - numbered = "\n\n".join(f"{n}. {finding}" for n, finding in enumerate(findings, 1)) - prompt = ( - "A code reviewer reported the numbered findings below. Another reviewer then reviewed the same change; its report follows them. " - "List the number of each finding that the other report also raises as 🔴 or 🟡: the same defect, at the same place or from the same cause, however it is worded. " - "A finding it raises only as 🟢 or ❓, or not at all, is not listed.\n\n" - f"The findings:\n{numbered}\n\nThe other report:\n{replay_report}" - ) - args = ["--json-schema", json.dumps(MATCH_SCHEMA), *rung.flags(), "--tools", ""] - reply = claude_reply(effects.claude, args, root, prompt, "claude") - if isinstance(reply, Failure | Limited): - return reply - output = structured_output(reply) - numbers = output.get("found") if isinstance(output, dict) else None - if not isinstance(numbers, list) or not all(isinstance(n, int) for n in numbers): - return Failure(f"the match replied: {json.dumps(reply)}") - return {n - 1 for n in numbers if 1 <= n <= len(findings)} diff --git a/bin/review_replay/omit_comparison.py b/bin/review_replay/omit_comparison.py new file mode 100644 index 00000000..5bd4ac78 --- /dev/null +++ b/bin/review_replay/omit_comparison.py @@ -0,0 +1,56 @@ +"""The experiment behind review-replay's --omit-claude-md: three replays of a pass's rounds with CLAUDE.md, one without, and the report that holds the one without to what the others found: docs/decisions.md → `agents/: a reviewer starts without CLAUDE.md only where a replay of its pass missed no more findings than the most an unchanged replay missed`.""" +import sys +import typing + +from failure import Failure +from claude_code.reply import Limited +from rung_replay.search import replayed_on +from review_replay.rounds import Round +from model_routing.ladder import Rung + + +def compared(reviewed: str, project: Rung, chosen: list[Round], found_on: typing.Callable[[Round, bool], set[int] | Failure | Limited]) -> str | Limited: + """Replays chosen three times with CLAUDE.md and once without, as found_on(round, bare) finds, and reports the comparison of the rounds no replay failed on; the Limited where claude hit its usage limit.""" + runs: list[list[set[int] | Failure]] = [] + for bare in (False, False, False, True): + def replay_one(one: Round, _: Rung, bare: bool = bare) -> set[int] | Failure | Limited: + found = found_on(one, bare) + if isinstance(found, Failure): + print(f" {one.root} {one.at} {'without' if bare else 'with'} the instruction files failed, so the round is left out: {found.reason}", file=sys.stderr, flush=True) + return found + + found: list[set[int] | Failure] | Limited = replayed_on(chosen, project, replay_one, f"{reviewed} rounds {'without' if bare else 'with'} the instruction files") + if isinstance(found, Limited): + return found + runs.append(found) + # A failed replay found nothing, which would lower the bar where it had CLAUDE.md and count as misses where it had none. + kept = [(one, *found) for one, *found in zip(chosen, *runs) if not any(isinstance(each, Failure) for each in found)] + first, second, third, without = ([typing.cast(set[int], found[run]) for _, *found in kept] for run in range(4)) + return comparison(reviewed, project, [one for one, *_ in kept], (first, second, third), without, len(chosen) - len(kept)) + + +def comparison(reviewed: str, reference: Rung, rounds: list[Round], with_it: tuple[list[set[int]], list[set[int]], list[set[int]]], without: list[set[int]], left_out: int) -> str: + """What the three replays with CLAUDE.md and the one without found in rounds. Each replay with it missed some findings the other two found: the control, the noise of an unchanged reviewer. The one without it is held to the findings the first two found, as the third is, and passes where it missed no more than the most a replay with it did. left_out counts the rounds a replay failed on, which rounds leaves out.""" + first, second, third = with_it + + def shared(one: list[set[int]], other: list[set[int]]) -> list[set[int]]: + return [found_by_one & found_by_other for found_by_one, found_by_other in zip(one, other)] + + def missing(held: list[set[int]], run: list[set[int]]) -> list[tuple[Round, int]]: + return [(found_round, index) for found_round, held_in_round, found in zip(rounds, held, run) for index in sorted(held_in_round - found)] + + def count(run: list[set[int]]) -> int: + return sum(len(found) for found in run) + + both = shared(first, second) + missed = missing(both, without) + control = [len(missing(shared(one, other), run)) for run, one, other in ((first, second, third), (second, first, third), (third, first, second))] + passes = len(missed) <= max(control) + return "\n".join([ + f"Replayed {len(rounds)} {reviewed} rounds of {reference.model} projects, which hold {sum(len(found_round.findings) for found_round in rounds)} 🔴 and 🟡 findings, four times on {reference}: three with CLAUDE.md, one without.", + *([f"Left out {left_out} more, which a replay failed on; stderr says why."] if left_out else []), + f"With CLAUDE.md the three replays found {count(first)}, {count(second)} and {count(third)} of them, and missed {control[0]}, {control[1]} and {control[2]} of the findings that both of the other two found.", + *([f"Without CLAUDE.md it found {count(without)} of them, and missed {len(missed)} of the {count(both)} findings that both of the first two found{':' if missed else '.'}"] if count(both) else [f"Without CLAUDE.md it found {count(without)} of them. No evidence: the first two replays with CLAUDE.md found none in common, so there is no finding to hold it to."]), + *(f" {found_round.root} {found_round.at}: {found_round.findings[index].splitlines()[0]}" for found_round, index in missed), + *([f"It {'passes' if passes else 'fails'}: {len(missed)} missed is {'no more' if passes else 'more'} than the most a replay with CLAUDE.md missed, {max(control)}."] if count(both) else []), + ]) diff --git a/bin/review_replay/recall.py b/bin/review_replay/recall.py index 0e50519f..77fe957c 100644 --- a/bin/review_replay/recall.py +++ b/bin/review_replay/recall.py @@ -1,10 +1,14 @@ """The bar review-replay holds a rung to, and its report: docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`.""" import dataclasses from rung_replay.command import down_from +from failure import Failure from fractions import Fraction from review_replay.rounds import is_red, Round from model_routing.ladder import LADDER, Rung +from claude_code.reply import Limited import math +from rung_replay.search import replayed_on +import typing # A rung qualifies with a share of the archived findings up to this far below the project's rung's. The project's rung found 69 to 78% of its own findings again, and its two replays of the same rounds differed by up to 4 points. MARGIN = Fraction(1, 10) @@ -91,3 +95,33 @@ def recall_report(reviewed: str, project: Rung, recall: Recall) -> str: def percent(fraction: Fraction) -> str: return f"{float(fraction):.0%}" + + +def recalled(chosen: list[Round], project: Rung, replay_one: typing.Callable[[Round, Rung], set[int] | Failure | Limited], what: str) -> Recall | Limited: + """Replays chosen twice on the project's rung, then each candidate, highest first, on the rounds neither of those failed on, until one doesn't qualify.""" + runs: list[list[set[int] | Failure]] = [] + for _ in range(2): + found: list[set[int] | Failure] | Limited = replayed_on(chosen, project, replay_one, what) + if isinstance(found, Limited): + return found + runs.append(found) + rounds: list[Round] = [] + reference: Reference = ([], []) + for one, first, second in zip(chosen, *runs): + if not isinstance(first, Failure) and not isinstance(second, Failure): + rounds.append(one) + reference[0].append(first) + reference[1].append(second) + recalls: list[RungRecall] = [] + lowest = None + for rung in candidates(project) if enough(rounds) and needed(rounds, reference) > 0 else []: + on_rung: list[set[int] | Failure] | Limited = replayed_on(rounds, rung, replay_one, what) + if isinstance(on_rung, Limited): + return on_rung + # A failed replay on a candidate found nothing, so it lowers that rung's count and never lifts it to what a rung needs. + one = tried(rung, rounds, reference, [set() if isinstance(found, Failure) else found for found in on_rung]) + recalls.append(one) + if not one.qualifies: + break + lowest = rung + return Recall(rounds, reference, len(chosen) - len(rounds), recalls, lowest) diff --git a/bin/review_replay/replay.py b/bin/review_replay/replay.py new file mode 100644 index 00000000..5fc68bc6 --- /dev/null +++ b/bin/review_replay/replay.py @@ -0,0 +1,179 @@ +"""Replay one archived review round on one rung: the code the round reviewed, found again in a throwaway clone, the reviewer's report on it, and the match of that report to the round's archived findings: docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`.""" +import dataclasses +import json +import os +import pathlib +import shutil +import sys +import tempfile + +from review_replay.rounds import authored_by, lost_inputs, reviewed_path, Round +from claude_code.reply import claude_reply, Limited +from git_cli.git_repo import default_ref +from loop_effects.effects import Effects +from failure import Failure +from rung_replay.search import first_reply +from review_rules.reviewer_context import principles_line, reviewer_context +from review_rules.review_folder import REPORTS, review_folder +from model_routing.ladder import Rung +from claude_code.entries import structured_output + +# Starts claude without its instruction files, as `omitClaudeMd` starts a subagent. +NO_CLAUDE_MD = ["--settings", json.dumps({"env": {"CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1"}})] +MATCH_SCHEMA = {"type": "object", "properties": {"found": {"type": "array", "items": {"type": "integer"}}}, "required": ["found"], "additionalProperties": False} + + +@dataclasses.dataclass(frozen=True) +class Code: + """Where a round's reviewer read the code, the commit it read, the default branch, and where that branch stood as the chunk began.""" + path: str + reviewed: str + default: str + base: str + + +def codes_found(rounds: list[Round], effects: Effects) -> dict[Round, Code]: + """The code each round reviewed, as code_of finds it; stderr names each round left out, and why.""" + codes = {} + for found in rounds: + if isinstance(code := code_of(found, effects), Failure): + print(f"{found.root} {found.at} is left out: {code.reason}", file=sys.stderr, flush=True) + else: + codes[found] = code + return codes + + +def code_of(found: Round, effects: Effects) -> Code | Failure: + """Where, and at which commits, found's reviewer read the code, as the archive lets it be found again: the folder rounds.reviewed_path guesses from the prompt, and the commits authored by then. + + Recording the reviewed commit at review time would need no such reading, but then no round archived before that change could be replayed. A wrong guess shows every rung the same wrong code: the project's rung finds fewer of that round's archived findings, which lowers the bar, and the rungs below find fewer too. + """ + if not os.path.isdir(found.root): + return Failure("the project has no checkout") + path = reviewed_path(found.prompt, found.root, os.path.exists) + if path is None: + return Failure("the prompt names neither one removed worktree nor the checkout") + ref = default_ref(cwd=found.root, guarded=True, run=effects.git) + if isinstance(ref, Failure): + return ref + default = ref.removeprefix("origin/") + log = effects.git("log", "--format=%H %at", f"refs/remotes/origin/{default}", cwd=found.root) + main_line = effects.git("log", "--first-parent", "--format=%H %at", f"refs/remotes/origin/{default}", cwd=found.root) + if isinstance(log, Failure): + return log + if isinstance(main_line, Failure): + return main_line + reviewed, base = authored_by(commit_log(log), found.seconds), authored_by(commit_log(main_line), found.began) + if reviewed is None or base is None: + return Failure("no commit was authored by then") + return Code(path, reviewed, default, base) + + +def commit_log(listed: str) -> list[tuple[str, float]]: + return [(commit, float(authored)) for commit, authored in (line.split() for line in listed.splitlines())] + + +def findings_replayed(found: Round, code: Code, rung: Rung, project: Rung, home: pathlib.Path, effects: Effects, bare: bool) -> set[int] | Failure | Limited: + """The indexes of the round's findings that its replay on rung reports, as claude on the project's rung matches them, or the Failure of the review or the match; where bare, the replay runs without CLAUDE.md, as `launch` says.""" + reviewed: str | Failure | Limited = first_reply(lambda: review(found, code, rung, home, effects, bare)) + if isinstance(reviewed, (Failure, Limited)): + return reviewed + return first_reply(lambda: match(found.findings, reviewed, found.root, project, effects)) + + +@dataclasses.dataclass(frozen=True) +class Copies: + """The throwaway copies a replay hands its reviewer: clone, of the project's checkout, and reports, of the chunk's review folder.""" + clone: str + reports: str + + +# The shell makes and removes the clone itself, and is tested on a temporary home as judge-replay's is; rounds.py's core takes the filesystem as an argument. +def review(found: Round, code: Code, rung: Rung, home: pathlib.Path, effects: Effects, bare: bool) -> str | Failure | Limited: + try: + (home / REPORTS).mkdir(mode=0o700, parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="review-replay-", ignore_cleanup_errors=True) as clone, tempfile.TemporaryDirectory(prefix="review-replay-", dir=home / REPORTS, ignore_cleanup_errors=True) as reports: + return reviewed_in(found, code, rung, Copies(clone, reports), home, effects, bare) + except OSError as failure: + return Failure(f"could not prepare the replay: {failure}") + + +def reviewed_in(found: Round, code: Code, rung: Rung, copies: Copies, home: pathlib.Path, effects: Effects, bare: bool) -> str | Failure | Limited: + """The report of found's reviewer on rung, replayed on code cloned into copies.clone, with its review folder copied into copies.reports; without CLAUDE.md where bare.""" + folder = pathlib.Path(review_folder(str(home), code.path)) + copy_reports(found, folder, copies.reports) + prompt = redirected(found.prompt, home, folder, code.path, copies) + if isinstance(prompt, Failure): + return prompt + if lost := lost_inputs(prompt, copies.reports, lambda name: (pathlib.Path(copies.reports) / name).exists()): + return Failure(f"the prompt names {', '.join(lost)}, which the review folder did not hold from the chunk's start until the round's") + if isinstance(cloned := clone_reviewed(found, code, copies.clone, effects), Failure): + return cloned + launched = launch(found.reviewer, rung, prompt, copies.clone, home, bare) + if isinstance(launched, Failure): + return launched + reply = claude_reply(effects.claude, launched[0], copies.clone, launched[1], "claude") + if isinstance(reply, Failure | Limited): + return reply + return reply["result"] if isinstance(reply.get("result"), str) else Failure("the reviewer replied with no report") + + +def launch(reviewer: str, rung: Rung, prompt: str, clone: str, home: pathlib.Path, bare: bool) -> tuple[list[str], str] | Failure: + """claude's arguments and prompt for reviewer on rung. Where bare, it starts without CLAUDE.md, and the prompt holds what the SubagentStart hook and the header's last line hand such a reviewer.""" + args = ["--agent", reviewer, *rung.flags()] + if not bare: + return args, prompt + context = reviewer_context(home) + if isinstance(context, Failure): + return context + return [*args, *NO_CLAUDE_MD], "\n".join([f"{context}\n\n{prompt}", *([principles] if (principles := principles_line(clone)) else [])]) + + +def copy_reports(found: Round, folder: pathlib.Path, reports: str) -> None: + """Copies into reports each report written to folder after found's chunk began and before its round did.""" + # The round's own report, any later one, and one from an earlier attempt at the item in the same worktree would each hand the reviewer the findings it is to find. + for written in folder.iterdir() if folder.is_dir() else (): + if written.is_file() and found.began <= written.stat().st_mtime < found.seconds: + shutil.copy2(written, reports) + + +def redirected(prompt: str, home: pathlib.Path, folder: pathlib.Path, worktree: str, copies: Copies) -> str | Failure: + """prompt naming copies.reports in place of the review folder, in either spelling, and copies.clone in place of the worktree; a Failure where it still names the review folder.""" + # Replace the folder's path first: outside home it contains the worktree's path, which replacing the worktree's would otherwise turn into the clone's. + prompt = prompt.replace(str(folder), copies.reports).replace(f"~/{folder.relative_to(home)}", copies.reports).replace(worktree, copies.clone) + # A spelling the rewrite misses, such as $HOME/, would leave the reviewer the live folder and the findings in it. + if REPORTS in prompt.replace(copies.reports, ""): + return Failure("the prompt names the review folder in a spelling the replay does not redirect") + return prompt + + +def clone_reviewed(found: Round, code: Code, clone: str, effects: Effects) -> Failure | None: + """Clones found's checkout into clone, on the commit its reviewer read, with the default branch where it stood as the chunk began.""" + for command in ( + ("clone", "--quiet", "--shared", "--no-checkout", found.root, clone), + ("-C", clone, "update-ref", f"refs/remotes/origin/{code.default}", code.base), + ("-C", clone, "update-ref", f"refs/heads/{code.default}", code.base), + ("-C", clone, "checkout", "--quiet", *(("-B", found.branch) if found.branch and found.branch != code.default else ("--detach",)), code.reviewed), + ): + if isinstance(ran := effects.git(*command, cwd=found.root), Failure): + return ran + return None + + +def match(findings: tuple[str, ...], replay_report: str, root: str, rung: Rung, effects: Effects) -> set[int] | Failure | Limited: + numbered = "\n\n".join(f"{n}. {finding}" for n, finding in enumerate(findings, 1)) + prompt = ( + "A code reviewer reported the numbered findings below. Another reviewer then reviewed the same change; its report follows them. " + "List the number of each finding that the other report also raises as 🔴 or 🟡: the same defect, at the same place or from the same cause, however it is worded. " + "A finding it raises only as 🟢 or ❓, or not at all, is not listed.\n\n" + f"The findings:\n{numbered}\n\nThe other report:\n{replay_report}" + ) + args = ["--json-schema", json.dumps(MATCH_SCHEMA), *rung.flags(), "--tools", ""] + reply = claude_reply(effects.claude, args, root, prompt, "claude") + if isinstance(reply, Failure | Limited): + return reply + output = structured_output(reply) + numbers = output.get("found") if isinstance(output, dict) else None + if not isinstance(numbers, list) or not all(isinstance(n, int) for n in numbers): + return Failure(f"the match replied: {json.dumps(reply)}") + return {n - 1 for n in numbers if 1 <= n <= len(findings)} diff --git a/bin/review_replay/rounds.py b/bin/review_replay/rounds.py index 2d89d8b4..d2f5866b 100644 --- a/bin/review_replay/rounds.py +++ b/bin/review_replay/rounds.py @@ -1,12 +1,16 @@ -"""Which archived rounds of a review pass to replay, the code each reviewed, and the report with --omit-claude-md: docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`.""" +"""Which archived rounds of a review pass to replay, and the code each reviewed: docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`.""" import dataclasses import datetime +import json import pathlib import re import typing -from claude_code.entries import AssistantTurn, Entry, Text, UserText -from model_routing.ladder import Rung +from claude_code.entries import AssistantTurn, Entry, parse_entry, Text, UserText +from transcript_archive.project_key import checkout_of +from failure import Failure +from claude_code.transcript import first_timestamped_line, read_transcript +from rung_replay.command import projects_on # A line that opens with a 🔴 or 🟡 mark, after any list, table, heading or bold markup, as review-findings.md has reviewers report them. FINDING = re.compile(r"^[\s#>*+|-]*(?:\d+[.)]\s*)?[*_]*\s*(?:🔴|🟡)") @@ -107,28 +111,31 @@ def sample(rounds: list[Round], latest: int) -> list[Round]: return sorted(rounds, key=lambda found: (found.seconds, found.root))[-latest:] -def comparison(reviewed: str, reference: Rung, rounds: list[Round], with_it: tuple[list[set[int]], list[set[int]], list[set[int]]], without: list[set[int]], left_out: int) -> str: - """What the three replays with CLAUDE.md and the one without found in rounds. Each replay with it missed some findings the other two found: the control, the noise of an unchanged reviewer. The one without it is held to the findings the first two found, as the third is, and passes where it missed no more than the most a replay with it did. left_out counts the rounds a replay failed on, which rounds leaves out.""" - first, second, third = with_it +def archived_rounds(reviewer: str, model: str, root: pathlib.Path, home: pathlib.Path) -> list[Round] | Failure: + projects = projects_on(model, home) + if isinstance(projects, Failure): + return projects + found = [] + try: + for key in projects: + for chunk in sorted(path for path in (root / key).glob("*") if path.is_dir()): + found += chunk_rounds(reviewer, checkout_of(key, home), chunk) + except (OSError, UnicodeDecodeError, ValueError) as failure: + return Failure(f"could not read the archive {root}: {failure}") + return found - def shared(one: list[set[int]], other: list[set[int]]) -> list[set[int]]: - return [found_by_one & found_by_other for found_by_one, found_by_other in zip(one, other)] - def missing(held: list[set[int]], run: list[set[int]]) -> list[tuple[Round, int]]: - return [(found_round, index) for found_round, held_in_round, found in zip(rounds, held, run) for index in sorted(held_in_round - found)] - - def count(run: list[set[int]]) -> int: - return sum(len(found) for found in run) - - both = shared(first, second) - missed = missing(both, without) - control = [len(missing(shared(one, other), run)) for run, one, other in ((first, second, third), (second, first, third), (third, first, second))] - passes = len(missed) <= max(control) - return "\n".join([ - f"Replayed {len(rounds)} {reviewed} rounds of {reference.model} projects, which hold {sum(len(found_round.findings) for found_round in rounds)} 🔴 and 🟡 findings, four times on {reference}: three with CLAUDE.md, one without.", - *([f"Left out {left_out} more, which a replay failed on; stderr says why."] if left_out else []), - f"With CLAUDE.md the three replays found {count(first)}, {count(second)} and {count(third)} of them, and missed {control[0]}, {control[1]} and {control[2]} of the findings that both of the other two found.", - *([f"Without CLAUDE.md it found {count(without)} of them, and missed {len(missed)} of the {count(both)} findings that both of the first two found{':' if missed else '.'}"] if count(both) else [f"Without CLAUDE.md it found {count(without)} of them. No evidence: the first two replays with CLAUDE.md found none in common, so there is no finding to hold it to."]), - *(f" {found_round.root} {found_round.at}: {found_round.findings[index].splitlines()[0]}" for found_round, index in missed), - *([f"It {'passes' if passes else 'fails'}: {len(missed)} missed is {'no more' if passes else 'more'} than the most a replay with CLAUDE.md missed, {max(control)}."] if count(both) else []), - ]) +def chunk_rounds(reviewer: str, checkout: str, chunk: pathlib.Path) -> list[Round]: + copies = sorted(chunk.glob("*.jsonl")) + began = parse_entry(first_timestamped_line(copies[0].read_bytes())) if copies else None + if not began or not began.timestamp: + return [] + found = [] + for meta in sorted(chunk.glob("session/*/subagents/*.meta.json")): + described = json.loads(meta.read_text(encoding="utf-8")) + if not isinstance(described, dict) or described.get("agentType") != reviewer: + continue + entries = read_transcript(str(meta.with_name(meta.name.removesuffix(".meta.json") + ".jsonl"))) + if not isinstance(entries, Failure) and (one := round_from(reviewer, checkout, stamp_seconds(began.timestamp), entries)): + found.append(one) + return found diff --git a/bin/review_replay/test_review_replay.py b/bin/review_replay/test_review_replay.py index 4c95f8cd..a7df700a 100644 --- a/bin/review_replay/test_review_replay.py +++ b/bin/review_replay/test_review_replay.py @@ -18,7 +18,7 @@ from review_replay.rounds import authored_by, lost_inputs, recorded_findings, re from loop_effects.effects import Effects, system_effects from claude_code.entries import entry_from from failure import Failure -from review_replay import main, recall +from review_replay import main, recall, replay from review_rules.reviewer_context import reviewer_context from review_rules.review_passes import REVIEWERS from model_routing.ladder import Rung @@ -124,17 +124,17 @@ class Launch(unittest.TestCase): self.clone.mkdir() def test_runs_the_reviewer_as_archived_with_claude_md(self): - self.assertEqual(main.launch("prose-reviewer", REFERENCE, "Review it.", str(self.clone), self.home, False), (["--agent", "prose-reviewer", *REFERENCE.flags()], "Review it.")) + self.assertEqual(replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.clone), self.home, False), (["--agent", "prose-reviewer", *REFERENCE.flags()], "Review it.")) def test_drops_claude_md_and_hands_the_reviewer_what_the_hook_and_the_header_would(self): (self.clone / "AGENTS.md").write_text("") - args, prompt = main.launch("prose-reviewer", REFERENCE, "Review it.", str(self.clone), self.home, True) + args, prompt = replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.clone), self.home, True) self.assertEqual(args, ["--agent", "prose-reviewer", *REFERENCE.flags(), "--settings", json.dumps({"env": {"CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1"}})]) self.assertEqual(prompt, f"{reviewer_context(self.home)}\n\nReview it.\nPrinciples: read {self.clone}/AGENTS.md first, for the project principles and the decision titles.") def test_fails_without_the_sections_to_hand_over(self): (self.home / ".claude" / "CLAUDE.md").write_text("") - self.assertIsInstance(main.launch("prose-reviewer", REFERENCE, "Review it.", str(self.clone), self.home, True), Failure) + self.assertIsInstance(replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.clone), self.home, True), Failure) class Match(unittest.TestCase): @@ -145,14 +145,14 @@ class Match(unittest.TestCase): self.asked.append((args, cwd, prompt)) return subprocess.CompletedProcess(args, 0, stdout, "") - return main.match(("🔴 a", "🟡 b"), "the other report", "/code/p", REFERENCE, dataclasses.replace(system_effects(), claude=claude)) + return replay.match(("🔴 a", "🟡 b"), "the other report", "/code/p", REFERENCE, dataclasses.replace(system_effects(), claude=claude)) def test_asks_for_the_numbered_findings_the_other_report_raises_as_json_with_no_tools(self): self.run_match("{}") [(args, cwd, prompt)] = self.asked self.assertEqual(cwd, "/code/p") self.assertIn("1. 🔴 a\n\n2. 🟡 b\n\nThe other report:\nthe other report", prompt) - self.assertEqual(json.loads(args[args.index("--json-schema") + 1]), main.MATCH_SCHEMA) + self.assertEqual(json.loads(args[args.index("--json-schema") + 1]), replay.MATCH_SCHEMA) self.assertEqual(args[args.index("--tools") + 1], "") self.assertEqual(args[args.index("--model"):args.index("--model") + 4], REFERENCE.flags()) @@ -498,7 +498,7 @@ class Main(unittest.TestCase): def test_leaves_out_a_round_whose_checkout_names_no_default_branch(self): run_git(self.checkout, "remote", "set-head", "origin", "-d") - self.assert_left_out("git symbolic-ref --short refs/remotes/origin/HEAD: ") + self.assert_left_out("origin records no default branch") def test_leaves_out_a_round_whose_checkout_is_gone(self): shutil.rmtree(self.checkout) diff --git a/bin/review_rules/reviewer_context.py b/bin/review_rules/reviewer_context.py index 39d31ed3..46df0598 100644 --- a/bin/review_rules/reviewer_context.py +++ b/bin/review_rules/reviewer_context.py @@ -1,9 +1,11 @@ """What a reviewer under agents/ is handed in place of the instruction files: `omitClaudeMd` starts it without them, and most of `~/.claude/CLAUDE.md` is the worker's. docs/decisions.md → `agents/: a reviewer starts without CLAUDE.md only where a replay of its pass missed no more findings than the most an unchanged replay missed` says why, and which reviewers.""" import json import pathlib +import re from failure import Failure from review_rules.review_passes import REVIEWERS +from records.markdown_text import section # The sections of ~/.claude/CLAUDE.md a reviewer needs, in the file's order: the code-health goal the agents cite, the bar its own prose meets, and the words its reports use. SECTIONS = ("Shared goal", "Prose", "Vocabulary") @@ -16,15 +18,10 @@ def reviewer_context(home: pathlib.Path) -> str | Failure: text = path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError) as failure: return Failure(f"could not read {path}: {failure}") - found: dict[str, list[str]] = {} - lines: list[str] = [] - for line in text.splitlines(): - if line.startswith("## "): - lines = found.setdefault(line[3:].strip(), []) - lines.append(line) - if missing := [name for name in SECTIONS if name not in found]: + found = {name: section(text, re.escape(name)) for name in SECTIONS} + if missing := [name for name, lines in found.items() if lines is None]: return Failure(f"{path} holds no section {', '.join(missing)}") - return "\n\n".join(["You start without `~/.claude/CLAUDE.md`. These sections of it hold for you:", *("\n".join(found[name]).strip() for name in SECTIONS)]) + return "\n\n".join(["You start without `~/.claude/CLAUDE.md`. These sections of it hold for you:", *("\n".join([f"## {name}", *lines]).strip() for name, lines in found.items() if lines is not None)]) def starts_bare(agent: str, home: pathlib.Path) -> bool: diff --git a/bin/review_rules/test_reviewer_context.py b/bin/review_rules/test_reviewer_context.py index c6b8f8bf..318d3761 100644 --- a/bin/review_rules/test_reviewer_context.py +++ b/bin/review_rules/test_reviewer_context.py @@ -26,6 +26,10 @@ class ReviewerContext(unittest.TestCase): def test_holds_the_shared_goal_prose_and_vocabulary_sections_and_none_of_the_workers(self): self.assertEqual(reviewer_context(self.home), "You start without `~/.claude/CLAUDE.md`. These sections of it hold for you:\n\n## Shared goal\n\nCode health.\n\n## Prose\n\nWrite short.\n\n## Vocabulary\n\n- **chunk** — a unit.") + def test_a_heading_inside_a_code_fence_does_not_end_a_section(self): + (self.home / ".claude" / "CLAUDE.md").write_text("## Shared goal\n\n```\n## Not a heading\n```\n\nstill the goal\n\n## Prose\n\np\n\n## Vocabulary\n\nv\n") + self.assertIn("## Shared goal\n\n```\n## Not a heading\n```\n\nstill the goal\n\n## Prose", reviewer_context(self.home)) + def test_fails_on_a_missing_section_or_file(self): (self.home / ".claude" / "CLAUDE.md").write_text("## Prose\n\nWrite short.\n") self.assertEqual(reviewer_context(self.home), Failure(f"{self.home / '.claude' / 'CLAUDE.md'} holds no section Shared goal, Vocabulary")) diff --git a/bin/routing_figures/dollars.py b/bin/routing_figures/dollars.py index f360efcd..ea1aa429 100644 --- a/bin/routing_figures/dollars.py +++ b/bin/routing_figures/dollars.py @@ -3,8 +3,8 @@ import collections import dataclasses import typing -from claude_code.entries import AssistantTurn, CostState, Entry, ToolCall -from routing_figures.prices import listed +from claude_code.entries import messages as claude_messages, CostState, Entry, ToolCall +from model_routing.prices import listed DIMENSIONS = ("project", "subagent type", "rung", "tool") # Each of DIMENSIONS to its labels' dollars. @@ -33,17 +33,12 @@ class Spent(typing.NamedTuple): def messages(entries: list[Entry]) -> list[Message]: """The assistant messages, counted once each, each with the tools whose results it took in, or `prompt` where it took none in.""" - # A message's usage is the same in each of its entries (none of 1,681 multi-entry messages in the archive of 2026-10-28 differed), so the first is kept, where reading_cost's tokens_in keeps the last. - kept: dict[str, tuple[AssistantTurn, list[str]]] = {} - for entry in entries: - if isinstance(entry, AssistantTurn): - found = kept.setdefault(entry.message_id or entry.uuid, (entry, [])) - found[1].extend(part.name for part in entry.parts if isinstance(part, ToolCall)) counted = [] before = [PROMPT] - for turn, called in kept.values(): + for turns in claude_messages(entries): + turn = turns[0] counted.append(Message(turn.timestamp, before, turn.model, listed(turn.model, turn.usage))) - before = called or [PROMPT] + before = [part.name for each in turns for part in each.parts if isinstance(part, ToolCall)] or [PROMPT] return counted diff --git a/bin/transcript_archive/outcomes.py b/bin/transcript_archive/outcomes.py index cef18a3b..41f8f3de 100644 --- a/bin/transcript_archive/outcomes.py +++ b/bin/transcript_archive/outcomes.py @@ -1,8 +1,10 @@ import re import typing +from model_routing.prices import PRICES from records.markdown_text import quoted from records.utc_stamp import STAMPED +from claude_code.tokens import Tokens from model_routing.route import WorkerLine TASK = "Task: " @@ -33,6 +35,9 @@ JUDGE_COST_AT = re.compile(rf"^{re.escape(JUDGE)}(.+)\n{re.escape(JUDGE_COST)}([ CONFIG = "Config: " # Opens the block of what the work since the previous outcome that recorded a spend spent; an outcome that spent nothing has none. SPENT = "Spent since " +KINDS = "input, cache write 5m, cache write 1h, cache read, output" +# Who spent, such as `worker`, `judge` or `Explore "Search the hooks"`, and the model they ran on. +Spender = tuple[str, str] # Opens the quote of a typed answer of the human's that reversed a choice the worker took on its own recommendation. REVERSED = "Reversed:" REVERSED_AT = re.compile(rf"^{re.escape(REVERSED)}\n((?:> .*\n)*> .*)", re.MULTILINE) @@ -41,7 +46,7 @@ REVERSED_AT = re.compile(rf"^{re.escape(REVERSED)}\n((?:> .*\n)*> .*)", re.MULTI class Annotation(typing.NamedTuple): """What a verdict, an interrupt or a quoted answer records beside its judge lines, so a change to the setup can be judged before and after: docs/decisions.md → `loop-claude: each verdict records the claude-config release and commit that ran it…`. Each part is left out where empty; the cycle always gives config.""" config: str = "" - # The block spend.spent_block renders. + # The block spent_block renders. spent: str = "" # The human's whole typed answer that holds the judge's quote of the reversal, as loop-claude checked it. reversed: str = "" @@ -54,6 +59,17 @@ def annotation_lines(annotation: Annotation) -> str: return f"{reversed_block}{spent_block}{config_line}" +# Nothing reads a Spent block yet: the first tool that derives dollars from one adds its parser here, beside this writer. +def spent_block(since: str, found: dict[Spender, Tokens]) -> str: + """The archive's block for what the stretch spent, then the list prices in force for each model in it; empty where nothing was spent.""" + if not found: + return "" + spenders = [f"- {name}, {model}: {', '.join(map(str, (tokens.input, tokens.write_5m, tokens.write_1h, tokens.read, tokens.output)))}" for (name, model), tokens in found.items()] + priced = [f"- {model}: {price.input:g}, {price.write_5m:g}, {price.write_1h:g}, {price.read:g}, {price.output:g}" for model in dict.fromkeys(model for _, model in found) if (price := PRICES.get(model))] + prices = ["", f"List prices in dollars per million tokens of {KINDS}:", *priced] if priced else [] + return "\n".join([f"{SPENT}{since or 'the chunk began'}, in tokens of {KINDS}:", *spenders, *prices]) + + def reversals(outcomes: str) -> list[str]: """Each reversing answer the outcomes quote, unquoted, in the order they were written.""" return [unquoted(found[1]) for found in REVERSED_AT.finditer(outcomes)] diff --git a/docs/decisions.md b/docs/decisions.md index 73600a8a..2e82f00b 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -144,7 +144,7 @@ Valid while `~/.claude` is a checkout of this repo. 2026-11-12, lilleman, todo item 305. Serves goals 1, 1.2 and 1.4. A change to the setup can be judged only by comparing chunks run before it with chunks run after, so each verdict, interrupt and quoted answer in loop-claude's archive records three things beside its judge lines. - `Config: ()`: `git describe` and the commit of `~/.claude`, so outcomes group by the setup that produced them. Where git can't tell, it records `unknown` and why, on one line. -- A `Spent since` block: the raw tokens each thread spent, by model and by kind (input, cache write 5m, cache write 1h, cache read, output), then the list prices in force, from `bin/routing_figures/prices.py`. The threads are the worker's main thread, each subagent by its agent type and description, and the judge run that ruled. Each message counts once, by its id. The stretch runs from the previous outcome that recorded a spend, so what a `/talk` or a stopped run spent between rulings is counted once. It keeps tokens, since prices change. The prices recorded beside them give what the stretch cost when it ran. Claude Code doesn't report thinking apart from output, so it is left in output. The judge's tokens come from its json reply's `usage`, until item 303 archives the judge's own transcript. +- A `Spent since` block: the raw tokens each thread spent, by model and by kind (input, cache write 5m, cache write 1h, cache read, output), then the list prices in force, from `bin/model_routing/prices.py`. The threads are the worker's main thread, each subagent by its agent type and description, and the judge run that ruled. Each message counts once, by its id. The stretch runs from the previous outcome that recorded a spend, so what a `/talk` or a stopped run spent between rulings is counted once. It keeps tokens, since prices change. The prices recorded beside them give what the stretch cost when it ran. Claude Code doesn't report thinking apart from output, so it is left in output. The judge's tokens come from its json reply's `usage`, until item 303 archives the judge's own transcript. - A `Reversed:` quote: the human's typed answer that overrides a choice the worker took on its own recommendation and recorded. The judge quotes the latest such `[human]` line; loop-claude keeps the whole typed answer that holds the quote verbatim, and drops a quote no typed answer holds. `score-outcomes` counts the distinct answers per finished chunk. That count is goal 1.2's counter-signal, and the 2026-11-08 entry of `loop-claude: a judge decides each interrupt…` is valid only while it stays low. It counts only chunks that record a Config line, since earlier judges didn't look for reversals. Valid while score-outcomes reads the reversals from the archive. diff --git a/reviewer-merge-trials/analyze_metered.py b/reviewer-merge-trials/analyze_metered.py index cf6718df..9eadf409 100644 --- a/reviewer-merge-trials/analyze_metered.py +++ b/reviewer-merge-trials/analyze_metered.py @@ -1,7 +1,7 @@ """Run analyze.py's matching and new-finding count, and also sum what those calls cost. Usage: [BASELINE=] analyze_metered.py -... [case ids...]""" import pathlib, runpy, sys, threading sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) -import analyze, review_replay.main as rm +import analyze, review_replay.replay as rm from claude_code.reply import claude_reply SPENT = [0.0]; LOCK = threading.Lock() def metered(*a, **k): diff --git a/reviewer-merge-trials/driver.py b/reviewer-merge-trials/driver.py index 9dfccb36..b6edc462 100644 --- a/reviewer-merge-trials/driver.py +++ b/reviewer-merge-trials/driver.py @@ -2,11 +2,11 @@ import concurrent.futures, json, os, pathlib, pickle, re, shutil, sys, tempfile sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "bin")) from claude_code.reply import claude_reply, Limited -from review_replay.main import clone_reviewed, Copies, copy_reports, match, redirected +from review_replay.replay import clone_reviewed, Copies, copy_reports, match, redirected from failure import Failure from review_replay.rounds import lost_inputs, Round from review_rules.review_folder import REPORTS, review_folder -from review_replay import main as rm +from review_replay import replay as rm from model_routing.ladder import Rung from loop_effects.effects import system_effects diff --git a/skills/larv-review/SKILL.md b/skills/larv-review/SKILL.md index 33d6b73d..e75b0b57 100644 --- a/skills/larv-review/SKILL.md +++ b/skills/larv-review/SKILL.md @@ -47,6 +47,8 @@ A pass is one kind of review, a row of the table, named by its first column. Its | Product-owner | product-owner-reviewer | changes an external surface or contract: API routes and request/response shapes, events, webhooks, published file formats, CLI flags and exit codes, UI, error or deprecation text a consumer reads, README and docs an outside reader lands on | | Prose | prose-reviewer | changes prose a reader reads, such as docs, comments, help or error text, or the plan's text for an item the chunk ships, splits or answers; one round | +The table's first two columns are also code: `PASS_AGENTS` in `bin/review_rules/review_passes.py`, which `bin/repo_checks/test_reviewer_launches.py` holds to the table. Change a row in both. + A comment is prose, not code, in any file, and so is a docstring. A change to config is one that changes what runs, so a comment in a workflow or compose file fires only the prose pass. Markdown an agent follows is config: `CLAUDE.md`, `AGENTS.md`, `workflow-bench/*-prompt.md`, and files under `agents/`, `principles/`, `rules/` or `skills/`. **Plan rows.** A `todo.md` row or detail that the chunk files from a finding, re-scores or moves between releases gets no review: the chunk that ships the item reads it.