Answer architecture round 1: 1 🔴, 8 🟡 (0 misread); one reader per message, prices and Spent block moved, one table of larv-review commands

This commit is contained in:
2026-11-12 20:05:52 +01:00
parent 9066f477a4
commit 5c37027046
28 changed files with 503 additions and 412 deletions
+1 -1
View File
@@ -115,7 +115,7 @@ def checkout(run: Run) -> tuple[Forge, str, str] | Failure:
def default_branch(run: Run) -> str | Failure:
"""origin's default branch, such as main."""
"""origin's default branch, such as main. Not git_cli's default_ref: that falls back to a local main or master and asks origin where nothing is recorded, but a pull request's base is origin's alone, and this tells the user how to record it."""
default = run(["git", "symbolic-ref", "--short", "refs/remotes/origin/HEAD"])
if isinstance(default, Failure):
return Failure(f"origin records no default branch; set it with `git remote set-head origin --auto`: {default.reason}")
+9
View File
@@ -145,6 +145,15 @@ def tool_calls(entries: typing.Iterable[Entry]) -> list[tuple[AssistantTurn, Too
return [(entry, part) for entry in entries if isinstance(entry, AssistantTurn) for part in entry.parts if isinstance(part, ToolCall)]
def messages(entries: typing.Iterable[Entry]) -> list[list[AssistantTurn]]:
"""The assistant turns grouped by message, in the order each message first appears. A message's usage repeats in each of its entries, so read it from the first of a group: none of 1,681 multi-entry messages in the archive of 2026-10-28 differed."""
grouped: dict[str, list[AssistantTurn]] = {}
for entry in entries:
if isinstance(entry, AssistantTurn):
grouped.setdefault(entry.message_id or entry.uuid, []).append(entry)
return list(grouped.values())
def field(found: dict, key: str) -> str:
return value if isinstance(value := found.get(key), str) else ""
+7
View File
@@ -148,6 +148,13 @@ class EntryKinds(unittest.TestCase):
uuid="u", parts=(entries.Text("Starting."), entries.ToolCall("toolu_1", "Bash", {"command": "ls"})), message_id="msg_1", model="claude-opus-5-5", usage={"input_tokens": 2},
))
def test_messages_group_the_turns_that_share_a_message_id_and_fall_back_to_the_uuid_where_there_is_none(self):
first = entries.AssistantTurn(uuid="a", message_id="msg_1", usage={"input_tokens": 1})
other = entries.AssistantTurn(uuid="b", message_id="msg_2")
again = entries.AssistantTurn(uuid="c", message_id="msg_1", usage={"input_tokens": 2})
bare = entries.AssistantTurn(uuid="d")
self.assertEqual(entries.messages([first, entries.HumanTurn(text="hi"), other, again, bare]), [[first, again], [other], [bare]])
def test_parses_tool_results_and_cost_states_and_reads_queued_notifications_and_other_kinds_as_plain_entries(self):
self.assertEqual(entries.entry_from(human([{"type": "tool_result", "tool_use_id": "toolu_1", "content": "hi", "is_error": True}])), entries.ToolResults(results=(entries.ToolResult("toolu_1", "hi", True),)))
self.assertEqual(entries.entry_from(queued("<task-notification>done</task-notification>", "task-notification")), entries.Entry())
+45 -33
View File
@@ -1,6 +1,6 @@
"""Hold larv-review's mechanical rules: the chunk's review folder, the paired ruling's base and vote count, each pass's rounds, the scoring run's trigger and setup; the comprehension history's format, its latest places and its count of how often seats agree; and the header of each reviewer's prompt.
usage: larv-review review-folder | ruling-base | round-count | needs-scoring-run [<item ID>...] | scoring-run-owed | record-ruling <votes file> | prepare-scoring [--preloaded <text>] | record-scoring <scores file> [<item ID>...] | places | agreement | reviewer-header <pass> [--rerun] | reviewer-context
usage: larv-review {commands}
The comprehension check records how hard a project is to read, two ways, and no score blocks a merge. In a
paired ruling, simulated readers, the seats, read the branch's head beside a base and vote better,
@@ -167,45 +167,57 @@ from larv_review.scoring_setup import prepare_scoring
from larv_review.history_commands import record_ruling, record_scoring, show_agreement, show_places
from larv_review.reviewer_header import reviewer_header
USAGE = "usage: larv-review review-folder | ruling-base | round-count | needs-scoring-run [<item ID>...] | scoring-run-owed | record-ruling <votes file> | prepare-scoring [--preloaded <text>] | record-scoring <scores file> [<item ID>...] | places | agreement | reviewer-header <pass> [--rerun] | reviewer-context"
class Command(typing.NamedTuple):
"""One subcommand: how the usage line spells it, whether the arguments after its name fit, and what it does with them."""
shape: str
fits: typing.Callable[[list[str]], bool]
run: typing.Callable[[list[str]], Failure | None]
def no_arguments(given: list[str]) -> bool:
return not given
def are_ids(given: list[str]) -> bool:
return all(re.fullmatch(r"[0-9]+", id_) for id_ in given)
# One row per subcommand, in the order the usage line and the docstring above list them: main, USAGE and `--help` all read this table.
COMMANDS = {
"review-folder": Command("review-folder", no_arguments, lambda given: make_review_folder()),
"ruling-base": Command("ruling-base", no_arguments, lambda given: with_default_branch(show_ruling_base, stale_ok=False)),
"round-count": Command("round-count", no_arguments, lambda given: with_default_branch(round_count, stale_ok=True)),
"needs-scoring-run": Command("needs-scoring-run [<item ID>...]", are_ids, lambda given: with_default_branch(lambda default: needs_scoring_run(default, [int(id_) for id_ in given]), stale_ok=False)),
"scoring-run-owed": Command("scoring-run-owed", no_arguments, lambda given: with_default_branch(owed, stale_ok=False)),
"record-ruling": Command("record-ruling <votes file>", lambda given: len(given) == 1, lambda given: record_ruling(pathlib.Path(given[0]))),
"prepare-scoring": Command("prepare-scoring [--preloaded <text>]", lambda given: not given or len(given) == 2 and given[0] == "--preloaded", lambda given: prepared_scoring(given[1] if given else None)),
"record-scoring": Command("record-scoring <scores file> [<item ID>...]", lambda given: bool(given) and are_ids(given[1:]), lambda given: with_default_branch(lambda default: record_scoring(default, pathlib.Path(given[0]), [int(id_) for id_ in given[1:]]), stale_ok=False)),
"places": Command("places", no_arguments, lambda given: show_places()),
"agreement": Command("agreement", no_arguments, lambda given: show_agreement()),
"reviewer-header": Command("reviewer-header <pass> [--rerun]", lambda given: given[:1] != [] and given[0] in REVIEWERS and given[1:] in ([], ["--rerun"]), lambda given: with_default_branch(lambda default: reviewer_header(default, given[0], given[1:] == ["--rerun"]), stale_ok=True)),
"reviewer-context": Command("reviewer-context", no_arguments, lambda given: printed(hook_reply(sys.stdin.read(), pathlib.Path.home()))),
}
# The commands that take item IDs, each with the number of arguments before the IDs.
ID_COMMANDS = {"needs-scoring-run": 0, "record-scoring": 1}
SHAPES = " | ".join(command.shape for command in COMMANDS.values())
USAGE = f"usage: larv-review {SHAPES}"
def main(argv: list[str]) -> int:
if argv in (["-h"], ["--help"]):
print(__doc__.strip().format(cap=CAP, release_cap=RELEASE_REVIEW_CAP, floor=FLOOR, pairs=MIN_PAIRS))
print(__doc__.strip().format(commands=SHAPES, cap=CAP, release_cap=RELEASE_REVIEW_CAP, floor=FLOOR, pairs=MIN_PAIRS))
return 0
command, *given = argv or [""]
if command == "review-folder" and not given:
done = make_review_folder()
elif command == "ruling-base" and not given:
done = with_default_branch(show_ruling_base, stale_ok=False)
elif command == "round-count" and not given:
done = with_default_branch(round_count, stale_ok=True)
elif command == "needs-scoring-run" and all(re.fullmatch(r"[0-9]+", id_) for id_ in given):
done = with_default_branch(lambda default: needs_scoring_run(default, [int(id_) for id_ in given]), stale_ok=False)
elif command == "scoring-run-owed" and not given:
done = with_default_branch(owed, stale_ok=False)
elif command == "record-ruling" and len(given) == 1:
done = record_ruling(pathlib.Path(given[0]))
elif command == "prepare-scoring" and (not given or len(given) == 2 and given[0] == "--preloaded"):
done = prepared_scoring(given[1] if given else None)
elif command == "record-scoring" and given and all(re.fullmatch(r"[0-9]+", id_) for id_ in given[1:]):
done = with_default_branch(lambda default: record_scoring(default, pathlib.Path(given[0]), [int(id_) for id_ in given[1:]]), stale_ok=False)
elif command == "places" and not given:
done = show_places()
elif command == "agreement" and not given:
done = show_agreement()
elif command == "reviewer-header" and given[:1] and given[0] in REVIEWERS and given[1:] in ([], ["--rerun"]):
done = with_default_branch(lambda default: reviewer_header(default, given[0], given[1:] == ["--rerun"]), stale_ok=True)
elif command == "reviewer-context" and not given:
done = printed(hook_reply(sys.stdin.read(), pathlib.Path.home()))
elif command in ("needs-scoring-run", "record-scoring") and given:
named = given[1:] if command == "record-scoring" else given
print(f"{command} takes item IDs, the numbers in todo.md's ID column, not {' '.join(id_ for id_ in named if not re.fullmatch(r'[0-9]+', id_))}; {USAGE}", file=sys.stderr)
return 1
elif not argv:
if not argv:
print(USAGE, file=sys.stderr)
return 1
name, *given = argv
command = COMMANDS.get(name)
if command and command.fits(given):
done = command.run(given)
elif command and name in ID_COMMANDS and (named := [id_ for id_ in given[ID_COMMANDS[name]:] if not re.fullmatch(r"[0-9]+", id_)]):
print(f"{name} takes item IDs, the numbers in todo.md's ID column, not {' '.join(named)}; {USAGE}", file=sys.stderr)
return 1
else:
print(f"unknown arguments {' '.join(argv)}; {USAGE}", file=sys.stderr)
return 1
+12 -26
View File
@@ -5,7 +5,7 @@ import re
import shutil
from failure import Failure
from records.markdown_text import FENCE, listed
from records.markdown_text import FENCE, listed, section
from git_cli.git_repo import git
from review_rules.review_folder import made_review_folder, review_folder
@@ -23,26 +23,6 @@ FORBIDDEN_FILES = ("todo.md", "docs/decisions.md", "docs/comprehension-history.m
FORBIDDEN = "todo.md, docs/decisions.md, docs/comprehension-history.md or docs/reading-cost.md (the plan, the decision log, past panels' records and past chunks' reading cost)"
TEST_FILE = re.compile(r"(^|/)(tests?/|test_[^/]*$|[^/]*_test\.[^/]*$|[^/]*\.(test|spec)\.[^/]*$)")
TEST_FILES = "test files being `test_*`, `*_test.*`, `*.test.*`, `*.spec.*` and those under a `test` or `tests` folder"
HEADING = re.compile(r"^(#+)\s+(.*?)\s*$")
def section(text: str, title: str) -> list[str] | None:
"""The lines under the first heading whose text fullmatches the pattern title, up to the next heading of its level or above; None where none does."""
found: list[str] | None = None
level = 0
fenced = False
for line in text.splitlines():
if FENCE.match(line):
fenced = not fenced
heading = None if fenced else HEADING.match(line)
if found is None:
if heading and re.fullmatch(title, heading.group(2)):
found, level = [], len(heading.group(1))
elif heading and len(heading.group(1)) <= level:
break
else:
found.append(line)
return found
def bullets(lines: list[str]) -> list[list[str]]:
@@ -62,12 +42,18 @@ def bullets(lines: list[str]) -> list[list[str]]:
return groups
ANCHOR = re.compile(r"- \*\*\d+ — ")
DIMENSIONS = ("Navigation", "Locality", "Shape", "Self-sufficiency")
def score_bullets(skill: str) -> tuple[list[str], list[str]] | Failure:
"""The six anchor bullets and the four dimension bullets of SKILL.md → "The score", without bold, each on one line."""
groups = [[line.replace("**", "") for line in group] for group in bullets(section(skill, "The score") or [])]
if [len(group) for group in groups[:2]] != [6, 4]:
return Failure(f"{PANEL / 'SKILL.md'} → \"The score\" no longer opens with six anchor bullets and then four dimension bullets")
return groups[0], groups[1]
"""The six anchor bullets and the four dimension bullets of SKILL.md → "The score", without bold, each on one line. A bullet is one by how it opens, `- **<n> — ` or `- **<dimension>**`, so other bullets and their order in the section change nothing."""
lines = [line.replace("**", "") for group in bullets(section(skill, "The score") or []) for line in group if ANCHOR.match(line) or line.startswith(tuple(f"- **{name}**" for name in DIMENSIONS))]
anchors = [line for line in lines if line[2].isdigit()]
dimensions = [line for line in lines if not line[2].isdigit()]
if len(anchors) != 6 or [line.split(" — ")[0] for line in dimensions] != [f"- {name}" for name in DIMENSIONS]:
return Failure(f"{PANEL / 'SKILL.md'} → \"The score\" no longer holds six bullets opening `- **<n> — ` and four opening `- **<dimension>**` for Navigation, Locality, Shape and Self-sufficiency, in that order")
return anchors, dimensions
def project_values(text: str, source: str) -> dict[str, str] | Failure:
+12
View File
@@ -872,6 +872,18 @@ class LarvReviewTool(unittest.TestCase):
def test_review_folder_takes_no_arguments(self):
self.assertIn("unknown arguments", self.run_tool("review-folder", "x").stderr)
def test_names_the_arguments_that_are_not_item_ids(self):
for args, named in ((("needs-scoring-run", "3", "x"), "x"), (("record-scoring", "scores.md", "4", "y1"), "y1")):
with self.subTest(args=args):
ran = self.run_tool(*args)
self.assertEqual(ran.returncode, 1)
self.assertIn(f"takes item IDs, the numbers in todo.md's ID column, not {named};", ran.stderr)
def test_help_and_the_usage_line_list_the_same_commands(self):
listed = self.run_tool("--help").stdout.splitlines()[2].removeprefix("usage: larv-review ").split(" | ")
self.assertEqual([shape.split()[0] for shape in listed], ["review-folder", "ruling-base", "round-count", "needs-scoring-run", "scoring-run-owed", "record-ruling", "prepare-scoring", "record-scoring", "places", "agreement", "reviewer-header", "reviewer-context"])
self.assertIn(f"usage: larv-review {' | '.join(listed)}", self.run_tool().stderr)
def test_takes_a_local_main_without_origin_and_reads_origin_as_last_fetched_when_the_fetch_fails(self):
self.commit("Answer prose round 1: a")
self.git("remote", "set-url", "origin", str(self.checkout.parent / "gone.git"))
+12 -8
View File
@@ -9,7 +9,7 @@ import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from larv_review.scoring_setup import bullets, CLAUDE_MD, file_globs, preloaded_files, prepare_scoring, project_values, removed_setup, score_bullets, section
from larv_review.scoring_setup import bullets, CLAUDE_MD, file_globs, preloaded_files, prepare_scoring, project_values, removed_setup, score_bullets
from failure import Failure
from test_env import isolated_env
@@ -27,6 +27,17 @@ class ScoreBullets(unittest.TestCase):
self.assertEqual([line.split(" — ")[0] for line in dimensions], ["- Navigation", "- Locality", "- Shape", "- Self-sufficiency"])
self.assertEqual(dimensions[1], "- Locality — understanding and changing one piece without holding the rest: hidden state, order dependence, action at a distance.")
def test_reads_the_bullets_by_how_they_open_whatever_else_the_section_holds(self):
skill = SKILL.read_text()
with_extra = skill.replace("## The score\n\n", "## The score\n\n- A bullet before the anchors.\n\n", 1)
self.assertEqual(score_bullets(with_extra), score_bullets(skill))
def test_fails_where_an_anchor_or_a_dimension_is_missing(self):
skill = SKILL.read_text()
for dropped in ("- **10 — Duress-proof.**", "- **Shape** —"):
with self.subTest(dropped=dropped):
self.assertIsInstance(score_bullets(skill.replace(dropped, "- Dropped:", 1)), Failure)
class ProjectValues(unittest.TestCase):
def test_reads_each_spelling_the_projects_use(self):
@@ -56,13 +67,6 @@ class FileGlobs(unittest.TestCase):
self.assertEqual(file_globs(paths), "`bin/**` except test files, `docs/**`, `README.md` and `ci.sh`, test files being `test_*`, `*_test.*`, `*.test.*`, `*.spec.*` and those under a `test` or `tests` folder")
class Section(unittest.TestCase):
def test_holds_deeper_headings_and_fenced_lines_and_ends_at_its_own_level(self):
text = "# P\n\n## Scoring run\n\na\n\n### Sub\n\n```\n## fenced\n```\n\n## Next\n\nb\n"
self.assertEqual(section(text, "Scoring run"), ["", "a", "", "### Sub", "", "```", "## fenced", "```", ""])
self.assertIsNone(section(text, "Missing"))
class Bullets(unittest.TestCase):
def test_joins_continuations_and_splits_runs_at_other_lines(self):
lines = [" stray", "- a ", " more", "Text", "- b", "", "- c"]
+1 -1
View File
@@ -181,7 +181,7 @@ def annotation_of(cycle: Cycle, transcript_path: str, entries: list[Entry], judg
if isinstance(subagents, Failure):
return subagents
spent = spend.spent(since, entries, subagents, judge_of(cycle).model, judged.tokens)
return outcomes.Annotation(cycle.config, spend.spent_block(since, spent), judged.reversed)
return outcomes.Annotation(cycle.config, outcomes.spent_block(since, spent), judged.reversed)
def ruling_outcome(cycle: Cycle, ruling: Interrupt | QuotedAnswer, quoted_answers: int, annotation: outcomes.Annotation) -> str:
+4 -22
View File
@@ -7,16 +7,12 @@ import json
import pathlib
import typing
from claude_code.entries import AssistantTurn, Entry
from claude_code.entries import AssistantTurn, Entry, messages
from failure import Failure
from routing_figures.prices import PRICES
from claude_code.transcript import read_transcript
from transcript_archive.outcomes import SPENT
from transcript_archive.outcomes import Spender
from claude_code.tokens import Tokens, tokens_of
KINDS = "input, cache write 5m, cache write 1h, cache read, output"
# Who spent, such as `worker`, `judge` or `Explore "Search the hooks"`, and the model they ran on.
Spender = tuple[str, str]
WORKER = "worker"
JUDGE = "judge"
@@ -38,14 +34,10 @@ def spent(since: str, worker: list[Entry], subagents: list[Subagent], judge_mode
def thread_spend(entries: list[Entry], since: str) -> dict[str, Tokens]:
"""Each model's tokens in one transcript's messages written after since. A message's usage repeats in each entry of it, so each message counts once, by its id."""
"""Each model's tokens in one transcript's messages written after since. Each message counts once."""
start = moment(since) if since else None
counted: dict[str, AssistantTurn] = {}
for entry in entries:
if isinstance(entry, AssistantTurn) and entry.model and (start is None or later(entry.timestamp, start)):
counted.setdefault(entry.message_id or entry.uuid, entry)
found: dict[str, Tokens] = {}
for turn in counted.values():
for turn, *_ in messages(entry for entry in entries if isinstance(entry, AssistantTurn) and entry.model and (start is None or later(entry.timestamp, start))):
found[turn.model] = found.get(turn.model, Tokens()).plus(tokens_of(turn.usage))
return found
@@ -84,13 +76,3 @@ def subagent_name(transcript: pathlib.Path) -> str:
# One line for the type, since a line break could forge an archive line such as `Config: `; json.dumps escapes the description's.
kind, description = " ".join(str(meta.get("agentType") or "subagent").split()), str(meta.get("description") or "")
return f"{kind} {json.dumps(description, ensure_ascii=False)}" if description else kind
def spent_block(since: str, found: dict[Spender, Tokens]) -> str:
"""The archive's block for what the stretch spent, then the list prices in force for each model in it; empty where nothing was spent."""
if not found:
return ""
spenders = [f"- {name}, {model}: {', '.join(map(str, (tokens.input, tokens.write_5m, tokens.write_1h, tokens.read, tokens.output)))}" for (name, model), tokens in found.items()]
priced = [f"- {model}: {price.input:g}, {price.write_5m:g}, {price.write_1h:g}, {price.read:g}, {price.output:g}" for model in dict.fromkeys(model for _, model in found) if (price := PRICES.get(model))]
prices = ["", f"List prices in dollars per million tokens of {KINDS}:", *priced] if priced else []
return "\n".join([f"{SPENT}{since or 'the chunk began'}, in tokens of {KINDS}:", *spenders, *prices])
+3 -3
View File
@@ -1690,14 +1690,14 @@ class Spend(unittest.TestCase):
def test_renders_the_spend_then_the_list_prices_of_each_model_it_knows(self):
found = {("worker", "claude-opus-5-5"): Tokens(1, 2, 3, 4, 5), ("judge", "unpriced"): Tokens(input=6)}
self.assertEqual(spend.spent_block("2026-11-01T10:00:00Z", found), """Spent since 2026-11-01T10:00:00Z, in tokens of input, cache write 5m, cache write 1h, cache read, output:
self.assertEqual(outcomes.spent_block("2026-11-01T10:00:00Z", found), """Spent since 2026-11-01T10:00:00Z, in tokens of input, cache write 5m, cache write 1h, cache read, output:
- worker, claude-opus-5-5: 1, 2, 3, 4, 5
- judge, unpriced: 6, 0, 0, 0, 0
List prices in dollars per million tokens of input, cache write 5m, cache write 1h, cache read, output:
- claude-opus-5-5: 4, 5, 8, 0.2, 20""")
self.assertEqual(spend.spent_block("", {("judge", "unpriced"): Tokens(input=6)}), "Spent since the chunk began, in tokens of input, cache write 5m, cache write 1h, cache read, output:\n- judge, unpriced: 6, 0, 0, 0, 0")
self.assertEqual(spend.spent_block("", {}), "")
self.assertEqual(outcomes.spent_block("", {("judge", "unpriced"): Tokens(input=6)}), "Spent since the chunk began, in tokens of input, cache write 5m, cache write 1h, cache read, output:\n- judge, unpriced: 6, 0, 0, 0, 0")
self.assertEqual(outcomes.spent_block("", {}), "")
def test_keeps_the_whole_typed_answer_that_holds_the_judges_quote_and_its_tokens(self):
shown = "[human] Use A.\n[human] No, keep the old name. Use X.\nworking: keep the new name"
+2 -3
View File
@@ -2,7 +2,7 @@ import collections
import dataclasses
import re
from claude_code.entries import AssistantTurn, Entry, parse_entry, tool_calls, ToolCall, usage_count
from claude_code.entries import AssistantTurn, Entry, messages, parse_entry, tool_calls, ToolCall, usage_count
from git_cli.git_command import commits
from reading_cost.shell_reads import Reads, shell_reads
from command_lines.shell_command import resolved, simple_commands
@@ -46,8 +46,7 @@ def commit_time(worker: list[Entry], home: str) -> str | None:
def tokens_in(turns: list[AssistantTurn]) -> int:
usages = {turn.message_id or turn.uuid: turn.usage for turn in turns}
return sum(usage_count(usage, "input_tokens") + usage_count(usage, "cache_creation_input_tokens") for usage in usages.values())
return sum(usage_count(first.usage, "input_tokens") + usage_count(first.usage, "cache_creation_input_tokens") for first, *_ in messages(turns))
def reads_of(call: ToolCall, cwd: str, home: str) -> Reads:
+27 -3
View File
@@ -2,10 +2,12 @@ import re
import typing
FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})", re.M)
HEADING = re.compile(r"^(#+)\s+(.*?)\s*$")
def sections(text: str) -> dict[str, list[str]]:
blocks: list[tuple[str | None, list[str]]] = []
def fenced_lines(text: str) -> list[tuple[str, bool]]:
"""Each line of text with whether a code fence holds it, the fence's own lines included. A fence closes only on a line of its own marker, as long as the opener or longer."""
found = []
opener = ""
for line in text.splitlines():
if fence := FENCE.match(line):
@@ -14,7 +16,13 @@ def sections(text: str) -> dict[str, list[str]]:
opener = marker
elif marker[0] == opener[0] and len(marker) >= len(opener) and not line[fence.end():].strip():
opener = ""
fenced = bool(opener or fence)
found.append((line, bool(opener or fence)))
return found
def sections(text: str) -> dict[str, list[str]]:
blocks: list[tuple[str | None, list[str]]] = []
for line, fenced in fenced_lines(text):
if not fenced and line.startswith("## "):
blocks.append((line.strip()[3:], []))
elif not fenced and line.startswith("# "):
@@ -28,6 +36,22 @@ def sections(text: str) -> dict[str, list[str]]:
return found
def section(text: str, title: str) -> list[str] | None:
"""The lines under the first heading whose text fullmatches the pattern title, up to the next heading of its level or above, blank lines kept; None where none does."""
found: list[str] | None = None
level = 0
for line, fenced in fenced_lines(text):
heading = None if fenced else HEADING.match(line)
if found is None:
if heading and re.fullmatch(title, heading.group(2)):
found, level = [], len(heading.group(1))
elif heading and len(heading.group(1)) <= level:
break
else:
found.append(line)
return found
# Blockquoted, so a passage holding blank lines still ends visibly where the text after it starts.
def quoted(passage: str) -> str:
return "\n".join(f"> {line}" for line in passage.splitlines())
+11
View File
@@ -21,6 +21,17 @@ class Sections(unittest.TestCase):
self.assertEqual(markdown_text.quoted("merged\n\nPR #1"), "> merged\n> \n> PR #1")
class Section(unittest.TestCase):
def test_holds_deeper_headings_and_fenced_lines_and_ends_at_its_own_level(self):
text = "# P\n\n## Scoring run\n\na\n\n### Sub\n\n```\n## fenced\n```\n\n## Next\n\nb\n"
self.assertEqual(markdown_text.section(text, "Scoring run"), ["", "a", "", "### Sub", "", "```", "## fenced", "```", ""])
self.assertIsNone(markdown_text.section(text, "Missing"))
def test_a_fence_closes_only_on_its_own_marker_here_too(self):
text = "## A\n````md\n```\n## Quoted\n````\n## B\n"
self.assertEqual(markdown_text.section(text, "A"), ["````md", "```", "## Quoted", "````"])
class Listed(unittest.TestCase):
def test_joins_names_with_commas_and_the_conjunction_before_the_last(self):
self.assertEqual(markdown_text.listed(["a"], "and"), "a")
+11 -256
View File
@@ -50,47 +50,26 @@ review or match leaves its round out, except on a rung below the project's, wher
nothing. Exit 1, saying why, on bad arguments, an unreadable archive or models file, no archived
round to replay, or a usage limit.
"""
import dataclasses
import json
import os
import pathlib
import shutil
import sys
import tempfile
import typing
from transcript_archive.location import archive_root
from rung_replay.command import arguments_or_exit, failed, limited_exit, projects_on
from review_replay.rounds import authored_by, comparison, lost_inputs, reviewed_path, Round, round_from, sample, stamp_seconds
from review_replay.recall import candidates, enough, needed, Recall, recall_report, Reference, RungRecall, tried
from transcript_archive.project_key import checkout_of
from claude_code.reply import claude_reply, Limited
from review_replay.rounds import archived_rounds, Round, sample
from rung_replay.command import arguments_or_exit, failed, limited_exit
from review_replay.replay import codes_found, findings_replayed
from review_replay.omit_comparison import compared
from loop_effects.effects import Effects
from failure import Failure
from rung_replay.search import first_reply, replayed_on
from claude_code.transcript import first_timestamped_line, read_transcript
from claude_code.entries import parse_entry, structured_output
from review_rules.reviewer_context import principles_line, reviewer_context
from claude_code.reply import Limited
from model_routing.ladder import project_rung, Rung
from review_rules.review_folder import REPORTS, review_folder
from review_replay.recall import recall_report, recalled
from review_rules.review_passes import ReviewedPass, REVIEWERS
TOOL = "review-replay"
OMIT = "--omit-claude-md"
# Starts claude without its instruction files, as `omitClaudeMd` starts a subagent.
NO_CLAUDE_MD = ["--settings", json.dumps({"env": {"CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1"}})]
USAGE = f"{TOOL} <pass> <model> [{OMIT}], <pass> one of {', '.join(REVIEWERS)} and <model> a model on the routing ladder"
LATEST = 10
MATCH_SCHEMA = {"type": "object", "properties": {"found": {"type": "array", "items": {"type": "integer"}}}, "required": ["found"], "additionalProperties": False}
@dataclasses.dataclass(frozen=True)
class Code:
"""Where a round's reviewer read the code, the commit it read, the default branch, and where that branch stood as the chunk began."""
path: str
reviewed: str
default: str
base: str
def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, effects: Effects) -> int:
@@ -106,7 +85,11 @@ def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, eff
return failed(TOOL, f"no archived {reviewed} round of projects on {project.model} in {archive_root(home, env)} holds a 🔴 or 🟡 on code it can find")
chosen = sample(list(codes), LATEST)
if without:
return compared(reviewed, project, chosen, lambda found, bare: findings_replayed(found, codes[found], project, project, home, effects, bare))
report = compared(reviewed, project, chosen, lambda found, bare: findings_replayed(found, codes[found], project, project, home, effects, bare))
if isinstance(report, Limited):
return limited_exit(TOOL, report)
print(report)
return 0
def replay_one(found: Round, rung: Rung) -> set[int] | Failure | Limited:
matched = findings_replayed(found, codes[found], rung, project, home, effects, False)
@@ -121,57 +104,6 @@ def main(argv: list[str], env: typing.Mapping[str, str], home: pathlib.Path, eff
return 0
def recalled(chosen: list[Round], project: Rung, replay_one: typing.Callable[[Round, Rung], set[int] | Failure | Limited], what: str) -> Recall | Limited:
"""Replays chosen twice on the project's rung, then each candidate, highest first, on the rounds neither of those failed on, until one doesn't qualify."""
runs: list[list[set[int] | Failure]] = []
for _ in range(2):
found: list[set[int] | Failure] | Limited = replayed_on(chosen, project, replay_one, what)
if isinstance(found, Limited):
return found
runs.append(found)
rounds: list[Round] = []
reference: Reference = ([], [])
for one, first, second in zip(chosen, *runs):
if not isinstance(first, Failure) and not isinstance(second, Failure):
rounds.append(one)
reference[0].append(first)
reference[1].append(second)
recalls: list[RungRecall] = []
lowest = None
for rung in candidates(project) if enough(rounds) and needed(rounds, reference) > 0 else []:
on_rung: list[set[int] | Failure] | Limited = replayed_on(rounds, rung, replay_one, what)
if isinstance(on_rung, Limited):
return on_rung
# A failed replay on a candidate found nothing, so it lowers that rung's count and never lifts it to what a rung needs.
one = tried(rung, rounds, reference, [set() if isinstance(found, Failure) else found for found in on_rung])
recalls.append(one)
if not one.qualifies:
break
lowest = rung
return Recall(rounds, reference, len(chosen) - len(rounds), recalls, lowest)
def compared(reviewed: str, project: Rung, chosen: list[Round], found_on: typing.Callable[[Round, bool], set[int] | Failure | Limited]) -> int:
"""Replays chosen three times with CLAUDE.md and once without, as found_on(round, bare) finds, and prints the comparison of the rounds no replay failed on."""
runs: list[list[set[int] | Failure]] = []
for bare in (False, False, False, True):
def replay_one(one: Round, _: Rung, bare: bool = bare) -> set[int] | Failure | Limited:
found = found_on(one, bare)
if isinstance(found, Failure):
print(f" {one.root} {one.at} {'without' if bare else 'with'} the instruction files failed, so the round is left out: {found.reason}", file=sys.stderr, flush=True)
return found
found: list[set[int] | Failure] | Limited = replayed_on(chosen, project, replay_one, f"{reviewed} rounds {'without' if bare else 'with'} the instruction files")
if isinstance(found, Limited):
return limited_exit(TOOL, found)
runs.append(found)
# A failed replay found nothing, which would lower the bar where it had CLAUDE.md and count as misses where it had none.
kept = [(one, *found) for one, *found in zip(chosen, *runs) if not any(isinstance(each, Failure) for each in found)]
first, second, third, without = ([typing.cast(set[int], found[run]) for _, *found in kept] for run in range(4))
print(comparison(reviewed, project, [one for one, *_ in kept], (first, second, third), without, len(chosen) - len(kept)))
return 0
def pass_and_model(argv: list[str]) -> tuple[ReviewedPass, Rung, bool] | None:
"""The pass, the project's rung of the model, and whether OMIT follows, as argv names them in `<pass> <model> [--omit-claude-md]`; None for any other argv."""
if argv[2:] not in ([], [OMIT]) or len(argv) < 2:
@@ -179,180 +111,3 @@ def pass_and_model(argv: list[str]) -> tuple[ReviewedPass, Rung, bool] | None:
reviewed = next((kind for kind in REVIEWERS if kind == argv[0]), None)
project = project_rung(argv[1])
return (reviewed, project, argv[2:] == [OMIT]) if reviewed and project else None
def codes_found(rounds: list[Round], effects: Effects) -> dict[Round, Code]:
"""The code each round reviewed, as code_of finds it; stderr names each round left out, and why."""
codes = {}
for found in rounds:
if isinstance(code := code_of(found, effects), Failure):
print(f"{found.root} {found.at} is left out: {code.reason}", file=sys.stderr, flush=True)
else:
codes[found] = code
return codes
def archived_rounds(reviewer: str, model: str, root: pathlib.Path, home: pathlib.Path) -> list[Round] | Failure:
projects = projects_on(model, home)
if isinstance(projects, Failure):
return projects
found = []
try:
for key in projects:
for chunk in sorted(path for path in (root / key).glob("*") if path.is_dir()):
found += chunk_rounds(reviewer, checkout_of(key, home), chunk)
except (OSError, UnicodeDecodeError, ValueError) as failure:
return Failure(f"could not read the archive {root}: {failure}")
return found
def chunk_rounds(reviewer: str, checkout: str, chunk: pathlib.Path) -> list[Round]:
copies = sorted(chunk.glob("*.jsonl"))
began = parse_entry(first_timestamped_line(copies[0].read_bytes())) if copies else None
if not began or not began.timestamp:
return []
found = []
for meta in sorted(chunk.glob("session/*/subagents/*.meta.json")):
described = json.loads(meta.read_text(encoding="utf-8"))
if not isinstance(described, dict) or described.get("agentType") != reviewer:
continue
entries = read_transcript(str(meta.with_name(meta.name.removesuffix(".meta.json") + ".jsonl")))
if not isinstance(entries, Failure) and (one := round_from(reviewer, checkout, stamp_seconds(began.timestamp), entries)):
found.append(one)
return found
def code_of(found: Round, effects: Effects) -> Code | Failure:
"""Where, and at which commits, found's reviewer read the code, as the archive lets it be found again: the folder rounds.reviewed_path guesses from the prompt, and the commits authored by then.
Recording the reviewed commit at review time would need no such reading, but then no round archived before that change could be replayed. A wrong guess shows every rung the same wrong code: the project's rung finds fewer of that round's archived findings, which lowers the bar, and the rungs below find fewer too.
"""
if not os.path.isdir(found.root):
return Failure("the project has no checkout")
path = reviewed_path(found.prompt, found.root, os.path.exists)
if path is None:
return Failure("the prompt names neither one removed worktree nor the checkout")
default = effects.git("symbolic-ref", "--short", "refs/remotes/origin/HEAD", cwd=found.root)
if isinstance(default, Failure):
return default
default = default.strip().removeprefix("origin/")
log = effects.git("log", "--format=%H %at", f"refs/remotes/origin/{default}", cwd=found.root)
main_line = effects.git("log", "--first-parent", "--format=%H %at", f"refs/remotes/origin/{default}", cwd=found.root)
if isinstance(log, Failure):
return log
if isinstance(main_line, Failure):
return main_line
reviewed, base = authored_by(commit_log(log), found.seconds), authored_by(commit_log(main_line), found.began)
if reviewed is None or base is None:
return Failure("no commit was authored by then")
return Code(path, reviewed, default, base)
def commit_log(listed: str) -> list[tuple[str, float]]:
return [(commit, float(authored)) for commit, authored in (line.split() for line in listed.splitlines())]
def findings_replayed(found: Round, code: Code, rung: Rung, project: Rung, home: pathlib.Path, effects: Effects, bare: bool) -> set[int] | Failure | Limited:
"""The indexes of the round's findings that its replay on rung reports, as claude on the project's rung matches them, or the Failure of the review or the match; where bare, the replay runs without CLAUDE.md, as `launch` says."""
reviewed: str | Failure | Limited = first_reply(lambda: review(found, code, rung, home, effects, bare))
if isinstance(reviewed, (Failure, Limited)):
return reviewed
return first_reply(lambda: match(found.findings, reviewed, found.root, project, effects))
@dataclasses.dataclass(frozen=True)
class Copies:
"""The throwaway copies a replay hands its reviewer: clone, of the project's checkout, and reports, of the chunk's review folder."""
clone: str
reports: str
# The shell makes and removes the clone itself, and is tested on a temporary home as judge-replay's is; rounds.py's core takes the filesystem as an argument.
def review(found: Round, code: Code, rung: Rung, home: pathlib.Path, effects: Effects, bare: bool) -> str | Failure | Limited:
try:
(home / REPORTS).mkdir(mode=0o700, parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="review-replay-", ignore_cleanup_errors=True) as clone, tempfile.TemporaryDirectory(prefix="review-replay-", dir=home / REPORTS, ignore_cleanup_errors=True) as reports:
return reviewed_in(found, code, rung, Copies(clone, reports), home, effects, bare)
except OSError as failure:
return Failure(f"could not prepare the replay: {failure}")
def reviewed_in(found: Round, code: Code, rung: Rung, copies: Copies, home: pathlib.Path, effects: Effects, bare: bool) -> str | Failure | Limited:
"""The report of found's reviewer on rung, replayed on code cloned into copies.clone, with its review folder copied into copies.reports; without CLAUDE.md where bare."""
folder = pathlib.Path(review_folder(str(home), code.path))
copy_reports(found, folder, copies.reports)
prompt = redirected(found.prompt, home, folder, code.path, copies)
if isinstance(prompt, Failure):
return prompt
if lost := lost_inputs(prompt, copies.reports, lambda name: (pathlib.Path(copies.reports) / name).exists()):
return Failure(f"the prompt names {', '.join(lost)}, which the review folder did not hold from the chunk's start until the round's")
if isinstance(cloned := clone_reviewed(found, code, copies.clone, effects), Failure):
return cloned
launched = launch(found.reviewer, rung, prompt, copies.clone, home, bare)
if isinstance(launched, Failure):
return launched
reply = claude_reply(effects.claude, launched[0], copies.clone, launched[1], "claude")
if isinstance(reply, Failure | Limited):
return reply
return reply["result"] if isinstance(reply.get("result"), str) else Failure("the reviewer replied with no report")
def launch(reviewer: str, rung: Rung, prompt: str, clone: str, home: pathlib.Path, bare: bool) -> tuple[list[str], str] | Failure:
"""claude's arguments and prompt for reviewer on rung. Where bare, it starts without CLAUDE.md, and the prompt holds what the SubagentStart hook and the header's last line hand such a reviewer."""
args = ["--agent", reviewer, *rung.flags()]
if not bare:
return args, prompt
context = reviewer_context(home)
if isinstance(context, Failure):
return context
return [*args, *NO_CLAUDE_MD], "\n".join([f"{context}\n\n{prompt}", *([principles] if (principles := principles_line(clone)) else [])])
def copy_reports(found: Round, folder: pathlib.Path, reports: str) -> None:
"""Copies into reports each report written to folder after found's chunk began and before its round did."""
# The round's own report, any later one, and one from an earlier attempt at the item in the same worktree would each hand the reviewer the findings it is to find.
for written in folder.iterdir() if folder.is_dir() else ():
if written.is_file() and found.began <= written.stat().st_mtime < found.seconds:
shutil.copy2(written, reports)
def redirected(prompt: str, home: pathlib.Path, folder: pathlib.Path, worktree: str, copies: Copies) -> str | Failure:
"""prompt naming copies.reports in place of the review folder, in either spelling, and copies.clone in place of the worktree; a Failure where it still names the review folder."""
# Replace the folder's path first: outside home it contains the worktree's path, which replacing the worktree's would otherwise turn into the clone's.
prompt = prompt.replace(str(folder), copies.reports).replace(f"~/{folder.relative_to(home)}", copies.reports).replace(worktree, copies.clone)
# A spelling the rewrite misses, such as $HOME/, would leave the reviewer the live folder and the findings in it.
if REPORTS in prompt.replace(copies.reports, ""):
return Failure("the prompt names the review folder in a spelling the replay does not redirect")
return prompt
def clone_reviewed(found: Round, code: Code, clone: str, effects: Effects) -> Failure | None:
"""Clones found's checkout into clone, on the commit its reviewer read, with the default branch where it stood as the chunk began."""
for command in (
("clone", "--quiet", "--shared", "--no-checkout", found.root, clone),
("-C", clone, "update-ref", f"refs/remotes/origin/{code.default}", code.base),
("-C", clone, "update-ref", f"refs/heads/{code.default}", code.base),
("-C", clone, "checkout", "--quiet", *(("-B", found.branch) if found.branch and found.branch != code.default else ("--detach",)), code.reviewed),
):
if isinstance(ran := effects.git(*command, cwd=found.root), Failure):
return ran
return None
def match(findings: tuple[str, ...], replay_report: str, root: str, rung: Rung, effects: Effects) -> set[int] | Failure | Limited:
numbered = "\n\n".join(f"{n}. {finding}" for n, finding in enumerate(findings, 1))
prompt = (
"A code reviewer reported the numbered findings below. Another reviewer then reviewed the same change; its report follows them. "
"List the number of each finding that the other report also raises as 🔴 or 🟡: the same defect, at the same place or from the same cause, however it is worded. "
"A finding it raises only as 🟢 or ❓, or not at all, is not listed.\n\n"
f"The findings:\n{numbered}\n\nThe other report:\n{replay_report}"
)
args = ["--json-schema", json.dumps(MATCH_SCHEMA), *rung.flags(), "--tools", ""]
reply = claude_reply(effects.claude, args, root, prompt, "claude")
if isinstance(reply, Failure | Limited):
return reply
output = structured_output(reply)
numbers = output.get("found") if isinstance(output, dict) else None
if not isinstance(numbers, list) or not all(isinstance(n, int) for n in numbers):
return Failure(f"the match replied: {json.dumps(reply)}")
return {n - 1 for n in numbers if 1 <= n <= len(findings)}
+56
View File
@@ -0,0 +1,56 @@
"""The experiment behind review-replay's --omit-claude-md: three replays of a pass's rounds with CLAUDE.md, one without, and the report that holds the one without to what the others found: docs/decisions.md → `agents/: a reviewer starts without CLAUDE.md only where a replay of its pass missed no more findings than the most an unchanged replay missed`."""
import sys
import typing
from failure import Failure
from claude_code.reply import Limited
from rung_replay.search import replayed_on
from review_replay.rounds import Round
from model_routing.ladder import Rung
def compared(reviewed: str, project: Rung, chosen: list[Round], found_on: typing.Callable[[Round, bool], set[int] | Failure | Limited]) -> str | Limited:
"""Replays chosen three times with CLAUDE.md and once without, as found_on(round, bare) finds, and reports the comparison of the rounds no replay failed on; the Limited where claude hit its usage limit."""
runs: list[list[set[int] | Failure]] = []
for bare in (False, False, False, True):
def replay_one(one: Round, _: Rung, bare: bool = bare) -> set[int] | Failure | Limited:
found = found_on(one, bare)
if isinstance(found, Failure):
print(f" {one.root} {one.at} {'without' if bare else 'with'} the instruction files failed, so the round is left out: {found.reason}", file=sys.stderr, flush=True)
return found
found: list[set[int] | Failure] | Limited = replayed_on(chosen, project, replay_one, f"{reviewed} rounds {'without' if bare else 'with'} the instruction files")
if isinstance(found, Limited):
return found
runs.append(found)
# A failed replay found nothing, which would lower the bar where it had CLAUDE.md and count as misses where it had none.
kept = [(one, *found) for one, *found in zip(chosen, *runs) if not any(isinstance(each, Failure) for each in found)]
first, second, third, without = ([typing.cast(set[int], found[run]) for _, *found in kept] for run in range(4))
return comparison(reviewed, project, [one for one, *_ in kept], (first, second, third), without, len(chosen) - len(kept))
def comparison(reviewed: str, reference: Rung, rounds: list[Round], with_it: tuple[list[set[int]], list[set[int]], list[set[int]]], without: list[set[int]], left_out: int) -> str:
"""What the three replays with CLAUDE.md and the one without found in rounds. Each replay with it missed some findings the other two found: the control, the noise of an unchanged reviewer. The one without it is held to the findings the first two found, as the third is, and passes where it missed no more than the most a replay with it did. left_out counts the rounds a replay failed on, which rounds leaves out."""
first, second, third = with_it
def shared(one: list[set[int]], other: list[set[int]]) -> list[set[int]]:
return [found_by_one & found_by_other for found_by_one, found_by_other in zip(one, other)]
def missing(held: list[set[int]], run: list[set[int]]) -> list[tuple[Round, int]]:
return [(found_round, index) for found_round, held_in_round, found in zip(rounds, held, run) for index in sorted(held_in_round - found)]
def count(run: list[set[int]]) -> int:
return sum(len(found) for found in run)
both = shared(first, second)
missed = missing(both, without)
control = [len(missing(shared(one, other), run)) for run, one, other in ((first, second, third), (second, first, third), (third, first, second))]
passes = len(missed) <= max(control)
return "\n".join([
f"Replayed {len(rounds)} {reviewed} rounds of {reference.model} projects, which hold {sum(len(found_round.findings) for found_round in rounds)} 🔴 and 🟡 findings, four times on {reference}: three with CLAUDE.md, one without.",
*([f"Left out {left_out} more, which a replay failed on; stderr says why."] if left_out else []),
f"With CLAUDE.md the three replays found {count(first)}, {count(second)} and {count(third)} of them, and missed {control[0]}, {control[1]} and {control[2]} of the findings that both of the other two found.",
*([f"Without CLAUDE.md it found {count(without)} of them, and missed {len(missed)} of the {count(both)} findings that both of the first two found{':' if missed else '.'}"] if count(both) else [f"Without CLAUDE.md it found {count(without)} of them. No evidence: the first two replays with CLAUDE.md found none in common, so there is no finding to hold it to."]),
*(f" {found_round.root} {found_round.at}: {found_round.findings[index].splitlines()[0]}" for found_round, index in missed),
*([f"It {'passes' if passes else 'fails'}: {len(missed)} missed is {'no more' if passes else 'more'} than the most a replay with CLAUDE.md missed, {max(control)}."] if count(both) else []),
])
+34
View File
@@ -1,10 +1,14 @@
"""The bar review-replay holds a rung to, and its report: docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`."""
import dataclasses
from rung_replay.command import down_from
from failure import Failure
from fractions import Fraction
from review_replay.rounds import is_red, Round
from model_routing.ladder import LADDER, Rung
from claude_code.reply import Limited
import math
from rung_replay.search import replayed_on
import typing
# A rung qualifies with a share of the archived findings up to this far below the project's rung's. The project's rung found 69 to 78% of its own findings again, and its two replays of the same rounds differed by up to 4 points.
MARGIN = Fraction(1, 10)
@@ -91,3 +95,33 @@ def recall_report(reviewed: str, project: Rung, recall: Recall) -> str:
def percent(fraction: Fraction) -> str:
return f"{float(fraction):.0%}"
def recalled(chosen: list[Round], project: Rung, replay_one: typing.Callable[[Round, Rung], set[int] | Failure | Limited], what: str) -> Recall | Limited:
"""Replays chosen twice on the project's rung, then each candidate, highest first, on the rounds neither of those failed on, until one doesn't qualify."""
runs: list[list[set[int] | Failure]] = []
for _ in range(2):
found: list[set[int] | Failure] | Limited = replayed_on(chosen, project, replay_one, what)
if isinstance(found, Limited):
return found
runs.append(found)
rounds: list[Round] = []
reference: Reference = ([], [])
for one, first, second in zip(chosen, *runs):
if not isinstance(first, Failure) and not isinstance(second, Failure):
rounds.append(one)
reference[0].append(first)
reference[1].append(second)
recalls: list[RungRecall] = []
lowest = None
for rung in candidates(project) if enough(rounds) and needed(rounds, reference) > 0 else []:
on_rung: list[set[int] | Failure] | Limited = replayed_on(rounds, rung, replay_one, what)
if isinstance(on_rung, Limited):
return on_rung
# A failed replay on a candidate found nothing, so it lowers that rung's count and never lifts it to what a rung needs.
one = tried(rung, rounds, reference, [set() if isinstance(found, Failure) else found for found in on_rung])
recalls.append(one)
if not one.qualifies:
break
lowest = rung
return Recall(rounds, reference, len(chosen) - len(rounds), recalls, lowest)
+179
View File
@@ -0,0 +1,179 @@
"""Replay one archived review round on one rung: the code the round reviewed, found again in a throwaway clone, the reviewer's report on it, and the match of that report to the round's archived findings: docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`."""
import dataclasses
import json
import os
import pathlib
import shutil
import sys
import tempfile
from review_replay.rounds import authored_by, lost_inputs, reviewed_path, Round
from claude_code.reply import claude_reply, Limited
from git_cli.git_repo import default_ref
from loop_effects.effects import Effects
from failure import Failure
from rung_replay.search import first_reply
from review_rules.reviewer_context import principles_line, reviewer_context
from review_rules.review_folder import REPORTS, review_folder
from model_routing.ladder import Rung
from claude_code.entries import structured_output
# Starts claude without its instruction files, as `omitClaudeMd` starts a subagent.
NO_CLAUDE_MD = ["--settings", json.dumps({"env": {"CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1"}})]
MATCH_SCHEMA = {"type": "object", "properties": {"found": {"type": "array", "items": {"type": "integer"}}}, "required": ["found"], "additionalProperties": False}
@dataclasses.dataclass(frozen=True)
class Code:
"""Where a round's reviewer read the code, the commit it read, the default branch, and where that branch stood as the chunk began."""
path: str
reviewed: str
default: str
base: str
def codes_found(rounds: list[Round], effects: Effects) -> dict[Round, Code]:
"""The code each round reviewed, as code_of finds it; stderr names each round left out, and why."""
codes = {}
for found in rounds:
if isinstance(code := code_of(found, effects), Failure):
print(f"{found.root} {found.at} is left out: {code.reason}", file=sys.stderr, flush=True)
else:
codes[found] = code
return codes
def code_of(found: Round, effects: Effects) -> Code | Failure:
"""Where, and at which commits, found's reviewer read the code, as the archive lets it be found again: the folder rounds.reviewed_path guesses from the prompt, and the commits authored by then.
Recording the reviewed commit at review time would need no such reading, but then no round archived before that change could be replayed. A wrong guess shows every rung the same wrong code: the project's rung finds fewer of that round's archived findings, which lowers the bar, and the rungs below find fewer too.
"""
if not os.path.isdir(found.root):
return Failure("the project has no checkout")
path = reviewed_path(found.prompt, found.root, os.path.exists)
if path is None:
return Failure("the prompt names neither one removed worktree nor the checkout")
ref = default_ref(cwd=found.root, guarded=True, run=effects.git)
if isinstance(ref, Failure):
return ref
default = ref.removeprefix("origin/")
log = effects.git("log", "--format=%H %at", f"refs/remotes/origin/{default}", cwd=found.root)
main_line = effects.git("log", "--first-parent", "--format=%H %at", f"refs/remotes/origin/{default}", cwd=found.root)
if isinstance(log, Failure):
return log
if isinstance(main_line, Failure):
return main_line
reviewed, base = authored_by(commit_log(log), found.seconds), authored_by(commit_log(main_line), found.began)
if reviewed is None or base is None:
return Failure("no commit was authored by then")
return Code(path, reviewed, default, base)
def commit_log(listed: str) -> list[tuple[str, float]]:
return [(commit, float(authored)) for commit, authored in (line.split() for line in listed.splitlines())]
def findings_replayed(found: Round, code: Code, rung: Rung, project: Rung, home: pathlib.Path, effects: Effects, bare: bool) -> set[int] | Failure | Limited:
"""The indexes of the round's findings that its replay on rung reports, as claude on the project's rung matches them, or the Failure of the review or the match; where bare, the replay runs without CLAUDE.md, as `launch` says."""
reviewed: str | Failure | Limited = first_reply(lambda: review(found, code, rung, home, effects, bare))
if isinstance(reviewed, (Failure, Limited)):
return reviewed
return first_reply(lambda: match(found.findings, reviewed, found.root, project, effects))
@dataclasses.dataclass(frozen=True)
class Copies:
"""The throwaway copies a replay hands its reviewer: clone, of the project's checkout, and reports, of the chunk's review folder."""
clone: str
reports: str
# The shell makes and removes the clone itself, and is tested on a temporary home as judge-replay's is; rounds.py's core takes the filesystem as an argument.
def review(found: Round, code: Code, rung: Rung, home: pathlib.Path, effects: Effects, bare: bool) -> str | Failure | Limited:
try:
(home / REPORTS).mkdir(mode=0o700, parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="review-replay-", ignore_cleanup_errors=True) as clone, tempfile.TemporaryDirectory(prefix="review-replay-", dir=home / REPORTS, ignore_cleanup_errors=True) as reports:
return reviewed_in(found, code, rung, Copies(clone, reports), home, effects, bare)
except OSError as failure:
return Failure(f"could not prepare the replay: {failure}")
def reviewed_in(found: Round, code: Code, rung: Rung, copies: Copies, home: pathlib.Path, effects: Effects, bare: bool) -> str | Failure | Limited:
"""The report of found's reviewer on rung, replayed on code cloned into copies.clone, with its review folder copied into copies.reports; without CLAUDE.md where bare."""
folder = pathlib.Path(review_folder(str(home), code.path))
copy_reports(found, folder, copies.reports)
prompt = redirected(found.prompt, home, folder, code.path, copies)
if isinstance(prompt, Failure):
return prompt
if lost := lost_inputs(prompt, copies.reports, lambda name: (pathlib.Path(copies.reports) / name).exists()):
return Failure(f"the prompt names {', '.join(lost)}, which the review folder did not hold from the chunk's start until the round's")
if isinstance(cloned := clone_reviewed(found, code, copies.clone, effects), Failure):
return cloned
launched = launch(found.reviewer, rung, prompt, copies.clone, home, bare)
if isinstance(launched, Failure):
return launched
reply = claude_reply(effects.claude, launched[0], copies.clone, launched[1], "claude")
if isinstance(reply, Failure | Limited):
return reply
return reply["result"] if isinstance(reply.get("result"), str) else Failure("the reviewer replied with no report")
def launch(reviewer: str, rung: Rung, prompt: str, clone: str, home: pathlib.Path, bare: bool) -> tuple[list[str], str] | Failure:
"""claude's arguments and prompt for reviewer on rung. Where bare, it starts without CLAUDE.md, and the prompt holds what the SubagentStart hook and the header's last line hand such a reviewer."""
args = ["--agent", reviewer, *rung.flags()]
if not bare:
return args, prompt
context = reviewer_context(home)
if isinstance(context, Failure):
return context
return [*args, *NO_CLAUDE_MD], "\n".join([f"{context}\n\n{prompt}", *([principles] if (principles := principles_line(clone)) else [])])
def copy_reports(found: Round, folder: pathlib.Path, reports: str) -> None:
"""Copies into reports each report written to folder after found's chunk began and before its round did."""
# The round's own report, any later one, and one from an earlier attempt at the item in the same worktree would each hand the reviewer the findings it is to find.
for written in folder.iterdir() if folder.is_dir() else ():
if written.is_file() and found.began <= written.stat().st_mtime < found.seconds:
shutil.copy2(written, reports)
def redirected(prompt: str, home: pathlib.Path, folder: pathlib.Path, worktree: str, copies: Copies) -> str | Failure:
"""prompt naming copies.reports in place of the review folder, in either spelling, and copies.clone in place of the worktree; a Failure where it still names the review folder."""
# Replace the folder's path first: outside home it contains the worktree's path, which replacing the worktree's would otherwise turn into the clone's.
prompt = prompt.replace(str(folder), copies.reports).replace(f"~/{folder.relative_to(home)}", copies.reports).replace(worktree, copies.clone)
# A spelling the rewrite misses, such as $HOME/, would leave the reviewer the live folder and the findings in it.
if REPORTS in prompt.replace(copies.reports, ""):
return Failure("the prompt names the review folder in a spelling the replay does not redirect")
return prompt
def clone_reviewed(found: Round, code: Code, clone: str, effects: Effects) -> Failure | None:
"""Clones found's checkout into clone, on the commit its reviewer read, with the default branch where it stood as the chunk began."""
for command in (
("clone", "--quiet", "--shared", "--no-checkout", found.root, clone),
("-C", clone, "update-ref", f"refs/remotes/origin/{code.default}", code.base),
("-C", clone, "update-ref", f"refs/heads/{code.default}", code.base),
("-C", clone, "checkout", "--quiet", *(("-B", found.branch) if found.branch and found.branch != code.default else ("--detach",)), code.reviewed),
):
if isinstance(ran := effects.git(*command, cwd=found.root), Failure):
return ran
return None
def match(findings: tuple[str, ...], replay_report: str, root: str, rung: Rung, effects: Effects) -> set[int] | Failure | Limited:
numbered = "\n\n".join(f"{n}. {finding}" for n, finding in enumerate(findings, 1))
prompt = (
"A code reviewer reported the numbered findings below. Another reviewer then reviewed the same change; its report follows them. "
"List the number of each finding that the other report also raises as 🔴 or 🟡: the same defect, at the same place or from the same cause, however it is worded. "
"A finding it raises only as 🟢 or ❓, or not at all, is not listed.\n\n"
f"The findings:\n{numbered}\n\nThe other report:\n{replay_report}"
)
args = ["--json-schema", json.dumps(MATCH_SCHEMA), *rung.flags(), "--tools", ""]
reply = claude_reply(effects.claude, args, root, prompt, "claude")
if isinstance(reply, Failure | Limited):
return reply
output = structured_output(reply)
numbers = output.get("found") if isinstance(output, dict) else None
if not isinstance(numbers, list) or not all(isinstance(n, int) for n in numbers):
return Failure(f"the match replied: {json.dumps(reply)}")
return {n - 1 for n in numbers if 1 <= n <= len(findings)}
+33 -26
View File
@@ -1,12 +1,16 @@
"""Which archived rounds of a review pass to replay, the code each reviewed, and the report with --omit-claude-md: docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`."""
"""Which archived rounds of a review pass to replay, and the code each reviewed: docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`."""
import dataclasses
import datetime
import json
import pathlib
import re
import typing
from claude_code.entries import AssistantTurn, Entry, Text, UserText
from model_routing.ladder import Rung
from claude_code.entries import AssistantTurn, Entry, parse_entry, Text, UserText
from transcript_archive.project_key import checkout_of
from failure import Failure
from claude_code.transcript import first_timestamped_line, read_transcript
from rung_replay.command import projects_on
# A line that opens with a 🔴 or 🟡 mark, after any list, table, heading or bold markup, as review-findings.md has reviewers report them.
FINDING = re.compile(r"^[\s#>*+|-]*(?:\d+[.)]\s*)?[*_]*\s*(?:🔴|🟡)")
@@ -107,28 +111,31 @@ def sample(rounds: list[Round], latest: int) -> list[Round]:
return sorted(rounds, key=lambda found: (found.seconds, found.root))[-latest:]
def comparison(reviewed: str, reference: Rung, rounds: list[Round], with_it: tuple[list[set[int]], list[set[int]], list[set[int]]], without: list[set[int]], left_out: int) -> str:
"""What the three replays with CLAUDE.md and the one without found in rounds. Each replay with it missed some findings the other two found: the control, the noise of an unchanged reviewer. The one without it is held to the findings the first two found, as the third is, and passes where it missed no more than the most a replay with it did. left_out counts the rounds a replay failed on, which rounds leaves out."""
first, second, third = with_it
def archived_rounds(reviewer: str, model: str, root: pathlib.Path, home: pathlib.Path) -> list[Round] | Failure:
projects = projects_on(model, home)
if isinstance(projects, Failure):
return projects
found = []
try:
for key in projects:
for chunk in sorted(path for path in (root / key).glob("*") if path.is_dir()):
found += chunk_rounds(reviewer, checkout_of(key, home), chunk)
except (OSError, UnicodeDecodeError, ValueError) as failure:
return Failure(f"could not read the archive {root}: {failure}")
return found
def shared(one: list[set[int]], other: list[set[int]]) -> list[set[int]]:
return [found_by_one & found_by_other for found_by_one, found_by_other in zip(one, other)]
def missing(held: list[set[int]], run: list[set[int]]) -> list[tuple[Round, int]]:
return [(found_round, index) for found_round, held_in_round, found in zip(rounds, held, run) for index in sorted(held_in_round - found)]
def count(run: list[set[int]]) -> int:
return sum(len(found) for found in run)
both = shared(first, second)
missed = missing(both, without)
control = [len(missing(shared(one, other), run)) for run, one, other in ((first, second, third), (second, first, third), (third, first, second))]
passes = len(missed) <= max(control)
return "\n".join([
f"Replayed {len(rounds)} {reviewed} rounds of {reference.model} projects, which hold {sum(len(found_round.findings) for found_round in rounds)} 🔴 and 🟡 findings, four times on {reference}: three with CLAUDE.md, one without.",
*([f"Left out {left_out} more, which a replay failed on; stderr says why."] if left_out else []),
f"With CLAUDE.md the three replays found {count(first)}, {count(second)} and {count(third)} of them, and missed {control[0]}, {control[1]} and {control[2]} of the findings that both of the other two found.",
*([f"Without CLAUDE.md it found {count(without)} of them, and missed {len(missed)} of the {count(both)} findings that both of the first two found{':' if missed else '.'}"] if count(both) else [f"Without CLAUDE.md it found {count(without)} of them. No evidence: the first two replays with CLAUDE.md found none in common, so there is no finding to hold it to."]),
*(f" {found_round.root} {found_round.at}: {found_round.findings[index].splitlines()[0]}" for found_round, index in missed),
*([f"It {'passes' if passes else 'fails'}: {len(missed)} missed is {'no more' if passes else 'more'} than the most a replay with CLAUDE.md missed, {max(control)}."] if count(both) else []),
])
def chunk_rounds(reviewer: str, checkout: str, chunk: pathlib.Path) -> list[Round]:
copies = sorted(chunk.glob("*.jsonl"))
began = parse_entry(first_timestamped_line(copies[0].read_bytes())) if copies else None
if not began or not began.timestamp:
return []
found = []
for meta in sorted(chunk.glob("session/*/subagents/*.meta.json")):
described = json.loads(meta.read_text(encoding="utf-8"))
if not isinstance(described, dict) or described.get("agentType") != reviewer:
continue
entries = read_transcript(str(meta.with_name(meta.name.removesuffix(".meta.json") + ".jsonl")))
if not isinstance(entries, Failure) and (one := round_from(reviewer, checkout, stamp_seconds(began.timestamp), entries)):
found.append(one)
return found
+7 -7
View File
@@ -18,7 +18,7 @@ from review_replay.rounds import authored_by, lost_inputs, recorded_findings, re
from loop_effects.effects import Effects, system_effects
from claude_code.entries import entry_from
from failure import Failure
from review_replay import main, recall
from review_replay import main, recall, replay
from review_rules.reviewer_context import reviewer_context
from review_rules.review_passes import REVIEWERS
from model_routing.ladder import Rung
@@ -124,17 +124,17 @@ class Launch(unittest.TestCase):
self.clone.mkdir()
def test_runs_the_reviewer_as_archived_with_claude_md(self):
self.assertEqual(main.launch("prose-reviewer", REFERENCE, "Review it.", str(self.clone), self.home, False), (["--agent", "prose-reviewer", *REFERENCE.flags()], "Review it."))
self.assertEqual(replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.clone), self.home, False), (["--agent", "prose-reviewer", *REFERENCE.flags()], "Review it."))
def test_drops_claude_md_and_hands_the_reviewer_what_the_hook_and_the_header_would(self):
(self.clone / "AGENTS.md").write_text("")
args, prompt = main.launch("prose-reviewer", REFERENCE, "Review it.", str(self.clone), self.home, True)
args, prompt = replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.clone), self.home, True)
self.assertEqual(args, ["--agent", "prose-reviewer", *REFERENCE.flags(), "--settings", json.dumps({"env": {"CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1"}})])
self.assertEqual(prompt, f"{reviewer_context(self.home)}\n\nReview it.\nPrinciples: read {self.clone}/AGENTS.md first, for the project principles and the decision titles.")
def test_fails_without_the_sections_to_hand_over(self):
(self.home / ".claude" / "CLAUDE.md").write_text("")
self.assertIsInstance(main.launch("prose-reviewer", REFERENCE, "Review it.", str(self.clone), self.home, True), Failure)
self.assertIsInstance(replay.launch("prose-reviewer", REFERENCE, "Review it.", str(self.clone), self.home, True), Failure)
class Match(unittest.TestCase):
@@ -145,14 +145,14 @@ class Match(unittest.TestCase):
self.asked.append((args, cwd, prompt))
return subprocess.CompletedProcess(args, 0, stdout, "")
return main.match(("🔴 a", "🟡 b"), "the other report", "/code/p", REFERENCE, dataclasses.replace(system_effects(), claude=claude))
return replay.match(("🔴 a", "🟡 b"), "the other report", "/code/p", REFERENCE, dataclasses.replace(system_effects(), claude=claude))
def test_asks_for_the_numbered_findings_the_other_report_raises_as_json_with_no_tools(self):
self.run_match("{}")
[(args, cwd, prompt)] = self.asked
self.assertEqual(cwd, "/code/p")
self.assertIn("1. 🔴 a\n\n2. 🟡 b\n\nThe other report:\nthe other report", prompt)
self.assertEqual(json.loads(args[args.index("--json-schema") + 1]), main.MATCH_SCHEMA)
self.assertEqual(json.loads(args[args.index("--json-schema") + 1]), replay.MATCH_SCHEMA)
self.assertEqual(args[args.index("--tools") + 1], "")
self.assertEqual(args[args.index("--model"):args.index("--model") + 4], REFERENCE.flags())
@@ -498,7 +498,7 @@ class Main(unittest.TestCase):
def test_leaves_out_a_round_whose_checkout_names_no_default_branch(self):
run_git(self.checkout, "remote", "set-head", "origin", "-d")
self.assert_left_out("git symbolic-ref --short refs/remotes/origin/HEAD: ")
self.assert_left_out("origin records no default branch")
def test_leaves_out_a_round_whose_checkout_is_gone(self):
shutil.rmtree(self.checkout)
+5 -8
View File
@@ -1,9 +1,11 @@
"""What a reviewer under agents/ is handed in place of the instruction files: `omitClaudeMd` starts it without them, and most of `~/.claude/CLAUDE.md` is the worker's. docs/decisions.md → `agents/: a reviewer starts without CLAUDE.md only where a replay of its pass missed no more findings than the most an unchanged replay missed` says why, and which reviewers."""
import json
import pathlib
import re
from failure import Failure
from review_rules.review_passes import REVIEWERS
from records.markdown_text import section
# The sections of ~/.claude/CLAUDE.md a reviewer needs, in the file's order: the code-health goal the agents cite, the bar its own prose meets, and the words its reports use.
SECTIONS = ("Shared goal", "Prose", "Vocabulary")
@@ -16,15 +18,10 @@ def reviewer_context(home: pathlib.Path) -> str | Failure:
text = path.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError) as failure:
return Failure(f"could not read {path}: {failure}")
found: dict[str, list[str]] = {}
lines: list[str] = []
for line in text.splitlines():
if line.startswith("## "):
lines = found.setdefault(line[3:].strip(), [])
lines.append(line)
if missing := [name for name in SECTIONS if name not in found]:
found = {name: section(text, re.escape(name)) for name in SECTIONS}
if missing := [name for name, lines in found.items() if lines is None]:
return Failure(f"{path} holds no section {', '.join(missing)}")
return "\n\n".join(["You start without `~/.claude/CLAUDE.md`. These sections of it hold for you:", *("\n".join(found[name]).strip() for name in SECTIONS)])
return "\n\n".join(["You start without `~/.claude/CLAUDE.md`. These sections of it hold for you:", *("\n".join([f"## {name}", *lines]).strip() for name, lines in found.items() if lines is not None)])
def starts_bare(agent: str, home: pathlib.Path) -> bool:
@@ -26,6 +26,10 @@ class ReviewerContext(unittest.TestCase):
def test_holds_the_shared_goal_prose_and_vocabulary_sections_and_none_of_the_workers(self):
self.assertEqual(reviewer_context(self.home), "You start without `~/.claude/CLAUDE.md`. These sections of it hold for you:\n\n## Shared goal\n\nCode health.\n\n## Prose\n\nWrite short.\n\n## Vocabulary\n\n- **chunk** — a unit.")
def test_a_heading_inside_a_code_fence_does_not_end_a_section(self):
(self.home / ".claude" / "CLAUDE.md").write_text("## Shared goal\n\n```\n## Not a heading\n```\n\nstill the goal\n\n## Prose\n\np\n\n## Vocabulary\n\nv\n")
self.assertIn("## Shared goal\n\n```\n## Not a heading\n```\n\nstill the goal\n\n## Prose", reviewer_context(self.home))
def test_fails_on_a_missing_section_or_file(self):
(self.home / ".claude" / "CLAUDE.md").write_text("## Prose\n\nWrite short.\n")
self.assertEqual(reviewer_context(self.home), Failure(f"{self.home / '.claude' / 'CLAUDE.md'} holds no section Shared goal, Vocabulary"))
+5 -10
View File
@@ -3,8 +3,8 @@ import collections
import dataclasses
import typing
from claude_code.entries import AssistantTurn, CostState, Entry, ToolCall
from routing_figures.prices import listed
from claude_code.entries import messages as claude_messages, CostState, Entry, ToolCall
from model_routing.prices import listed
DIMENSIONS = ("project", "subagent type", "rung", "tool")
# Each of DIMENSIONS to its labels' dollars.
@@ -33,17 +33,12 @@ class Spent(typing.NamedTuple):
def messages(entries: list[Entry]) -> list[Message]:
"""The assistant messages, counted once each, each with the tools whose results it took in, or `prompt` where it took none in."""
# A message's usage is the same in each of its entries (none of 1,681 multi-entry messages in the archive of 2026-10-28 differed), so the first is kept, where reading_cost's tokens_in keeps the last.
kept: dict[str, tuple[AssistantTurn, list[str]]] = {}
for entry in entries:
if isinstance(entry, AssistantTurn):
found = kept.setdefault(entry.message_id or entry.uuid, (entry, []))
found[1].extend(part.name for part in entry.parts if isinstance(part, ToolCall))
counted = []
before = [PROMPT]
for turn, called in kept.values():
for turns in claude_messages(entries):
turn = turns[0]
counted.append(Message(turn.timestamp, before, turn.model, listed(turn.model, turn.usage)))
before = called or [PROMPT]
before = [part.name for each in turns for part in each.parts if isinstance(part, ToolCall)] or [PROMPT]
return counted
+17 -1
View File
@@ -1,8 +1,10 @@
import re
import typing
from model_routing.prices import PRICES
from records.markdown_text import quoted
from records.utc_stamp import STAMPED
from claude_code.tokens import Tokens
from model_routing.route import WorkerLine
TASK = "Task: "
@@ -33,6 +35,9 @@ JUDGE_COST_AT = re.compile(rf"^{re.escape(JUDGE)}(.+)\n{re.escape(JUDGE_COST)}([
CONFIG = "Config: "
# Opens the block of what the work since the previous outcome that recorded a spend spent; an outcome that spent nothing has none.
SPENT = "Spent since "
KINDS = "input, cache write 5m, cache write 1h, cache read, output"
# Who spent, such as `worker`, `judge` or `Explore "Search the hooks"`, and the model they ran on.
Spender = tuple[str, str]
# Opens the quote of a typed answer of the human's that reversed a choice the worker took on its own recommendation.
REVERSED = "Reversed:"
REVERSED_AT = re.compile(rf"^{re.escape(REVERSED)}\n((?:> .*\n)*> .*)", re.MULTILINE)
@@ -41,7 +46,7 @@ REVERSED_AT = re.compile(rf"^{re.escape(REVERSED)}\n((?:> .*\n)*> .*)", re.MULTI
class Annotation(typing.NamedTuple):
"""What a verdict, an interrupt or a quoted answer records beside its judge lines, so a change to the setup can be judged before and after: docs/decisions.md → `loop-claude: each verdict records the claude-config release and commit that ran it…`. Each part is left out where empty; the cycle always gives config."""
config: str = ""
# The block spend.spent_block renders.
# The block spent_block renders.
spent: str = ""
# The human's whole typed answer that holds the judge's quote of the reversal, as loop-claude checked it.
reversed: str = ""
@@ -54,6 +59,17 @@ def annotation_lines(annotation: Annotation) -> str:
return f"{reversed_block}{spent_block}{config_line}"
# Nothing reads a Spent block yet: the first tool that derives dollars from one adds its parser here, beside this writer.
def spent_block(since: str, found: dict[Spender, Tokens]) -> str:
"""The archive's block for what the stretch spent, then the list prices in force for each model in it; empty where nothing was spent."""
if not found:
return ""
spenders = [f"- {name}, {model}: {', '.join(map(str, (tokens.input, tokens.write_5m, tokens.write_1h, tokens.read, tokens.output)))}" for (name, model), tokens in found.items()]
priced = [f"- {model}: {price.input:g}, {price.write_5m:g}, {price.write_1h:g}, {price.read:g}, {price.output:g}" for model in dict.fromkeys(model for _, model in found) if (price := PRICES.get(model))]
prices = ["", f"List prices in dollars per million tokens of {KINDS}:", *priced] if priced else []
return "\n".join([f"{SPENT}{since or 'the chunk began'}, in tokens of {KINDS}:", *spenders, *prices])
def reversals(outcomes: str) -> list[str]:
"""Each reversing answer the outcomes quote, unquoted, in the order they were written."""
return [unquoted(found[1]) for found in REVERSED_AT.finditer(outcomes)]
+1 -1
View File
@@ -144,7 +144,7 @@ Valid while `~/.claude` is a checkout of this repo.
2026-11-12, lilleman, todo item 305. Serves goals 1, 1.2 and 1.4. A change to the setup can be judged only by comparing chunks run before it with chunks run after, so each verdict, interrupt and quoted answer in loop-claude's archive records three things beside its judge lines.
- `Config: <release> (<commit>)`: `git describe` and the commit of `~/.claude`, so outcomes group by the setup that produced them. Where git can't tell, it records `unknown` and why, on one line.
- A `Spent since` block: the raw tokens each thread spent, by model and by kind (input, cache write 5m, cache write 1h, cache read, output), then the list prices in force, from `bin/routing_figures/prices.py`. The threads are the worker's main thread, each subagent by its agent type and description, and the judge run that ruled. Each message counts once, by its id. The stretch runs from the previous outcome that recorded a spend, so what a `/talk` or a stopped run spent between rulings is counted once. It keeps tokens, since prices change. The prices recorded beside them give what the stretch cost when it ran. Claude Code doesn't report thinking apart from output, so it is left in output. The judge's tokens come from its json reply's `usage`, until item 303 archives the judge's own transcript.
- A `Spent since` block: the raw tokens each thread spent, by model and by kind (input, cache write 5m, cache write 1h, cache read, output), then the list prices in force, from `bin/model_routing/prices.py`. The threads are the worker's main thread, each subagent by its agent type and description, and the judge run that ruled. Each message counts once, by its id. The stretch runs from the previous outcome that recorded a spend, so what a `/talk` or a stopped run spent between rulings is counted once. It keeps tokens, since prices change. The prices recorded beside them give what the stretch cost when it ran. Claude Code doesn't report thinking apart from output, so it is left in output. The judge's tokens come from its json reply's `usage`, until item 303 archives the judge's own transcript.
- A `Reversed:` quote: the human's typed answer that overrides a choice the worker took on its own recommendation and recorded. The judge quotes the latest such `[human]` line; loop-claude keeps the whole typed answer that holds the quote verbatim, and drops a quote no typed answer holds. `score-outcomes` counts the distinct answers per finished chunk. That count is goal 1.2's counter-signal, and the 2026-11-08 entry of `loop-claude: a judge decides each interrupt…` is valid only while it stays low. It counts only chunks that record a Config line, since earlier judges didn't look for reversals.
Valid while score-outcomes reads the reversals from the archive.
+1 -1
View File
@@ -1,7 +1,7 @@
"""Run analyze.py's matching and new-finding count, and also sum what those calls cost. Usage: [BASELINE=<candidate>] analyze_metered.py <candidate> <variant>-<run>... [case ids...]"""
import pathlib, runpy, sys, threading
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
import analyze, review_replay.main as rm
import analyze, review_replay.replay as rm
from claude_code.reply import claude_reply
SPENT = [0.0]; LOCK = threading.Lock()
def metered(*a, **k):
+2 -2
View File
@@ -2,11 +2,11 @@
import concurrent.futures, json, os, pathlib, pickle, re, shutil, sys, tempfile
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "bin"))
from claude_code.reply import claude_reply, Limited
from review_replay.main import clone_reviewed, Copies, copy_reports, match, redirected
from review_replay.replay import clone_reviewed, Copies, copy_reports, match, redirected
from failure import Failure
from review_replay.rounds import lost_inputs, Round
from review_rules.review_folder import REPORTS, review_folder
from review_replay import main as rm
from review_replay import replay as rm
from model_routing.ladder import Rung
from loop_effects.effects import system_effects
+2
View File
@@ -47,6 +47,8 @@ A pass is one kind of review, a row of the table, named by its first column. Its
| Product-owner | product-owner-reviewer | changes an external surface or contract: API routes and request/response shapes, events, webhooks, published file formats, CLI flags and exit codes, UI, error or deprecation text a consumer reads, README and docs an outside reader lands on |
| Prose | prose-reviewer | changes prose a reader reads, such as docs, comments, help or error text, or the plan's text for an item the chunk ships, splits or answers; one round |
The table's first two columns are also code: `PASS_AGENTS` in `bin/review_rules/review_passes.py`, which `bin/repo_checks/test_reviewer_launches.py` holds to the table. Change a row in both.
A comment is prose, not code, in any file, and so is a docstring. A change to config is one that changes what runs, so a comment in a workflow or compose file fires only the prose pass. Markdown an agent follows is config: `CLAUDE.md`, `AGENTS.md`, `workflow-bench/*-prompt.md`, and files under `agents/`, `principles/`, `rules/` or `skills/`.
**Plan rows.** A `todo.md` row or detail that the chunk files from a finding, re-scores or moves between releases gets no review: the chunk that ships the item reads it.