Files
claude-config/bin/review_replay/replay.py
T

203 lines
12 KiB
Python

"""Replay one archived review round on one rung: the code the round reviewed, found again in a throwaway clone, the reviewer's report on it, and the match of that report to the round's archived findings: docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`."""
import dataclasses
import json
import os
import pathlib
import shutil
import tempfile
import typing
from review_replay.rounds import authored_by, lost_inputs, reviewed_path, Round
from review_replay.changes import Change
from claude_code.reply import claude_reply, Limited
from git_cli.git_repo import default_ref
from loop_effects.effects import Effects
from failure import Failure
from rung_replay.search import first_reply
from record_store.stores import local_file, store
from project_config.main import located
from review_rules.reviewer_context import principles_line, reviewer_context
from text_files.compressed import read_stored, SUFFIX
from review_rules.review_folder import REPORTS, review_folder
from model_routing.ladder import Rung
from claude_code.entries import structured_output
MATCH_SCHEMA = {"type": "object", "properties": {"found": {"type": "array", "items": {"type": "integer"}}}, "required": ["found"], "additionalProperties": False}
@dataclasses.dataclass(frozen=True)
class Code:
"""Where a round's reviewer read the code, the commit it read, the default branch, and where that branch stood as the chunk began."""
path: str
reviewed: str
default: str
base: str
def codes_found(rounds: list[Round], effects: Effects, say: typing.Callable[[str], None]) -> dict[Round, Code]:
"""The code each round reviewed, as code_of finds it; say is told each round left out, and why."""
codes = {}
for found in rounds:
if isinstance(code := code_of(found, effects), Failure):
say(f"{found.root} {found.at} is left out: {code.reason}")
else:
codes[found] = code
return codes
def code_of(found: Round, effects: Effects) -> Code | Failure:
"""Where, and at which commits, found's reviewer read the code, as the archive lets it be found again: the folder rounds.reviewed_path guesses from the prompt, and the commits authored by then.
Recording the reviewed commit at review time would need no such reading, but then no round archived before that change could be replayed. A wrong guess shows every rung the same wrong code: the project's rung finds fewer of that round's archived findings, which lowers the bar, and the rungs below find fewer too.
"""
if not os.path.isdir(found.root):
return Failure("the project has no checkout")
path = reviewed_path(found.prompt, found.root, os.path.exists)
if path is None:
return Failure("the prompt names neither one removed worktree nor the checkout")
ref = default_ref(cwd=found.root, guarded=True, run=effects.git)
if isinstance(ref, Failure):
return ref
default = ref.removeprefix("origin/")
log = effects.git("log", "--format=%H %at", f"refs/remotes/origin/{default}", cwd=found.root)
main_line = effects.git("log", "--first-parent", "--format=%H %at", f"refs/remotes/origin/{default}", cwd=found.root)
if isinstance(log, Failure):
return log
if isinstance(main_line, Failure):
return main_line
reviewed, base = authored_by(commit_log(log), found.seconds), authored_by(commit_log(main_line), found.began)
if reviewed is None or base is None:
return Failure("no commit was authored by then")
return Code(path, reviewed, default, base)
def commit_log(listed: str) -> list[tuple[str, float]]:
return [(commit, float(authored)) for commit, authored in (line.split() for line in listed.splitlines())]
def findings_replayed(found: Round, code: Code, rung: Rung, project: Rung, home: pathlib.Path, effects: Effects, change: Change | None, scratch: pathlib.Path | None) -> set[int] | Failure | Limited:
"""The indexes of the round's findings that its replay on rung reports, as claude on the project's rung matches them, or the Failure of the review or the match; where change is given, the replay runs with it, as `launch` says. The clone is made under scratch, or under the system's temporary folder where it is None."""
reviewed: str | Failure | Limited = first_reply(lambda: review(found, code, rung, home, effects, change, scratch))
if isinstance(reviewed, (Failure, Limited)):
return reviewed
return first_reply(lambda: match(found.findings, reviewed, found.root, project, effects))
@dataclasses.dataclass(frozen=True)
class Copies:
"""The throwaway copies a replay hands its reviewer: clone, of the project's checkout, and reports, of the chunk's review folder."""
clone: str
reports: str
# The shell makes and removes the clone itself, and is tested on a temporary home as judge-replay's is.
def review(found: Round, code: Code, rung: Rung, home: pathlib.Path, effects: Effects, change: Change | None, scratch: pathlib.Path | None) -> str | Failure | Limited:
try:
(home / REPORTS).mkdir(mode=0o700, parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="review-replay-", dir=scratch, ignore_cleanup_errors=True) as clone, tempfile.TemporaryDirectory(prefix="review-replay-", dir=home / REPORTS, ignore_cleanup_errors=True) as reports:
return reviewed_in(found, code, rung, Copies(clone, reports), home, effects, change)
except OSError as failure:
return Failure(f"could not prepare the replay: {failure}")
def reviewed_in(found: Round, code: Code, rung: Rung, copies: Copies, home: pathlib.Path, effects: Effects, change: Change | None) -> str | Failure | Limited:
"""The report of found's reviewer on rung, replayed on code cloned into copies.clone, with its review folder copied into copies.reports; with change where given."""
folder = pathlib.Path(review_folder(str(home), code.path))
copy_reports(found, folder, copies.reports)
prompt = redirected(found.prompt, home, folder, code.path, copies)
if isinstance(prompt, Failure):
return prompt
if lost := lost_inputs(prompt, copies.reports, lambda name: (pathlib.Path(copies.reports) / name).exists()):
return Failure(f"the prompt names {', '.join(lost)}, which the review folder did not hold from the chunk's start until the round's")
if isinstance(cloned := clone_reviewed(found, code, copies.clone, effects), Failure):
return cloned
launched = launch(found.reviewer, rung, prompt, found.root, copies, home, change)
if isinstance(launched, Failure):
return launched
reply = claude_reply(effects.claude, launched[0], copies.clone, launched[1], "claude")
if isinstance(reply, Failure | Limited):
return reply
return reply["result"] if isinstance(reply.get("result"), str) else Failure("the reviewer replied with no report")
def launch(reviewer: str, rung: Rung, prompt: str, checkout: str, copies: Copies, home: pathlib.Path, change: Change | None) -> tuple[list[str], str] | Failure:
"""claude's arguments and prompt for reviewer on rung, in copies.clone, a copy of the project whose checkout is checkout. Where change is given, claude starts under the environment change holds, which claude's settings set; where change turns the instruction files off, the prompt holds what the SubagentStart hook and the header's last line hand such a reviewer."""
args = ["--agent", reviewer, *rung.flags()]
if change is None:
return args, prompt
# Another --settings would replace this one, so the whole environment goes in one.
args += ["--settings", json.dumps({"env": change.env})]
if not change.without_instruction_files:
return args, prompt
context = reviewer_context(home)
if isinstance(context, Failure):
return context
config = located(checkout, home)
if isinstance(config, Failure):
return config
record = store(config, "decisions")
# No commit holds a decision log outside the checkout, so the reviewer reads it as it stands today.
decisions = local_file(record, pathlib.Path(copies.reports, "decisions.md")) if record.tracked is None else pathlib.Path(copies.clone, record.tracked)
if isinstance(decisions, Failure):
return decisions
tracked = config.tracked("working-rules")
# Likewise for an `AGENTS.md` outside the checkout.
rules = config.paths["working-rules"] if tracked is None else pathlib.Path(copies.clone, tracked)
return args, "\n".join([f"{context}\n\n{prompt}", *([principles] if (principles := principles_line(rules, decisions)) else [])])
def copy_reports(found: Round, folder: pathlib.Path, reports: str) -> None:
"""Copies into reports each report written to folder after found's chunk began and before its round did."""
# The round's own report, any later one, and one from an earlier attempt at the item in the same worktree would each hand the reviewer the findings it is to find.
for written in folder.iterdir() if folder.is_dir() else ():
if not written.is_file() or not found.began <= written.stat().st_mtime < found.seconds:
continue
name = written.name.removesuffix(SUFFIX)
if name == written.name:
shutil.copy2(written, reports)
# A gzipped report. Where a plain report of the same name stands beside it, that one is newer and the gzip is skipped; the plain report is copied only if it was written before the round.
elif not (folder / name).is_file():
(pathlib.Path(reports) / name).write_bytes(read_stored(folder / name))
def redirected(prompt: str, home: pathlib.Path, folder: pathlib.Path, worktree: str, copies: Copies) -> str | Failure:
"""prompt naming copies.reports in place of the review folder, in either spelling, and copies.clone in place of the worktree; a Failure where it still names the review folder."""
# Replace the folder's path first: outside home it contains the worktree's path, which replacing the worktree's would otherwise turn into the clone's.
prompt = prompt.replace(str(folder), copies.reports).replace(f"~/{folder.relative_to(home)}", copies.reports).replace(worktree, copies.clone)
# A spelling the rewrite misses, such as $HOME/, would leave the reviewer the live folder and the findings in it.
if REPORTS in prompt.replace(copies.reports, ""):
return Failure("the prompt names the review folder in a spelling the replay does not redirect")
return prompt
def clone_reviewed(found: Round, code: Code, clone: str, effects: Effects) -> Failure | None:
"""Clones found's checkout into clone, on the commit its reviewer read, with the default branch where it stood as the chunk began."""
for command in (
("clone", "--quiet", "--shared", "--no-checkout", found.root, clone),
("-C", clone, "update-ref", f"refs/remotes/origin/{code.default}", code.base),
("-C", clone, "update-ref", f"refs/heads/{code.default}", code.base),
("-C", clone, "checkout", "--quiet", *(("-B", found.branch) if found.branch and found.branch != code.default else ("--detach",)), code.reviewed),
):
if isinstance(ran := effects.git(*command, cwd=found.root), Failure):
return ran
return None
def match(findings: tuple[str, ...], replay_report: str, root: str, rung: Rung, effects: Effects) -> set[int] | Failure | Limited:
numbered = "\n\n".join(f"{n}. {finding}" for n, finding in enumerate(findings, 1))
prompt = (
"A code reviewer reported the numbered findings below. Another reviewer then reviewed the same change; its report follows them. "
"List the number of each finding that the other report also raises as 🔴 or 🟡: the same defect, at the same place or from the same cause, however it is worded. "
"A finding it raises only as 🟢 or ❓, or not at all, is not listed.\n\n"
f"The findings:\n{numbered}\n\nThe other report:\n{replay_report}"
)
args = ["--json-schema", json.dumps(MATCH_SCHEMA), *rung.flags(), "--tools", ""]
reply = claude_reply(effects.claude, args, root, prompt, "claude")
if isinstance(reply, Failure | Limited):
return reply
output = structured_output(reply)
numbers = output.get("found") if isinstance(output, dict) else None
if not isinstance(numbers, list) or not all(isinstance(n, int) for n in numbers):
return Failure(f"the match replied: {json.dumps(reply)}")
return {n - 1 for n in numbers if 1 <= n <= len(findings)}