203 lines
12 KiB
Python
203 lines
12 KiB
Python
"""Replay one archived review round on one rung: the code the round reviewed, found again in a throwaway clone, the reviewer's report on it, and the match of that report to the round's archived findings: docs/decisions.md → `routing: a review pass runs below the project's rung only on a rung whose replay of its rounds finds a share of their 🔴 and 🟡 findings no more than a margin below the project's rung's own replay`."""
|
|
import dataclasses
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import tempfile
|
|
import typing
|
|
|
|
from review_replay.rounds import authored_by, lost_inputs, reviewed_path, Round
|
|
from review_replay.changes import Change
|
|
from claude_code.reply import claude_reply, Limited
|
|
from git_cli.git_repo import default_ref
|
|
from loop_effects.effects import Effects
|
|
from failure import Failure
|
|
from rung_replay.search import first_reply
|
|
from record_store.stores import local_file, store
|
|
from project_config.main import located
|
|
from review_rules.reviewer_context import principles_line, reviewer_context
|
|
from text_files.compressed import read_stored, SUFFIX
|
|
from review_rules.review_folder import REPORTS, review_folder
|
|
from model_routing.ladder import Rung
|
|
from claude_code.entries import structured_output
|
|
|
|
MATCH_SCHEMA = {"type": "object", "properties": {"found": {"type": "array", "items": {"type": "integer"}}}, "required": ["found"], "additionalProperties": False}
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Code:
|
|
"""Where a round's reviewer read the code, the commit it read, the default branch, and where that branch stood as the chunk began."""
|
|
path: str
|
|
reviewed: str
|
|
default: str
|
|
base: str
|
|
|
|
|
|
def codes_found(rounds: list[Round], effects: Effects, say: typing.Callable[[str], None]) -> dict[Round, Code]:
|
|
"""The code each round reviewed, as code_of finds it; say is told each round left out, and why."""
|
|
codes = {}
|
|
for found in rounds:
|
|
if isinstance(code := code_of(found, effects), Failure):
|
|
say(f"{found.root} {found.at} is left out: {code.reason}")
|
|
else:
|
|
codes[found] = code
|
|
return codes
|
|
|
|
|
|
def code_of(found: Round, effects: Effects) -> Code | Failure:
|
|
"""Where, and at which commits, found's reviewer read the code, as the archive lets it be found again: the folder rounds.reviewed_path guesses from the prompt, and the commits authored by then.
|
|
|
|
Recording the reviewed commit at review time would need no such reading, but then no round archived before that change could be replayed. A wrong guess shows every rung the same wrong code: the project's rung finds fewer of that round's archived findings, which lowers the bar, and the rungs below find fewer too.
|
|
"""
|
|
if not os.path.isdir(found.root):
|
|
return Failure("the project has no checkout")
|
|
path = reviewed_path(found.prompt, found.root, os.path.exists)
|
|
if path is None:
|
|
return Failure("the prompt names neither one removed worktree nor the checkout")
|
|
ref = default_ref(cwd=found.root, guarded=True, run=effects.git)
|
|
if isinstance(ref, Failure):
|
|
return ref
|
|
default = ref.removeprefix("origin/")
|
|
log = effects.git("log", "--format=%H %at", f"refs/remotes/origin/{default}", cwd=found.root)
|
|
main_line = effects.git("log", "--first-parent", "--format=%H %at", f"refs/remotes/origin/{default}", cwd=found.root)
|
|
if isinstance(log, Failure):
|
|
return log
|
|
if isinstance(main_line, Failure):
|
|
return main_line
|
|
reviewed, base = authored_by(commit_log(log), found.seconds), authored_by(commit_log(main_line), found.began)
|
|
if reviewed is None or base is None:
|
|
return Failure("no commit was authored by then")
|
|
return Code(path, reviewed, default, base)
|
|
|
|
|
|
def commit_log(listed: str) -> list[tuple[str, float]]:
|
|
return [(commit, float(authored)) for commit, authored in (line.split() for line in listed.splitlines())]
|
|
|
|
|
|
def findings_replayed(found: Round, code: Code, rung: Rung, project: Rung, home: pathlib.Path, effects: Effects, change: Change | None, scratch: pathlib.Path | None) -> set[int] | Failure | Limited:
|
|
"""The indexes of the round's findings that its replay on rung reports, as claude on the project's rung matches them, or the Failure of the review or the match; where change is given, the replay runs with it, as `launch` says. The clone is made under scratch, or under the system's temporary folder where it is None."""
|
|
reviewed: str | Failure | Limited = first_reply(lambda: review(found, code, rung, home, effects, change, scratch))
|
|
if isinstance(reviewed, (Failure, Limited)):
|
|
return reviewed
|
|
return first_reply(lambda: match(found.findings, reviewed, found.root, project, effects))
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Copies:
|
|
"""The throwaway copies a replay hands its reviewer: clone, of the project's checkout, and reports, of the chunk's review folder."""
|
|
clone: str
|
|
reports: str
|
|
|
|
|
|
# The shell makes and removes the clone itself, and is tested on a temporary home as judge-replay's is.
|
|
def review(found: Round, code: Code, rung: Rung, home: pathlib.Path, effects: Effects, change: Change | None, scratch: pathlib.Path | None) -> str | Failure | Limited:
|
|
try:
|
|
(home / REPORTS).mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
with tempfile.TemporaryDirectory(prefix="review-replay-", dir=scratch, ignore_cleanup_errors=True) as clone, tempfile.TemporaryDirectory(prefix="review-replay-", dir=home / REPORTS, ignore_cleanup_errors=True) as reports:
|
|
return reviewed_in(found, code, rung, Copies(clone, reports), home, effects, change)
|
|
except OSError as failure:
|
|
return Failure(f"could not prepare the replay: {failure}")
|
|
|
|
|
|
def reviewed_in(found: Round, code: Code, rung: Rung, copies: Copies, home: pathlib.Path, effects: Effects, change: Change | None) -> str | Failure | Limited:
|
|
"""The report of found's reviewer on rung, replayed on code cloned into copies.clone, with its review folder copied into copies.reports; with change where given."""
|
|
folder = pathlib.Path(review_folder(str(home), code.path))
|
|
copy_reports(found, folder, copies.reports)
|
|
prompt = redirected(found.prompt, home, folder, code.path, copies)
|
|
if isinstance(prompt, Failure):
|
|
return prompt
|
|
if lost := lost_inputs(prompt, copies.reports, lambda name: (pathlib.Path(copies.reports) / name).exists()):
|
|
return Failure(f"the prompt names {', '.join(lost)}, which the review folder did not hold from the chunk's start until the round's")
|
|
if isinstance(cloned := clone_reviewed(found, code, copies.clone, effects), Failure):
|
|
return cloned
|
|
launched = launch(found.reviewer, rung, prompt, found.root, copies, home, change)
|
|
if isinstance(launched, Failure):
|
|
return launched
|
|
reply = claude_reply(effects.claude, launched[0], copies.clone, launched[1], "claude")
|
|
if isinstance(reply, Failure | Limited):
|
|
return reply
|
|
return reply["result"] if isinstance(reply.get("result"), str) else Failure("the reviewer replied with no report")
|
|
|
|
|
|
def launch(reviewer: str, rung: Rung, prompt: str, checkout: str, copies: Copies, home: pathlib.Path, change: Change | None) -> tuple[list[str], str] | Failure:
|
|
"""claude's arguments and prompt for reviewer on rung, in copies.clone, a copy of the project whose checkout is checkout. Where change is given, claude starts under the environment change holds, which claude's settings set; where change turns the instruction files off, the prompt holds what the SubagentStart hook and the header's last line hand such a reviewer."""
|
|
args = ["--agent", reviewer, *rung.flags()]
|
|
if change is None:
|
|
return args, prompt
|
|
# Another --settings would replace this one, so the whole environment goes in one.
|
|
args += ["--settings", json.dumps({"env": change.env})]
|
|
if not change.without_instruction_files:
|
|
return args, prompt
|
|
context = reviewer_context(home)
|
|
if isinstance(context, Failure):
|
|
return context
|
|
config = located(checkout, home)
|
|
if isinstance(config, Failure):
|
|
return config
|
|
record = store(config, "decisions")
|
|
# No commit holds a decision log outside the checkout, so the reviewer reads it as it stands today.
|
|
decisions = local_file(record, pathlib.Path(copies.reports, "decisions.md")) if record.tracked is None else pathlib.Path(copies.clone, record.tracked)
|
|
if isinstance(decisions, Failure):
|
|
return decisions
|
|
tracked = config.tracked("working-rules")
|
|
# Likewise for an `AGENTS.md` outside the checkout.
|
|
rules = config.paths["working-rules"] if tracked is None else pathlib.Path(copies.clone, tracked)
|
|
return args, "\n".join([f"{context}\n\n{prompt}", *([principles] if (principles := principles_line(rules, decisions)) else [])])
|
|
|
|
|
|
def copy_reports(found: Round, folder: pathlib.Path, reports: str) -> None:
|
|
"""Copies into reports each report written to folder after found's chunk began and before its round did."""
|
|
# The round's own report, any later one, and one from an earlier attempt at the item in the same worktree would each hand the reviewer the findings it is to find.
|
|
for written in folder.iterdir() if folder.is_dir() else ():
|
|
if not written.is_file() or not found.began <= written.stat().st_mtime < found.seconds:
|
|
continue
|
|
name = written.name.removesuffix(SUFFIX)
|
|
if name == written.name:
|
|
shutil.copy2(written, reports)
|
|
# A gzipped report. Where a plain report of the same name stands beside it, that one is newer and the gzip is skipped; the plain report is copied only if it was written before the round.
|
|
elif not (folder / name).is_file():
|
|
(pathlib.Path(reports) / name).write_bytes(read_stored(folder / name))
|
|
|
|
|
|
def redirected(prompt: str, home: pathlib.Path, folder: pathlib.Path, worktree: str, copies: Copies) -> str | Failure:
|
|
"""prompt naming copies.reports in place of the review folder, in either spelling, and copies.clone in place of the worktree; a Failure where it still names the review folder."""
|
|
# Replace the folder's path first: outside home it contains the worktree's path, which replacing the worktree's would otherwise turn into the clone's.
|
|
prompt = prompt.replace(str(folder), copies.reports).replace(f"~/{folder.relative_to(home)}", copies.reports).replace(worktree, copies.clone)
|
|
# A spelling the rewrite misses, such as $HOME/, would leave the reviewer the live folder and the findings in it.
|
|
if REPORTS in prompt.replace(copies.reports, ""):
|
|
return Failure("the prompt names the review folder in a spelling the replay does not redirect")
|
|
return prompt
|
|
|
|
|
|
def clone_reviewed(found: Round, code: Code, clone: str, effects: Effects) -> Failure | None:
|
|
"""Clones found's checkout into clone, on the commit its reviewer read, with the default branch where it stood as the chunk began."""
|
|
for command in (
|
|
("clone", "--quiet", "--shared", "--no-checkout", found.root, clone),
|
|
("-C", clone, "update-ref", f"refs/remotes/origin/{code.default}", code.base),
|
|
("-C", clone, "update-ref", f"refs/heads/{code.default}", code.base),
|
|
("-C", clone, "checkout", "--quiet", *(("-B", found.branch) if found.branch and found.branch != code.default else ("--detach",)), code.reviewed),
|
|
):
|
|
if isinstance(ran := effects.git(*command, cwd=found.root), Failure):
|
|
return ran
|
|
return None
|
|
|
|
|
|
def match(findings: tuple[str, ...], replay_report: str, root: str, rung: Rung, effects: Effects) -> set[int] | Failure | Limited:
|
|
numbered = "\n\n".join(f"{n}. {finding}" for n, finding in enumerate(findings, 1))
|
|
prompt = (
|
|
"A code reviewer reported the numbered findings below. Another reviewer then reviewed the same change; its report follows them. "
|
|
"List the number of each finding that the other report also raises as 🔴 or 🟡: the same defect, at the same place or from the same cause, however it is worded. "
|
|
"A finding it raises only as 🟢 or ❓, or not at all, is not listed.\n\n"
|
|
f"The findings:\n{numbered}\n\nThe other report:\n{replay_report}"
|
|
)
|
|
args = ["--json-schema", json.dumps(MATCH_SCHEMA), *rung.flags(), "--tools", ""]
|
|
reply = claude_reply(effects.claude, args, root, prompt, "claude")
|
|
if isinstance(reply, Failure | Limited):
|
|
return reply
|
|
output = structured_output(reply)
|
|
numbers = output.get("found") if isinstance(output, dict) else None
|
|
if not isinstance(numbers, list) or not all(isinstance(n, int) for n in numbers):
|
|
return Failure(f"the match replied: {json.dumps(reply)}")
|
|
return {n - 1 for n in numbers if 1 <= n <= len(findings)}
|