#!/usr/bin/env python3 """Run larv-review bench runs: BENCH_SOURCES= bench.py -- ... The 2026-09-30 bench ran two versions of larv-review, main and rebuild. Each reviewed four real chunks, the cases, three times; compare.py then rules whether each version reviews at least as well as the other. The bench's inputs are untracked: - versions// beside this script: a copy of ~/.claude at that version, which each run reads in place of ~/.claude (run-prompt.md). - $BENCH_SOURCES/-main/: a clone of the case's project; CASES names the chunk's base and head commits in it. - $BENCH_SOURCES/-main.md: main's own review of the case, run on 2026-09-29. - $BENCH_SOURCES/comparison.md: what each case's real review found, one table row per issue, its ID led by the case's letter in compare.py's PREFIX. Each run leaves its clone in /, its report in .md and its claude stream in logs/. """ import concurrent.futures import functools import json import os import pathlib import shutil import subprocess import sys import time BENCH = pathlib.Path(__file__).resolve().parent REVIEW_GIT = BENCH / "review_git.py" CASES = { "cjk": ("151bf38^", "6c49912"), "35b": ("692433b", "1d369ec"), "brlan": ("59ce55b^", "92e466a"), "mypy": ("89a66d3", "b32d131"), } ATTEMPTS = 4 def permission_flags(clone, version_dir, report): """The tools a review needs, each limited to what it may touch; the run denies every other call.""" # A permission rule writes an absolute path as "//"; a trailing "/**" matches everything under it. trees = (f"/{clone}/**", f"/{version_dir}/**") # git can reach outside the clone through an argument, an option or a hook, so the run gets no bare git: # review_git.py is its one shell command (see its docstring). The clone's own .git and .claude stay # unwritable, lest an edit plant a hook or widen the run's tools. allowed = [ "Agent", "Skill(security-review)", f"Edit(/{clone}/**)", f"Edit(/{report})", # Claude Code, not the wrapper, refuses what a shell adds to this command: a redirect, `$(…)`, backticks, # or a second command after `;`, `&` or `&&`. It lets through `cd && …`, which is why run() names # the clone in REVIEW_GIT_CLONE, and a pipe into a reader such as `| head`. f"Bash(python3 {REVIEW_GIT} *)", *(f"{tool}({tree})" for tool in ("Read", "Grep", "Glob") for tree in trees), ] denied = ["Bash(git *)", f"Edit(/{clone}/.git/**)", f"Edit(/{clone}/.claude/**)"] # No settings source: ~/.claude/settings.json would add ~/code to the trees and run its hooks, and a case's # committed .claude/settings.json or settings.local.json could run its own. --strict-mcp-config ignores a # committed .mcp.json. return ["--setting-sources", "", "--strict-mcp-config", "--permission-mode", "dontAsk", "--permission-prompts", "none", "--allowedTools", *allowed, "--disallowedTools", *denied] def git(repo, *args): return subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True).stdout.strip() def fresh_clone(case, dest, sources): if dest.exists(): shutil.rmtree(dest) source = sources / f"{case}-main" base_ref, head_ref = CASES[case] base = git(source, "rev-parse", base_ref) head = git(source, "rev-parse", head_ref) subprocess.run(["git", "clone", "-q", "--no-local", str(source), str(dest)], check=True) git(dest, "checkout", "-q", "-B", "chunk", head) for branch in git(dest, "for-each-ref", "--format=%(refname:short)", "refs/heads").split(): if branch != "chunk": git(dest, "branch", "-D", branch) for tag in git(dest, "tag").split(): git(dest, "tag", "-d", tag) git(dest, "remote", "remove", "origin") git(dest, "reflog", "expire", "--expire=now", "--all") git(dest, "gc", "-q", "--prune=now") return base, head def rate_limit_reset(log): resets = [0] for line in log.read_text().splitlines(): if '"status":"rejected"' in line: resets.append(json.loads(line)["rate_limit_info"]["resetsAt"]) return max(resets) def run(name, sources): """One review run of --, whose case's clone is in the folder sources.""" case, version, _ = name.split("-") clone = BENCH / name report = BENCH / f"{name}.md" for attempt in range(1, ATTEMPTS + 1): if report.exists(): return f"{name}: done" base, head = fresh_clone(case, clone, sources) version_dir = BENCH / "versions" / version prompt = (BENCH / "run-prompt.md").read_text().format( version_dir=version_dir, base=base, head=head, case=case, report=report, review_git=REVIEW_GIT) log = BENCH / "logs" / f"{name}.{attempt}.jsonl" log.parent.mkdir(exist_ok=True) with log.open("w") as out: subprocess.run( ["claude", "-p", prompt, *permission_flags(clone, version_dir, report), "--model", "claude-opus-5-5", "--output-format", "stream-json", "--verbose"], cwd=clone, env={**os.environ, "REVIEW_GIT_CLONE": str(clone)}, stdout=out, stderr=subprocess.STDOUT) if report.exists(): return f"{name}: done on attempt {attempt}" time.sleep(max(1200, rate_limit_reset(log) + 120 - time.time())) return f"{name}: FAILED after {ATTEMPTS} attempts" def attempted(name, run): """run's result line for name, or a line naming the failure, so one failed run doesn't stop the other runs' results printing.""" try: return run(name) except (KeyError, ValueError, subprocess.CalledProcessError, OSError) as failure: return f"{name}: FAILED: {type(failure).__name__}: {failure}" def main(names, run): """Prints the result line of each run named, which run makes.""" with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: for result in pool.map(lambda name: attempted(name, run), names): print(result, flush=True) if __name__ == "__main__": main(sys.argv[1:], functools.partial(run, sources=pathlib.Path(os.environ["BENCH_SOURCES"])))