124 lines
5.1 KiB
Python
Executable File
124 lines
5.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Run larv-review bench runs: BENCH_SOURCES=<dir> bench.py <case>-<version>-<n> ...
|
|
|
|
The 2026-09-30 bench ran two versions of larv-review, main and rebuild. Each reviewed four real chunks,
|
|
the cases, three times; compare.py then rules whether each version reviews at least as well as the
|
|
other. The bench's inputs are untracked:
|
|
- versions/<version>/ beside this script: a copy of ~/.claude at that version, which each run reads
|
|
in place of ~/.claude (run-prompt.md).
|
|
- $BENCH_SOURCES/<case>-main/: a clone of the case's project; CASES names the chunk's base and head
|
|
commits in it.
|
|
- $BENCH_SOURCES/<case>-main.md: main's own review of the case, run on 2026-09-29.
|
|
- $BENCH_SOURCES/comparison.md: what each case's real review found, one table row per issue, its ID
|
|
led by the case's letter in compare.py's PREFIX.
|
|
Each run leaves its clone in <run>/, its report in <run>.md and its claude stream in logs/.
|
|
"""
|
|
import concurrent.futures
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
BENCH = pathlib.Path(__file__).resolve().parent
|
|
REVIEW_GIT = BENCH / "review_git.py"
|
|
SOURCES = pathlib.Path(os.environ["BENCH_SOURCES"])
|
|
CASES = {
|
|
"cjk": ("151bf38^", "6c49912"),
|
|
"35b": ("692433b", "1d369ec"),
|
|
"brlan": ("59ce55b^", "92e466a"),
|
|
"mypy": ("89a66d3", "b32d131"),
|
|
}
|
|
ATTEMPTS = 4
|
|
|
|
|
|
def permission_flags(clone, version_dir, report):
|
|
"""The tools a review needs, each limited to what it may touch; the run denies every other call."""
|
|
# A permission rule writes an absolute path as "//<path>"; a trailing "/**" matches everything under it.
|
|
trees = (f"/{clone}/**", f"/{version_dir}/**")
|
|
# git can reach outside the clone through an argument, an option or a hook, so the run gets no bare git:
|
|
# review_git.py is its one shell command (see its docstring). The clone's own .git and .claude stay
|
|
# unwritable, lest an edit plant a hook or widen the run's tools.
|
|
allowed = [
|
|
"Agent", "Skill(security-review)", f"Edit(/{clone}/**)", f"Edit(/{report})",
|
|
# Claude Code, not the wrapper, refuses what a shell adds to this command: a redirect, `$(…)`, backticks,
|
|
# or a second command after `;`, `&` or `&&`. It lets through `cd <dir> && …`, which is why run() names
|
|
# the clone in REVIEW_GIT_CLONE, and a pipe into a reader such as `| head`.
|
|
f"Bash(python3 {REVIEW_GIT} *)",
|
|
*(f"{tool}({tree})" for tool in ("Read", "Grep", "Glob") for tree in trees),
|
|
]
|
|
denied = ["Bash(git *)", f"Edit(/{clone}/.git/**)", f"Edit(/{clone}/.claude/**)"]
|
|
# No settings source: ~/.claude/settings.json would add ~/code to the trees and run its hooks, and a case's
|
|
# committed .claude/settings.json or settings.local.json could run its own. --strict-mcp-config ignores a
|
|
# committed .mcp.json.
|
|
return ["--setting-sources", "", "--strict-mcp-config", "--permission-mode", "dontAsk", "--permission-prompts", "none",
|
|
"--allowedTools", *allowed, "--disallowedTools", *denied]
|
|
|
|
|
|
def git(repo, *args):
|
|
return subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True).stdout.strip()
|
|
|
|
|
|
def fresh_clone(case, dest):
|
|
if dest.exists():
|
|
shutil.rmtree(dest)
|
|
source = SOURCES / f"{case}-main"
|
|
base_ref, head_ref = CASES[case]
|
|
base = git(source, "rev-parse", base_ref)
|
|
head = git(source, "rev-parse", head_ref)
|
|
subprocess.run(["git", "clone", "-q", "--no-local", str(source), str(dest)], check=True)
|
|
git(dest, "checkout", "-q", "-B", "chunk", head)
|
|
for branch in git(dest, "for-each-ref", "--format=%(refname:short)", "refs/heads").split():
|
|
if branch != "chunk":
|
|
git(dest, "branch", "-D", branch)
|
|
for tag in git(dest, "tag").split():
|
|
git(dest, "tag", "-d", tag)
|
|
git(dest, "remote", "remove", "origin")
|
|
git(dest, "reflog", "expire", "--expire=now", "--all")
|
|
git(dest, "gc", "-q", "--prune=now")
|
|
return base, head
|
|
|
|
|
|
def rate_limit_reset(log):
|
|
resets = [0]
|
|
for line in log.read_text().splitlines():
|
|
if '"status":"rejected"' in line:
|
|
resets.append(json.loads(line)["rate_limit_info"]["resetsAt"])
|
|
return max(resets)
|
|
|
|
|
|
def run(name):
|
|
case, version, _ = name.split("-")
|
|
clone = BENCH / name
|
|
report = BENCH / f"{name}.md"
|
|
for attempt in range(1, ATTEMPTS + 1):
|
|
if report.exists():
|
|
return f"{name}: done"
|
|
base, head = fresh_clone(case, clone)
|
|
version_dir = BENCH / "versions" / version
|
|
prompt = (BENCH / "run-prompt.md").read_text().format(
|
|
version_dir=version_dir, base=base, head=head, case=case, report=report, review_git=REVIEW_GIT)
|
|
log = BENCH / "logs" / f"{name}.{attempt}.jsonl"
|
|
log.parent.mkdir(exist_ok=True)
|
|
with log.open("w") as out:
|
|
subprocess.run(
|
|
["claude", "-p", prompt, *permission_flags(clone, version_dir, report),
|
|
"--model", "claude-opus-5-5", "--output-format", "stream-json", "--verbose"],
|
|
cwd=clone, env={**os.environ, "REVIEW_GIT_CLONE": str(clone)}, stdout=out, stderr=subprocess.STDOUT)
|
|
if report.exists():
|
|
return f"{name}: done on attempt {attempt}"
|
|
time.sleep(max(1200, rate_limit_reset(log) + 120 - time.time()))
|
|
return f"{name}: FAILED after {ATTEMPTS} attempts"
|
|
|
|
|
|
def main():
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
|
for result in pool.map(run, sys.argv[1:]):
|
|
print(result, flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|