Files
claude-config/workflow-bench/bench.py
T

97 lines
3.4 KiB
Python
Executable File

#!/usr/bin/env python3
"""Run larv-review bench runs: BENCH_SOURCES=<dir> bench.py <case>-<version>-<n> ...
The 2026-09-30 bench had two versions of larv-review, main and rebuild, each review four real chunks,
the cases, three times; compare.py then rules which version reviews better. Its inputs are untracked:
- versions/<version>/ beside this script: a copy of ~/.claude at that version, which each run reads
in place of ~/.claude (run-prompt.md).
- $BENCH_SOURCES/<case>-main/: a clone of the case's project; CASES names the chunk's base and head
commits in it.
- $BENCH_SOURCES/<case>-main.md: main's own review of the case, run on 2026-09-29.
- $BENCH_SOURCES/comparison.md: what each case's real review found, one table row per issue, its ID
led by the case's letter in compare.py's PREFIX.
Each run leaves its clone in <run>/, its report in <run>.md and its claude stream in logs/.
"""
import concurrent.futures
import json
import os
import pathlib
import shutil
import subprocess
import sys
import time
BENCH = pathlib.Path(__file__).resolve().parent
SOURCES = pathlib.Path(os.environ["BENCH_SOURCES"])
CASES = {
"cjk": ("151bf38^", "6c49912"),
"35b": ("692433b", "1d369ec"),
"brlan": ("59ce55b^", "92e466a"),
"mypy": ("89a66d3", "b32d131"),
}
ATTEMPTS = 4
def git(repo, *args):
return subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True).stdout.strip()
def fresh_clone(case, dest):
if dest.exists():
shutil.rmtree(dest)
source = SOURCES / f"{case}-main"
base_ref, head_ref = CASES[case]
base = git(source, "rev-parse", base_ref)
head = git(source, "rev-parse", head_ref)
subprocess.run(["git", "clone", "-q", "--no-local", str(source), str(dest)], check=True)
git(dest, "checkout", "-q", "-B", "chunk", head)
for branch in git(dest, "for-each-ref", "--format=%(refname:short)", "refs/heads").split():
if branch != "chunk":
git(dest, "branch", "-D", branch)
for tag in git(dest, "tag").split():
git(dest, "tag", "-d", tag)
git(dest, "remote", "remove", "origin")
git(dest, "reflog", "expire", "--expire=now", "--all")
git(dest, "gc", "-q", "--prune=now")
return base, head
def rate_limit_reset(log):
resets = [0]
for line in log.read_text().splitlines():
if '"status":"rejected"' in line:
resets.append(json.loads(line)["rate_limit_info"]["resetsAt"])
return max(resets)
def run(name):
case, version, _ = name.split("-")
clone = BENCH / name
report = BENCH / f"{name}.md"
for attempt in range(1, ATTEMPTS + 1):
if report.exists():
return f"{name}: done"
base, head = fresh_clone(case, clone)
prompt = (BENCH / "run-prompt.md").read_text().format(
version_dir=BENCH / "versions" / version, base=base, head=head, case=case, report=report)
log = BENCH / "logs" / f"{name}.{attempt}.jsonl"
log.parent.mkdir(exist_ok=True)
with log.open("w") as out:
subprocess.run(
["claude", "-p", prompt, "--dangerously-skip-permissions", "--model", "claude-opus-5-5", "--output-format", "stream-json", "--verbose"],
cwd=clone, stdout=out, stderr=subprocess.STDOUT)
if report.exists():
return f"{name}: done on attempt {attempt}"
time.sleep(max(1200, rate_limit_reset(log) + 120 - time.time()))
return f"{name}: FAILED after {ATTEMPTS} attempts"
def main():
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
for result in pool.map(run, sys.argv[1:]):
print(result, flush=True)
if __name__ == "__main__":
main()