#!/usr/bin/env python3 """Blind-compare each case's six runs: BENCH_SOURCES= compare.py ...""" import concurrent.futures import json import pathlib import random import re import shutil import subprocess import sys import bench BENCH = bench.BENCH TRUTH_END = {"cjk": "160c2a7", "35b": "a4b6b48", "brlan": "8f6e22d", "mypy": "628ee30"} PREFIX = {"cjk": "C", "35b": "B", "brlan": "R", "mypy": "M"} def runs(case, version): """The version's first three runs as (report, clone), main's 2026-09-29 run last.""" found = [] for n in range(1, 4): report = BENCH / f"{case}-{version}-{n}.md" if report.exists(): found.append((report, BENCH / f"{case}-{version}-{n}")) if version == "main": found.append((bench.SOURCES / f"{case}-main.md", bench.SOURCES / f"{case}-main")) return found[:3] def blind(text): """The report, with VERSION_DIR in place of every name that tells which version wrote it.""" return re.sub(r"\S*versions/(main|rebuild)\S*|\b(main|rebuild)\b(?=[`\s]*(process|version))", "VERSION_DIR", text) def prepare(case): folder = BENCH / "compare" / case if folder.exists(): shutil.rmtree(folder) folder.mkdir(parents=True) source = bench.SOURCES / f"{case}-main" head = bench.git(source, "rev-parse", bench.CASES[case][1]) (folder / "truth.diff").write_text(bench.git(source, "diff", head, TRUTH_END[case]) + "\n") rows = [line.split("|")[1:3] for line in (bench.SOURCES / "comparison.md").read_text().splitlines() if re.match(rf"\| {PREFIX[case]}\d+ \|", line)] (folder / "truth-issues.md").write_text("".join(f"-{i.strip()}: {t.strip()}\n" for i, t in rows)) groups = ["main", "rebuild"] random.shuffle(groups) mapping = {} for label, version in zip("AB", groups): for n, (report, clone) in enumerate(runs(case, version), 1): run = folder / f"{label}{n}" run.mkdir() (run / "report.md").write_text(blind(report.read_text())) (run / "fixes.diff").write_text(bench.git(clone, "diff", head, "chunk") + "\n") mapping[f"{label}{n}"] = str(report) (BENCH / "compare" / f"{case}.mapping.json").write_text(json.dumps(mapping, indent="\t") + "\n") return folder def compare(case): folder = prepare(case) ruling = BENCH / "compare" / f"{case}.ruling.md" prompt = (BENCH / "compare-prompt.md").read_text().format(dir=folder, ruling=ruling) with (BENCH / "logs" / f"compare-{case}.jsonl").open("w") as out: subprocess.run( ["claude", "-p", prompt, "--model", "claude-opus-5-5", "--allowedTools", "Read,Grep,Glob,Write", "--output-format", "stream-json", "--verbose"], cwd=folder, stdout=out, stderr=subprocess.STDOUT) return f"{case}: {'ruled' if ruling.exists() else 'FAILED'}" if __name__ == "__main__": with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: for result in pool.map(compare, sys.argv[1:]): print(result, flush=True)