76 lines
2.8 KiB
Python
Executable File
76 lines
2.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Blind-compare each case's six runs: BENCH_SOURCES=<as for bench.py> compare.py <case> ..."""
|
|
import concurrent.futures
|
|
import json
|
|
import pathlib
|
|
import random
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
import bench
|
|
|
|
BENCH = bench.BENCH
|
|
TRUTH_END = {"cjk": "160c2a7", "35b": "a4b6b48", "brlan": "8f6e22d", "mypy": "628ee30"}
|
|
PREFIX = {"cjk": "C", "35b": "B", "brlan": "R", "mypy": "M"}
|
|
|
|
|
|
def runs(case, version):
|
|
"""The version's first three runs as (report, clone), main's 2026-09-29 run last."""
|
|
found = []
|
|
for n in range(1, 4):
|
|
report = BENCH / f"{case}-{version}-{n}.md"
|
|
if report.exists():
|
|
found.append((report, BENCH / f"{case}-{version}-{n}"))
|
|
if version == "main":
|
|
found.append((bench.SOURCES / f"{case}-main.md", bench.SOURCES / f"{case}-main"))
|
|
return found[:3]
|
|
|
|
|
|
def blind(text):
|
|
"""The report with every name that tells which version wrote it replaced."""
|
|
return re.sub(r"\S*versions/(main|rebuild)\S*|\b(main|rebuild)\b(?=[`\s]*(process|version))", "VERSION_DIR", text)
|
|
|
|
|
|
def prepare(case):
|
|
folder = BENCH / "compare" / case
|
|
if folder.exists():
|
|
shutil.rmtree(folder)
|
|
folder.mkdir(parents=True)
|
|
source = bench.SOURCES / f"{case}-main"
|
|
head = bench.git(source, "rev-parse", bench.CASES[case][1])
|
|
(folder / "truth.diff").write_text(bench.git(source, "diff", head, TRUTH_END[case]) + "\n")
|
|
rows = [line.split("|")[1:3] for line in (bench.SOURCES / "comparison.md").read_text().splitlines()
|
|
if re.match(rf"\| {PREFIX[case]}\d+ \|", line)]
|
|
(folder / "truth-issues.md").write_text("".join(f"-{i.strip()}: {t.strip()}\n" for i, t in rows))
|
|
groups = ["main", "rebuild"]
|
|
random.shuffle(groups)
|
|
mapping = {}
|
|
for label, version in zip("AB", groups):
|
|
for n, (report, clone) in enumerate(runs(case, version), 1):
|
|
run = folder / f"{label}{n}"
|
|
run.mkdir()
|
|
(run / "report.md").write_text(blind(report.read_text()))
|
|
(run / "fixes.diff").write_text(bench.git(clone, "diff", head, "chunk") + "\n")
|
|
mapping[f"{label}{n}"] = str(report)
|
|
(BENCH / "compare" / f"{case}.mapping.json").write_text(json.dumps(mapping, indent="\t") + "\n")
|
|
return folder
|
|
|
|
|
|
def compare(case):
|
|
folder = prepare(case)
|
|
ruling = BENCH / "compare" / f"{case}.ruling.md"
|
|
prompt = (BENCH / "compare-prompt.md").read_text().format(dir=folder, ruling=ruling)
|
|
with (BENCH / "logs" / f"compare-{case}.jsonl").open("w") as out:
|
|
subprocess.run(
|
|
["claude", "-p", prompt, "--model", "claude-opus-5-5", "--allowedTools", "Read,Grep,Glob,Write", "--output-format", "stream-json", "--verbose"],
|
|
cwd=folder, stdout=out, stderr=subprocess.STDOUT)
|
|
return f"{case}: {'ruled' if ruling.exists() else 'FAILED'}"
|
|
|
|
|
|
if __name__ == "__main__":
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
|
|
for result in pool.map(compare, sys.argv[1:]):
|
|
print(result, flush=True)
|