Files
claude-config/workflow-bench/compare.py
T

76 lines
2.8 KiB
Python
Executable File

#!/usr/bin/env python3
"""Blind-compare each case's six runs: BENCH_SOURCES=<as for bench.py> compare.py <case> ..."""
import concurrent.futures
import json
import pathlib
import random
import re
import shutil
import subprocess
import sys
import bench
BENCH = bench.BENCH
TRUTH_END = {"cjk": "160c2a7", "35b": "a4b6b48", "brlan": "8f6e22d", "mypy": "628ee30"}
PREFIX = {"cjk": "C", "35b": "B", "brlan": "R", "mypy": "M"}
def runs(case, version):
"""The version's first three runs as (report, clone), main's 2026-09-29 run last."""
found = []
for n in range(1, 4):
report = BENCH / f"{case}-{version}-{n}.md"
if report.exists():
found.append((report, BENCH / f"{case}-{version}-{n}"))
if version == "main":
found.append((bench.SOURCES / f"{case}-main.md", bench.SOURCES / f"{case}-main"))
return found[:3]
def blind(text):
"""The report with every name that tells which version wrote it replaced."""
return re.sub(r"\S*versions/(main|rebuild)\S*|\b(main|rebuild)\b(?=[`\s]*(process|version))", "VERSION_DIR", text)
def prepare(case):
folder = BENCH / "compare" / case
if folder.exists():
shutil.rmtree(folder)
folder.mkdir(parents=True)
source = bench.SOURCES / f"{case}-main"
head = bench.git(source, "rev-parse", bench.CASES[case][1])
(folder / "truth.diff").write_text(bench.git(source, "diff", head, TRUTH_END[case]) + "\n")
rows = [line.split("|")[1:3] for line in (bench.SOURCES / "comparison.md").read_text().splitlines()
if re.match(rf"\| {PREFIX[case]}\d+ \|", line)]
(folder / "truth-issues.md").write_text("".join(f"-{i.strip()}: {t.strip()}\n" for i, t in rows))
groups = ["main", "rebuild"]
random.shuffle(groups)
mapping = {}
for label, version in zip("AB", groups):
for n, (report, clone) in enumerate(runs(case, version), 1):
run = folder / f"{label}{n}"
run.mkdir()
(run / "report.md").write_text(blind(report.read_text()))
(run / "fixes.diff").write_text(bench.git(clone, "diff", head, "chunk") + "\n")
mapping[f"{label}{n}"] = str(report)
(BENCH / "compare" / f"{case}.mapping.json").write_text(json.dumps(mapping, indent="\t") + "\n")
return folder
def compare(case):
folder = prepare(case)
ruling = BENCH / "compare" / f"{case}.ruling.md"
prompt = (BENCH / "compare-prompt.md").read_text().format(dir=folder, ruling=ruling)
with (BENCH / "logs" / f"compare-{case}.jsonl").open("w") as out:
subprocess.run(
["claude", "-p", prompt, "--model", "claude-opus-5-5", "--allowedTools", "Read,Grep,Glob,Write", "--output-format", "stream-json", "--verbose"],
cwd=folder, stdout=out, stderr=subprocess.STDOUT)
return f"{case}: {'ruled' if ruling.exists() else 'FAILED'}"
if __name__ == "__main__":
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
for result in pool.map(compare, sys.argv[1:]):
print(result, flush=True)