#!/usr/bin/env python3 """Bench the advisor tool on archived chunks: advisor_bench.py run|grade -- ..., then advisor_bench.py summary The question: does a worker below the project's rung, Sonnet 5.5 with Opus 5.5 as its advisor, beat the same Sonnet alone, or match Opus 5.5 at medium for less? Each case is an archived L 1 or L 2 chunk of this repo (CASES holds its todo.md item, the commit it started from and the commit it merged at). `run` clones the repo at the start commit and hands the arm's worker the item's text from todo.md, with the bench rules of advisor-run-prompt.md in place of loop-claude's prompt. It saves, beside the clone in : the claude stream (.jsonl), the worker's change (.diff) and the result of the tests of the packages that either change touches (.tests). `grade` has a judge on Opus 5.5 at medium, blind to the arm, rule on the .diff against the merged commit's change (.verdict); `summary` prints a table per arm. Run each arm of each case once; the table shows a direction, not a significance. """ import concurrent.futures import dataclasses import functools import json import pathlib import re import subprocess import sys import typing BENCH = pathlib.Path(__file__).resolve().parent REPO = BENCH.parent RUN_HEAVY = pathlib.Path.home() / ".claude/bin/run-heavy" SUITE = ["uvx", "--python", sys.executable, "--with", "pytest-xdist==3.8.0", "pytest==9.2.0", "-n", "4", "-q"] CASES = { "290": ("cda45a931", "509804da0"), "251": ("b74322892", "d13c4fb0d"), "218": ("62d57344d", "c424fa66a"), "289": ("034356f34", "62d1f5edd"), "229": ("e2aca90c9", "2321f4930"), "214": ("39dd51bb4", "4764d3d52"), } @dataclasses.dataclass(frozen=True) class Arm: model: str effort: str advisor: str | None = None def flags(self): return ["--model", self.model, "--effort", self.effort, *(["--advisor", self.advisor] if self.advisor else [])] ARMS = { "sonnet-advisor": Arm("claude-sonnet-5-5", "high", "claude-opus-5-5"), "sonnet": Arm("claude-sonnet-5-5", "high"), "opus-medium": Arm("claude-opus-5-5", "medium"), } VERDICTS = ("pass", "partial", "fail") JUDGE = Arm("claude-opus-5-5", "medium") @dataclasses.dataclass(frozen=True) class Bench: """What a bench's steps act on, which the script's entry point hands them.""" # The repo whose archived chunks the bench reruns. repo: pathlib.Path # Each case's start commit and the commit it merged at, by todo item. cases: dict # Runs claude and the tests, as subprocess.run does. process: typing.Callable @dataclasses.dataclass(frozen=True) class Outcome: cost: float seconds: float advisor_calls: int def item_text(todo, item): """The item's heading and detail in todo.md, up to the next item's heading.""" found = re.search(rf"^### {item}\. .*?(?=^### \d+\. |\Z)", todo, re.DOTALL | re.MULTILINE) if not found: raise ValueError(f"item {item} has no detail in todo.md") return found.group().strip() def read_result(lines): """The cost, seconds and advisor calls of a claude stream's first result line, which is the worker's: a background task's notification makes the later ones. The cost counts the advisor's tokens.""" for line in lines: try: entry = json.loads(line) except ValueError: continue if entry.get("type") == "result": if entry.get("is_error"): raise ValueError(f"the run ended in an error: {entry.get('result')}") iterations = entry.get("usage", {}).get("iterations", []) return Outcome( cost=entry["total_cost_usd"], seconds=entry["duration_ms"] / 1000, advisor_calls=sum(1 for each in iterations if each.get("type") == "advisor_message")) raise ValueError("the stream has no result line") def verdict_of(reply): """The verdict in the judge's `claude -p --output-format json` reply.""" found = re.search(r"\{.*\}", json.loads(reply)["result"], re.DOTALL) if not found: raise ValueError("the judge's reply has no JSON object") verdict = json.loads(found.group())["verdict"] if verdict not in VERDICTS: raise ValueError(f"the judge's verdict {verdict!r} is none of {VERDICTS}") return verdict def split_name(name): """The case, arm and run number of a run's name, such as 290-sonnet-advisor-1.""" case, rest = name.split("-", 1) arm, number = rest.rsplit("-", 1) return case, arm, number def suite_targets(changed): """The tests to run for a change: the package, or the lone module, under bin/ of each changed file, else all of bin/.""" found = {"/".join(path.split("/")[:2]) for path in changed if path.startswith("bin/")} return sorted(found) or ["bin"] def summary(rows): """A markdown table with one row per arm, in ARMS' order: its runs, passes, partials, mean cost, mean advisor calls and mean seconds.""" lines = ["| arm | runs | pass | partial | mean cost $ | mean advisor calls | mean seconds |", "|---|---|---|---|---|---|---|"] for name in ARMS: own = [row for row in rows if row["arm"] == name] if own: passes, partials = (sum(row["verdict"] == kind for row in own) for kind in ("pass", "partial")) cost, advisor_calls, seconds = (sum(row[key] for row in own) / len(own) for key in ("cost", "advisor_calls", "seconds")) lines.append(f"| {name} | {len(own)} | {passes} | {partials} | {cost:.2f} | {advisor_calls:.1f} | {seconds:.0f} |") return "\n".join(lines) def git(repo, *args): return subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True).stdout def fresh_clone(repo, base, dest): """A clone of repo at base, with no remote to push to.""" subprocess.run(["git", "clone", "-q", "--no-local", str(repo), str(dest)], check=True) git(dest, "checkout", "-q", "--detach", base) git(dest, "remote", "remove", "origin") def run(bench, work, name): """One worker run of --; the files it leaves are in the module docstring.""" case, arm, _ = split_name(name) base = bench.cases[case][0] clone = work / name if clone.exists(): raise ValueError(f"{name} already ran: remove {clone}") fresh_clone(bench.repo, base, clone) prompt = (BENCH / "advisor-run-prompt.md").read_text().format(base=base, item=case, text=item_text(git(bench.repo, "show", f"{base}:todo.md"), case)) with (work / f"{name}.jsonl").open("w") as out: bench.process( ["claude", "-p", prompt, *ARMS[arm].flags(), "--setting-sources", "project", "--permission-mode", "bypassPermissions", "--no-session-persistence", "--output-format", "stream-json", "--verbose"], cwd=clone, stdout=out, stderr=subprocess.STDOUT) read_result((work / f"{name}.jsonl").read_text().splitlines()) save_change(bench, work, name, clone, bench.cases[case]) return f"{name}: ran" def save_change(bench, work, name, clone, commits): """Save the clone's change since the case's start commit as .diff, and as .tests the result of the tests of the packages that the clone's change or the merged change touches.""" base, merged = commits git(clone, "add", "--all") (work / f"{name}.diff").write_text(git(clone, "diff", "--cached", base)) changed = [*git(clone, "diff", "--cached", "--name-only", base).split(), *git(bench.repo, "diff", "--name-only", base, merged).split()] targets = [target for target in suite_targets(changed) if (clone / target).exists()] or ["bin"] tests = bench.process([str(RUN_HEAVY), *SUITE, *targets], cwd=clone, capture_output=True, text=True) (work / f"{name}.tests").write_text(f"exit {tests.returncode}\n{tests.stdout[-1500:]}") def grade(bench, work, name): """Have the judge rule on one run's diff, blind to its arm, and save the verdict beside it.""" case = split_name(name)[0] base, merged = bench.cases[case] prompt = (BENCH / "advisor-judge-prompt.md").read_text().format( text=item_text(git(bench.repo, "show", f"{base}:todo.md"), case), reference=git(bench.repo, "diff", base, merged, "--", ".", ":!todo.md", ":!docs", ":!CHANGELOG.md"), diff=(work / f"{name}.diff").read_text(), tests=(work / f"{name}.tests").read_text()) reply = bench.process( ["claude", "-p", prompt, *JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", "", "--strict-mcp-config"], capture_output=True, text=True, check=True).stdout verdict = verdict_of(reply) (work / f"{name}.verdict").write_text(json.dumps({"verdict": verdict, "reply": json.loads(reply)["result"]})) return f"{name}: {verdict}" def attempted(step, work, name): """step's result line, or a line naming the failure, so one failed run doesn't stop the other runs' results printing.""" try: return step(work, name) except (KeyError, ValueError, subprocess.CalledProcessError, OSError) as failure: return f"{name}: FAILED: {type(failure).__name__}: {failure}" def rows_of(work): rows = [] for verdict_file in sorted(work.glob("*.verdict")): name = verdict_file.stem try: outcome = read_result((work / f"{name}.jsonl").read_text().splitlines()) except ValueError as failure: raise ValueError(f"{name}: {failure}") from failure rows.append({"arm": split_name(name)[1], "verdict": json.loads(verdict_file.read_text())["verdict"], **dataclasses.asdict(outcome)}) return rows def steps_of(bench): """The steps main runs by name, each on bench.""" return {"run": functools.partial(run, bench), "grade": functools.partial(grade, bench)} def main(argv, steps, say): """Runs the step argv names on each run it names, and hands say each result line; or hands say the summary.""" work, step, names = pathlib.Path(argv[0]), argv[1], argv[2:] if step == "summary": say(summary(rows_of(work))) return with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: for result in pool.map(lambda name: attempted(steps[step], work, name), names): say(result) if __name__ == "__main__": main(sys.argv[1:], steps_of(Bench(REPO, CASES, subprocess.run)), functools.partial(print, flush=True))