Files
2026-11-20 18:02:12 +01:00

230 lines
9.6 KiB
Python

#!/usr/bin/env python3
"""Bench the advisor tool on archived chunks: advisor_bench.py <work dir> run|grade <case>-<arm>-<n> ..., then advisor_bench.py <work dir> summary
The question: does a worker below the project's rung, Sonnet 5.5 with Opus 5.5 as its advisor, beat the same
Sonnet alone, or match Opus 5.5 at medium for less? Each case is an archived L 1 or L 2 chunk of this repo
(CASES holds its todo.md item, the commit it started from and the commit it merged at). `run` clones the repo at
the start commit and hands the arm's worker the item's text from todo.md, with the bench rules of
advisor-run-prompt.md in place of loop-claude's prompt. It saves, beside the clone in <work dir>: the claude
stream (.jsonl), the worker's change (.diff) and the result of the tests of the packages that either change touches (.tests).
`grade` has a judge on Opus 5.5 at medium, blind to the arm, rule on the .diff against the merged commit's change
(.verdict); `summary` prints a table per arm.
Run each arm of each case once; the table shows a direction, not a significance.
"""
import concurrent.futures
import dataclasses
import functools
import json
import pathlib
import re
import subprocess
import sys
import typing
BENCH = pathlib.Path(__file__).resolve().parent
REPO = BENCH.parent
RUN_HEAVY = pathlib.Path.home() / ".claude/bin/run-heavy"
SUITE = ["uvx", "--python", sys.executable, "--with", "pytest-xdist==3.8.0", "pytest==9.2.0", "-n", "4", "-q"]
CASES = {
"290": ("cda45a931", "509804da0"),
"251": ("b74322892", "d13c4fb0d"),
"218": ("62d57344d", "c424fa66a"),
"289": ("034356f34", "62d1f5edd"),
"229": ("e2aca90c9", "2321f4930"),
"214": ("39dd51bb4", "4764d3d52"),
}
@dataclasses.dataclass(frozen=True)
class Arm:
model: str
effort: str
advisor: str | None = None
def flags(self):
return ["--model", self.model, "--effort", self.effort, *(["--advisor", self.advisor] if self.advisor else [])]
ARMS = {
"sonnet-advisor": Arm("claude-sonnet-5-5", "high", "claude-opus-5-5"),
"sonnet": Arm("claude-sonnet-5-5", "high"),
"opus-medium": Arm("claude-opus-5-5", "medium"),
}
VERDICTS = ("pass", "partial", "fail")
JUDGE = Arm("claude-opus-5-5", "medium")
@dataclasses.dataclass(frozen=True)
class Bench:
"""What a bench's steps act on, which the script's entry point hands them."""
# The repo whose archived chunks the bench reruns.
repo: pathlib.Path
# Each case's start commit and the commit it merged at, by todo item.
cases: dict
# Runs claude and the tests, as subprocess.run does.
process: typing.Callable
@dataclasses.dataclass(frozen=True)
class Outcome:
cost: float
seconds: float
advisor_calls: int
def item_text(todo, item):
"""The item's heading and detail in todo.md, up to the next item's heading."""
found = re.search(rf"^### {item}\. .*?(?=^### \d+\. |\Z)", todo, re.DOTALL | re.MULTILINE)
if not found:
raise ValueError(f"item {item} has no detail in todo.md")
return found.group().strip()
def read_result(lines):
"""The cost, seconds and advisor calls of a claude stream's first result line, which is the worker's: a background task's notification makes the later ones. The cost counts the advisor's tokens."""
for line in lines:
try:
entry = json.loads(line)
except ValueError:
continue
if entry.get("type") == "result":
if entry.get("is_error"):
raise ValueError(f"the run ended in an error: {entry.get('result')}")
iterations = entry.get("usage", {}).get("iterations", [])
return Outcome(
cost=entry["total_cost_usd"], seconds=entry["duration_ms"] / 1000,
advisor_calls=sum(1 for each in iterations if each.get("type") == "advisor_message"))
raise ValueError("the stream has no result line")
def verdict_of(reply):
"""The verdict in the judge's `claude -p --output-format json` reply."""
found = re.search(r"\{.*\}", json.loads(reply)["result"], re.DOTALL)
if not found:
raise ValueError("the judge's reply has no JSON object")
verdict = json.loads(found.group())["verdict"]
if verdict not in VERDICTS:
raise ValueError(f"the judge's verdict {verdict!r} is none of {VERDICTS}")
return verdict
def split_name(name):
"""The case, arm and run number of a run's name, such as 290-sonnet-advisor-1."""
case, rest = name.split("-", 1)
arm, number = rest.rsplit("-", 1)
return case, arm, number
def suite_targets(changed):
"""The tests to run for a change: the package, or the lone module, under bin/ of each changed file, else all of bin/."""
found = {"/".join(path.split("/")[:2]) for path in changed if path.startswith("bin/")}
return sorted(found) or ["bin"]
def summary(rows):
"""A markdown table with one row per arm, in ARMS' order: its runs, passes, partials, mean cost, mean advisor calls and mean seconds."""
lines = ["| arm | runs | pass | partial | mean cost $ | mean advisor calls | mean seconds |", "|---|---|---|---|---|---|---|"]
for name in ARMS:
own = [row for row in rows if row["arm"] == name]
if own:
passes, partials = (sum(row["verdict"] == kind for row in own) for kind in ("pass", "partial"))
cost, advisor_calls, seconds = (sum(row[key] for row in own) / len(own) for key in ("cost", "advisor_calls", "seconds"))
lines.append(f"| {name} | {len(own)} | {passes} | {partials} | {cost:.2f} | {advisor_calls:.1f} | {seconds:.0f} |")
return "\n".join(lines)
def git(repo, *args):
return subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True).stdout
def fresh_clone(repo, base, dest):
"""A clone of repo at base, with no remote to push to."""
subprocess.run(["git", "clone", "-q", "--no-local", str(repo), str(dest)], check=True)
git(dest, "checkout", "-q", "--detach", base)
git(dest, "remote", "remove", "origin")
def run(bench, work, name):
"""One worker run of <case>-<arm>-<n>; the files it leaves are in the module docstring."""
case, arm, _ = split_name(name)
base = bench.cases[case][0]
clone = work / name
if clone.exists():
raise ValueError(f"{name} already ran: remove {clone}")
fresh_clone(bench.repo, base, clone)
prompt = (BENCH / "advisor-run-prompt.md").read_text().format(base=base, item=case, text=item_text(git(bench.repo, "show", f"{base}:todo.md"), case))
with (work / f"{name}.jsonl").open("w") as out:
bench.process(
["claude", "-p", prompt, *ARMS[arm].flags(), "--setting-sources", "project", "--permission-mode", "bypassPermissions",
"--no-session-persistence", "--output-format", "stream-json", "--verbose"],
cwd=clone, stdout=out, stderr=subprocess.STDOUT)
read_result((work / f"{name}.jsonl").read_text().splitlines())
save_change(bench, work, name, clone, bench.cases[case])
return f"{name}: ran"
def save_change(bench, work, name, clone, commits):
"""Save the clone's change since the case's start commit as <name>.diff, and as <name>.tests the result of the tests of the packages that the clone's change or the merged change touches."""
base, merged = commits
git(clone, "add", "--all")
(work / f"{name}.diff").write_text(git(clone, "diff", "--cached", base))
changed = [*git(clone, "diff", "--cached", "--name-only", base).split(), *git(bench.repo, "diff", "--name-only", base, merged).split()]
targets = [target for target in suite_targets(changed) if (clone / target).exists()] or ["bin"]
tests = bench.process([str(RUN_HEAVY), *SUITE, *targets], cwd=clone, capture_output=True, text=True)
(work / f"{name}.tests").write_text(f"exit {tests.returncode}\n{tests.stdout[-1500:]}")
def grade(bench, work, name):
"""Have the judge rule on one run's diff, blind to its arm, and save the verdict beside it."""
case = split_name(name)[0]
base, merged = bench.cases[case]
prompt = (BENCH / "advisor-judge-prompt.md").read_text().format(
text=item_text(git(bench.repo, "show", f"{base}:todo.md"), case), reference=git(bench.repo, "diff", base, merged, "--", ".", ":!todo.md", ":!docs", ":!CHANGELOG.md"),
diff=(work / f"{name}.diff").read_text(), tests=(work / f"{name}.tests").read_text())
reply = bench.process(
["claude", "-p", prompt, *JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", "", "--strict-mcp-config"],
capture_output=True, text=True, check=True).stdout
verdict = verdict_of(reply)
(work / f"{name}.verdict").write_text(json.dumps({"verdict": verdict, "reply": json.loads(reply)["result"]}))
return f"{name}: {verdict}"
def attempted(step, work, name):
"""step's result line, or a line naming the failure, so one failed run doesn't stop the other runs' results printing."""
try:
return step(work, name)
except (KeyError, ValueError, subprocess.CalledProcessError, OSError) as failure:
return f"{name}: FAILED: {type(failure).__name__}: {failure}"
def rows_of(work):
rows = []
for verdict_file in sorted(work.glob("*.verdict")):
name = verdict_file.stem
try:
outcome = read_result((work / f"{name}.jsonl").read_text().splitlines())
except ValueError as failure:
raise ValueError(f"{name}: {failure}") from failure
rows.append({"arm": split_name(name)[1], "verdict": json.loads(verdict_file.read_text())["verdict"], **dataclasses.asdict(outcome)})
return rows
def steps_of(bench):
"""The steps main runs by name, each on bench."""
return {"run": functools.partial(run, bench), "grade": functools.partial(grade, bench)}
def main(argv, steps, say):
"""Runs the step argv names on each run it names, and hands say each result line; or hands say the summary."""
work, step, names = pathlib.Path(argv[0]), argv[1], argv[2:]
if step == "summary":
say(summary(rows_of(work)))
return
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
for result in pool.map(lambda name: attempted(steps[step], work, name), names):
say(result)
if __name__ == "__main__":
main(sys.argv[1:], steps_of(Bench(REPO, CASES, subprocess.run)), functools.partial(print, flush=True))