211 lines
8.9 KiB
Python
211 lines
8.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Bench the advisor tool on archived chunks: advisor_bench.py <work dir> run|grade <case>-<arm>-<n> ..., then advisor_bench.py <work dir> summary
|
|
|
|
The question: does a worker below the project's rung, Sonnet 5.5 with Opus 5.5 as its advisor, beat the same
|
|
Sonnet alone, or match Opus 5.5 at medium for less? Each case is an archived L 1 or L 2 chunk of this repo
|
|
(CASES holds its todo.md item, the commit it started from and the commit it merged at). `run` clones the repo at
|
|
the start commit and hands the arm's worker the item's text from todo.md, with the bench rules of
|
|
advisor-run-prompt.md in place of loop-claude's prompt. It saves, beside the clone in <work dir>: the claude
|
|
stream (.jsonl), the worker's change (.diff) and the result of the tests of the packages that either change touches (.tests).
|
|
`grade` has a judge on Opus 5.5 at medium, blind to the arm, rule on the .diff against the merged commit's change
|
|
(.verdict); `summary` prints a table per arm.
|
|
Run each arm of each case once; the table shows a direction, not a significance.
|
|
"""
|
|
import concurrent.futures
|
|
import dataclasses
|
|
import json
|
|
import pathlib
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
BENCH = pathlib.Path(__file__).resolve().parent
|
|
REPO = BENCH.parent
|
|
RUN_HEAVY = pathlib.Path.home() / ".claude/bin/run-heavy"
|
|
SUITE = ["uvx", "--python", sys.executable, "--with", "pytest-xdist==3.8.0", "pytest==9.2.0", "-n", "4", "-q"]
|
|
CASES = {
|
|
"290": ("cda45a931", "509804da0"),
|
|
"251": ("b74322892", "d13c4fb0d"),
|
|
"218": ("62d57344d", "c424fa66a"),
|
|
"289": ("034356f34", "62d1f5edd"),
|
|
"229": ("e2aca90c9", "2321f4930"),
|
|
"214": ("39dd51bb4", "4764d3d52"),
|
|
}
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Arm:
|
|
model: str
|
|
effort: str
|
|
advisor: str | None = None
|
|
|
|
def flags(self):
|
|
return ["--model", self.model, "--effort", self.effort, *(["--advisor", self.advisor] if self.advisor else [])]
|
|
|
|
|
|
ARMS = {
|
|
"sonnet-advisor": Arm("claude-sonnet-5-5", "high", "claude-opus-5-5"),
|
|
"sonnet": Arm("claude-sonnet-5-5", "high"),
|
|
"opus-medium": Arm("claude-opus-5-5", "medium"),
|
|
}
|
|
VERDICTS = ("pass", "partial", "fail")
|
|
JUDGE = Arm("claude-opus-5-5", "medium")
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Outcome:
|
|
cost: float
|
|
seconds: float
|
|
advisor_calls: int
|
|
|
|
|
|
def item_text(todo, item):
|
|
"""The item's heading and detail in todo.md, up to the next item's heading."""
|
|
found = re.search(rf"^### {item}\. .*?(?=^### \d+\. |\Z)", todo, re.DOTALL | re.MULTILINE)
|
|
if not found:
|
|
raise ValueError(f"item {item} has no detail in todo.md")
|
|
return found.group().strip()
|
|
|
|
|
|
def read_result(lines):
|
|
"""The cost, seconds and advisor calls of a claude stream's first result line, which is the worker's: a background task's notification makes the later ones. The cost counts the advisor's tokens."""
|
|
for line in lines:
|
|
try:
|
|
entry = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
if entry.get("type") == "result":
|
|
if entry.get("is_error"):
|
|
raise ValueError(f"the run ended in an error: {entry.get('result')}")
|
|
iterations = entry.get("usage", {}).get("iterations", [])
|
|
return Outcome(
|
|
cost=entry["total_cost_usd"], seconds=entry["duration_ms"] / 1000,
|
|
advisor_calls=sum(1 for each in iterations if each.get("type") == "advisor_message"))
|
|
raise ValueError("the stream has no result line")
|
|
|
|
|
|
def verdict_of(reply):
|
|
"""The verdict in the judge's `claude -p --output-format json` reply."""
|
|
found = re.search(r"\{.*\}", json.loads(reply)["result"], re.DOTALL)
|
|
if not found:
|
|
raise ValueError("the judge's reply has no JSON object")
|
|
verdict = json.loads(found.group())["verdict"]
|
|
if verdict not in VERDICTS:
|
|
raise ValueError(f"the judge's verdict {verdict!r} is none of {VERDICTS}")
|
|
return verdict
|
|
|
|
|
|
def split_name(name):
|
|
"""The case, arm and run number of a run's name, such as 290-sonnet-advisor-1."""
|
|
case, rest = name.split("-", 1)
|
|
arm, number = rest.rsplit("-", 1)
|
|
return case, arm, number
|
|
|
|
|
|
def suite_targets(changed):
|
|
"""The tests to run for a change: the package, or the lone module, under bin/ of each changed file, else all of bin/."""
|
|
found = {"/".join(path.split("/")[:2]) for path in changed if path.startswith("bin/")}
|
|
return sorted(found) or ["bin"]
|
|
|
|
|
|
def summary(rows):
|
|
"""A markdown table with one row per arm, in ARMS' order: its runs, passes, partials, mean cost, mean advisor calls and mean seconds."""
|
|
lines = ["| arm | runs | pass | partial | mean cost $ | mean advisor calls | mean seconds |", "|---|---|---|---|---|---|---|"]
|
|
for name in ARMS:
|
|
own = [row for row in rows if row["arm"] == name]
|
|
if own:
|
|
passes, partials = (sum(row["verdict"] == kind for row in own) for kind in ("pass", "partial"))
|
|
cost, advisor_calls, seconds = (sum(row[key] for row in own) / len(own) for key in ("cost", "advisor_calls", "seconds"))
|
|
lines.append(f"| {name} | {len(own)} | {passes} | {partials} | {cost:.2f} | {advisor_calls:.1f} | {seconds:.0f} |")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def git(repo, *args):
|
|
return subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True).stdout
|
|
|
|
|
|
def fresh_clone(base, dest):
|
|
"""A clone of this repo at base, with no remote to push to."""
|
|
subprocess.run(["git", "clone", "-q", "--no-local", str(REPO), str(dest)], check=True)
|
|
git(dest, "checkout", "-q", "--detach", base)
|
|
git(dest, "remote", "remove", "origin")
|
|
|
|
|
|
def run(work, name):
|
|
"""One worker run of <case>-<arm>-<n>; the files it leaves are in the module docstring."""
|
|
case, arm, _ = split_name(name)
|
|
base = CASES[case][0]
|
|
clone = work / name
|
|
if clone.exists():
|
|
raise ValueError(f"{name} already ran: remove {clone}")
|
|
fresh_clone(base, clone)
|
|
prompt = (BENCH / "advisor-run-prompt.md").read_text().format(base=base, item=case, text=item_text(git(REPO, "show", f"{base}:todo.md"), case))
|
|
with (work / f"{name}.jsonl").open("w") as out:
|
|
subprocess.run(
|
|
["claude", "-p", prompt, *ARMS[arm].flags(), "--setting-sources", "project", "--permission-mode", "bypassPermissions",
|
|
"--no-session-persistence", "--output-format", "stream-json", "--verbose"],
|
|
cwd=clone, stdout=out, stderr=subprocess.STDOUT)
|
|
read_result((work / f"{name}.jsonl").read_text().splitlines())
|
|
save_change(work, name, clone, CASES[case])
|
|
return f"{name}: ran"
|
|
|
|
|
|
def save_change(work, name, clone, commits):
|
|
"""Save the clone's change since the case's start commit as <name>.diff, and the result of the tests of the packages that it or the merged change touches as <name>.tests."""
|
|
base, merged = commits
|
|
git(clone, "add", "--all")
|
|
(work / f"{name}.diff").write_text(git(clone, "diff", "--cached", base))
|
|
changed = [*git(clone, "diff", "--cached", "--name-only", base).split(), *git(REPO, "diff", "--name-only", base, merged).split()]
|
|
targets = [target for target in suite_targets(changed) if (clone / target).exists()] or ["bin"]
|
|
tests = subprocess.run([str(RUN_HEAVY), *SUITE, *targets], cwd=clone, capture_output=True, text=True)
|
|
(work / f"{name}.tests").write_text(f"exit {tests.returncode}\n{tests.stdout[-1500:]}")
|
|
|
|
|
|
def grade(work, name, cases=None):
|
|
"""Have the judge rule on one run's diff, blind to its arm, and save the verdict beside it. cases stands in for CASES where another bench grades its own."""
|
|
case = split_name(name)[0]
|
|
base, merged = (cases or CASES)[case]
|
|
prompt = (BENCH / "advisor-judge-prompt.md").read_text().format(
|
|
text=item_text(git(REPO, "show", f"{base}:todo.md"), case), reference=git(REPO, "diff", base, merged, "--", ".", ":!todo.md", ":!docs", ":!CHANGELOG.md"),
|
|
diff=(work / f"{name}.diff").read_text(), tests=(work / f"{name}.tests").read_text())
|
|
reply = subprocess.run(
|
|
["claude", "-p", prompt, *JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", ""],
|
|
capture_output=True, text=True, check=True).stdout
|
|
verdict = verdict_of(reply)
|
|
(work / f"{name}.verdict").write_text(json.dumps({"verdict": verdict, "reply": json.loads(reply)["result"]}))
|
|
return f"{name}: {verdict}"
|
|
|
|
|
|
def attempted(step, work, name):
|
|
"""step's result line, or a line naming the failure, so one failed run doesn't stop the other runs' results printing."""
|
|
try:
|
|
return step(work, name)
|
|
except (KeyError, ValueError, subprocess.CalledProcessError, OSError) as failure:
|
|
return f"{name}: FAILED: {type(failure).__name__}: {failure}"
|
|
|
|
|
|
def rows_of(work):
|
|
rows = []
|
|
for verdict_file in sorted(work.glob("*.verdict")):
|
|
name = verdict_file.stem
|
|
try:
|
|
outcome = read_result((work / f"{name}.jsonl").read_text().splitlines())
|
|
except ValueError as failure:
|
|
raise ValueError(f"{name}: {failure}") from failure
|
|
rows.append({"arm": split_name(name)[1], "verdict": json.loads(verdict_file.read_text())["verdict"], **dataclasses.asdict(outcome)})
|
|
return rows
|
|
|
|
|
|
def main(argv):
|
|
work, step, names = pathlib.Path(argv[0]), argv[1], argv[2:]
|
|
if step == "summary":
|
|
print(summary(rows_of(work)))
|
|
return
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
|
|
for result in pool.map(lambda name: attempted({"run": run, "grade": grade}[step], work, name), names):
|
|
print(result, flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(sys.argv[1:])
|