223 lines
11 KiB
Python
223 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Bench how a worker's step to a stronger model hands the chunk over: handoff_bench.py <work dir> start <case>-<n> ..., then run|grade <case>-<arm>-<n> ..., then summary
|
|
|
|
The question: when loop-claude steps a worker to a rung on another model, which handoff serves the stronger model
|
|
best? Each case is an archived chunk of this repo, as in advisor_bench.py, whose judge and files this bench shares.
|
|
- `start` runs WEAK on the item in the clone <work dir>/<case>-<n> until it has spent START_BUDGET, which
|
|
stands in for a run that ends asking for a higher rung. A worker that finishes within the budget leaves no
|
|
handoff, so `start` fails on it. It saves the change so far (<case>-start-<n>.diff), and a copy of the clone
|
|
(.tree), which every arm starts from, and a copy of the session's transcript (.session.jsonl), which the resume
|
|
and compact arms continue.
|
|
- `run` puts the clone and the session back as `start` left them, then continues on STRONG, one arm of a case at a
|
|
time. The resume arm resumes WEAK's session with handoff-resume-prompt.md. The compact arm does the same once WEAK
|
|
has compacted the session with `/compact`, whose stream it saves as <name>.compact.jsonl. The brief arm starts a
|
|
session that has the item and handoff-brief-prompt.md, and no transcript.
|
|
- `grade` is advisor_bench.py's, over these cases. `summary` counts an arm's cost and seconds from its own runs,
|
|
the compaction included, and leaves out the start, which the arms share (`outcome`).
|
|
Run each arm of each case once; the table shows a direction, not a significance.
|
|
"""
|
|
import collections
|
|
import concurrent.futures
|
|
import dataclasses
|
|
import functools
|
|
import json
|
|
import pathlib
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import typing
|
|
import uuid
|
|
|
|
import advisor_bench
|
|
from advisor_bench import Arm
|
|
|
|
BENCH = pathlib.Path(__file__).resolve().parent
|
|
# The ladder's first step that changes the model: bin/model_routing/routing-ladder.json.
|
|
WEAK = Arm("claude-sonnet-5-5", "high")
|
|
STRONG = Arm("claude-opus-5-5", "medium")
|
|
ARMS = ("resume", "compact", "brief")
|
|
# Claude Code keeps a session's transcript at <CLAUDE_PROJECTS>/<the working directory's slug>/<session id>.jsonl.
|
|
CLAUDE_PROJECTS = pathlib.Path.home() / ".claude/projects"
|
|
# One per case and run number: its arms share one clone, so they run one at a time.
|
|
CLONE_LOCKS: collections.defaultdict = collections.defaultdict(threading.Lock)
|
|
# L 3 chunks, by todo item. advisor_bench's L 1 and L 2 chunks are too small: Sonnet 5.5 finished five of the six for
|
|
# under $0.20, before any handoff.
|
|
CASES = {
|
|
"143": ("45cb331a4", "545f3c347"),
|
|
"163": ("d4cf6e7fe", "d1b0fe3a4"),
|
|
"198": ("4a0ba5921", "fd60b6da"),
|
|
"210": ("4764d3d52", "dcbdc970"),
|
|
"227": ("eb7b50698", "3bde9cd6d"),
|
|
"239": ("d583f8d48", "47a40757"),
|
|
}
|
|
# Dollars WEAK may spend before the handoff. In a trial, $0.36 stopped Sonnet 5.5 on items 163 and 227 after 11 to 14
|
|
# turns, as the result line's `num_turns` counts them, with about 250 lines of diff.
|
|
START_BUDGET = 0.36
|
|
CLAUDE_FLAGS = ["--setting-sources", "project", "--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"]
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class Handoff:
|
|
"""What this bench's steps act on, which the script's entry point hands them."""
|
|
bench: advisor_bench.Bench
|
|
# Dollars WEAK may spend before the handoff.
|
|
budget: float
|
|
# Where Claude Code keeps each session's transcript.
|
|
projects: pathlib.Path
|
|
# Makes the ID of the session `start` opens, as uuid.uuid4 does.
|
|
new_session: typing.Callable
|
|
|
|
|
|
def stopped_at_budget(lines):
|
|
"""Whether a claude stream's first result line, the worker's own, says the run stopped at --max-budget-usd."""
|
|
for line in lines:
|
|
try:
|
|
entry = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
if isinstance(entry, dict) and entry.get("type") == "result":
|
|
return entry.get("subtype") == "error_max_budget_usd"
|
|
return False
|
|
|
|
|
|
def run_prompt(bench, case):
|
|
"""advisor_bench's prompt for the case's item, as the item's text stood at the start commit."""
|
|
base = bench.cases[case][0]
|
|
text = advisor_bench.item_text(advisor_bench.git(bench.repo, "show", f"{base}:todo.md"), case)
|
|
return (BENCH / "advisor-run-prompt.md").read_text().format(base=base, item=case, text=text)
|
|
|
|
|
|
def claude(process, prompt, flags, clone, stream):
|
|
with stream.open("w") as out:
|
|
process(["claude", "-p", prompt, *flags, *CLAUDE_FLAGS], cwd=clone, stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT)
|
|
|
|
|
|
def start(handoff, work, name):
|
|
"""WEAK's capped run of <case>-<n>, and the tree and session it leaves the arms; the files are in the module docstring."""
|
|
case, number = name.split("-")
|
|
bench = handoff.bench
|
|
base = bench.cases[case][0]
|
|
clone, saved = work / name, work / f"{case}-start-{number}"
|
|
if clone.exists():
|
|
raise ValueError(f"{name} already started: remove {clone}")
|
|
advisor_bench.fresh_clone(bench.repo, base, clone)
|
|
session = str(handoff.new_session())
|
|
stream = saved.with_suffix(".jsonl")
|
|
claude(bench.process, run_prompt(bench, case), [*WEAK.flags(), "--session-id", session, "--max-budget-usd", str(handoff.budget)], clone, stream)
|
|
if not stopped_at_budget(stream.read_text().splitlines()):
|
|
raise ValueError("the weak worker finished within its budget, or its run failed, so there is no handoff to bench")
|
|
transcripts = list(handoff.projects.glob(f"*/{session}.jsonl"))
|
|
if len(transcripts) != 1:
|
|
raise ValueError(f"expected one transcript of session {session} under {handoff.projects}, found {len(transcripts)}")
|
|
shutil.copy(transcripts[0], saved.with_suffix(".session.jsonl"))
|
|
saved.with_suffix(".session").write_text(str(transcripts[0]))
|
|
# The worker may have committed against the bench rules, so the arms get its whole change since base, uncommitted.
|
|
advisor_bench.git(clone, "add", "--all")
|
|
saved.with_suffix(".diff").write_text(advisor_bench.git(clone, "diff", "--cached", base))
|
|
advisor_bench.git(clone, "reset", "-q", "--mixed", base)
|
|
shutil.copytree(clone, saved.with_suffix(".tree"), symlinks=True)
|
|
return f"{name}: started"
|
|
|
|
|
|
def run(handoff, work, name):
|
|
"""STRONG's run of one arm, <case>-<arm>-<n>, from the tree and session `start` left."""
|
|
case, arm, number = advisor_bench.split_name(name)
|
|
bench = handoff.bench
|
|
if arm not in ARMS:
|
|
raise ValueError(f"the arm {arm!r} is none of {ARMS}")
|
|
clone, saved = work / f"{case}-{number}", work / f"{case}-start-{number}"
|
|
if not saved.with_suffix(".tree").exists():
|
|
raise ValueError(f"start {case}-{number} first")
|
|
stream = work / f"{name}.jsonl"
|
|
if stream.exists():
|
|
raise ValueError(f"{name} already ran: remove {stream}")
|
|
with CLONE_LOCKS[clone]:
|
|
shutil.rmtree(clone)
|
|
shutil.copytree(saved.with_suffix(".tree"), clone, symlinks=True)
|
|
transcript = pathlib.Path(saved.with_suffix(".session").read_text())
|
|
shutil.copy(saved.with_suffix(".session.jsonl"), transcript)
|
|
resumed = ["--resume", transcript.stem]
|
|
if arm == "compact":
|
|
claude(bench.process, "/compact", [*WEAK.flags(), *resumed], clone, work / f"{name}.compact.jsonl")
|
|
finished(work / f"{name}.compact.jsonl")
|
|
if arm == "brief":
|
|
claude(bench.process, run_prompt(bench, case) + "\n" + (BENCH / "handoff-brief-prompt.md").read_text(), [*STRONG.flags(), "--no-session-persistence"], clone, stream)
|
|
else:
|
|
prompt = (BENCH / "handoff-resume-prompt.md").read_text().format(rung=f"{STRONG.model} at {STRONG.effort} effort")
|
|
claude(bench.process, prompt, [*STRONG.flags(), *resumed], clone, stream)
|
|
outcome(work, name)
|
|
advisor_bench.save_change(bench, work, name, clone, bench.cases[case])
|
|
return f"{name}: ran"
|
|
|
|
|
|
def results(stream):
|
|
"""The result lines of a claude stream. A resumed run may open with one that holds no work, and a background task's notification adds one."""
|
|
found = []
|
|
for line in stream.read_text().splitlines():
|
|
try:
|
|
entry = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
if isinstance(entry, dict) and entry.get("type") == "result":
|
|
found.append(entry)
|
|
return found
|
|
|
|
|
|
def finished(stream):
|
|
"""The result lines of a run that ended well; a run without one, or whose last one is an error, raises."""
|
|
ran = results(stream)
|
|
if not ran or ran[-1].get("is_error"):
|
|
raise ValueError(f"the run ended in an error: {ran[-1].get('result')}" if ran else "the stream has no result line")
|
|
return ran
|
|
|
|
|
|
def outcome(work, name):
|
|
"""The cost and seconds of an arm's run, its compaction included. A resumed session's cost counts from the session's start, so the start's cost comes off it."""
|
|
case, arm, number = advisor_bench.split_name(name)
|
|
ran = finished(work / f"{name}.jsonl")
|
|
compacted = results(work / f"{name}.compact.jsonl") if arm == "compact" else []
|
|
before = 0 if arm == "brief" else results(work / f"{case}-start-{number}.jsonl")[-1]["total_cost_usd"]
|
|
return {"cost": ran[-1]["total_cost_usd"] - before, "seconds": sum(entry["duration_ms"] for entry in [*compacted, *ran]) / 1000}
|
|
|
|
|
|
def rows_of(work):
|
|
"""One row per graded run: its arm, verdict, cost and seconds."""
|
|
rows = []
|
|
for verdict_file in sorted(work.glob("*.verdict")):
|
|
name = verdict_file.stem
|
|
rows.append({"arm": advisor_bench.split_name(name)[1], "verdict": json.loads(verdict_file.read_text())["verdict"], **outcome(work, name)})
|
|
return rows
|
|
|
|
|
|
def summary(rows):
|
|
"""A markdown table with one row per arm, in ARMS' order: its runs, passes, partials, mean cost and mean seconds."""
|
|
lines = ["| arm | runs | pass | partial | mean cost $ | mean seconds |", "|---|---|---|---|---|---|"]
|
|
for name in ARMS:
|
|
own = [row for row in rows if row["arm"] == name]
|
|
if own:
|
|
passes, partials = (sum(row["verdict"] == kind for row in own) for kind in ("pass", "partial"))
|
|
cost, seconds = (sum(row[key] for row in own) / len(own) for key in ("cost", "seconds"))
|
|
lines.append(f"| {name} | {len(own)} | {passes} | {partials} | {cost:.2f} | {seconds:.0f} |")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def steps_of(handoff):
|
|
"""The steps main runs by name, each on handoff; the grade is advisor_bench's, over handoff's cases."""
|
|
return {"start": functools.partial(start, handoff), "run": functools.partial(run, handoff), "grade": functools.partial(advisor_bench.grade, handoff.bench)}
|
|
|
|
|
|
def main(argv, steps, say):
|
|
"""Runs the step argv names on each run it names, and hands say each result line; or hands say the summary."""
|
|
work, step, names = pathlib.Path(argv[0]), argv[1], argv[2:]
|
|
if step == "summary":
|
|
say(summary(rows_of(work)))
|
|
return
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
|
|
for result in pool.map(lambda name: advisor_bench.attempted(steps[step], work, name), names):
|
|
say(result)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(sys.argv[1:], steps_of(Handoff(advisor_bench.Bench(advisor_bench.REPO, CASES, subprocess.run), START_BUDGET, CLAUDE_PROJECTS, uuid.uuid4)), functools.partial(print, flush=True))
|