Bench a model step's handoff: resume, compact or a fresh session with a brief
This commit is contained in:
@@ -184,9 +184,13 @@
|
||||
!workflow-bench/bench.py
|
||||
!workflow-bench/compare-prompt.md
|
||||
!workflow-bench/compare.py
|
||||
!workflow-bench/handoff-brief-prompt.md
|
||||
!workflow-bench/handoff-resume-prompt.md
|
||||
!workflow-bench/handoff_bench.py
|
||||
!workflow-bench/review_git.py
|
||||
!workflow-bench/run-prompt.md
|
||||
!workflow-bench/setup.cfg
|
||||
!workflow-bench/test_advisor_bench.py
|
||||
!workflow-bench/test_bench.py
|
||||
!workflow-bench/test_handoff_bench.py
|
||||
!workflow-bench/test_review_git.py
|
||||
|
||||
@@ -146,19 +146,25 @@ def run(work, name):
|
||||
"--no-session-persistence", "--output-format", "stream-json", "--verbose"],
|
||||
cwd=clone, stdout=out, stderr=subprocess.STDOUT)
|
||||
read_result((work / f"{name}.jsonl").read_text().splitlines())
|
||||
git(clone, "add", "--all")
|
||||
(work / f"{name}.diff").write_text(git(clone, "diff", "--cached", base))
|
||||
changed = [*git(clone, "diff", "--cached", "--name-only", base).split(), *git(REPO, "diff", "--name-only", base, CASES[case][1]).split()]
|
||||
targets = [target for target in suite_targets(changed) if (clone / target).exists()] or ["bin"]
|
||||
tests = subprocess.run([str(RUN_HEAVY), *SUITE, *targets], cwd=clone, capture_output=True, text=True)
|
||||
(work / f"{name}.tests").write_text(f"exit {tests.returncode}\n{tests.stdout[-1500:]}")
|
||||
save_change(work, name, clone, CASES[case])
|
||||
return f"{name}: ran"
|
||||
|
||||
|
||||
def grade(work, name):
|
||||
"""Have the judge rule on one run's diff, blind to its arm, and save the verdict beside it."""
|
||||
def save_change(work, name, clone, commits):
|
||||
"""Save the clone's change since the case's start commit as <name>.diff, and the result of the tests of the packages that it or the merged change touches as <name>.tests."""
|
||||
base, merged = commits
|
||||
git(clone, "add", "--all")
|
||||
(work / f"{name}.diff").write_text(git(clone, "diff", "--cached", base))
|
||||
changed = [*git(clone, "diff", "--cached", "--name-only", base).split(), *git(REPO, "diff", "--name-only", base, merged).split()]
|
||||
targets = [target for target in suite_targets(changed) if (clone / target).exists()] or ["bin"]
|
||||
tests = subprocess.run([str(RUN_HEAVY), *SUITE, *targets], cwd=clone, capture_output=True, text=True)
|
||||
(work / f"{name}.tests").write_text(f"exit {tests.returncode}\n{tests.stdout[-1500:]}")
|
||||
|
||||
|
||||
def grade(work, name, cases=None):
|
||||
"""Have the judge rule on one run's diff, blind to its arm, and save the verdict beside it. cases stands in for CASES where another bench grades its own."""
|
||||
case = split_name(name)[0]
|
||||
base, merged = CASES[case]
|
||||
base, merged = (cases or CASES)[case]
|
||||
prompt = (BENCH / "advisor-judge-prompt.md").read_text().format(
|
||||
text=item_text(git(REPO, "show", f"{base}:todo.md"), case), reference=git(REPO, "diff", base, merged, "--", ".", ":!todo.md", ":!docs", ":!CHANGELOG.md"),
|
||||
diff=(work / f"{name}.diff").read_text(), tests=(work / f"{name}.tests").read_text())
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
|
||||
## Handoff
|
||||
|
||||
A worker on a weaker model began this item and stopped before it was done, asking for a stronger model. You start fresh, without its transcript. Its unfinished change is in the clone, uncommitted: read `git status` and `git diff` first. Keep what holds, redo what doesn't, and finish the item.
|
||||
@@ -0,0 +1 @@
|
||||
Continue the task; the previous run ended before the item was done. Your request for a higher rung went through: you now run on {rung}. The bench rules still hold.
|
||||
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bench how a worker's step to a stronger model hands the chunk over: handoff_bench.py <work dir> start <case>-<n> ..., then run|grade <case>-<arm>-<n> ..., then summary
|
||||
|
||||
The question: when loop-claude steps a worker to a rung on another model, which handoff serves the stronger model
|
||||
best? Each case is an archived chunk of this repo, as in advisor_bench.py, whose judge and files this bench shares.
|
||||
- `start` runs WEAK on the item in the clone <work dir>/<case>-<n> until it has spent START_BUDGET, which
|
||||
stands in for a run that ends asking for a higher rung. A worker that finishes within the budget leaves no
|
||||
handoff, so `start` fails on it. It saves the change so far (<case>-start-<n>.diff), and a copy of the clone
|
||||
(.tree) and of the session's transcript (.session.jsonl), which every arm starts from.
|
||||
- `run` puts the clone and the session back as `start` left them, then continues on STRONG, one arm of a case at a
|
||||
time. The resume arm resumes WEAK's session with handoff-resume-prompt.md. The compact arm does the same once WEAK
|
||||
has compacted the session with `/compact`, whose stream it saves as <name>.compact.jsonl. The brief arm starts a
|
||||
session that has the item and handoff-brief-prompt.md, and no transcript.
|
||||
- `grade` is advisor_bench.py's, over these cases. `summary` counts an arm's cost and seconds from its own runs,
|
||||
the compaction included, and leaves out the start, which the arms share.
|
||||
Run each arm of each case once; the table shows a direction, not a significance.
|
||||
"""
|
||||
import concurrent.futures
|
||||
import collections
|
||||
import json
|
||||
import pathlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
|
||||
import advisor_bench
|
||||
from advisor_bench import Arm
|
||||
|
||||
BENCH = pathlib.Path(__file__).resolve().parent
|
||||
# The ladder's first step that changes the model: bin/model_routing/routing-ladder.json.
|
||||
WEAK = Arm("claude-sonnet-5-5", "high")
|
||||
STRONG = Arm("claude-opus-5-5", "medium")
|
||||
ARMS = ("resume", "compact", "brief")
|
||||
# Claude Code keeps a session's transcript at <CLAUDE_PROJECTS>/<the working directory's slug>/<session id>.jsonl.
|
||||
CLAUDE_PROJECTS = pathlib.Path.home() / ".claude/projects"
|
||||
# One per case and run number: its arms share one clone, so they run one at a time.
|
||||
CLONE_LOCKS: collections.defaultdict = collections.defaultdict(threading.Lock)
|
||||
# L 3 chunks, by todo item. advisor_bench's L 1 and L 2 chunks are too small: Sonnet 5.5 finished five of the six for
|
||||
# under $0.20, before any handoff.
|
||||
CASES = {
|
||||
"143": ("45cb331a4", "545f3c347"),
|
||||
"163": ("d4cf6e7fe", "d1b0fe3a4"),
|
||||
"198": ("4a0ba5921", "fd60b6da"),
|
||||
"210": ("4764d3d52", "dcbdc970"),
|
||||
"227": ("eb7b50698", "3bde9cd6d"),
|
||||
"239": ("d583f8d48", "47a40757"),
|
||||
}
|
||||
# Dollars WEAK may spend before the handoff. In a trial, $0.36 stopped Sonnet 5.5 on items 163 and 227 after 11 to 14
|
||||
# turns, with about 250 lines of diff.
|
||||
START_BUDGET = 0.36
|
||||
CLAUDE_FLAGS = ["--setting-sources", "project", "--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"]
|
||||
|
||||
|
||||
def stopped_at_budget(lines):
|
||||
"""Whether a claude stream's first result line, the worker's own, says the run stopped at --max-budget-usd."""
|
||||
for line in lines:
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if entry.get("type") == "result":
|
||||
return entry.get("subtype") == "error_max_budget_usd"
|
||||
return False
|
||||
|
||||
|
||||
def run_prompt(case):
|
||||
"""advisor_bench's prompt for the case's item, as the item's text stood at the start commit."""
|
||||
base = CASES[case][0]
|
||||
text = advisor_bench.item_text(advisor_bench.git(advisor_bench.REPO, "show", f"{base}:todo.md"), case)
|
||||
return (BENCH / "advisor-run-prompt.md").read_text().format(base=base, item=case, text=text)
|
||||
|
||||
|
||||
def claude(prompt, flags, clone, stream):
|
||||
with stream.open("w") as out:
|
||||
subprocess.run(["claude", "-p", prompt, *flags, *CLAUDE_FLAGS], cwd=clone, stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT)
|
||||
|
||||
|
||||
def start(work, name):
|
||||
"""WEAK's capped run of <case>-<n>, and the tree and session it leaves the arms; the files are in the module docstring."""
|
||||
case, number = name.split("-")
|
||||
base = CASES[case][0]
|
||||
clone, saved = work / name, work / f"{case}-start-{number}"
|
||||
if clone.exists():
|
||||
raise ValueError(f"{name} already started: remove {clone}")
|
||||
advisor_bench.fresh_clone(base, clone)
|
||||
session = str(uuid.uuid4())
|
||||
stream = saved.with_suffix(".jsonl")
|
||||
claude(run_prompt(case), [*WEAK.flags(), "--session-id", session, "--max-budget-usd", str(START_BUDGET)], clone, stream)
|
||||
if not stopped_at_budget(stream.read_text().splitlines()):
|
||||
raise ValueError("the weak worker finished within its budget, or its run failed, so there is no handoff to bench")
|
||||
transcripts = list(CLAUDE_PROJECTS.glob(f"*/{session}.jsonl"))
|
||||
if len(transcripts) != 1:
|
||||
raise ValueError(f"expected one transcript of session {session} under {CLAUDE_PROJECTS}, found {len(transcripts)}")
|
||||
shutil.copy(transcripts[0], saved.with_suffix(".session.jsonl"))
|
||||
saved.with_suffix(".session").write_text(str(transcripts[0]))
|
||||
# The worker may have committed against the bench rules, so the arms get its whole change since base, uncommitted.
|
||||
advisor_bench.git(clone, "add", "--all")
|
||||
saved.with_suffix(".diff").write_text(advisor_bench.git(clone, "diff", "--cached", base))
|
||||
advisor_bench.git(clone, "reset", "-q", "--mixed", base)
|
||||
shutil.copytree(clone, saved.with_suffix(".tree"), symlinks=True)
|
||||
return f"{name}: started"
|
||||
|
||||
|
||||
def run(work, name):
|
||||
"""STRONG's run of one arm, <case>-<arm>-<n>, from the tree and session `start` left."""
|
||||
case, arm, number = advisor_bench.split_name(name)
|
||||
if arm not in ARMS:
|
||||
raise ValueError(f"the arm {arm!r} is none of {ARMS}")
|
||||
clone, saved = work / f"{case}-{number}", work / f"{case}-start-{number}"
|
||||
if not saved.with_suffix(".tree").exists():
|
||||
raise ValueError(f"start {case}-{number} first")
|
||||
stream = work / f"{name}.jsonl"
|
||||
if stream.exists():
|
||||
raise ValueError(f"{name} already ran: remove {stream}")
|
||||
with CLONE_LOCKS[clone]:
|
||||
shutil.rmtree(clone)
|
||||
shutil.copytree(saved.with_suffix(".tree"), clone, symlinks=True)
|
||||
transcript = pathlib.Path(saved.with_suffix(".session").read_text())
|
||||
shutil.copy(saved.with_suffix(".session.jsonl"), transcript)
|
||||
resumed = ["--resume", transcript.stem]
|
||||
if arm == "compact":
|
||||
claude("/compact", [*WEAK.flags(), *resumed], clone, work / f"{name}.compact.jsonl")
|
||||
if arm == "brief":
|
||||
claude(run_prompt(case) + "\n" + (BENCH / "handoff-brief-prompt.md").read_text(), [*STRONG.flags(), "--no-session-persistence"], clone, stream)
|
||||
else:
|
||||
prompt = (BENCH / "handoff-resume-prompt.md").read_text().format(rung=f"{STRONG.model} at {STRONG.effort} effort")
|
||||
claude(prompt, [*STRONG.flags(), *resumed], clone, stream)
|
||||
outcome(work, name)
|
||||
advisor_bench.save_change(work, name, clone, CASES[case])
|
||||
return f"{name}: ran"
|
||||
|
||||
|
||||
def outcome(work, name):
|
||||
"""The cost and seconds of an arm's run: its STRONG stream's, plus its compaction's where it made one."""
|
||||
streams = [stream for stream in (work / f"{name}.compact.jsonl", work / f"{name}.jsonl") if stream.exists()]
|
||||
results = [advisor_bench.read_result(stream.read_text().splitlines()) for stream in streams]
|
||||
return {"cost": sum(result.cost for result in results), "seconds": sum(result.seconds for result in results)}
|
||||
|
||||
|
||||
def rows_of(work):
|
||||
"""One row per graded run: its arm, verdict, cost and seconds."""
|
||||
rows = []
|
||||
for verdict_file in sorted(work.glob("*.verdict")):
|
||||
name = verdict_file.stem
|
||||
rows.append({"arm": advisor_bench.split_name(name)[1], "verdict": json.loads(verdict_file.read_text())["verdict"], **outcome(work, name)})
|
||||
return rows
|
||||
|
||||
|
||||
def summary(rows):
|
||||
"""A markdown table with one row per arm, in ARMS' order: its runs, passes, partials, mean cost and mean seconds."""
|
||||
lines = ["| arm | runs | pass | partial | mean cost $ | mean seconds |", "|---|---|---|---|---|---|"]
|
||||
for name in ARMS:
|
||||
own = [row for row in rows if row["arm"] == name]
|
||||
if own:
|
||||
passes, partials = (sum(row["verdict"] == kind for row in own) for kind in ("pass", "partial"))
|
||||
cost, seconds = (sum(row[key] for row in own) / len(own) for key in ("cost", "seconds"))
|
||||
lines.append(f"| {name} | {len(own)} | {passes} | {partials} | {cost:.2f} | {seconds:.0f} |")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main(argv):
|
||||
work, step, names = pathlib.Path(argv[0]), argv[1], argv[2:]
|
||||
if step == "summary":
|
||||
print(summary(rows_of(work)))
|
||||
return
|
||||
steps = {"start": start, "run": run, "grade": lambda work, name: advisor_bench.grade(work, name, CASES)}
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
|
||||
for result in pool.map(lambda name: advisor_bench.attempted(steps[step], work, name), names):
|
||||
print(result, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
@@ -0,0 +1,224 @@
|
||||
import json
|
||||
import pathlib
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
import unittest.mock
|
||||
import uuid
|
||||
|
||||
import advisor_bench
|
||||
import handoff_bench
|
||||
from test_advisor_bench import commit, TODO
|
||||
|
||||
SESSION = uuid.UUID("11111111-2222-4333-8444-555555555555")
|
||||
|
||||
|
||||
def result_line(**fields):
|
||||
return json.dumps({"type": "result", "subtype": "success", "total_cost_usd": 0.5, "duration_ms": 12000, **fields})
|
||||
|
||||
|
||||
class StoppedAtBudget(unittest.TestCase):
|
||||
def test_holds_where_the_first_result_is_the_budget_stop(self):
|
||||
self.assertTrue(handoff_bench.stopped_at_budget(["not json", result_line(subtype="error_max_budget_usd", is_error=True)]))
|
||||
|
||||
def test_fails_where_the_worker_finished_within_the_budget(self):
|
||||
self.assertFalse(handoff_bench.stopped_at_budget([result_line()]))
|
||||
|
||||
def test_fails_where_the_stream_has_no_result(self):
|
||||
self.assertFalse(handoff_bench.stopped_at_budget(['{"type": "assistant"}']))
|
||||
|
||||
|
||||
class Summary(unittest.TestCase):
|
||||
def test_prints_one_row_per_arm_with_passes_and_means(self):
|
||||
rows = [
|
||||
{"arm": "brief", "verdict": "pass", "cost": 0.5, "seconds": 100.0},
|
||||
{"arm": "brief", "verdict": "partial", "cost": 1.5, "seconds": 300.0},
|
||||
{"arm": "resume", "verdict": "fail", "cost": 2.0, "seconds": 50.0},
|
||||
{"arm": "compact", "verdict": "pass", "cost": 0.25, "seconds": 9.0},
|
||||
]
|
||||
self.assertEqual(handoff_bench.summary(rows), "\n".join([
|
||||
"| arm | runs | pass | partial | mean cost $ | mean seconds |",
|
||||
"|---|---|---|---|---|---|",
|
||||
"| resume | 1 | 0 | 0 | 2.00 | 50 |",
|
||||
"| compact | 1 | 1 | 0 | 0.25 | 9 |",
|
||||
"| brief | 2 | 1 | 1 | 1.00 | 200 |",
|
||||
]))
|
||||
|
||||
|
||||
class Orchestration(unittest.TestCase):
|
||||
"""start, run and main on a throwaway repo, with claude and run-heavy faked. Case 7 starts at a commit holding bin/pkg/mod.py and merges a change to it. The fake claude writes a session's transcript as Claude Code does, and each resumed run adds a line to it."""
|
||||
|
||||
def setUp(self):
|
||||
scratch = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(scratch.cleanup)
|
||||
root = pathlib.Path(scratch.name)
|
||||
self.repo, self.work, self.projects = root / "repo", root / "work", root / "projects"
|
||||
self.work.mkdir()
|
||||
subprocess.run(["git", "init", "-q", str(self.repo)], check=True)
|
||||
self.base = self.commit({"todo.md": TODO, "bin/pkg/mod.py": "x = 1\n"})
|
||||
merged = self.commit({"bin/pkg/mod.py": "x = 2\n"})
|
||||
self.calls = []
|
||||
self.start_result = {"subtype": "error_max_budget_usd", "is_error": True}
|
||||
real_run = subprocess.run
|
||||
for patch in (
|
||||
unittest.mock.patch.object(advisor_bench, "REPO", self.repo),
|
||||
unittest.mock.patch.object(handoff_bench, "CASES", {"7": (self.base, merged)}),
|
||||
unittest.mock.patch.object(handoff_bench, "START_BUDGET", 0.25),
|
||||
unittest.mock.patch.object(handoff_bench, "CLAUDE_PROJECTS", self.projects),
|
||||
unittest.mock.patch.object(handoff_bench.uuid, "uuid4", return_value=SESSION),
|
||||
unittest.mock.patch.object(advisor_bench.subprocess, "run", side_effect=lambda args, **kwargs: self.fake(real_run, args, **kwargs)),
|
||||
):
|
||||
patch.start()
|
||||
self.addCleanup(patch.stop)
|
||||
self.transcript = self.projects / "slug" / f"{SESSION}.jsonl"
|
||||
|
||||
def commit(self, files):
|
||||
for path, text in files.items():
|
||||
(self.repo / path).parent.mkdir(parents=True, exist_ok=True)
|
||||
(self.repo / path).write_text(text)
|
||||
subprocess.run(["git", "-C", str(self.repo), "add", "."], check=True)
|
||||
commit(self.repo, "a commit")
|
||||
return advisor_bench.git(self.repo, "rev-parse", "HEAD").strip()
|
||||
|
||||
def fake(self, real_run, args, **kwargs):
|
||||
if args[0] == "claude":
|
||||
cwd = pathlib.Path(kwargs["cwd"])
|
||||
# What the run found as it began: the session's transcript and the files under bin/pkg.
|
||||
found = (self.transcript.read_text() if self.transcript.exists() else None, sorted(path.name for path in (cwd / "bin/pkg").iterdir()))
|
||||
self.calls.append((args, kwargs, found))
|
||||
if "--max-budget-usd" in args:
|
||||
(cwd / "bin/pkg/mod.py").write_text("x = 3\n")
|
||||
(cwd / "bin/pkg/new.py").write_text("n = 1\n")
|
||||
self.transcript.parent.mkdir(parents=True)
|
||||
self.transcript.write_text("weak\n")
|
||||
kwargs["stdout"].write(result_line(**self.start_result) + "\n")
|
||||
elif args[2] == "/compact":
|
||||
self.transcript.write_text(self.transcript.read_text() + "compacted\n")
|
||||
kwargs["stdout"].write(result_line(total_cost_usd=0.25, duration_ms=3000) + "\n")
|
||||
else:
|
||||
if "--resume" in args:
|
||||
self.transcript.write_text(self.transcript.read_text() + "strong\n")
|
||||
(cwd / "bin/pkg" / f"done_{len(self.calls)}.py").write_text("d = 1\n")
|
||||
kwargs["stdout"].write(result_line() + "\n")
|
||||
return subprocess.CompletedProcess(args, 0)
|
||||
if args[0] == str(advisor_bench.RUN_HEAVY):
|
||||
return subprocess.CompletedProcess(args, 0, stdout="3 passed\n", stderr="")
|
||||
return real_run(args, **kwargs)
|
||||
|
||||
def started(self):
|
||||
handoff_bench.start(self.work, "7-1")
|
||||
self.calls.clear()
|
||||
|
||||
def test_start_runs_the_weak_worker_to_its_budget_and_saves_the_tree_and_the_session(self):
|
||||
self.assertEqual(handoff_bench.start(self.work, "7-1"), "7-1: started")
|
||||
(claude, kwargs, _), = self.calls
|
||||
self.assertEqual(kwargs["cwd"], self.work / "7-1")
|
||||
self.assertEqual(kwargs["stdin"], subprocess.DEVNULL)
|
||||
self.assertEqual(claude[:2] + claude[3:], [
|
||||
"claude", "-p", *handoff_bench.WEAK.flags(), "--session-id", str(SESSION), "--max-budget-usd", "0.25", "--setting-sources", "project",
|
||||
"--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"])
|
||||
self.assertIn("Do the seventh thing.", claude[2])
|
||||
self.assertEqual((self.work / "7-start-1.session").read_text(), str(self.transcript))
|
||||
self.assertEqual((self.work / "7-start-1.session.jsonl").read_text(), "weak\n")
|
||||
self.assertIn("-x = 1\n+x = 3\n", (self.work / "7-start-1.diff").read_text())
|
||||
tree = self.work / "7-start-1.tree"
|
||||
self.assertEqual(((tree / "bin/pkg/mod.py").read_text(), (tree / "bin/pkg/new.py").read_text()), ("x = 3\n", "n = 1\n"))
|
||||
self.assertEqual(advisor_bench.git(tree, "status", "--porcelain"), " M bin/pkg/mod.py\n?? bin/pkg/new.py\n")
|
||||
self.assertEqual(advisor_bench.git(tree, "rev-parse", "HEAD").strip(), self.base)
|
||||
|
||||
def test_start_refuses_a_worker_that_finished_within_its_budget(self):
|
||||
self.start_result = {}
|
||||
with self.assertRaisesRegex(ValueError, "finished within its budget"):
|
||||
handoff_bench.start(self.work, "7-1")
|
||||
|
||||
def test_start_refuses_a_name_that_already_started(self):
|
||||
(self.work / "7-1").mkdir()
|
||||
with self.assertRaisesRegex(ValueError, "already started"):
|
||||
handoff_bench.start(self.work, "7-1")
|
||||
|
||||
def test_start_refuses_a_session_whose_transcript_it_cannot_find(self):
|
||||
self.transcript = self.work / "elsewhere" / f"{SESSION}.jsonl"
|
||||
with self.assertRaisesRegex(ValueError, "found 0"):
|
||||
handoff_bench.start(self.work, "7-1")
|
||||
|
||||
def test_the_resume_arm_resumes_the_weak_workers_session_on_the_strong_rung(self):
|
||||
self.started()
|
||||
self.assertEqual(handoff_bench.run(self.work, "7-resume-1"), "7-resume-1: ran")
|
||||
(claude, kwargs, _), = self.calls
|
||||
self.assertEqual(kwargs["cwd"], self.work / "7-1")
|
||||
self.assertEqual(claude[:2] + claude[3:], [
|
||||
"claude", "-p", *handoff_bench.STRONG.flags(), "--resume", str(SESSION), "--setting-sources", "project",
|
||||
"--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"])
|
||||
self.assertEqual(claude[2], (handoff_bench.BENCH / "handoff-resume-prompt.md").read_text().format(rung="claude-opus-5-5 at medium effort"))
|
||||
diff = (self.work / "7-resume-1.diff").read_text()
|
||||
self.assertIn("+x = 3\n", diff)
|
||||
self.assertIn("+++ b/bin/pkg/done_1.py", diff)
|
||||
self.assertEqual((self.work / "7-resume-1.tests").read_text(), "exit 0\n3 passed\n")
|
||||
|
||||
def test_the_compact_arm_has_the_weak_model_compact_the_session_before_the_strong_one_resumes_it(self):
|
||||
self.started()
|
||||
handoff_bench.run(self.work, "7-compact-1")
|
||||
(compact, _, _), (claude, _, found) = self.calls
|
||||
self.assertEqual(compact[:2] + compact[3:], [
|
||||
"claude", "-p", *handoff_bench.WEAK.flags(), "--resume", str(SESSION), "--setting-sources", "project",
|
||||
"--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"])
|
||||
self.assertEqual(compact[2], "/compact")
|
||||
self.assertEqual(claude[3:7], [*handoff_bench.STRONG.flags()][:4])
|
||||
self.assertIn("--resume", claude)
|
||||
self.assertEqual(found[0], "weak\ncompacted\n")
|
||||
self.assertEqual(handoff_bench.outcome(self.work, "7-compact-1"), {"cost": 0.75, "seconds": 15.0})
|
||||
|
||||
def test_the_brief_arm_starts_a_fresh_session_with_the_item_and_the_brief(self):
|
||||
self.started()
|
||||
self.assertEqual(handoff_bench.run(self.work, "7-brief-1"), "7-brief-1: ran")
|
||||
(claude, kwargs, _), = self.calls
|
||||
self.assertEqual(kwargs["cwd"], self.work / "7-1")
|
||||
self.assertEqual(claude[:2] + claude[3:], [
|
||||
"claude", "-p", *handoff_bench.STRONG.flags(), "--no-session-persistence", "--setting-sources", "project",
|
||||
"--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"])
|
||||
self.assertIn("Do the seventh thing.", claude[2])
|
||||
self.assertIn("## Handoff", claude[2])
|
||||
self.assertIn("+++ b/bin/pkg/done_1.py", (self.work / "7-brief-1.diff").read_text())
|
||||
|
||||
def test_each_arm_begins_on_the_tree_and_the_session_that_start_left(self):
|
||||
self.started()
|
||||
for arm in ("resume", "compact", "brief"):
|
||||
handoff_bench.run(self.work, f"7-{arm}-1")
|
||||
self.assertEqual([found for _, _, found in self.calls if found[0] != "weak\ncompacted\n"], [("weak\n", ["mod.py", "new.py"])] * 3)
|
||||
self.assertNotIn("done_1.py", (self.work / "7-brief-1.diff").read_text())
|
||||
|
||||
def test_run_refuses_an_arm_that_already_ran_or_never_started(self):
|
||||
with self.assertRaisesRegex(ValueError, "start 7-1 first"):
|
||||
handoff_bench.run(self.work, "7-brief-1")
|
||||
self.started()
|
||||
handoff_bench.run(self.work, "7-brief-1")
|
||||
with self.assertRaisesRegex(ValueError, "already ran"):
|
||||
handoff_bench.run(self.work, "7-brief-1")
|
||||
|
||||
def test_run_refuses_an_unknown_arm(self):
|
||||
self.started()
|
||||
with self.assertRaisesRegex(ValueError, "arm"):
|
||||
handoff_bench.run(self.work, "7-sonnet-1")
|
||||
|
||||
def test_run_refuses_an_arm_whose_run_ended_in_an_error_before_it_tests(self):
|
||||
self.started()
|
||||
with unittest.mock.patch.object(advisor_bench, "save_change") as save_change, unittest.mock.patch.object(advisor_bench, "read_result", side_effect=ValueError("the run ended in an error")):
|
||||
with self.assertRaisesRegex(ValueError, "ended in an error"):
|
||||
handoff_bench.run(self.work, "7-brief-1")
|
||||
save_change.assert_not_called()
|
||||
|
||||
def test_main_grades_with_the_handoff_cases_and_prints_the_summary(self):
|
||||
self.started()
|
||||
handoff_bench.run(self.work, "7-brief-1")
|
||||
with unittest.mock.patch.object(advisor_bench, "grade", return_value="7-brief-1: pass") as grade, unittest.mock.patch("builtins.print") as printed:
|
||||
handoff_bench.main([str(self.work), "grade", "7-brief-1"])
|
||||
grade.assert_called_once_with(self.work, "7-brief-1", handoff_bench.CASES)
|
||||
printed.assert_called_once_with("7-brief-1: pass", flush=True)
|
||||
(self.work / "7-brief-1.verdict").write_text(json.dumps({"verdict": "pass"}))
|
||||
with unittest.mock.patch("builtins.print") as printed:
|
||||
handoff_bench.main([str(self.work), "summary"])
|
||||
self.assertIn("| brief | 1 | 1 | 0 | 0.50 | 12 |", printed.call_args.args[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user