Files
claude-config/workflow-bench/test_handoff_bench.py
T

256 lines
13 KiB
Python

import json
import pathlib
import subprocess
import tempfile
import unittest
import uuid
import advisor_bench
from test_advisor_bench import commit, TODO
import handoff_bench
SESSION = uuid.UUID("11111111-2222-4333-8444-555555555555")
def result_line(**fields):
return json.dumps({"type": "result", "subtype": "success", "total_cost_usd": 0.5, "duration_ms": 12000, **fields})
class StoppedAtBudget(unittest.TestCase):
def test_holds_where_the_first_result_is_the_budget_stop(self):
self.assertTrue(handoff_bench.stopped_at_budget(["not json", '{"type": "assistant"}', result_line(subtype="error_max_budget_usd", is_error=True)]))
def test_fails_where_the_worker_finished_within_the_budget(self):
self.assertFalse(handoff_bench.stopped_at_budget([result_line()]))
def test_fails_where_the_stream_has_no_result(self):
self.assertFalse(handoff_bench.stopped_at_budget(['{"type": "assistant"}', "[]"]))
class Summary(unittest.TestCase):
def test_prints_one_row_per_arm_with_passes_and_means(self):
rows = [
{"arm": "brief", "verdict": "pass", "cost": 0.5, "seconds": 100.0},
{"arm": "brief", "verdict": "partial", "cost": 1.5, "seconds": 300.0},
{"arm": "resume", "verdict": "fail", "cost": 2.0, "seconds": 50.0},
{"arm": "compact", "verdict": "pass", "cost": 0.25, "seconds": 9.0},
]
self.assertEqual(handoff_bench.summary(rows), "\n".join([
"| arm | runs | pass | partial | mean cost $ | mean seconds |",
"|---|---|---|---|---|---|",
"| resume | 1 | 0 | 0 | 2.00 | 50 |",
"| compact | 1 | 1 | 0 | 0.25 | 9 |",
"| brief | 2 | 1 | 1 | 1.00 | 200 |",
]))
class Orchestration(unittest.TestCase):
"""start, run and main on a throwaway repo, with claude and run-heavy faked. Case 7 starts at a commit holding bin/pkg/mod.py and merges a change to it. The fake claude writes a session's transcript as Claude Code does, and each resumed run adds a line to it."""
def setUp(self):
scratch = tempfile.TemporaryDirectory()
self.addCleanup(scratch.cleanup)
root = pathlib.Path(scratch.name)
self.repo, self.work, self.projects = root / "repo", root / "work", root / "projects"
self.work.mkdir()
subprocess.run(["git", "init", "-q", str(self.repo)], check=True)
(self.repo / "link").symlink_to("todo.md")
self.base = self.commit({"todo.md": TODO, "bin/pkg/mod.py": "x = 1\n", "bin/other/o.py": "o = 1\n"})
merged = self.commit({"bin/pkg/mod.py": "x = 2\n"})
self.commit({"bin/other/o.py": "o = 2\n"})
self.calls, self.heavy = [], []
self.start_result = {"subtype": "error_max_budget_usd", "is_error": True, "total_cost_usd": 1.0}
self.compact_result, self.strong_result = {}, {}
# The prompts the judge was given.
self.judged = []
self.handoff = handoff_bench.Handoff(advisor_bench.Bench(self.repo, {"7": (self.base, merged)}, self.fake), 0.25, self.projects, lambda: SESSION)
self.transcript = self.projects / "slug" / f"{SESSION}.jsonl"
def commit(self, files):
for path, text in files.items():
(self.repo / path).parent.mkdir(parents=True, exist_ok=True)
(self.repo / path).write_text(text)
subprocess.run(["git", "-C", str(self.repo), "add", "."], check=True)
commit(self.repo, "a commit")
return advisor_bench.git(self.repo, "rev-parse", "HEAD").strip()
def fake(self, args, **kwargs):
"""Stands in for subprocess.run where the bench runs claude or the tests."""
if args[0] == "claude" and "--tools" in args:
self.judged.append(args[2])
return subprocess.CompletedProcess(args, 0, stdout=json.dumps({"result": '{"verdict": "pass"}'}), stderr="")
if args[0] == "claude":
cwd = pathlib.Path(kwargs["cwd"])
# What the run found as it began: the session's transcript and the files under bin/pkg.
found = (self.transcript.read_text() if self.transcript.exists() else None, sorted(path.name for path in (cwd / "bin/pkg").iterdir()))
self.calls.append((args, kwargs, found))
if "--max-budget-usd" in args:
(cwd / "bin/pkg/mod.py").write_text("x = 3\n")
# Against the bench rules, the worker commits a part of its change.
subprocess.run(["git", "-C", str(cwd), "add", "bin/pkg/mod.py"], check=True)
commit(cwd, "against the rules")
(cwd / "bin/pkg/new.py").write_text("n = 1\n")
self.transcript.parent.mkdir(parents=True)
self.transcript.write_text("weak\n")
kwargs["stdout"].write(result_line(**self.start_result) + "\n")
elif args[2] == "/compact":
self.transcript.write_text(self.transcript.read_text() + "compacted\n")
kwargs["stdout"].write(result_line(**self.compact_result, total_cost_usd=1.25, duration_ms=3000) + "\n")
else:
if "--resume" in args:
self.transcript.write_text(self.transcript.read_text() + "strong\n")
# A resumed session counts its cost from the session's start, and may open with a result that holds no work.
kwargs["stdout"].write(result_line(total_cost_usd=1.0, duration_ms=0) + "\n")
(cwd / "bin/pkg" / f"done_{len(self.calls)}.py").write_text("d = 1\n")
kwargs["stdout"].write("a line that is not JSON\n")
kwargs["stdout"].write(result_line(**self.strong_result, total_cost_usd=3.0 if "--resume" in args else 0.5) + "\n")
return subprocess.CompletedProcess(args, 0)
if args[0] == str(advisor_bench.RUN_HEAVY):
self.heavy.append(args)
return subprocess.CompletedProcess(args, 0, stdout="3 passed\n", stderr="")
raise AssertionError(f"the bench ran {args}")
def started(self):
handoff_bench.start(self.handoff, self.work, "7-1")
self.calls.clear()
def test_start_runs_the_weak_worker_to_its_budget_and_saves_the_tree_and_the_session(self):
self.assertEqual(handoff_bench.start(self.handoff, self.work, "7-1"), "7-1: started")
(claude, kwargs, _), = self.calls
self.assertEqual(kwargs["cwd"], self.work / "7-1")
self.assertEqual((kwargs["stdin"], kwargs["stderr"]), (subprocess.DEVNULL, subprocess.STDOUT))
self.assertEqual(claude[:2] + claude[3:], [
"claude", "-p", *handoff_bench.WEAK.flags(), "--session-id", str(SESSION), "--max-budget-usd", "0.25", "--setting-sources", "project",
"--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"])
self.assertIn("Do the seventh thing.", claude[2])
self.assertIn(f"a throwaway clone at `{self.base}`. Implement todo item 7 below", claude[2])
self.assertEqual((self.work / "7-start-1.session").read_text(), str(self.transcript))
self.assertEqual((self.work / "7-start-1.session.jsonl").read_text(), "weak\n")
self.assertIn("-x = 1\n+x = 3\n", (self.work / "7-start-1.diff").read_text())
self.assertIn("+n = 1\n", (self.work / "7-start-1.diff").read_text())
tree = self.work / "7-start-1.tree"
self.assertEqual(((tree / "bin/pkg/mod.py").read_text(), (tree / "bin/pkg/new.py").read_text()), ("x = 3\n", "n = 1\n"))
self.assertEqual(advisor_bench.git(tree, "status", "--porcelain"), " M bin/pkg/mod.py\n?? bin/pkg/new.py\n")
self.assertEqual(advisor_bench.git(tree, "rev-parse", "HEAD").strip(), self.base)
self.assertTrue((tree / "link").is_symlink())
def test_start_refuses_a_worker_that_finished_within_its_budget(self):
self.start_result = {}
with self.assertRaisesRegex(ValueError, "^the weak worker finished within its budget, or its run failed, so there is no handoff to bench$"):
handoff_bench.start(self.handoff, self.work, "7-1")
def test_start_refuses_a_name_that_already_started(self):
(self.work / "7-1").mkdir()
with self.assertRaisesRegex(ValueError, "already started"):
handoff_bench.start(self.handoff, self.work, "7-1")
def test_start_refuses_a_session_whose_transcript_it_cannot_find(self):
self.transcript = self.work / "elsewhere" / f"{SESSION}.jsonl"
with self.assertRaisesRegex(ValueError, "found 0"):
handoff_bench.start(self.handoff, self.work, "7-1")
def test_the_resume_arm_resumes_the_weak_workers_session_on_the_strong_rung(self):
self.started()
self.assertEqual(handoff_bench.run(self.handoff, self.work, "7-resume-1"), "7-resume-1: ran")
(claude, kwargs, _), = self.calls
self.assertEqual(kwargs["cwd"], self.work / "7-1")
self.assertEqual(claude[:2] + claude[3:], [
"claude", "-p", *handoff_bench.STRONG.flags(), "--resume", str(SESSION), "--setting-sources", "project",
"--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"])
self.assertEqual(claude[2], (handoff_bench.BENCH / "handoff-resume-prompt.md").read_text().format(rung="claude-opus-5-5 at medium effort"))
diff = (self.work / "7-resume-1.diff").read_text()
self.assertIn("+x = 3\n", diff)
self.assertIn("+++ b/bin/pkg/done_1.py", diff)
self.assertEqual((self.work / "7-resume-1.tests").read_text(), "exit 0\n3 passed\n")
self.assertEqual(self.heavy[-1][-1], "bin/pkg")
self.assertNotIn("bin/other", self.heavy[-1])
self.assertTrue((self.work / "7-1" / "link").is_symlink())
self.assertEqual(handoff_bench.outcome(self.work, "7-resume-1"), {"cost": 2.0, "seconds": 12.0})
def test_the_compact_arm_has_the_weak_model_compact_the_session_before_the_strong_one_resumes_it(self):
self.started()
handoff_bench.run(self.handoff, self.work, "7-compact-1")
(compact, _, _), (claude, _, found) = self.calls
self.assertEqual(compact[:2] + compact[3:], [
"claude", "-p", *handoff_bench.WEAK.flags(), "--resume", str(SESSION), "--setting-sources", "project",
"--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"])
self.assertEqual(compact[2], "/compact")
self.assertEqual(claude[3:7], [*handoff_bench.STRONG.flags()][:4])
self.assertIn("--resume", claude)
self.assertEqual(found[0], "weak\ncompacted\n")
self.assertEqual(handoff_bench.outcome(self.work, "7-compact-1"), {"cost": 2.0, "seconds": 15.0})
def test_the_brief_arm_starts_a_fresh_session_with_the_item_and_the_brief(self):
self.started()
self.assertEqual(handoff_bench.run(self.handoff, self.work, "7-brief-1"), "7-brief-1: ran")
(claude, kwargs, _), = self.calls
self.assertEqual(kwargs["cwd"], self.work / "7-1")
self.assertEqual(claude[:2] + claude[3:], [
"claude", "-p", *handoff_bench.STRONG.flags(), "--no-session-persistence", "--setting-sources", "project",
"--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"])
self.assertIn("### 7. Do the seventh thing.", claude[2])
self.assertTrue(claude[2].endswith("- End with a two-line summary of what you changed.\n\n\n## Handoff\n\n" + (handoff_bench.BENCH / "handoff-brief-prompt.md").read_text().split("## Handoff\n\n")[1]), claude[2][-400:])
self.assertIn("+++ b/bin/pkg/done_1.py", (self.work / "7-brief-1.diff").read_text())
def test_each_arm_begins_on_the_tree_and_the_session_that_start_left(self):
self.started()
for arm in ("resume", "compact", "brief"):
handoff_bench.run(self.handoff, self.work, f"7-{arm}-1")
self.assertEqual([found for _, _, found in self.calls if found[0] != "weak\ncompacted\n"], [("weak\n", ["mod.py", "new.py"])] * 3)
self.assertNotIn("done_1.py", (self.work / "7-brief-1.diff").read_text())
def test_run_refuses_an_arm_that_already_ran_or_never_started(self):
with self.assertRaisesRegex(ValueError, "start 7-1 first"):
handoff_bench.run(self.handoff, self.work, "7-brief-1")
self.started()
handoff_bench.run(self.handoff, self.work, "7-brief-1")
with self.assertRaisesRegex(ValueError, "already ran"):
handoff_bench.run(self.handoff, self.work, "7-brief-1")
def test_run_refuses_an_unknown_arm(self):
self.started()
with self.assertRaisesRegex(ValueError, "arm"):
handoff_bench.run(self.handoff, self.work, "7-sonnet-1")
def test_run_refuses_an_arm_whose_run_ended_in_an_error_before_it_tests(self):
self.started()
self.strong_result = {"is_error": True, "result": "overloaded"}
with self.assertRaisesRegex(ValueError, "ended in an error: overloaded"):
handoff_bench.run(self.handoff, self.work, "7-brief-1")
self.assertEqual(self.heavy, [])
self.assertFalse((self.work / "7-brief-1.diff").exists())
def test_run_refuses_a_compact_arm_whose_compaction_failed_before_the_strong_model_runs(self):
self.started()
self.compact_result = {"is_error": True, "result": "no conversation to compact"}
with self.assertRaisesRegex(ValueError, "^the run ended in an error: no conversation to compact$"):
handoff_bench.run(self.handoff, self.work, "7-compact-1")
self.assertEqual([args[2] for args, _, _ in self.calls], ["/compact"])
self.assertFalse((self.work / "7-compact-1.jsonl").exists())
def test_outcome_refuses_a_stream_with_no_result_line(self):
self.started()
(self.work / "7-brief-1.jsonl").write_text("not json\n[]\n")
with self.assertRaisesRegex(ValueError, "^the stream has no result line$"):
handoff_bench.outcome(self.work, "7-brief-1")
def test_main_grades_with_the_handoff_cases_and_prints_the_summary(self):
steps, printed = handoff_bench.steps_of(self.handoff), []
handoff_bench.main([str(self.work), "start", "7-1"], steps, printed.append)
handoff_bench.main([str(self.work), "run", "7-brief-1"], steps, printed.append)
self.assertEqual(printed, ["7-1: started", "7-brief-1: ran"])
printed.clear()
handoff_bench.main([str(self.work), "grade", "7-brief-1"], steps, printed.append)
self.assertEqual(printed, ["7-brief-1: pass"])
# The judge saw the handoff case's merged change, so the grade ran over the handoff's cases.
(prompt,) = self.judged
self.assertIn("-x = 1\n+x = 2\n", prompt)
(self.work / "7-brief-1.verdict").write_text(json.dumps({"verdict": "pass"}))
printed.clear()
handoff_bench.main([str(self.work), "summary"], steps, printed.append)
self.assertIn("| brief | 1 | 1 | 0 | 0.50 | 12 |", printed[-1])
if __name__ == "__main__":
unittest.main()