From e42eb5b28825729cbc464ea5531a99d590f5ab7c Mon Sep 17 00:00:00 2001 From: Lilleman auf Larv Date: Tue, 17 Nov 2026 02:57:22 +0100 Subject: [PATCH] Bench a model step's handoff: resume, compact or a fresh session with a brief --- .gitignore | 4 + workflow-bench/advisor_bench.py | 24 ++- workflow-bench/handoff-brief-prompt.md | 4 + workflow-bench/handoff-resume-prompt.md | 1 + workflow-bench/handoff_bench.py | 175 ++++++++++++++++++ workflow-bench/test_handoff_bench.py | 224 ++++++++++++++++++++++++ 6 files changed, 423 insertions(+), 9 deletions(-) create mode 100644 workflow-bench/handoff-brief-prompt.md create mode 100644 workflow-bench/handoff-resume-prompt.md create mode 100644 workflow-bench/handoff_bench.py create mode 100644 workflow-bench/test_handoff_bench.py diff --git a/.gitignore b/.gitignore index 94c1be88..19e7cb16 100644 --- a/.gitignore +++ b/.gitignore @@ -184,9 +184,13 @@ !workflow-bench/bench.py !workflow-bench/compare-prompt.md !workflow-bench/compare.py +!workflow-bench/handoff-brief-prompt.md +!workflow-bench/handoff-resume-prompt.md +!workflow-bench/handoff_bench.py !workflow-bench/review_git.py !workflow-bench/run-prompt.md !workflow-bench/setup.cfg !workflow-bench/test_advisor_bench.py !workflow-bench/test_bench.py +!workflow-bench/test_handoff_bench.py !workflow-bench/test_review_git.py diff --git a/workflow-bench/advisor_bench.py b/workflow-bench/advisor_bench.py index a4ec633f..9c40f1e3 100644 --- a/workflow-bench/advisor_bench.py +++ b/workflow-bench/advisor_bench.py @@ -146,19 +146,25 @@ def run(work, name): "--no-session-persistence", "--output-format", "stream-json", "--verbose"], cwd=clone, stdout=out, stderr=subprocess.STDOUT) read_result((work / f"{name}.jsonl").read_text().splitlines()) - git(clone, "add", "--all") - (work / f"{name}.diff").write_text(git(clone, "diff", "--cached", base)) - changed = [*git(clone, "diff", "--cached", "--name-only", base).split(), *git(REPO, "diff", "--name-only", base, CASES[case][1]).split()] - targets = [target for target in suite_targets(changed) if (clone / target).exists()] or ["bin"] - tests = subprocess.run([str(RUN_HEAVY), *SUITE, *targets], cwd=clone, capture_output=True, text=True) - (work / f"{name}.tests").write_text(f"exit {tests.returncode}\n{tests.stdout[-1500:]}") + save_change(work, name, clone, CASES[case]) return f"{name}: ran" -def grade(work, name): - """Have the judge rule on one run's diff, blind to its arm, and save the verdict beside it.""" +def save_change(work, name, clone, commits): + """Save the clone's change since the case's start commit as .diff, and the result of the tests of the packages that it or the merged change touches as .tests.""" + base, merged = commits + git(clone, "add", "--all") + (work / f"{name}.diff").write_text(git(clone, "diff", "--cached", base)) + changed = [*git(clone, "diff", "--cached", "--name-only", base).split(), *git(REPO, "diff", "--name-only", base, merged).split()] + targets = [target for target in suite_targets(changed) if (clone / target).exists()] or ["bin"] + tests = subprocess.run([str(RUN_HEAVY), *SUITE, *targets], cwd=clone, capture_output=True, text=True) + (work / f"{name}.tests").write_text(f"exit {tests.returncode}\n{tests.stdout[-1500:]}") + + +def grade(work, name, cases=None): + """Have the judge rule on one run's diff, blind to its arm, and save the verdict beside it. cases stands in for CASES where another bench grades its own.""" case = split_name(name)[0] - base, merged = CASES[case] + base, merged = (cases or CASES)[case] prompt = (BENCH / "advisor-judge-prompt.md").read_text().format( text=item_text(git(REPO, "show", f"{base}:todo.md"), case), reference=git(REPO, "diff", base, merged, "--", ".", ":!todo.md", ":!docs", ":!CHANGELOG.md"), diff=(work / f"{name}.diff").read_text(), tests=(work / f"{name}.tests").read_text()) diff --git a/workflow-bench/handoff-brief-prompt.md b/workflow-bench/handoff-brief-prompt.md new file mode 100644 index 00000000..067e9f89 --- /dev/null +++ b/workflow-bench/handoff-brief-prompt.md @@ -0,0 +1,4 @@ + +## Handoff + +A worker on a weaker model began this item and stopped before it was done, asking for a stronger model. You start fresh, without its transcript. Its unfinished change is in the clone, uncommitted: read `git status` and `git diff` first. Keep what holds, redo what doesn't, and finish the item. diff --git a/workflow-bench/handoff-resume-prompt.md b/workflow-bench/handoff-resume-prompt.md new file mode 100644 index 00000000..2d271159 --- /dev/null +++ b/workflow-bench/handoff-resume-prompt.md @@ -0,0 +1 @@ +Continue the task; the previous run ended before the item was done. Your request for a higher rung went through: you now run on {rung}. The bench rules still hold. diff --git a/workflow-bench/handoff_bench.py b/workflow-bench/handoff_bench.py new file mode 100644 index 00000000..09c8947a --- /dev/null +++ b/workflow-bench/handoff_bench.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Bench how a worker's step to a stronger model hands the chunk over: handoff_bench.py start - ..., then run|grade -- ..., then summary + +The question: when loop-claude steps a worker to a rung on another model, which handoff serves the stronger model +best? Each case is an archived chunk of this repo, as in advisor_bench.py, whose judge and files this bench shares. +- `start` runs WEAK on the item in the clone /- until it has spent START_BUDGET, which + stands in for a run that ends asking for a higher rung. A worker that finishes within the budget leaves no + handoff, so `start` fails on it. It saves the change so far (-start-.diff), and a copy of the clone + (.tree) and of the session's transcript (.session.jsonl), which every arm starts from. +- `run` puts the clone and the session back as `start` left them, then continues on STRONG, one arm of a case at a + time. The resume arm resumes WEAK's session with handoff-resume-prompt.md. The compact arm does the same once WEAK + has compacted the session with `/compact`, whose stream it saves as .compact.jsonl. The brief arm starts a + session that has the item and handoff-brief-prompt.md, and no transcript. +- `grade` is advisor_bench.py's, over these cases. `summary` counts an arm's cost and seconds from its own runs, + the compaction included, and leaves out the start, which the arms share. +Run each arm of each case once; the table shows a direction, not a significance. +""" +import concurrent.futures +import collections +import json +import pathlib +import shutil +import subprocess +import sys +import threading +import uuid + +import advisor_bench +from advisor_bench import Arm + +BENCH = pathlib.Path(__file__).resolve().parent +# The ladder's first step that changes the model: bin/model_routing/routing-ladder.json. +WEAK = Arm("claude-sonnet-5-5", "high") +STRONG = Arm("claude-opus-5-5", "medium") +ARMS = ("resume", "compact", "brief") +# Claude Code keeps a session's transcript at //.jsonl. +CLAUDE_PROJECTS = pathlib.Path.home() / ".claude/projects" +# One per case and run number: its arms share one clone, so they run one at a time. +CLONE_LOCKS: collections.defaultdict = collections.defaultdict(threading.Lock) +# L 3 chunks, by todo item. advisor_bench's L 1 and L 2 chunks are too small: Sonnet 5.5 finished five of the six for +# under $0.20, before any handoff. +CASES = { + "143": ("45cb331a4", "545f3c347"), + "163": ("d4cf6e7fe", "d1b0fe3a4"), + "198": ("4a0ba5921", "fd60b6da"), + "210": ("4764d3d52", "dcbdc970"), + "227": ("eb7b50698", "3bde9cd6d"), + "239": ("d583f8d48", "47a40757"), +} +# Dollars WEAK may spend before the handoff. In a trial, $0.36 stopped Sonnet 5.5 on items 163 and 227 after 11 to 14 +# turns, with about 250 lines of diff. +START_BUDGET = 0.36 +CLAUDE_FLAGS = ["--setting-sources", "project", "--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"] + + +def stopped_at_budget(lines): + """Whether a claude stream's first result line, the worker's own, says the run stopped at --max-budget-usd.""" + for line in lines: + try: + entry = json.loads(line) + except ValueError: + continue + if entry.get("type") == "result": + return entry.get("subtype") == "error_max_budget_usd" + return False + + +def run_prompt(case): + """advisor_bench's prompt for the case's item, as the item's text stood at the start commit.""" + base = CASES[case][0] + text = advisor_bench.item_text(advisor_bench.git(advisor_bench.REPO, "show", f"{base}:todo.md"), case) + return (BENCH / "advisor-run-prompt.md").read_text().format(base=base, item=case, text=text) + + +def claude(prompt, flags, clone, stream): + with stream.open("w") as out: + subprocess.run(["claude", "-p", prompt, *flags, *CLAUDE_FLAGS], cwd=clone, stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT) + + +def start(work, name): + """WEAK's capped run of -, and the tree and session it leaves the arms; the files are in the module docstring.""" + case, number = name.split("-") + base = CASES[case][0] + clone, saved = work / name, work / f"{case}-start-{number}" + if clone.exists(): + raise ValueError(f"{name} already started: remove {clone}") + advisor_bench.fresh_clone(base, clone) + session = str(uuid.uuid4()) + stream = saved.with_suffix(".jsonl") + claude(run_prompt(case), [*WEAK.flags(), "--session-id", session, "--max-budget-usd", str(START_BUDGET)], clone, stream) + if not stopped_at_budget(stream.read_text().splitlines()): + raise ValueError("the weak worker finished within its budget, or its run failed, so there is no handoff to bench") + transcripts = list(CLAUDE_PROJECTS.glob(f"*/{session}.jsonl")) + if len(transcripts) != 1: + raise ValueError(f"expected one transcript of session {session} under {CLAUDE_PROJECTS}, found {len(transcripts)}") + shutil.copy(transcripts[0], saved.with_suffix(".session.jsonl")) + saved.with_suffix(".session").write_text(str(transcripts[0])) + # The worker may have committed against the bench rules, so the arms get its whole change since base, uncommitted. + advisor_bench.git(clone, "add", "--all") + saved.with_suffix(".diff").write_text(advisor_bench.git(clone, "diff", "--cached", base)) + advisor_bench.git(clone, "reset", "-q", "--mixed", base) + shutil.copytree(clone, saved.with_suffix(".tree"), symlinks=True) + return f"{name}: started" + + +def run(work, name): + """STRONG's run of one arm, --, from the tree and session `start` left.""" + case, arm, number = advisor_bench.split_name(name) + if arm not in ARMS: + raise ValueError(f"the arm {arm!r} is none of {ARMS}") + clone, saved = work / f"{case}-{number}", work / f"{case}-start-{number}" + if not saved.with_suffix(".tree").exists(): + raise ValueError(f"start {case}-{number} first") + stream = work / f"{name}.jsonl" + if stream.exists(): + raise ValueError(f"{name} already ran: remove {stream}") + with CLONE_LOCKS[clone]: + shutil.rmtree(clone) + shutil.copytree(saved.with_suffix(".tree"), clone, symlinks=True) + transcript = pathlib.Path(saved.with_suffix(".session").read_text()) + shutil.copy(saved.with_suffix(".session.jsonl"), transcript) + resumed = ["--resume", transcript.stem] + if arm == "compact": + claude("/compact", [*WEAK.flags(), *resumed], clone, work / f"{name}.compact.jsonl") + if arm == "brief": + claude(run_prompt(case) + "\n" + (BENCH / "handoff-brief-prompt.md").read_text(), [*STRONG.flags(), "--no-session-persistence"], clone, stream) + else: + prompt = (BENCH / "handoff-resume-prompt.md").read_text().format(rung=f"{STRONG.model} at {STRONG.effort} effort") + claude(prompt, [*STRONG.flags(), *resumed], clone, stream) + outcome(work, name) + advisor_bench.save_change(work, name, clone, CASES[case]) + return f"{name}: ran" + + +def outcome(work, name): + """The cost and seconds of an arm's run: its STRONG stream's, plus its compaction's where it made one.""" + streams = [stream for stream in (work / f"{name}.compact.jsonl", work / f"{name}.jsonl") if stream.exists()] + results = [advisor_bench.read_result(stream.read_text().splitlines()) for stream in streams] + return {"cost": sum(result.cost for result in results), "seconds": sum(result.seconds for result in results)} + + +def rows_of(work): + """One row per graded run: its arm, verdict, cost and seconds.""" + rows = [] + for verdict_file in sorted(work.glob("*.verdict")): + name = verdict_file.stem + rows.append({"arm": advisor_bench.split_name(name)[1], "verdict": json.loads(verdict_file.read_text())["verdict"], **outcome(work, name)}) + return rows + + +def summary(rows): + """A markdown table with one row per arm, in ARMS' order: its runs, passes, partials, mean cost and mean seconds.""" + lines = ["| arm | runs | pass | partial | mean cost $ | mean seconds |", "|---|---|---|---|---|---|"] + for name in ARMS: + own = [row for row in rows if row["arm"] == name] + if own: + passes, partials = (sum(row["verdict"] == kind for row in own) for kind in ("pass", "partial")) + cost, seconds = (sum(row[key] for row in own) / len(own) for key in ("cost", "seconds")) + lines.append(f"| {name} | {len(own)} | {passes} | {partials} | {cost:.2f} | {seconds:.0f} |") + return "\n".join(lines) + + +def main(argv): + work, step, names = pathlib.Path(argv[0]), argv[1], argv[2:] + if step == "summary": + print(summary(rows_of(work))) + return + steps = {"start": start, "run": run, "grade": lambda work, name: advisor_bench.grade(work, name, CASES)} + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: + for result in pool.map(lambda name: advisor_bench.attempted(steps[step], work, name), names): + print(result, flush=True) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/workflow-bench/test_handoff_bench.py b/workflow-bench/test_handoff_bench.py new file mode 100644 index 00000000..8c9f5b45 --- /dev/null +++ b/workflow-bench/test_handoff_bench.py @@ -0,0 +1,224 @@ +import json +import pathlib +import subprocess +import tempfile +import unittest +import unittest.mock +import uuid + +import advisor_bench +import handoff_bench +from test_advisor_bench import commit, TODO + +SESSION = uuid.UUID("11111111-2222-4333-8444-555555555555") + + +def result_line(**fields): + return json.dumps({"type": "result", "subtype": "success", "total_cost_usd": 0.5, "duration_ms": 12000, **fields}) + + +class StoppedAtBudget(unittest.TestCase): + def test_holds_where_the_first_result_is_the_budget_stop(self): + self.assertTrue(handoff_bench.stopped_at_budget(["not json", result_line(subtype="error_max_budget_usd", is_error=True)])) + + def test_fails_where_the_worker_finished_within_the_budget(self): + self.assertFalse(handoff_bench.stopped_at_budget([result_line()])) + + def test_fails_where_the_stream_has_no_result(self): + self.assertFalse(handoff_bench.stopped_at_budget(['{"type": "assistant"}'])) + + +class Summary(unittest.TestCase): + def test_prints_one_row_per_arm_with_passes_and_means(self): + rows = [ + {"arm": "brief", "verdict": "pass", "cost": 0.5, "seconds": 100.0}, + {"arm": "brief", "verdict": "partial", "cost": 1.5, "seconds": 300.0}, + {"arm": "resume", "verdict": "fail", "cost": 2.0, "seconds": 50.0}, + {"arm": "compact", "verdict": "pass", "cost": 0.25, "seconds": 9.0}, + ] + self.assertEqual(handoff_bench.summary(rows), "\n".join([ + "| arm | runs | pass | partial | mean cost $ | mean seconds |", + "|---|---|---|---|---|---|", + "| resume | 1 | 0 | 0 | 2.00 | 50 |", + "| compact | 1 | 1 | 0 | 0.25 | 9 |", + "| brief | 2 | 1 | 1 | 1.00 | 200 |", + ])) + + +class Orchestration(unittest.TestCase): + """start, run and main on a throwaway repo, with claude and run-heavy faked. Case 7 starts at a commit holding bin/pkg/mod.py and merges a change to it. The fake claude writes a session's transcript as Claude Code does, and each resumed run adds a line to it.""" + + def setUp(self): + scratch = tempfile.TemporaryDirectory() + self.addCleanup(scratch.cleanup) + root = pathlib.Path(scratch.name) + self.repo, self.work, self.projects = root / "repo", root / "work", root / "projects" + self.work.mkdir() + subprocess.run(["git", "init", "-q", str(self.repo)], check=True) + self.base = self.commit({"todo.md": TODO, "bin/pkg/mod.py": "x = 1\n"}) + merged = self.commit({"bin/pkg/mod.py": "x = 2\n"}) + self.calls = [] + self.start_result = {"subtype": "error_max_budget_usd", "is_error": True} + real_run = subprocess.run + for patch in ( + unittest.mock.patch.object(advisor_bench, "REPO", self.repo), + unittest.mock.patch.object(handoff_bench, "CASES", {"7": (self.base, merged)}), + unittest.mock.patch.object(handoff_bench, "START_BUDGET", 0.25), + unittest.mock.patch.object(handoff_bench, "CLAUDE_PROJECTS", self.projects), + unittest.mock.patch.object(handoff_bench.uuid, "uuid4", return_value=SESSION), + unittest.mock.patch.object(advisor_bench.subprocess, "run", side_effect=lambda args, **kwargs: self.fake(real_run, args, **kwargs)), + ): + patch.start() + self.addCleanup(patch.stop) + self.transcript = self.projects / "slug" / f"{SESSION}.jsonl" + + def commit(self, files): + for path, text in files.items(): + (self.repo / path).parent.mkdir(parents=True, exist_ok=True) + (self.repo / path).write_text(text) + subprocess.run(["git", "-C", str(self.repo), "add", "."], check=True) + commit(self.repo, "a commit") + return advisor_bench.git(self.repo, "rev-parse", "HEAD").strip() + + def fake(self, real_run, args, **kwargs): + if args[0] == "claude": + cwd = pathlib.Path(kwargs["cwd"]) + # What the run found as it began: the session's transcript and the files under bin/pkg. + found = (self.transcript.read_text() if self.transcript.exists() else None, sorted(path.name for path in (cwd / "bin/pkg").iterdir())) + self.calls.append((args, kwargs, found)) + if "--max-budget-usd" in args: + (cwd / "bin/pkg/mod.py").write_text("x = 3\n") + (cwd / "bin/pkg/new.py").write_text("n = 1\n") + self.transcript.parent.mkdir(parents=True) + self.transcript.write_text("weak\n") + kwargs["stdout"].write(result_line(**self.start_result) + "\n") + elif args[2] == "/compact": + self.transcript.write_text(self.transcript.read_text() + "compacted\n") + kwargs["stdout"].write(result_line(total_cost_usd=0.25, duration_ms=3000) + "\n") + else: + if "--resume" in args: + self.transcript.write_text(self.transcript.read_text() + "strong\n") + (cwd / "bin/pkg" / f"done_{len(self.calls)}.py").write_text("d = 1\n") + kwargs["stdout"].write(result_line() + "\n") + return subprocess.CompletedProcess(args, 0) + if args[0] == str(advisor_bench.RUN_HEAVY): + return subprocess.CompletedProcess(args, 0, stdout="3 passed\n", stderr="") + return real_run(args, **kwargs) + + def started(self): + handoff_bench.start(self.work, "7-1") + self.calls.clear() + + def test_start_runs_the_weak_worker_to_its_budget_and_saves_the_tree_and_the_session(self): + self.assertEqual(handoff_bench.start(self.work, "7-1"), "7-1: started") + (claude, kwargs, _), = self.calls + self.assertEqual(kwargs["cwd"], self.work / "7-1") + self.assertEqual(kwargs["stdin"], subprocess.DEVNULL) + self.assertEqual(claude[:2] + claude[3:], [ + "claude", "-p", *handoff_bench.WEAK.flags(), "--session-id", str(SESSION), "--max-budget-usd", "0.25", "--setting-sources", "project", + "--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"]) + self.assertIn("Do the seventh thing.", claude[2]) + self.assertEqual((self.work / "7-start-1.session").read_text(), str(self.transcript)) + self.assertEqual((self.work / "7-start-1.session.jsonl").read_text(), "weak\n") + self.assertIn("-x = 1\n+x = 3\n", (self.work / "7-start-1.diff").read_text()) + tree = self.work / "7-start-1.tree" + self.assertEqual(((tree / "bin/pkg/mod.py").read_text(), (tree / "bin/pkg/new.py").read_text()), ("x = 3\n", "n = 1\n")) + self.assertEqual(advisor_bench.git(tree, "status", "--porcelain"), " M bin/pkg/mod.py\n?? bin/pkg/new.py\n") + self.assertEqual(advisor_bench.git(tree, "rev-parse", "HEAD").strip(), self.base) + + def test_start_refuses_a_worker_that_finished_within_its_budget(self): + self.start_result = {} + with self.assertRaisesRegex(ValueError, "finished within its budget"): + handoff_bench.start(self.work, "7-1") + + def test_start_refuses_a_name_that_already_started(self): + (self.work / "7-1").mkdir() + with self.assertRaisesRegex(ValueError, "already started"): + handoff_bench.start(self.work, "7-1") + + def test_start_refuses_a_session_whose_transcript_it_cannot_find(self): + self.transcript = self.work / "elsewhere" / f"{SESSION}.jsonl" + with self.assertRaisesRegex(ValueError, "found 0"): + handoff_bench.start(self.work, "7-1") + + def test_the_resume_arm_resumes_the_weak_workers_session_on_the_strong_rung(self): + self.started() + self.assertEqual(handoff_bench.run(self.work, "7-resume-1"), "7-resume-1: ran") + (claude, kwargs, _), = self.calls + self.assertEqual(kwargs["cwd"], self.work / "7-1") + self.assertEqual(claude[:2] + claude[3:], [ + "claude", "-p", *handoff_bench.STRONG.flags(), "--resume", str(SESSION), "--setting-sources", "project", + "--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"]) + self.assertEqual(claude[2], (handoff_bench.BENCH / "handoff-resume-prompt.md").read_text().format(rung="claude-opus-5-5 at medium effort")) + diff = (self.work / "7-resume-1.diff").read_text() + self.assertIn("+x = 3\n", diff) + self.assertIn("+++ b/bin/pkg/done_1.py", diff) + self.assertEqual((self.work / "7-resume-1.tests").read_text(), "exit 0\n3 passed\n") + + def test_the_compact_arm_has_the_weak_model_compact_the_session_before_the_strong_one_resumes_it(self): + self.started() + handoff_bench.run(self.work, "7-compact-1") + (compact, _, _), (claude, _, found) = self.calls + self.assertEqual(compact[:2] + compact[3:], [ + "claude", "-p", *handoff_bench.WEAK.flags(), "--resume", str(SESSION), "--setting-sources", "project", + "--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"]) + self.assertEqual(compact[2], "/compact") + self.assertEqual(claude[3:7], [*handoff_bench.STRONG.flags()][:4]) + self.assertIn("--resume", claude) + self.assertEqual(found[0], "weak\ncompacted\n") + self.assertEqual(handoff_bench.outcome(self.work, "7-compact-1"), {"cost": 0.75, "seconds": 15.0}) + + def test_the_brief_arm_starts_a_fresh_session_with_the_item_and_the_brief(self): + self.started() + self.assertEqual(handoff_bench.run(self.work, "7-brief-1"), "7-brief-1: ran") + (claude, kwargs, _), = self.calls + self.assertEqual(kwargs["cwd"], self.work / "7-1") + self.assertEqual(claude[:2] + claude[3:], [ + "claude", "-p", *handoff_bench.STRONG.flags(), "--no-session-persistence", "--setting-sources", "project", + "--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"]) + self.assertIn("Do the seventh thing.", claude[2]) + self.assertIn("## Handoff", claude[2]) + self.assertIn("+++ b/bin/pkg/done_1.py", (self.work / "7-brief-1.diff").read_text()) + + def test_each_arm_begins_on_the_tree_and_the_session_that_start_left(self): + self.started() + for arm in ("resume", "compact", "brief"): + handoff_bench.run(self.work, f"7-{arm}-1") + self.assertEqual([found for _, _, found in self.calls if found[0] != "weak\ncompacted\n"], [("weak\n", ["mod.py", "new.py"])] * 3) + self.assertNotIn("done_1.py", (self.work / "7-brief-1.diff").read_text()) + + def test_run_refuses_an_arm_that_already_ran_or_never_started(self): + with self.assertRaisesRegex(ValueError, "start 7-1 first"): + handoff_bench.run(self.work, "7-brief-1") + self.started() + handoff_bench.run(self.work, "7-brief-1") + with self.assertRaisesRegex(ValueError, "already ran"): + handoff_bench.run(self.work, "7-brief-1") + + def test_run_refuses_an_unknown_arm(self): + self.started() + with self.assertRaisesRegex(ValueError, "arm"): + handoff_bench.run(self.work, "7-sonnet-1") + + def test_run_refuses_an_arm_whose_run_ended_in_an_error_before_it_tests(self): + self.started() + with unittest.mock.patch.object(advisor_bench, "save_change") as save_change, unittest.mock.patch.object(advisor_bench, "read_result", side_effect=ValueError("the run ended in an error")): + with self.assertRaisesRegex(ValueError, "ended in an error"): + handoff_bench.run(self.work, "7-brief-1") + save_change.assert_not_called() + + def test_main_grades_with_the_handoff_cases_and_prints_the_summary(self): + self.started() + handoff_bench.run(self.work, "7-brief-1") + with unittest.mock.patch.object(advisor_bench, "grade", return_value="7-brief-1: pass") as grade, unittest.mock.patch("builtins.print") as printed: + handoff_bench.main([str(self.work), "grade", "7-brief-1"]) + grade.assert_called_once_with(self.work, "7-brief-1", handoff_bench.CASES) + printed.assert_called_once_with("7-brief-1: pass", flush=True) + (self.work / "7-brief-1.verdict").write_text(json.dumps({"verdict": "pass"})) + with unittest.mock.patch("builtins.print") as printed: + handoff_bench.main([str(self.work), "summary"]) + self.assertIn("| brief | 1 | 1 | 0 | 0.50 | 12 |", printed.call_args.args[0]) + + +if __name__ == "__main__": + unittest.main()