Hold the handoff bench's prompts, saved change and steps to their tests
This commit is contained in:
@@ -15,8 +15,8 @@ best? Each case is an archived chunk of this repo, as in advisor_bench.py, whose
|
||||
the compaction included, and leaves out the start, which the arms share (`outcome`).
|
||||
Run each arm of each case once; the table shows a direction, not a significance.
|
||||
"""
|
||||
import concurrent.futures
|
||||
import collections
|
||||
import concurrent.futures
|
||||
import json
|
||||
import pathlib
|
||||
import shutil
|
||||
|
||||
@@ -7,8 +7,8 @@ import unittest.mock
|
||||
import uuid
|
||||
|
||||
import advisor_bench
|
||||
import handoff_bench
|
||||
from test_advisor_bench import commit, TODO
|
||||
import handoff_bench
|
||||
|
||||
SESSION = uuid.UUID("11111111-2222-4333-8444-555555555555")
|
||||
|
||||
@@ -55,9 +55,11 @@ class Orchestration(unittest.TestCase):
|
||||
self.repo, self.work, self.projects = root / "repo", root / "work", root / "projects"
|
||||
self.work.mkdir()
|
||||
subprocess.run(["git", "init", "-q", str(self.repo)], check=True)
|
||||
self.base = self.commit({"todo.md": TODO, "bin/pkg/mod.py": "x = 1\n"})
|
||||
(self.repo / "link").symlink_to("todo.md")
|
||||
self.base = self.commit({"todo.md": TODO, "bin/pkg/mod.py": "x = 1\n", "bin/other/o.py": "o = 1\n"})
|
||||
merged = self.commit({"bin/pkg/mod.py": "x = 2\n"})
|
||||
self.calls = []
|
||||
self.commit({"bin/other/o.py": "o = 2\n"})
|
||||
self.calls, self.heavy = [], []
|
||||
self.start_result = {"subtype": "error_max_budget_usd", "is_error": True, "total_cost_usd": 1.0}
|
||||
self.strong_result = {}
|
||||
real_run = subprocess.run
|
||||
@@ -89,6 +91,9 @@ class Orchestration(unittest.TestCase):
|
||||
self.calls.append((args, kwargs, found))
|
||||
if "--max-budget-usd" in args:
|
||||
(cwd / "bin/pkg/mod.py").write_text("x = 3\n")
|
||||
# Against the bench rules, the worker commits a part of its change.
|
||||
real_run(["git", "-C", str(cwd), "add", "bin/pkg/mod.py"], check=True)
|
||||
commit(cwd, "against the rules")
|
||||
(cwd / "bin/pkg/new.py").write_text("n = 1\n")
|
||||
self.transcript.parent.mkdir(parents=True)
|
||||
self.transcript.write_text("weak\n")
|
||||
@@ -102,9 +107,11 @@ class Orchestration(unittest.TestCase):
|
||||
# A resumed session counts its cost from the session's start, and may open with a result that holds no work.
|
||||
kwargs["stdout"].write(result_line(total_cost_usd=1.0, duration_ms=0) + "\n")
|
||||
(cwd / "bin/pkg" / f"done_{len(self.calls)}.py").write_text("d = 1\n")
|
||||
kwargs["stdout"].write("a line that is not JSON\n")
|
||||
kwargs["stdout"].write(result_line(**self.strong_result, total_cost_usd=3.0 if "--resume" in args else 0.5) + "\n")
|
||||
return subprocess.CompletedProcess(args, 0)
|
||||
if args[0] == str(advisor_bench.RUN_HEAVY):
|
||||
self.heavy.append(args)
|
||||
return subprocess.CompletedProcess(args, 0, stdout="3 passed\n", stderr="")
|
||||
return real_run(args, **kwargs)
|
||||
|
||||
@@ -116,11 +123,12 @@ class Orchestration(unittest.TestCase):
|
||||
self.assertEqual(handoff_bench.start(self.work, "7-1"), "7-1: started")
|
||||
(claude, kwargs, _), = self.calls
|
||||
self.assertEqual(kwargs["cwd"], self.work / "7-1")
|
||||
self.assertEqual(kwargs["stdin"], subprocess.DEVNULL)
|
||||
self.assertEqual((kwargs["stdin"], kwargs["stderr"]), (subprocess.DEVNULL, subprocess.STDOUT))
|
||||
self.assertEqual(claude[:2] + claude[3:], [
|
||||
"claude", "-p", *handoff_bench.WEAK.flags(), "--session-id", str(SESSION), "--max-budget-usd", "0.25", "--setting-sources", "project",
|
||||
"--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"])
|
||||
self.assertIn("Do the seventh thing.", claude[2])
|
||||
self.assertIn(f"a throwaway clone at `{self.base}`. Implement todo item 7 below", claude[2])
|
||||
self.assertEqual((self.work / "7-start-1.session").read_text(), str(self.transcript))
|
||||
self.assertEqual((self.work / "7-start-1.session.jsonl").read_text(), "weak\n")
|
||||
self.assertIn("-x = 1\n+x = 3\n", (self.work / "7-start-1.diff").read_text())
|
||||
@@ -128,10 +136,11 @@ class Orchestration(unittest.TestCase):
|
||||
self.assertEqual(((tree / "bin/pkg/mod.py").read_text(), (tree / "bin/pkg/new.py").read_text()), ("x = 3\n", "n = 1\n"))
|
||||
self.assertEqual(advisor_bench.git(tree, "status", "--porcelain"), " M bin/pkg/mod.py\n?? bin/pkg/new.py\n")
|
||||
self.assertEqual(advisor_bench.git(tree, "rev-parse", "HEAD").strip(), self.base)
|
||||
self.assertTrue((tree / "link").is_symlink())
|
||||
|
||||
def test_start_refuses_a_worker_that_finished_within_its_budget(self):
|
||||
self.start_result = {}
|
||||
with self.assertRaisesRegex(ValueError, "finished within its budget"):
|
||||
with self.assertRaisesRegex(ValueError, "^the weak worker finished within its budget, or its run failed, so there is no handoff to bench$"):
|
||||
handoff_bench.start(self.work, "7-1")
|
||||
|
||||
def test_start_refuses_a_name_that_already_started(self):
|
||||
@@ -157,6 +166,9 @@ class Orchestration(unittest.TestCase):
|
||||
self.assertIn("+x = 3\n", diff)
|
||||
self.assertIn("+++ b/bin/pkg/done_1.py", diff)
|
||||
self.assertEqual((self.work / "7-resume-1.tests").read_text(), "exit 0\n3 passed\n")
|
||||
self.assertEqual(self.heavy[-1][-1], "bin/pkg")
|
||||
self.assertNotIn("bin/other", self.heavy[-1])
|
||||
self.assertTrue((self.work / "7-1" / "link").is_symlink())
|
||||
self.assertEqual(handoff_bench.outcome(self.work, "7-resume-1"), {"cost": 2.0, "seconds": 12.0})
|
||||
|
||||
def test_the_compact_arm_has_the_weak_model_compact_the_session_before_the_strong_one_resumes_it(self):
|
||||
@@ -180,7 +192,7 @@ class Orchestration(unittest.TestCase):
|
||||
self.assertEqual(claude[:2] + claude[3:], [
|
||||
"claude", "-p", *handoff_bench.STRONG.flags(), "--no-session-persistence", "--setting-sources", "project",
|
||||
"--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"])
|
||||
self.assertIn("Do the seventh thing.", claude[2])
|
||||
self.assertEqual(claude[2], handoff_bench.run_prompt("7") + "\n" + (handoff_bench.BENCH / "handoff-brief-prompt.md").read_text())
|
||||
self.assertIn("## Handoff", claude[2])
|
||||
self.assertIn("+++ b/bin/pkg/done_1.py", (self.work / "7-brief-1.diff").read_text())
|
||||
|
||||
@@ -215,12 +227,14 @@ class Orchestration(unittest.TestCase):
|
||||
def test_outcome_refuses_a_stream_with_no_result_line(self):
|
||||
self.started()
|
||||
(self.work / "7-brief-1.jsonl").write_text("not json\n[]\n")
|
||||
with self.assertRaisesRegex(ValueError, "no result line"):
|
||||
with self.assertRaisesRegex(ValueError, "^the stream has no result line$"):
|
||||
handoff_bench.outcome(self.work, "7-brief-1")
|
||||
|
||||
def test_main_grades_with_the_handoff_cases_and_prints_the_summary(self):
|
||||
self.started()
|
||||
handoff_bench.run(self.work, "7-brief-1")
|
||||
with unittest.mock.patch("builtins.print") as printed:
|
||||
handoff_bench.main([str(self.work), "start", "7-1"])
|
||||
handoff_bench.main([str(self.work), "run", "7-brief-1"])
|
||||
self.assertEqual([call.args[0] for call in printed.call_args_list], ["7-1: started", "7-brief-1: ran"])
|
||||
with unittest.mock.patch.object(advisor_bench, "grade", return_value="7-brief-1: pass") as grade, unittest.mock.patch("builtins.print") as printed:
|
||||
handoff_bench.main([str(self.work), "grade", "7-brief-1"])
|
||||
grade.assert_called_once_with(self.work, "7-brief-1", handoff_bench.CASES)
|
||||
|
||||
Reference in New Issue
Block a user