Pin the advisor bench's argv, kwargs, diff exclusions and failures in its tests

This commit is contained in:
2026-11-12 05:20:39 +01:00
parent 45fd66d3ab
commit e1e99645be
+76 -30
View File
@@ -76,6 +76,10 @@ class ReadResult(unittest.TestCase):
def test_counts_no_advisor_call_where_the_run_has_no_iterations(self):
self.assertEqual(advisor_bench.read_result([result_line(usage={})]).advisor_calls, 0)
def test_counts_no_advisor_call_where_the_run_has_no_usage(self):
line = json.dumps({"type": "result", "total_cost_usd": 0.5, "num_turns": 9, "duration_ms": 12000})
self.assertEqual(advisor_bench.read_result([line]).advisor_calls, 0)
def test_refuses_a_stream_with_no_result_line(self):
with self.assertRaises(ValueError) as refused:
advisor_bench.read_result(['{"type": "assistant"}'])
@@ -90,6 +94,9 @@ class Verdict(unittest.TestCase):
for reply in ('```json\n{"verdict": "fail", "why": "x"}\n```', 'Ruling: {"verdict": "fail", "why": "x"} Done.'):
self.assertEqual(advisor_bench.verdict_of(json.dumps({"result": reply})), "fail")
def test_reads_a_verdict_from_a_reply_that_spans_lines(self):
self.assertEqual(advisor_bench.verdict_of(json.dumps({"result": '{\n"verdict": "pass",\n"why": "x"\n}'})), "pass")
def test_refuses_a_reply_with_no_json_object(self):
with self.assertRaises(ValueError) as refused:
advisor_bench.verdict_of(json.dumps({"result": "I cannot rule."}))
@@ -122,7 +129,10 @@ def commit(repo, message):
class Orchestration(unittest.TestCase):
"""run, grade, main and rows_of, on a throwaway repo whose case 7 changes bin/pkg/mod.py, with claude and run-heavy faked."""
"""run, grade, main and rows_of, on a throwaway repo, with claude and run-heavy faked.
Case 7 starts at a commit holding bin/pkg/mod.py, and merges a change to bin/pkg/mod.py, a new bin/other/mod.py, and
todo.md, docs and CHANGELOG.md; a later commit then changes bin/pkg/mod.py again and adds bin/later/mod.py."""
def setUp(self):
scratch = tempfile.TemporaryDirectory()
@@ -130,16 +140,10 @@ class Orchestration(unittest.TestCase):
root = pathlib.Path(scratch.name)
self.repo, self.work = root / "repo", root / "work"
self.work.mkdir()
(self.repo / "bin/pkg").mkdir(parents=True)
subprocess.run(["git", "init", "-q", str(self.repo)], check=True)
(self.repo / "todo.md").write_text(TODO)
(self.repo / "bin/pkg/mod.py").write_text("x = 1\n")
subprocess.run(["git", "-C", str(self.repo), "add", "."], check=True)
commit(self.repo, "start")
base = advisor_bench.git(self.repo, "rev-parse", "HEAD").strip()
(self.repo / "bin/pkg/mod.py").write_text("x = 2\n")
commit(self.repo, "merged")
merged = advisor_bench.git(self.repo, "rev-parse", "HEAD").strip()
base = self.commit({"todo.md": TODO, "bin/pkg/mod.py": "x = 1\n", "docs/notes.md": "a\n", "CHANGELOG.md": "a\n"})
merged = self.commit({"todo.md": TODO + "\nmore\n", "bin/pkg/mod.py": "x = 2\n", "bin/other/mod.py": "z = 1\n", "docs/notes.md": "b\n", "CHANGELOG.md": "b\n"})
self.commit({"bin/pkg/mod.py": "x = 5\n", "bin/later/mod.py": "y = 1\n"})
self.calls = []
real_run = subprocess.run
for patch in (
@@ -149,31 +153,51 @@ class Orchestration(unittest.TestCase):
patch.start()
self.addCleanup(patch.stop)
self.tests_output = "3 passed\n"
self.base = base
def commit(self, files):
for path, text in files.items():
(self.repo / path).parent.mkdir(parents=True, exist_ok=True)
(self.repo / path).write_text(text)
subprocess.run(["git", "-C", str(self.repo), "add", "."], check=True)
commit(self.repo, "a commit")
return advisor_bench.git(self.repo, "rev-parse", "HEAD").strip()
def fake(self, real_run, args, **kwargs):
if args[0] == "claude":
self.calls.append((args, kwargs.get("cwd")))
self.calls.append((args, kwargs))
if "--tools" in args:
return subprocess.CompletedProcess(args, 0, stdout=json.dumps({"result": '{"verdict": "pass", "why": "ok"}'}), stderr="")
pathlib.Path(kwargs["cwd"], "bin/pkg/mod.py").write_text("x = 3\n")
pathlib.Path(kwargs["cwd"], "bin/pkg/new.py").write_text("n = 1\n")
kwargs["stdout"].write(result_line() + "\n")
return subprocess.CompletedProcess(args, 0)
if args[0] == str(advisor_bench.RUN_HEAVY):
self.calls.append((args, kwargs.get("cwd")))
self.calls.append((args, kwargs))
return subprocess.CompletedProcess(args, 4, stdout=self.tests_output, stderr="")
return real_run(args, **kwargs)
def test_run_clones_at_the_start_commit_and_saves_the_stream_the_diff_and_the_tests(self):
self.assertEqual(advisor_bench.run(self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: ran")
(claude, claude_cwd), (tests, tests_cwd) = self.calls
(claude, claude_kwargs), (tests, tests_kwargs) = self.calls
clone = self.work / "7-sonnet-advisor-1"
self.assertEqual((claude_cwd, tests_cwd), (clone, clone))
self.assertEqual(claude[claude.index("--model"):claude.index("--model") + 6], advisor_bench.ARMS["sonnet-advisor"].flags())
self.assertEqual((claude_kwargs["cwd"], tests_kwargs["cwd"]), (clone, clone))
self.assertEqual(claude[:2] + claude[3:], [
"claude", "-p", *advisor_bench.ARMS["sonnet-advisor"].flags(), "--setting-sources", "project", "--permission-mode", "bypassPermissions",
"--no-session-persistence", "--output-format", "stream-json", "--verbose"])
self.assertIn("Do the seventh thing.", claude[2])
self.assertEqual(tests, [str(advisor_bench.RUN_HEAVY), *advisor_bench.SUITE, "bin/pkg"])
self.assertIn(f"`{self.base}`", claude[2])
self.assertIn("todo item 7 ", claude[2])
self.assertEqual(tests, [str(advisor_bench.RUN_HEAVY), *advisor_bench.SUITE, "bin/other", "bin/pkg"])
self.assertEqual((tests_kwargs["capture_output"], tests_kwargs["text"]), (True, True))
self.assertEqual(claude_kwargs["stdout"].name, str(self.work / "7-sonnet-advisor-1.jsonl"))
self.assertEqual(claude_kwargs["stderr"], subprocess.STDOUT)
self.assertEqual(advisor_bench.git(clone, "remote"), "")
self.assertEqual(advisor_bench.git(clone, "rev-parse", "HEAD").strip(), self.base)
self.assertEqual(json.loads((self.work / "7-sonnet-advisor-1.jsonl").read_text())["num_turns"], 9)
self.assertEqual((self.work / "7-sonnet-advisor-1.diff").read_text().splitlines()[-2:], ["-x = 1", "+x = 3"])
diff = (self.work / "7-sonnet-advisor-1.diff").read_text()
self.assertIn("-x = 1\n+x = 3\n", diff)
self.assertIn("+++ b/bin/pkg/new.py", diff)
self.assertEqual((self.work / "7-sonnet-advisor-1.tests").read_text(), "exit 4\n3 passed\n")
def test_run_keeps_the_last_1500_characters_of_the_tests(self):
@@ -186,23 +210,42 @@ class Orchestration(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "7-sonnet-1 already ran: remove .*7-sonnet-1"):
advisor_bench.run(self.work, "7-sonnet-1")
def test_a_clone_of_a_missing_repo_fails(self):
with unittest.mock.patch.object(advisor_bench, "REPO", self.work / "missing"), self.assertRaises(subprocess.CalledProcessError):
advisor_bench.fresh_clone(self.base, self.work / "clone")
def test_git_fails_where_git_does(self):
with self.assertRaises(subprocess.CalledProcessError):
advisor_bench.git(self.repo, "rev-parse", "no-such-revision")
def test_grade_saves_the_verdict_and_shows_the_judge_both_changes_but_not_the_arm(self):
advisor_bench.run(self.work, "7-sonnet-advisor-1")
self.calls.clear()
self.assertEqual(advisor_bench.grade(self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: pass")
((judge, _),) = self.calls
self.assertEqual(judge[judge.index("--model"):judge.index("--model") + 4], advisor_bench.JUDGE.flags())
self.assertIn("+x = 2", judge[2])
self.assertIn("+x = 3", judge[2])
self.assertIn("3 passed", judge[2])
self.assertNotIn("sonnet", judge[2])
self.assertEqual(json.loads((self.work / "7-sonnet-advisor-1.verdict").read_text())["verdict"], "pass")
((judge, kwargs),) = self.calls
self.assertEqual(judge[:2] + judge[3:], [
"claude", "-p", *advisor_bench.JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", ""])
self.assertEqual((kwargs["capture_output"], kwargs["text"], kwargs["check"]), (True, True, True))
prompt = judge[2]
self.assertIn("Do the seventh thing.", prompt)
self.assertIn("-x = 1\n+x = 2\n", prompt)
self.assertIn("+++ b/bin/other/mod.py", prompt)
for left_out in ("x = 5", "later", "docs/notes.md", "CHANGELOG.md", "todo.md"):
self.assertNotIn(left_out, prompt)
self.assertIn("+x = 3", prompt)
self.assertIn("3 passed", prompt)
self.assertNotIn("sonnet", prompt)
self.assertEqual(
json.loads((self.work / "7-sonnet-advisor-1.verdict").read_text()),
{"verdict": "pass", "reply": '{"verdict": "pass", "why": "ok"}'})
def test_attempted_names_a_failure_in_place_of_raising_it(self):
def step(work, name):
raise KeyError("9")
self.assertEqual(advisor_bench.attempted(step, self.work, "9-sonnet-1"), "9-sonnet-1: FAILED: KeyError: '9'")
self.assertEqual(advisor_bench.attempted(lambda work, name: f"{name}: ok", self.work, "9-sonnet-1"), "9-sonnet-1: ok")
def test_attempted_hands_the_step_the_work_dir_and_the_name(self):
self.assertEqual(advisor_bench.attempted(lambda work, name: f"{work.name} {name}", self.work, "9-sonnet-1"), "work 9-sonnet-1")
def test_rows_of_pairs_each_verdict_with_its_runs_outcome(self):
(self.work / "7-sonnet-advisor-1.jsonl").write_text(result_line(total_cost_usd=2.0) + "\n")
@@ -224,11 +267,14 @@ class Orchestration(unittest.TestCase):
with unittest.mock.patch.object(advisor_bench, "run", step):
self.assertEqual(self.run_main("run", "7-a-1", "7-b-1"), "7-a-1: FAILED: ValueError: no\n7-b-1: ran\n")
def test_main_grades_on_grade_and_prints_the_summary_on_summary(self):
with unittest.mock.patch.object(advisor_bench, "grade", lambda work, name: f"{name}: graded"):
self.assertEqual(self.run_main("grade", "7-a-1"), "7-a-1: graded\n")
with unittest.mock.patch.object(advisor_bench, "summary", return_value="a table"):
self.assertEqual(self.run_main("summary"), "a table\n")
def test_main_grades_on_grade(self):
with unittest.mock.patch.object(advisor_bench, "grade", lambda work, name: f"{work.name} {name}: graded"):
self.assertEqual(self.run_main("grade", "7-a-1"), "work 7-a-1: graded\n")
def test_main_prints_the_work_dirs_summary_on_summary(self):
(self.work / "7-sonnet-1.jsonl").write_text(result_line() + "\n")
(self.work / "7-sonnet-1.verdict").write_text(json.dumps({"verdict": "pass"}))
self.assertIn("| sonnet | 1 | 1 | 0 | 0.50 | 0.0 | 12 |", self.run_main("summary"))
if __name__ == "__main__":