Test the advisor bench's run, grade, main and rows_of against a throwaway repo, and kill the mutants of its pure functions

This commit is contained in:
2026-11-12 05:18:44 +01:00
parent 66b9651d68
commit 45fd66d3ab
+130 -4
View File
@@ -1,5 +1,11 @@
import contextlib
import io
import json
import pathlib
import subprocess
import tempfile
import unittest
import unittest.mock
import advisor_bench
@@ -60,7 +66,7 @@ class ItemText(unittest.TestCase):
class ReadResult(unittest.TestCase):
def test_reads_cost_turns_seconds_and_counts_advisor_calls(self):
iterations = [{"type": "message"}, {"type": "advisor_message"}, {"type": "message"}, {"type": "advisor_message"}]
iterations = [{"type": "message"}, {"type": "advisor_message"}, {"type": "message"}, {"type": "message"}, {"type": "advisor_message"}]
outcome = advisor_bench.read_result(["not json", result_line(usage={"iterations": iterations})])
self.assertEqual(outcome, advisor_bench.Outcome(cost=0.5, turns=9, seconds=12.0, advisor_calls=2))
@@ -71,8 +77,9 @@ class ReadResult(unittest.TestCase):
self.assertEqual(advisor_bench.read_result([result_line(usage={})]).advisor_calls, 0)
def test_refuses_a_stream_with_no_result_line(self):
with self.assertRaisesRegex(ValueError, "no result"):
with self.assertRaises(ValueError) as refused:
advisor_bench.read_result(['{"type": "assistant"}'])
self.assertEqual(str(refused.exception), "the stream has no result line")
class Verdict(unittest.TestCase):
@@ -84,8 +91,9 @@ class Verdict(unittest.TestCase):
self.assertEqual(advisor_bench.verdict_of(json.dumps({"result": reply})), "fail")
def test_refuses_a_reply_with_no_json_object(self):
with self.assertRaisesRegex(ValueError, "no JSON"):
with self.assertRaises(ValueError) as refused:
advisor_bench.verdict_of(json.dumps({"result": "I cannot rule."}))
self.assertEqual(str(refused.exception), "the judge's reply has no JSON object")
def test_refuses_a_verdict_outside_pass_partial_fail(self):
with self.assertRaisesRegex(ValueError, "verdict"):
@@ -99,11 +107,129 @@ class Summary(unittest.TestCase):
{"arm": "sonnet", "verdict": "fail", "cost": 3.0, "advisor_calls": 0, "seconds": 300.0},
{"arm": "opus-medium", "verdict": "pass", "cost": 4.0, "advisor_calls": 0, "seconds": 200.0},
]
self.assertEqual(advisor_bench.summary(rows).splitlines()[2:], [
rows.append({"arm": "sonnet-advisor", "verdict": "partial", "cost": 1.0, "advisor_calls": 2, "seconds": 50.0})
self.assertEqual(advisor_bench.summary(rows).splitlines(), [
"| arm | runs | pass | partial | mean cost $ | mean advisor calls | mean seconds |",
"|---|---|---|---|---|---|---|",
"| sonnet-advisor | 1 | 0 | 1 | 1.00 | 2.0 | 50 |",
"| sonnet | 2 | 1 | 0 | 2.00 | 0.0 | 200 |",
"| opus-medium | 1 | 1 | 0 | 4.00 | 0.0 | 200 |",
])
def commit(repo, message):
subprocess.run(["git", "-C", str(repo), "-c", "user.name=bench", "-c", "user.email=bench@example.com", "commit", "-q", "-a", "-m", message], check=True)
class Orchestration(unittest.TestCase):
"""run, grade, main and rows_of, on a throwaway repo whose case 7 changes bin/pkg/mod.py, with claude and run-heavy faked."""
def setUp(self):
scratch = tempfile.TemporaryDirectory()
self.addCleanup(scratch.cleanup)
root = pathlib.Path(scratch.name)
self.repo, self.work = root / "repo", root / "work"
self.work.mkdir()
(self.repo / "bin/pkg").mkdir(parents=True)
subprocess.run(["git", "init", "-q", str(self.repo)], check=True)
(self.repo / "todo.md").write_text(TODO)
(self.repo / "bin/pkg/mod.py").write_text("x = 1\n")
subprocess.run(["git", "-C", str(self.repo), "add", "."], check=True)
commit(self.repo, "start")
base = advisor_bench.git(self.repo, "rev-parse", "HEAD").strip()
(self.repo / "bin/pkg/mod.py").write_text("x = 2\n")
commit(self.repo, "merged")
merged = advisor_bench.git(self.repo, "rev-parse", "HEAD").strip()
self.calls = []
real_run = subprocess.run
for patch in (
unittest.mock.patch.object(advisor_bench, "REPO", self.repo), unittest.mock.patch.object(advisor_bench, "CASES", {"7": (base, merged)}),
unittest.mock.patch.object(advisor_bench.subprocess, "run", side_effect=lambda args, **kwargs: self.fake(real_run, args, **kwargs)),
):
patch.start()
self.addCleanup(patch.stop)
self.tests_output = "3 passed\n"
def fake(self, real_run, args, **kwargs):
if args[0] == "claude":
self.calls.append((args, kwargs.get("cwd")))
if "--tools" in args:
return subprocess.CompletedProcess(args, 0, stdout=json.dumps({"result": '{"verdict": "pass", "why": "ok"}'}), stderr="")
pathlib.Path(kwargs["cwd"], "bin/pkg/mod.py").write_text("x = 3\n")
kwargs["stdout"].write(result_line() + "\n")
return subprocess.CompletedProcess(args, 0)
if args[0] == str(advisor_bench.RUN_HEAVY):
self.calls.append((args, kwargs.get("cwd")))
return subprocess.CompletedProcess(args, 4, stdout=self.tests_output, stderr="")
return real_run(args, **kwargs)
def test_run_clones_at_the_start_commit_and_saves_the_stream_the_diff_and_the_tests(self):
self.assertEqual(advisor_bench.run(self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: ran")
(claude, claude_cwd), (tests, tests_cwd) = self.calls
clone = self.work / "7-sonnet-advisor-1"
self.assertEqual((claude_cwd, tests_cwd), (clone, clone))
self.assertEqual(claude[claude.index("--model"):claude.index("--model") + 6], advisor_bench.ARMS["sonnet-advisor"].flags())
self.assertIn("Do the seventh thing.", claude[2])
self.assertEqual(tests, [str(advisor_bench.RUN_HEAVY), *advisor_bench.SUITE, "bin/pkg"])
self.assertEqual(advisor_bench.git(clone, "remote"), "")
self.assertEqual(json.loads((self.work / "7-sonnet-advisor-1.jsonl").read_text())["num_turns"], 9)
self.assertEqual((self.work / "7-sonnet-advisor-1.diff").read_text().splitlines()[-2:], ["-x = 1", "+x = 3"])
self.assertEqual((self.work / "7-sonnet-advisor-1.tests").read_text(), "exit 4\n3 passed\n")
def test_run_keeps_the_last_1500_characters_of_the_tests(self):
self.tests_output = "a" * 2000 + "b" * 1500
advisor_bench.run(self.work, "7-sonnet-1")
self.assertEqual((self.work / "7-sonnet-1.tests").read_text(), "exit 4\n" + "b" * 1500)
def test_run_refuses_a_name_that_already_ran(self):
advisor_bench.run(self.work, "7-sonnet-1")
with self.assertRaisesRegex(ValueError, "7-sonnet-1 already ran: remove .*7-sonnet-1"):
advisor_bench.run(self.work, "7-sonnet-1")
def test_grade_saves_the_verdict_and_shows_the_judge_both_changes_but_not_the_arm(self):
advisor_bench.run(self.work, "7-sonnet-advisor-1")
self.calls.clear()
self.assertEqual(advisor_bench.grade(self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: pass")
((judge, _),) = self.calls
self.assertEqual(judge[judge.index("--model"):judge.index("--model") + 4], advisor_bench.JUDGE.flags())
self.assertIn("+x = 2", judge[2])
self.assertIn("+x = 3", judge[2])
self.assertIn("3 passed", judge[2])
self.assertNotIn("sonnet", judge[2])
self.assertEqual(json.loads((self.work / "7-sonnet-advisor-1.verdict").read_text())["verdict"], "pass")
def test_attempted_names_a_failure_in_place_of_raising_it(self):
def step(work, name):
raise KeyError("9")
self.assertEqual(advisor_bench.attempted(step, self.work, "9-sonnet-1"), "9-sonnet-1: FAILED: KeyError: '9'")
self.assertEqual(advisor_bench.attempted(lambda work, name: f"{name}: ok", self.work, "9-sonnet-1"), "9-sonnet-1: ok")
def test_rows_of_pairs_each_verdict_with_its_runs_outcome(self):
(self.work / "7-sonnet-advisor-1.jsonl").write_text(result_line(total_cost_usd=2.0) + "\n")
(self.work / "7-sonnet-advisor-1.verdict").write_text(json.dumps({"verdict": "fail"}))
(self.work / "7-sonnet-2.jsonl").write_text(result_line() + "\n")
self.assertEqual(advisor_bench.rows_of(self.work), [{"arm": "sonnet-advisor", "verdict": "fail", "cost": 2.0, "turns": 9, "seconds": 12.0, "advisor_calls": 0}])
def run_main(self, *argv):
out = io.StringIO()
with contextlib.redirect_stdout(out):
advisor_bench.main([str(self.work), *argv])
return out.getvalue()
def test_main_prints_each_runs_line_though_an_earlier_run_failed(self):
def step(work, name):
if name == "7-a-1":
raise ValueError("no")
return f"{name}: ran"
with unittest.mock.patch.object(advisor_bench, "run", step):
self.assertEqual(self.run_main("run", "7-a-1", "7-b-1"), "7-a-1: FAILED: ValueError: no\n7-b-1: ran\n")
def test_main_grades_on_grade_and_prints_the_summary_on_summary(self):
with unittest.mock.patch.object(advisor_bench, "grade", lambda work, name: f"{name}: graded"):
self.assertEqual(self.run_main("grade", "7-a-1"), "7-a-1: graded\n")
with unittest.mock.patch.object(advisor_bench, "summary", return_value="a table"):
self.assertEqual(self.run_main("summary"), "a table\n")
if __name__ == "__main__":
unittest.main()