Test the advisor bench's run, grade, main and rows_of against a throwaway repo, and kill the mutants of its pure functions
This commit is contained in:
@@ -1,5 +1,11 @@
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import pathlib
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
import unittest.mock
|
||||
|
||||
import advisor_bench
|
||||
|
||||
@@ -60,7 +66,7 @@ class ItemText(unittest.TestCase):
|
||||
|
||||
class ReadResult(unittest.TestCase):
|
||||
def test_reads_cost_turns_seconds_and_counts_advisor_calls(self):
|
||||
iterations = [{"type": "message"}, {"type": "advisor_message"}, {"type": "message"}, {"type": "advisor_message"}]
|
||||
iterations = [{"type": "message"}, {"type": "advisor_message"}, {"type": "message"}, {"type": "message"}, {"type": "advisor_message"}]
|
||||
outcome = advisor_bench.read_result(["not json", result_line(usage={"iterations": iterations})])
|
||||
self.assertEqual(outcome, advisor_bench.Outcome(cost=0.5, turns=9, seconds=12.0, advisor_calls=2))
|
||||
|
||||
@@ -71,8 +77,9 @@ class ReadResult(unittest.TestCase):
|
||||
self.assertEqual(advisor_bench.read_result([result_line(usage={})]).advisor_calls, 0)
|
||||
|
||||
def test_refuses_a_stream_with_no_result_line(self):
|
||||
with self.assertRaisesRegex(ValueError, "no result"):
|
||||
with self.assertRaises(ValueError) as refused:
|
||||
advisor_bench.read_result(['{"type": "assistant"}'])
|
||||
self.assertEqual(str(refused.exception), "the stream has no result line")
|
||||
|
||||
|
||||
class Verdict(unittest.TestCase):
|
||||
@@ -84,8 +91,9 @@ class Verdict(unittest.TestCase):
|
||||
self.assertEqual(advisor_bench.verdict_of(json.dumps({"result": reply})), "fail")
|
||||
|
||||
def test_refuses_a_reply_with_no_json_object(self):
|
||||
with self.assertRaisesRegex(ValueError, "no JSON"):
|
||||
with self.assertRaises(ValueError) as refused:
|
||||
advisor_bench.verdict_of(json.dumps({"result": "I cannot rule."}))
|
||||
self.assertEqual(str(refused.exception), "the judge's reply has no JSON object")
|
||||
|
||||
def test_refuses_a_verdict_outside_pass_partial_fail(self):
|
||||
with self.assertRaisesRegex(ValueError, "verdict"):
|
||||
@@ -99,11 +107,129 @@ class Summary(unittest.TestCase):
|
||||
{"arm": "sonnet", "verdict": "fail", "cost": 3.0, "advisor_calls": 0, "seconds": 300.0},
|
||||
{"arm": "opus-medium", "verdict": "pass", "cost": 4.0, "advisor_calls": 0, "seconds": 200.0},
|
||||
]
|
||||
self.assertEqual(advisor_bench.summary(rows).splitlines()[2:], [
|
||||
rows.append({"arm": "sonnet-advisor", "verdict": "partial", "cost": 1.0, "advisor_calls": 2, "seconds": 50.0})
|
||||
self.assertEqual(advisor_bench.summary(rows).splitlines(), [
|
||||
"| arm | runs | pass | partial | mean cost $ | mean advisor calls | mean seconds |",
|
||||
"|---|---|---|---|---|---|---|",
|
||||
"| sonnet-advisor | 1 | 0 | 1 | 1.00 | 2.0 | 50 |",
|
||||
"| sonnet | 2 | 1 | 0 | 2.00 | 0.0 | 200 |",
|
||||
"| opus-medium | 1 | 1 | 0 | 4.00 | 0.0 | 200 |",
|
||||
])
|
||||
|
||||
|
||||
def commit(repo, message):
|
||||
subprocess.run(["git", "-C", str(repo), "-c", "user.name=bench", "-c", "user.email=bench@example.com", "commit", "-q", "-a", "-m", message], check=True)
|
||||
|
||||
|
||||
class Orchestration(unittest.TestCase):
|
||||
"""run, grade, main and rows_of, on a throwaway repo whose case 7 changes bin/pkg/mod.py, with claude and run-heavy faked."""
|
||||
|
||||
def setUp(self):
|
||||
scratch = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(scratch.cleanup)
|
||||
root = pathlib.Path(scratch.name)
|
||||
self.repo, self.work = root / "repo", root / "work"
|
||||
self.work.mkdir()
|
||||
(self.repo / "bin/pkg").mkdir(parents=True)
|
||||
subprocess.run(["git", "init", "-q", str(self.repo)], check=True)
|
||||
(self.repo / "todo.md").write_text(TODO)
|
||||
(self.repo / "bin/pkg/mod.py").write_text("x = 1\n")
|
||||
subprocess.run(["git", "-C", str(self.repo), "add", "."], check=True)
|
||||
commit(self.repo, "start")
|
||||
base = advisor_bench.git(self.repo, "rev-parse", "HEAD").strip()
|
||||
(self.repo / "bin/pkg/mod.py").write_text("x = 2\n")
|
||||
commit(self.repo, "merged")
|
||||
merged = advisor_bench.git(self.repo, "rev-parse", "HEAD").strip()
|
||||
self.calls = []
|
||||
real_run = subprocess.run
|
||||
for patch in (
|
||||
unittest.mock.patch.object(advisor_bench, "REPO", self.repo), unittest.mock.patch.object(advisor_bench, "CASES", {"7": (base, merged)}),
|
||||
unittest.mock.patch.object(advisor_bench.subprocess, "run", side_effect=lambda args, **kwargs: self.fake(real_run, args, **kwargs)),
|
||||
):
|
||||
patch.start()
|
||||
self.addCleanup(patch.stop)
|
||||
self.tests_output = "3 passed\n"
|
||||
|
||||
def fake(self, real_run, args, **kwargs):
|
||||
if args[0] == "claude":
|
||||
self.calls.append((args, kwargs.get("cwd")))
|
||||
if "--tools" in args:
|
||||
return subprocess.CompletedProcess(args, 0, stdout=json.dumps({"result": '{"verdict": "pass", "why": "ok"}'}), stderr="")
|
||||
pathlib.Path(kwargs["cwd"], "bin/pkg/mod.py").write_text("x = 3\n")
|
||||
kwargs["stdout"].write(result_line() + "\n")
|
||||
return subprocess.CompletedProcess(args, 0)
|
||||
if args[0] == str(advisor_bench.RUN_HEAVY):
|
||||
self.calls.append((args, kwargs.get("cwd")))
|
||||
return subprocess.CompletedProcess(args, 4, stdout=self.tests_output, stderr="")
|
||||
return real_run(args, **kwargs)
|
||||
|
||||
def test_run_clones_at_the_start_commit_and_saves_the_stream_the_diff_and_the_tests(self):
|
||||
self.assertEqual(advisor_bench.run(self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: ran")
|
||||
(claude, claude_cwd), (tests, tests_cwd) = self.calls
|
||||
clone = self.work / "7-sonnet-advisor-1"
|
||||
self.assertEqual((claude_cwd, tests_cwd), (clone, clone))
|
||||
self.assertEqual(claude[claude.index("--model"):claude.index("--model") + 6], advisor_bench.ARMS["sonnet-advisor"].flags())
|
||||
self.assertIn("Do the seventh thing.", claude[2])
|
||||
self.assertEqual(tests, [str(advisor_bench.RUN_HEAVY), *advisor_bench.SUITE, "bin/pkg"])
|
||||
self.assertEqual(advisor_bench.git(clone, "remote"), "")
|
||||
self.assertEqual(json.loads((self.work / "7-sonnet-advisor-1.jsonl").read_text())["num_turns"], 9)
|
||||
self.assertEqual((self.work / "7-sonnet-advisor-1.diff").read_text().splitlines()[-2:], ["-x = 1", "+x = 3"])
|
||||
self.assertEqual((self.work / "7-sonnet-advisor-1.tests").read_text(), "exit 4\n3 passed\n")
|
||||
|
||||
def test_run_keeps_the_last_1500_characters_of_the_tests(self):
|
||||
self.tests_output = "a" * 2000 + "b" * 1500
|
||||
advisor_bench.run(self.work, "7-sonnet-1")
|
||||
self.assertEqual((self.work / "7-sonnet-1.tests").read_text(), "exit 4\n" + "b" * 1500)
|
||||
|
||||
def test_run_refuses_a_name_that_already_ran(self):
|
||||
advisor_bench.run(self.work, "7-sonnet-1")
|
||||
with self.assertRaisesRegex(ValueError, "7-sonnet-1 already ran: remove .*7-sonnet-1"):
|
||||
advisor_bench.run(self.work, "7-sonnet-1")
|
||||
|
||||
def test_grade_saves_the_verdict_and_shows_the_judge_both_changes_but_not_the_arm(self):
|
||||
advisor_bench.run(self.work, "7-sonnet-advisor-1")
|
||||
self.calls.clear()
|
||||
self.assertEqual(advisor_bench.grade(self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: pass")
|
||||
((judge, _),) = self.calls
|
||||
self.assertEqual(judge[judge.index("--model"):judge.index("--model") + 4], advisor_bench.JUDGE.flags())
|
||||
self.assertIn("+x = 2", judge[2])
|
||||
self.assertIn("+x = 3", judge[2])
|
||||
self.assertIn("3 passed", judge[2])
|
||||
self.assertNotIn("sonnet", judge[2])
|
||||
self.assertEqual(json.loads((self.work / "7-sonnet-advisor-1.verdict").read_text())["verdict"], "pass")
|
||||
|
||||
def test_attempted_names_a_failure_in_place_of_raising_it(self):
|
||||
def step(work, name):
|
||||
raise KeyError("9")
|
||||
self.assertEqual(advisor_bench.attempted(step, self.work, "9-sonnet-1"), "9-sonnet-1: FAILED: KeyError: '9'")
|
||||
self.assertEqual(advisor_bench.attempted(lambda work, name: f"{name}: ok", self.work, "9-sonnet-1"), "9-sonnet-1: ok")
|
||||
|
||||
def test_rows_of_pairs_each_verdict_with_its_runs_outcome(self):
|
||||
(self.work / "7-sonnet-advisor-1.jsonl").write_text(result_line(total_cost_usd=2.0) + "\n")
|
||||
(self.work / "7-sonnet-advisor-1.verdict").write_text(json.dumps({"verdict": "fail"}))
|
||||
(self.work / "7-sonnet-2.jsonl").write_text(result_line() + "\n")
|
||||
self.assertEqual(advisor_bench.rows_of(self.work), [{"arm": "sonnet-advisor", "verdict": "fail", "cost": 2.0, "turns": 9, "seconds": 12.0, "advisor_calls": 0}])
|
||||
|
||||
def run_main(self, *argv):
|
||||
out = io.StringIO()
|
||||
with contextlib.redirect_stdout(out):
|
||||
advisor_bench.main([str(self.work), *argv])
|
||||
return out.getvalue()
|
||||
|
||||
def test_main_prints_each_runs_line_though_an_earlier_run_failed(self):
|
||||
def step(work, name):
|
||||
if name == "7-a-1":
|
||||
raise ValueError("no")
|
||||
return f"{name}: ran"
|
||||
with unittest.mock.patch.object(advisor_bench, "run", step):
|
||||
self.assertEqual(self.run_main("run", "7-a-1", "7-b-1"), "7-a-1: FAILED: ValueError: no\n7-b-1: ran\n")
|
||||
|
||||
def test_main_grades_on_grade_and_prints_the_summary_on_summary(self):
|
||||
with unittest.mock.patch.object(advisor_bench, "grade", lambda work, name: f"{name}: graded"):
|
||||
self.assertEqual(self.run_main("grade", "7-a-1"), "7-a-1: graded\n")
|
||||
with unittest.mock.patch.object(advisor_bench, "summary", return_value="a table"):
|
||||
self.assertEqual(self.run_main("summary"), "a table\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user