import json import pathlib import shutil import subprocess import tempfile import unittest import advisor_bench TODO = """| table | ## Details ### 7. Do the seventh thing. Its detail. ### 8. Do the eighth thing. Another detail. """ def result_line(**fields): return json.dumps({"type": "result", "total_cost_usd": 0.5, "num_turns": 9, "duration_ms": 12000, "usage": {"iterations": []}, **fields}) class Arms(unittest.TestCase): def test_the_advisor_arm_names_its_model_effort_and_advisor(self): self.assertEqual( advisor_bench.ARMS["sonnet-advisor"].flags(), ["--model", "claude-sonnet-5-5", "--effort", "high", "--advisor", "claude-opus-5-5"]) def test_an_arm_without_an_advisor_passes_no_advisor_flag(self): self.assertEqual(advisor_bench.ARMS["opus-medium"].flags(), ["--model", "claude-opus-5-5", "--effort", "medium"]) self.assertNotIn("--advisor", advisor_bench.ARMS["sonnet"].flags()) class SplitName(unittest.TestCase): def test_keeps_a_hyphen_in_the_arm(self): self.assertEqual(advisor_bench.split_name("290-sonnet-advisor-2"), ("290", "sonnet-advisor", "2")) class SuiteTargets(unittest.TestCase): def test_names_the_package_of_each_changed_file_under_bin_once(self): self.assertEqual( advisor_bench.suite_targets(["bin/rule_hook/rules.py", "bin/rule_hook/test_rule_hook.py", "bin/shell_command.py", "README.md"]), ["bin/rule_hook", "bin/shell_command.py"]) def test_falls_back_to_the_whole_of_bin_where_no_changed_file_is_in_it(self): self.assertEqual(advisor_bench.suite_targets(["README.md"]), ["bin"]) class ItemText(unittest.TestCase): def test_reads_the_heading_and_detail_up_to_the_next_item(self): self.assertEqual(advisor_bench.item_text(TODO, 7), "### 7. Do the seventh thing.\n\nIts detail.") def test_reads_the_last_item_to_the_end(self): self.assertEqual(advisor_bench.item_text(TODO, 8), "### 8. Do the eighth thing.\n\nAnother detail.") def test_refuses_an_item_with_no_detail(self): with self.assertRaisesRegex(ValueError, "item 9"): advisor_bench.item_text(TODO, 9) class ReadResult(unittest.TestCase): def test_reads_cost_seconds_and_counts_advisor_calls(self): iterations = [{"type": "message"}, {"type": "advisor_message"}, {"type": "message"}, {"type": "message"}, {"type": "advisor_message"}] outcome = advisor_bench.read_result(["not json", result_line(usage={"iterations": iterations})]) self.assertEqual(outcome, advisor_bench.Outcome(cost=0.5, seconds=12.0, advisor_calls=2)) def test_reads_the_first_result_where_a_background_task_makes_later_ones(self): self.assertEqual(advisor_bench.read_result([result_line(total_cost_usd=0.5), result_line(total_cost_usd=2.0)]).cost, 0.5) def test_counts_no_advisor_call_where_the_run_has_no_iterations(self): self.assertEqual(advisor_bench.read_result([result_line(usage={})]).advisor_calls, 0) def test_counts_no_advisor_call_where_the_run_has_no_usage(self): line = json.dumps({"type": "result", "total_cost_usd": 0.5, "num_turns": 9, "duration_ms": 12000}) self.assertEqual(advisor_bench.read_result([line]).advisor_calls, 0) def test_refuses_a_run_that_ended_in_an_error(self): with self.assertRaisesRegex(ValueError, "the run ended in an error: API Error"): advisor_bench.read_result([result_line(is_error=True, result="API Error")]) def test_refuses_a_stream_with_no_result_line(self): with self.assertRaises(ValueError) as refused: advisor_bench.read_result(['{"type": "assistant"}']) self.assertEqual(str(refused.exception), "the stream has no result line") class Verdict(unittest.TestCase): def test_reads_the_verdict_from_a_json_reply(self): self.assertEqual(advisor_bench.verdict_of(json.dumps({"result": '{"verdict": "partial", "why": "no test"}'})), "partial") def test_reads_the_verdict_from_a_reply_in_a_code_fence_or_after_prose(self): for reply in ('```json\n{"verdict": "fail", "why": "x"}\n```', 'Ruling: {"verdict": "fail", "why": "x"} Done.'): self.assertEqual(advisor_bench.verdict_of(json.dumps({"result": reply})), "fail") def test_reads_a_verdict_from_a_reply_that_spans_lines(self): self.assertEqual(advisor_bench.verdict_of(json.dumps({"result": '{\n"verdict": "pass",\n"why": "x"\n}'})), "pass") def test_refuses_a_reply_with_no_json_object(self): with self.assertRaises(ValueError) as refused: advisor_bench.verdict_of(json.dumps({"result": "I cannot rule."})) self.assertEqual(str(refused.exception), "the judge's reply has no JSON object") def test_refuses_a_verdict_outside_pass_partial_fail(self): with self.assertRaisesRegex(ValueError, "verdict"): advisor_bench.verdict_of(json.dumps({"result": '{"verdict": "great", "why": "x"}'})) class Summary(unittest.TestCase): def test_prints_one_row_per_arm_with_passes_and_mean_cost(self): rows = [ {"arm": "sonnet", "verdict": "pass", "cost": 1.0, "advisor_calls": 0, "seconds": 100.0}, {"arm": "sonnet", "verdict": "fail", "cost": 3.0, "advisor_calls": 0, "seconds": 300.0}, {"arm": "opus-medium", "verdict": "pass", "cost": 4.0, "advisor_calls": 0, "seconds": 200.0}, ] rows.append({"arm": "sonnet-advisor", "verdict": "partial", "cost": 1.0, "advisor_calls": 2, "seconds": 50.0}) self.assertEqual(advisor_bench.summary(rows).splitlines(), [ "| arm | runs | pass | partial | mean cost $ | mean advisor calls | mean seconds |", "|---|---|---|---|---|---|---|", "| sonnet-advisor | 1 | 0 | 1 | 1.00 | 2.0 | 50 |", "| sonnet | 2 | 1 | 0 | 2.00 | 0.0 | 200 |", "| opus-medium | 1 | 1 | 0 | 4.00 | 0.0 | 200 |", ]) def commit(repo, message): subprocess.run(["git", "-C", str(repo), "-c", "user.name=bench", "-c", "user.email=bench@example.com", "commit", "-q", "-a", "-m", message], check=True) class Orchestration(unittest.TestCase): """run, grade, main and rows_of, on a throwaway repo, with claude and run-heavy faked. Case 7 starts at a commit holding bin/pkg/mod.py, and merges a change to bin/pkg/mod.py, a new bin/other/mod.py, and todo.md, docs and CHANGELOG.md; a later commit then changes bin/pkg/mod.py again and adds bin/later/mod.py, which the judge must not see.""" def setUp(self): scratch = tempfile.TemporaryDirectory() self.addCleanup(scratch.cleanup) root = pathlib.Path(scratch.name) self.repo, self.work = root / "repo", root / "work" self.work.mkdir() subprocess.run(["git", "init", "-q", str(self.repo)], check=True) base = self.commit({"todo.md": TODO, "bin/pkg/mod.py": "x = 1\n", "docs/notes.md": "a\n", "CHANGELOG.md": "a\n"}) merged = self.commit({"todo.md": TODO + "\nmore\n", "bin/pkg/mod.py": "x = 2\n", "bin/other/mod.py": "z = 1\n", "docs/notes.md": "b\n", "CHANGELOG.md": "b\n"}) self.commit({"bin/pkg/mod.py": "x = 5\n", "bin/later/mod.py": "y = 1\n"}) self.calls = [] self.bench = advisor_bench.Bench(self.repo, {"7": (base, merged)}, self.fake) self.tests_output = "3 passed\n" self.worker_result = {} # Whether the fake worker removes bin/ in place of changing it. self.worker_removes_bin = False self.base = base def commit(self, files): for path, text in files.items(): (self.repo / path).parent.mkdir(parents=True, exist_ok=True) (self.repo / path).write_text(text) subprocess.run(["git", "-C", str(self.repo), "add", "."], check=True) commit(self.repo, "a commit") return advisor_bench.git(self.repo, "rev-parse", "HEAD").strip() def fake(self, args, **kwargs): """Stands in for subprocess.run where the bench runs claude or the tests.""" if args[0] == "claude": self.calls.append((args, kwargs)) if "--tools" in args: return subprocess.CompletedProcess(args, 0, stdout=json.dumps({"result": '{"verdict": "pass", "why": "ok"}'}), stderr="") if self.worker_removes_bin: shutil.rmtree(pathlib.Path(kwargs["cwd"], "bin")) else: pathlib.Path(kwargs["cwd"], "bin/pkg/mod.py").write_text("x = 3\n") pathlib.Path(kwargs["cwd"], "bin/pkg/new.py").write_text("n = 1\n") pathlib.Path(kwargs["cwd"], "bin/third").mkdir() pathlib.Path(kwargs["cwd"], "bin/third/mod.py").write_text("t = 1\n") subprocess.run(["git", "-C", str(kwargs["cwd"]), "add", "bin/third"], check=True) commit(kwargs["cwd"], "the worker commits") kwargs["stdout"].write(result_line(**self.worker_result) + "\n") return subprocess.CompletedProcess(args, 0) if args[0] == str(advisor_bench.RUN_HEAVY): self.calls.append((args, kwargs)) return subprocess.CompletedProcess(args, 4, stdout=self.tests_output, stderr="") raise AssertionError(f"the bench ran {args}") def test_run_clones_at_the_start_commit_and_saves_the_stream_the_diff_and_the_tests(self): self.assertEqual(advisor_bench.run(self.bench, self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: ran") (claude, claude_kwargs), (tests, tests_kwargs) = self.calls clone = self.work / "7-sonnet-advisor-1" self.assertEqual((claude_kwargs["cwd"], tests_kwargs["cwd"]), (clone, clone)) self.assertEqual(claude[:2] + claude[3:], [ "claude", "-p", *advisor_bench.ARMS["sonnet-advisor"].flags(), "--setting-sources", "project", "--permission-mode", "bypassPermissions", "--no-session-persistence", "--output-format", "stream-json", "--verbose"]) self.assertIn("Do the seventh thing.", claude[2]) self.assertIn(f"`{self.base}`", claude[2]) self.assertIn("todo item 7 ", claude[2]) self.assertEqual(tests, [str(advisor_bench.RUN_HEAVY), *advisor_bench.SUITE, "bin/pkg", "bin/third"]) self.assertEqual((tests_kwargs["capture_output"], tests_kwargs["text"]), (True, True)) self.assertEqual(claude_kwargs["stdout"].name, str(self.work / "7-sonnet-advisor-1.jsonl")) self.assertEqual(claude_kwargs["stderr"], subprocess.STDOUT) self.assertEqual(advisor_bench.git(clone, "remote"), "") self.assertEqual(advisor_bench.git(clone, "rev-parse", "HEAD~").strip(), self.base) self.assertEqual(json.loads((self.work / "7-sonnet-advisor-1.jsonl").read_text())["total_cost_usd"], 0.5) diff = (self.work / "7-sonnet-advisor-1.diff").read_text() self.assertIn("-x = 1\n+x = 3\n", diff) self.assertIn("+++ b/bin/pkg/new.py", diff) self.assertIn("+++ b/bin/third/mod.py", diff) self.assertEqual((self.work / "7-sonnet-advisor-1.tests").read_text(), "exit 4\n3 passed\n") def test_run_tests_all_of_bin_where_no_touched_package_is_in_the_clone(self): self.worker_removes_bin = True advisor_bench.run(self.bench, self.work, "7-sonnet-1") self.assertEqual(self.calls[1][0][-1:], ["bin"]) def test_run_refuses_a_run_that_ended_in_an_error_before_it_tests(self): self.worker_result = {"is_error": True, "result": "API Error"} with self.assertRaisesRegex(ValueError, "API Error"): advisor_bench.run(self.bench, self.work, "7-sonnet-1") self.assertEqual(len(self.calls), 1) self.assertFalse((self.work / "7-sonnet-1.tests").exists()) def test_run_keeps_the_last_1500_characters_of_the_tests(self): self.tests_output = "a" * 2000 + "b" * 1500 advisor_bench.run(self.bench, self.work, "7-sonnet-1") self.assertEqual((self.work / "7-sonnet-1.tests").read_text(), "exit 4\n" + "b" * 1500) def test_run_refuses_a_name_that_already_ran(self): advisor_bench.run(self.bench, self.work, "7-sonnet-1") with self.assertRaisesRegex(ValueError, "7-sonnet-1 already ran: remove .*7-sonnet-1"): advisor_bench.run(self.bench, self.work, "7-sonnet-1") def test_a_clone_of_a_missing_repo_fails(self): with self.assertRaises(subprocess.CalledProcessError): advisor_bench.fresh_clone(self.work / "missing", self.base, self.work / "clone") def test_git_fails_where_git_does(self): with self.assertRaises(subprocess.CalledProcessError): advisor_bench.git(self.repo, "rev-parse", "no-such-revision") def test_grade_saves_the_verdict_and_shows_the_judge_both_changes_but_not_the_arm(self): advisor_bench.run(self.bench, self.work, "7-sonnet-advisor-1") self.calls.clear() self.assertEqual(advisor_bench.grade(self.bench, self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: pass") ((judge, kwargs),) = self.calls self.assertEqual(judge[:2] + judge[3:], [ "claude", "-p", *advisor_bench.JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", "", "--strict-mcp-config"]) self.assertEqual((kwargs["capture_output"], kwargs["text"], kwargs["check"]), (True, True, True)) prompt = judge[2] self.assertIn("Do the seventh thing.", prompt) self.assertIn("-x = 1\n+x = 2\n", prompt) self.assertIn("+++ b/bin/other/mod.py", prompt) for left_out in ("x = 5", "later", "docs/notes.md", "CHANGELOG.md", "todo.md"): self.assertNotIn(left_out, prompt) self.assertIn("+x = 3", prompt) self.assertIn("3 passed", prompt) self.assertNotIn("sonnet", prompt) self.assertEqual( json.loads((self.work / "7-sonnet-advisor-1.verdict").read_text()), {"verdict": "pass", "reply": '{"verdict": "pass", "why": "ok"}'}) def test_attempted_names_a_failure_in_place_of_raising_it(self): def step(work, name): raise KeyError("9") self.assertEqual(advisor_bench.attempted(step, self.work, "9-sonnet-1"), "9-sonnet-1: FAILED: KeyError: '9'") def test_attempted_hands_the_step_the_work_dir_and_the_name(self): self.assertEqual(advisor_bench.attempted(lambda work, name: f"{work.name} {name}", self.work, "9-sonnet-1"), "work 9-sonnet-1") def test_rows_of_pairs_each_verdict_with_its_runs_outcome(self): (self.work / "7-sonnet-advisor-1.jsonl").write_text(result_line(total_cost_usd=2.0) + "\n") (self.work / "7-sonnet-advisor-1.verdict").write_text(json.dumps({"verdict": "fail"})) (self.work / "7-sonnet-2.jsonl").write_text(result_line() + "\n") self.assertEqual(advisor_bench.rows_of(self.work), [{"arm": "sonnet-advisor", "verdict": "fail", "cost": 2.0, "seconds": 12.0, "advisor_calls": 0}]) def test_rows_of_names_a_run_whose_stream_has_no_result(self): (self.work / "7-sonnet-1.jsonl").write_text("") (self.work / "7-sonnet-1.verdict").write_text(json.dumps({"verdict": "pass"})) with self.assertRaisesRegex(ValueError, "^7-sonnet-1: the stream has no result line$"): advisor_bench.rows_of(self.work) def run_main(self, steps, *argv): said = [] advisor_bench.main([str(self.work), *argv], steps, said.append) return "".join(f"{line}\n" for line in said) def test_main_prints_each_runs_line_though_an_earlier_run_failed(self): def step(work, name): if name == "7-a-1": raise ValueError("no") return f"{name}: ran" self.assertEqual(self.run_main({"run": step}, "run", "7-a-1", "7-b-1"), "7-a-1: FAILED: ValueError: no\n7-b-1: ran\n") def test_main_grades_on_grade(self): self.assertEqual(self.run_main({"grade": lambda work, name: f"{work.name} {name}: graded"}, "grade", "7-a-1"), "work 7-a-1: graded\n") def test_the_steps_run_and_grade_on_the_bench_they_are_given(self): steps = advisor_bench.steps_of(self.bench) self.assertEqual(self.run_main(steps, "run", "7-sonnet-1"), "7-sonnet-1: ran\n") self.assertEqual(self.run_main(steps, "grade", "7-sonnet-1"), "7-sonnet-1: pass\n") def test_main_prints_the_work_dirs_summary_on_summary(self): (self.work / "7-sonnet-1.jsonl").write_text(result_line() + "\n") (self.work / "7-sonnet-1.verdict").write_text(json.dumps({"verdict": "pass"})) self.assertIn("| sonnet | 1 | 1 | 0 | 0.50 | 0.0 | 12 |", self.run_main({}, "summary")) if __name__ == "__main__": unittest.main()