Pin the advisor bench's argv, kwargs, diff exclusions and failures in its tests
This commit is contained in:
@@ -76,6 +76,10 @@ class ReadResult(unittest.TestCase):
|
||||
def test_counts_no_advisor_call_where_the_run_has_no_iterations(self):
|
||||
self.assertEqual(advisor_bench.read_result([result_line(usage={})]).advisor_calls, 0)
|
||||
|
||||
def test_counts_no_advisor_call_where_the_run_has_no_usage(self):
|
||||
line = json.dumps({"type": "result", "total_cost_usd": 0.5, "num_turns": 9, "duration_ms": 12000})
|
||||
self.assertEqual(advisor_bench.read_result([line]).advisor_calls, 0)
|
||||
|
||||
def test_refuses_a_stream_with_no_result_line(self):
|
||||
with self.assertRaises(ValueError) as refused:
|
||||
advisor_bench.read_result(['{"type": "assistant"}'])
|
||||
@@ -90,6 +94,9 @@ class Verdict(unittest.TestCase):
|
||||
for reply in ('```json\n{"verdict": "fail", "why": "x"}\n```', 'Ruling: {"verdict": "fail", "why": "x"} Done.'):
|
||||
self.assertEqual(advisor_bench.verdict_of(json.dumps({"result": reply})), "fail")
|
||||
|
||||
def test_reads_a_verdict_from_a_reply_that_spans_lines(self):
|
||||
self.assertEqual(advisor_bench.verdict_of(json.dumps({"result": '{\n"verdict": "pass",\n"why": "x"\n}'})), "pass")
|
||||
|
||||
def test_refuses_a_reply_with_no_json_object(self):
|
||||
with self.assertRaises(ValueError) as refused:
|
||||
advisor_bench.verdict_of(json.dumps({"result": "I cannot rule."}))
|
||||
@@ -122,7 +129,10 @@ def commit(repo, message):
|
||||
|
||||
|
||||
class Orchestration(unittest.TestCase):
|
||||
"""run, grade, main and rows_of, on a throwaway repo whose case 7 changes bin/pkg/mod.py, with claude and run-heavy faked."""
|
||||
"""run, grade, main and rows_of, on a throwaway repo, with claude and run-heavy faked.
|
||||
|
||||
Case 7 starts at a commit holding bin/pkg/mod.py, and merges a change to bin/pkg/mod.py, a new bin/other/mod.py, and
|
||||
todo.md, docs and CHANGELOG.md; a later commit then changes bin/pkg/mod.py again and adds bin/later/mod.py."""
|
||||
|
||||
def setUp(self):
|
||||
scratch = tempfile.TemporaryDirectory()
|
||||
@@ -130,16 +140,10 @@ class Orchestration(unittest.TestCase):
|
||||
root = pathlib.Path(scratch.name)
|
||||
self.repo, self.work = root / "repo", root / "work"
|
||||
self.work.mkdir()
|
||||
(self.repo / "bin/pkg").mkdir(parents=True)
|
||||
subprocess.run(["git", "init", "-q", str(self.repo)], check=True)
|
||||
(self.repo / "todo.md").write_text(TODO)
|
||||
(self.repo / "bin/pkg/mod.py").write_text("x = 1\n")
|
||||
subprocess.run(["git", "-C", str(self.repo), "add", "."], check=True)
|
||||
commit(self.repo, "start")
|
||||
base = advisor_bench.git(self.repo, "rev-parse", "HEAD").strip()
|
||||
(self.repo / "bin/pkg/mod.py").write_text("x = 2\n")
|
||||
commit(self.repo, "merged")
|
||||
merged = advisor_bench.git(self.repo, "rev-parse", "HEAD").strip()
|
||||
base = self.commit({"todo.md": TODO, "bin/pkg/mod.py": "x = 1\n", "docs/notes.md": "a\n", "CHANGELOG.md": "a\n"})
|
||||
merged = self.commit({"todo.md": TODO + "\nmore\n", "bin/pkg/mod.py": "x = 2\n", "bin/other/mod.py": "z = 1\n", "docs/notes.md": "b\n", "CHANGELOG.md": "b\n"})
|
||||
self.commit({"bin/pkg/mod.py": "x = 5\n", "bin/later/mod.py": "y = 1\n"})
|
||||
self.calls = []
|
||||
real_run = subprocess.run
|
||||
for patch in (
|
||||
@@ -149,31 +153,51 @@ class Orchestration(unittest.TestCase):
|
||||
patch.start()
|
||||
self.addCleanup(patch.stop)
|
||||
self.tests_output = "3 passed\n"
|
||||
self.base = base
|
||||
|
||||
def commit(self, files):
|
||||
for path, text in files.items():
|
||||
(self.repo / path).parent.mkdir(parents=True, exist_ok=True)
|
||||
(self.repo / path).write_text(text)
|
||||
subprocess.run(["git", "-C", str(self.repo), "add", "."], check=True)
|
||||
commit(self.repo, "a commit")
|
||||
return advisor_bench.git(self.repo, "rev-parse", "HEAD").strip()
|
||||
|
||||
def fake(self, real_run, args, **kwargs):
|
||||
if args[0] == "claude":
|
||||
self.calls.append((args, kwargs.get("cwd")))
|
||||
self.calls.append((args, kwargs))
|
||||
if "--tools" in args:
|
||||
return subprocess.CompletedProcess(args, 0, stdout=json.dumps({"result": '{"verdict": "pass", "why": "ok"}'}), stderr="")
|
||||
pathlib.Path(kwargs["cwd"], "bin/pkg/mod.py").write_text("x = 3\n")
|
||||
pathlib.Path(kwargs["cwd"], "bin/pkg/new.py").write_text("n = 1\n")
|
||||
kwargs["stdout"].write(result_line() + "\n")
|
||||
return subprocess.CompletedProcess(args, 0)
|
||||
if args[0] == str(advisor_bench.RUN_HEAVY):
|
||||
self.calls.append((args, kwargs.get("cwd")))
|
||||
self.calls.append((args, kwargs))
|
||||
return subprocess.CompletedProcess(args, 4, stdout=self.tests_output, stderr="")
|
||||
return real_run(args, **kwargs)
|
||||
|
||||
def test_run_clones_at_the_start_commit_and_saves_the_stream_the_diff_and_the_tests(self):
|
||||
self.assertEqual(advisor_bench.run(self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: ran")
|
||||
(claude, claude_cwd), (tests, tests_cwd) = self.calls
|
||||
(claude, claude_kwargs), (tests, tests_kwargs) = self.calls
|
||||
clone = self.work / "7-sonnet-advisor-1"
|
||||
self.assertEqual((claude_cwd, tests_cwd), (clone, clone))
|
||||
self.assertEqual(claude[claude.index("--model"):claude.index("--model") + 6], advisor_bench.ARMS["sonnet-advisor"].flags())
|
||||
self.assertEqual((claude_kwargs["cwd"], tests_kwargs["cwd"]), (clone, clone))
|
||||
self.assertEqual(claude[:2] + claude[3:], [
|
||||
"claude", "-p", *advisor_bench.ARMS["sonnet-advisor"].flags(), "--setting-sources", "project", "--permission-mode", "bypassPermissions",
|
||||
"--no-session-persistence", "--output-format", "stream-json", "--verbose"])
|
||||
self.assertIn("Do the seventh thing.", claude[2])
|
||||
self.assertEqual(tests, [str(advisor_bench.RUN_HEAVY), *advisor_bench.SUITE, "bin/pkg"])
|
||||
self.assertIn(f"`{self.base}`", claude[2])
|
||||
self.assertIn("todo item 7 ", claude[2])
|
||||
self.assertEqual(tests, [str(advisor_bench.RUN_HEAVY), *advisor_bench.SUITE, "bin/other", "bin/pkg"])
|
||||
self.assertEqual((tests_kwargs["capture_output"], tests_kwargs["text"]), (True, True))
|
||||
self.assertEqual(claude_kwargs["stdout"].name, str(self.work / "7-sonnet-advisor-1.jsonl"))
|
||||
self.assertEqual(claude_kwargs["stderr"], subprocess.STDOUT)
|
||||
self.assertEqual(advisor_bench.git(clone, "remote"), "")
|
||||
self.assertEqual(advisor_bench.git(clone, "rev-parse", "HEAD").strip(), self.base)
|
||||
self.assertEqual(json.loads((self.work / "7-sonnet-advisor-1.jsonl").read_text())["num_turns"], 9)
|
||||
self.assertEqual((self.work / "7-sonnet-advisor-1.diff").read_text().splitlines()[-2:], ["-x = 1", "+x = 3"])
|
||||
diff = (self.work / "7-sonnet-advisor-1.diff").read_text()
|
||||
self.assertIn("-x = 1\n+x = 3\n", diff)
|
||||
self.assertIn("+++ b/bin/pkg/new.py", diff)
|
||||
self.assertEqual((self.work / "7-sonnet-advisor-1.tests").read_text(), "exit 4\n3 passed\n")
|
||||
|
||||
def test_run_keeps_the_last_1500_characters_of_the_tests(self):
|
||||
@@ -186,23 +210,42 @@ class Orchestration(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, "7-sonnet-1 already ran: remove .*7-sonnet-1"):
|
||||
advisor_bench.run(self.work, "7-sonnet-1")
|
||||
|
||||
def test_a_clone_of_a_missing_repo_fails(self):
|
||||
with unittest.mock.patch.object(advisor_bench, "REPO", self.work / "missing"), self.assertRaises(subprocess.CalledProcessError):
|
||||
advisor_bench.fresh_clone(self.base, self.work / "clone")
|
||||
|
||||
def test_git_fails_where_git_does(self):
|
||||
with self.assertRaises(subprocess.CalledProcessError):
|
||||
advisor_bench.git(self.repo, "rev-parse", "no-such-revision")
|
||||
|
||||
def test_grade_saves_the_verdict_and_shows_the_judge_both_changes_but_not_the_arm(self):
|
||||
advisor_bench.run(self.work, "7-sonnet-advisor-1")
|
||||
self.calls.clear()
|
||||
self.assertEqual(advisor_bench.grade(self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: pass")
|
||||
((judge, _),) = self.calls
|
||||
self.assertEqual(judge[judge.index("--model"):judge.index("--model") + 4], advisor_bench.JUDGE.flags())
|
||||
self.assertIn("+x = 2", judge[2])
|
||||
self.assertIn("+x = 3", judge[2])
|
||||
self.assertIn("3 passed", judge[2])
|
||||
self.assertNotIn("sonnet", judge[2])
|
||||
self.assertEqual(json.loads((self.work / "7-sonnet-advisor-1.verdict").read_text())["verdict"], "pass")
|
||||
((judge, kwargs),) = self.calls
|
||||
self.assertEqual(judge[:2] + judge[3:], [
|
||||
"claude", "-p", *advisor_bench.JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", ""])
|
||||
self.assertEqual((kwargs["capture_output"], kwargs["text"], kwargs["check"]), (True, True, True))
|
||||
prompt = judge[2]
|
||||
self.assertIn("Do the seventh thing.", prompt)
|
||||
self.assertIn("-x = 1\n+x = 2\n", prompt)
|
||||
self.assertIn("+++ b/bin/other/mod.py", prompt)
|
||||
for left_out in ("x = 5", "later", "docs/notes.md", "CHANGELOG.md", "todo.md"):
|
||||
self.assertNotIn(left_out, prompt)
|
||||
self.assertIn("+x = 3", prompt)
|
||||
self.assertIn("3 passed", prompt)
|
||||
self.assertNotIn("sonnet", prompt)
|
||||
self.assertEqual(
|
||||
json.loads((self.work / "7-sonnet-advisor-1.verdict").read_text()),
|
||||
{"verdict": "pass", "reply": '{"verdict": "pass", "why": "ok"}'})
|
||||
|
||||
def test_attempted_names_a_failure_in_place_of_raising_it(self):
|
||||
def step(work, name):
|
||||
raise KeyError("9")
|
||||
self.assertEqual(advisor_bench.attempted(step, self.work, "9-sonnet-1"), "9-sonnet-1: FAILED: KeyError: '9'")
|
||||
self.assertEqual(advisor_bench.attempted(lambda work, name: f"{name}: ok", self.work, "9-sonnet-1"), "9-sonnet-1: ok")
|
||||
|
||||
def test_attempted_hands_the_step_the_work_dir_and_the_name(self):
|
||||
self.assertEqual(advisor_bench.attempted(lambda work, name: f"{work.name} {name}", self.work, "9-sonnet-1"), "work 9-sonnet-1")
|
||||
|
||||
def test_rows_of_pairs_each_verdict_with_its_runs_outcome(self):
|
||||
(self.work / "7-sonnet-advisor-1.jsonl").write_text(result_line(total_cost_usd=2.0) + "\n")
|
||||
@@ -224,11 +267,14 @@ class Orchestration(unittest.TestCase):
|
||||
with unittest.mock.patch.object(advisor_bench, "run", step):
|
||||
self.assertEqual(self.run_main("run", "7-a-1", "7-b-1"), "7-a-1: FAILED: ValueError: no\n7-b-1: ran\n")
|
||||
|
||||
def test_main_grades_on_grade_and_prints_the_summary_on_summary(self):
|
||||
with unittest.mock.patch.object(advisor_bench, "grade", lambda work, name: f"{name}: graded"):
|
||||
self.assertEqual(self.run_main("grade", "7-a-1"), "7-a-1: graded\n")
|
||||
with unittest.mock.patch.object(advisor_bench, "summary", return_value="a table"):
|
||||
self.assertEqual(self.run_main("summary"), "a table\n")
|
||||
def test_main_grades_on_grade(self):
|
||||
with unittest.mock.patch.object(advisor_bench, "grade", lambda work, name: f"{work.name} {name}: graded"):
|
||||
self.assertEqual(self.run_main("grade", "7-a-1"), "work 7-a-1: graded\n")
|
||||
|
||||
def test_main_prints_the_work_dirs_summary_on_summary(self):
|
||||
(self.work / "7-sonnet-1.jsonl").write_text(result_line() + "\n")
|
||||
(self.work / "7-sonnet-1.verdict").write_text(json.dumps({"verdict": "pass"}))
|
||||
self.assertIn("| sonnet | 1 | 1 | 0 | 0.50 | 0.0 | 12 |", self.run_main("summary"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user