Answer stability round 1: 0 🔴, 6 🟡 (0 misread); the bench tests only the packages the clone holds, refuses a run that ended in an error, and pins a worker's committed change
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# Claude Code configuration
|
||||
|
||||
My personal `~/.claude`, version controlled: `CLAUDE.md` and `rules/` for instructions, the words under `CLAUDE.md` → "Vocabulary"; `.claude/rules/` for the rules that load in this repo alone; `agents/` and `skills/`; `principles/` and `skills/larv-review/standards/` for what the reviewer agents hold projects to; `docs/` for decisions and the comprehension history; `settings.json` for the harness and `.claude/settings.json` for the harness inside this repo; `CHANGELOG.md` and `todo.md`; the tools in `bin/`; `workflow-bench/`, the scripts and prompts of the 2026-09-30 bench of larv-review; `reviewer-merge-trials/`, the 2026-11-09 trials of merging reviewers, with their method, results and scripts; `judge-field-trials/`, the 2026-11-11 trials of which facts code could read in place of loop-claude's judge, with their results and scripts; and `presentation/`, a page that explains this setup to newcomers, with the script that prints it to PDF.
|
||||
My personal `~/.claude`, version controlled: `CLAUDE.md` and `rules/` for instructions, the words under `CLAUDE.md` → "Vocabulary"; `.claude/rules/` for the rules that load in this repo alone; `agents/` and `skills/`; `principles/` and `skills/larv-review/standards/` for what the reviewer agents hold projects to; `docs/` for decisions and the comprehension history; `settings.json` for the harness and `.claude/settings.json` for the harness inside this repo; `CHANGELOG.md` and `todo.md`; the tools in `bin/`; `workflow-bench/`, the scripts and prompts of the 2026-09-30 bench of larv-review and of the 2026-11-12 bench of the advisor tool; `reviewer-merge-trials/`, the 2026-11-09 trials of merging reviewers, with their method, results and scripts; `judge-field-trials/`, the 2026-11-11 trials of which facts code could read in place of loop-claude's judge, with their results and scripts; and `presentation/`, a page that explains this setup to newcomers, with the script that prints it to PDF.
|
||||
|
||||
## Goals
|
||||
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bench the advisor tool on archived chunks: advisor_bench.py <work dir> run|grade <case>-<arm>-<n> ..., then advisor_bench.py <work dir> summary
|
||||
|
||||
Item 320's question: does a worker below the project's rung, Sonnet 5.5 with Opus 5.5 as its advisor, beat the
|
||||
same Sonnet alone, or match Opus 5.5 at medium for less? Each case is an archived L 1 or L 2 chunk of this
|
||||
repo (CASES: its todo.md item, the commit it started from, the commit it merged at). `run` clones the repo at
|
||||
the start commit, hands the arm's worker that item as loop-claude's worker would see it, and saves, beside the
|
||||
clone in <work dir>: the claude stream (.jsonl), the worker's change (.diff) and the result of the tests the worker's and the shipped change touch (.tests).
|
||||
The question: does a worker below the project's rung, Sonnet 5.5 with Opus 5.5 as its advisor, beat the same
|
||||
Sonnet alone, or match Opus 5.5 at medium for less? Each case is an archived L 1 or L 2 chunk of this repo
|
||||
(CASES holds its todo.md item, the commit it started from and the commit it merged at). `run` clones the repo at
|
||||
the start commit and hands the arm's worker the item's text from todo.md, with the bench rules of
|
||||
advisor-run-prompt.md in place of loop-claude's prompt. It saves, beside the clone in <work dir>: the claude
|
||||
stream (.jsonl), the worker's change (.diff) and the result of the tests of the packages that either change touches (.tests).
|
||||
`grade` has a judge on Opus 5.5 at medium, blind to the arm, rule on the .diff against the merged commit's change
|
||||
(.verdict); `summary` prints a table per arm.
|
||||
Run each arm of each case once; the table shows a direction, not a significance.
|
||||
@@ -21,8 +22,7 @@ import sys
|
||||
BENCH = pathlib.Path(__file__).resolve().parent
|
||||
REPO = BENCH.parent
|
||||
RUN_HEAVY = pathlib.Path.home() / ".claude/bin/run-heavy"
|
||||
SUITE = ["uvx", "--python", "/usr/bin/python3", "--with", "pytest-xdist==3.8.0", "pytest==9.2.0", "-n", "4", "-q"]
|
||||
# Each case is the item's ID, the commit its chunk started from and the commit it merged at.
|
||||
SUITE = ["uvx", "--python", sys.executable, "--with", "pytest-xdist==3.8.0", "pytest==9.2.0", "-n", "4", "-q"]
|
||||
CASES = {
|
||||
"290": ("cda45a931", "509804da0"),
|
||||
"251": ("b74322892", "d13c4fb0d"),
|
||||
@@ -55,7 +55,6 @@ JUDGE = Arm("claude-opus-5-5", "medium")
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Outcome:
|
||||
cost: float
|
||||
turns: int
|
||||
seconds: float
|
||||
advisor_calls: int
|
||||
|
||||
@@ -69,16 +68,18 @@ def item_text(todo, item):
|
||||
|
||||
|
||||
def read_result(lines):
|
||||
"""The cost, turns, seconds and advisor calls of a claude stream's first result line, the worker's own: a background task's notification makes later ones. Its cost counts the advisor's tokens."""
|
||||
"""The cost, seconds and advisor calls of a claude stream's first result line, which is the worker's: a background task's notification makes the later ones. The cost counts the advisor's tokens."""
|
||||
for line in lines:
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if entry.get("type") == "result":
|
||||
if entry.get("is_error"):
|
||||
raise ValueError(f"the run ended in an error: {entry.get('result')}")
|
||||
iterations = entry.get("usage", {}).get("iterations", [])
|
||||
return Outcome(
|
||||
cost=entry["total_cost_usd"], turns=entry["num_turns"], seconds=entry["duration_ms"] / 1000,
|
||||
cost=entry["total_cost_usd"], seconds=entry["duration_ms"] / 1000,
|
||||
advisor_calls=sum(1 for each in iterations if each.get("type") == "advisor_message"))
|
||||
raise ValueError("the stream has no result line")
|
||||
|
||||
@@ -144,10 +145,12 @@ def run(work, name):
|
||||
["claude", "-p", prompt, *ARMS[arm].flags(), "--setting-sources", "project", "--permission-mode", "bypassPermissions",
|
||||
"--no-session-persistence", "--output-format", "stream-json", "--verbose"],
|
||||
cwd=clone, stdout=out, stderr=subprocess.STDOUT)
|
||||
read_result((work / f"{name}.jsonl").read_text().splitlines())
|
||||
git(clone, "add", "--all")
|
||||
(work / f"{name}.diff").write_text(git(clone, "diff", "--cached", base))
|
||||
changed = [*git(clone, "diff", "--cached", "--name-only", base).split(), *git(REPO, "diff", "--name-only", base, CASES[case][1]).split()]
|
||||
tests = subprocess.run([str(RUN_HEAVY), *SUITE, *suite_targets(changed)], cwd=clone, capture_output=True, text=True)
|
||||
targets = [target for target in suite_targets(changed) if (clone / target).exists()] or ["bin"]
|
||||
tests = subprocess.run([str(RUN_HEAVY), *SUITE, *targets], cwd=clone, capture_output=True, text=True)
|
||||
(work / f"{name}.tests").write_text(f"exit {tests.returncode}\n{tests.stdout[-1500:]}")
|
||||
return f"{name}: ran"
|
||||
|
||||
@@ -179,7 +182,10 @@ def rows_of(work):
|
||||
rows = []
|
||||
for verdict_file in sorted(work.glob("*.verdict")):
|
||||
name = verdict_file.stem
|
||||
outcome = read_result((work / f"{name}.jsonl").read_text().splitlines())
|
||||
try:
|
||||
outcome = read_result((work / f"{name}.jsonl").read_text().splitlines())
|
||||
except ValueError as failure:
|
||||
raise ValueError(f"{name}: {failure}") from failure
|
||||
rows.append({"arm": split_name(name)[1], "verdict": json.loads(verdict_file.read_text())["verdict"], **dataclasses.asdict(outcome)})
|
||||
return rows
|
||||
|
||||
|
||||
@@ -65,13 +65,13 @@ class ItemText(unittest.TestCase):
|
||||
|
||||
|
||||
class ReadResult(unittest.TestCase):
|
||||
def test_reads_cost_turns_seconds_and_counts_advisor_calls(self):
|
||||
def test_reads_cost_seconds_and_counts_advisor_calls(self):
|
||||
iterations = [{"type": "message"}, {"type": "advisor_message"}, {"type": "message"}, {"type": "message"}, {"type": "advisor_message"}]
|
||||
outcome = advisor_bench.read_result(["not json", result_line(usage={"iterations": iterations})])
|
||||
self.assertEqual(outcome, advisor_bench.Outcome(cost=0.5, turns=9, seconds=12.0, advisor_calls=2))
|
||||
self.assertEqual(outcome, advisor_bench.Outcome(cost=0.5, seconds=12.0, advisor_calls=2))
|
||||
|
||||
def test_reads_the_first_result_where_a_background_task_makes_later_ones(self):
|
||||
self.assertEqual(advisor_bench.read_result([result_line(num_turns=9), result_line(num_turns=2)]).turns, 9)
|
||||
self.assertEqual(advisor_bench.read_result([result_line(total_cost_usd=0.5), result_line(total_cost_usd=2.0)]).cost, 0.5)
|
||||
|
||||
def test_counts_no_advisor_call_where_the_run_has_no_iterations(self):
|
||||
self.assertEqual(advisor_bench.read_result([result_line(usage={})]).advisor_calls, 0)
|
||||
@@ -80,6 +80,10 @@ class ReadResult(unittest.TestCase):
|
||||
line = json.dumps({"type": "result", "total_cost_usd": 0.5, "num_turns": 9, "duration_ms": 12000})
|
||||
self.assertEqual(advisor_bench.read_result([line]).advisor_calls, 0)
|
||||
|
||||
def test_refuses_a_run_that_ended_in_an_error(self):
|
||||
with self.assertRaisesRegex(ValueError, "the run ended in an error: API Error"):
|
||||
advisor_bench.read_result([result_line(is_error=True, result="API Error")])
|
||||
|
||||
def test_refuses_a_stream_with_no_result_line(self):
|
||||
with self.assertRaises(ValueError) as refused:
|
||||
advisor_bench.read_result(['{"type": "assistant"}'])
|
||||
@@ -132,7 +136,7 @@ class Orchestration(unittest.TestCase):
|
||||
"""run, grade, main and rows_of, on a throwaway repo, with claude and run-heavy faked.
|
||||
|
||||
Case 7 starts at a commit holding bin/pkg/mod.py, and merges a change to bin/pkg/mod.py, a new bin/other/mod.py, and
|
||||
todo.md, docs and CHANGELOG.md; a later commit then changes bin/pkg/mod.py again and adds bin/later/mod.py."""
|
||||
todo.md, docs and CHANGELOG.md; a later commit then changes bin/pkg/mod.py again and adds bin/later/mod.py, which the judge must not see."""
|
||||
|
||||
def setUp(self):
|
||||
scratch = tempfile.TemporaryDirectory()
|
||||
@@ -153,6 +157,7 @@ class Orchestration(unittest.TestCase):
|
||||
patch.start()
|
||||
self.addCleanup(patch.stop)
|
||||
self.tests_output = "3 passed\n"
|
||||
self.worker_result = {}
|
||||
self.base = base
|
||||
|
||||
def commit(self, files):
|
||||
@@ -170,7 +175,11 @@ class Orchestration(unittest.TestCase):
|
||||
return subprocess.CompletedProcess(args, 0, stdout=json.dumps({"result": '{"verdict": "pass", "why": "ok"}'}), stderr="")
|
||||
pathlib.Path(kwargs["cwd"], "bin/pkg/mod.py").write_text("x = 3\n")
|
||||
pathlib.Path(kwargs["cwd"], "bin/pkg/new.py").write_text("n = 1\n")
|
||||
kwargs["stdout"].write(result_line() + "\n")
|
||||
pathlib.Path(kwargs["cwd"], "bin/third").mkdir()
|
||||
pathlib.Path(kwargs["cwd"], "bin/third/mod.py").write_text("t = 1\n")
|
||||
subprocess.run(["git", "-C", str(kwargs["cwd"]), "add", "bin/third"], check=True)
|
||||
commit(kwargs["cwd"], "the worker commits")
|
||||
kwargs["stdout"].write(result_line(**self.worker_result) + "\n")
|
||||
return subprocess.CompletedProcess(args, 0)
|
||||
if args[0] == str(advisor_bench.RUN_HEAVY):
|
||||
self.calls.append((args, kwargs))
|
||||
@@ -188,18 +197,31 @@ class Orchestration(unittest.TestCase):
|
||||
self.assertIn("Do the seventh thing.", claude[2])
|
||||
self.assertIn(f"`{self.base}`", claude[2])
|
||||
self.assertIn("todo item 7 ", claude[2])
|
||||
self.assertEqual(tests, [str(advisor_bench.RUN_HEAVY), *advisor_bench.SUITE, "bin/other", "bin/pkg"])
|
||||
self.assertEqual(tests, [str(advisor_bench.RUN_HEAVY), *advisor_bench.SUITE, "bin/pkg", "bin/third"])
|
||||
self.assertEqual((tests_kwargs["capture_output"], tests_kwargs["text"]), (True, True))
|
||||
self.assertEqual(claude_kwargs["stdout"].name, str(self.work / "7-sonnet-advisor-1.jsonl"))
|
||||
self.assertEqual(claude_kwargs["stderr"], subprocess.STDOUT)
|
||||
self.assertEqual(advisor_bench.git(clone, "remote"), "")
|
||||
self.assertEqual(advisor_bench.git(clone, "rev-parse", "HEAD").strip(), self.base)
|
||||
self.assertEqual(json.loads((self.work / "7-sonnet-advisor-1.jsonl").read_text())["num_turns"], 9)
|
||||
self.assertEqual(advisor_bench.git(clone, "rev-parse", "HEAD~").strip(), self.base)
|
||||
self.assertEqual(json.loads((self.work / "7-sonnet-advisor-1.jsonl").read_text())["total_cost_usd"], 0.5)
|
||||
diff = (self.work / "7-sonnet-advisor-1.diff").read_text()
|
||||
self.assertIn("-x = 1\n+x = 3\n", diff)
|
||||
self.assertIn("+++ b/bin/pkg/new.py", diff)
|
||||
self.assertIn("+++ b/bin/third/mod.py", diff)
|
||||
self.assertEqual((self.work / "7-sonnet-advisor-1.tests").read_text(), "exit 4\n3 passed\n")
|
||||
|
||||
def test_run_tests_all_of_bin_where_no_touched_package_is_in_the_clone(self):
|
||||
with unittest.mock.patch.object(advisor_bench, "suite_targets", return_value=["bin/absent"]):
|
||||
advisor_bench.run(self.work, "7-sonnet-1")
|
||||
self.assertEqual(self.calls[1][0][-1:], ["bin"])
|
||||
|
||||
def test_run_refuses_a_run_that_ended_in_an_error_before_it_tests(self):
|
||||
self.worker_result = {"is_error": True, "result": "API Error"}
|
||||
with self.assertRaisesRegex(ValueError, "API Error"):
|
||||
advisor_bench.run(self.work, "7-sonnet-1")
|
||||
self.assertEqual(len(self.calls), 1)
|
||||
self.assertFalse((self.work / "7-sonnet-1.tests").exists())
|
||||
|
||||
def test_run_keeps_the_last_1500_characters_of_the_tests(self):
|
||||
self.tests_output = "a" * 2000 + "b" * 1500
|
||||
advisor_bench.run(self.work, "7-sonnet-1")
|
||||
@@ -251,7 +273,13 @@ class Orchestration(unittest.TestCase):
|
||||
(self.work / "7-sonnet-advisor-1.jsonl").write_text(result_line(total_cost_usd=2.0) + "\n")
|
||||
(self.work / "7-sonnet-advisor-1.verdict").write_text(json.dumps({"verdict": "fail"}))
|
||||
(self.work / "7-sonnet-2.jsonl").write_text(result_line() + "\n")
|
||||
self.assertEqual(advisor_bench.rows_of(self.work), [{"arm": "sonnet-advisor", "verdict": "fail", "cost": 2.0, "turns": 9, "seconds": 12.0, "advisor_calls": 0}])
|
||||
self.assertEqual(advisor_bench.rows_of(self.work), [{"arm": "sonnet-advisor", "verdict": "fail", "cost": 2.0, "seconds": 12.0, "advisor_calls": 0}])
|
||||
|
||||
def test_rows_of_names_a_run_whose_stream_has_no_result(self):
|
||||
(self.work / "7-sonnet-1.jsonl").write_text("")
|
||||
(self.work / "7-sonnet-1.verdict").write_text(json.dumps({"verdict": "pass"}))
|
||||
with self.assertRaisesRegex(ValueError, "^7-sonnet-1: the stream has no result line$"):
|
||||
advisor_bench.rows_of(self.work)
|
||||
|
||||
def run_main(self, *argv):
|
||||
out = io.StringIO()
|
||||
|
||||
Reference in New Issue
Block a user