Answer stability round 1: 0 🔴, 6 🟡 (0 misread); the bench tests only the packages the clone holds, refuses a run that ended in an error, and pins a worker's committed change

This commit is contained in:
2026-11-12 05:30:15 +01:00
parent 46436f2ee0
commit 6817b074bd
3 changed files with 56 additions and 22 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
# Claude Code configuration
My personal `~/.claude`, version controlled: `CLAUDE.md` and `rules/` for instructions, the words under `CLAUDE.md` → "Vocabulary"; `.claude/rules/` for the rules that load in this repo alone; `agents/` and `skills/`; `principles/` and `skills/larv-review/standards/` for what the reviewer agents hold projects to; `docs/` for decisions and the comprehension history; `settings.json` for the harness and `.claude/settings.json` for the harness inside this repo; `CHANGELOG.md` and `todo.md`; the tools in `bin/`; `workflow-bench/`, the scripts and prompts of the 2026-09-30 bench of larv-review; `reviewer-merge-trials/`, the 2026-11-09 trials of merging reviewers, with their method, results and scripts; `judge-field-trials/`, the 2026-11-11 trials of which facts code could read in place of loop-claude's judge, with their results and scripts; and `presentation/`, a page that explains this setup to newcomers, with the script that prints it to PDF.
My personal `~/.claude`, version controlled: `CLAUDE.md` and `rules/` for instructions, the words under `CLAUDE.md` → "Vocabulary"; `.claude/rules/` for the rules that load in this repo alone; `agents/` and `skills/`; `principles/` and `skills/larv-review/standards/` for what the reviewer agents hold projects to; `docs/` for decisions and the comprehension history; `settings.json` for the harness and `.claude/settings.json` for the harness inside this repo; `CHANGELOG.md` and `todo.md`; the tools in `bin/`; `workflow-bench/`, the scripts and prompts of the 2026-09-30 bench of larv-review and of the 2026-11-12 bench of the advisor tool; `reviewer-merge-trials/`, the 2026-11-09 trials of merging reviewers, with their method, results and scripts; `judge-field-trials/`, the 2026-11-11 trials of which facts code could read in place of loop-claude's judge, with their results and scripts; and `presentation/`, a page that explains this setup to newcomers, with the script that prints it to PDF.
## Goals
+18 -12
View File
@@ -1,11 +1,12 @@
#!/usr/bin/env python3
"""Bench the advisor tool on archived chunks: advisor_bench.py <work dir> run|grade <case>-<arm>-<n> ..., then advisor_bench.py <work dir> summary
Item 320's question: does a worker below the project's rung, Sonnet 5.5 with Opus 5.5 as its advisor, beat the
same Sonnet alone, or match Opus 5.5 at medium for less? Each case is an archived L 1 or L 2 chunk of this
repo (CASES: its todo.md item, the commit it started from, the commit it merged at). `run` clones the repo at
the start commit, hands the arm's worker that item as loop-claude's worker would see it, and saves, beside the
clone in <work dir>: the claude stream (.jsonl), the worker's change (.diff) and the result of the tests the worker's and the shipped change touch (.tests).
The question: does a worker below the project's rung, Sonnet 5.5 with Opus 5.5 as its advisor, beat the same
Sonnet alone, or match Opus 5.5 at medium for less? Each case is an archived L 1 or L 2 chunk of this repo
(CASES holds its todo.md item, the commit it started from and the commit it merged at). `run` clones the repo at
the start commit and hands the arm's worker the item's text from todo.md, with the bench rules of
advisor-run-prompt.md in place of loop-claude's prompt. It saves, beside the clone in <work dir>: the claude
stream (.jsonl), the worker's change (.diff) and the result of the tests of the packages that either change touches (.tests).
`grade` has a judge on Opus 5.5 at medium, blind to the arm, rule on the .diff against the merged commit's change
(.verdict); `summary` prints a table per arm.
Run each arm of each case once; the table shows a direction, not a significance.
@@ -21,8 +22,7 @@ import sys
BENCH = pathlib.Path(__file__).resolve().parent
REPO = BENCH.parent
RUN_HEAVY = pathlib.Path.home() / ".claude/bin/run-heavy"
SUITE = ["uvx", "--python", "/usr/bin/python3", "--with", "pytest-xdist==3.8.0", "pytest==9.2.0", "-n", "4", "-q"]
# Each case is the item's ID, the commit its chunk started from and the commit it merged at.
SUITE = ["uvx", "--python", sys.executable, "--with", "pytest-xdist==3.8.0", "pytest==9.2.0", "-n", "4", "-q"]
CASES = {
"290": ("cda45a931", "509804da0"),
"251": ("b74322892", "d13c4fb0d"),
@@ -55,7 +55,6 @@ JUDGE = Arm("claude-opus-5-5", "medium")
@dataclasses.dataclass(frozen=True)
class Outcome:
cost: float
turns: int
seconds: float
advisor_calls: int
@@ -69,16 +68,18 @@ def item_text(todo, item):
def read_result(lines):
"""The cost, turns, seconds and advisor calls of a claude stream's first result line, the worker's own: a background task's notification makes later ones. Its cost counts the advisor's tokens."""
"""The cost, seconds and advisor calls of a claude stream's first result line, which is the worker's: a background task's notification makes the later ones. The cost counts the advisor's tokens."""
for line in lines:
try:
entry = json.loads(line)
except ValueError:
continue
if entry.get("type") == "result":
if entry.get("is_error"):
raise ValueError(f"the run ended in an error: {entry.get('result')}")
iterations = entry.get("usage", {}).get("iterations", [])
return Outcome(
cost=entry["total_cost_usd"], turns=entry["num_turns"], seconds=entry["duration_ms"] / 1000,
cost=entry["total_cost_usd"], seconds=entry["duration_ms"] / 1000,
advisor_calls=sum(1 for each in iterations if each.get("type") == "advisor_message"))
raise ValueError("the stream has no result line")
@@ -144,10 +145,12 @@ def run(work, name):
["claude", "-p", prompt, *ARMS[arm].flags(), "--setting-sources", "project", "--permission-mode", "bypassPermissions",
"--no-session-persistence", "--output-format", "stream-json", "--verbose"],
cwd=clone, stdout=out, stderr=subprocess.STDOUT)
read_result((work / f"{name}.jsonl").read_text().splitlines())
git(clone, "add", "--all")
(work / f"{name}.diff").write_text(git(clone, "diff", "--cached", base))
changed = [*git(clone, "diff", "--cached", "--name-only", base).split(), *git(REPO, "diff", "--name-only", base, CASES[case][1]).split()]
tests = subprocess.run([str(RUN_HEAVY), *SUITE, *suite_targets(changed)], cwd=clone, capture_output=True, text=True)
targets = [target for target in suite_targets(changed) if (clone / target).exists()] or ["bin"]
tests = subprocess.run([str(RUN_HEAVY), *SUITE, *targets], cwd=clone, capture_output=True, text=True)
(work / f"{name}.tests").write_text(f"exit {tests.returncode}\n{tests.stdout[-1500:]}")
return f"{name}: ran"
@@ -179,7 +182,10 @@ def rows_of(work):
rows = []
for verdict_file in sorted(work.glob("*.verdict")):
name = verdict_file.stem
outcome = read_result((work / f"{name}.jsonl").read_text().splitlines())
try:
outcome = read_result((work / f"{name}.jsonl").read_text().splitlines())
except ValueError as failure:
raise ValueError(f"{name}: {failure}") from failure
rows.append({"arm": split_name(name)[1], "verdict": json.loads(verdict_file.read_text())["verdict"], **dataclasses.asdict(outcome)})
return rows
+37 -9
View File
@@ -65,13 +65,13 @@ class ItemText(unittest.TestCase):
class ReadResult(unittest.TestCase):
def test_reads_cost_turns_seconds_and_counts_advisor_calls(self):
def test_reads_cost_seconds_and_counts_advisor_calls(self):
iterations = [{"type": "message"}, {"type": "advisor_message"}, {"type": "message"}, {"type": "message"}, {"type": "advisor_message"}]
outcome = advisor_bench.read_result(["not json", result_line(usage={"iterations": iterations})])
self.assertEqual(outcome, advisor_bench.Outcome(cost=0.5, turns=9, seconds=12.0, advisor_calls=2))
self.assertEqual(outcome, advisor_bench.Outcome(cost=0.5, seconds=12.0, advisor_calls=2))
def test_reads_the_first_result_where_a_background_task_makes_later_ones(self):
self.assertEqual(advisor_bench.read_result([result_line(num_turns=9), result_line(num_turns=2)]).turns, 9)
self.assertEqual(advisor_bench.read_result([result_line(total_cost_usd=0.5), result_line(total_cost_usd=2.0)]).cost, 0.5)
def test_counts_no_advisor_call_where_the_run_has_no_iterations(self):
self.assertEqual(advisor_bench.read_result([result_line(usage={})]).advisor_calls, 0)
@@ -80,6 +80,10 @@ class ReadResult(unittest.TestCase):
line = json.dumps({"type": "result", "total_cost_usd": 0.5, "num_turns": 9, "duration_ms": 12000})
self.assertEqual(advisor_bench.read_result([line]).advisor_calls, 0)
def test_refuses_a_run_that_ended_in_an_error(self):
with self.assertRaisesRegex(ValueError, "the run ended in an error: API Error"):
advisor_bench.read_result([result_line(is_error=True, result="API Error")])
def test_refuses_a_stream_with_no_result_line(self):
with self.assertRaises(ValueError) as refused:
advisor_bench.read_result(['{"type": "assistant"}'])
@@ -132,7 +136,7 @@ class Orchestration(unittest.TestCase):
"""run, grade, main and rows_of, on a throwaway repo, with claude and run-heavy faked.
Case 7 starts at a commit holding bin/pkg/mod.py, and merges a change to bin/pkg/mod.py, a new bin/other/mod.py, and
todo.md, docs and CHANGELOG.md; a later commit then changes bin/pkg/mod.py again and adds bin/later/mod.py."""
todo.md, docs and CHANGELOG.md; a later commit then changes bin/pkg/mod.py again and adds bin/later/mod.py, which the judge must not see."""
def setUp(self):
scratch = tempfile.TemporaryDirectory()
@@ -153,6 +157,7 @@ class Orchestration(unittest.TestCase):
patch.start()
self.addCleanup(patch.stop)
self.tests_output = "3 passed\n"
self.worker_result = {}
self.base = base
def commit(self, files):
@@ -170,7 +175,11 @@ class Orchestration(unittest.TestCase):
return subprocess.CompletedProcess(args, 0, stdout=json.dumps({"result": '{"verdict": "pass", "why": "ok"}'}), stderr="")
pathlib.Path(kwargs["cwd"], "bin/pkg/mod.py").write_text("x = 3\n")
pathlib.Path(kwargs["cwd"], "bin/pkg/new.py").write_text("n = 1\n")
kwargs["stdout"].write(result_line() + "\n")
pathlib.Path(kwargs["cwd"], "bin/third").mkdir()
pathlib.Path(kwargs["cwd"], "bin/third/mod.py").write_text("t = 1\n")
subprocess.run(["git", "-C", str(kwargs["cwd"]), "add", "bin/third"], check=True)
commit(kwargs["cwd"], "the worker commits")
kwargs["stdout"].write(result_line(**self.worker_result) + "\n")
return subprocess.CompletedProcess(args, 0)
if args[0] == str(advisor_bench.RUN_HEAVY):
self.calls.append((args, kwargs))
@@ -188,18 +197,31 @@ class Orchestration(unittest.TestCase):
self.assertIn("Do the seventh thing.", claude[2])
self.assertIn(f"`{self.base}`", claude[2])
self.assertIn("todo item 7 ", claude[2])
self.assertEqual(tests, [str(advisor_bench.RUN_HEAVY), *advisor_bench.SUITE, "bin/other", "bin/pkg"])
self.assertEqual(tests, [str(advisor_bench.RUN_HEAVY), *advisor_bench.SUITE, "bin/pkg", "bin/third"])
self.assertEqual((tests_kwargs["capture_output"], tests_kwargs["text"]), (True, True))
self.assertEqual(claude_kwargs["stdout"].name, str(self.work / "7-sonnet-advisor-1.jsonl"))
self.assertEqual(claude_kwargs["stderr"], subprocess.STDOUT)
self.assertEqual(advisor_bench.git(clone, "remote"), "")
self.assertEqual(advisor_bench.git(clone, "rev-parse", "HEAD").strip(), self.base)
self.assertEqual(json.loads((self.work / "7-sonnet-advisor-1.jsonl").read_text())["num_turns"], 9)
self.assertEqual(advisor_bench.git(clone, "rev-parse", "HEAD~").strip(), self.base)
self.assertEqual(json.loads((self.work / "7-sonnet-advisor-1.jsonl").read_text())["total_cost_usd"], 0.5)
diff = (self.work / "7-sonnet-advisor-1.diff").read_text()
self.assertIn("-x = 1\n+x = 3\n", diff)
self.assertIn("+++ b/bin/pkg/new.py", diff)
self.assertIn("+++ b/bin/third/mod.py", diff)
self.assertEqual((self.work / "7-sonnet-advisor-1.tests").read_text(), "exit 4\n3 passed\n")
def test_run_tests_all_of_bin_where_no_touched_package_is_in_the_clone(self):
with unittest.mock.patch.object(advisor_bench, "suite_targets", return_value=["bin/absent"]):
advisor_bench.run(self.work, "7-sonnet-1")
self.assertEqual(self.calls[1][0][-1:], ["bin"])
def test_run_refuses_a_run_that_ended_in_an_error_before_it_tests(self):
self.worker_result = {"is_error": True, "result": "API Error"}
with self.assertRaisesRegex(ValueError, "API Error"):
advisor_bench.run(self.work, "7-sonnet-1")
self.assertEqual(len(self.calls), 1)
self.assertFalse((self.work / "7-sonnet-1.tests").exists())
def test_run_keeps_the_last_1500_characters_of_the_tests(self):
self.tests_output = "a" * 2000 + "b" * 1500
advisor_bench.run(self.work, "7-sonnet-1")
@@ -251,7 +273,13 @@ class Orchestration(unittest.TestCase):
(self.work / "7-sonnet-advisor-1.jsonl").write_text(result_line(total_cost_usd=2.0) + "\n")
(self.work / "7-sonnet-advisor-1.verdict").write_text(json.dumps({"verdict": "fail"}))
(self.work / "7-sonnet-2.jsonl").write_text(result_line() + "\n")
self.assertEqual(advisor_bench.rows_of(self.work), [{"arm": "sonnet-advisor", "verdict": "fail", "cost": 2.0, "turns": 9, "seconds": 12.0, "advisor_calls": 0}])
self.assertEqual(advisor_bench.rows_of(self.work), [{"arm": "sonnet-advisor", "verdict": "fail", "cost": 2.0, "seconds": 12.0, "advisor_calls": 0}])
def test_rows_of_names_a_run_whose_stream_has_no_result(self):
(self.work / "7-sonnet-1.jsonl").write_text("")
(self.work / "7-sonnet-1.verdict").write_text(json.dumps({"verdict": "pass"}))
with self.assertRaisesRegex(ValueError, "^7-sonnet-1: the stream has no result line$"):
advisor_bench.rows_of(self.work)
def run_main(self, *argv):
out = io.StringIO()