Net a resumed arm's cost of the start it resumes

This commit is contained in:
2026-11-17 03:44:53 +01:00
parent e42eb5b288
commit 1bc2ce89e7
2 changed files with 39 additions and 11 deletions
+22 -5
View File
@@ -12,7 +12,7 @@ best? Each case is an archived chunk of this repo, as in advisor_bench.py, whose
has compacted the session with `/compact`, whose stream it saves as <name>.compact.jsonl. The brief arm starts a
session that has the item and handoff-brief-prompt.md, and no transcript.
- `grade` is advisor_bench.py's, over these cases. `summary` counts an arm's cost and seconds from its own runs,
the compaction included, and leaves out the start, which the arms share.
the compaction included, and leaves out the start, which the arms share (`outcome`).
Run each arm of each case once; the table shows a direction, not a significance.
"""
import concurrent.futures
@@ -132,11 +132,28 @@ def run(work, name):
return f"{name}: ran"
def results(stream):
"""The result lines of a claude stream. A resumed run may open with one that holds no work, and a background task's notification adds one."""
found = []
for line in stream.read_text().splitlines():
try:
entry = json.loads(line)
except ValueError:
continue
if isinstance(entry, dict) and entry.get("type") == "result":
found.append(entry)
return found
def outcome(work, name):
"""The cost and seconds of an arm's run: its STRONG stream's, plus its compaction's where it made one."""
streams = [stream for stream in (work / f"{name}.compact.jsonl", work / f"{name}.jsonl") if stream.exists()]
results = [advisor_bench.read_result(stream.read_text().splitlines()) for stream in streams]
return {"cost": sum(result.cost for result in results), "seconds": sum(result.seconds for result in results)}
"""The cost and seconds of an arm's run, its compaction included. A resumed session's cost counts from the session's start, so the start's cost comes off it."""
case, arm, number = advisor_bench.split_name(name)
ran = results(work / f"{name}.jsonl")
if not ran or ran[-1].get("is_error"):
raise ValueError(f"the run ended in an error: {ran[-1].get('result')}" if ran else "the stream has no result line")
compacted = results(work / f"{name}.compact.jsonl") if arm == "compact" else []
before = 0 if arm == "brief" else results(work / f"{case}-start-{number}.jsonl")[-1]["total_cost_usd"]
return {"cost": ran[-1]["total_cost_usd"] - before, "seconds": sum(entry["duration_ms"] for entry in [*compacted, *ran]) / 1000}
def rows_of(work):
+17 -6
View File
@@ -58,7 +58,8 @@ class Orchestration(unittest.TestCase):
self.base = self.commit({"todo.md": TODO, "bin/pkg/mod.py": "x = 1\n"})
merged = self.commit({"bin/pkg/mod.py": "x = 2\n"})
self.calls = []
self.start_result = {"subtype": "error_max_budget_usd", "is_error": True}
self.start_result = {"subtype": "error_max_budget_usd", "is_error": True, "total_cost_usd": 1.0}
self.strong_result = {}
real_run = subprocess.run
for patch in (
unittest.mock.patch.object(advisor_bench, "REPO", self.repo),
@@ -94,12 +95,14 @@ class Orchestration(unittest.TestCase):
kwargs["stdout"].write(result_line(**self.start_result) + "\n")
elif args[2] == "/compact":
self.transcript.write_text(self.transcript.read_text() + "compacted\n")
kwargs["stdout"].write(result_line(total_cost_usd=0.25, duration_ms=3000) + "\n")
kwargs["stdout"].write(result_line(total_cost_usd=1.25, duration_ms=3000) + "\n")
else:
if "--resume" in args:
self.transcript.write_text(self.transcript.read_text() + "strong\n")
# A resumed session counts its cost from the session's start, and may open with a result that holds no work.
kwargs["stdout"].write(result_line(total_cost_usd=1.0, duration_ms=0) + "\n")
(cwd / "bin/pkg" / f"done_{len(self.calls)}.py").write_text("d = 1\n")
kwargs["stdout"].write(result_line() + "\n")
kwargs["stdout"].write(result_line(**self.strong_result, total_cost_usd=3.0 if "--resume" in args else 0.5) + "\n")
return subprocess.CompletedProcess(args, 0)
if args[0] == str(advisor_bench.RUN_HEAVY):
return subprocess.CompletedProcess(args, 0, stdout="3 passed\n", stderr="")
@@ -154,6 +157,7 @@ class Orchestration(unittest.TestCase):
self.assertIn("+x = 3\n", diff)
self.assertIn("+++ b/bin/pkg/done_1.py", diff)
self.assertEqual((self.work / "7-resume-1.tests").read_text(), "exit 0\n3 passed\n")
self.assertEqual(handoff_bench.outcome(self.work, "7-resume-1"), {"cost": 2.0, "seconds": 12.0})
def test_the_compact_arm_has_the_weak_model_compact_the_session_before_the_strong_one_resumes_it(self):
self.started()
@@ -166,7 +170,7 @@ class Orchestration(unittest.TestCase):
self.assertEqual(claude[3:7], [*handoff_bench.STRONG.flags()][:4])
self.assertIn("--resume", claude)
self.assertEqual(found[0], "weak\ncompacted\n")
self.assertEqual(handoff_bench.outcome(self.work, "7-compact-1"), {"cost": 0.75, "seconds": 15.0})
self.assertEqual(handoff_bench.outcome(self.work, "7-compact-1"), {"cost": 2.0, "seconds": 15.0})
def test_the_brief_arm_starts_a_fresh_session_with_the_item_and_the_brief(self):
self.started()
@@ -202,11 +206,18 @@ class Orchestration(unittest.TestCase):
def test_run_refuses_an_arm_whose_run_ended_in_an_error_before_it_tests(self):
self.started()
with unittest.mock.patch.object(advisor_bench, "save_change") as save_change, unittest.mock.patch.object(advisor_bench, "read_result", side_effect=ValueError("the run ended in an error")):
with self.assertRaisesRegex(ValueError, "ended in an error"):
self.strong_result = {"is_error": True, "result": "overloaded"}
with unittest.mock.patch.object(advisor_bench, "save_change") as save_change:
with self.assertRaisesRegex(ValueError, "ended in an error: overloaded"):
handoff_bench.run(self.work, "7-brief-1")
save_change.assert_not_called()
def test_outcome_refuses_a_stream_with_no_result_line(self):
self.started()
(self.work / "7-brief-1.jsonl").write_text("not json\n[]\n")
with self.assertRaisesRegex(ValueError, "no result line"):
handoff_bench.outcome(self.work, "7-brief-1")
def test_main_grades_with_the_handoff_cases_and_prints_the_summary(self):
self.started()
handoff_bench.run(self.work, "7-brief-1")