Net a resumed arm's cost of the start it resumes
This commit is contained in:
@@ -12,7 +12,7 @@ best? Each case is an archived chunk of this repo, as in advisor_bench.py, whose
|
||||
has compacted the session with `/compact`, whose stream it saves as <name>.compact.jsonl. The brief arm starts a
|
||||
session that has the item and handoff-brief-prompt.md, and no transcript.
|
||||
- `grade` is advisor_bench.py's, over these cases. `summary` counts an arm's cost and seconds from its own runs,
|
||||
the compaction included, and leaves out the start, which the arms share.
|
||||
the compaction included, and leaves out the start, which the arms share (`outcome`).
|
||||
Run each arm of each case once; the table shows a direction, not a significance.
|
||||
"""
|
||||
import concurrent.futures
|
||||
@@ -132,11 +132,28 @@ def run(work, name):
|
||||
return f"{name}: ran"
|
||||
|
||||
|
||||
def results(stream):
|
||||
"""The result lines of a claude stream. A resumed run may open with one that holds no work, and a background task's notification adds one."""
|
||||
found = []
|
||||
for line in stream.read_text().splitlines():
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(entry, dict) and entry.get("type") == "result":
|
||||
found.append(entry)
|
||||
return found
|
||||
|
||||
|
||||
def outcome(work, name):
|
||||
"""The cost and seconds of an arm's run: its STRONG stream's, plus its compaction's where it made one."""
|
||||
streams = [stream for stream in (work / f"{name}.compact.jsonl", work / f"{name}.jsonl") if stream.exists()]
|
||||
results = [advisor_bench.read_result(stream.read_text().splitlines()) for stream in streams]
|
||||
return {"cost": sum(result.cost for result in results), "seconds": sum(result.seconds for result in results)}
|
||||
"""The cost and seconds of an arm's run, its compaction included. A resumed session's cost counts from the session's start, so the start's cost comes off it."""
|
||||
case, arm, number = advisor_bench.split_name(name)
|
||||
ran = results(work / f"{name}.jsonl")
|
||||
if not ran or ran[-1].get("is_error"):
|
||||
raise ValueError(f"the run ended in an error: {ran[-1].get('result')}" if ran else "the stream has no result line")
|
||||
compacted = results(work / f"{name}.compact.jsonl") if arm == "compact" else []
|
||||
before = 0 if arm == "brief" else results(work / f"{case}-start-{number}.jsonl")[-1]["total_cost_usd"]
|
||||
return {"cost": ran[-1]["total_cost_usd"] - before, "seconds": sum(entry["duration_ms"] for entry in [*compacted, *ran]) / 1000}
|
||||
|
||||
|
||||
def rows_of(work):
|
||||
|
||||
@@ -58,7 +58,8 @@ class Orchestration(unittest.TestCase):
|
||||
self.base = self.commit({"todo.md": TODO, "bin/pkg/mod.py": "x = 1\n"})
|
||||
merged = self.commit({"bin/pkg/mod.py": "x = 2\n"})
|
||||
self.calls = []
|
||||
self.start_result = {"subtype": "error_max_budget_usd", "is_error": True}
|
||||
self.start_result = {"subtype": "error_max_budget_usd", "is_error": True, "total_cost_usd": 1.0}
|
||||
self.strong_result = {}
|
||||
real_run = subprocess.run
|
||||
for patch in (
|
||||
unittest.mock.patch.object(advisor_bench, "REPO", self.repo),
|
||||
@@ -94,12 +95,14 @@ class Orchestration(unittest.TestCase):
|
||||
kwargs["stdout"].write(result_line(**self.start_result) + "\n")
|
||||
elif args[2] == "/compact":
|
||||
self.transcript.write_text(self.transcript.read_text() + "compacted\n")
|
||||
kwargs["stdout"].write(result_line(total_cost_usd=0.25, duration_ms=3000) + "\n")
|
||||
kwargs["stdout"].write(result_line(total_cost_usd=1.25, duration_ms=3000) + "\n")
|
||||
else:
|
||||
if "--resume" in args:
|
||||
self.transcript.write_text(self.transcript.read_text() + "strong\n")
|
||||
# A resumed session counts its cost from the session's start, and may open with a result that holds no work.
|
||||
kwargs["stdout"].write(result_line(total_cost_usd=1.0, duration_ms=0) + "\n")
|
||||
(cwd / "bin/pkg" / f"done_{len(self.calls)}.py").write_text("d = 1\n")
|
||||
kwargs["stdout"].write(result_line() + "\n")
|
||||
kwargs["stdout"].write(result_line(**self.strong_result, total_cost_usd=3.0 if "--resume" in args else 0.5) + "\n")
|
||||
return subprocess.CompletedProcess(args, 0)
|
||||
if args[0] == str(advisor_bench.RUN_HEAVY):
|
||||
return subprocess.CompletedProcess(args, 0, stdout="3 passed\n", stderr="")
|
||||
@@ -154,6 +157,7 @@ class Orchestration(unittest.TestCase):
|
||||
self.assertIn("+x = 3\n", diff)
|
||||
self.assertIn("+++ b/bin/pkg/done_1.py", diff)
|
||||
self.assertEqual((self.work / "7-resume-1.tests").read_text(), "exit 0\n3 passed\n")
|
||||
self.assertEqual(handoff_bench.outcome(self.work, "7-resume-1"), {"cost": 2.0, "seconds": 12.0})
|
||||
|
||||
def test_the_compact_arm_has_the_weak_model_compact_the_session_before_the_strong_one_resumes_it(self):
|
||||
self.started()
|
||||
@@ -166,7 +170,7 @@ class Orchestration(unittest.TestCase):
|
||||
self.assertEqual(claude[3:7], [*handoff_bench.STRONG.flags()][:4])
|
||||
self.assertIn("--resume", claude)
|
||||
self.assertEqual(found[0], "weak\ncompacted\n")
|
||||
self.assertEqual(handoff_bench.outcome(self.work, "7-compact-1"), {"cost": 0.75, "seconds": 15.0})
|
||||
self.assertEqual(handoff_bench.outcome(self.work, "7-compact-1"), {"cost": 2.0, "seconds": 15.0})
|
||||
|
||||
def test_the_brief_arm_starts_a_fresh_session_with_the_item_and_the_brief(self):
|
||||
self.started()
|
||||
@@ -202,11 +206,18 @@ class Orchestration(unittest.TestCase):
|
||||
|
||||
def test_run_refuses_an_arm_whose_run_ended_in_an_error_before_it_tests(self):
|
||||
self.started()
|
||||
with unittest.mock.patch.object(advisor_bench, "save_change") as save_change, unittest.mock.patch.object(advisor_bench, "read_result", side_effect=ValueError("the run ended in an error")):
|
||||
with self.assertRaisesRegex(ValueError, "ended in an error"):
|
||||
self.strong_result = {"is_error": True, "result": "overloaded"}
|
||||
with unittest.mock.patch.object(advisor_bench, "save_change") as save_change:
|
||||
with self.assertRaisesRegex(ValueError, "ended in an error: overloaded"):
|
||||
handoff_bench.run(self.work, "7-brief-1")
|
||||
save_change.assert_not_called()
|
||||
|
||||
def test_outcome_refuses_a_stream_with_no_result_line(self):
|
||||
self.started()
|
||||
(self.work / "7-brief-1.jsonl").write_text("not json\n[]\n")
|
||||
with self.assertRaisesRegex(ValueError, "no result line"):
|
||||
handoff_bench.outcome(self.work, "7-brief-1")
|
||||
|
||||
def test_main_grades_with_the_handoff_cases_and_prints_the_summary(self):
|
||||
self.started()
|
||||
handoff_bench.run(self.work, "7-brief-1")
|
||||
|
||||
Reference in New Issue
Block a user