Answer principles round 2: 0 🔴, 2 🟡 (0 misread); workflow-bench hands its effects in as arguments, and no Python file of the repo patches a global
CI / tests (pull_request) Successful in 3m46s

This commit is contained in:
2026-11-19 12:55:35 +01:00
parent cb3135bc8c
commit 762614353c
18 changed files with 335 additions and 231 deletions
+1 -1
View File
@@ -219,7 +219,7 @@ Install puts only `loop-claude` on `PATH`; call the others by their path, such a
### Packages under `bin/`
`bin/` is the import root: each launcher puts it on `sys.path`, so a module imports another by its path under `bin/`, such as `from claude_code.entries import parse_entry`. Each tool's package holds its tests. Beside the packages, `bin/`'s root holds only the launchers, the tools of one file with their tests, `test_env.py`, `failure.py` and `setup.cfg`. `failure.py`'s `Failure`, which modules across `bin/` return for a failed step, belongs to no single package. `agent_cycle/`, `atlassian/`, `branch_chunks/`, `chunk_driver/`, `claude_code/`, `command_lines/`, `forge/`, `git_cli/`, `loop_effects/`, `loop_state/`, `machine_load/`, `model_routing/`, `notifier/`, `plan/`, `plan_store/`, `record_store/`, `release_plan/`, `review_rules/`, `rung_replay/`, `text_files/` and `transcript_archive/` serve more than one tool. No package imports one that imports it back, and a package's `doubles.py` holds the test doubles that other packages' tests use, so no module imports a `test_*` module of another package; `repo_checks/test_package_cycles.py` checks both. No test swaps a global with `mock.patch`: the code under test takes each effect as an argument, and the test hands it a double, which `repo_checks/test_no_patched_globals.py` checks.
`bin/` is the import root: each launcher puts it on `sys.path`, so a module imports another by its path under `bin/`, such as `from claude_code.entries import parse_entry`. Each tool's package holds its tests. Beside the packages, `bin/`'s root holds only the launchers, the tools of one file with their tests, `test_env.py`, `failure.py` and `setup.cfg`. `failure.py`'s `Failure`, which modules across `bin/` return for a failed step, belongs to no single package. `agent_cycle/`, `atlassian/`, `branch_chunks/`, `chunk_driver/`, `claude_code/`, `command_lines/`, `forge/`, `git_cli/`, `loop_effects/`, `loop_state/`, `machine_load/`, `model_routing/`, `notifier/`, `plan/`, `plan_store/`, `record_store/`, `release_plan/`, `review_rules/`, `rung_replay/`, `text_files/` and `transcript_archive/` serve more than one tool. No package imports one that imports it back, and a package's `doubles.py` holds the test doubles that other packages' tests use, so no module imports a `test_*` module of another package; `repo_checks/test_package_cycles.py` checks both. No test swaps a global with `mock.patch`: the code under test takes each effect as an argument, and the test hands it a double, which `repo_checks/test_no_patched_globals.py` checks in every Python file of the repo, `workflow-bench/`'s included.
- `adf_codec/` converts markdown to ADF, the rich text of Jira Cloud and Confluence, and back, in a throwaway container running a pinned `@larvit/adf-codec`. It builds the container's image inside a heavy slot.
- `agent_cycle/` runs one cycle (`run.py`): a headless claude worker, then the judge of its transcript. It also holds the interrupt, the tmux window and the routing of the worker's rung. The tool that drives the cycles supplies the prompts and file names of its own workflow (`worker_prompts.py`), and reads which items a worker took in. `loop_claude` and `do_epic` drive it; `agent_cycle` doesn't import either. `doubles.py` holds the doubles its tests share with `chunk_driver`'s.
+12 -2
View File
@@ -13,11 +13,12 @@ PAGE = 100
MAX_PAGES = 100
# The key of the status category Jira Cloud gives every done status.
DONE = "done"
# How a Failure names an answer in which a field is missing or of another type.
SHAPE = "is not in the shape Jira Cloud's REST API version 3 documents"
# What a failed create ends with: a request Jira never answered, or answered brokenly, may still have filed the issue.
PERHAPS_FILED = "unless Jira refused the request, the issue may exist: look under {under} before filing it again"
# What a failed move ends with, for the same reason.
SHAPE = "is not in the shape Jira Cloud's REST API version 3 documents"
PERHAPS_MOVED ="unless Jira refused the request, the issue may have moved: look at {key} before finishing the item again"
PERHAPS_MOVED = "unless Jira refused the request, the issue may have moved: look at {key} before finishing the item again"
@dataclasses.dataclass(frozen=True)
@@ -72,6 +73,15 @@ def offer_of(raw: object) -> tuple[str, str, str, str] | None:
return (id_, name, status, key) if isinstance(id_, str) and isinstance(name, str) and isinstance(status, str) and isinstance(key, str) else None
def status_category(issue: object) -> str | Failure:
"""The key of the category of the status in Jira's answer about one issue, or how the answer is not in the shape Jira documents."""
fields = issue.get("fields") if isinstance(issue, dict) else None
status = fields.get("status") if isinstance(fields, dict) else None
category = status.get("statusCategory") if isinstance(status, dict) else None
key = category.get("key") if isinstance(category, dict) else None
return key if isinstance(key, str) else Failure("its status category has no key that is text")
def finishing_transition(site: str, key: str, finish: Finish, authorization: str, http: Http) -> str | Failure:
"""The ID of the transition finish names, as Jira offers it now for the issue named key, which is a plan item or an epic; or why none of those offered finishes it."""
offered = http("GET", f"{site}/rest/api/3/issue/{key}/transitions", authorization, None)
+10
View File
@@ -53,5 +53,15 @@ class TheHeadOfAnEntry(unittest.TestCase):
entry_of({"key": "ABC-1", "fields": Faulty()}, "plan")
class TheCategoryOfAnIssuesStatus(unittest.TestCase):
def test_is_the_key_jira_gives_the_category(self):
self.assertEqual(jira.status_category({"fields": {"status": {"statusCategory": {"key": "done"}}}}), "done")
def test_an_answer_in_another_shape_is_a_failure_naming_the_field(self):
for wrong in (["ABC-1"], {}, {"fields": []}, {"fields": {}}, {"fields": {"status": "Done"}}, {"fields": {"status": {}}}, {"fields": {"status": {"statusCategory": "done"}}}, {"fields": {"status": {"statusCategory": {}}}}, {"fields": {"status": {"statusCategory": {"key": 3}}}}):
with self.subTest(wrong=wrong):
self.assertEqual(jira.status_category(wrong), Failure("its status category has no key that is text"))
if __name__ == "__main__":
unittest.main()
+5 -6
View File
@@ -5,7 +5,7 @@ import typing
from adf_codec.convert import adfs_to_markdown, markdown_to_adf, Refused
from atlassian.client import basic_auth, Http, jira_json, ToAdf, ToMarkdown
from atlassian.jira import DONE, entry_of, finishing_transition, ISSUE_KEY, MAX_PAGES, move_refusal, PERHAPS_FILED, PERHAPS_MOVED, project_of, rank_refusal, searched, unshaped
from atlassian.jira import DONE, entry_of, finishing_transition, ISSUE_KEY, MAX_PAGES, move_refusal, PERHAPS_FILED, PERHAPS_MOVED, project_of, rank_refusal, searched, SHAPE, status_category, unshaped
from project_config.settings import Epic, Finish, JIRA_FINISH_STATUS, JIRA_FINISH_TRANSITION
from do_epic.backend.protocol import EpicIssue, no_child, no_parent, no_words, unfit, unfit_label
from failure import Failure
@@ -127,11 +127,10 @@ class JiraEpic:
answered = self.http("GET", f"{self.epic.site}/rest/api/3/issue/{self.epic_key}?fields=status", authorization, None)
if isinstance(answered, Failure):
return Failure(f"cannot read {self.epic_key}'s status: {answered.reason}")
try:
category = answered["fields"]["status"]["statusCategory"]["key"] # type: ignore[index]
except (KeyError, TypeError) as failure:
return Failure(f"{self.epic_key}'s status is not in the shape Jira Cloud's REST API version 3 documents: {failure!r}")
return bool(category == DONE)
category = status_category(answered)
if isinstance(category, Failure):
return Failure(f"{self.epic_key}'s status {SHAPE}: {category.reason}")
return category == DONE
def close(self, summary: str) -> None | Failure:
if self.finish is None:
+1 -1
View File
@@ -212,7 +212,7 @@ class Jira(Case):
self.assertEqual(self.new_store().close("Shipped."), Failure("posted the summary on ABC-12, but cannot move ABC-12 by 'Mark as done': Jira answered 503; close the epic in Jira rather than posting the summary again"))
def test_says_why_the_epics_status_cannot_be_read(self) -> None:
for answer, reason in ((Failure("Jira answered 503"), "cannot read ABC-12's status: Jira answered 503"), ({"fields": {}}, "ABC-12's status is not in the shape Jira Cloud's REST API version 3 documents: KeyError('status')"), (["ABC-12"], "ABC-12's status is not in the shape Jira Cloud's REST API version 3 documents: TypeError('list indices must be integers or slices, not str')")):
for answer, reason in ((Failure("Jira answered 503"), "cannot read ABC-12's status: Jira answered 503"), ({"fields": {}}, "ABC-12's status is not in the shape Jira Cloud's REST API version 3 documents: its status category has no key that is text"), (["ABC-12"], "ABC-12's status is not in the shape Jira Cloud's REST API version 3 documents: its status category has no key that is text"), ({"fields": {"status": {"statusCategory": {"key": 3}}}}, "ABC-12's status is not in the shape Jira Cloud's REST API version 3 documents: its status category has no key that is text")):
self.jira.answers[("GET", EPIC_STATUS)] = answer
self.assertEqual(self.new_store().closed(), Failure(reason))
+6 -6
View File
@@ -165,8 +165,8 @@ def start_chunks(epic: Epic, issues: list[EpicIssue], finishing: bool, own: str
now = start_chunk(epic, chunk if chunk.issue not in parked else Chunk(chunk.issue, None), own, windows, run)
if isinstance(now, Failure):
return now
if isinstance(now, str):
waits.append(Waits(chunk.issue, now))
if isinstance(now, Waits):
waits.append(now)
else:
windows = now
started.append(chunk.issue)
@@ -176,8 +176,8 @@ def start_chunks(epic: Epic, issues: list[EpicIssue], finishing: bool, own: str
return Started(started, sorted(waits, key=lambda wait: order.index(wait.issue)))
def start_chunk(epic: Epic, chunk: Chunk, own: str | None, windows: list[tmux.Listed], run: tmux.Tmux) -> list[tmux.Listed] | str | Failure:
"""Starts chunk's window beside windows, the epic's session's, and returns the session's windows afterwards; the reason where its repo takes no chunk now, and a Failure where tmux started none, or where the repo's lock cannot be taken. It holds that lock from the check to the start. A chunk without a repo starts in the epic's folder. So does a parked chunk, since it takes no room in its repo."""
def start_chunk(epic: Epic, chunk: Chunk, own: str | None, windows: list[tmux.Listed], run: tmux.Tmux) -> list[tmux.Listed] | Waits | Failure:
"""Starts chunk's window beside windows, the epic's session's, and returns the session's windows afterwards; a Waits with the reason where its repo takes no chunk now, and a Failure where tmux started none, or where the repo's lock cannot be taken. It holds that lock from the check to the start. A chunk without a repo starts in the epic's folder. So does a parked chunk, since it takes no room in its repo."""
session = tmux.session_name(tmux.EPIC, epic.key)
handed = {**{name: value for name, value in epic.env.items() if name.startswith(HANDED)}, PARKS: "1"}
command = [str(epic.launcher), CHUNK_FLAG, chunk.issue, epic.key]
@@ -185,13 +185,13 @@ def start_chunk(epic: Epic, chunk: Chunk, own: str | None, windows: list[tmux.Li
return tmux.start_chunk(tmux.Window(session, chunk.issue, ""), str(epic.folder), handed, command, windows, False, run)
located = epic.locate(chunk.repo)
if isinstance(located, Failure):
return located.reason
return Waits(chunk.issue, located.reason)
checkout, limit = located
with starting(project_lock(str(checkout), epic.home)) as unlocked:
if unlocked:
return unlocked
if reason := refusal(chunk.repo, room_in(str(checkout), limit, (session, own), epic.home, run)):
return reason
return Waits(chunk.issue, reason)
return tmux.start_chunk(tmux.Window(session, chunk.issue, ""), str(checkout), handed, command, windows, False, run)
+49 -15
View File
@@ -72,23 +72,45 @@ class PullView:
said: tuple[str, ...]
def text(value: object) -> str:
"""value, which a forge's answer must hold as a string. Raises TypeError where it is none."""
if not isinstance(value, str):
raise TypeError(f"{value!r} is not a string")
# What `held` gives where the answer lacks the key asked for: no text, so the check of the field refuses it.
MISSING = object()
def held(value: object, *path: str) -> object:
"""What a forge's answer holds along path, one key per level; MISSING where a level is no object or lacks its key."""
for key in path:
value = value.get(key, MISSING) if isinstance(value, dict) else MISSING
return value
def pull_view(printed: str) -> PullView | Failure:
"""The pull request that gh printed, with each field checked here, so what decides on it meets no other shape."""
unshaped = Failure(f"gh pr view printed no review decision: {printed.strip()[:200]}")
try:
pull = json.loads(printed)
author = text(pull["author"]["login"])
reviews = tuple(Review(text(review["author"]["login"]), text(review["state"]), text(review["submittedAt"] or ""), text(review["commit"]["oid"]) if review["state"] == CHANGES_REQUESTED else "") for review in pull["reviews"])
commented = tuple(text(comment["createdAt"]) for comment in pull["comments"] if text(comment["author"]["login"]) == author)
return PullView(bool(pull["isDraft"]), text(pull["reviewDecision"] or ""), reviews, tuple(review.submitted_at for review in reviews if review.author == author) + commented)
except (ValueError, KeyError, TypeError):
return Failure(f"gh pr view printed no review decision: {printed.strip()[:200]}")
except ValueError:
return unshaped
author, draft, decision, listed, comments = held(pull, "author", "login"), held(pull, "isDraft"), held(pull, "reviewDecision") or "", held(pull, "reviews"), held(pull, "comments")
if not isinstance(author, str) or draft is MISSING or not isinstance(decision, str) or not isinstance(listed, list) or not isinstance(comments, list):
return unshaped
reviews = []
for review in listed:
reviewer, state, submitted_at = held(review, "author", "login"), held(review, "state"), held(review, "submittedAt") or ""
commit = held(review, "commit", "oid") if state == CHANGES_REQUESTED else ""
if not isinstance(reviewer, str) or not isinstance(state, str) or not isinstance(submitted_at, str) or not isinstance(commit, str):
return unshaped
reviews.append(Review(reviewer, state, submitted_at, commit))
commented = []
for comment in comments:
commenter, created_at = held(comment, "author", "login"), held(comment, "createdAt")
if not isinstance(commenter, str):
return unshaped
if commenter != author:
continue
if not isinstance(created_at, str):
return unshaped
commented.append(created_at)
return PullView(bool(draft), decision, tuple(reviews), tuple(review.submitted_at for review in reviews if review.author == author) + tuple(commented))
def answered(pull: PullView, head: str) -> bool:
@@ -135,12 +157,24 @@ class Thread:
def threads_read(printed: str) -> tuple[str, tuple[Thread, ...]] | Failure:
"""Who opened the pull request and its review threads, as gh printed THREADS' answer, with each field checked here."""
unshaped = Failure(f"gh api graphql printed no review threads: {printed.strip()[:200]}")
try:
pull = json.loads(printed)["data"]["repository"]["pullRequest"]
threads = tuple(Thread(text(thread["path"]), bool(thread["isResolved"]), tuple(text(comment["author"]["login"]) for comment in thread["comments"]["nodes"])) for thread in pull["reviewThreads"]["nodes"])
return text(pull["author"]["login"]), threads
except (ValueError, KeyError, TypeError):
return Failure(f"gh api graphql printed no review threads: {printed.strip()[:200]}")
pull = held(json.loads(printed), "data", "repository", "pullRequest")
except ValueError:
return unshaped
author, nodes = held(pull, "author", "login"), held(pull, "reviewThreads", "nodes")
if not isinstance(author, str) or not isinstance(nodes, list):
return unshaped
threads = []
for thread in nodes:
path, resolved, comments = held(thread, "path"), held(thread, "isResolved"), held(thread, "comments", "nodes")
if not isinstance(path, str) or resolved is MISSING or not isinstance(comments, list):
return unshaped
last = tuple(login for comment in comments if isinstance(login := held(comment, "author", "login"), str))
if len(last) != len(comments):
return unshaped
threads.append(Thread(path, bool(resolved), last))
return author, tuple(threads)
def waits(author: str, threads: tuple[Thread, ...]) -> list[str]:
+2 -2
View File
@@ -35,7 +35,7 @@ class ThePullRequestAsViewed(unittest.TestCase):
self.assertEqual(reviews.pull_view(printed), PullView(False, CHANGES_REQUESTED, (Review("rev", CHANGES_REQUESTED, "2026-11-02T10:00:00Z", "abc"), Review("me", "PENDING", "", "")), ("", "2026-11-02T11:00:00Z")))
def test_an_answer_in_another_shape_is_a_failure_naming_what_gh_printed(self):
unshaped = ("not json", "[]", json.dumps(PULL | {"reviews": None}), json.dumps(PULL | {"author": None}), json.dumps(PULL | {"reviews": [ASKED | {"commit": None}]}), json.dumps(PULL | {"reviews": [ASKED | {"submittedAt": 7}]}), json.dumps({key: value for key, value in PULL.items() if key != "isDraft"}))
unshaped = ("not json", "[]", json.dumps(PULL | {"reviews": None}), json.dumps(PULL | {"author": None}), json.dumps(PULL | {"reviews": [ASKED | {"commit": None}]}), json.dumps(PULL | {"reviews": [ASKED | {"submittedAt": 7}]}), json.dumps(PULL | {"comments": [{"author": {"login": 7}, "createdAt": "2026-11-02T11:00:00Z"}]}), json.dumps(PULL | {"comments": [{"author": {"login": PULL["author"]["login"]}, "createdAt": None}]}), json.dumps(PULL | {"reviews": [{key: value for key, value in ASKED.items() if key != "submittedAt"}]}), json.dumps({key: value for key, value in PULL.items() if key != "isDraft"}))
for printed in unshaped:
with self.subTest(printed=printed):
self.assertEqual(reviews.pull_view(printed), Failure(f"gh pr view printed no review decision: {printed[:200]}"))
@@ -78,7 +78,7 @@ class TheThreadsThatWait(unittest.TestCase):
self.assertEqual(reviews.waiting_threads("7", lambda args: printed), ["a.py", "b.py"])
def test_an_answer_in_another_shape_is_a_failure_naming_what_gh_printed(self):
for printed in ("not json", "{}", json.dumps({"data": {"repository": {"pullRequest": {"author": {"login": "me"}, "reviewThreads": {"nodes": [{"isResolved": False}]}}}}})):
for printed in ("not json", "{}", json.dumps({"data": {"repository": {"pullRequest": {"author": {"login": "me"}, "reviewThreads": {"nodes": [{"isResolved": False}]}}}}}), json.dumps({"data": {"repository": {"pullRequest": {"author": {"login": "me"}, "reviewThreads": {"nodes": [{"isResolved": False, "path": "a.py", "comments": {"nodes": [{"author": {"login": 7}}]}}]}}}}}), json.dumps({"data": {"repository": {"pullRequest": {"author": {"login": "me"}, "reviewThreads": {"nodes": [{"path": "a.py", "comments": {"nodes": []}}]}}}}})):
with self.subTest(printed=printed):
self.assertEqual(reviews.waiting_threads("7", lambda args: printed), Failure(f"gh api graphql printed no review threads: {printed[:200]}"))
+3 -2
View File
@@ -20,10 +20,11 @@ def patches(source: str) -> list[int]:
class NoPatchedGlobals(unittest.TestCase):
def test_no_python_file_under_bin_patches_a_global(self):
def test_no_python_file_patches_a_global(self):
# A file not yet staged counts too, so a new suite is checked before its first commit.
listed = subprocess.run(["git", "ls-files", "--cached", "--others", "--exclude-standard", "bin/*.py"],capture_output=True, check=True, cwd=ROOT, text=True).stdout.split()
listed = subprocess.run(["git", "ls-files", "--cached", "--others", "--exclude-standard", "*.py"], capture_output=True, check=True, cwd=ROOT, text=True).stdout.split()
self.assertIn("bin/repo_checks/test_no_patched_globals.py", listed)
self.assertIn("workflow-bench/test_bench.py", listed)
patching = {name for name in listed if patches((ROOT / name).read_text(encoding="utf-8"))}
self.assertEqual(patching, set(), "hand the code under test its effect as an argument instead")
+2 -1
View File
@@ -1,9 +1,10 @@
#!/usr/bin/env python3
import pathlib
import sys
import time
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
from wait_for import main
sys.exit(main.main(sys.argv[1:]))
sys.exit(main.main(sys.argv[1:], time))
+45 -46
View File
@@ -13,7 +13,6 @@ Exit 0 on the line, 1 on the timeout, bad arguments or a file it cannot read, sa
import pathlib
import re
import sys
import time
import typing
from failure import Failure
@@ -28,25 +27,41 @@ class Clock(typing.Protocol):
def sleep(self, seconds: float) -> None: ...
def text_of(path: pathlib.Path) -> str | None | Failure:
"""What the file holds; None where it doesn't exist."""
try:
return path.read_text(errors="replace")
except FileNotFoundError:
return None
except OSError as error:
return Failure(f"cannot read {path}: {error.strerror}")
def main(argv: list[str], clock: Clock) -> int:
if argv[:1] in (["-h"], ["--help"]):
print(__doc__.strip())
return 0
parsed = arguments(argv)
done = parsed if isinstance(parsed, Failure) else waited(*parsed, clock)
if isinstance(done, Failure):
print(f"wait-for: {done.reason}", file=sys.stderr)
return 1
print(done)
return 0
def line_in(text: str, pattern: re.Pattern[str] | None) -> str | None:
"""The first line of text matching pattern, or without one its last line; None where there is none."""
lines = text.splitlines()
if pattern is None:
return lines[-1] if lines else None
return next((line for line in lines if pattern.search(line)), None)
def arguments(argv: list[str]) -> tuple[str, str | None, float] | Failure:
operands = list(argv)
timeout = float(TIMEOUT_SECONDS)
if "--timeout" in operands:
at = operands.index("--timeout")
given = operands[at + 1:at + 2]
if not given or not re.fullmatch(r"[0-9]+(\.[0-9]+)?", given[0]) or float(given[0]) <= 0:
return Failure(f"give --timeout a number of seconds above 0; {USAGE}")
timeout = float(given[0])
del operands[at:at + 2]
if not 1 <= len(operands) <= 2:
return Failure(USAGE)
pattern = operands[1] if len(operands) == 2 else None
if pattern is not None:
try:
re.compile(pattern)
except re.error as error:
return Failure(f"cannot read the pattern `{pattern}`: {error}")
return operands[0], pattern, timeout
def waited(file: str, pattern: str | None, timeout: float, clock: Clock = time) -> str | Failure:
def waited(file: str, pattern: str | None, timeout: float, clock: Clock) -> str | Failure:
path = pathlib.Path(file)
compiled = re.compile(pattern) if pattern is not None else None
deadline = clock.monotonic() + timeout
@@ -72,35 +87,19 @@ def waited(file: str, pattern: str | None, timeout: float, clock: Clock = time)
clock.sleep(min(POLL_SECONDS, left))
def arguments(argv: list[str]) -> tuple[str, str | None, float] | Failure:
operands = list(argv)
timeout = float(TIMEOUT_SECONDS)
if "--timeout" in operands:
at = operands.index("--timeout")
given = operands[at + 1:at + 2]
if not given or not re.fullmatch(r"[0-9]+(\.[0-9]+)?", given[0]) or float(given[0]) <= 0:
return Failure(f"give --timeout a number of seconds above 0; {USAGE}")
timeout = float(given[0])
del operands[at:at + 2]
if not 1 <= len(operands) <= 2:
return Failure(USAGE)
pattern = operands[1] if len(operands) == 2 else None
if pattern is not None:
try:
re.compile(pattern)
except re.error as error:
return Failure(f"cannot read the pattern `{pattern}`: {error}")
return operands[0], pattern, timeout
def text_of(path: pathlib.Path) -> str | None | Failure:
"""What the file holds; None where it doesn't exist."""
try:
return path.read_text(errors="replace")
except FileNotFoundError:
return None
except OSError as error:
return Failure(f"cannot read {path}: {error.strerror}")
def main(argv: list[str]) -> int:
if argv[:1] in (["-h"], ["--help"]):
print(__doc__.strip())
return 0
parsed = arguments(argv)
done = parsed if isinstance(parsed, Failure) else waited(*parsed)
if isinstance(done, Failure):
print(f"wait-for: {done.reason}", file=sys.stderr)
return 1
print(done)
return 0
def line_in(text: str, pattern: re.Pattern[str] | None) -> str | None:
"""The first line of text matching pattern, or without one its last line; None where there is none."""
lines = text.splitlines()
if pattern is None:
return lines[-1] if lines else None
return next((line for line in lines if pattern.search(line)), None)
+1 -1
View File
@@ -116,7 +116,7 @@ class WaitFor(unittest.TestCase):
with self.subTest(argv=argv):
out, err = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
code = main.main(argv)
code = main.main(argv, Clock())
self.assertEqual((code, out.getvalue(), err.getvalue()), expected)
def test_arguments_take_the_timeout_anywhere_and_refuse_a_bad_one(self):
+41 -22
View File
@@ -13,11 +13,13 @@ Run each arm of each case once; the table shows a direction, not a significance.
"""
import concurrent.futures
import dataclasses
import functools
import json
import pathlib
import re
import subprocess
import sys
import typing
BENCH = pathlib.Path(__file__).resolve().parent
REPO = BENCH.parent
@@ -52,6 +54,17 @@ VERDICTS = ("pass", "partial", "fail")
JUDGE = Arm("claude-opus-5-5", "medium")
@dataclasses.dataclass(frozen=True)
class Bench:
"""What a bench's steps act on, which the script's entry point hands them."""
# The repo whose archived chunks the bench reruns.
repo: pathlib.Path
# Each case's start commit and the commit it merged at, by todo item.
cases: dict
# Runs claude and the tests, as subprocess.run does.
process: typing.Callable
@dataclasses.dataclass(frozen=True)
class Outcome:
cost: float
@@ -124,51 +137,51 @@ def git(repo, *args):
return subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True).stdout
def fresh_clone(base, dest):
"""A clone of this repo at base, with no remote to push to."""
subprocess.run(["git", "clone", "-q", "--no-local", str(REPO), str(dest)], check=True)
def fresh_clone(repo, base, dest):
"""A clone of repo at base, with no remote to push to."""
subprocess.run(["git", "clone", "-q", "--no-local", str(repo), str(dest)], check=True)
git(dest, "checkout", "-q", "--detach", base)
git(dest, "remote", "remove", "origin")
def run(work, name):
def run(bench, work, name):
"""One worker run of <case>-<arm>-<n>; the files it leaves are in the module docstring."""
case, arm, _ = split_name(name)
base = CASES[case][0]
base = bench.cases[case][0]
clone = work / name
if clone.exists():
raise ValueError(f"{name} already ran: remove {clone}")
fresh_clone(base, clone)
prompt = (BENCH / "advisor-run-prompt.md").read_text().format(base=base, item=case, text=item_text(git(REPO, "show", f"{base}:todo.md"), case))
fresh_clone(bench.repo, base, clone)
prompt = (BENCH / "advisor-run-prompt.md").read_text().format(base=base, item=case, text=item_text(git(bench.repo, "show", f"{base}:todo.md"), case))
with (work / f"{name}.jsonl").open("w") as out:
subprocess.run(
bench.process(
["claude", "-p", prompt, *ARMS[arm].flags(), "--setting-sources", "project", "--permission-mode", "bypassPermissions",
"--no-session-persistence", "--output-format", "stream-json", "--verbose"],
cwd=clone, stdout=out, stderr=subprocess.STDOUT)
read_result((work / f"{name}.jsonl").read_text().splitlines())
save_change(work, name, clone, CASES[case])
save_change(bench, work, name, clone, bench.cases[case])
return f"{name}: ran"
def save_change(work, name, clone, commits):
def save_change(bench, work, name, clone, commits):
"""Save the clone's change since the case's start commit as <name>.diff, and as <name>.tests the result of the tests of the packages that the clone's change or the merged change touches."""
base, merged = commits
git(clone, "add", "--all")
(work / f"{name}.diff").write_text(git(clone, "diff", "--cached", base))
changed = [*git(clone, "diff", "--cached", "--name-only", base).split(), *git(REPO, "diff", "--name-only", base, merged).split()]
changed = [*git(clone, "diff", "--cached", "--name-only", base).split(), *git(bench.repo, "diff", "--name-only", base, merged).split()]
targets = [target for target in suite_targets(changed) if (clone / target).exists()] or ["bin"]
tests = subprocess.run([str(RUN_HEAVY), *SUITE, *targets], cwd=clone, capture_output=True, text=True)
tests = bench.process([str(RUN_HEAVY), *SUITE, *targets], cwd=clone, capture_output=True, text=True)
(work / f"{name}.tests").write_text(f"exit {tests.returncode}\n{tests.stdout[-1500:]}")
def grade(work, name, cases=None):
"""Have the judge rule on one run's diff, blind to its arm, and save the verdict beside it. cases stands in for CASES where another bench grades its own."""
def grade(bench, work, name):
"""Have the judge rule on one run's diff, blind to its arm, and save the verdict beside it."""
case = split_name(name)[0]
base, merged = (cases or CASES)[case]
base, merged = bench.cases[case]
prompt = (BENCH / "advisor-judge-prompt.md").read_text().format(
text=item_text(git(REPO, "show", f"{base}:todo.md"), case), reference=git(REPO, "diff", base, merged, "--", ".", ":!todo.md", ":!docs", ":!CHANGELOG.md"),
text=item_text(git(bench.repo, "show", f"{base}:todo.md"), case), reference=git(bench.repo, "diff", base, merged, "--", ".", ":!todo.md", ":!docs", ":!CHANGELOG.md"),
diff=(work / f"{name}.diff").read_text(), tests=(work / f"{name}.tests").read_text())
reply = subprocess.run(
reply = bench.process(
["claude", "-p", prompt, *JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", ""],
capture_output=True, text=True, check=True).stdout
verdict = verdict_of(reply)
@@ -196,15 +209,21 @@ def rows_of(work):
return rows
def main(argv):
def steps_of(bench):
"""The steps main runs by name, each on bench."""
return {"run": functools.partial(run, bench), "grade": functools.partial(grade, bench)}
def main(argv, steps, say):
"""Runs the step argv names on each run it names, and hands say each result line; or hands say the summary."""
work, step, names = pathlib.Path(argv[0]), argv[1], argv[2:]
if step == "summary":
print(summary(rows_of(work)))
say(summary(rows_of(work)))
return
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
for result in pool.map(lambda name: attempted({"run": run, "grade": grade}[step], work, name), names):
print(result, flush=True)
for result in pool.map(lambda name: attempted(steps[step], work, name), names):
say(result)
if __name__ == "__main__":
main(sys.argv[1:])
main(sys.argv[1:], steps_of(Bench(REPO, CASES, subprocess.run)), functools.partial(print, flush=True))
+12 -10
View File
@@ -14,6 +14,7 @@ other. The bench's inputs are untracked:
Each run leaves its clone in <run>/, its report in <run>.md and its claude stream in logs/.
"""
import concurrent.futures
import functools
import json
import os
import pathlib
@@ -24,7 +25,6 @@ import time
BENCH = pathlib.Path(__file__).resolve().parent
REVIEW_GIT = BENCH / "review_git.py"
SOURCES = pathlib.Path(os.environ["BENCH_SOURCES"])
CASES = {
"cjk": ("151bf38^", "6c49912"),
"35b": ("692433b", "1d369ec"),
@@ -61,10 +61,10 @@ def git(repo, *args):
return subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True, text=True).stdout.strip()
def fresh_clone(case, dest):
def fresh_clone(case, dest, sources):
if dest.exists():
shutil.rmtree(dest)
source = SOURCES / f"{case}-main"
source = sources / f"{case}-main"
base_ref, head_ref = CASES[case]
base = git(source, "rev-parse", base_ref)
head = git(source, "rev-parse", head_ref)
@@ -89,14 +89,15 @@ def rate_limit_reset(log):
return max(resets)
def run(name):
def run(name, sources):
"""One review run of <case>-<version>-<n>, whose case's clone is in the folder sources."""
case, version, _ = name.split("-")
clone = BENCH / name
report = BENCH / f"{name}.md"
for attempt in range(1, ATTEMPTS + 1):
if report.exists():
return f"{name}: done"
base, head = fresh_clone(case, clone)
base, head = fresh_clone(case, clone, sources)
version_dir = BENCH / "versions" / version
prompt = (BENCH / "run-prompt.md").read_text().format(
version_dir=version_dir, base=base, head=head, case=case, report=report, review_git=REVIEW_GIT)
@@ -113,19 +114,20 @@ def run(name):
return f"{name}: FAILED after {ATTEMPTS} attempts"
def attempted(name):
"""run's result line, or a line naming the failure, so one failed run doesn't stop the other runs' results printing."""
def attempted(name, run):
"""run's result line for name, or a line naming the failure, so one failed run doesn't stop the other runs' results printing."""
try:
return run(name)
except (KeyError, ValueError, subprocess.CalledProcessError, OSError) as failure:
return f"{name}: FAILED: {type(failure).__name__}: {failure}"
def main():
def main(names, run):
"""Prints the result line of each run named, which run makes."""
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
for result in pool.map(attempted, sys.argv[1:]):
for result in pool.map(lambda name: attempted(name, run), names):
print(result, flush=True)
if __name__ == "__main__":
main()
main(sys.argv[1:], functools.partial(run, sources=pathlib.Path(os.environ["BENCH_SOURCES"])))
+44 -22
View File
@@ -18,12 +18,15 @@ Run each arm of each case once; the table shows a direction, not a significance.
"""
import collections
import concurrent.futures
import dataclasses
import functools
import json
import pathlib
import shutil
import subprocess
import sys
import threading
import typing
import uuid
import advisor_bench
@@ -54,6 +57,18 @@ START_BUDGET = 0.36
CLAUDE_FLAGS = ["--setting-sources", "project", "--permission-mode", "bypassPermissions", "--output-format", "stream-json", "--verbose"]
@dataclasses.dataclass(frozen=True)
class Handoff:
"""What this bench's steps act on, which the script's entry point hands them."""
bench: advisor_bench.Bench
# Dollars WEAK may spend before the handoff.
budget: float
# Where Claude Code keeps each session's transcript.
projects: pathlib.Path
# Makes the ID of the session `start` opens, as uuid.uuid4 does.
new_session: typing.Callable
def stopped_at_budget(lines):
"""Whether a claude stream's first result line, the worker's own, says the run stopped at --max-budget-usd."""
for line in lines:
@@ -66,34 +81,35 @@ def stopped_at_budget(lines):
return False
def run_prompt(case):
def run_prompt(bench, case):
"""advisor_bench's prompt for the case's item, as the item's text stood at the start commit."""
base = CASES[case][0]
text = advisor_bench.item_text(advisor_bench.git(advisor_bench.REPO, "show", f"{base}:todo.md"), case)
base = bench.cases[case][0]
text = advisor_bench.item_text(advisor_bench.git(bench.repo, "show", f"{base}:todo.md"), case)
return (BENCH / "advisor-run-prompt.md").read_text().format(base=base, item=case, text=text)
def claude(prompt, flags, clone, stream):
def claude(process, prompt, flags, clone, stream):
with stream.open("w") as out:
subprocess.run(["claude", "-p", prompt, *flags, *CLAUDE_FLAGS], cwd=clone, stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT)
process(["claude", "-p", prompt, *flags, *CLAUDE_FLAGS], cwd=clone, stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT)
def start(work, name):
def start(handoff, work, name):
"""WEAK's capped run of <case>-<n>, and the tree and session it leaves the arms; the files are in the module docstring."""
case, number = name.split("-")
base = CASES[case][0]
bench = handoff.bench
base = bench.cases[case][0]
clone, saved = work / name, work / f"{case}-start-{number}"
if clone.exists():
raise ValueError(f"{name} already started: remove {clone}")
advisor_bench.fresh_clone(base, clone)
session = str(uuid.uuid4())
advisor_bench.fresh_clone(bench.repo, base, clone)
session = str(handoff.new_session())
stream = saved.with_suffix(".jsonl")
claude(run_prompt(case), [*WEAK.flags(), "--session-id", session, "--max-budget-usd", str(START_BUDGET)], clone, stream)
claude(bench.process, run_prompt(bench, case), [*WEAK.flags(), "--session-id", session, "--max-budget-usd", str(handoff.budget)], clone, stream)
if not stopped_at_budget(stream.read_text().splitlines()):
raise ValueError("the weak worker finished within its budget, or its run failed, so there is no handoff to bench")
transcripts = list(CLAUDE_PROJECTS.glob(f"*/{session}.jsonl"))
transcripts = list(handoff.projects.glob(f"*/{session}.jsonl"))
if len(transcripts) != 1:
raise ValueError(f"expected one transcript of session {session} under {CLAUDE_PROJECTS}, found {len(transcripts)}")
raise ValueError(f"expected one transcript of session {session} under {handoff.projects}, found {len(transcripts)}")
shutil.copy(transcripts[0], saved.with_suffix(".session.jsonl"))
saved.with_suffix(".session").write_text(str(transcripts[0]))
# The worker may have committed against the bench rules, so the arms get its whole change since base, uncommitted.
@@ -104,9 +120,10 @@ def start(work, name):
return f"{name}: started"
def run(work, name):
def run(handoff, work, name):
"""STRONG's run of one arm, <case>-<arm>-<n>, from the tree and session `start` left."""
case, arm, number = advisor_bench.split_name(name)
bench = handoff.bench
if arm not in ARMS:
raise ValueError(f"the arm {arm!r} is none of {ARMS}")
clone, saved = work / f"{case}-{number}", work / f"{case}-start-{number}"
@@ -122,15 +139,15 @@ def run(work, name):
shutil.copy(saved.with_suffix(".session.jsonl"), transcript)
resumed = ["--resume", transcript.stem]
if arm == "compact":
claude("/compact", [*WEAK.flags(), *resumed], clone, work / f"{name}.compact.jsonl")
claude(bench.process, "/compact", [*WEAK.flags(), *resumed], clone, work / f"{name}.compact.jsonl")
finished(work / f"{name}.compact.jsonl")
if arm == "brief":
claude(run_prompt(case) + "\n" + (BENCH / "handoff-brief-prompt.md").read_text(), [*STRONG.flags(), "--no-session-persistence"], clone, stream)
claude(bench.process, run_prompt(bench, case) + "\n" + (BENCH / "handoff-brief-prompt.md").read_text(), [*STRONG.flags(), "--no-session-persistence"], clone, stream)
else:
prompt = (BENCH / "handoff-resume-prompt.md").read_text().format(rung=f"{STRONG.model} at {STRONG.effort} effort")
claude(prompt, [*STRONG.flags(), *resumed], clone, stream)
claude(bench.process, prompt, [*STRONG.flags(), *resumed], clone, stream)
outcome(work, name)
advisor_bench.save_change(work, name, clone, CASES[case])
advisor_bench.save_change(bench, work, name, clone, bench.cases[case])
return f"{name}: ran"
@@ -185,16 +202,21 @@ def summary(rows):
return "\n".join(lines)
def main(argv):
def steps_of(handoff):
"""The steps main runs by name, each on handoff; the grade is advisor_bench's, over handoff's cases."""
return {"start": functools.partial(start, handoff), "run": functools.partial(run, handoff), "grade": functools.partial(advisor_bench.grade, handoff.bench)}
def main(argv, steps, say):
"""Runs the step argv names on each run it names, and hands say each result line; or hands say the summary."""
work, step, names = pathlib.Path(argv[0]), argv[1], argv[2:]
if step == "summary":
print(summary(rows_of(work)))
say(summary(rows_of(work)))
return
steps = {"start": start, "run": run, "grade": lambda work, name: advisor_bench.grade(work, name, CASES)}
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
for result in pool.map(lambda name: advisor_bench.attempted(steps[step], work, name), names):
print(result, flush=True)
say(result)
if __name__ == "__main__":
main(sys.argv[1:])
main(sys.argv[1:], steps_of(Handoff(advisor_bench.Bench(advisor_bench.REPO, CASES, subprocess.run), START_BUDGET, CLAUDE_PROJECTS, uuid.uuid4)), functools.partial(print, flush=True))
+38 -38
View File
@@ -1,11 +1,9 @@
import contextlib
import io
import json
import pathlib
import shutil
import subprocess
import tempfile
import unittest
import unittest.mock
import advisor_bench
@@ -149,15 +147,11 @@ class Orchestration(unittest.TestCase):
merged = self.commit({"todo.md": TODO + "\nmore\n", "bin/pkg/mod.py": "x = 2\n", "bin/other/mod.py": "z = 1\n", "docs/notes.md": "b\n", "CHANGELOG.md": "b\n"})
self.commit({"bin/pkg/mod.py": "x = 5\n", "bin/later/mod.py": "y = 1\n"})
self.calls = []
real_run = subprocess.run
for patch in (
unittest.mock.patch.object(advisor_bench, "REPO", self.repo), unittest.mock.patch.object(advisor_bench, "CASES", {"7": (base, merged)}),
unittest.mock.patch.object(advisor_bench.subprocess, "run", side_effect=lambda args, **kwargs: self.fake(real_run, args, **kwargs)),
):
patch.start()
self.addCleanup(patch.stop)
self.bench = advisor_bench.Bench(self.repo, {"7": (base, merged)}, self.fake)
self.tests_output = "3 passed\n"
self.worker_result = {}
# Whether the fake worker removes bin/ in place of changing it.
self.worker_removes_bin = False
self.base = base
def commit(self, files):
@@ -168,26 +162,30 @@ class Orchestration(unittest.TestCase):
commit(self.repo, "a commit")
return advisor_bench.git(self.repo, "rev-parse", "HEAD").strip()
def fake(self, real_run, args, **kwargs):
def fake(self, args, **kwargs):
"""Stands in for subprocess.run where the bench runs claude or the tests."""
if args[0] == "claude":
self.calls.append((args, kwargs))
if "--tools" in args:
return subprocess.CompletedProcess(args, 0, stdout=json.dumps({"result": '{"verdict": "pass", "why": "ok"}'}), stderr="")
pathlib.Path(kwargs["cwd"], "bin/pkg/mod.py").write_text("x = 3\n")
pathlib.Path(kwargs["cwd"], "bin/pkg/new.py").write_text("n = 1\n")
pathlib.Path(kwargs["cwd"], "bin/third").mkdir()
pathlib.Path(kwargs["cwd"], "bin/third/mod.py").write_text("t = 1\n")
subprocess.run(["git", "-C", str(kwargs["cwd"]), "add", "bin/third"], check=True)
if self.worker_removes_bin:
shutil.rmtree(pathlib.Path(kwargs["cwd"], "bin"))
else:
pathlib.Path(kwargs["cwd"], "bin/pkg/mod.py").write_text("x = 3\n")
pathlib.Path(kwargs["cwd"], "bin/pkg/new.py").write_text("n = 1\n")
pathlib.Path(kwargs["cwd"], "bin/third").mkdir()
pathlib.Path(kwargs["cwd"], "bin/third/mod.py").write_text("t = 1\n")
subprocess.run(["git", "-C", str(kwargs["cwd"]), "add", "bin/third"], check=True)
commit(kwargs["cwd"], "the worker commits")
kwargs["stdout"].write(result_line(**self.worker_result) + "\n")
return subprocess.CompletedProcess(args, 0)
if args[0] == str(advisor_bench.RUN_HEAVY):
self.calls.append((args, kwargs))
return subprocess.CompletedProcess(args, 4, stdout=self.tests_output, stderr="")
return real_run(args, **kwargs)
raise AssertionError(f"the bench ran {args}")
def test_run_clones_at_the_start_commit_and_saves_the_stream_the_diff_and_the_tests(self):
self.assertEqual(advisor_bench.run(self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: ran")
self.assertEqual(advisor_bench.run(self.bench, self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: ran")
(claude, claude_kwargs), (tests, tests_kwargs) = self.calls
clone = self.work / "7-sonnet-advisor-1"
self.assertEqual((claude_kwargs["cwd"], tests_kwargs["cwd"]), (clone, clone))
@@ -211,39 +209,39 @@ class Orchestration(unittest.TestCase):
self.assertEqual((self.work / "7-sonnet-advisor-1.tests").read_text(), "exit 4\n3 passed\n")
def test_run_tests_all_of_bin_where_no_touched_package_is_in_the_clone(self):
with unittest.mock.patch.object(advisor_bench, "suite_targets", return_value=["bin/absent"]):
advisor_bench.run(self.work, "7-sonnet-1")
self.worker_removes_bin = True
advisor_bench.run(self.bench, self.work, "7-sonnet-1")
self.assertEqual(self.calls[1][0][-1:], ["bin"])
def test_run_refuses_a_run_that_ended_in_an_error_before_it_tests(self):
self.worker_result = {"is_error": True, "result": "API Error"}
with self.assertRaisesRegex(ValueError, "API Error"):
advisor_bench.run(self.work, "7-sonnet-1")
advisor_bench.run(self.bench, self.work, "7-sonnet-1")
self.assertEqual(len(self.calls), 1)
self.assertFalse((self.work / "7-sonnet-1.tests").exists())
def test_run_keeps_the_last_1500_characters_of_the_tests(self):
self.tests_output = "a" * 2000 + "b" * 1500
advisor_bench.run(self.work, "7-sonnet-1")
advisor_bench.run(self.bench, self.work, "7-sonnet-1")
self.assertEqual((self.work / "7-sonnet-1.tests").read_text(), "exit 4\n" + "b" * 1500)
def test_run_refuses_a_name_that_already_ran(self):
advisor_bench.run(self.work, "7-sonnet-1")
advisor_bench.run(self.bench, self.work, "7-sonnet-1")
with self.assertRaisesRegex(ValueError, "7-sonnet-1 already ran: remove .*7-sonnet-1"):
advisor_bench.run(self.work, "7-sonnet-1")
advisor_bench.run(self.bench, self.work, "7-sonnet-1")
def test_a_clone_of_a_missing_repo_fails(self):
with unittest.mock.patch.object(advisor_bench, "REPO", self.work / "missing"), self.assertRaises(subprocess.CalledProcessError):
advisor_bench.fresh_clone(self.base, self.work / "clone")
with self.assertRaises(subprocess.CalledProcessError):
advisor_bench.fresh_clone(self.work / "missing", self.base, self.work / "clone")
def test_git_fails_where_git_does(self):
with self.assertRaises(subprocess.CalledProcessError):
advisor_bench.git(self.repo, "rev-parse", "no-such-revision")
def test_grade_saves_the_verdict_and_shows_the_judge_both_changes_but_not_the_arm(self):
advisor_bench.run(self.work, "7-sonnet-advisor-1")
advisor_bench.run(self.bench, self.work, "7-sonnet-advisor-1")
self.calls.clear()
self.assertEqual(advisor_bench.grade(self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: pass")
self.assertEqual(advisor_bench.grade(self.bench, self.work, "7-sonnet-advisor-1"), "7-sonnet-advisor-1: pass")
((judge, kwargs),) = self.calls
self.assertEqual(judge[:2] + judge[3:], [
"claude", "-p", *advisor_bench.JUDGE.flags(), "--setting-sources", "", "--no-session-persistence", "--output-format", "json", "--tools", ""])
@@ -281,28 +279,30 @@ class Orchestration(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "^7-sonnet-1: the stream has no result line$"):
advisor_bench.rows_of(self.work)
def run_main(self, *argv):
out = io.StringIO()
with contextlib.redirect_stdout(out):
advisor_bench.main([str(self.work), *argv])
return out.getvalue()
def run_main(self, steps, *argv):
said = []
advisor_bench.main([str(self.work), *argv], steps, said.append)
return "".join(f"{line}\n" for line in said)
def test_main_prints_each_runs_line_though_an_earlier_run_failed(self):
def step(work, name):
if name == "7-a-1":
raise ValueError("no")
return f"{name}: ran"
with unittest.mock.patch.object(advisor_bench, "run", step):
self.assertEqual(self.run_main("run", "7-a-1", "7-b-1"), "7-a-1: FAILED: ValueError: no\n7-b-1: ran\n")
self.assertEqual(self.run_main({"run": step}, "run", "7-a-1", "7-b-1"), "7-a-1: FAILED: ValueError: no\n7-b-1: ran\n")
def test_main_grades_on_grade(self):
with unittest.mock.patch.object(advisor_bench, "grade", lambda work, name: f"{work.name} {name}: graded"):
self.assertEqual(self.run_main("grade", "7-a-1"), "work 7-a-1: graded\n")
self.assertEqual(self.run_main({"grade": lambda work, name: f"{work.name} {name}: graded"}, "grade", "7-a-1"), "work 7-a-1: graded\n")
def test_the_steps_run_and_grade_on_the_bench_they_are_given(self):
steps = advisor_bench.steps_of(self.bench)
self.assertEqual(self.run_main(steps, "run", "7-sonnet-1"), "7-sonnet-1: ran\n")
self.assertEqual(self.run_main(steps, "grade", "7-sonnet-1"), "7-sonnet-1: pass\n")
def test_main_prints_the_work_dirs_summary_on_summary(self):
(self.work / "7-sonnet-1.jsonl").write_text(result_line() + "\n")
(self.work / "7-sonnet-1.verdict").write_text(json.dumps({"verdict": "pass"}))
self.assertIn("| sonnet | 1 | 1 | 0 | 0.50 | 0.0 | 12 |", self.run_main("summary"))
self.assertIn("| sonnet | 1 | 1 | 0 | 0.50 | 0.0 | 12 |", self.run_main({}, "summary"))
if __name__ == "__main__":
+22 -12
View File
@@ -1,13 +1,23 @@
import contextlib
import functools
import io
import os
import pathlib
import subprocess
import sys
import unittest
import unittest.mock
with unittest.mock.patch.dict(os.environ, {"BENCH_SOURCES": "/nonexistent"}):
import bench
import bench
def answering(*answers):
"""A run that gives the next of answers on each call, raising the one that is a failure."""
left = list(answers)
def run(name):
answer = left.pop(0)
if isinstance(answer, Exception):
raise answer
return answer
return run
class Main(unittest.TestCase):
@@ -17,27 +27,27 @@ class Main(unittest.TestCase):
raise KeyError("a")
return f"{name}: done"
out = io.StringIO()
with unittest.mock.patch.object(bench, "run", side_effect=run), unittest.mock.patch.object(sys, "argv", ["bench.py", "a-v-1", "b-v-1"]), contextlib.redirect_stdout(out):
bench.main()
with contextlib.redirect_stdout(out):
bench.main(["a-v-1", "b-v-1"], run)
self.assertEqual(out.getvalue().splitlines(), ["a-v-1: FAILED: KeyError: 'a'", "b-v-1: done"])
class Attempted(unittest.TestCase):
def test_returns_the_run_result_of_a_run_that_finished(self):
with unittest.mock.patch.object(bench, "run", return_value="a-v-1: done"):
self.assertEqual(bench.attempted("a-v-1"), "a-v-1: done")
self.assertEqual(bench.attempted("a-v-1", answering("a-v-1: done")), "a-v-1: done")
def test_returns_a_failed_run_as_a_line_naming_the_run_and_its_error_so_later_runs_still_report(self):
failures = [KeyError("nope"), ValueError("not enough values to unpack"), subprocess.CalledProcessError(128, ["git", "clone"]), FileNotFoundError("claude")]
for failure in failures:
with self.subTest(failure=type(failure).__name__), unittest.mock.patch.object(bench, "run", side_effect=[failure, "b-v-1: done"]):
results = [bench.attempted("a-v-1"), bench.attempted("b-v-1")]
with self.subTest(failure=type(failure).__name__):
run = answering(failure, "b-v-1: done")
results = [bench.attempted("a-v-1", run), bench.attempted("b-v-1", run)]
self.assertTrue(results[0].startswith("a-v-1: FAILED: "), results[0])
self.assertIn(type(failure).__name__, results[0])
self.assertEqual(results[1], "b-v-1: done")
def test_a_run_of_an_unknown_case_fails_as_a_line(self):
self.assertRegex(bench.attempted("nope-v-1"), r"^nope-v-1: FAILED: KeyError")
self.assertRegex(bench.attempted("nope-v-1", functools.partial(bench.run, sources=pathlib.Path("/nonexistent"))), r"^nope-v-1: FAILED: KeyError")
if __name__ == "__main__":
+41 -44
View File
@@ -3,7 +3,6 @@ import pathlib
import subprocess
import tempfile
import unittest
import unittest.mock
import uuid
import advisor_bench
@@ -62,17 +61,9 @@ class Orchestration(unittest.TestCase):
self.calls, self.heavy = [], []
self.start_result = {"subtype": "error_max_budget_usd", "is_error": True, "total_cost_usd": 1.0}
self.compact_result, self.strong_result = {}, {}
real_run = subprocess.run
for patch in (
unittest.mock.patch.object(advisor_bench, "REPO", self.repo),
unittest.mock.patch.object(handoff_bench, "CASES", {"7": (self.base, merged)}),
unittest.mock.patch.object(handoff_bench, "START_BUDGET", 0.25),
unittest.mock.patch.object(handoff_bench, "CLAUDE_PROJECTS", self.projects),
unittest.mock.patch.object(handoff_bench.uuid, "uuid4", return_value=SESSION),
unittest.mock.patch.object(advisor_bench.subprocess, "run", side_effect=lambda args, **kwargs: self.fake(real_run, args, **kwargs)),
):
patch.start()
self.addCleanup(patch.stop)
# The prompts the judge was given.
self.judged = []
self.handoff = handoff_bench.Handoff(advisor_bench.Bench(self.repo, {"7": (self.base, merged)}, self.fake), 0.25, self.projects, lambda: SESSION)
self.transcript = self.projects / "slug" / f"{SESSION}.jsonl"
def commit(self, files):
@@ -83,7 +74,11 @@ class Orchestration(unittest.TestCase):
commit(self.repo, "a commit")
return advisor_bench.git(self.repo, "rev-parse", "HEAD").strip()
def fake(self, real_run, args, **kwargs):
def fake(self, args, **kwargs):
"""Stands in for subprocess.run where the bench runs claude or the tests."""
if args[0] == "claude" and "--tools" in args:
self.judged.append(args[2])
return subprocess.CompletedProcess(args, 0, stdout=json.dumps({"result": '{"verdict": "pass"}'}), stderr="")
if args[0] == "claude":
cwd = pathlib.Path(kwargs["cwd"])
# What the run found as it began: the session's transcript and the files under bin/pkg.
@@ -92,7 +87,7 @@ class Orchestration(unittest.TestCase):
if "--max-budget-usd" in args:
(cwd / "bin/pkg/mod.py").write_text("x = 3\n")
# Against the bench rules, the worker commits a part of its change.
real_run(["git", "-C", str(cwd), "add", "bin/pkg/mod.py"], check=True)
subprocess.run(["git", "-C", str(cwd), "add", "bin/pkg/mod.py"], check=True)
commit(cwd, "against the rules")
(cwd / "bin/pkg/new.py").write_text("n = 1\n")
self.transcript.parent.mkdir(parents=True)
@@ -113,14 +108,14 @@ class Orchestration(unittest.TestCase):
if args[0] == str(advisor_bench.RUN_HEAVY):
self.heavy.append(args)
return subprocess.CompletedProcess(args, 0, stdout="3 passed\n", stderr="")
return real_run(args, **kwargs)
raise AssertionError(f"the bench ran {args}")
def started(self):
handoff_bench.start(self.work, "7-1")
handoff_bench.start(self.handoff, self.work, "7-1")
self.calls.clear()
def test_start_runs_the_weak_worker_to_its_budget_and_saves_the_tree_and_the_session(self):
self.assertEqual(handoff_bench.start(self.work, "7-1"), "7-1: started")
self.assertEqual(handoff_bench.start(self.handoff, self.work, "7-1"), "7-1: started")
(claude, kwargs, _), = self.calls
self.assertEqual(kwargs["cwd"], self.work / "7-1")
self.assertEqual((kwargs["stdin"], kwargs["stderr"]), (subprocess.DEVNULL, subprocess.STDOUT))
@@ -142,21 +137,21 @@ class Orchestration(unittest.TestCase):
def test_start_refuses_a_worker_that_finished_within_its_budget(self):
self.start_result = {}
with self.assertRaisesRegex(ValueError, "^the weak worker finished within its budget, or its run failed, so there is no handoff to bench$"):
handoff_bench.start(self.work, "7-1")
handoff_bench.start(self.handoff, self.work, "7-1")
def test_start_refuses_a_name_that_already_started(self):
(self.work / "7-1").mkdir()
with self.assertRaisesRegex(ValueError, "already started"):
handoff_bench.start(self.work, "7-1")
handoff_bench.start(self.handoff, self.work, "7-1")
def test_start_refuses_a_session_whose_transcript_it_cannot_find(self):
self.transcript = self.work / "elsewhere" / f"{SESSION}.jsonl"
with self.assertRaisesRegex(ValueError, "found 0"):
handoff_bench.start(self.work, "7-1")
handoff_bench.start(self.handoff, self.work, "7-1")
def test_the_resume_arm_resumes_the_weak_workers_session_on_the_strong_rung(self):
self.started()
self.assertEqual(handoff_bench.run(self.work, "7-resume-1"), "7-resume-1: ran")
self.assertEqual(handoff_bench.run(self.handoff, self.work, "7-resume-1"), "7-resume-1: ran")
(claude, kwargs, _), = self.calls
self.assertEqual(kwargs["cwd"], self.work / "7-1")
self.assertEqual(claude[:2] + claude[3:], [
@@ -174,7 +169,7 @@ class Orchestration(unittest.TestCase):
def test_the_compact_arm_has_the_weak_model_compact_the_session_before_the_strong_one_resumes_it(self):
self.started()
handoff_bench.run(self.work, "7-compact-1")
handoff_bench.run(self.handoff, self.work, "7-compact-1")
(compact, _, _), (claude, _, found) = self.calls
self.assertEqual(compact[:2] + compact[3:], [
"claude", "-p", *handoff_bench.WEAK.flags(), "--resume", str(SESSION), "--setting-sources", "project",
@@ -187,7 +182,7 @@ class Orchestration(unittest.TestCase):
def test_the_brief_arm_starts_a_fresh_session_with_the_item_and_the_brief(self):
self.started()
self.assertEqual(handoff_bench.run(self.work, "7-brief-1"), "7-brief-1: ran")
self.assertEqual(handoff_bench.run(self.handoff, self.work, "7-brief-1"), "7-brief-1: ran")
(claude, kwargs, _), = self.calls
self.assertEqual(kwargs["cwd"], self.work / "7-1")
self.assertEqual(claude[:2] + claude[3:], [
@@ -200,36 +195,36 @@ class Orchestration(unittest.TestCase):
def test_each_arm_begins_on_the_tree_and_the_session_that_start_left(self):
self.started()
for arm in ("resume", "compact", "brief"):
handoff_bench.run(self.work, f"7-{arm}-1")
handoff_bench.run(self.handoff, self.work, f"7-{arm}-1")
self.assertEqual([found for _, _, found in self.calls if found[0] != "weak\ncompacted\n"], [("weak\n", ["mod.py", "new.py"])] * 3)
self.assertNotIn("done_1.py", (self.work / "7-brief-1.diff").read_text())
def test_run_refuses_an_arm_that_already_ran_or_never_started(self):
with self.assertRaisesRegex(ValueError, "start 7-1 first"):
handoff_bench.run(self.work, "7-brief-1")
handoff_bench.run(self.handoff, self.work, "7-brief-1")
self.started()
handoff_bench.run(self.work, "7-brief-1")
handoff_bench.run(self.handoff, self.work, "7-brief-1")
with self.assertRaisesRegex(ValueError, "already ran"):
handoff_bench.run(self.work, "7-brief-1")
handoff_bench.run(self.handoff, self.work, "7-brief-1")
def test_run_refuses_an_unknown_arm(self):
self.started()
with self.assertRaisesRegex(ValueError, "arm"):
handoff_bench.run(self.work, "7-sonnet-1")
handoff_bench.run(self.handoff, self.work, "7-sonnet-1")
def test_run_refuses_an_arm_whose_run_ended_in_an_error_before_it_tests(self):
self.started()
self.strong_result = {"is_error": True, "result": "overloaded"}
with unittest.mock.patch.object(advisor_bench, "save_change") as save_change:
with self.assertRaisesRegex(ValueError, "ended in an error: overloaded"):
handoff_bench.run(self.work, "7-brief-1")
save_change.assert_not_called()
with self.assertRaisesRegex(ValueError, "ended in an error: overloaded"):
handoff_bench.run(self.handoff, self.work, "7-brief-1")
self.assertEqual(self.heavy, [])
self.assertFalse((self.work / "7-brief-1.diff").exists())
def test_run_refuses_a_compact_arm_whose_compaction_failed_before_the_strong_model_runs(self):
self.started()
self.compact_result = {"is_error": True, "result": "no conversation to compact"}
with self.assertRaisesRegex(ValueError, "^the run ended in an error: no conversation to compact$"):
handoff_bench.run(self.work, "7-compact-1")
handoff_bench.run(self.handoff, self.work, "7-compact-1")
self.assertEqual([args[2] for args, _, _ in self.calls], ["/compact"])
self.assertFalse((self.work / "7-compact-1.jsonl").exists())
@@ -240,18 +235,20 @@ class Orchestration(unittest.TestCase):
handoff_bench.outcome(self.work, "7-brief-1")
def test_main_grades_with_the_handoff_cases_and_prints_the_summary(self):
with unittest.mock.patch("builtins.print") as printed:
handoff_bench.main([str(self.work), "start", "7-1"])
handoff_bench.main([str(self.work), "run", "7-brief-1"])
self.assertEqual([call.args[0] for call in printed.call_args_list], ["7-1: started", "7-brief-1: ran"])
with unittest.mock.patch.object(advisor_bench, "grade", return_value="7-brief-1: pass") as grade, unittest.mock.patch("builtins.print") as printed:
handoff_bench.main([str(self.work), "grade", "7-brief-1"])
grade.assert_called_once_with(self.work, "7-brief-1", handoff_bench.CASES)
printed.assert_called_once_with("7-brief-1: pass", flush=True)
steps, printed = handoff_bench.steps_of(self.handoff), []
handoff_bench.main([str(self.work), "start", "7-1"], steps, printed.append)
handoff_bench.main([str(self.work), "run", "7-brief-1"], steps, printed.append)
self.assertEqual(printed, ["7-1: started", "7-brief-1: ran"])
printed.clear()
handoff_bench.main([str(self.work), "grade", "7-brief-1"], steps, printed.append)
self.assertEqual(printed, ["7-brief-1: pass"])
# The judge saw the handoff case's merged change, so the grade ran over the handoff's cases.
(prompt,) = self.judged
self.assertIn("-x = 1\n+x = 2\n", prompt)
(self.work / "7-brief-1.verdict").write_text(json.dumps({"verdict": "pass"}))
with unittest.mock.patch("builtins.print") as printed:
handoff_bench.main([str(self.work), "summary"])
self.assertIn("| brief | 1 | 1 | 0 | 0.50 | 12 |", printed.call_args.args[0])
printed.clear()
handoff_bench.main([str(self.work), "summary"], steps, printed.append)
self.assertIn("| brief | 1 | 1 | 0 | 0.50 | 12 |", printed[-1])
if __name__ == "__main__":