Answer stability round 1: 0 🔴, 4 🟡 (0 misread); a cycle's close takes the task from the cycle, a resolved thread is skipped unread, and tests run git in their own environment
CI / tests (pull_request) Successful in 3m56s

This commit is contained in:
2026-11-19 14:04:11 +01:00
parent aafe75e16f
commit 914b8dc2d4
19 changed files with 143 additions and 87 deletions
+6 -5
View File
@@ -206,13 +206,14 @@ class Blamed(unittest.TestCase):
class Unsupported(unittest.TestCase):
def test_says_which_files_the_config_puts_where_no_commits_are_read(self):
checkout = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory())).resolve()
self.assertIsNone(unsupported(str(checkout), checkout), "no git checkout, so no config names it")
run = functools.partial(git, env=isolated_env())
self.assertIsNone(unsupported(str(checkout), checkout, run), "no git checkout, so no config names it")
folder = configured(self, checkout)
home = folder.parents[1]
self.assertIsNone(unsupported(str(checkout), home))
self.assertIsNone(unsupported(str(checkout), home, run))
config = folder / "config.toml"
config.write_text(config.read_text().replace('"checkout:todo.md"', '"folder:todo.md"').replace('"checkout:docs/reading-cost.md"', '"checkout:records/reading-cost.md"'))
self.assertEqual(unsupported(str(checkout), home), f"the project's config puts {folder}/todo.md, {checkout}/records/reading-cost.md elsewhere than todo.md, docs/comprehension-history.md, docs/reading-cost.md in the checkout, the only paths whose commits are read so far")
self.assertEqual(unsupported(str(checkout), home, run), f"the project's config puts {folder}/todo.md, {checkout}/records/reading-cost.md elsewhere than todo.md, docs/comprehension-history.md, docs/reading-cost.md in the checkout, the only paths whose commits are read so far")
class TracedTo(unittest.TestCase):
@@ -256,8 +257,8 @@ class BranchChunks(unittest.TestCase):
return self.git("rev-parse", "--short", "HEAD").strip()
def read(self, run=None, titles=BARE):
"""branch_chunks over main, run by git_repo's git, or else by run."""
return branch_chunks("main", functools.partial(run or git, cwd=str(self.checkout)), titles)
"""branch_chunks over main, run by git_repo's git, or else by run, in the test's own environment."""
return branch_chunks("main", functools.partial(run or git, cwd=str(self.checkout), env=self.env), titles)
def fixing_history(self):
"""Merges seven chunks onto main, then checks out a branch off it. Returns each of main's chunks by label, with the chunks that fixed it.
+4 -7
View File
@@ -21,8 +21,6 @@ class Run(typing.Protocol):
# Every run is a dataclass, which is what lets closing a cycle replace its fields.
__dataclass_fields__: typing.ClassVar[dict[str, typing.Any]]
@property
def task(self) -> str: ...
@property
def unfinished_evidence(self) -> str: ...
@property
@@ -35,7 +33,6 @@ class Run(typing.Protocol):
class ChunkRun:
"""The run of one chunk, for an entry point that never hands it to another process, such as do-epic."""
cycle_number: int = 0
task: str = ""
# The evidence of the latest unfinished verdict; empty before the chunk's first one, and once it finishes.
unfinished_evidence: str = ""
# Set by each unfinished verdict; None before the chunk's first one, and once it finishes.
@@ -44,19 +41,19 @@ class ChunkRun:
unfinished_run: int = 0
def closed_cycle[R: Run](run: R, project: ProjectRung | None, concluded: Concluded, archive_folder: pathlib.Path, driver: str, loop: Loop) -> R | Stop:
"""The run after a cycle of driver, the entry point, concluded, with the step-up archived; the stop that needs the human, archived, once the chunk has had UNFINISHED_LIMIT unfinished verdicts running."""
def closed_cycle[R: Run](run: R, task: str, project: ProjectRung | None, concluded: Concluded, archive_folder: pathlib.Path, driver: str, loop: Loop) -> R | Stop:
"""The run after a cycle of driver, the entry point, concluded on the chunk's task, with the step-up archived; the stop that needs the human, archived, once the chunk has had UNFINISHED_LIMIT unfinished verdicts running."""
after = after_verdict(run, project, concluded)
# A finished chunk drops its route, so it has no step-up, and its run of unfinished verdicts restarts.
if after.route is None:
return after
# The unfinished verdict, archived as the cycle closed, names the route its cycle ended on; the step-up names the one the next cycle runs on.
path = transcript_file(loop.session)
if path and (failure := archived_step(archive_folder, path, after.task, step_up(concluded.route, after.route), loop.effects)):
if path and (failure := archived_step(archive_folder, path, task, step_up(concluded.route, after.route), loop.effects)):
return Stop(1, failure.reason)
if limit := reached_limit(after, driver):
# Archived with the route the next cycle would run on, so a rerun resumes there.
return archived_stop(archive_folder, loop.session, after.task, after.route, limit, limit.reason, loop.effects)
return archived_stop(archive_folder, loop.session, task, after.route, limit, limit.reason, loop.effects)
return after
+9 -9
View File
@@ -32,23 +32,23 @@ class Steps(unittest.TestCase):
FABLE_MEDIUM = Rung("claude-fable-5-2", "medium")
def test_a_finished_verdict_clears_the_route_and_the_evidence(self):
run_with = chunk_run.ChunkRun(cycle_number=1, task="do it", unfinished_evidence="old", route=Route(self.OPUS))
run_with = chunk_run.ChunkRun(cycle_number=1, unfinished_evidence="old", route=Route(self.OPUS))
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Finished("x"), run_with.route)), dataclasses.replace(run_with, route=None, unfinished_evidence=""))
self.assertEqual(chunk_run.after_verdict(dataclasses.replace(run_with, unfinished_run=3), OPUS_PROJECT, run.Concluded(Finished("x"), run_with.route)).unfinished_run, 0)
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", taken_in=TakenIn(highest_likelihood=4)), run_with.route)), dataclasses.replace(run_with, route=Route(self.FABLE_MEDIUM, unfinished=1), unfinished_evidence="new", unfinished_run=1))
def test_steps_from_the_route_its_cycle_ended_on_which_a_ruling_may_have_raised(self):
run_with = chunk_run.ChunkRun(cycle_number=1, task="do it", unfinished_evidence="", route=Route(self.OPUS, 3, 3, 4))
run_with = chunk_run.ChunkRun(cycle_number=1, unfinished_evidence="", route=Route(self.OPUS, 3, 3, 4))
raised = Route(self.FABLE_MEDIUM, 3, 3, 4, escalated_at=5)
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new"), raised)).route, dataclasses.replace(raised, unfinished=1))
def test_keeps_the_route_its_cycle_ended_on_off_the_ladder(self):
off_ladder = Route(Rung("worker-model"))
run_with = chunk_run.ChunkRun(cycle_number=1, task="do it", unfinished_evidence="")
run_with = chunk_run.ChunkRun(cycle_number=1, unfinished_evidence="")
self.assertEqual(chunk_run.after_verdict(run_with, None, run.Concluded(Unfinished("new", "round 2: 🔴", TakenIn(highest_likelihood=4), entries_read=9), off_ladder)).route, off_ladder)
def test_an_unfinished_verdicts_escalation_marks_the_transcript_entries_the_judge_read(self):
run_with = chunk_run.ChunkRun(cycle_number=1, task="do it", unfinished_evidence="", route=Route(self.FABLE_LOW, 3, 3, 4, escalated_at=2))
run_with = chunk_run.ChunkRun(cycle_number=1, unfinished_evidence="", route=Route(self.FABLE_LOW, 3, 3, 4, escalated_at=2))
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", "round 2: 🔴", entries_read=9), run_with.route)).route, Route(self.FABLE_MEDIUM, 3, 3, 4, escalated_at=9))
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", entries_read=9), run_with.route)).route, Route(self.FABLE_LOW, 3, 3, 4, unfinished=1, escalated_at=2))
@@ -62,14 +62,14 @@ class Steps(unittest.TestCase):
before = Route(self.FABLE_LOW, 3, 3, 4)
with contextlib.redirect_stderr(io.StringIO()):
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW, replace=replace_file))
after = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, task="Do it.", unfinished_evidence=""), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
after = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, unfinished_evidence=""), "Do it.", OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
self.assertEqual(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_MEDIUM, 3, 3, 4))
with contextlib.redirect_stderr(io.StringIO()):
chunk_run.closed_cycle(after, OPUS_PROJECT, run.Concluded(Finished("done"), after.route), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
chunk_run.closed_cycle(after, "Do it.", OPUS_PROJECT, run.Concluded(Finished("done"), after.route), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
self.assertEqual(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_MEDIUM, 3, 3, 4))
blocked = pathlib.Path(home, "blocked")
blocked.write_text("")
stop = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, task="Do it.", unfinished_evidence=""), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), blocked, "a-driver", a_loop(FakeEffects().effects(), session=opened))
stop = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, unfinished_evidence=""), "Do it.", OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), blocked, "a-driver", a_loop(FakeEffects().effects(), session=opened))
self.assertEqual((stop.code, stop.reason.split(":")[0]), (1, "could not archive the transcript"))
def outcomes_after(self, unfinished_run: int, taken_in: TakenIn) -> tuple[object, str]:
@@ -82,7 +82,7 @@ class Steps(unittest.TestCase):
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
before = Route(self.FABLE_LOW, 3, 3, 4)
with contextlib.redirect_stderr(io.StringIO()):
closed = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, task="Do it.", unfinished_evidence="", unfinished_run=unfinished_run), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=taken_in), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
closed = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, unfinished_evidence="", unfinished_run=unfinished_run), "Do it.", OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=taken_in), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
return closed, "".join(file.read_text() for file in archive.rglob("*") if file.is_file() and file.suffix != ".jsonl")
def test_a_step_up_archived_first_for_its_transcript_names_the_chunks_task(self):
@@ -109,5 +109,5 @@ class Steps(unittest.TestCase):
before = Route(self.FABLE_LOW, 3, 3, 4)
with contextlib.redirect_stderr(io.StringIO()):
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW, replace=replace_file))
chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, task="Do it.", unfinished_evidence=""), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_escape_and_impact=6)), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, unfinished_evidence=""), "Do it.", OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_escape_and_impact=6)), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
self.assertEqual(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_LOW, 3, 3, 4, escape_and_impact_taken_in=6))
+1 -1
View File
@@ -271,7 +271,7 @@ def chunk_run(key: str, epic: EpicRun) -> Stop:
concluded = run_cycle(cycle)
if isinstance(concluded, Stop):
return concluded
closed = closed_cycle(run, cycle.project, concluded, cycle.project_archive, chunk_prompt.DRIVER, loop)
closed = closed_cycle(run, cycle.task, cycle.project, concluded, cycle.project_archive, chunk_prompt.DRIVER, loop)
if isinstance(closed, Stop):
return closed
run = closed
+2 -2
View File
@@ -6,7 +6,7 @@ import pathlib
import typing
from do_epic.epic_record import CAP
from chunk_driver.claims import CHUNK_FLAG, epic_chunks, HANDED, LOCK, loop_chunks, project_lock, starting
from chunk_driver.claims import CHUNK_FLAG, epic_chunks, HANDED, loop_chunks, project_lock, starting
from do_epic.backend.protocol import EpicIssue
from failure import Failure
from do_epic.repo_pick import HumanTask, recorded, Repo
@@ -16,7 +16,7 @@ from chunk_pr.approval import PARKS
from do_epic.split import SPLIT_LABEL
from agent_cycle import tmux
# Under the home folder, a folder per epic key holds the epic's LOCK. A chunk without a repo works there.
# Under the home folder, a folder per epic key holds the epic's lock file, which `claims.LOCK` names. A chunk without a repo works there.
STATE = ".local/state/do-epic"
# Why a human's task with a live window waits, as the epic's status says.
HUMAN_WAIT = "it is a human's task; do-epic polls Jira until you close it"
+3 -2
View File
@@ -10,9 +10,10 @@ from do_epic.config import DoEpicConfig, load
from do_epic.epic_record import EpicRecord
from failure import Failure
from loop_effects.doubles import FakeEffects, judged, Worker
from do_epic import findings, finish, main, scheduler
from do_epic import findings, finish, main
from project_config.settings import Finish
from do_epic.test_scheduler import issue
from chunk_driver.claims import LOCK
from do_epic.backend.fake import MemoryEpic
from loop_state.paths import paths_from
from claude_code.session import session_named
@@ -215,7 +216,7 @@ class TheFinishingChunk(DoEpicTest):
def test_stops_where_the_epics_issues_cannot_be_read_or_its_lock_cannot_be_taken(self) -> None:
self.issues = [issue("ABC-20", "do-epic-claude", done=True)]
(self.home / ".local/state/do-epic/ABC-12" / scheduler.LOCK).mkdir(parents=True)
(self.home / ".local/state/do-epic/ABC-12" / LOCK).mkdir(parents=True)
code, _, err = self.run_main(["ABC-12"], FakeEffects(on_terminal=False))
self.assertEqual(code, 1)
self.assertIn("cannot lock ", err)
+20
View File
@@ -26,6 +26,7 @@ from chunk_driver.this_loop import Loop
from do_epic.backend.fake import MemoryEpic
from do_epic import parked
from loop_state.paths import paths_from
from text_files.file_writes import replace_file
from model_routing.route import Route
from model_routing.ladder import Rung
from do_epic.test_do_epic import SETTINGS
@@ -410,6 +411,25 @@ class AChunksRun(DoEpicTest):
self.assertTrue(any("Z Stepped up\n" in text for text in outcomes) and any("Z Stopped: " in text for text in outcomes), outcomes)
self.assertEqual([text for text in outcomes if not text.startswith("Task: Work the Jira issue ABC-21 of the epic ABC-12 as one chunk")], [])
def test_a_step_up_archived_beside_a_new_copy_of_the_transcript_opens_with_the_chunks_task(self) -> None:
fake = FakeEffects(streams=[self.worker()] * 6, claude=[judged("unfinished", escalation="working")] + [judged("unfinished")] * 5, git=UNREAD)
def grown_after_each_copy(path: pathlib.Path, data: bytes) -> Failure | None:
"""Writes the file, and adds a line to the worker's transcript after each copy of it, as a worker that still writes does."""
unwritten = replace_file(path, data)
if path.suffix == ".jsonl":
for transcript in (paths_from(self.home).claude_code_transcripts.parent / "projects" / "-fake").glob("*.jsonl"):
with open(transcript, "a", encoding="utf-8") as handle:
handle.write(json.dumps({"type": "queue-operation", "timestamp": str(path)}) + "\n")
return unwritten
fake.replace_file = grown_after_each_copy
code, out, _ = self.run_main(["--chunk", "ABC-21", "ABC-12"], fake, MODELS)
self.assertEqual(code, 2, out)
outcomes = [path.read_text(encoding="utf-8") for path in (self.home / ".claude/do-epic-transcripts/ABC-12").glob("*/*.outcomes.md")]
self.assertTrue(any("Z Stepped up\n" in text for text in outcomes) and any("Z Stopped: " in text for text in outcomes), outcomes)
self.assertEqual([text for text in outcomes if not text.startswith("Task: Work the Jira issue ABC-21 of the epic ABC-12 as one chunk")], [])
def test_a_finished_chunk_moves_its_issue_to_done_before_the_chunk_it_blocks_starts(self) -> None:
store = MemoryEpic("ABC-12", [self.issues[0], self.issues[1], issue("ABC-22", "do-epic-claude", blockers=("ABC-21",))])
self.backend = lambda config, key: (store, EpicRecord(Description(self.description))) # type: ignore[method-assign,arg-type]
+12 -7
View File
@@ -148,15 +148,14 @@ def waiting_threads(number: str, run: Run) -> list[str] | Failure:
@dataclasses.dataclass(frozen=True)
class Thread:
"""One review thread of a pull request."""
"""One review thread of a pull request that nobody resolved."""
path: str
resolved: bool
# Who wrote its last comment; empty for a thread without one.
last: tuple[str, ...]
def threads_read(printed: str) -> tuple[str, tuple[Thread, ...]] | Failure:
"""Who opened the pull request and its review threads, as gh printed THREADS' answer, with each field checked here."""
"""Who opened the pull request and its review threads that nobody resolved, as gh printed THREADS' answer, with each field checked here."""
unshaped = Failure(f"gh api graphql printed no review threads: {printed.strip()[:200]}")
try:
pull = held(json.loads(printed), "data", "repository", "pullRequest")
@@ -167,16 +166,22 @@ def threads_read(printed: str) -> tuple[str, tuple[Thread, ...]] | Failure:
return unshaped
threads = []
for thread in nodes:
path, resolved, comments = held(thread, "path"), held(thread, "isResolved"), held(thread, "comments", "nodes")
if not isinstance(path, str) or resolved is MISSING or not isinstance(comments, list):
resolved = held(thread, "isResolved")
if resolved is MISSING:
return unshaped
# A resolved thread is skipped unread: its last comment may be a deleted account's, whose author gh prints as null.
if resolved:
continue
path, comments = held(thread, "path"), held(thread, "comments", "nodes")
if not isinstance(path, str) or not isinstance(comments, list):
return unshaped
last = tuple(login for comment in comments if isinstance(login := held(comment, "author", "login"), str))
if len(last) != len(comments):
return unshaped
threads.append(Thread(path, bool(resolved), last))
threads.append(Thread(path, last))
return author, tuple(threads)
def waits(author: str, threads: tuple[Thread, ...]) -> list[str]:
"""The files of the threads that wait for an answer from author, who opened the pull request."""
return sorted({thread.path for thread in threads if not thread.resolved and any(last != author for last in thread.last)})
return sorted({thread.path for thread in threads if any(last != author for last in thread.last)})
+14 -2
View File
@@ -77,8 +77,20 @@ class TheThreadsThatWait(unittest.TestCase):
nodes = [{"isResolved": resolved, "path": path, "comments": {"nodes": [{"author": {"login": last}}]}} for path, resolved, last in threads]
return json.dumps({"data": {"repository": {"pullRequest": {"author": {"login": "me"}, "reviewThreads": {"nodes": nodes}}}}})
def test_parses_what_gh_prints_into_the_author_and_each_thread(self):
self.assertEqual(reviews.threads_read(self.printed(("a.py", False, "rev"), ("b.py", True, "me"))), ("me", (Thread("a.py", False, ("rev",)), Thread("b.py", True, ("me",)))))
def test_parses_what_gh_prints_into_the_author_and_each_thread_nobody_resolved(self):
self.assertEqual(reviews.threads_read(self.printed(("a.py", False, "rev"), ("b.py", True, "me"), ("c.py", False, "me"))), ("me", (Thread("a.py", ("rev",)), Thread("c.py", ("me",)))))
def pull_with(self, *nodes: dict[str, object]) -> str:
return json.dumps({"data": {"repository": {"pullRequest": {"author": {"login": "me"}, "reviewThreads": {"nodes": nodes}}}}})
def test_skips_a_resolved_thread_unread_so_a_deleted_accounts_comment_there_doesnt_fail_the_read(self):
deleted = {"comments": {"nodes": [{"author": None}]}}
printed = self.pull_with({"isResolved": True, "path": "b.py", **deleted}, {"isResolved": True}, {"isResolved": False, "path": "a.py", "comments": {"nodes": [{"author": {"login": "rev"}}]}})
self.assertEqual(reviews.waiting_threads("7", lambda args: printed), ["a.py"])
def test_a_deleted_accounts_last_word_in_a_thread_nobody_resolved_is_a_failure(self):
printed = self.pull_with({"isResolved": False, "path": "a.py", "comments": {"nodes": [{"author": None}]}})
self.assertEqual(reviews.waiting_threads("7", lambda args: printed), Failure(f"gh api graphql printed no review threads: {printed[:200]}"))
def test_a_thread_waits_while_unresolved_with_a_reviewers_last_word(self):
printed = self.printed(("b.py", False, "rev"), ("a.py", False, "rev"), ("a.py", False, "rev"), ("c.py", True, "rev"), ("d.py", False, "me"))
+4 -7
View File
@@ -5,7 +5,6 @@ import pathlib
import subprocess
import sys
import tempfile
import time
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
@@ -25,6 +24,7 @@ from chunk_driver.drivers import LOOP_CLAUDE
from judge_replay import main
from rung_replay.search import Miss, Replay, Tried
from model_routing.ladder import Rung
from rung_replay.doubles import Together
TOOL = pathlib.Path(__file__).resolve().parents[1] / "judge-replay"
TRANSCRIPT = pathlib.Path("/archive/p/c/a.jsonl")
@@ -245,19 +245,16 @@ class Main(unittest.TestCase):
for n in range(2, 8):
(chunk / f"t{n}.jsonl").write_text(json.dumps({"type": "assistant", "message": {"content": [{"type": "text", "text": f"said {FINISHED} {n}"}]}}) + "\n")
(chunk / f"t{n}.outcomes.md").write_text(f"Task: Do it.\n\n2026-10-14T10:00:0{n}Z {FINISHED}\n\nEvidence:\n> e\n")
running, most = [], []
together = Together(4)
def reply(rung, interrupted):
running.append(rung)
most.append(len(running))
time.sleep(0.05)
running.remove(rung)
together(rung)
return self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}")
code, _, err = self.run_main(["claude-opus-5-5"], reply)
self.assertEqual(code, 0, err)
self.assertIn("replaying 8 verdicts on ", err)
self.assertEqual(max(most), 4)
self.assertEqual(set(together.most.values()), {4})
def test_the_prompt_and_the_reply_schema_of_a_rebuilt_run_name_loop_claude_and_the_records_and_show_the_transcript_once(self):
code, _, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
+11 -11
View File
@@ -6,7 +6,7 @@ import shutil
from review_rules.comprehension_history import DIMENSIONS
from failure import Failure
from text_files.markdown_text import FENCE, listed, section
from git_cli.git_repo import git
from git_cli.git_repo import git, Git
from project_config.main import located
from review_rules.review_folder import made_review_folder, review_folder
from project_config.settings import ProjectConfig
@@ -127,14 +127,14 @@ def scoring_folder(worktree: str, home: pathlib.Path) -> pathlib.Path:
return pathlib.Path(review_folder(str(home), worktree)) / "scoring"
def prepare_scoring(worktree: str, home: pathlib.Path) -> str | Failure:
def prepare_scoring(worktree: str, home: pathlib.Path, run: Git = git) -> str | Failure:
"""Makes the scoring run's worktree at worktree's HEAD and writes each seat's prompt beside it, in the chunk's review folder; what it made, as the tool prints it."""
folder = scoring_folder(worktree, home)
# A setup record-scoring never removed, as where the seats failed, would otherwise block every later run.
replaced = folder.exists()
if replaced and isinstance(removed := removed_setup(worktree, home), Failure):
if replaced and isinstance(removed := removed_setup(worktree, home, run), Failure):
return removed
config = located(worktree, home)
config = located(worktree, home, run)
if isinstance(config, Failure):
return config
# Where AGENTS.md is missing, this repo keeps the values in seat-prompts.md: .claude/rules/this-repo-keeps-no-agents-md.md.
@@ -156,9 +156,9 @@ def prepare_scoring(worktree: str, home: pathlib.Path) -> str | Failure:
score = score_bullets(skill)
if isinstance(score, Failure):
return score
listing = git("worktree", "list", "--porcelain", cwd=worktree)
branch = git("symbolic-ref", "--short", "-q", "HEAD", cwd=worktree)
head = git("rev-parse", "--short", "HEAD", cwd=worktree)
listing = run("worktree", "list", "--porcelain", cwd=worktree)
branch = run("symbolic-ref", "--short", "-q", "HEAD", cwd=worktree)
head = run("rev-parse", "--short", "HEAD", cwd=worktree)
if isinstance(listing, Failure):
return listing
if isinstance(head, Failure):
@@ -169,8 +169,8 @@ def prepare_scoring(worktree: str, home: pathlib.Path) -> str | Failure:
return made
checkout = folder / "checkout"
# The seats score the chunk's head, which holds origin's default branch since larv-review §4 step 1, as larv-review → "Scoring run" says.
added = git("worktree", "add", "--quiet", "--detach", str(checkout), "HEAD", cwd=worktree)
tracked = added if isinstance(added, Failure) else git("ls-files", cwd=str(checkout))
added = run("worktree", "add", "--quiet", "--detach", str(checkout), "HEAD", cwd=worktree)
tracked = added if isinstance(added, Failure) else run("ls-files", cwd=str(checkout))
if isinstance(tracked, Failure):
shutil.rmtree(folder, ignore_errors=True)
return tracked
@@ -200,12 +200,12 @@ def prepare_scoring(worktree: str, home: pathlib.Path) -> str | Failure:
return before + f"Scoring worktree, at {head.strip()}: {checkout}\nPrompts, each for one {SEAT_AGENT} agent:\n" + "\n".join(written)
def removed_setup(worktree: str, home: pathlib.Path) -> Failure | None:
def removed_setup(worktree: str, home: pathlib.Path, run: Git = git) -> Failure | None:
"""Removes the scoring worktree and prompts prepare_scoring made for the chunk at worktree, where they exist."""
folder = scoring_folder(worktree, home)
if not folder.exists():
return None
if (folder / "checkout").exists() and isinstance(removed := git("worktree", "remove", "--force", str(folder / "checkout"), cwd=worktree), Failure):
if (folder / "checkout").exists() and isinstance(removed := run("worktree", "remove", "--force", str(folder / "checkout"), cwd=worktree), Failure):
return Failure(f"could not remove the scoring worktree {folder / 'checkout'}: {removed.reason}")
shutil.rmtree(folder, ignore_errors=True)
return None
+4 -2
View File
@@ -1,3 +1,4 @@
import functools
import os
import pathlib
import re
@@ -10,6 +11,7 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from larv_review.scoring_setup import bullets, file_globs, prepare_scoring, project_values, removed_setup, score_bullets
from failure import Failure
from git_cli.git_repo import git
from test_env import isolated_env
from project_config.doubles import write_config
@@ -102,7 +104,7 @@ class PrepareScoring(unittest.TestCase):
def prepare(self):
"""What prepare-scoring returns in the checkout, run in this process so mutation tests see the lines it runs."""
return prepare_scoring(str(self.checkout), self.root)
return prepare_scoring(str(self.checkout), self.root, functools.partial(git, env=self.env))
def run_tool(self, *args):
return subprocess.run([sys.executable, str(TOOL), *args], capture_output=True, cwd=self.checkout, env=self.env, text=True)
@@ -137,7 +139,7 @@ class PrepareScoring(unittest.TestCase):
self.commit()
checkout = self.root / ".local" / "state" / "larv-review" / "proj" / "scoring" / "checkout"
checkout.mkdir(parents=True)
removed = removed_setup(str(self.checkout), self.root)
removed = removed_setup(str(self.checkout), self.root, functools.partial(git, env=self.env))
self.assertIsInstance(removed, Failure)
self.assertTrue(removed.reason.startswith(f"could not remove the scoring worktree {checkout}: "), removed.reason)
self.assertTrue(checkout.exists())
+2 -2
View File
@@ -5,7 +5,7 @@ import re
import typing
from touch_predictions.beside import Beside, chunks_beside
from chunk_driver.claims import CHUNK_FLAG, epic_chunks, HANDED, LOCK, loop_session, project_lock, starting
from chunk_driver.claims import CHUNK_FLAG, epic_chunks, HANDED, loop_session, project_lock, starting
from loop_claude.chunk import chunk_items
from git_cli.git_repo import default_ref
from loop_effects.effects import Effects, say
@@ -161,6 +161,6 @@ def held_beside(chunk: str, root: str, paths: Paths, effects: Effects) -> list[i
def wait_for_starts(root: str, home: pathlib.Path) -> None:
"""Returns once no loop of the project is starting windows: start_chunks holds LOCK while it starts them. Where the lock cannot be taken, it returns at once."""
"""Returns once no loop of the project is starting windows: start_chunks holds the lock file `claims.LOCK` names while it starts them. Where the lock cannot be taken, it returns at once."""
with starting(project_lock(root, home)):
pass
+2 -2
View File
@@ -41,7 +41,7 @@ class Steps:
exec_if_edited: typing.Callable[[Loop, str, LoopRun], Stop | None]
next_cycle: typing.Callable[[LoopRun, Loop], Cycle | Stop]
run_cycle: typing.Callable[[Cycle], Stop | Concluded]
closed_cycle: typing.Callable[[LoopRun, ProjectRung | None, Concluded, pathlib.Path, str, Loop], LoopRun | Stop]
closed_cycle: typing.Callable[[LoopRun, str, ProjectRung | None, Concluded, pathlib.Path, str, Loop], LoopRun | Stop]
check_routing: typing.Callable[[pathlib.Path, Rung, Loop], Stop | None]
@@ -93,7 +93,7 @@ def run_loop(argv: list[str], env: typing.Mapping[str, str], paths: Paths, launc
concluded = steps.run_cycle(cycle)
if isinstance(concluded, Stop):
return concluded
closed = steps.closed_cycle(loop_run, cycle.project, concluded, cycle.project_archive, fresh_prompt.DRIVER, loop)
closed = steps.closed_cycle(loop_run, cycle.task, cycle.project, concluded, cycle.project_archive, fresh_prompt.DRIVER, loop)
if isinstance(closed, Stop):
return closed
loop_run = closed
+4 -4
View File
@@ -194,8 +194,8 @@ class WindowStarts(unittest.TestCase):
def test_a_window_loop_hands_each_step_of_a_cycle_what_the_steps_before_it_made(self):
env = {"TMUX_PANE": "%4", "LOOP_CLAUDE_TRANSCRIPTS": "/t"}
on_ladder = unittest.mock.Mock(project=OPUS_PROJECT, project_archive=pathlib.Path("/t/r"))
off_ladder = unittest.mock.Mock(project=None, project_archive=pathlib.Path("/t/r"), route=unittest.mock.Mock(rung=Rung("my-model")))
on_ladder = unittest.mock.Mock(task="the cycle's task", project=OPUS_PROJECT, project_archive=pathlib.Path("/t/r"))
off_ladder = unittest.mock.Mock(task="the cycle's task", project=None, project_archive=pathlib.Path("/t/r"), route=unittest.mock.Mock(rung=Rung("my-model")))
stop, calls = self.window_loop([on_ladder, off_ladder, Stop(2, "the human is needed")], env)
self.assertEqual(stop, Stop(2, "the human is needed"))
before, first, second, third = (main.LoopRun(number, "/r", "do it", "", unfinished_run=max(number - 1, 0), chunk="item-3") for number in range(4))
@@ -204,12 +204,12 @@ class WindowStarts(unittest.TestCase):
("exec_if_edited", self.loop, "print-1", before),
("next_cycle", first, self.loop),
("run_cycle", on_ladder),
("closed_cycle", first, OPUS_PROJECT, self.unfinished, pathlib.Path("/t/r"), "loop-claude", self.loop),
("closed_cycle", first, "the cycle's task", OPUS_PROJECT, self.unfinished, pathlib.Path("/t/r"), "loop-claude", self.loop),
("check_routing", pathlib.Path("/t"), Rung("claude-opus-5-5", "medium"), self.loop),
("exec_if_edited", self.loop, "print-1", dataclasses.replace(first, unfinished_run=1)),
("next_cycle", second, self.loop),
("run_cycle", off_ladder),
("closed_cycle", second, None, self.unfinished, pathlib.Path("/t/r"), "loop-claude", self.loop),
("closed_cycle", second, "the cycle's task", None, self.unfinished, pathlib.Path("/t/r"), "loop-claude", self.loop),
("check_routing", pathlib.Path("/t"), Rung("my-model"), self.loop),
("exec_if_edited", self.loop, "print-1", dataclasses.replace(second, unfinished_run=2)),
("next_cycle", third, self.loop),
+5 -8
View File
@@ -9,7 +9,6 @@ import shutil
import subprocess
import sys
import tempfile
import time
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
@@ -24,6 +23,7 @@ from failure import Failure
from review_rules.reviewer_context import reviewer_context
from review_rules.review_passes import REVIEWERS
from model_routing.ladder import Rung
from rung_replay.doubles import Together
TOOL = pathlib.Path(__file__).resolve().parents[1] / "review-replay"
REFERENCE = Rung("claude-opus-5-5", "medium")
@@ -306,21 +306,18 @@ class Main(unittest.TestCase):
def test_replays_no_more_rounds_at_once_than_asked_on_any_rung(self):
self.write_chunk("proj", "c2", "2026-10-14T10:30:00Z")
self.limits["at_once"] = 1
self.limits["at_once"] = 2
for argv, rungs in ((["stability", "claude-opus-5-5"], {REFERENCE, Rung("claude-opus-5-5", "low"), Rung("claude-sonnet-5-5", "high")}), (["stability", "claude-opus-5-5", "--omit-claude-md"], {REFERENCE})):
with self.subTest(argv=argv):
running, most = [], {}
together = Together(2)
def review(rung):
running.append(rung)
most[rung] = max(most.get(rung, 0), len(running))
time.sleep(0.05)
running.remove(rung)
together(rung)
return {"type": "result", "result": f"review on {rung.model} {rung.effort}"}
code, _, err = self.run_main(argv, lambda rung: {"found": [0, 1]}, review=review)
self.assertEqual(code, 0, err)
self.assertEqual(most, dict.fromkeys(rungs, 1))
self.assertEqual(together.most, dict.fromkeys(rungs, 2))
def test_without_claude_md_replays_each_round_three_times_with_it_and_once_without_on_the_project_rung(self):
finds = iter(([0, 1], [0], [0, 1], [1]))
+8 -7
View File
@@ -1,6 +1,7 @@
import collections
import contextlib
import dataclasses
import functools
import io
import json
import os
@@ -319,7 +320,7 @@ class BranchHistory(unittest.TestCase):
folder = configured(self, checkout)
config = folder / "config.toml"
config.write_text(config.read_text().replace('"checkout:todo.md"', '"folder:todo.md"'))
found = archive.branch_history(str(checkout), folder.parents[1], BARE, git_repo.git)
found = archive.branch_history(str(checkout), folder.parents[1], BARE, functools.partial(git_repo.git, env=isolated_env()))
assert isinstance(found, Failure), found
self.assertEqual(found.reason, f"its chunks' items and later fixes are not counted: the project's config puts {folder}/todo.md elsewhere than todo.md, docs/comprehension-history.md, docs/reading-cost.md in the checkout, the only paths whose commits are read so far")
@@ -365,7 +366,7 @@ class Archive(unittest.TestCase):
env = {**isolated_env(), "GIT_CONFIG_GLOBAL": os.devnull, "HOME": str(home)}
self.ship_then_fix(home / "code" / "log", env)
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
found = archive.measure_archive(self.root, str(home), "", git_repo.git)
found = archive.measure_archive(self.root, str(home), "", functools.partial(git_repo.git, env=env))
self.assertEqual([(chunk.dollars["project"].keys(), chunk.fixed_later) for chunk in found.chunks], [({"log": 0}.keys(), 1)])
def test_reads_a_projects_round_titles_and_its_branchs_history_through_its_configs_templates(self):
@@ -376,9 +377,9 @@ class Archive(unittest.TestCase):
config = write_config(home, home / "code" / "log") / "config.toml"
worker = [message(f"m{n}", "2026-10-14T09:00:00Z", call("Bash", command=f"git commit -m '{round_title.format(n)}'")) for n in (1, 2)]
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), worker)
bare = archive.measure_archive(self.root, str(home), "", git_repo.git)
bare = archive.measure_archive(self.root, str(home), "", functools.partial(git_repo.git, env=env))
config.write_text(config.read_text() + 'commit-title = "PAY-{id} - {summary}"\n')
templated = archive.measure_archive(self.root, str(home), "", git_repo.git)
templated = archive.measure_archive(self.root, str(home), "", functools.partial(git_repo.git, env=env))
self.assertEqual([(chunk.rounds_past_first, chunk.fixed_later) for chunk in bare.chunks], [(0, 0)])
self.assertEqual([(chunk.rounds_past_first, chunk.fixed_later) for chunk in templated.chunks], [(1, 1)])
@@ -389,7 +390,7 @@ class Archive(unittest.TestCase):
config = write_config(home, home / "code" / "log") / "config.toml"
config.write_text(config.read_text() + 'commit-title = "PAY-{id}"\n')
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
found = archive.measure_archive(self.root, str(home), "", git_repo.git)
found = archive.measure_archive(self.root, str(home), "", functools.partial(git_repo.git, env=env))
self.assertEqual(found.chunks, [])
self.assertEqual(len(found.unread), 1)
self.assertRegex(found.unread[0], r"^log: none of its chunks is measured: .*`commit-title` is 'PAY-\{id\}'")
@@ -403,7 +404,7 @@ class Archive(unittest.TestCase):
def recording(*args, **options):
made.append((args[0], options))
return git_repo.git(*args, **options)
return git_repo.git(*args, **options, env=env)
archive.measure_archive(self.root, str(home), "", recording)
self.assertTrue(all(options.get("guarded") is True for _, options in made))
read = [options.get("timeout") for command, options in made if command in ("log", "ls-tree", "show", "diff", "blame")]
@@ -416,7 +417,7 @@ class Archive(unittest.TestCase):
(home / "code" / "log").mkdir(parents=True)
subprocess.run(["git", "init", "-q"], cwd=home / "code" / "log", check=True, env=env, capture_output=True)
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
found = archive.measure_archive(self.root, str(home), "", git_repo.git)
found = archive.measure_archive(self.root, str(home), "", functools.partial(git_repo.git, env=env))
self.assertEqual(found.unread, ("log: no origin remote, and no main or master branch to take as the default branch",))
def test_names_each_project_whose_history_it_cannot_read(self):
+26
View File
@@ -0,0 +1,26 @@
"""Test doubles for what a replay runs at once."""
import collections
import threading
import typing
class Together:
"""What a fake replay calls to show how many replays run at once: each call returns once `parties` calls are inside it together, so it neither sleeps nor reads a clock."""
def __init__(self, parties: int) -> None:
# The timeout only ends a test whose replays never run `parties` at once: it breaks the barrier, which raises in every call.
self.barrier = threading.Barrier(parties, timeout=60)
self.lock = threading.Lock()
self.inside = 0
# The most calls inside at once, by the key each call gave.
self.most: dict[typing.Hashable, int] = collections.defaultdict(int)
def __call__(self, key: typing.Hashable) -> None:
with self.lock:
self.inside += 1
self.most[key] = max(self.most[key], self.inside)
try:
self.barrier.wait()
finally:
with self.lock:
self.inside -= 1
+6 -9
View File
@@ -5,7 +5,6 @@ import pathlib
import subprocess
import sys
import tempfile
import time
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
@@ -18,6 +17,7 @@ from failure import Failure
from claude_code.reply import Limited
from model_routing.ladder_file import read_ladder, SHIPPED
from model_routing.ladder import Rung
from rung_replay.doubles import Together
LADDER = read_ladder(SHIPPED)
REFERENCE = Rung("claude-opus-5-5", "medium")
@@ -63,18 +63,15 @@ class Replayed(unittest.TestCase):
def test_says_what_it_replays_on_each_rung_and_replays_no_more_cases_at_once_than_asked(self):
high = Rung("claude-fable-5-2", "low")
said, running, most = [], [], {}
said, together = [], Together(2)
def replay_one(case, rung):
running.append(case)
most[rung] = max(most.get(rung, 0), len(running))
time.sleep(0.02)
running.remove(case)
together(rung)
return {0}
replayed(["a", "b", "c"], REFERENCE, [MIDDLE], replay_one, lambda _, reply: sorted(reply), "verdicts", said.append, 1, [high])
self.assertEqual(said, [f"replaying 3 verdicts on {REFERENCE}", f"replaying 3 verdicts on {high}", f"replaying 3 verdicts on {MIDDLE}"])
self.assertEqual(most, {REFERENCE: 1, high: 1, MIDDLE: 1})
replayed(["a", "b", "c", "d"], REFERENCE, [MIDDLE], replay_one, lambda _, reply: sorted(reply), "verdicts", said.append, 2, [high])
self.assertEqual(said, [f"replaying 4 verdicts on {REFERENCE}", f"replaying 4 verdicts on {high}", f"replaying 4 verdicts on {MIDDLE}"])
self.assertEqual(together.most, {REFERENCE: 2, high: 2, MIDDLE: 2})
def test_stops_at_a_usage_limit(self):
limit = Limited(None, "limit")