Answer stability round 1: 0 🔴, 4 🟡 (0 misread); a cycle's close takes the task from the cycle, a resolved thread is skipped unread, and tests run git in their own environment
CI / tests (pull_request) Successful in 3m56s
CI / tests (pull_request) Successful in 3m56s
This commit is contained in:
@@ -206,13 +206,14 @@ class Blamed(unittest.TestCase):
|
||||
class Unsupported(unittest.TestCase):
|
||||
def test_says_which_files_the_config_puts_where_no_commits_are_read(self):
|
||||
checkout = pathlib.Path(self.enterContext(tempfile.TemporaryDirectory())).resolve()
|
||||
self.assertIsNone(unsupported(str(checkout), checkout), "no git checkout, so no config names it")
|
||||
run = functools.partial(git, env=isolated_env())
|
||||
self.assertIsNone(unsupported(str(checkout), checkout, run), "no git checkout, so no config names it")
|
||||
folder = configured(self, checkout)
|
||||
home = folder.parents[1]
|
||||
self.assertIsNone(unsupported(str(checkout), home))
|
||||
self.assertIsNone(unsupported(str(checkout), home, run))
|
||||
config = folder / "config.toml"
|
||||
config.write_text(config.read_text().replace('"checkout:todo.md"', '"folder:todo.md"').replace('"checkout:docs/reading-cost.md"', '"checkout:records/reading-cost.md"'))
|
||||
self.assertEqual(unsupported(str(checkout), home), f"the project's config puts {folder}/todo.md, {checkout}/records/reading-cost.md elsewhere than todo.md, docs/comprehension-history.md, docs/reading-cost.md in the checkout, the only paths whose commits are read so far")
|
||||
self.assertEqual(unsupported(str(checkout), home, run), f"the project's config puts {folder}/todo.md, {checkout}/records/reading-cost.md elsewhere than todo.md, docs/comprehension-history.md, docs/reading-cost.md in the checkout, the only paths whose commits are read so far")
|
||||
|
||||
|
||||
class TracedTo(unittest.TestCase):
|
||||
@@ -256,8 +257,8 @@ class BranchChunks(unittest.TestCase):
|
||||
return self.git("rev-parse", "--short", "HEAD").strip()
|
||||
|
||||
def read(self, run=None, titles=BARE):
|
||||
"""branch_chunks over main, run by git_repo's git, or else by run."""
|
||||
return branch_chunks("main", functools.partial(run or git, cwd=str(self.checkout)), titles)
|
||||
"""branch_chunks over main, run by git_repo's git, or else by run, in the test's own environment."""
|
||||
return branch_chunks("main", functools.partial(run or git, cwd=str(self.checkout), env=self.env), titles)
|
||||
|
||||
def fixing_history(self):
|
||||
"""Merges seven chunks onto main, then checks out a branch off it. Returns each of main's chunks by label, with the chunks that fixed it.
|
||||
|
||||
@@ -21,8 +21,6 @@ class Run(typing.Protocol):
|
||||
# Every run is a dataclass, which is what lets closing a cycle replace its fields.
|
||||
__dataclass_fields__: typing.ClassVar[dict[str, typing.Any]]
|
||||
|
||||
@property
|
||||
def task(self) -> str: ...
|
||||
@property
|
||||
def unfinished_evidence(self) -> str: ...
|
||||
@property
|
||||
@@ -35,7 +33,6 @@ class Run(typing.Protocol):
|
||||
class ChunkRun:
|
||||
"""The run of one chunk, for an entry point that never hands it to another process, such as do-epic."""
|
||||
cycle_number: int = 0
|
||||
task: str = ""
|
||||
# The evidence of the latest unfinished verdict; empty before the chunk's first one, and once it finishes.
|
||||
unfinished_evidence: str = ""
|
||||
# Set by each unfinished verdict; None before the chunk's first one, and once it finishes.
|
||||
@@ -44,19 +41,19 @@ class ChunkRun:
|
||||
unfinished_run: int = 0
|
||||
|
||||
|
||||
def closed_cycle[R: Run](run: R, project: ProjectRung | None, concluded: Concluded, archive_folder: pathlib.Path, driver: str, loop: Loop) -> R | Stop:
|
||||
"""The run after a cycle of driver, the entry point, concluded, with the step-up archived; the stop that needs the human, archived, once the chunk has had UNFINISHED_LIMIT unfinished verdicts running."""
|
||||
def closed_cycle[R: Run](run: R, task: str, project: ProjectRung | None, concluded: Concluded, archive_folder: pathlib.Path, driver: str, loop: Loop) -> R | Stop:
|
||||
"""The run after a cycle of driver, the entry point, concluded on the chunk's task, with the step-up archived; the stop that needs the human, archived, once the chunk has had UNFINISHED_LIMIT unfinished verdicts running."""
|
||||
after = after_verdict(run, project, concluded)
|
||||
# A finished chunk drops its route, so it has no step-up, and its run of unfinished verdicts restarts.
|
||||
if after.route is None:
|
||||
return after
|
||||
# The unfinished verdict, archived as the cycle closed, names the route its cycle ended on; the step-up names the one the next cycle runs on.
|
||||
path = transcript_file(loop.session)
|
||||
if path and (failure := archived_step(archive_folder, path, after.task, step_up(concluded.route, after.route), loop.effects)):
|
||||
if path and (failure := archived_step(archive_folder, path, task, step_up(concluded.route, after.route), loop.effects)):
|
||||
return Stop(1, failure.reason)
|
||||
if limit := reached_limit(after, driver):
|
||||
# Archived with the route the next cycle would run on, so a rerun resumes there.
|
||||
return archived_stop(archive_folder, loop.session, after.task, after.route, limit, limit.reason, loop.effects)
|
||||
return archived_stop(archive_folder, loop.session, task, after.route, limit, limit.reason, loop.effects)
|
||||
return after
|
||||
|
||||
|
||||
|
||||
@@ -32,23 +32,23 @@ class Steps(unittest.TestCase):
|
||||
FABLE_MEDIUM = Rung("claude-fable-5-2", "medium")
|
||||
|
||||
def test_a_finished_verdict_clears_the_route_and_the_evidence(self):
|
||||
run_with = chunk_run.ChunkRun(cycle_number=1, task="do it", unfinished_evidence="old", route=Route(self.OPUS))
|
||||
run_with = chunk_run.ChunkRun(cycle_number=1, unfinished_evidence="old", route=Route(self.OPUS))
|
||||
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Finished("x"), run_with.route)), dataclasses.replace(run_with, route=None, unfinished_evidence=""))
|
||||
self.assertEqual(chunk_run.after_verdict(dataclasses.replace(run_with, unfinished_run=3), OPUS_PROJECT, run.Concluded(Finished("x"), run_with.route)).unfinished_run, 0)
|
||||
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", taken_in=TakenIn(highest_likelihood=4)), run_with.route)), dataclasses.replace(run_with, route=Route(self.FABLE_MEDIUM, unfinished=1), unfinished_evidence="new", unfinished_run=1))
|
||||
|
||||
def test_steps_from_the_route_its_cycle_ended_on_which_a_ruling_may_have_raised(self):
|
||||
run_with = chunk_run.ChunkRun(cycle_number=1, task="do it", unfinished_evidence="", route=Route(self.OPUS, 3, 3, 4))
|
||||
run_with = chunk_run.ChunkRun(cycle_number=1, unfinished_evidence="", route=Route(self.OPUS, 3, 3, 4))
|
||||
raised = Route(self.FABLE_MEDIUM, 3, 3, 4, escalated_at=5)
|
||||
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new"), raised)).route, dataclasses.replace(raised, unfinished=1))
|
||||
|
||||
def test_keeps_the_route_its_cycle_ended_on_off_the_ladder(self):
|
||||
off_ladder = Route(Rung("worker-model"))
|
||||
run_with = chunk_run.ChunkRun(cycle_number=1, task="do it", unfinished_evidence="")
|
||||
run_with = chunk_run.ChunkRun(cycle_number=1, unfinished_evidence="")
|
||||
self.assertEqual(chunk_run.after_verdict(run_with, None, run.Concluded(Unfinished("new", "round 2: 🔴", TakenIn(highest_likelihood=4), entries_read=9), off_ladder)).route, off_ladder)
|
||||
|
||||
def test_an_unfinished_verdicts_escalation_marks_the_transcript_entries_the_judge_read(self):
|
||||
run_with = chunk_run.ChunkRun(cycle_number=1, task="do it", unfinished_evidence="", route=Route(self.FABLE_LOW, 3, 3, 4, escalated_at=2))
|
||||
run_with = chunk_run.ChunkRun(cycle_number=1, unfinished_evidence="", route=Route(self.FABLE_LOW, 3, 3, 4, escalated_at=2))
|
||||
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", "round 2: 🔴", entries_read=9), run_with.route)).route, Route(self.FABLE_MEDIUM, 3, 3, 4, escalated_at=9))
|
||||
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", entries_read=9), run_with.route)).route, Route(self.FABLE_LOW, 3, 3, 4, unfinished=1, escalated_at=2))
|
||||
|
||||
@@ -62,14 +62,14 @@ class Steps(unittest.TestCase):
|
||||
before = Route(self.FABLE_LOW, 3, 3, 4)
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW, replace=replace_file))
|
||||
after = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, task="Do it.", unfinished_evidence=""), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
||||
after = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, unfinished_evidence=""), "Do it.", OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
||||
self.assertEqual(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_MEDIUM, 3, 3, 4))
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
chunk_run.closed_cycle(after, OPUS_PROJECT, run.Concluded(Finished("done"), after.route), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
||||
chunk_run.closed_cycle(after, "Do it.", OPUS_PROJECT, run.Concluded(Finished("done"), after.route), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
||||
self.assertEqual(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_MEDIUM, 3, 3, 4))
|
||||
blocked = pathlib.Path(home, "blocked")
|
||||
blocked.write_text("")
|
||||
stop = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, task="Do it.", unfinished_evidence=""), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), blocked, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
||||
stop = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, unfinished_evidence=""), "Do it.", OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), blocked, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
||||
self.assertEqual((stop.code, stop.reason.split(":")[0]), (1, "could not archive the transcript"))
|
||||
|
||||
def outcomes_after(self, unfinished_run: int, taken_in: TakenIn) -> tuple[object, str]:
|
||||
@@ -82,7 +82,7 @@ class Steps(unittest.TestCase):
|
||||
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
||||
before = Route(self.FABLE_LOW, 3, 3, 4)
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
closed = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, task="Do it.", unfinished_evidence="", unfinished_run=unfinished_run), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=taken_in), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
||||
closed = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, unfinished_evidence="", unfinished_run=unfinished_run), "Do it.", OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=taken_in), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
||||
return closed, "".join(file.read_text() for file in archive.rglob("*") if file.is_file() and file.suffix != ".jsonl")
|
||||
|
||||
def test_a_step_up_archived_first_for_its_transcript_names_the_chunks_task(self):
|
||||
@@ -109,5 +109,5 @@ class Steps(unittest.TestCase):
|
||||
before = Route(self.FABLE_LOW, 3, 3, 4)
|
||||
with contextlib.redirect_stderr(io.StringIO()):
|
||||
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW, replace=replace_file))
|
||||
chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, task="Do it.", unfinished_evidence=""), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_escape_and_impact=6)), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
||||
chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, unfinished_evidence=""), "Do it.", OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_escape_and_impact=6)), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
||||
self.assertEqual(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_LOW, 3, 3, 4, escape_and_impact_taken_in=6))
|
||||
|
||||
+1
-1
@@ -271,7 +271,7 @@ def chunk_run(key: str, epic: EpicRun) -> Stop:
|
||||
concluded = run_cycle(cycle)
|
||||
if isinstance(concluded, Stop):
|
||||
return concluded
|
||||
closed = closed_cycle(run, cycle.project, concluded, cycle.project_archive, chunk_prompt.DRIVER, loop)
|
||||
closed = closed_cycle(run, cycle.task, cycle.project, concluded, cycle.project_archive, chunk_prompt.DRIVER, loop)
|
||||
if isinstance(closed, Stop):
|
||||
return closed
|
||||
run = closed
|
||||
|
||||
@@ -6,7 +6,7 @@ import pathlib
|
||||
import typing
|
||||
|
||||
from do_epic.epic_record import CAP
|
||||
from chunk_driver.claims import CHUNK_FLAG, epic_chunks, HANDED, LOCK, loop_chunks, project_lock, starting
|
||||
from chunk_driver.claims import CHUNK_FLAG, epic_chunks, HANDED, loop_chunks, project_lock, starting
|
||||
from do_epic.backend.protocol import EpicIssue
|
||||
from failure import Failure
|
||||
from do_epic.repo_pick import HumanTask, recorded, Repo
|
||||
@@ -16,7 +16,7 @@ from chunk_pr.approval import PARKS
|
||||
from do_epic.split import SPLIT_LABEL
|
||||
from agent_cycle import tmux
|
||||
|
||||
# Under the home folder, a folder per epic key holds the epic's LOCK. A chunk without a repo works there.
|
||||
# Under the home folder, a folder per epic key holds the epic's lock file, which `claims.LOCK` names. A chunk without a repo works there.
|
||||
STATE = ".local/state/do-epic"
|
||||
# Why a human's task with a live window waits, as the epic's status says.
|
||||
HUMAN_WAIT = "it is a human's task; do-epic polls Jira until you close it"
|
||||
|
||||
@@ -10,9 +10,10 @@ from do_epic.config import DoEpicConfig, load
|
||||
from do_epic.epic_record import EpicRecord
|
||||
from failure import Failure
|
||||
from loop_effects.doubles import FakeEffects, judged, Worker
|
||||
from do_epic import findings, finish, main, scheduler
|
||||
from do_epic import findings, finish, main
|
||||
from project_config.settings import Finish
|
||||
from do_epic.test_scheduler import issue
|
||||
from chunk_driver.claims import LOCK
|
||||
from do_epic.backend.fake import MemoryEpic
|
||||
from loop_state.paths import paths_from
|
||||
from claude_code.session import session_named
|
||||
@@ -215,7 +216,7 @@ class TheFinishingChunk(DoEpicTest):
|
||||
|
||||
def test_stops_where_the_epics_issues_cannot_be_read_or_its_lock_cannot_be_taken(self) -> None:
|
||||
self.issues = [issue("ABC-20", "do-epic-claude", done=True)]
|
||||
(self.home / ".local/state/do-epic/ABC-12" / scheduler.LOCK).mkdir(parents=True)
|
||||
(self.home / ".local/state/do-epic/ABC-12" / LOCK).mkdir(parents=True)
|
||||
code, _, err = self.run_main(["ABC-12"], FakeEffects(on_terminal=False))
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("cannot lock ", err)
|
||||
|
||||
@@ -26,6 +26,7 @@ from chunk_driver.this_loop import Loop
|
||||
from do_epic.backend.fake import MemoryEpic
|
||||
from do_epic import parked
|
||||
from loop_state.paths import paths_from
|
||||
from text_files.file_writes import replace_file
|
||||
from model_routing.route import Route
|
||||
from model_routing.ladder import Rung
|
||||
from do_epic.test_do_epic import SETTINGS
|
||||
@@ -410,6 +411,25 @@ class AChunksRun(DoEpicTest):
|
||||
self.assertTrue(any("Z Stepped up\n" in text for text in outcomes) and any("Z Stopped: " in text for text in outcomes), outcomes)
|
||||
self.assertEqual([text for text in outcomes if not text.startswith("Task: Work the Jira issue ABC-21 of the epic ABC-12 as one chunk")], [])
|
||||
|
||||
def test_a_step_up_archived_beside_a_new_copy_of_the_transcript_opens_with_the_chunks_task(self) -> None:
|
||||
fake = FakeEffects(streams=[self.worker()] * 6, claude=[judged("unfinished", escalation="working")] + [judged("unfinished")] * 5, git=UNREAD)
|
||||
|
||||
def grown_after_each_copy(path: pathlib.Path, data: bytes) -> Failure | None:
|
||||
"""Writes the file, and adds a line to the worker's transcript after each copy of it, as a worker that still writes does."""
|
||||
unwritten = replace_file(path, data)
|
||||
if path.suffix == ".jsonl":
|
||||
for transcript in (paths_from(self.home).claude_code_transcripts.parent / "projects" / "-fake").glob("*.jsonl"):
|
||||
with open(transcript, "a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps({"type": "queue-operation", "timestamp": str(path)}) + "\n")
|
||||
return unwritten
|
||||
|
||||
fake.replace_file = grown_after_each_copy
|
||||
code, out, _ = self.run_main(["--chunk", "ABC-21", "ABC-12"], fake, MODELS)
|
||||
self.assertEqual(code, 2, out)
|
||||
outcomes = [path.read_text(encoding="utf-8") for path in (self.home / ".claude/do-epic-transcripts/ABC-12").glob("*/*.outcomes.md")]
|
||||
self.assertTrue(any("Z Stepped up\n" in text for text in outcomes) and any("Z Stopped: " in text for text in outcomes), outcomes)
|
||||
self.assertEqual([text for text in outcomes if not text.startswith("Task: Work the Jira issue ABC-21 of the epic ABC-12 as one chunk")], [])
|
||||
|
||||
def test_a_finished_chunk_moves_its_issue_to_done_before_the_chunk_it_blocks_starts(self) -> None:
|
||||
store = MemoryEpic("ABC-12", [self.issues[0], self.issues[1], issue("ABC-22", "do-epic-claude", blockers=("ABC-21",))])
|
||||
self.backend = lambda config, key: (store, EpicRecord(Description(self.description))) # type: ignore[method-assign,arg-type]
|
||||
|
||||
+12
-7
@@ -148,15 +148,14 @@ def waiting_threads(number: str, run: Run) -> list[str] | Failure:
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Thread:
|
||||
"""One review thread of a pull request."""
|
||||
"""One review thread of a pull request that nobody resolved."""
|
||||
path: str
|
||||
resolved: bool
|
||||
# Who wrote its last comment; empty for a thread without one.
|
||||
last: tuple[str, ...]
|
||||
|
||||
|
||||
def threads_read(printed: str) -> tuple[str, tuple[Thread, ...]] | Failure:
|
||||
"""Who opened the pull request and its review threads, as gh printed THREADS' answer, with each field checked here."""
|
||||
"""Who opened the pull request and its review threads that nobody resolved, as gh printed THREADS' answer, with each field checked here."""
|
||||
unshaped = Failure(f"gh api graphql printed no review threads: {printed.strip()[:200]}")
|
||||
try:
|
||||
pull = held(json.loads(printed), "data", "repository", "pullRequest")
|
||||
@@ -167,16 +166,22 @@ def threads_read(printed: str) -> tuple[str, tuple[Thread, ...]] | Failure:
|
||||
return unshaped
|
||||
threads = []
|
||||
for thread in nodes:
|
||||
path, resolved, comments = held(thread, "path"), held(thread, "isResolved"), held(thread, "comments", "nodes")
|
||||
if not isinstance(path, str) or resolved is MISSING or not isinstance(comments, list):
|
||||
resolved = held(thread, "isResolved")
|
||||
if resolved is MISSING:
|
||||
return unshaped
|
||||
# A resolved thread is skipped unread: its last comment may be a deleted account's, whose author gh prints as null.
|
||||
if resolved:
|
||||
continue
|
||||
path, comments = held(thread, "path"), held(thread, "comments", "nodes")
|
||||
if not isinstance(path, str) or not isinstance(comments, list):
|
||||
return unshaped
|
||||
last = tuple(login for comment in comments if isinstance(login := held(comment, "author", "login"), str))
|
||||
if len(last) != len(comments):
|
||||
return unshaped
|
||||
threads.append(Thread(path, bool(resolved), last))
|
||||
threads.append(Thread(path, last))
|
||||
return author, tuple(threads)
|
||||
|
||||
|
||||
def waits(author: str, threads: tuple[Thread, ...]) -> list[str]:
|
||||
"""The files of the threads that wait for an answer from author, who opened the pull request."""
|
||||
return sorted({thread.path for thread in threads if not thread.resolved and any(last != author for last in thread.last)})
|
||||
return sorted({thread.path for thread in threads if any(last != author for last in thread.last)})
|
||||
|
||||
@@ -77,8 +77,20 @@ class TheThreadsThatWait(unittest.TestCase):
|
||||
nodes = [{"isResolved": resolved, "path": path, "comments": {"nodes": [{"author": {"login": last}}]}} for path, resolved, last in threads]
|
||||
return json.dumps({"data": {"repository": {"pullRequest": {"author": {"login": "me"}, "reviewThreads": {"nodes": nodes}}}}})
|
||||
|
||||
def test_parses_what_gh_prints_into_the_author_and_each_thread(self):
|
||||
self.assertEqual(reviews.threads_read(self.printed(("a.py", False, "rev"), ("b.py", True, "me"))), ("me", (Thread("a.py", False, ("rev",)), Thread("b.py", True, ("me",)))))
|
||||
def test_parses_what_gh_prints_into_the_author_and_each_thread_nobody_resolved(self):
|
||||
self.assertEqual(reviews.threads_read(self.printed(("a.py", False, "rev"), ("b.py", True, "me"), ("c.py", False, "me"))), ("me", (Thread("a.py", ("rev",)), Thread("c.py", ("me",)))))
|
||||
|
||||
def pull_with(self, *nodes: dict[str, object]) -> str:
|
||||
return json.dumps({"data": {"repository": {"pullRequest": {"author": {"login": "me"}, "reviewThreads": {"nodes": nodes}}}}})
|
||||
|
||||
def test_skips_a_resolved_thread_unread_so_a_deleted_accounts_comment_there_doesnt_fail_the_read(self):
|
||||
deleted = {"comments": {"nodes": [{"author": None}]}}
|
||||
printed = self.pull_with({"isResolved": True, "path": "b.py", **deleted}, {"isResolved": True}, {"isResolved": False, "path": "a.py", "comments": {"nodes": [{"author": {"login": "rev"}}]}})
|
||||
self.assertEqual(reviews.waiting_threads("7", lambda args: printed), ["a.py"])
|
||||
|
||||
def test_a_deleted_accounts_last_word_in_a_thread_nobody_resolved_is_a_failure(self):
|
||||
printed = self.pull_with({"isResolved": False, "path": "a.py", "comments": {"nodes": [{"author": None}]}})
|
||||
self.assertEqual(reviews.waiting_threads("7", lambda args: printed), Failure(f"gh api graphql printed no review threads: {printed[:200]}"))
|
||||
|
||||
def test_a_thread_waits_while_unresolved_with_a_reviewers_last_word(self):
|
||||
printed = self.printed(("b.py", False, "rev"), ("a.py", False, "rev"), ("a.py", False, "rev"), ("c.py", True, "rev"), ("d.py", False, "me"))
|
||||
|
||||
@@ -5,7 +5,6 @@ import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
@@ -25,6 +24,7 @@ from chunk_driver.drivers import LOOP_CLAUDE
|
||||
from judge_replay import main
|
||||
from rung_replay.search import Miss, Replay, Tried
|
||||
from model_routing.ladder import Rung
|
||||
from rung_replay.doubles import Together
|
||||
|
||||
TOOL = pathlib.Path(__file__).resolve().parents[1] / "judge-replay"
|
||||
TRANSCRIPT = pathlib.Path("/archive/p/c/a.jsonl")
|
||||
@@ -245,19 +245,16 @@ class Main(unittest.TestCase):
|
||||
for n in range(2, 8):
|
||||
(chunk / f"t{n}.jsonl").write_text(json.dumps({"type": "assistant", "message": {"content": [{"type": "text", "text": f"said {FINISHED} {n}"}]}}) + "\n")
|
||||
(chunk / f"t{n}.outcomes.md").write_text(f"Task: Do it.\n\n2026-10-14T10:00:0{n}Z {FINISHED}\n\nEvidence:\n> e\n")
|
||||
running, most = [], []
|
||||
together = Together(4)
|
||||
|
||||
def reply(rung, interrupted):
|
||||
running.append(rung)
|
||||
most.append(len(running))
|
||||
time.sleep(0.05)
|
||||
running.remove(rung)
|
||||
together(rung)
|
||||
return self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}")
|
||||
|
||||
code, _, err = self.run_main(["claude-opus-5-5"], reply)
|
||||
self.assertEqual(code, 0, err)
|
||||
self.assertIn("replaying 8 verdicts on ", err)
|
||||
self.assertEqual(max(most), 4)
|
||||
self.assertEqual(set(together.most.values()), {4})
|
||||
|
||||
def test_the_prompt_and_the_reply_schema_of_a_rebuilt_run_name_loop_claude_and_the_records_and_show_the_transcript_once(self):
|
||||
code, _, err = self.run_main(["claude-opus-5-5"], lambda rung, interrupted: self.verdict(interrupted, f"said {INTERRUPTED if interrupted else FINISHED}"))
|
||||
|
||||
@@ -6,7 +6,7 @@ import shutil
|
||||
from review_rules.comprehension_history import DIMENSIONS
|
||||
from failure import Failure
|
||||
from text_files.markdown_text import FENCE, listed, section
|
||||
from git_cli.git_repo import git
|
||||
from git_cli.git_repo import git, Git
|
||||
from project_config.main import located
|
||||
from review_rules.review_folder import made_review_folder, review_folder
|
||||
from project_config.settings import ProjectConfig
|
||||
@@ -127,14 +127,14 @@ def scoring_folder(worktree: str, home: pathlib.Path) -> pathlib.Path:
|
||||
return pathlib.Path(review_folder(str(home), worktree)) / "scoring"
|
||||
|
||||
|
||||
def prepare_scoring(worktree: str, home: pathlib.Path) -> str | Failure:
|
||||
def prepare_scoring(worktree: str, home: pathlib.Path, run: Git = git) -> str | Failure:
|
||||
"""Makes the scoring run's worktree at worktree's HEAD and writes each seat's prompt beside it, in the chunk's review folder; what it made, as the tool prints it."""
|
||||
folder = scoring_folder(worktree, home)
|
||||
# A setup record-scoring never removed, as where the seats failed, would otherwise block every later run.
|
||||
replaced = folder.exists()
|
||||
if replaced and isinstance(removed := removed_setup(worktree, home), Failure):
|
||||
if replaced and isinstance(removed := removed_setup(worktree, home, run), Failure):
|
||||
return removed
|
||||
config = located(worktree, home)
|
||||
config = located(worktree, home, run)
|
||||
if isinstance(config, Failure):
|
||||
return config
|
||||
# Where AGENTS.md is missing, this repo keeps the values in seat-prompts.md: .claude/rules/this-repo-keeps-no-agents-md.md.
|
||||
@@ -156,9 +156,9 @@ def prepare_scoring(worktree: str, home: pathlib.Path) -> str | Failure:
|
||||
score = score_bullets(skill)
|
||||
if isinstance(score, Failure):
|
||||
return score
|
||||
listing = git("worktree", "list", "--porcelain", cwd=worktree)
|
||||
branch = git("symbolic-ref", "--short", "-q", "HEAD", cwd=worktree)
|
||||
head = git("rev-parse", "--short", "HEAD", cwd=worktree)
|
||||
listing = run("worktree", "list", "--porcelain", cwd=worktree)
|
||||
branch = run("symbolic-ref", "--short", "-q", "HEAD", cwd=worktree)
|
||||
head = run("rev-parse", "--short", "HEAD", cwd=worktree)
|
||||
if isinstance(listing, Failure):
|
||||
return listing
|
||||
if isinstance(head, Failure):
|
||||
@@ -169,8 +169,8 @@ def prepare_scoring(worktree: str, home: pathlib.Path) -> str | Failure:
|
||||
return made
|
||||
checkout = folder / "checkout"
|
||||
# The seats score the chunk's head, which holds origin's default branch since larv-review §4 step 1, as larv-review → "Scoring run" says.
|
||||
added = git("worktree", "add", "--quiet", "--detach", str(checkout), "HEAD", cwd=worktree)
|
||||
tracked = added if isinstance(added, Failure) else git("ls-files", cwd=str(checkout))
|
||||
added = run("worktree", "add", "--quiet", "--detach", str(checkout), "HEAD", cwd=worktree)
|
||||
tracked = added if isinstance(added, Failure) else run("ls-files", cwd=str(checkout))
|
||||
if isinstance(tracked, Failure):
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
return tracked
|
||||
@@ -200,12 +200,12 @@ def prepare_scoring(worktree: str, home: pathlib.Path) -> str | Failure:
|
||||
return before + f"Scoring worktree, at {head.strip()}: {checkout}\nPrompts, each for one {SEAT_AGENT} agent:\n" + "\n".join(written)
|
||||
|
||||
|
||||
def removed_setup(worktree: str, home: pathlib.Path) -> Failure | None:
|
||||
def removed_setup(worktree: str, home: pathlib.Path, run: Git = git) -> Failure | None:
|
||||
"""Removes the scoring worktree and prompts prepare_scoring made for the chunk at worktree, where they exist."""
|
||||
folder = scoring_folder(worktree, home)
|
||||
if not folder.exists():
|
||||
return None
|
||||
if (folder / "checkout").exists() and isinstance(removed := git("worktree", "remove", "--force", str(folder / "checkout"), cwd=worktree), Failure):
|
||||
if (folder / "checkout").exists() and isinstance(removed := run("worktree", "remove", "--force", str(folder / "checkout"), cwd=worktree), Failure):
|
||||
return Failure(f"could not remove the scoring worktree {folder / 'checkout'}: {removed.reason}")
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
return None
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import functools
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
@@ -10,6 +11,7 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from larv_review.scoring_setup import bullets, file_globs, prepare_scoring, project_values, removed_setup, score_bullets
|
||||
from failure import Failure
|
||||
from git_cli.git_repo import git
|
||||
from test_env import isolated_env
|
||||
from project_config.doubles import write_config
|
||||
|
||||
@@ -102,7 +104,7 @@ class PrepareScoring(unittest.TestCase):
|
||||
|
||||
def prepare(self):
|
||||
"""What prepare-scoring returns in the checkout, run in this process so mutation tests see the lines it runs."""
|
||||
return prepare_scoring(str(self.checkout), self.root)
|
||||
return prepare_scoring(str(self.checkout), self.root, functools.partial(git, env=self.env))
|
||||
|
||||
def run_tool(self, *args):
|
||||
return subprocess.run([sys.executable, str(TOOL), *args], capture_output=True, cwd=self.checkout, env=self.env, text=True)
|
||||
@@ -137,7 +139,7 @@ class PrepareScoring(unittest.TestCase):
|
||||
self.commit()
|
||||
checkout = self.root / ".local" / "state" / "larv-review" / "proj" / "scoring" / "checkout"
|
||||
checkout.mkdir(parents=True)
|
||||
removed = removed_setup(str(self.checkout), self.root)
|
||||
removed = removed_setup(str(self.checkout), self.root, functools.partial(git, env=self.env))
|
||||
self.assertIsInstance(removed, Failure)
|
||||
self.assertTrue(removed.reason.startswith(f"could not remove the scoring worktree {checkout}: "), removed.reason)
|
||||
self.assertTrue(checkout.exists())
|
||||
|
||||
@@ -5,7 +5,7 @@ import re
|
||||
import typing
|
||||
|
||||
from touch_predictions.beside import Beside, chunks_beside
|
||||
from chunk_driver.claims import CHUNK_FLAG, epic_chunks, HANDED, LOCK, loop_session, project_lock, starting
|
||||
from chunk_driver.claims import CHUNK_FLAG, epic_chunks, HANDED, loop_session, project_lock, starting
|
||||
from loop_claude.chunk import chunk_items
|
||||
from git_cli.git_repo import default_ref
|
||||
from loop_effects.effects import Effects, say
|
||||
@@ -161,6 +161,6 @@ def held_beside(chunk: str, root: str, paths: Paths, effects: Effects) -> list[i
|
||||
|
||||
|
||||
def wait_for_starts(root: str, home: pathlib.Path) -> None:
|
||||
"""Returns once no loop of the project is starting windows: start_chunks holds LOCK while it starts them. Where the lock cannot be taken, it returns at once."""
|
||||
"""Returns once no loop of the project is starting windows: start_chunks holds the lock file `claims.LOCK` names while it starts them. Where the lock cannot be taken, it returns at once."""
|
||||
with starting(project_lock(root, home)):
|
||||
pass
|
||||
|
||||
@@ -41,7 +41,7 @@ class Steps:
|
||||
exec_if_edited: typing.Callable[[Loop, str, LoopRun], Stop | None]
|
||||
next_cycle: typing.Callable[[LoopRun, Loop], Cycle | Stop]
|
||||
run_cycle: typing.Callable[[Cycle], Stop | Concluded]
|
||||
closed_cycle: typing.Callable[[LoopRun, ProjectRung | None, Concluded, pathlib.Path, str, Loop], LoopRun | Stop]
|
||||
closed_cycle: typing.Callable[[LoopRun, str, ProjectRung | None, Concluded, pathlib.Path, str, Loop], LoopRun | Stop]
|
||||
check_routing: typing.Callable[[pathlib.Path, Rung, Loop], Stop | None]
|
||||
|
||||
|
||||
@@ -93,7 +93,7 @@ def run_loop(argv: list[str], env: typing.Mapping[str, str], paths: Paths, launc
|
||||
concluded = steps.run_cycle(cycle)
|
||||
if isinstance(concluded, Stop):
|
||||
return concluded
|
||||
closed = steps.closed_cycle(loop_run, cycle.project, concluded, cycle.project_archive, fresh_prompt.DRIVER, loop)
|
||||
closed = steps.closed_cycle(loop_run, cycle.task, cycle.project, concluded, cycle.project_archive, fresh_prompt.DRIVER, loop)
|
||||
if isinstance(closed, Stop):
|
||||
return closed
|
||||
loop_run = closed
|
||||
|
||||
@@ -194,8 +194,8 @@ class WindowStarts(unittest.TestCase):
|
||||
|
||||
def test_a_window_loop_hands_each_step_of_a_cycle_what_the_steps_before_it_made(self):
|
||||
env = {"TMUX_PANE": "%4", "LOOP_CLAUDE_TRANSCRIPTS": "/t"}
|
||||
on_ladder = unittest.mock.Mock(project=OPUS_PROJECT, project_archive=pathlib.Path("/t/r"))
|
||||
off_ladder = unittest.mock.Mock(project=None, project_archive=pathlib.Path("/t/r"), route=unittest.mock.Mock(rung=Rung("my-model")))
|
||||
on_ladder = unittest.mock.Mock(task="the cycle's task", project=OPUS_PROJECT, project_archive=pathlib.Path("/t/r"))
|
||||
off_ladder = unittest.mock.Mock(task="the cycle's task", project=None, project_archive=pathlib.Path("/t/r"), route=unittest.mock.Mock(rung=Rung("my-model")))
|
||||
stop, calls = self.window_loop([on_ladder, off_ladder, Stop(2, "the human is needed")], env)
|
||||
self.assertEqual(stop, Stop(2, "the human is needed"))
|
||||
before, first, second, third = (main.LoopRun(number, "/r", "do it", "", unfinished_run=max(number - 1, 0), chunk="item-3") for number in range(4))
|
||||
@@ -204,12 +204,12 @@ class WindowStarts(unittest.TestCase):
|
||||
("exec_if_edited", self.loop, "print-1", before),
|
||||
("next_cycle", first, self.loop),
|
||||
("run_cycle", on_ladder),
|
||||
("closed_cycle", first, OPUS_PROJECT, self.unfinished, pathlib.Path("/t/r"), "loop-claude", self.loop),
|
||||
("closed_cycle", first, "the cycle's task", OPUS_PROJECT, self.unfinished, pathlib.Path("/t/r"), "loop-claude", self.loop),
|
||||
("check_routing", pathlib.Path("/t"), Rung("claude-opus-5-5", "medium"), self.loop),
|
||||
("exec_if_edited", self.loop, "print-1", dataclasses.replace(first, unfinished_run=1)),
|
||||
("next_cycle", second, self.loop),
|
||||
("run_cycle", off_ladder),
|
||||
("closed_cycle", second, None, self.unfinished, pathlib.Path("/t/r"), "loop-claude", self.loop),
|
||||
("closed_cycle", second, "the cycle's task", None, self.unfinished, pathlib.Path("/t/r"), "loop-claude", self.loop),
|
||||
("check_routing", pathlib.Path("/t"), Rung("my-model"), self.loop),
|
||||
("exec_if_edited", self.loop, "print-1", dataclasses.replace(second, unfinished_run=2)),
|
||||
("next_cycle", third, self.loop),
|
||||
|
||||
@@ -9,7 +9,6 @@ import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
@@ -24,6 +23,7 @@ from failure import Failure
|
||||
from review_rules.reviewer_context import reviewer_context
|
||||
from review_rules.review_passes import REVIEWERS
|
||||
from model_routing.ladder import Rung
|
||||
from rung_replay.doubles import Together
|
||||
|
||||
TOOL = pathlib.Path(__file__).resolve().parents[1] / "review-replay"
|
||||
REFERENCE = Rung("claude-opus-5-5", "medium")
|
||||
@@ -306,21 +306,18 @@ class Main(unittest.TestCase):
|
||||
|
||||
def test_replays_no_more_rounds_at_once_than_asked_on_any_rung(self):
|
||||
self.write_chunk("proj", "c2", "2026-10-14T10:30:00Z")
|
||||
self.limits["at_once"] = 1
|
||||
self.limits["at_once"] = 2
|
||||
for argv, rungs in ((["stability", "claude-opus-5-5"], {REFERENCE, Rung("claude-opus-5-5", "low"), Rung("claude-sonnet-5-5", "high")}), (["stability", "claude-opus-5-5", "--omit-claude-md"], {REFERENCE})):
|
||||
with self.subTest(argv=argv):
|
||||
running, most = [], {}
|
||||
together = Together(2)
|
||||
|
||||
def review(rung):
|
||||
running.append(rung)
|
||||
most[rung] = max(most.get(rung, 0), len(running))
|
||||
time.sleep(0.05)
|
||||
running.remove(rung)
|
||||
together(rung)
|
||||
return {"type": "result", "result": f"review on {rung.model} {rung.effort}"}
|
||||
|
||||
code, _, err = self.run_main(argv, lambda rung: {"found": [0, 1]}, review=review)
|
||||
self.assertEqual(code, 0, err)
|
||||
self.assertEqual(most, dict.fromkeys(rungs, 1))
|
||||
self.assertEqual(together.most, dict.fromkeys(rungs, 2))
|
||||
|
||||
def test_without_claude_md_replays_each_round_three_times_with_it_and_once_without_on_the_project_rung(self):
|
||||
finds = iter(([0, 1], [0], [0, 1], [1]))
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import collections
|
||||
import contextlib
|
||||
import dataclasses
|
||||
import functools
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
@@ -319,7 +320,7 @@ class BranchHistory(unittest.TestCase):
|
||||
folder = configured(self, checkout)
|
||||
config = folder / "config.toml"
|
||||
config.write_text(config.read_text().replace('"checkout:todo.md"', '"folder:todo.md"'))
|
||||
found = archive.branch_history(str(checkout), folder.parents[1], BARE, git_repo.git)
|
||||
found = archive.branch_history(str(checkout), folder.parents[1], BARE, functools.partial(git_repo.git, env=isolated_env()))
|
||||
assert isinstance(found, Failure), found
|
||||
self.assertEqual(found.reason, f"its chunks' items and later fixes are not counted: the project's config puts {folder}/todo.md elsewhere than todo.md, docs/comprehension-history.md, docs/reading-cost.md in the checkout, the only paths whose commits are read so far")
|
||||
|
||||
@@ -365,7 +366,7 @@ class Archive(unittest.TestCase):
|
||||
env = {**isolated_env(), "GIT_CONFIG_GLOBAL": os.devnull, "HOME": str(home)}
|
||||
self.ship_then_fix(home / "code" / "log", env)
|
||||
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
|
||||
found = archive.measure_archive(self.root, str(home), "", git_repo.git)
|
||||
found = archive.measure_archive(self.root, str(home), "", functools.partial(git_repo.git, env=env))
|
||||
self.assertEqual([(chunk.dollars["project"].keys(), chunk.fixed_later) for chunk in found.chunks], [({"log": 0}.keys(), 1)])
|
||||
|
||||
def test_reads_a_projects_round_titles_and_its_branchs_history_through_its_configs_templates(self):
|
||||
@@ -376,9 +377,9 @@ class Archive(unittest.TestCase):
|
||||
config = write_config(home, home / "code" / "log") / "config.toml"
|
||||
worker = [message(f"m{n}", "2026-10-14T09:00:00Z", call("Bash", command=f"git commit -m '{round_title.format(n)}'")) for n in (1, 2)]
|
||||
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), worker)
|
||||
bare = archive.measure_archive(self.root, str(home), "", git_repo.git)
|
||||
bare = archive.measure_archive(self.root, str(home), "", functools.partial(git_repo.git, env=env))
|
||||
config.write_text(config.read_text() + 'commit-title = "PAY-{id} - {summary}"\n')
|
||||
templated = archive.measure_archive(self.root, str(home), "", git_repo.git)
|
||||
templated = archive.measure_archive(self.root, str(home), "", functools.partial(git_repo.git, env=env))
|
||||
self.assertEqual([(chunk.rounds_past_first, chunk.fixed_later) for chunk in bare.chunks], [(0, 0)])
|
||||
self.assertEqual([(chunk.rounds_past_first, chunk.fixed_later) for chunk in templated.chunks], [(1, 1)])
|
||||
|
||||
@@ -389,7 +390,7 @@ class Archive(unittest.TestCase):
|
||||
config = write_config(home, home / "code" / "log") / "config.toml"
|
||||
config.write_text(config.read_text() + 'commit-title = "PAY-{id}"\n')
|
||||
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
|
||||
found = archive.measure_archive(self.root, str(home), "", git_repo.git)
|
||||
found = archive.measure_archive(self.root, str(home), "", functools.partial(git_repo.git, env=env))
|
||||
self.assertEqual(found.chunks, [])
|
||||
self.assertEqual(len(found.unread), 1)
|
||||
self.assertRegex(found.unread[0], r"^log: none of its chunks is measured: .*`commit-title` is 'PAY-\{id\}'")
|
||||
@@ -403,7 +404,7 @@ class Archive(unittest.TestCase):
|
||||
|
||||
def recording(*args, **options):
|
||||
made.append((args[0], options))
|
||||
return git_repo.git(*args, **options)
|
||||
return git_repo.git(*args, **options, env=env)
|
||||
archive.measure_archive(self.root, str(home), "", recording)
|
||||
self.assertTrue(all(options.get("guarded") is True for _, options in made))
|
||||
read = [options.get("timeout") for command, options in made if command in ("log", "ls-tree", "show", "diff", "blame")]
|
||||
@@ -416,7 +417,7 @@ class Archive(unittest.TestCase):
|
||||
(home / "code" / "log").mkdir(parents=True)
|
||||
subprocess.run(["git", "init", "-q"], cwd=home / "code" / "log", check=True, env=env, capture_output=True)
|
||||
self.chunk("log", "c1", stamped("2026-10-14T10:00:00Z", outcomes.finished(LOW_ROUTE, "m", "j")), [message("m1", "2026-10-14T09:00:00Z")])
|
||||
found = archive.measure_archive(self.root, str(home), "", git_repo.git)
|
||||
found = archive.measure_archive(self.root, str(home), "", functools.partial(git_repo.git, env=env))
|
||||
self.assertEqual(found.unread, ("log: no origin remote, and no main or master branch to take as the default branch",))
|
||||
|
||||
def test_names_each_project_whose_history_it_cannot_read(self):
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Test doubles for what a replay runs at once."""
|
||||
import collections
|
||||
import threading
|
||||
import typing
|
||||
|
||||
|
||||
class Together:
|
||||
"""What a fake replay calls to show how many replays run at once: each call returns once `parties` calls are inside it together, so it neither sleeps nor reads a clock."""
|
||||
|
||||
def __init__(self, parties: int) -> None:
|
||||
# The timeout only ends a test whose replays never run `parties` at once: it breaks the barrier, which raises in every call.
|
||||
self.barrier = threading.Barrier(parties, timeout=60)
|
||||
self.lock = threading.Lock()
|
||||
self.inside = 0
|
||||
# The most calls inside at once, by the key each call gave.
|
||||
self.most: dict[typing.Hashable, int] = collections.defaultdict(int)
|
||||
|
||||
def __call__(self, key: typing.Hashable) -> None:
|
||||
with self.lock:
|
||||
self.inside += 1
|
||||
self.most[key] = max(self.most[key], self.inside)
|
||||
try:
|
||||
self.barrier.wait()
|
||||
finally:
|
||||
with self.lock:
|
||||
self.inside -= 1
|
||||
@@ -5,7 +5,6 @@ import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
@@ -18,6 +17,7 @@ from failure import Failure
|
||||
from claude_code.reply import Limited
|
||||
from model_routing.ladder_file import read_ladder, SHIPPED
|
||||
from model_routing.ladder import Rung
|
||||
from rung_replay.doubles import Together
|
||||
|
||||
LADDER = read_ladder(SHIPPED)
|
||||
REFERENCE = Rung("claude-opus-5-5", "medium")
|
||||
@@ -63,18 +63,15 @@ class Replayed(unittest.TestCase):
|
||||
|
||||
def test_says_what_it_replays_on_each_rung_and_replays_no_more_cases_at_once_than_asked(self):
|
||||
high = Rung("claude-fable-5-2", "low")
|
||||
said, running, most = [], [], {}
|
||||
said, together = [], Together(2)
|
||||
|
||||
def replay_one(case, rung):
|
||||
running.append(case)
|
||||
most[rung] = max(most.get(rung, 0), len(running))
|
||||
time.sleep(0.02)
|
||||
running.remove(case)
|
||||
together(rung)
|
||||
return {0}
|
||||
|
||||
replayed(["a", "b", "c"], REFERENCE, [MIDDLE], replay_one, lambda _, reply: sorted(reply), "verdicts", said.append, 1, [high])
|
||||
self.assertEqual(said, [f"replaying 3 verdicts on {REFERENCE}", f"replaying 3 verdicts on {high}", f"replaying 3 verdicts on {MIDDLE}"])
|
||||
self.assertEqual(most, {REFERENCE: 1, high: 1, MIDDLE: 1})
|
||||
replayed(["a", "b", "c", "d"], REFERENCE, [MIDDLE], replay_one, lambda _, reply: sorted(reply), "verdicts", said.append, 2, [high])
|
||||
self.assertEqual(said, [f"replaying 4 verdicts on {REFERENCE}", f"replaying 4 verdicts on {high}", f"replaying 4 verdicts on {MIDDLE}"])
|
||||
self.assertEqual(together.most, {REFERENCE: 2, high: 2, MIDDLE: 2})
|
||||
|
||||
def test_stops_at_a_usage_limit(self):
|
||||
limit = Limited(None, "limit")
|
||||
|
||||
Reference in New Issue
Block a user