205 lines
11 KiB
Python
205 lines
11 KiB
Python
import contextlib
|
|
import dataclasses
|
|
import functools
|
|
import io
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import sys
|
|
import tempfile
|
|
import types
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from chunk_driver.doubles import a_loop
|
|
from project_config.doubles import configured
|
|
from agent_cycle.run import Cycle
|
|
from chunk_driver import cycle_start
|
|
from failure import Failure
|
|
from loop_effects.doubles import FakeEffects, NOW, STAMP
|
|
from git_cli import git_repo
|
|
from chunk_driver.project.model_choice import Models
|
|
from transcript_archive import outcomes, write
|
|
from loop_state.paths import paths_from
|
|
from model_routing.ladder import ProjectRung, Rung
|
|
from model_routing.ladder_file import read_ladder, SHIPPED
|
|
from text_files.file_writes import replace_file
|
|
from model_routing.route import Route, WorkerLine
|
|
from claude_code import session
|
|
from agent_cycle.stop import Stop
|
|
from agent_cycle.verdicts import TakenIn
|
|
from agent_cycle.worker_prompts import WorkerPrompts
|
|
|
|
OPUS_PROJECT = ProjectRung(read_ladder(SHIPPED), Rung("claude-opus-5-5", "medium"))
|
|
|
|
|
|
class SetupVersion(unittest.TestCase):
|
|
def test_names_the_release_and_the_commit_or_why_git_cannot_tell(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
git = functools.partial(git_repo.git, env={**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1", "GIT_CEILING_DIRECTORIES": root})
|
|
gone = cycle_start.setup_version(pathlib.Path(root), git)
|
|
self.assertRegex(gone, r"^unknown: [^\n]*not a git repository")
|
|
for step in (["init", "-q"], ["-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", "one"]):
|
|
self.assertIsInstance(git(*step, cwd=root), str)
|
|
head = git("rev-parse", "HEAD", cwd=root).strip()
|
|
self.assertEqual(cycle_start.setup_version(pathlib.Path(root), git), f"{head[:7]} ({head})")
|
|
git("tag", "v0.1.0", cwd=root)
|
|
self.assertEqual(cycle_start.setup_version(pathlib.Path(root), git), f"v0.1.0 ({head})")
|
|
git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", "two", cwd=root)
|
|
head = git("rev-parse", "HEAD", cwd=root).strip()
|
|
self.assertEqual(cycle_start.setup_version(pathlib.Path(root), git), f"v0.1.0-1-g{head[:7]} ({head})")
|
|
|
|
def test_names_why_git_cannot_name_the_commit_on_one_line(self):
|
|
def fake_git(*args, cwd=None):
|
|
return Failure("fatal: bad HEAD\nhint: repair it") if args[0] == "rev-parse" else "v0.1.0\n"
|
|
|
|
self.assertEqual(cycle_start.setup_version(pathlib.Path("/r"), fake_git), "unknown: fatal: bad HEAD hint: repair it")
|
|
|
|
|
|
class ProjectRouting(unittest.TestCase):
|
|
def setUp(self):
|
|
temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temp.cleanup)
|
|
self.home = pathlib.Path(temp.name)
|
|
self.paths = paths_from(self.home)
|
|
self.checkout = self.home / "code" / "acme"
|
|
self.checkout.mkdir(parents=True)
|
|
self.config = configured(self, self.checkout, home=self.home)
|
|
self.paths.models_file.parent.mkdir(exist_ok=True)
|
|
|
|
def routing(self, fake=None, env=None):
|
|
return cycle_start.project_routing(a_loop((fake or FakeEffects()).effects(), root=str(self.checkout), paths=self.paths, env=env or {}), "a-driver")
|
|
|
|
def test_routes_on_the_projects_saved_model_and_names_the_shipped_ladder_or_the_projects_own_as_its_file(self):
|
|
self.paths.models_file.write_text(json.dumps({"acme": "claude-opus-5-5"}))
|
|
shipped = read_ladder(SHIPPED)
|
|
self.assertEqual(self.routing(), cycle_start.Routing(Models("claude-opus-5-5", "claude-opus-5-5"), shipped, SHIPPED))
|
|
self.assertEqual(self.routing(env={"LOOP_CLAUDE_JUDGE_MODEL": "j"}), cycle_start.Routing(Models("j", "claude-opus-5-5"), shipped, SHIPPED))
|
|
own = self.config / "routing-ladder.json"
|
|
own.write_text('[{"model": "claude-fable-5-2", "effort": "low"}]', encoding="utf-8")
|
|
routing = self.routing()
|
|
assert isinstance(routing, cycle_start.Routing)
|
|
self.assertEqual((routing.ladder_file, [str(rung) for rung in routing.ladder.rungs]), (own, ["claude-fable-5-2 at low effort"]))
|
|
|
|
def test_stops_with_exit_1_naming_the_ladder_file_that_cannot_be_read(self):
|
|
self.paths.models_file.write_text(json.dumps({"acme": "claude-opus-5-5"}))
|
|
own = self.config / "routing-ladder.json"
|
|
own.write_text("not json", encoding="utf-8")
|
|
stop = self.routing()
|
|
assert isinstance(stop, Stop)
|
|
self.assertEqual(stop.code, 1)
|
|
self.assertIn(str(own), stop.reason)
|
|
|
|
def test_stops_with_exit_1_naming_the_config_that_cannot_be_read(self):
|
|
self.paths.models_file.write_text(json.dumps({"acme": "claude-opus-5-5"}))
|
|
(self.config / "config.toml").write_text("not toml", encoding="utf-8")
|
|
stop = self.routing()
|
|
assert isinstance(stop, Stop)
|
|
self.assertEqual(stop.code, 1)
|
|
self.assertIn(str(self.config / "config.toml"), stop.reason)
|
|
|
|
def test_asks_for_the_model_where_none_is_saved_and_off_a_terminal_stops_naming_the_driver_to_rerun(self):
|
|
stop = self.routing(FakeEffects(on_terminal=False))
|
|
assert isinstance(stop, Stop)
|
|
self.assertEqual(stop.code, 2)
|
|
self.assertIn("pick acme's model", stop.reason)
|
|
self.assertIn("rerun a-driver on a terminal", stop.reason)
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(self.routing(FakeEffects(inputs=["claude-opus-5-5"])), cycle_start.Routing(Models("claude-opus-5-5", "claude-opus-5-5"), read_ladder(SHIPPED), SHIPPED))
|
|
|
|
|
|
class CycleStart(unittest.TestCase):
|
|
ROUTING = cycle_start.Routing(Models("judge-model", "claude-opus-5-5"), OPUS_PROJECT.ladder, SHIPPED)
|
|
|
|
def test_announces_the_route_the_chunks_run_holds_and_else_the_one_its_archive_last_names(self):
|
|
def never():
|
|
raise AssertionError("not asked")
|
|
|
|
kept = Route(Rung("claude-opus-5-5", "xhigh"), 3, 4, 5)
|
|
with tempfile.TemporaryDirectory() as home:
|
|
opened = session.session_named("A proj a-driver", pathlib.Path(home, "projects"))
|
|
loop = a_loop(FakeEffects().effects(), session=opened)
|
|
archive = pathlib.Path(home, "archive")
|
|
with contextlib.redirect_stdout(io.StringIO()) as said:
|
|
self.assertEqual(cycle_start.announced_route(4, kept, self.ROUTING, archive, never, loop), kept)
|
|
self.assertEqual(said.getvalue(), f"\n{STAMP} === cycle 4, worker on claude-opus-5-5 at xhigh effort ===\n")
|
|
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
|
|
transcript.parent.mkdir(parents=True)
|
|
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(kept.worker_line(), "working", "j", ""), NOW, replace=replace_file))
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
self.assertEqual(cycle_start.announced_route(1, None, self.ROUTING, archive, never, loop), kept)
|
|
|
|
def test_builds_the_cycle_from_the_loop_the_routing_and_what_the_entry_point_knows(self):
|
|
asked = []
|
|
|
|
def git(*args, cwd=None):
|
|
asked.append(cwd)
|
|
return "v0.1.0\n" if args[0] == "describe" else "abc\n"
|
|
|
|
effects = dataclasses.replace(FakeEffects().effects(), git=git)
|
|
loop, route, archive = a_loop(effects), Route(Rung("claude-opus-5-5", "xhigh"), 3, 4, 5), pathlib.Path("/t/r")
|
|
prompts = WorkerPrompts("a-driver", lambda *args: "fresh", lambda *args: "resumed", lambda *args: "", "the sources", "the destinations", "the records named")
|
|
def taken_in(entries):
|
|
return TakenIn()
|
|
|
|
cycle = cycle_start.cycle_of(loop, self.ROUTING, route, archive, task="the task", facts="the facts", premises="the premises", records="the records", unfinished_evidence="the evidence", prompts=prompts, taken_in=taken_in)
|
|
self.assertEqual(cycle, Cycle(config="v0.1.0 (abc)", effects=effects, facts="the facts", judge=Rung("judge-model"), premises="the premises", project=OPUS_PROJECT, project_archive=archive, records="the records", root="/r",
|
|
route=route, session=loop.session, task="the task", prompts=prompts, taken_in=taken_in, unfinished_evidence="the evidence", window=loop.window))
|
|
self.assertEqual(asked, ["/h/.claude", "/h/.claude"])
|
|
|
|
|
|
class Routes(unittest.TestCase):
|
|
OPUS = Rung("claude-opus-5-5", "medium")
|
|
SONNET = Rung("claude-sonnet-5-5", "high")
|
|
|
|
def test_keeps_a_chunks_route_or_its_archived_one_at_or_above_the_rung_its_item_picks_and_else_routes_afresh(self):
|
|
item = types.SimpleNamespace(id=3, likelihood=4, escape_and_impact=5)
|
|
archived = Route(Rung("claude-opus-5-5", "xhigh"), 3, 4, 5)
|
|
def never():
|
|
raise AssertionError("not asked")
|
|
self.assertEqual(cycle_start.routed(archived, None, "worker-model", never, never), Route(Rung("worker-model")))
|
|
self.assertEqual(cycle_start.routed(archived, OPUS_PROJECT, "claude-opus-5-5", never, never), archived)
|
|
self.assertEqual(cycle_start.routed(None, OPUS_PROJECT, "claude-opus-5-5", lambda: archived, never), archived)
|
|
self.assertEqual(cycle_start.routed(None, OPUS_PROJECT, "claude-opus-5-5", lambda: None, lambda: item), Route(Rung("claude-fable-5-2", "medium"), 3, 4, 5))
|
|
self.assertEqual(cycle_start.routed(Route(self.SONNET), OPUS_PROJECT, "claude-opus-5-5", never, lambda: None), Route(self.OPUS))
|
|
cheap = Route(self.SONNET, 4, 2, 3)
|
|
self.assertEqual(cycle_start.routed(cheap, OPUS_PROJECT, "claude-opus-5-5", never, never), cheap)
|
|
self.assertEqual(cycle_start.routed(Route(Rung("claude-haiku-5-5", "max"), 4, 2, 3), OPUS_PROJECT, "claude-opus-5-5", never, lambda: None), Route(self.OPUS))
|
|
|
|
def test_resumes_on_the_archived_route_and_says_why_it_routes_afresh(self):
|
|
with tempfile.TemporaryDirectory() as home:
|
|
opened = session.session_named("A proj a-driver", pathlib.Path(home, "projects"))
|
|
archive = pathlib.Path(home, "archive")
|
|
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
|
|
route = Route(Rung("claude-fable-5-2", "low"), 3, 4, 5)
|
|
with contextlib.redirect_stderr(io.StringIO()) as stderr:
|
|
self.assertIsNone(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)))
|
|
transcript.parent.mkdir(parents=True)
|
|
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
|
self.assertIsNone(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)))
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(route.worker_line(), "working", "j", ""), NOW, replace=replace_file))
|
|
with contextlib.redirect_stderr(stderr):
|
|
self.assertEqual(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), route)
|
|
self.assertEqual(stderr.getvalue(), "")
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.stepped_up(WorkerLine('{"model": "claude-opus-5-5", "rung": 3}')), NOW + 1, replace=replace_file))
|
|
with contextlib.redirect_stderr(io.StringIO()) as stderr:
|
|
self.assertIsNone(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)))
|
|
self.assertEqual(stderr.getvalue(), f'{STAMP} routing the chunk afresh: the archive\'s latest Worker: line names no route that can be read: {{"model": "claude-opus-5-5", "rung": 3}}\n')
|
|
(chunk,) = archive.iterdir()
|
|
chunk.chmod(0)
|
|
try:
|
|
with contextlib.redirect_stderr(io.StringIO()) as stderr:
|
|
self.assertIsNone(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)))
|
|
finally:
|
|
chunk.chmod(0o700)
|
|
self.assertTrue(stderr.getvalue().startswith(f"{STAMP} routing the chunk afresh: could not read the archive"), stderr.getvalue())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|