288 lines
18 KiB
Python
288 lines
18 KiB
Python
import contextlib
|
|
import dataclasses
|
|
import io
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
import unittest.mock
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2]))
|
|
|
|
from chunk_driver.doubles import a_loop
|
|
from loop_effects.doubles import CWD, FakeEffects, ran, STAMP
|
|
from failure import Failure
|
|
from agent_cycle.verdicts import Finished, Unfinished
|
|
from loop_claude.loop import main, reload, usage
|
|
from loop_claude.doubles import OPUS_PROJECT
|
|
from loop_state import paths
|
|
from model_routing.route import Route
|
|
from model_routing.ladder import Rung
|
|
from claude_code import session
|
|
from agent_cycle.stop import Stop
|
|
from chunk_driver import this_loop
|
|
from agent_cycle import tmux
|
|
|
|
|
|
class LoopRuns(unittest.TestCase):
|
|
def test_stops_naming_the_reload_when_it_hands_over_anything_but_the_state_with_a_task(self):
|
|
for handed in (["3", "2", "/r", "do it"], ["3"], ["[1]"], [json.dumps({"cycle_number": 3, "root": "/r", "unfinished_evidence": ""})], [json.dumps({"cycle_number": 3, "root": "/r", "task": "", "unfinished_evidence": ""})], [json.dumps({"cycle_number": "3", "root": "/r", "task": "do it", "unfinished_evidence": ""})], [json.dumps({"cycle_number": 3, "root": "", "task": "do it", "unfinished_evidence": ""})], [json.dumps({"cycle_number": 3, "root": "/r", "task": "do it", "turns": 1, "unfinished_evidence": ""})], [json.dumps({"cycle_number": 3, "failures": 0, "root": "/r", "task": "do it", "unfinished_evidence": ""})]):
|
|
with self.subTest(handed=handed):
|
|
self.assertEqual(main.loop_run_from([main.RELOAD_FLAG, *handed], CWD, FakeEffects().effects()), Stop(1, f"the edited loop-claude cannot read the state the running one handed it, likely because the edit changed that handover; rerun loop-claude <task>. Handed: {' '.join(handed)}"))
|
|
|
|
def test_decodes_every_field_of_the_state_and_its_route(self):
|
|
state = main.LoopRun(cycle_number=2, root="/r", task="do it", unfinished_evidence="old", route=Route(Rung("claude-fable-5-2", "low"), 3, 4, 5, 1, 7, 9), unfinished_run=2)
|
|
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(state))), state)
|
|
self.assertEqual(set(reload.LOOP_RUN_FIELDS), {field.name for field in dataclasses.fields(main.LoopRun)})
|
|
|
|
def test_rejects_a_field_of_another_type_or_an_unknown_field(self):
|
|
state = dataclasses.asdict(main.LoopRun(cycle_number=2, root="/r", task="do it", unfinished_evidence="", route=Route(Rung("claude-fable-5-2", "low"), 3, 4, 5, 1, 7, 9), unfinished_run=2))
|
|
for path, wrong in ((("unfinished_run",), "2"), (("unfinished_run",), True), (("route", "unfinished"), 1.0), (("route", "escalated_at"), "9"), (("route", "item"), True), (("route", "rung"), "claude-opus-5-5"), (("route", "rung", "model"), None), (("route",), "r"), (("route", "extra"), 1), (("route", "model"), "claude-opus-5-5"), (("route", "effort"), "high")):
|
|
with self.subTest(path=path, wrong=wrong):
|
|
handed = json.loads(json.dumps(state))
|
|
target = handed
|
|
for key in path[:-1]:
|
|
target = target[key]
|
|
target[path[-1]] = wrong
|
|
self.assertIsNone(reload.decode_handed(json.dumps(handed)))
|
|
|
|
def test_rejects_a_state_or_rung_missing_a_required_field(self):
|
|
state = dataclasses.asdict(main.LoopRun(cycle_number=2, root="/r", task="do it", unfinished_evidence="", route=Route(Rung("claude-fable-5-2", "low"))))
|
|
for path in (("cycle_number",), ("root",), ("task",), ("unfinished_evidence",), ("route", "rung"), ("route", "rung", "model")):
|
|
with self.subTest(path=path):
|
|
handed = json.loads(json.dumps(state))
|
|
target = handed
|
|
for key in path[:-1]:
|
|
target = target[key]
|
|
del target[path[-1]]
|
|
self.assertIsNone(reload.decode_handed(json.dumps(handed)))
|
|
|
|
def test_decodes_a_rung_without_an_effort(self):
|
|
state = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(Rung("some-model")))
|
|
self.assertEqual(reload.decode_handed(json.dumps({**dataclasses.asdict(state), "route": {"rung": {"model": "some-model"}}})), state)
|
|
|
|
def test_rejects_text_that_is_not_json(self):
|
|
self.assertIsNone(reload.decode_handed("not json"))
|
|
|
|
def test_decodes_a_route_inside_the_state_or_none(self):
|
|
state = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="", route=Route(Rung("claude-fable-5-2", "low"), 3, 4, 5, 1))
|
|
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(state))), state)
|
|
unrouted = dataclasses.replace(state, route=None)
|
|
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(unrouted))), unrouted)
|
|
handed = dataclasses.asdict(state)
|
|
handed["route"]["rung"]["effort"] = 1
|
|
self.assertIsNone(reload.decode_handed(json.dumps(handed)))
|
|
del handed["route"]["rung"]
|
|
self.assertIsNone(reload.decode_handed(json.dumps(handed)))
|
|
off_scale = dataclasses.asdict(state)
|
|
off_scale["route"]["likelihood"] = 9
|
|
self.assertIsNone(reload.decode_handed(json.dumps(off_scale)))
|
|
older = dataclasses.asdict(state)
|
|
del older["route"]["escape_and_impact_taken_in"]
|
|
self.assertEqual(reload.decode_handed(json.dumps(older)), state)
|
|
taken = dataclasses.replace(state, route=dataclasses.replace(state.route, escape_and_impact_taken_in=7))
|
|
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(taken))), taken)
|
|
for taken_in in (9, 5):
|
|
with self.subTest(taken_in=taken_in):
|
|
handed = dataclasses.asdict(state)
|
|
handed["route"]["escape_and_impact_taken_in"] = taken_in
|
|
self.assertIsNone(reload.decode_handed(json.dumps(handed)))
|
|
|
|
def test_decodes_a_state_from_before_a_field_with_a_default_joined_it(self):
|
|
self.assertEqual(reload.decode_handed(json.dumps({"cycle_number": 3, "root": "/r", "task": "do it", "unfinished_evidence": ""})), main.LoopRun(3, "/r", "do it", ""))
|
|
|
|
def test_drops_the_items_taken_in_that_a_loop_started_before_0_11_0_hands_on(self):
|
|
state = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="")
|
|
self.assertEqual(reload.decode_handed(json.dumps(dataclasses.asdict(state) | {"items_taken_in": [12, 14]})), state)
|
|
|
|
def test_drops_the_slot_that_a_loop_started_before_chunks_ran_in_windows_hands_on(self):
|
|
state = main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="")
|
|
self.assertEqual(reload.decode_handed(json.dumps({key: value for key, value in dataclasses.asdict(state).items() if key != "chunk"} | {"slot": 1})), state)
|
|
|
|
def test_stops_when_the_reload_cannot_start_python(self):
|
|
with tempfile.TemporaryDirectory() as root:
|
|
launcher = pathlib.Path(root, "loop-claude")
|
|
launcher.write_text("")
|
|
fake = FakeEffects(python=[ran()])
|
|
self.assertEqual(reload.exec_if_edited(a_loop(fake.effects(), launcher=launcher), "stale", main.LoopRun(cycle_number=1, root="/r", task="do it", unfinished_evidence="")), Stop(1, "could not reload loop-claude: No such file or directory; rerun loop-claude <task>"))
|
|
|
|
|
|
def steps(start_chunks, **others):
|
|
"""The steps of a loop that starts chunks through start_chunks, and takes each other step through the double given for it, else fails."""
|
|
def unused(*args):
|
|
raise AssertionError(args)
|
|
|
|
return main.Steps(**{"fingerprint": unused, "exec_if_edited": unused, "next_cycle": unused, "run_cycle": unused, "closed_cycle": unused, "check_routing": unused, **others}, start_chunks=start_chunks)
|
|
|
|
|
|
class WindowStarts(unittest.TestCase):
|
|
"""The loop-claude the human runs, which starts the windows, and the stop of a window's loop whose chunk finished."""
|
|
HOME = pathlib.Path("/h")
|
|
|
|
def run_loop(self, argv, fake, env=None, started=("item-3", "item-4")):
|
|
"""run_loop's stop and what it said on stdout, with start_chunks answering `started`; self.asked holds what start_chunks was asked."""
|
|
self.asked = []
|
|
|
|
def start(*args):
|
|
self.asked.append(args[:2])
|
|
return list(started) if isinstance(started, tuple) else started
|
|
|
|
with contextlib.redirect_stdout(io.StringIO()) as said:
|
|
stop = main.run_loop(argv, env or {}, paths.paths_from(self.HOME), pathlib.Path("/b/loop-claude"), CWD, fake.effects(), steps(start))
|
|
return stop, said.getvalue()
|
|
|
|
def test_refuses_to_start_anything_on_a_tmux_older_than_needed(self):
|
|
fake = FakeEffects(git=["/r\n"], tmux_version="tmux 3.1c")
|
|
self.assertEqual(self.run_loop(["do it"], fake), (Stop(1, "loop-claude runs each chunk in a tmux window, and needs tmux 3.2 or later; it found tmux 3.1c"), ""))
|
|
self.assertEqual((fake.tmux_calls, self.asked), ([["-V"]], []))
|
|
|
|
def test_prints_the_help_without_tmux(self):
|
|
fake = FakeEffects(tmux_version="")
|
|
self.assertEqual(self.run_loop(["--help"], fake)[0], Stop(0, usage.HELP))
|
|
self.assertEqual(fake.tmux_calls, [])
|
|
|
|
def test_starts_the_chunks_then_attaches_to_the_session_or_inside_tmux_switches_to_it(self):
|
|
for env, shown in (({}, ["attach-session", "-t", "=loop-r"]), ({"TMUX": "/tmp/tmux-1000/default,1,0"}, ["switch-client", "-t", "=loop-r"])):
|
|
with self.subTest(shown=shown):
|
|
fake = FakeEffects(git=["/r\n"])
|
|
stop, said = self.run_loop(["do", "it"], fake, env)
|
|
self.assertEqual((stop, said), (Stop(0, ""), f"{STAMP} started item-3, item-4 in the tmux session loop-r\n"))
|
|
self.assertEqual(fake.tmux_calls, [["-V"], shown])
|
|
self.assertEqual(self.asked, [(main.LoopRun(0, "/r", "do it", ""), False)])
|
|
|
|
def test_hands_the_window_starts_its_paths_environment_launcher_and_effects(self):
|
|
fake = FakeEffects(git=["/r\n"], on_terminal=False)
|
|
effects, env, launcher = fake.effects(), {"LOOP_CLAUDE_X": "1"}, pathlib.Path("/b/loop-claude")
|
|
asked = []
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
main.run_loop(["do it"], env, paths.paths_from(self.HOME), launcher, CWD, effects, steps(lambda *args: asked.append(args) or ["item-3"]))
|
|
self.assertEqual(asked, [(main.LoopRun(0, "/r", "do it", ""), False, paths.paths_from(self.HOME), env, launcher, effects)])
|
|
|
|
def test_a_window_loop_runs_its_chunk_in_its_own_session_and_window_and_hands_on_once_it_finished(self):
|
|
fake = FakeEffects(git=["/r\n"])
|
|
effects, env = fake.effects(), {"TMUX_PANE": "%4"}
|
|
launcher = pathlib.Path("/b/loop-claude")
|
|
cycle = unittest.mock.Mock(project=OPUS_PROJECT)
|
|
unfinished = unittest.mock.Mock(verdict=unittest.mock.Mock(spec=Unfinished))
|
|
finished = unittest.mock.Mock(verdict=unittest.mock.Mock(spec=Finished))
|
|
closed = [main.LoopRun(1, "/r", "do it", "", chunk="item-3"), main.LoopRun(2, "/r", "do it", "", chunk="item-3")]
|
|
concluded, cycles, started = [unfinished, finished], [], []
|
|
taken = steps(lambda *args: started.append(args) or [], fingerprint=lambda launcher: "", exec_if_edited=lambda *args: None, next_cycle=lambda *args: cycles.append(args) or cycle,
|
|
run_cycle=lambda cycle: concluded.pop(0), closed_cycle=lambda *args: closed.pop(0), check_routing=lambda *args: None)
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
stop = main.run_loop(["--chunk", "item-3", "do it"], env, paths.paths_from(self.HOME), launcher, CWD, effects, taken)
|
|
loop = this_loop.Loop(root="/r", session=session.session_named("A r loop-claude item-3", self.HOME / ".claude" / "projects"), window=tmux.Window("loop-r", "item-3", "%4"), paths=paths.paths_from(self.HOME), env=env, launcher=launcher, effects=effects)
|
|
self.assertEqual(stop, Stop(0, ""))
|
|
self.assertEqual(cycles, [(main.LoopRun(1, "/r", "do it", "", chunk="item-3"), loop), (main.LoopRun(2, "/r", "do it", "", chunk="item-3"), loop)])
|
|
self.assertEqual(started, [(main.LoopRun(2, "/r", "do it", "", chunk="item-3"), True, loop.paths, env, launcher, effects)])
|
|
|
|
def test_a_window_loop_given_no_task_prints_the_help(self):
|
|
self.assertEqual(main.loop_run_from(["--chunk", "item-3"], CWD, FakeEffects(git=["/r\n"]).effects()), Stop(1, usage.HELP))
|
|
|
|
def test_a_window_loop_outside_a_checkout_stops(self):
|
|
self.assertIsInstance(main.loop_run_from(["--chunk", "item-3", "do it"], CWD, FakeEffects(git=[Failure("git cannot start")]).effects()), Stop)
|
|
|
|
def test_says_that_nothing_started_where_the_session_already_runs_every_chunk_that_may(self):
|
|
self.assertEqual(self.run_loop(["do it"], FakeEffects(git=["/r\n"]), started=())[1], f"{STAMP} the tmux session loop-r runs every chunk that may run now, so `do it` started in no new window\n")
|
|
|
|
def test_off_a_terminal_it_prints_how_to_attach(self):
|
|
fake = FakeEffects(git=["/r\n"], on_terminal=False)
|
|
stop, said = self.run_loop(["do it"], fake)
|
|
self.assertEqual(stop, Stop(0, ""))
|
|
self.assertTrue(said.endswith(f"{STAMP} not on a terminal; see the chunks with `tmux attach -t =loop-r`\n"), said)
|
|
self.assertEqual(fake.tmux_calls, [["-V"]])
|
|
|
|
def test_stops_where_no_chunk_could_start(self):
|
|
self.assertEqual(self.run_loop(["do it"], FakeEffects(git=["/r\n"]), started=Failure("no plan"))[0], Stop(1, "no plan"))
|
|
|
|
def test_stops_where_tmux_cannot_show_the_session(self):
|
|
fake = FakeEffects(git=["/r\n"])
|
|
effects = dataclasses.replace(fake.effects(), tmux_on_terminal=lambda args: 1)
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
stop = main.run_loop(["do it"], {}, paths.paths_from(self.HOME), pathlib.Path("/b/loop-claude"), CWD, effects, steps(lambda *args: ["item-3"]))
|
|
self.assertEqual(stop, Stop(1, "`tmux attach-session -t =loop-r` failed; the chunks go on in the tmux session"))
|
|
|
|
def test_a_window_loop_outside_tmux_stops(self):
|
|
fake = FakeEffects(git=["/r\n"])
|
|
self.assertEqual(self.run_loop(["--chunk", "item-3", "do it"], fake)[0], Stop(1, "--chunk is how loop-claude starts itself in a tmux window; run loop-claude <task>"))
|
|
self.assertEqual(self.asked, [])
|
|
|
|
def test_reads_the_chunk_a_window_loop_is_started_with(self):
|
|
self.assertEqual(main.loop_run_from(["--chunk", "item-3", "do", "it"], CWD, FakeEffects(git=["/r\n"]).effects()), main.LoopRun(0, "/r", "do it", "", chunk="item-3"))
|
|
self.assertEqual(main.loop_run_from(["do", "it"], CWD, FakeEffects(git=["/r\n"]).effects()), main.LoopRun(0, "/r", "do it", ""))
|
|
|
|
def test_a_finished_chunk_starts_each_chunk_that_may_start_and_stops_without_a_reason(self):
|
|
loop_run = main.LoopRun(4, "/r", "do it", "", chunk="item-7")
|
|
for started, said, stop in ((["item-9", "item-12"], "the chunk finished; started item-9, item-12", Stop(0, "")), ([], "the chunk finished; started no other, since the chunks running fill what may run now", Stop(0, "")),
|
|
(Failure("no plan"), "", Stop(1, "the chunk finished, but the next could not start: no plan"))):
|
|
with self.subTest(started=started), contextlib.redirect_stdout(io.StringIO()) as printed:
|
|
loop, asked = a_loop(FakeEffects().effects(), env={"LOOP_CLAUDE_X": "1"}), []
|
|
self.assertEqual(main.handed_on(loop_run, loop, lambda *args: asked.append(args) or started), stop)
|
|
self.assertEqual(printed.getvalue(), f"{STAMP} {said}\n" if said else "")
|
|
self.assertEqual(asked, [(loop_run, True, loop.paths, loop.env, loop.launcher, loop.effects)])
|
|
|
|
|
|
class StopDms(unittest.TestCase):
|
|
def run_main(self, argv, fake):
|
|
with tempfile.TemporaryDirectory() as home, contextlib.redirect_stderr(io.StringIO()) as said, contextlib.redirect_stdout(io.StringIO()):
|
|
code = main.main(argv, {}, paths.paths_from(pathlib.Path(home)), pathlib.Path(home, "loop-claude"), CWD, fake.effects())
|
|
return code, said.getvalue()
|
|
|
|
def exit_code(self, run, fake):
|
|
with contextlib.redirect_stderr(io.StringIO()) as said, contextlib.redirect_stdout(io.StringIO()):
|
|
code = main.exit_code(run, CWD, fake.effects())
|
|
return code, said.getvalue()
|
|
|
|
def raising(self, error):
|
|
raise error
|
|
|
|
def test_dms_the_reason_of_a_stop_with_exit_2_such_as_a_chunk_left_unfinished_too_often(self):
|
|
fake = FakeEffects()
|
|
self.assertEqual(self.exit_code(lambda: Stop(2, "the human is needed to look at the chunk, but it stayed unfinished"), fake)[0], 2)
|
|
self.assertEqual(fake.dms, [f"loop-claude in {CWD} stopped: the human is needed to look at the chunk, but it stayed unfinished"])
|
|
|
|
def test_skips_the_dm_of_a_stop_without_a_reason_as_once_the_windows_started(self):
|
|
fake = FakeEffects()
|
|
self.assertEqual(self.exit_code(lambda: Stop(0, ""), fake), (0, ""))
|
|
self.assertEqual(fake.dms, [])
|
|
|
|
def test_skips_the_dm_where_ctrl_c_stopped_the_loop(self):
|
|
fake = FakeEffects()
|
|
self.assertEqual(self.exit_code(lambda: self.raising(KeyboardInterrupt()), fake)[0], 130)
|
|
self.assertEqual(fake.dms, [])
|
|
|
|
def test_dms_a_crash_before_it_reaches_the_terminal(self):
|
|
fake = FakeEffects()
|
|
with self.assertRaises(RuntimeError):
|
|
self.exit_code(lambda: self.raising(RuntimeError("boom")), fake)
|
|
self.assertEqual(fake.dms, [f"loop-claude in {CWD} crashed: RuntimeError('boom')"])
|
|
|
|
def test_dms_the_reason_where_a_loop_given_a_task_stops_with_exit_1(self):
|
|
fake = FakeEffects()
|
|
code, said = self.run_main([main.RELOAD_FLAG, "3"], fake)
|
|
self.assertEqual(code, 1)
|
|
self.assertEqual(fake.dms, [f"loop-claude in {CWD} stopped: the edited loop-claude cannot read the state the running one handed it, likely because the edit changed that handover; rerun loop-claude <task>. Handed: 3"])
|
|
|
|
def test_says_why_the_dm_of_a_stop_failed_after_the_reason(self):
|
|
fake = FakeEffects(dm_failures=[Failure("cannot reach https://chat.example: timed out")])
|
|
self.assertTrue(self.run_main([main.RELOAD_FLAG, "3"], fake)[1].endswith(f"Handed: 3\n{STAMP} could not DM you: cannot reach https://chat.example: timed out\n"))
|
|
|
|
def test_skips_the_dm_where_it_prints_its_help(self):
|
|
for argv, code in (([], 1), (["--help"], 0)):
|
|
with self.subTest(argv=argv):
|
|
fake = FakeEffects()
|
|
self.assertEqual(self.run_main(argv, fake)[0], code)
|
|
self.assertEqual(fake.dms, [])
|
|
|
|
|
|
class OwnLadder(unittest.TestCase):
|
|
def test_a_cycle_on_a_projects_own_ladder_names_its_file_and_says_where_the_workers_model_is_off_it(self):
|
|
file = pathlib.Path("/h/.claude-projects/acme/routing-ladder.json")
|
|
self.assertEqual(main.own_ladder(file, "claude-fable-5-2", False), f"routing on the project's own ladder, {file}.")
|
|
self.assertEqual(main.own_ladder(file, "claude-opus-5-5", True), f"routing on the project's own ladder, {file}. claude-opus-5-5 is not on it, so the worker runs on claude-opus-5-5 unrouted; LOOP_CLAUDE_WORKER_MODEL and LOOP_CLAUDE_JUDGE_MODEL name other models for a run.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|