114 lines
8.2 KiB
Python
114 lines
8.2 KiB
Python
import contextlib
|
|
import dataclasses
|
|
import io
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from chunk_driver.doubles import a_loop
|
|
from chunk_driver import chunk_run, cycle_start
|
|
from loop_effects.doubles import FakeEffects, NOW
|
|
from agent_cycle.verdicts import Finished, TakenIn, Unfinished
|
|
from transcript_archive import outcomes, write
|
|
from model_routing.ladder import ProjectRung, Rung
|
|
from model_routing.ladder_file import read_ladder, SHIPPED
|
|
from text_files.file_writes import replace_file
|
|
from model_routing.route import Route
|
|
from agent_cycle import run
|
|
from claude_code import session
|
|
from agent_cycle.stop import Stop
|
|
|
|
OPUS_PROJECT = ProjectRung(read_ladder(SHIPPED), Rung("claude-opus-5-5", "medium"))
|
|
|
|
|
|
class Steps(unittest.TestCase):
|
|
OPUS = Rung("claude-opus-5-5", "medium")
|
|
# The rungs an L 3 and an L 4 pick from OPUS.
|
|
FABLE_LOW = Rung("claude-fable-5-2", "low")
|
|
FABLE_MEDIUM = Rung("claude-fable-5-2", "medium")
|
|
|
|
def test_a_finished_verdict_clears_the_route_and_the_evidence(self):
|
|
run_with = chunk_run.ChunkRun(cycle_number=1, unfinished_evidence="old", route=Route(self.OPUS))
|
|
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Finished("x"), run_with.route)), dataclasses.replace(run_with, route=None, unfinished_evidence=""))
|
|
self.assertEqual(chunk_run.after_verdict(dataclasses.replace(run_with, unfinished_run=3), OPUS_PROJECT, run.Concluded(Finished("x"), run_with.route)).unfinished_run, 0)
|
|
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", taken_in=TakenIn(highest_likelihood=4)), run_with.route)), dataclasses.replace(run_with, route=Route(self.FABLE_MEDIUM, unfinished=1), unfinished_evidence="new", unfinished_run=1))
|
|
|
|
def test_steps_from_the_route_its_cycle_ended_on_which_a_ruling_may_have_raised(self):
|
|
run_with = chunk_run.ChunkRun(cycle_number=1, unfinished_evidence="", route=Route(self.OPUS, 3, 3, 4))
|
|
raised = Route(self.FABLE_MEDIUM, 3, 3, 4, escalated_at=5)
|
|
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new"), raised)).route, dataclasses.replace(raised, unfinished=1))
|
|
|
|
def test_keeps_the_route_its_cycle_ended_on_off_the_ladder(self):
|
|
off_ladder = Route(Rung("worker-model"))
|
|
run_with = chunk_run.ChunkRun(cycle_number=1, unfinished_evidence="")
|
|
self.assertEqual(chunk_run.after_verdict(run_with, None, run.Concluded(Unfinished("new", "round 2: 🔴", TakenIn(highest_likelihood=4), entries_read=9), off_ladder)).route, off_ladder)
|
|
|
|
def test_an_unfinished_verdicts_escalation_marks_the_transcript_entries_the_judge_read(self):
|
|
run_with = chunk_run.ChunkRun(cycle_number=1, unfinished_evidence="", route=Route(self.FABLE_LOW, 3, 3, 4, escalated_at=2))
|
|
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", "round 2: 🔴", entries_read=9), run_with.route)).route, Route(self.FABLE_MEDIUM, 3, 3, 4, escalated_at=9))
|
|
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", entries_read=9), run_with.route)).route, Route(self.FABLE_LOW, 3, 3, 4, unfinished=1, escalated_at=2))
|
|
|
|
def test_archives_a_step_up_as_it_happens_so_a_rerun_resumes_on_it(self):
|
|
with tempfile.TemporaryDirectory() as home:
|
|
opened = session.session_named("A proj a-driver", pathlib.Path(home, "projects"))
|
|
archive = pathlib.Path(home, "archive")
|
|
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
|
|
transcript.parent.mkdir(parents=True)
|
|
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
|
before = Route(self.FABLE_LOW, 3, 3, 4)
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW, replace=replace_file))
|
|
after = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, unfinished_evidence=""), "Do it.", OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
|
self.assertEqual(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_MEDIUM, 3, 3, 4))
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
chunk_run.closed_cycle(after, "Do it.", OPUS_PROJECT, run.Concluded(Finished("done"), after.route), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
|
self.assertEqual(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_MEDIUM, 3, 3, 4))
|
|
blocked = pathlib.Path(home, "blocked")
|
|
blocked.write_text("")
|
|
stop = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, unfinished_evidence=""), "Do it.", OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), blocked, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
|
self.assertEqual((stop.code, stop.reason.split(":")[0]), (1, "could not archive the transcript"))
|
|
|
|
def outcomes_after(self, unfinished_run: int, taken_in: TakenIn) -> tuple[object, str]:
|
|
"""What closed_cycle answers for a-driver's chunk `Do it.`, after an unfinished verdict that follows unfinished_run others, and the outcomes it archived for a transcript not archived before."""
|
|
with tempfile.TemporaryDirectory() as home:
|
|
opened = session.session_named("A proj a-driver", pathlib.Path(home, "projects"))
|
|
archive = pathlib.Path(home, "archive")
|
|
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
|
|
transcript.parent.mkdir(parents=True)
|
|
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
|
before = Route(self.FABLE_LOW, 3, 3, 4)
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
closed = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, unfinished_evidence="", unfinished_run=unfinished_run), "Do it.", OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=taken_in), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
|
return closed, "".join(file.read_text() for file in archive.rglob("*") if file.is_file() and file.suffix != ".jsonl")
|
|
|
|
def test_a_step_up_archived_first_for_its_transcript_names_the_chunks_task(self):
|
|
closed, outcomes_kept = self.outcomes_after(0, TakenIn(highest_likelihood=4))
|
|
self.assertEqual(closed.route.rung, self.FABLE_MEDIUM)
|
|
self.assertIn("Do it.\n", outcomes_kept)
|
|
|
|
def test_the_sixth_unfinished_verdict_running_stops_for_the_human_naming_the_driver_to_rerun_and_archives_why_with_the_task(self):
|
|
reason = "the human is needed to go on with the chunk, but the judge ruled it unfinished 6 cycles running; finish the chunk yourself, or rerun a-driver to give it 6 more cycles"
|
|
closed, outcomes_kept = self.outcomes_after(5, TakenIn())
|
|
self.assertEqual(closed, Stop(2, reason))
|
|
self.assertIn("Do it.\n", outcomes_kept)
|
|
self.assertIn(f"Stopped: {reason}\n", outcomes_kept)
|
|
closed, outcomes_kept = self.outcomes_after(4, TakenIn())
|
|
self.assertEqual((closed.unfinished_run, outcomes_kept), (5, ""))
|
|
|
|
def test_archives_a_higher_e_plus_i_taken_in_so_a_rerun_keeps_it(self):
|
|
with tempfile.TemporaryDirectory() as home:
|
|
opened = session.session_named("A proj a-driver", pathlib.Path(home, "projects"))
|
|
archive = pathlib.Path(home, "archive")
|
|
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
|
|
transcript.parent.mkdir(parents=True)
|
|
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
|
before = Route(self.FABLE_LOW, 3, 3, 4)
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW, replace=replace_file))
|
|
chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, unfinished_evidence=""), "Do it.", OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_escape_and_impact=6)), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
|
self.assertEqual(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_LOW, 3, 3, 4, escape_and_impact_taken_in=6))
|