86 lines
6.2 KiB
Python
86 lines
6.2 KiB
Python
import contextlib
|
|
import dataclasses
|
|
import io
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
|
|
|
from chunk_driver.doubles import a_loop
|
|
from chunk_driver import chunk_run, cycle_start
|
|
from loop_effects.doubles import FakeEffects, NOW
|
|
from agent_cycle.verdicts import Finished, TakenIn, Unfinished
|
|
from transcript_archive import outcomes, write
|
|
from model_routing.ladder import ProjectRung, Rung
|
|
from model_routing.ladder_file import read_ladder, SHIPPED
|
|
from text_files.file_writes import replace_file
|
|
from model_routing.route import Route
|
|
from agent_cycle import run
|
|
from claude_code import session
|
|
|
|
OPUS_PROJECT = ProjectRung(read_ladder(SHIPPED), Rung("claude-opus-5-5", "medium"))
|
|
|
|
|
|
class Steps(unittest.TestCase):
|
|
OPUS = Rung("claude-opus-5-5", "medium")
|
|
# The rungs an L 3 and an L 4 pick from OPUS.
|
|
FABLE_LOW = Rung("claude-fable-5-2", "low")
|
|
FABLE_MEDIUM = Rung("claude-fable-5-2", "medium")
|
|
|
|
def test_a_finished_verdict_clears_the_route_and_the_evidence(self):
|
|
run_with = chunk_run.ChunkRun(cycle_number=1, task="do it", unfinished_evidence="old", route=Route(self.OPUS))
|
|
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Finished("x"), run_with.route)), dataclasses.replace(run_with, route=None, unfinished_evidence=""))
|
|
self.assertEqual(chunk_run.after_verdict(dataclasses.replace(run_with, unfinished_run=3), OPUS_PROJECT, run.Concluded(Finished("x"), run_with.route)).unfinished_run, 0)
|
|
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", taken_in=TakenIn(highest_likelihood=4)), run_with.route)), dataclasses.replace(run_with, route=Route(self.FABLE_MEDIUM, unfinished=1), unfinished_evidence="new", unfinished_run=1))
|
|
|
|
def test_steps_from_the_route_its_cycle_ended_on_which_a_ruling_may_have_raised(self):
|
|
run_with = chunk_run.ChunkRun(cycle_number=1, task="do it", unfinished_evidence="", route=Route(self.OPUS, 3, 3, 4))
|
|
raised = Route(self.FABLE_MEDIUM, 3, 3, 4, escalated_at=5)
|
|
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new"), raised)).route, dataclasses.replace(raised, unfinished=1))
|
|
|
|
def test_keeps_the_route_its_cycle_ended_on_off_the_ladder(self):
|
|
off_ladder = Route(Rung("worker-model"))
|
|
run_with = chunk_run.ChunkRun(cycle_number=1, task="do it", unfinished_evidence="")
|
|
self.assertEqual(chunk_run.after_verdict(run_with, None, run.Concluded(Unfinished("new", "round 2: 🔴", TakenIn(highest_likelihood=4), entries_read=9), off_ladder)).route, off_ladder)
|
|
|
|
def test_an_unfinished_verdicts_escalation_marks_the_transcript_entries_the_judge_read(self):
|
|
run_with = chunk_run.ChunkRun(cycle_number=1, task="do it", unfinished_evidence="", route=Route(self.FABLE_LOW, 3, 3, 4, escalated_at=2))
|
|
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", "round 2: 🔴", entries_read=9), run_with.route)).route, Route(self.FABLE_MEDIUM, 3, 3, 4, escalated_at=9))
|
|
self.assertEqual(chunk_run.after_verdict(run_with, OPUS_PROJECT, run.Concluded(Unfinished("new", entries_read=9), run_with.route)).route, Route(self.FABLE_LOW, 3, 3, 4, unfinished=1, escalated_at=2))
|
|
|
|
def test_archives_a_step_up_as_it_happens_so_a_rerun_resumes_on_it(self):
|
|
with tempfile.TemporaryDirectory() as home:
|
|
opened = session.session_named("A proj a-driver", pathlib.Path(home, "projects"))
|
|
archive = pathlib.Path(home, "archive")
|
|
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
|
|
transcript.parent.mkdir(parents=True)
|
|
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
|
before = Route(self.FABLE_LOW, 3, 3, 4)
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW, replace=replace_file))
|
|
after = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, task="Do it.", unfinished_evidence=""), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
|
self.assertEqual(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_MEDIUM, 3, 3, 4))
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
chunk_run.closed_cycle(after, OPUS_PROJECT, run.Concluded(Finished("done"), after.route), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
|
self.assertEqual(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_MEDIUM, 3, 3, 4))
|
|
blocked = pathlib.Path(home, "blocked")
|
|
blocked.write_text("")
|
|
stop = chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, task="Do it.", unfinished_evidence=""), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_likelihood=4)), before), blocked, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
|
self.assertEqual((stop.code, stop.reason.split(":")[0]), (1, "could not archive the transcript"))
|
|
|
|
def test_archives_a_higher_e_plus_i_taken_in_so_a_rerun_keeps_it(self):
|
|
with tempfile.TemporaryDirectory() as home:
|
|
opened = session.session_named("A proj a-driver", pathlib.Path(home, "projects"))
|
|
archive = pathlib.Path(home, "archive")
|
|
transcript = pathlib.Path(home, "projects", "-fake", f"{opened.id}.jsonl")
|
|
transcript.parent.mkdir(parents=True)
|
|
transcript.write_text(json.dumps({"type": "queue-operation", "timestamp": "1"}) + "\n")
|
|
before = Route(self.FABLE_LOW, 3, 3, 4)
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertIsNone(write.archive(archive, str(transcript), "Do it.", outcomes.unfinished(before.worker_line(), "working", "j", ""), NOW, replace=replace_file))
|
|
chunk_run.closed_cycle(chunk_run.ChunkRun(cycle_number=1, task="Do it.", unfinished_evidence=""), OPUS_PROJECT, run.Concluded(Unfinished("working", taken_in=TakenIn(highest_escape_and_impact=6)), before), archive, "a-driver", a_loop(FakeEffects().effects(), session=opened))
|
|
self.assertEqual(cycle_start.archived_route(archive, a_loop(FakeEffects().effects(), session=opened)), Route(self.FABLE_LOW, 3, 3, 4, escape_and_impact_taken_in=6))
|