Files
claude-config/bin/chunk_driver/project/test_model_choice.py
T

110 lines
4.6 KiB
Python

import dataclasses
import fcntl
import json
import pathlib
import sys
import tempfile
import unittest
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2]))
from failure import Failure
from loop_effects.doubles import FakeEffects
from chunk_driver.drivers import LOOP_CLAUDE
from chunk_driver.project import model_choice
from loop_state.paths import paths_from
from text_files.file_writes import replace_file
from agent_cycle.stop import Stop
class Overridden(unittest.TestCase):
def test_the_environment_names_the_judges_or_the_workers_model_over_the_default(self):
self.assertEqual(model_choice.overridden({}, "saved"), model_choice.Models("saved", "saved"))
self.assertEqual(model_choice.overridden({"LOOP_CLAUDE_JUDGE_MODEL": "j", "LOOP_CLAUDE_WORKER_MODEL": ""}, "saved"), model_choice.Models("j", "saved"))
self.assertEqual(model_choice.overridden({"LOOP_CLAUDE_JUDGE_MODEL": "j", "LOOP_CLAUDE_WORKER_MODEL": "w"}, "saved"), model_choice.Models("j", "w"))
class SavingAModel(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
self.paths = paths_from(pathlib.Path(temp.name))
self.lock = self.paths.home / ".local/state/loop-claude/models.lock"
def test_adds_the_projects_model_to_those_the_file_holds(self):
self.paths.models_file.parent.mkdir()
self.paths.models_file.write_text(json.dumps({"other": "model-x"}))
self.assertIsNone(model_choice.save_model("proj", "model-b", self.paths, replace_file))
self.assertEqual(json.loads(self.paths.models_file.read_text()), {"other": "model-x", "proj": "model-b"})
def lock_is_held(self) -> bool:
with open(self.lock, "a") as other:
try:
fcntl.flock(other, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
return True
return False
def test_holds_the_models_lock_from_its_read_to_its_write(self):
self.paths.models_file.parent.mkdir()
held = []
class Watched(pathlib.PosixPath):
"""The models file, which notes whether the lock is held each time it is opened."""
def __fspath__(watched) -> str:
held.append(self.lock_is_held())
return super().__fspath__()
def replace(path: pathlib.Path, data: bytes) -> None:
held.append(self.lock_is_held())
paths = dataclasses.replace(self.paths, models_file=Watched(self.paths.models_file))
self.assertIsNone(model_choice.save_model("proj", "model-b", paths, replace))
self.assertEqual(held, [True, True])
self.assertFalse(self.lock_is_held())
def test_doesnt_save_anything_where_it_cannot_take_the_lock(self):
self.paths.models_file.parent.mkdir()
self.lock.parent.parent.mkdir(parents=True)
self.lock.parent.write_text("")
saved = model_choice.save_model("proj", "model-b", self.paths, replace_file)
assert isinstance(saved, Failure)
self.assertRegex(saved.reason, f"^could not save the model in {self.paths.models_file}: cannot lock {self.lock.parent}: \\w")
self.assertFalse(self.paths.models_file.exists())
def test_a_pick_is_saved_through_save_model_alone(self):
self.paths.models_file.parent.mkdir()
fake, held = FakeEffects(inputs=["my-model"]), []
def replace(path: pathlib.Path, data: bytes) -> Failure:
held.append(self.lock_is_held())
return Failure("could not save")
fake.replace_file = replace
self.assertEqual(model_choice.get_or_ask_model("proj", self.paths, LOOP_CLAUDE, fake.effects()), Stop(1, f"could not save the model in {self.paths.models_file}: could not save"))
self.assertEqual(held, [True])
class ModelPrompt(unittest.TestCase):
OFFERED = [("claude-opus-5-5", "Opus 5.5"), ("claude-haiku-4-5", "Haiku 4.5")]
def test_reads_a_number_as_the_offered_model_and_anything_else_as_a_name(self):
self.assertEqual(model_choice.model_picked(" 2 ", self.OFFERED), "claude-haiku-4-5")
self.assertEqual(model_choice.model_picked("opus", self.OFFERED), "opus")
self.assertEqual(model_choice.model_picked("3", []), "3")
self.assertEqual(model_choice.model_picked("3", self.OFFERED), Failure("pick 1 to 2"))
self.assertIsNone(model_choice.model_picked(" ", self.OFFERED))
def test_asks_again_until_an_answer_picks_a_model(self):
with tempfile.TemporaryDirectory() as empty:
fake = FakeEffects(inputs=["", "my-model"])
self.assertEqual(model_choice.ask_for_model("proj", pathlib.Path("/m.json"), pathlib.Path(empty), LOOP_CLAUDE, fake.effects()), "my-model")
self.assertEqual(len(fake.made("input")), 2)
fake = FakeEffects(inputs=[EOFError()])
self.assertEqual(model_choice.ask_for_model("proj", pathlib.Path("/m.json"), pathlib.Path(empty), LOOP_CLAUDE, fake.effects()), Stop(2, "the human is needed to pick proj's model, but the input ended"))
if __name__ == "__main__":
unittest.main()