133 lines
8.9 KiB
Python
133 lines
8.9 KiB
Python
import contextlib
|
|
import dataclasses
|
|
import io
|
|
import json
|
|
import pathlib
|
|
import tempfile
|
|
import unittest
|
|
|
|
from loop_effects.doubles import FakeEffects, NOW, ran, Streamed
|
|
from model_routing.ladder import Rung
|
|
from loop_effects.child_shells import Shell
|
|
from agent_cycle import stuck_workers, worker
|
|
from text_files.utc_stamp import utc
|
|
|
|
RUNG = Rung("claude-opus-5-5", "medium")
|
|
ASSISTANT = {"type": "assistant", "message": {"content": []}}
|
|
|
|
|
|
def verdict(said="stuck", process=8, why="pytest waits on a shell that ignores SIGINT"):
|
|
return ran(json.dumps({"type": "result", "structured_output": {"verdict": said, "process": process, "why": why}}))
|
|
|
|
|
|
def shell(pid=7, cpu=2.0, started=NOW - 1000, written=NOW - 1000, runs_busy_tool=False, processes=("8 S 2.0 s python3 -m pytest", "9 S 0.0 s sh -c trap '' INT; sleep 999"), output="collected 40 items", holds_slot=False):
|
|
return Shell(pid=pid, started=started, command="bin/run-tests", written=written, runs_busy_tool=runs_busy_tool, cpu=cpu, processes=processes, output=output, holds_slot=holds_slot)
|
|
|
|
|
|
def sign(run, quiet, idle=True):
|
|
return f"\n - shell 7 has run {run} s and written no output for {quiet} s{', and used no CPU for 300 s' if idle else ''}: bin/run-tests"
|
|
|
|
|
|
class StuckWorkers(unittest.TestCase):
|
|
def worker_with(self, *events, held=(), replies=(), archive=None, each_check=None):
|
|
"""Runs a worker whose claude prints events and holds the shells held, while each investigation replies in turn; returns the claude, the fake, what was said on stdout and stderr and the record's text. The archive folder is archive, or else one that doesn't exist yet, as for a project's first stop. Where each_check is given, each check finds the next of its shells alone."""
|
|
clock = [NOW]
|
|
streamed = Streamed(0, *events, clock=clock, held=held)
|
|
if each_check:
|
|
streamed.shells = lambda: [each_check.pop(0)]
|
|
fake = FakeEffects(streams=[streamed], claude=replies)
|
|
effects = dataclasses.replace(fake.effects(), now=lambda: clock[0])
|
|
with tempfile.TemporaryDirectory() as folder, contextlib.redirect_stdout(io.StringIO()) as said, contextlib.redirect_stderr(io.StringIO()) as failed:
|
|
archive = archive or pathlib.Path(folder, "transcripts", "project")
|
|
worker.run_worker("/r", [], {}, archive, RUNG, effects)
|
|
record = archive / stuck_workers.RECORD
|
|
return streamed, fake, said.getvalue(), failed.getvalue(), record.read_text() if record.exists() else ""
|
|
|
|
def test_a_silent_worker_whose_shell_used_no_cpu_since_the_last_check_is_investigated_once_and_the_verdict_said_and_recorded_without_a_kill(self):
|
|
streamed, fake, said, _, record = self.worker_with(ASSISTANT, None, None, None, held=[shell()], replies=[verdict()])
|
|
[(args, prompt)] = fake.made("claude")
|
|
self.assertEqual(args[args.index("--model") + 1 :][:3], ["claude-opus-5-5", "--effort", "medium"])
|
|
self.assertEqual(args[args.index("--tools") + 1 :][:2], ["", "--strict-mcp-config"])
|
|
self.assertEqual(json.loads(args[args.index("--json-schema") + 1])["properties"]["verdict"]["enum"], ["working", "waiting on a person", "stuck"])
|
|
self.assertTrue(prompt.endswith("\n\nShell 7: bin/run-tests\nIts processes, as pid, state, CPU and command line:\n8 S 2.0 s python3 -m pytest\n9 S 0.0 s sh -c trap '' INT; sleep 999\nThe end of its output:\ncollected 40 items"))
|
|
line = "stuck check: a headless claude judged the worker stuck, naming process 8 (python3 -m pytest): pytest waits on a shell that ignores SIGINT" + sign(1600, 1600)
|
|
self.assertEqual(said, f"<{NOW + 600}> {line}\n")
|
|
self.assertEqual(record, f"- {utc(NOW + 600)} {line}\n")
|
|
self.assertEqual(streamed.stopped, [])
|
|
|
|
def test_a_judged_shell_is_judged_again_only_after_it_used_cpu_and_went_idle_once_more(self):
|
|
cpu = [2.0, 2.0, 2.0, 3.0, 3.0]
|
|
_, fake, _, _, record = self.worker_with(ASSISTANT, *[None] * 5, each_check=[shell(cpu=each) for each in cpu], replies=[verdict("working", 8, "the tests run"), verdict()])
|
|
self.assertEqual(len(fake.made("claude")), 2)
|
|
self.assertEqual([line.split(" ")[1] for line in record.splitlines() if line.startswith("- ")], [utc(NOW + 600), utc(NOW + 1500)])
|
|
|
|
def test_an_old_shell_that_keeps_using_cpu_is_judged_once(self):
|
|
_, fake, _, _, _ = self.worker_with(ASSISTANT, None, None, None, each_check=[shell(cpu=each, started=NOW - stuck_workers.OLD) for each in (2.0, 3.0, 4.0)], replies=[verdict()])
|
|
self.assertEqual(len(fake.made("claude")), 1)
|
|
|
|
def test_a_shell_whose_investigation_failed_is_judged_at_the_next_check(self):
|
|
_, fake, said, _, _ = self.worker_with(ASSISTANT, None, None, None, held=[shell()], replies=[ran(code=1, stderr="no model"), verdict()])
|
|
self.assertEqual(len(fake.made("claude")), 2)
|
|
self.assertIn(f"<{NOW + 900}> stuck check: a headless claude judged the worker stuck", said)
|
|
|
|
def test_a_named_process_without_a_command_line_is_said_so(self):
|
|
_, _, said, _, _ = self.worker_with(ASSISTANT, None, None, held=[shell(processes=("8 Z 0.0 s ",))], replies=[verdict()])
|
|
self.assertIn("naming process 8 (no command line):", said)
|
|
|
|
def test_hands_the_investigation_none_for_a_shell_without_processes_or_output(self):
|
|
_, fake, _, _, _ = self.worker_with(ASSISTANT, None, None, held=[shell(processes=(), output="")], replies=[verdict()])
|
|
[(_, prompt)] = fake.made("claude")
|
|
self.assertTrue(prompt.endswith("\n\nShell 7: bin/run-tests\nIts processes, as pid, state, CPU and command line:\nnone\nThe end of its output:\nnone"))
|
|
|
|
def test_a_shell_older_than_a_chunk_usually_runs_is_investigated_at_the_first_check(self):
|
|
old = [shell(pid, started=NOW - stuck_workers.OLD, written=NOW + 290) for pid in (7, 10)]
|
|
_, fake, said, _, _ = self.worker_with(ASSISTANT, None, held=old, replies=[verdict("working", 7, "the tests still print")])
|
|
[(_, prompt)] = fake.made("claude")
|
|
self.assertIn("collected 40 items\n\nShell 10: bin/run-tests\n", prompt)
|
|
self.assertIn(f"stuck check: a headless claude judged the worker working, naming process 7 (bin/run-tests): the tests still print{sign(stuck_workers.OLD + 300, 10, idle=False)}{sign(stuck_workers.OLD + 300, 10, idle=False).replace('shell 7', 'shell 10')}\n", said)
|
|
|
|
def test_a_shell_holding_a_heavy_slot_is_investigated_and_the_verdict_said_on_one_line_with_each_sign_under_it(self):
|
|
_, _, said, _, _ = self.worker_with(ASSISTANT, None, None, held=[shell(holds_slot=True)], replies=[verdict(process=99, why="the tests\nhang")])
|
|
self.assertEqual(said, f"<{NOW + 600}> stuck check: a headless claude judged the worker stuck, naming process 99 (not in the evidence): the tests hang{sign(1600, 1600)}\n")
|
|
|
|
def test_a_worker_whose_shells_show_no_sign_is_not_investigated(self):
|
|
for events, held in (
|
|
((ASSISTANT, None, None), [shell(runs_busy_tool=True)]),
|
|
((ASSISTANT, None, None), [shell(written=NOW + 600 - stuck_workers.QUIET)]),
|
|
((ASSISTANT, None), [shell()]),
|
|
((ASSISTANT, None, None), []),
|
|
(({"type": "result"}, None, None), [shell()]),
|
|
((ASSISTANT, None), [shell(started=NOW + 300 - stuck_workers.OLD)]),
|
|
):
|
|
with self.subTest(events=events, held=held):
|
|
_, fake, said, _, record = self.worker_with(*events, held=held)
|
|
self.assertEqual((fake.made("claude"), record), ([], ""))
|
|
|
|
def test_a_shell_that_used_cpu_since_the_last_check_is_not_investigated(self):
|
|
clock = [NOW]
|
|
used = [shell(cpu=2.0), shell(cpu=2.5)]
|
|
streamed = Streamed(0, ASSISTANT, None, None, clock=clock)
|
|
streamed.shells = lambda: [used.pop(0)]
|
|
fake = FakeEffects(streams=[streamed])
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
worker.run_worker("/r", [], {}, pathlib.Path("/archive"), RUNG, dataclasses.replace(fake.effects(), now=lambda: clock[0]))
|
|
self.assertEqual(fake.made("claude"), [])
|
|
|
|
def test_an_investigation_that_fails_or_answers_outside_the_schema_is_said_on_stderr_and_recorded(self):
|
|
limited = ran(json.dumps({"type": "result", "api_error_status": 429, "result": "You've hit your limit"}))
|
|
for reply, reason in ((ran(code=1, stderr="no model"), "no model"), (verdict("hung"), "answered outside its schema: {'verdict': 'hung', 'process': 8, 'why': 'pytest waits on a shell that ignores SIGINT'}"), (ran("crashed"), "the stuck check printed no json reply; its last line was: crashed"), (limited, "claude hit a limit: You've hit your limit")):
|
|
with self.subTest(reason=reason):
|
|
_, _, said, failed, record = self.worker_with(ASSISTANT, None, None, held=[shell()], replies=[reply])
|
|
line = f"stuck check: could not judge whether the worker is stuck: {reason}{sign(1600, 1600)}"
|
|
self.assertEqual((said, failed, record), ("", f"<{NOW + 600}> {line}\n", f"- {utc(NOW + 600)} {line}\n"))
|
|
|
|
def test_a_verdict_that_cannot_be_recorded_is_still_said_and_the_failure_said_on_stderr(self):
|
|
with tempfile.NamedTemporaryFile() as blocker:
|
|
_, _, said, failed, _ = self.worker_with(ASSISTANT, None, None, held=[shell()], replies=[verdict()], archive=pathlib.Path(blocker.name))
|
|
self.assertIn("judged the worker stuck", said)
|
|
self.assertIn(f"could not record the stuck check in {blocker.name}/{stuck_workers.RECORD}", failed)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|