Kill the mutants of the stuck check, its rung and the /proc readings it hands the investigation
This commit is contained in:
@@ -20,44 +20,50 @@ def verdict(said="stuck", process=8, why="pytest waits on a shell that ignores S
|
||||
return ran(json.dumps({"type": "result", "structured_output": {"verdict": said, "process": process, "why": why}}))
|
||||
|
||||
|
||||
def shell(cpu=2.0, started=NOW - 1000, written=NOW - 1000, runs_busy_tool=False):
|
||||
return Shell(pid=7, started=started, command="bin/run-tests", written=written, runs_busy_tool=runs_busy_tool, cpu=cpu, processes=("8 S 2.0 s python3 -m pytest",), output="collected 40 items")
|
||||
def shell(pid=7, cpu=2.0, started=NOW - 1000, written=NOW - 1000, runs_busy_tool=False, processes=("8 S 2.0 s python3 -m pytest", "9 S 0.0 s sh -c trap '' INT; sleep 999"), output="collected 40 items"):
|
||||
return Shell(pid=pid, started=started, command="bin/run-tests", written=written, runs_busy_tool=runs_busy_tool, cpu=cpu, processes=processes, output=output)
|
||||
|
||||
|
||||
class StuckWorkers(unittest.TestCase):
|
||||
def worker_with(self, *events, held=(), replies=()):
|
||||
"""Runs a worker whose claude prints events and holds the shells held, while each investigation replies in turn; returns the claude, the fake, what was said on stdout and stderr and the record's text."""
|
||||
def worker_with(self, *events, held=(), replies=(), archive=None):
|
||||
"""Runs a worker whose claude prints events and holds the shells held, while each investigation replies in turn; returns the claude, the fake, what was said on stdout and stderr and the record's text. The archive folder is archive, or else one that doesn't exist yet, as for a project's first stop."""
|
||||
clock = [NOW]
|
||||
streamed = Streamed(0, *events, clock=clock, held=held)
|
||||
fake = FakeEffects(streams=[streamed], claude=replies)
|
||||
effects = dataclasses.replace(fake.effects(), now=lambda: clock[0])
|
||||
with tempfile.TemporaryDirectory() as folder, contextlib.redirect_stdout(io.StringIO()) as said, contextlib.redirect_stderr(io.StringIO()) as failed:
|
||||
archive = pathlib.Path(folder, "transcripts", "project")
|
||||
archive = archive or pathlib.Path(folder, "transcripts", "project")
|
||||
worker.run_worker("/r", [], {}, archive, RUNG, effects)
|
||||
record = archive / stuck_workers.RECORD
|
||||
return streamed, fake, said.getvalue(), failed.getvalue(), record.read_text() if record.exists() else ""
|
||||
|
||||
def test_a_silent_worker_whose_shell_used_no_cpu_since_the_last_check_is_investigated_and_the_verdict_said_and_recorded_without_a_kill(self):
|
||||
streamed, fake, said, _, record = self.worker_with(ASSISTANT, None, None, held=[shell()], replies=[verdict()])
|
||||
[(args, prompt)] = fake.made("claude")
|
||||
streamed, fake, said, _, record = self.worker_with(ASSISTANT, None, None, None, held=[shell()], replies=[verdict(), verdict()])
|
||||
(args, prompt), _ = fake.made("claude")
|
||||
self.assertEqual(args[args.index("--model") + 1 :][:3], ["claude-opus-5-5", "--effort", "medium"])
|
||||
self.assertEqual(args[args.index("--tools") + 1], "")
|
||||
for evidence in ("bin/run-tests", "8 S 2.0 s python3 -m pytest", "collected 40 items", "working", "waiting on a person", "stuck"):
|
||||
self.assertIn(evidence, prompt)
|
||||
line = "stuck check: shell 7 has run 1600 s, written nothing for 1600 s and used no CPU for 300 s: bin/run-tests; a headless claude judged the worker stuck, held by process 8: pytest waits on a shell that ignores SIGINT"
|
||||
self.assertEqual(said, f"<{NOW + 600}> {line}\n")
|
||||
self.assertEqual(record, f"- {utc(NOW + 600)} {line}\n")
|
||||
self.assertEqual(json.loads(args[args.index("--json-schema") + 1])["properties"]["verdict"]["enum"], ["working", "waiting on a person", "stuck"])
|
||||
self.assertTrue(prompt.endswith("\n\nShell 7: bin/run-tests\nIts processes, as pid, state, CPU and command line:\n8 S 2.0 s python3 -m pytest\n9 S 0.0 s sh -c trap '' INT; sleep 999\nThe end of its output:\ncollected 40 items"))
|
||||
first, second = (f"stuck check: shell 7 has run {run} s, written nothing for {run} s and used no CPU for 300 s: bin/run-tests; a headless claude judged the worker stuck, held by process 8: pytest waits on a shell that ignores SIGINT" for run in (1600, 1900))
|
||||
self.assertEqual(said, f"<{NOW + 600}> {first}\n<{NOW + 900}> {second}\n")
|
||||
self.assertEqual(record, f"- {utc(NOW + 600)} {first}\n- {utc(NOW + 900)} {second}\n")
|
||||
self.assertEqual(streamed.stopped, [])
|
||||
|
||||
def test_hands_the_investigation_none_for_a_shell_without_processes_or_output(self):
|
||||
_, fake, _, _, _ = self.worker_with(ASSISTANT, None, None, held=[shell(processes=(), output="")], replies=[verdict()])
|
||||
[(_, prompt)] = fake.made("claude")
|
||||
self.assertTrue(prompt.endswith("\n\nShell 7: bin/run-tests\nIts processes, as pid, state, CPU and command line:\nnone\nThe end of its output:\nnone"))
|
||||
|
||||
def test_a_shell_older_than_a_chunk_usually_runs_is_investigated_at_the_first_check(self):
|
||||
_, fake, said, _, _ = self.worker_with(ASSISTANT, None, held=[shell(started=NOW - stuck_workers.OLD, written=NOW + 290)], replies=[verdict("working", 7, "the tests still print")])
|
||||
old = [shell(pid, started=NOW - stuck_workers.OLD, written=NOW + 290) for pid in (7, 10)]
|
||||
_, fake, said, _, _ = self.worker_with(ASSISTANT, None, held=old, replies=[verdict("working", 7, "the tests still print")])
|
||||
self.assertEqual(len(fake.made("claude")), 1)
|
||||
self.assertIn(f"shell 7 has run {stuck_workers.OLD + 300} s, written nothing for 10 s: bin/run-tests; a headless claude judged the worker working, held by process 7: the tests still print", said)
|
||||
self.assertIn(f"stuck check: shell 7 has run {stuck_workers.OLD + 300} s, written nothing for 10 s: bin/run-tests; shell 10 has run {stuck_workers.OLD + 300} s, written nothing for 10 s: bin/run-tests; a headless claude judged the worker working, held by process 7: the tests still print", said)
|
||||
|
||||
def test_a_worker_whose_shells_show_no_sign_is_not_investigated(self):
|
||||
for events, held in (
|
||||
((ASSISTANT, None, None), [shell(runs_busy_tool=True)]),
|
||||
((ASSISTANT, None, None), [shell(written=NOW + 600 - 179)]),
|
||||
((ASSISTANT, None, None), [shell(written=NOW + 600 - stuck_workers.QUIET)]),
|
||||
((ASSISTANT, None), [shell()]),
|
||||
((ASSISTANT, None, None), []),
|
||||
(({"type": "result"}, None, None), [shell()]),
|
||||
@@ -78,12 +84,19 @@ class StuckWorkers(unittest.TestCase):
|
||||
self.assertEqual(fake.made("claude"), [])
|
||||
|
||||
def test_an_investigation_that_fails_or_answers_outside_the_schema_is_said_on_stderr(self):
|
||||
for reply, reason in ((ran(code=1, stderr="no model"), "no model"), (verdict("hung"), "answered outside its schema")):
|
||||
limited = ran(json.dumps({"type": "result", "api_error_status": 429, "result": "You've hit your limit"}))
|
||||
for reply, reason in ((ran(code=1, stderr="no model"), "no model"), (verdict("hung"), "answered outside its schema"), (ran("crashed"), "the stuck check printed no json reply"), (limited, "claude hit a limit: You've hit your limit")):
|
||||
with self.subTest(reason=reason):
|
||||
_, _, said, failed, record = self.worker_with(ASSISTANT, None, None, held=[shell()], replies=[reply])
|
||||
self.assertEqual((said, record), ("", ""))
|
||||
self.assertIn(f"could not judge whether the worker is stuck: {reason}", failed)
|
||||
|
||||
def test_a_verdict_that_cannot_be_recorded_is_still_said_and_the_failure_said_on_stderr(self):
|
||||
with tempfile.NamedTemporaryFile() as blocker:
|
||||
_, _, said, failed, _ = self.worker_with(ASSISTANT, None, None, held=[shell()], replies=[verdict()], archive=pathlib.Path(blocker.name))
|
||||
self.assertIn("judged the worker stuck", said)
|
||||
self.assertIn(f"could not record the stuck check in {blocker.name}/{stuck_workers.RECORD}", failed)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -32,7 +32,7 @@ from claude_code import entries, reply, session
|
||||
from failure import Failure
|
||||
from agent_cycle.verdicts import Finished, Interrupt, Judged, QuotedAnswer, TakenIn, Unfinished
|
||||
from git_cli.git_repo import git
|
||||
from agent_cycle import interrupt, judge, lingering_workers, retry, run, spend, states, worker, worker_prompts
|
||||
from agent_cycle import interrupt, judge, lingering_workers, retry, run, spend, states, stuck_workers, worker, worker_prompts
|
||||
from claude_code.reply import Limited
|
||||
import loop_claude.routed_item
|
||||
from loop_claude.loop import main, reload, routing_check, this_loop, usage
|
||||
@@ -384,6 +384,23 @@ Worker: {OFF_LADDER}
|
||||
self.assertEqual(run.run_cycle(cycle).verdict, Finished("working"))
|
||||
self.assertEqual(fake.worker_environments, [{"LOOP_CLAUDE_PROJECT_RUNG": "claude-opus-5-5 at medium effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-haiku-5-5 at max effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "5", "CLAUDE_CODE_AUTO_COMPACT_WINDOW": "200000", "BASH_MAX_OUTPUT_LENGTH": "6000"}])
|
||||
|
||||
def test_a_stuck_check_of_the_worker_runs_on_the_projects_rung_and_off_the_ladder_on_the_workers_own(self):
|
||||
for project, model in ((OPUS_PROJECT, "claude-opus-5-5"), (None, "claude-haiku-5-5")):
|
||||
with self.subTest(project=project):
|
||||
clock = [NOW]
|
||||
quiet = Shell(pid=7, started=NOW, command="bin/run-tests", written=NOW, runs_busy_tool=False)
|
||||
events = ({"type": "assistant", "message": {"content": []}}, None, None, {"type": "result"})
|
||||
stream = lambda args: (Worker(self.paths.claude_code_transcripts.parent)(args), Streamed(0, *events, clock=clock, held=[quiet]))[1]
|
||||
verdict = ran(json.dumps({"type": "result", "structured_output": {"verdict": "stuck", "process": 7, "why": "it waits"}}))
|
||||
fake = FakeEffects(streams=[stream], claude=[verdict, judged("finished")])
|
||||
cycle = dataclasses.replace(self.cycle(fake), project=project, route=Route(Rung("claude-haiku-5-5", "max")))
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
run.run_cycle(dataclasses.replace(cycle, effects=dataclasses.replace(cycle.effects, now=lambda: clock[0])))
|
||||
[investigator, _] = [args for args, _ in fake.made("claude")]
|
||||
self.assertEqual(investigator[investigator.index("--model") + 1], model)
|
||||
self.assertEqual(fake.worker_folders, [str(self.home)])
|
||||
self.assertIn("judged the worker stuck", (self.archive / stuck_workers.RECORD).read_text())
|
||||
|
||||
def test_the_claude_an_interrupt_resumes_is_handed_the_same_rungs_as_the_worker(self):
|
||||
def answer(args):
|
||||
with open(session.transcript_file(self.session), "a") as handle:
|
||||
|
||||
@@ -190,20 +190,30 @@ class ChildShells(unittest.TestCase):
|
||||
self.assertEqual(self.shell_of(to_pipe).written, self.shell_of(to_pipe).started)
|
||||
|
||||
def test_reads_the_end_of_a_shells_output_file_and_nothing_from_a_pipe(self):
|
||||
with tempfile.NamedTemporaryFile() as output:
|
||||
output.write(b"x" * child_shells.OUTPUT_BYTES + b"last line\n")
|
||||
output.flush()
|
||||
to_file = self.started("sleep 60; true", stdout=output)
|
||||
to_pipe = self.started("sleep 60; true", stdout=subprocess.PIPE)
|
||||
self.assertEqual(self.shell_of(to_file).output, "x" * (child_shells.OUTPUT_BYTES - 10) + "last line\n")
|
||||
self.assertEqual(self.shell_of(to_pipe).output, "")
|
||||
for written, want in ((b"x" * child_shells.OUTPUT_BYTES + b"last line\n", "x" * (child_shells.OUTPUT_BYTES - 10) + "last line\n"), (b"short \xff\n", "short \ufffd\n")):
|
||||
with self.subTest(want=want[-12:]), tempfile.NamedTemporaryFile() as output:
|
||||
output.write(written)
|
||||
output.flush()
|
||||
self.assertEqual(self.shell_of(self.started("sleep 60; true", stdout=output)).output, want)
|
||||
self.assertEqual(self.shell_of(self.started("sleep 60; true", stdout=subprocess.PIPE)).output, "")
|
||||
|
||||
def test_reads_each_process_below_a_shell_and_the_cpu_the_whole_tree_used(self):
|
||||
busy = self.started("timeout 5 sh -c 'while :; do :; done'; true")
|
||||
wait_until(lambda: self.shell_of(busy).cpu > 0.1)
|
||||
def test_reads_nothing_of_a_process_that_is_gone(self):
|
||||
gone = subprocess.Popen(["true"])
|
||||
gone.wait()
|
||||
self.assertEqual((child_shells.cpu_and_state(gone.pid), child_shells.output_tail(gone.pid)), ((0.0, "gone"), ""))
|
||||
|
||||
def test_reads_each_process_below_a_shell_and_the_user_and_kernel_cpu_the_whole_tree_used(self):
|
||||
# dd from /dev/zero spends its time in the kernel, and the sleep after it uses none, so the total must add both modes of every process.
|
||||
busy = self.started("dd if=/dev/zero of=/dev/null bs=1M & sleep 60; true")
|
||||
wait_until(lambda: len(child_shells.descendants(busy.pid)) == 2)
|
||||
before, started = self.shell_of(busy).cpu, time.monotonic()
|
||||
time.sleep(1)
|
||||
used = self.shell_of(busy).cpu - before
|
||||
self.assertGreater(used, 0.2 * (time.monotonic() - started))
|
||||
self.assertLess(used, 1.5 * (time.monotonic() - started))
|
||||
processes = self.shell_of(busy).processes
|
||||
self.assertEqual([line.split(" ", 4)[4] for line in processes], ["timeout 5 sh -c while :; do :; done", "sh -c while :; do :; done"])
|
||||
self.assertEqual(processes[1].split(" ")[1], "R")
|
||||
self.assertEqual([line.split(" ", 4)[4] for line in processes], ["dd if=/dev/zero of=/dev/null bs=1M", "sleep 60"])
|
||||
self.assertEqual([line.split(" ")[1] for line in processes], ["R", "S"])
|
||||
|
||||
def test_stops_a_shell_and_the_process_it_started_and_no_other(self):
|
||||
shell = self.started("sleep 60; true")
|
||||
|
||||
Reference in New Issue
Block a user