Kill the mutants of the stuck check, its rung and the /proc readings it hands the investigation

This commit is contained in:
2026-11-19 09:06:26 +01:00
parent 6b8fbc6ff5
commit a12710ebc4
3 changed files with 69 additions and 29 deletions
+29 -16
View File
@@ -20,44 +20,50 @@ def verdict(said="stuck", process=8, why="pytest waits on a shell that ignores S
return ran(json.dumps({"type": "result", "structured_output": {"verdict": said, "process": process, "why": why}}))
def shell(cpu=2.0, started=NOW - 1000, written=NOW - 1000, runs_busy_tool=False):
return Shell(pid=7, started=started, command="bin/run-tests", written=written, runs_busy_tool=runs_busy_tool, cpu=cpu, processes=("8 S 2.0 s python3 -m pytest",), output="collected 40 items")
def shell(pid=7, cpu=2.0, started=NOW - 1000, written=NOW - 1000, runs_busy_tool=False, processes=("8 S 2.0 s python3 -m pytest", "9 S 0.0 s sh -c trap '' INT; sleep 999"), output="collected 40 items"):
return Shell(pid=pid, started=started, command="bin/run-tests", written=written, runs_busy_tool=runs_busy_tool, cpu=cpu, processes=processes, output=output)
class StuckWorkers(unittest.TestCase):
def worker_with(self, *events, held=(), replies=()):
"""Runs a worker whose claude prints events and holds the shells held, while each investigation replies in turn; returns the claude, the fake, what was said on stdout and stderr and the record's text."""
def worker_with(self, *events, held=(), replies=(), archive=None):
"""Runs a worker whose claude prints events and holds the shells held, while each investigation replies in turn; returns the claude, the fake, what was said on stdout and stderr and the record's text. The archive folder is archive, or else one that doesn't exist yet, as for a project's first stop."""
clock = [NOW]
streamed = Streamed(0, *events, clock=clock, held=held)
fake = FakeEffects(streams=[streamed], claude=replies)
effects = dataclasses.replace(fake.effects(), now=lambda: clock[0])
with tempfile.TemporaryDirectory() as folder, contextlib.redirect_stdout(io.StringIO()) as said, contextlib.redirect_stderr(io.StringIO()) as failed:
archive = pathlib.Path(folder, "transcripts", "project")
archive = archive or pathlib.Path(folder, "transcripts", "project")
worker.run_worker("/r", [], {}, archive, RUNG, effects)
record = archive / stuck_workers.RECORD
return streamed, fake, said.getvalue(), failed.getvalue(), record.read_text() if record.exists() else ""
def test_a_silent_worker_whose_shell_used_no_cpu_since_the_last_check_is_investigated_and_the_verdict_said_and_recorded_without_a_kill(self):
streamed, fake, said, _, record = self.worker_with(ASSISTANT, None, None, held=[shell()], replies=[verdict()])
[(args, prompt)] = fake.made("claude")
streamed, fake, said, _, record = self.worker_with(ASSISTANT, None, None, None, held=[shell()], replies=[verdict(), verdict()])
(args, prompt), _ = fake.made("claude")
self.assertEqual(args[args.index("--model") + 1 :][:3], ["claude-opus-5-5", "--effort", "medium"])
self.assertEqual(args[args.index("--tools") + 1], "")
for evidence in ("bin/run-tests", "8 S 2.0 s python3 -m pytest", "collected 40 items", "working", "waiting on a person", "stuck"):
self.assertIn(evidence, prompt)
line = "stuck check: shell 7 has run 1600 s, written nothing for 1600 s and used no CPU for 300 s: bin/run-tests; a headless claude judged the worker stuck, held by process 8: pytest waits on a shell that ignores SIGINT"
self.assertEqual(said, f"<{NOW + 600}> {line}\n")
self.assertEqual(record, f"- {utc(NOW + 600)} {line}\n")
self.assertEqual(json.loads(args[args.index("--json-schema") + 1])["properties"]["verdict"]["enum"], ["working", "waiting on a person", "stuck"])
self.assertTrue(prompt.endswith("\n\nShell 7: bin/run-tests\nIts processes, as pid, state, CPU and command line:\n8 S 2.0 s python3 -m pytest\n9 S 0.0 s sh -c trap '' INT; sleep 999\nThe end of its output:\ncollected 40 items"))
first, second = (f"stuck check: shell 7 has run {run} s, written nothing for {run} s and used no CPU for 300 s: bin/run-tests; a headless claude judged the worker stuck, held by process 8: pytest waits on a shell that ignores SIGINT" for run in (1600, 1900))
self.assertEqual(said, f"<{NOW + 600}> {first}\n<{NOW + 900}> {second}\n")
self.assertEqual(record, f"- {utc(NOW + 600)} {first}\n- {utc(NOW + 900)} {second}\n")
self.assertEqual(streamed.stopped, [])
def test_hands_the_investigation_none_for_a_shell_without_processes_or_output(self):
_, fake, _, _, _ = self.worker_with(ASSISTANT, None, None, held=[shell(processes=(), output="")], replies=[verdict()])
[(_, prompt)] = fake.made("claude")
self.assertTrue(prompt.endswith("\n\nShell 7: bin/run-tests\nIts processes, as pid, state, CPU and command line:\nnone\nThe end of its output:\nnone"))
def test_a_shell_older_than_a_chunk_usually_runs_is_investigated_at_the_first_check(self):
_, fake, said, _, _ = self.worker_with(ASSISTANT, None, held=[shell(started=NOW - stuck_workers.OLD, written=NOW + 290)], replies=[verdict("working", 7, "the tests still print")])
old = [shell(pid, started=NOW - stuck_workers.OLD, written=NOW + 290) for pid in (7, 10)]
_, fake, said, _, _ = self.worker_with(ASSISTANT, None, held=old, replies=[verdict("working", 7, "the tests still print")])
self.assertEqual(len(fake.made("claude")), 1)
self.assertIn(f"shell 7 has run {stuck_workers.OLD + 300} s, written nothing for 10 s: bin/run-tests; a headless claude judged the worker working, held by process 7: the tests still print", said)
self.assertIn(f"stuck check: shell 7 has run {stuck_workers.OLD + 300} s, written nothing for 10 s: bin/run-tests; shell 10 has run {stuck_workers.OLD + 300} s, written nothing for 10 s: bin/run-tests; a headless claude judged the worker working, held by process 7: the tests still print", said)
def test_a_worker_whose_shells_show_no_sign_is_not_investigated(self):
for events, held in (
((ASSISTANT, None, None), [shell(runs_busy_tool=True)]),
((ASSISTANT, None, None), [shell(written=NOW + 600 - 179)]),
((ASSISTANT, None, None), [shell(written=NOW + 600 - stuck_workers.QUIET)]),
((ASSISTANT, None), [shell()]),
((ASSISTANT, None, None), []),
(({"type": "result"}, None, None), [shell()]),
@@ -78,12 +84,19 @@ class StuckWorkers(unittest.TestCase):
self.assertEqual(fake.made("claude"), [])
def test_an_investigation_that_fails_or_answers_outside_the_schema_is_said_on_stderr(self):
for reply, reason in ((ran(code=1, stderr="no model"), "no model"), (verdict("hung"), "answered outside its schema")):
limited = ran(json.dumps({"type": "result", "api_error_status": 429, "result": "You've hit your limit"}))
for reply, reason in ((ran(code=1, stderr="no model"), "no model"), (verdict("hung"), "answered outside its schema"), (ran("crashed"), "the stuck check printed no json reply"), (limited, "claude hit a limit: You've hit your limit")):
with self.subTest(reason=reason):
_, _, said, failed, record = self.worker_with(ASSISTANT, None, None, held=[shell()], replies=[reply])
self.assertEqual((said, record), ("", ""))
self.assertIn(f"could not judge whether the worker is stuck: {reason}", failed)
def test_a_verdict_that_cannot_be_recorded_is_still_said_and_the_failure_said_on_stderr(self):
with tempfile.NamedTemporaryFile() as blocker:
_, _, said, failed, _ = self.worker_with(ASSISTANT, None, None, held=[shell()], replies=[verdict()], archive=pathlib.Path(blocker.name))
self.assertIn("judged the worker stuck", said)
self.assertIn(f"could not record the stuck check in {blocker.name}/{stuck_workers.RECORD}", failed)
if __name__ == "__main__":
unittest.main()
+18 -1
View File
@@ -32,7 +32,7 @@ from claude_code import entries, reply, session
from failure import Failure
from agent_cycle.verdicts import Finished, Interrupt, Judged, QuotedAnswer, TakenIn, Unfinished
from git_cli.git_repo import git
from agent_cycle import interrupt, judge, lingering_workers, retry, run, spend, states, worker, worker_prompts
from agent_cycle import interrupt, judge, lingering_workers, retry, run, spend, states, stuck_workers, worker, worker_prompts
from claude_code.reply import Limited
import loop_claude.routed_item
from loop_claude.loop import main, reload, routing_check, this_loop, usage
@@ -384,6 +384,23 @@ Worker: {OFF_LADDER}
self.assertEqual(run.run_cycle(cycle).verdict, Finished("working"))
self.assertEqual(fake.worker_environments, [{"LOOP_CLAUDE_PROJECT_RUNG": "claude-opus-5-5 at medium effort", "LOOP_CLAUDE_WORKER_RUNG": "claude-haiku-5-5 at max effort", "LOOP_CLAUDE_ESCAPE_AND_IMPACT": "5", "CLAUDE_CODE_AUTO_COMPACT_WINDOW": "200000", "BASH_MAX_OUTPUT_LENGTH": "6000"}])
def test_a_stuck_check_of_the_worker_runs_on_the_projects_rung_and_off_the_ladder_on_the_workers_own(self):
for project, model in ((OPUS_PROJECT, "claude-opus-5-5"), (None, "claude-haiku-5-5")):
with self.subTest(project=project):
clock = [NOW]
quiet = Shell(pid=7, started=NOW, command="bin/run-tests", written=NOW, runs_busy_tool=False)
events = ({"type": "assistant", "message": {"content": []}}, None, None, {"type": "result"})
stream = lambda args: (Worker(self.paths.claude_code_transcripts.parent)(args), Streamed(0, *events, clock=clock, held=[quiet]))[1]
verdict = ran(json.dumps({"type": "result", "structured_output": {"verdict": "stuck", "process": 7, "why": "it waits"}}))
fake = FakeEffects(streams=[stream], claude=[verdict, judged("finished")])
cycle = dataclasses.replace(self.cycle(fake), project=project, route=Route(Rung("claude-haiku-5-5", "max")))
with contextlib.redirect_stdout(io.StringIO()):
run.run_cycle(dataclasses.replace(cycle, effects=dataclasses.replace(cycle.effects, now=lambda: clock[0])))
[investigator, _] = [args for args, _ in fake.made("claude")]
self.assertEqual(investigator[investigator.index("--model") + 1], model)
self.assertEqual(fake.worker_folders, [str(self.home)])
self.assertIn("judged the worker stuck", (self.archive / stuck_workers.RECORD).read_text())
def test_the_claude_an_interrupt_resumes_is_handed_the_same_rungs_as_the_worker(self):
def answer(args):
with open(session.transcript_file(self.session), "a") as handle:
+22 -12
View File
@@ -190,20 +190,30 @@ class ChildShells(unittest.TestCase):
self.assertEqual(self.shell_of(to_pipe).written, self.shell_of(to_pipe).started)
def test_reads_the_end_of_a_shells_output_file_and_nothing_from_a_pipe(self):
with tempfile.NamedTemporaryFile() as output:
output.write(b"x" * child_shells.OUTPUT_BYTES + b"last line\n")
output.flush()
to_file = self.started("sleep 60; true", stdout=output)
to_pipe = self.started("sleep 60; true", stdout=subprocess.PIPE)
self.assertEqual(self.shell_of(to_file).output, "x" * (child_shells.OUTPUT_BYTES - 10) + "last line\n")
self.assertEqual(self.shell_of(to_pipe).output, "")
for written, want in ((b"x" * child_shells.OUTPUT_BYTES + b"last line\n", "x" * (child_shells.OUTPUT_BYTES - 10) + "last line\n"), (b"short \xff\n", "short \ufffd\n")):
with self.subTest(want=want[-12:]), tempfile.NamedTemporaryFile() as output:
output.write(written)
output.flush()
self.assertEqual(self.shell_of(self.started("sleep 60; true", stdout=output)).output, want)
self.assertEqual(self.shell_of(self.started("sleep 60; true", stdout=subprocess.PIPE)).output, "")
def test_reads_each_process_below_a_shell_and_the_cpu_the_whole_tree_used(self):
busy = self.started("timeout 5 sh -c 'while :; do :; done'; true")
wait_until(lambda: self.shell_of(busy).cpu > 0.1)
def test_reads_nothing_of_a_process_that_is_gone(self):
gone = subprocess.Popen(["true"])
gone.wait()
self.assertEqual((child_shells.cpu_and_state(gone.pid), child_shells.output_tail(gone.pid)), ((0.0, "gone"), ""))
def test_reads_each_process_below_a_shell_and_the_user_and_kernel_cpu_the_whole_tree_used(self):
# dd from /dev/zero spends its time in the kernel, and the sleep after it uses none, so the total must add both modes of every process.
busy = self.started("dd if=/dev/zero of=/dev/null bs=1M & sleep 60; true")
wait_until(lambda: len(child_shells.descendants(busy.pid)) == 2)
before, started = self.shell_of(busy).cpu, time.monotonic()
time.sleep(1)
used = self.shell_of(busy).cpu - before
self.assertGreater(used, 0.2 * (time.monotonic() - started))
self.assertLess(used, 1.5 * (time.monotonic() - started))
processes = self.shell_of(busy).processes
self.assertEqual([line.split(" ", 4)[4] for line in processes], ["timeout 5 sh -c while :; do :; done", "sh -c while :; do :; done"])
self.assertEqual(processes[1].split(" ")[1], "R")
self.assertEqual([line.split(" ", 4)[4] for line in processes], ["dd if=/dev/zero of=/dev/null bs=1M", "sleep 60"])
self.assertEqual([line.split(" ")[1] for line in processes], ["R", "S"])
def test_stops_a_shell_and_the_process_it_started_and_no_other(self):
shell = self.started("sleep 60; true")