Answer product-owner round 2: 0 🔴, 1 🟡 (0 misread); judge a shell again once it used CPU and went idle, retry a failed judgment, and name a process without a command line

This commit is contained in:
2026-11-19 10:02:12 +01:00
parent bfcff168c2
commit a49a790a36
2 changed files with 43 additions and 16 deletions
+20 -14
View File
@@ -1,8 +1,8 @@
"""Reports whether a worker that has gone quiet before its final result is stuck, as a headless claude judges it.
A worker can stall on a command that hangs, as a full test run did on a shell that ignored SIGINT (item 402). The check runs on the tick `lingering_workers` uses: CHECK_EVERY seconds after the worker's last line. It looks for two signs in each of the worker's shells. One is a shell that used no CPU since the last check and wrote no output for QUIET seconds. The other is a shell older than OLD. A shell that runs a busy tool, as `Shell.runs_busy_tool` says, shows neither sign, since waiting for a slot or a merge is work. A shell that holds a heavy slot shows them, since a hung test run holds one.
A worker can stall on a command that hangs, as a full test run did on a shell that ignored SIGINT (item 402). The check runs on the tick `lingering_workers` uses: CHECK_EVERY seconds after the worker's last line. It looks for two signs in each of the worker's shells. One is a shell that used no CPU since the last check and wrote no output for QUIET seconds. The other is a shell older than OLD. A shell that runs a busy tool, as `Shell.runs_busy_tool` says, shows neither sign, since waiting for a slot or a merge is work. A shell that holds a heavy slot can show them, since a hung test run holds one.
Where a shell shows a sign, a headless claude judges the worker from each shell's command, processes and output tail. These hold what the worker's tmux pane shows, so each chunk's loop checks its own worker, and no process watches a whole tmux session. Each shell is judged once, so a stall that lasts all night costs one call.
Where a shell shows a sign, a headless claude judges the worker from each shell's command, processes and output tail. These hold what the worker's tmux pane shows, so each chunk's loop checks its own worker, and no process watches a whole tmux session. A shell is judged again only where it used CPU since its last judgment and then went idle, so a stall that lasts all night costs one call.
The check only reports, on the terminal and in RECORD in the project's transcript archive. Killing the process it names waits until the recorded verdicts match what the human would have done.
"""
@@ -43,25 +43,26 @@ PROMPT = f"""A headless Claude Code worker hasn't printed anything for {CHECK_EV
@dataclasses.dataclass
class Watch:
"""What checks a worker for stuck shells: the rung its investigations run on, the project's transcript archive, each shell's CPU seconds at the last check, and the shells already judged."""
"""What checks a worker for stuck shells: the rung its investigations run on, the project's transcript archive, each shell's CPU seconds at the last check, and each judged shell's CPU seconds when it was judged."""
rung: Rung
archive: pathlib.Path
cpu: dict[int, float] = dataclasses.field(default_factory=dict)
judged: set[int] = dataclasses.field(default_factory=set)
judged: dict[int, float] = dataclasses.field(default_factory=dict)
def check(self, proc: Streaming, root: str, effects: Effects) -> None:
"""Has a headless claude judge the worker whose claude is proc, where a shell not yet judged shows a sign. The verdict, or why there is none, goes to the terminal and to RECORD."""
"""Has a headless claude judge the worker whose claude is proc, where a shell shows a sign and is due a judgment. The verdict, or why there is none, goes to the terminal and to RECORD."""
now = effects.now()
shells = proc.shells()
signs = {shell.pid: sign for shell in shells if (sign := sign_of(shell, self.cpu.get(shell.pid), now))}
self.cpu = {shell.pid: shell.cpu for shell in shells}
if not signs.keys() - self.judged:
before, self.cpu = self.cpu, {shell.pid: shell.cpu for shell in shells}
signs = {shell.pid: sign for shell in shells if (sign := sign_of(shell, before.get(shell.pid), now))}
# A shell is judged again only where it used CPU since its judgment and then went idle, so one stall costs one call, and a stall after a `working` verdict still gets one.
if not any(shell.pid not in self.judged or self.judged[shell.pid] != shell.cpu and idle(shell, before.get(shell.pid), now) for shell in shells if shell.pid in signs):
return
self.judged |= signs.keys()
judged = investigated(shells, self.rung, root, effects)
if isinstance(judged, Failure):
line = f"stuck check: could not judge whether the worker is stuck: {judged.reason}"
else:
self.judged |= {shell.pid: shell.cpu for shell in shells if shell.pid in signs}
line = f"stuck check: a headless claude judged the worker {judged['verdict']}, naming process {judged['process']} ({command_of(judged['process'], shells)}): {judged['why']}"
line = " ".join(line.split()) + "".join(f"\n - {' '.join(sign.split())}" for sign in signs.values())
say(line, effects, stdout=not isinstance(judged, Failure))
@@ -78,20 +79,25 @@ def sign_of(shell: Shell, cpu_before: float | None, now: float) -> str:
"""The sign that shell may hold its worker stuck, given the CPU seconds it had used at the last check; empty where it shows none."""
if shell.runs_busy_tool:
return ""
idle = cpu_before == shell.cpu and now - shell.written > QUIET
if not (idle or now - shell.started > OLD):
quiet = idle(shell, cpu_before, now)
if not (quiet or now - shell.started > OLD):
return ""
return f"shell {shell.pid} has run {int(now - shell.started)} s and written no output for {int(now - shell.written)} s{f', and used no CPU for {CHECK_EVERY} s' if idle else ''}: {shell.command}"
return f"shell {shell.pid} has run {int(now - shell.started)} s and written no output for {int(now - shell.written)} s{f', and used no CPU for {CHECK_EVERY} s' if quiet else ''}: {shell.command}"
def idle(shell: Shell, cpu_before: float | None, now: float) -> bool:
"""Whether shell used no CPU since the last check, when it had used cpu_before, and wrote no output for QUIET seconds."""
return cpu_before == shell.cpu and now - shell.written > QUIET
def command_of(pid: int, shells: list[Shell]) -> str:
"""The command line of the process pid among shells and the processes below them, as the evidence shows it."""
"""The command line of the process pid among shells and the processes below them, as the evidence shows it; `not in the evidence` where neither holds pid, and `no command line` where its line is empty, as a defunct process's is."""
for shell in shells:
if shell.pid == pid:
return shell.command
for line in shell.processes:
if line.split(" ", 1)[0] == str(pid):
return line.split(" ", 4)[4]
return line.split(" ", 4)[4] or "no command line"
return "not in the evidence"
+23 -2
View File
@@ -29,10 +29,12 @@ def sign(run, quiet, idle=True):
class StuckWorkers(unittest.TestCase):
def worker_with(self, *events, held=(), replies=(), archive=None):
"""Runs a worker whose claude prints events and holds the shells held, while each investigation replies in turn; returns the claude, the fake, what was said on stdout and stderr and the record's text. The archive folder is archive, or else one that doesn't exist yet, as for a project's first stop."""
def worker_with(self, *events, held=(), replies=(), archive=None, each_check=None):
"""Runs a worker whose claude prints events and holds the shells held, while each investigation replies in turn; returns the claude, the fake, what was said on stdout and stderr and the record's text. The archive folder is archive, or else one that doesn't exist yet, as for a project's first stop. Where each_check is given, each check finds the next of its shells alone."""
clock = [NOW]
streamed = Streamed(0, *events, clock=clock, held=held)
if each_check:
streamed.shells = lambda: [each_check.pop(0)]
fake = FakeEffects(streams=[streamed], claude=replies)
effects = dataclasses.replace(fake.effects(), now=lambda: clock[0])
with tempfile.TemporaryDirectory() as folder, contextlib.redirect_stdout(io.StringIO()) as said, contextlib.redirect_stderr(io.StringIO()) as failed:
@@ -53,6 +55,25 @@ class StuckWorkers(unittest.TestCase):
self.assertEqual(record, f"- {utc(NOW + 600)} {line}\n")
self.assertEqual(streamed.stopped, [])
def test_a_judged_shell_is_judged_again_only_after_it_used_cpu_and_went_idle_once_more(self):
cpu = [2.0, 2.0, 2.0, 3.0, 3.0]
_, fake, _, _, record = self.worker_with(ASSISTANT, *[None] * 5, each_check=[shell(cpu=each) for each in cpu], replies=[verdict("working", 8, "the tests run"), verdict()])
self.assertEqual(len(fake.made("claude")), 2)
self.assertEqual([line.split(" ")[1] for line in record.splitlines() if line.startswith("- ")], [utc(NOW + 600), utc(NOW + 1500)])
def test_an_old_shell_that_keeps_using_cpu_is_judged_once(self):
_, fake, _, _, _ = self.worker_with(ASSISTANT, None, None, None, each_check=[shell(cpu=each, started=NOW - stuck_workers.OLD) for each in (2.0, 3.0, 4.0)], replies=[verdict()])
self.assertEqual(len(fake.made("claude")), 1)
def test_a_shell_whose_investigation_failed_is_judged_at_the_next_check(self):
_, fake, said, _, _ = self.worker_with(ASSISTANT, None, None, None, held=[shell()], replies=[ran(code=1, stderr="no model"), verdict()])
self.assertEqual(len(fake.made("claude")), 2)
self.assertIn(f"<{NOW + 900}> stuck check: a headless claude judged the worker stuck", said)
def test_a_named_process_without_a_command_line_is_said_so(self):
_, _, said, _, _ = self.worker_with(ASSISTANT, None, None, held=[shell(processes=("8 Z 0.0 s ",))], replies=[verdict()])
self.assertIn("naming process 8 (no command line):", said)
def test_hands_the_investigation_none_for_a_shell_without_processes_or_output(self):
_, fake, _, _, _ = self.worker_with(ASSISTANT, None, None, held=[shell(processes=(), output="")], replies=[verdict()])
[(_, prompt)] = fake.made("claude")