Answer product-owner round 1: 0 🔴, 3 🟡 (0 misread); check a shell that holds a heavy slot, name the process's command, and judge each shell once
This commit is contained in:
@@ -18,12 +18,12 @@ RECORD = "lingering-workers.md"
|
||||
|
||||
|
||||
def stopped_shells(proc: Streaming, printed_result: bool, record: pathlib.Path, effects: Effects) -> None:
|
||||
"""Stops the shells of a claude that is still alive CHECK_EVERY seconds after it printed its final result (printed_result), saying each stop on the terminal and in record. A claude whose latest line is not a result is working and keeps its shells, and so does one with a shell that wrote output within QUIET seconds or runs a busy tool, as `Shell.runs_busy_tool` says. A subagent's shells are children of the claude too, so they count like its own: a subagent that prints keeps them, and one inside a single quiet call loses it."""
|
||||
"""Stops the shells of a claude that is still alive CHECK_EVERY seconds after it printed its final result (printed_result), saying each stop on the terminal and in record. A claude whose latest line is not a result is working and keeps its shells, and so does one with a shell that wrote output within QUIET seconds or runs a busy tool or holds a heavy slot, as `Shell.runs_busy_tool` and `Shell.holds_slot` say. A subagent's shells are children of the claude too, so they count like its own: a subagent that prints keeps them, and one inside a single quiet call loses it."""
|
||||
if not printed_result:
|
||||
return
|
||||
now = effects.now()
|
||||
shells = proc.shells()
|
||||
if any(now - shell.written <= QUIET or shell.runs_busy_tool for shell in shells):
|
||||
if any(now - shell.written <= QUIET or shell.runs_busy_tool or shell.holds_slot for shell in shells):
|
||||
return
|
||||
for shell in shells:
|
||||
proc.stop(shell)
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
"""Asks a headless claude whether a worker that prints nothing before its final result is stuck, where one of its shells shows a sign of it, and reports the verdict.
|
||||
"""Reports whether a worker that has gone quiet before its final result is stuck, as a headless claude judges it.
|
||||
|
||||
A worker can stall on a command that hangs, as a full test run did on a shell that ignored SIGINT (item 402). The check runs on the tick `lingering_workers` uses, CHECK_EVERY seconds after the worker's last line, so a worker that prints is never checked. A shell shows a sign where it used no CPU since the last check and wrote nothing for QUIET seconds, or where it is older than OLD; one that runs a busy tool, as `Shell.runs_busy_tool` says, shows none, since a wait for a gate is work. Each chunk's loop checks its own worker: no process watches a whole tmux session, and the shells' commands, processes and output hold what the worker's pane shows.
|
||||
A worker can stall on a command that hangs, as a full test run did on a shell that ignored SIGINT (item 402). The check runs on the tick `lingering_workers` uses: CHECK_EVERY seconds after the worker's last line. It looks for two signs in each of the worker's shells. One is a shell that used no CPU since the last check and wrote no output for QUIET seconds. The other is a shell older than OLD. A shell that runs a busy tool, as `Shell.runs_busy_tool` says, shows neither sign, since waiting for a slot or a merge is work. A shell that holds a heavy slot shows them, since a hung test run holds one.
|
||||
|
||||
The verdict is only reported, on the terminal and in RECORD in the project's transcript archive, and kills nothing until its verdicts have matched what the human would have done.
|
||||
Where a shell shows a sign, a headless claude judges the worker from each shell's command, processes and output tail. These hold what the worker's tmux pane shows, so each chunk's loop checks its own worker, and no process watches a whole tmux session. Each shell is judged once, so a stall that lasts all night costs one call.
|
||||
|
||||
The check only reports, on the terminal and in RECORD in the project's transcript archive. Killing the process it names waits until the recorded verdicts match what the human would have done.
|
||||
"""
|
||||
import dataclasses
|
||||
import json
|
||||
@@ -32,32 +34,37 @@ SCHEMA = {
|
||||
"required": ["verdict", "process", "why"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
PROMPT = f"""A headless Claude Code worker has printed nothing for {CHECK_EVERY} seconds. Judge from its shells below whether it is `working`, `waiting on a person` or `stuck`, and name the pid of the process that holds it and why.
|
||||
A worker that waits on a gate, such as run-heavy's wait for a slot or a test run that still writes output or uses CPU, is working. Stuck means the process will not finish on its own, as a command waiting on a child that never exits.
|
||||
PROMPT = f"""A headless Claude Code worker hasn't printed anything for {CHECK_EVERY} seconds. Judge from its shells below whether it is `working`, `waiting on a person` or `stuck`. Name the pid of the process it waits on, and say why.
|
||||
- Working: it waits on a process that will finish, such as a test run that still writes output or uses CPU, or run-heavy waiting for a slot.
|
||||
- Waiting on a person: it waits on input or an approval only a person can give, such as a prompt read from a terminal.
|
||||
- Stuck: it waits on a process that will never finish on its own, such as a command waiting on a child that never exits.
|
||||
"""
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class Watch:
|
||||
"""What checks a worker for stuck shells: the rung its investigations run on, the project's transcript archive, and each shell's CPU seconds at the last check."""
|
||||
"""What checks a worker for stuck shells: the rung its investigations run on, the project's transcript archive, each shell's CPU seconds at the last check, and the shells already judged."""
|
||||
rung: Rung
|
||||
archive: pathlib.Path
|
||||
cpu: dict[int, float] = dataclasses.field(default_factory=dict)
|
||||
judged: set[int] = dataclasses.field(default_factory=set)
|
||||
|
||||
def check(self, proc: Streaming, root: str, effects: Effects) -> None:
|
||||
"""Investigates the worker whose claude is proc where a shell shows a sign, and says the verdict on the terminal and in RECORD; a failed investigation is said on stderr."""
|
||||
"""Has a headless claude judge the worker whose claude is proc, where a shell not yet judged shows a sign. The verdict, or why there is none, goes to the terminal and to RECORD."""
|
||||
now = effects.now()
|
||||
shells = proc.shells()
|
||||
signs = [sign for shell in shells if (sign := sign_of(shell, self.cpu.get(shell.pid), now))]
|
||||
signs = {shell.pid: sign for shell in shells if (sign := sign_of(shell, self.cpu.get(shell.pid), now))}
|
||||
self.cpu = {shell.pid: shell.cpu for shell in shells}
|
||||
if not signs:
|
||||
if not signs.keys() - self.judged:
|
||||
return
|
||||
self.judged |= signs.keys()
|
||||
judged = investigated(shells, self.rung, root, effects)
|
||||
if isinstance(judged, Failure):
|
||||
say(f"could not judge whether the worker is stuck: {judged.reason}", effects)
|
||||
return
|
||||
line = f"stuck check: {'; '.join(signs)}; a headless claude judged the worker {judged['verdict']}, held by process {judged['process']}: {judged['why']}"
|
||||
say(line, effects, stdout=True)
|
||||
line = f"stuck check: could not judge whether the worker is stuck: {judged.reason}"
|
||||
else:
|
||||
line = f"stuck check: a headless claude judged the worker {judged['verdict']}, naming process {judged['process']} ({command_of(judged['process'], shells)}): {judged['why']}"
|
||||
line = " ".join(line.split()) + "".join(f"\n - {' '.join(sign.split())}" for sign in signs.values())
|
||||
say(line, effects, stdout=not isinstance(judged, Failure))
|
||||
record = self.archive / RECORD
|
||||
try:
|
||||
record.parent.mkdir(parents=True, exist_ok=True)
|
||||
@@ -68,13 +75,24 @@ class Watch:
|
||||
|
||||
|
||||
def sign_of(shell: Shell, cpu_before: float | None, now: float) -> str:
|
||||
"""What says shell may hold its worker stuck, given the CPU seconds it had used at the last check; empty where nothing does."""
|
||||
"""The sign that shell may hold its worker stuck, given the CPU seconds it had used at the last check; empty where it shows none."""
|
||||
if shell.runs_busy_tool:
|
||||
return ""
|
||||
idle = cpu_before == shell.cpu and now - shell.written > QUIET
|
||||
if not (idle or now - shell.started > OLD):
|
||||
return ""
|
||||
return f"shell {shell.pid} has run {int(now - shell.started)} s, written nothing for {int(now - shell.written)} s{f' and used no CPU for {CHECK_EVERY} s' if idle else ''}: {shell.command}"
|
||||
return f"shell {shell.pid} has run {int(now - shell.started)} s and written no output for {int(now - shell.written)} s{f', and used no CPU for {CHECK_EVERY} s' if idle else ''}: {shell.command}"
|
||||
|
||||
|
||||
def command_of(pid: int, shells: list[Shell]) -> str:
|
||||
"""The command line of the process pid among shells and the processes below them, as the evidence shows it."""
|
||||
for shell in shells:
|
||||
if shell.pid == pid:
|
||||
return shell.command
|
||||
for line in shell.processes:
|
||||
if line.split(" ", 1)[0] == str(pid):
|
||||
return line.split(" ", 4)[4]
|
||||
return "not in the evidence"
|
||||
|
||||
|
||||
def investigated(shells: list[Shell], rung: Rung, root: str, effects: Effects) -> dict[str, typing.Any] | Failure:
|
||||
|
||||
@@ -20,8 +20,12 @@ def verdict(said="stuck", process=8, why="pytest waits on a shell that ignores S
|
||||
return ran(json.dumps({"type": "result", "structured_output": {"verdict": said, "process": process, "why": why}}))
|
||||
|
||||
|
||||
def shell(pid=7, cpu=2.0, started=NOW - 1000, written=NOW - 1000, runs_busy_tool=False, processes=("8 S 2.0 s python3 -m pytest", "9 S 0.0 s sh -c trap '' INT; sleep 999"), output="collected 40 items"):
|
||||
return Shell(pid=pid, started=started, command="bin/run-tests", written=written, runs_busy_tool=runs_busy_tool, cpu=cpu, processes=processes, output=output)
|
||||
def shell(pid=7, cpu=2.0, started=NOW - 1000, written=NOW - 1000, runs_busy_tool=False, processes=("8 S 2.0 s python3 -m pytest", "9 S 0.0 s sh -c trap '' INT; sleep 999"), output="collected 40 items", holds_slot=False):
|
||||
return Shell(pid=pid, started=started, command="bin/run-tests", written=written, runs_busy_tool=runs_busy_tool, cpu=cpu, processes=processes, output=output, holds_slot=holds_slot)
|
||||
|
||||
|
||||
def sign(run, quiet, idle=True):
|
||||
return f"\n - shell 7 has run {run} s and written no output for {quiet} s{', and used no CPU for 300 s' if idle else ''}: bin/run-tests"
|
||||
|
||||
|
||||
class StuckWorkers(unittest.TestCase):
|
||||
@@ -37,16 +41,16 @@ class StuckWorkers(unittest.TestCase):
|
||||
record = archive / stuck_workers.RECORD
|
||||
return streamed, fake, said.getvalue(), failed.getvalue(), record.read_text() if record.exists() else ""
|
||||
|
||||
def test_a_silent_worker_whose_shell_used_no_cpu_since_the_last_check_is_investigated_and_the_verdict_said_and_recorded_without_a_kill(self):
|
||||
streamed, fake, said, _, record = self.worker_with(ASSISTANT, None, None, None, held=[shell()], replies=[verdict(), verdict()])
|
||||
(args, prompt), _ = fake.made("claude")
|
||||
def test_a_silent_worker_whose_shell_used_no_cpu_since_the_last_check_is_investigated_once_and_the_verdict_said_and_recorded_without_a_kill(self):
|
||||
streamed, fake, said, _, record = self.worker_with(ASSISTANT, None, None, None, held=[shell()], replies=[verdict()])
|
||||
[(args, prompt)] = fake.made("claude")
|
||||
self.assertEqual(args[args.index("--model") + 1 :][:3], ["claude-opus-5-5", "--effort", "medium"])
|
||||
self.assertEqual(args[args.index("--tools") + 1], "")
|
||||
self.assertEqual(json.loads(args[args.index("--json-schema") + 1])["properties"]["verdict"]["enum"], ["working", "waiting on a person", "stuck"])
|
||||
self.assertTrue(prompt.endswith("\n\nShell 7: bin/run-tests\nIts processes, as pid, state, CPU and command line:\n8 S 2.0 s python3 -m pytest\n9 S 0.0 s sh -c trap '' INT; sleep 999\nThe end of its output:\ncollected 40 items"))
|
||||
first, second = (f"stuck check: shell 7 has run {run} s, written nothing for {run} s and used no CPU for 300 s: bin/run-tests; a headless claude judged the worker stuck, held by process 8: pytest waits on a shell that ignores SIGINT" for run in (1600, 1900))
|
||||
self.assertEqual(said, f"<{NOW + 600}> {first}\n<{NOW + 900}> {second}\n")
|
||||
self.assertEqual(record, f"- {utc(NOW + 600)} {first}\n- {utc(NOW + 900)} {second}\n")
|
||||
line = "stuck check: a headless claude judged the worker stuck, naming process 8 (python3 -m pytest): pytest waits on a shell that ignores SIGINT" + sign(1600, 1600)
|
||||
self.assertEqual(said, f"<{NOW + 600}> {line}\n")
|
||||
self.assertEqual(record, f"- {utc(NOW + 600)} {line}\n")
|
||||
self.assertEqual(streamed.stopped, [])
|
||||
|
||||
def test_hands_the_investigation_none_for_a_shell_without_processes_or_output(self):
|
||||
@@ -59,7 +63,11 @@ class StuckWorkers(unittest.TestCase):
|
||||
_, fake, said, _, _ = self.worker_with(ASSISTANT, None, held=old, replies=[verdict("working", 7, "the tests still print")])
|
||||
[(_, prompt)] = fake.made("claude")
|
||||
self.assertIn("collected 40 items\n\nShell 10: bin/run-tests\n", prompt)
|
||||
self.assertIn(f"stuck check: shell 7 has run {stuck_workers.OLD + 300} s, written nothing for 10 s: bin/run-tests; shell 10 has run {stuck_workers.OLD + 300} s, written nothing for 10 s: bin/run-tests; a headless claude judged the worker working, held by process 7: the tests still print", said)
|
||||
self.assertIn(f"stuck check: a headless claude judged the worker working, naming process 7 (bin/run-tests): the tests still print{sign(stuck_workers.OLD + 300, 10, idle=False)}{sign(stuck_workers.OLD + 300, 10, idle=False).replace('shell 7', 'shell 10')}\n", said)
|
||||
|
||||
def test_a_shell_holding_a_heavy_slot_is_investigated_and_the_verdict_said_on_one_line_with_each_sign_under_it(self):
|
||||
_, _, said, _, _ = self.worker_with(ASSISTANT, None, None, held=[shell(holds_slot=True)], replies=[verdict(process=99, why="the tests\nhang")])
|
||||
self.assertEqual(said, f"<{NOW + 600}> stuck check: a headless claude judged the worker stuck, naming process 99 (not in the evidence): the tests hang{sign(1600, 1600)}\n")
|
||||
|
||||
def test_a_worker_whose_shells_show_no_sign_is_not_investigated(self):
|
||||
for events, held in (
|
||||
@@ -84,13 +92,13 @@ class StuckWorkers(unittest.TestCase):
|
||||
worker.run_worker("/r", [], {}, pathlib.Path("/archive"), RUNG, dataclasses.replace(fake.effects(), now=lambda: clock[0]))
|
||||
self.assertEqual(fake.made("claude"), [])
|
||||
|
||||
def test_an_investigation_that_fails_or_answers_outside_the_schema_is_said_on_stderr(self):
|
||||
def test_an_investigation_that_fails_or_answers_outside_the_schema_is_said_on_stderr_and_recorded(self):
|
||||
limited = ran(json.dumps({"type": "result", "api_error_status": 429, "result": "You've hit your limit"}))
|
||||
for reply, reason in ((ran(code=1, stderr="no model"), "no model"), (verdict("hung"), "answered outside its schema"), (ran("crashed"), "the stuck check printed no json reply"), (limited, "claude hit a limit: You've hit your limit")):
|
||||
for reply, reason in ((ran(code=1, stderr="no model"), "no model"), (verdict("hung"), "answered outside its schema: {'verdict': 'hung', 'process': 8, 'why': 'pytest waits on a shell that ignores SIGINT'}"), (ran("crashed"), "the stuck check printed no json reply; its last line was: crashed"), (limited, "claude hit a limit: You've hit your limit")):
|
||||
with self.subTest(reason=reason):
|
||||
_, _, said, failed, record = self.worker_with(ASSISTANT, None, None, held=[shell()], replies=[reply])
|
||||
self.assertEqual((said, record), ("", ""))
|
||||
self.assertIn(f"could not judge whether the worker is stuck: {reason}", failed)
|
||||
line = f"stuck check: could not judge whether the worker is stuck: {reason}{sign(1600, 1600)}"
|
||||
self.assertEqual((said, failed, record), ("", f"<{NOW + 600}> {line}\n", f"- {utc(NOW + 600)} {line}\n"))
|
||||
|
||||
def test_a_verdict_that_cannot_be_recorded_is_still_said_and_the_failure_said_on_stderr(self):
|
||||
with tempfile.NamedTemporaryFile() as blocker:
|
||||
|
||||
@@ -850,8 +850,8 @@ class LingeringWorkers(unittest.TestCase):
|
||||
record = archive / lingering_workers.RECORD
|
||||
return streamed, said.getvalue(), record.read_text() if record.exists() else ""
|
||||
|
||||
def shell(self, written=NOW - 1000, runs_busy_tool=False):
|
||||
return Shell(pid=7, started=NOW - 1000, command="until false; do sleep 15; done", written=written, runs_busy_tool=runs_busy_tool)
|
||||
def shell(self, written=NOW - 1000, runs_busy_tool=False, holds_slot=False):
|
||||
return Shell(pid=7, started=NOW - 1000, command="until false; do sleep 15; done", written=written, runs_busy_tool=runs_busy_tool, holds_slot=holds_slot)
|
||||
|
||||
def test_a_worker_that_lingers_after_its_result_loses_its_quiet_shell_and_the_stop_is_said_and_recorded_in_utc(self):
|
||||
shell = self.shell()
|
||||
@@ -867,8 +867,8 @@ class LingeringWorkers(unittest.TestCase):
|
||||
streamed, said, record = self.worker_with(*events, held=[self.shell()])
|
||||
self.assertEqual((streamed.stopped, said, record), ([], "", ""))
|
||||
|
||||
def test_a_worker_keeps_every_shell_while_one_of_them_writes_output_or_runs_a_busy_tool(self):
|
||||
for busy in (self.shell(written=NOW + 200), self.shell(runs_busy_tool=True)):
|
||||
def test_a_worker_keeps_every_shell_while_one_of_them_writes_output_runs_a_busy_tool_or_holds_a_heavy_slot(self):
|
||||
for busy in (self.shell(written=NOW + 200), self.shell(runs_busy_tool=True), self.shell(holds_slot=True)):
|
||||
with self.subTest(busy=busy):
|
||||
streamed, _, _ = self.worker_with(self.RESULT, None, held=[self.shell(), busy])
|
||||
self.assertEqual(streamed.stopped, [])
|
||||
|
||||
@@ -27,14 +27,16 @@ class Shell:
|
||||
command: str
|
||||
# Epoch seconds of the last write to the file the shell's output goes to; the shell's start where the output goes to no file, which the check cannot read.
|
||||
written: float
|
||||
# Whether a process below the shell runs one of BUSY_TOOLS or holds a heavy slot, a file of slot_folder that run-heavy leaves open in the command it runs.
|
||||
# Whether a process below the shell runs one of BUSY_TOOLS.
|
||||
runs_busy_tool: bool
|
||||
# CPU seconds the shell and every process below it have used.
|
||||
cpu: float = 0.0
|
||||
# Each process below the shell, as `<pid> <state> <CPU seconds> s <command line>`, with the state letter /proc gives, such as S for sleeping.
|
||||
processes: tuple[str, ...] = ()
|
||||
# The last OUTPUT_BYTES of the file the shell's output goes to; empty where it goes to no file.
|
||||
# The last OUTPUT_BYTES of the file the shell's output goes to; empty where it goes to a pipe or a terminal.
|
||||
output: str = ""
|
||||
# Whether a process below the shell holds a heavy slot: a file of slot_folder, which run-heavy leaves open in the command it runs.
|
||||
holds_slot: bool = False
|
||||
|
||||
|
||||
def stat_of(pid: int) -> tuple[str, list[str]] | None:
|
||||
@@ -55,7 +57,7 @@ def parent_and_start(pid: int) -> tuple[int, int, str] | None:
|
||||
|
||||
|
||||
def cpu_and_state(pid: int) -> tuple[float, str]:
|
||||
"""The CPU seconds a process has used, in user and kernel mode, and its state letter; none and `gone` where it is gone."""
|
||||
"""The CPU seconds a process has used, in user and kernel mode, and its state letter; 0.0 and `gone` where it is gone."""
|
||||
found = stat_of(pid)
|
||||
if found is None:
|
||||
return 0.0, "gone"
|
||||
@@ -129,7 +131,7 @@ def command_of(shell_argv: list[str]) -> str:
|
||||
return (command.rsplit(" < /dev/null", 1)[0].removeprefix("'").removesuffix("'") if found else line)[:COMMAND_CHARS]
|
||||
|
||||
|
||||
def holds_slot(pid: int, slots: pathlib.Path) -> bool:
|
||||
def has_slot(pid: int, slots: pathlib.Path) -> bool:
|
||||
"""Whether pid has a file of the heavy slots' folder open. run-heavy replaces itself with its command once it holds a slot, and the command keeps the slot's file open."""
|
||||
try:
|
||||
return any(os.readlink(f"/proc/{pid}/fd/{fd}").startswith(f"{slots}/") for fd in os.listdir(f"/proc/{pid}/fd"))
|
||||
@@ -137,9 +139,14 @@ def holds_slot(pid: int, slots: pathlib.Path) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def runs_busy_tool(pid: int, slots: pathlib.Path) -> bool:
|
||||
"""Whether a process below pid holds a heavy slot or has one of BUSY_TOOLS as its program or its script, as `python3 /x/chunk-pr`. A command line that merely names a tool, such as a `pgrep -f chunk-pr`, doesn't count."""
|
||||
return any(holds_slot(each, slots) or any(os.path.basename(name) in BUSY_TOOLS for name in argv_of(each)[:2]) for each in descendants(pid))
|
||||
def runs_busy_tool(pid: int) -> bool:
|
||||
"""Whether a process below pid has one of BUSY_TOOLS as its program or its script, as `python3 /x/chunk-pr`. A command line that merely names a tool, such as a `pgrep -f chunk-pr`, doesn't count."""
|
||||
return any(os.path.basename(name) in BUSY_TOOLS for each in descendants(pid) for name in argv_of(each)[:2])
|
||||
|
||||
|
||||
def holds_slot(pid: int, slots: pathlib.Path) -> bool:
|
||||
"""Whether a process below pid holds a heavy slot."""
|
||||
return any(has_slot(each, slots) for each in descendants(pid))
|
||||
|
||||
|
||||
def shells_of(claude_pid: int) -> list[Shell]:
|
||||
@@ -156,7 +163,7 @@ def shells_of(claude_pid: int) -> list[Shell]:
|
||||
continue
|
||||
started = booted + found[1] / ticks
|
||||
cpu, processes = tree_of(pid)
|
||||
shells.append(Shell(pid, started, command_of(argv_of(pid)), output_written(pid, started), runs_busy_tool(pid, slots), cpu, processes, output_tail(pid)))
|
||||
shells.append(Shell(pid, started, command_of(argv_of(pid)), output_written(pid, started), runs_busy_tool(pid), cpu, processes, output_tail(pid), holds_slot(pid, slots)))
|
||||
return shells
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user