116 lines
7.3 KiB
Python
116 lines
7.3 KiB
Python
"""Reports whether a worker that has gone quiet before its final result is stuck, as a headless claude judges it.
|
|
|
|
A worker can stall on a command that hangs, as a full test run did on a shell that ignored SIGINT (item 402). The check runs on the tick `lingering_workers` uses: CHECK_EVERY seconds after the worker's last line. It looks for two signs in each of the worker's shells. One is a shell that used no CPU since the last check and wrote no output for QUIET seconds. The other is a shell older than OLD. A shell that runs a busy tool, as `Shell.runs_busy_tool` says, shows neither sign, since waiting for a slot or a merge is work. A shell that holds a heavy slot can show them, since a hung test run holds one.
|
|
|
|
Where a shell shows a sign, a headless claude judges the worker from each shell's command, processes and output tail. These hold what the worker's tmux pane shows, so each chunk's loop checks its own worker, and no process watches a whole tmux session. A shell is judged again only where it used CPU since its last judgment and then went idle, so a stall that lasts all night costs one call.
|
|
|
|
The check only reports, on the terminal and in RECORD in the project's transcript archive. Killing the process it names waits until the recorded verdicts match what the human would have done.
|
|
"""
|
|
import dataclasses
|
|
import json
|
|
import pathlib
|
|
import typing
|
|
|
|
from agent_cycle.lingering_workers import CHECK_EVERY, QUIET
|
|
from claude_code.reply import claude_reply, Limited
|
|
from loop_effects.effects import Effects, say, Streaming
|
|
from failure import Failure
|
|
from model_routing.ladder import Rung
|
|
from loop_effects.child_shells import Shell
|
|
from claude_code.entries import structured_output
|
|
from text_files.utc_stamp import utc
|
|
|
|
# Seconds after which a shell holds a worker longer than a chunk usually runs.
|
|
OLD = 4 * 3600
|
|
RECORD = "stuck-workers.md"
|
|
VERDICTS = ("working", "waiting on a person", "stuck")
|
|
SCHEMA = {
|
|
"type": "object",
|
|
"properties": {
|
|
"verdict": {"enum": list(VERDICTS)},
|
|
"process": {"type": "integer", "description": "The pid of the process that holds the worker, from the evidence."},
|
|
"why": {"type": "string", "description": "One sentence on why that process holds it."},
|
|
},
|
|
"required": ["verdict", "process", "why"],
|
|
"additionalProperties": False,
|
|
}
|
|
PROMPT = f"""A headless Claude Code worker hasn't printed anything for {CHECK_EVERY} seconds. Judge from its shells below whether it is `working`, `waiting on a person` or `stuck`. Name the pid of the process it waits on, and say why.
|
|
- Working: it waits on a process that will finish, such as a test run that still writes output or uses CPU, or run-heavy waiting for a slot.
|
|
- Waiting on a person: it waits on input or an approval only a person can give, such as a prompt read from a terminal.
|
|
- Stuck: it waits on a process that will never finish on its own, such as a command waiting on a child that never exits.
|
|
"""
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class Watch:
|
|
"""What checks a worker for stuck shells: the rung its investigations run on, the project's transcript archive, each shell's CPU seconds at the last check, and the CPU seconds of each shell that showed a sign at the latest judgment."""
|
|
rung: Rung
|
|
archive: pathlib.Path
|
|
cpu: dict[int, float] = dataclasses.field(default_factory=dict)
|
|
judged: dict[int, float] = dataclasses.field(default_factory=dict)
|
|
|
|
def check(self, proc: Streaming, root: str, effects: Effects) -> None:
|
|
"""Has a headless claude judge the worker whose claude is proc, where a shell shows a sign and is due a judgment. The verdict, or why there is none, goes to the terminal and to RECORD."""
|
|
now = effects.now()
|
|
shells = proc.shells()
|
|
before, self.cpu = self.cpu, {shell.pid: shell.cpu for shell in shells}
|
|
signs = {shell.pid: sign for shell in shells if (sign := sign_of(shell, before.get(shell.pid), now))}
|
|
# A shell is judged again only where it used CPU since its judgment and then went idle, so one stall costs one call, and a stall after a `working` verdict still gets one.
|
|
if not any(shell.pid not in self.judged or self.judged[shell.pid] != shell.cpu and idle(shell, before.get(shell.pid), now) for shell in shells if shell.pid in signs):
|
|
return
|
|
judged = investigated(shells, self.rung, root, effects)
|
|
if isinstance(judged, Failure):
|
|
line = f"stuck check: could not judge whether the worker is stuck: {judged.reason}"
|
|
else:
|
|
self.judged = {shell.pid: shell.cpu for shell in shells if shell.pid in signs}
|
|
line = f"stuck check: a headless claude judged the worker {judged['verdict']}, naming process {judged['process']} ({command_of(judged['process'], shells)}): {judged['why']}"
|
|
line = " ".join(line.split()) + "".join(f"\n - {' '.join(sign.split())}" for sign in signs.values())
|
|
say(line, effects, stdout=not isinstance(judged, Failure))
|
|
record = self.archive / RECORD
|
|
try:
|
|
record.parent.mkdir(parents=True, exist_ok=True)
|
|
with record.open("a") as handle:
|
|
handle.write(f"- {utc(now)} {line}\n")
|
|
except OSError as failure:
|
|
say(f"could not record the stuck check in {record}: {failure.strerror}", effects)
|
|
|
|
|
|
def sign_of(shell: Shell, cpu_before: float | None, now: float) -> str:
|
|
"""The sign that shell may hold its worker stuck, given the CPU seconds it had used at the last check; empty where it shows none."""
|
|
if shell.runs_busy_tool:
|
|
return ""
|
|
quiet = idle(shell, cpu_before, now)
|
|
if not (quiet or now - shell.started > OLD):
|
|
return ""
|
|
return f"shell {shell.pid} has run {int(now - shell.started)} s and written no output for {int(now - shell.written)} s{f', and used no CPU for {CHECK_EVERY} s' if quiet else ''}: {shell.command}"
|
|
|
|
|
|
def idle(shell: Shell, cpu_before: float | None, now: float) -> bool:
|
|
"""Whether shell used no CPU since the last check, when it had used cpu_before, and wrote no output for QUIET seconds."""
|
|
return cpu_before == shell.cpu and now - shell.written > QUIET
|
|
|
|
|
|
def command_of(pid: int, shells: list[Shell]) -> str:
|
|
"""The command line of the process pid among shells and the processes below them, as the evidence shows it; `not in the evidence` where neither holds pid, and `no command line` where its line is empty, as a defunct process's is."""
|
|
for shell in shells:
|
|
if shell.pid == pid:
|
|
return shell.command
|
|
for line in shell.processes:
|
|
if line.startswith(f"{pid} "):
|
|
return line.split(" ", 4)[4] or "no command line"
|
|
return "not in the evidence"
|
|
|
|
|
|
def investigated(shells: list[Shell], rung: Rung, root: str, effects: Effects) -> dict[str, typing.Any] | Failure:
|
|
"""A headless claude's verdict on the worker holding shells, as SCHEMA has it, or why there is none. It runs without tools, so it judges only the evidence it is handed."""
|
|
evidence = "\n\n".join(f"Shell {shell.pid}: {shell.command}\nIts processes, as pid, state, CPU and command line:\n{chr(10).join(shell.processes) or 'none'}\nThe end of its output:\n{shell.output or 'none'}" for shell in shells)
|
|
reply = claude_reply(effects.claude, ["--json-schema", json.dumps(SCHEMA), *rung.flags(), "--tools", "", "--strict-mcp-config"], root, PROMPT + "\n" + evidence, "the stuck check")
|
|
if isinstance(reply, Limited):
|
|
return Failure(f"claude hit a limit: {reply.said}")
|
|
if isinstance(reply, Failure):
|
|
return reply
|
|
output = structured_output(reply)
|
|
if not (isinstance(output, dict) and output.get("verdict") in VERDICTS and isinstance(output.get("process"), int) and isinstance(output.get("why"), str)):
|
|
return Failure(f"answered outside its schema: {output!r}")
|
|
return output
|