End a worker that lingers after its final result by stopping the quiet background shells it left
CI / tests (pull_request) Failing after 4m10s

This commit is contained in:
2026-11-18 16:53:30 +01:00
parent aa34fb8912
commit 72ffb9f748
9 changed files with 272 additions and 15 deletions
+1 -1
View File
@@ -183,7 +183,7 @@ Install puts only `loop-claude` on `PATH`; call the others by their path, such a
- `deploy-release [--cut] [<checkout>]` moves `~/.claude`, or the checkout named, to the newest release, as Update says; with `--cut`, run in the development clone, it first cuts the next release, as Release says.
- `judge-replay <model> [--offline]` replays loop-claude's judge on archived verdicts of the projects run on the model. It prints what each rung a review climbs to misses, and the lowest rung below those that returns every verdict, as each candidate above it does. With `--offline` it doesn't call a judge: it counts what a "?" rule, a decision table and a classifier trained on the archive decide of each verdict's kind from the worker's final message, and how many interrupts each misses. `judge-replay --help` states both replays.
- `larv-review <command>` holds larv-review's mechanical rules: the chunk's review folder, the paired ruling's base and vote count, each pass's round count and the opening of its round commit's title, the scoring run's trigger and setup; the comprehension history's format, its latest places and its count of how often seats agree; and each reviewer's prompt header. `larv-review --help` states them. `settings.json` → `hooks` runs `larv-review reviewer-context` as each subagent starts. A reviewer under `agents/` whose frontmatter has `omitClaudeMd: true` starts without `CLAUDE.md`, and the command hands it that file's "Shared goal", "Prose" and "Vocabulary" sections.
- `loop-claude <task>` runs a project's next chunks, each in its own tmux window, cycle after cycle; `loop-claude --help` states its contract.
- `loop-claude <task>` runs a project's next chunks, each in its own tmux window, cycle after cycle; `loop-claude --help` states its contract. A worker that printed its final result and waits on a background shell that stays quiet for 3 minutes loses that shell, as `bin/loop_claude/cycle/lingering_workers.py` says.
- `markdown-section <file> <fragment>` prints the section of a markdown file under the first heading that holds the fragment, to the next heading of its level or a higher one, instead of a `grep -n` and a guessed `sed -n` range; `markdown-section --help` says what it does where several headings match.
- `metric-bench [history]` benches cheap static metrics of a project's Python against the places its comprehension history names as hardest; `metric-bench --help` states the metrics and the scores.
- `mutate-changes [--base <ref>] [--prefix <shell text>] [--jobs <n>]` mutation-tests the lines a branch changes and prints each mutant its tests don't kill; `mutate-changes --help` says which tool it runs for each language.
@@ -0,0 +1,35 @@
"""Ends a worker whose claude -p printed its final result and then waits on a background shell that never ends.
Claude Code's own time limit for a background command cannot do this: `BASH_DEFAULT_TIMEOUT_MS` and `BASH_MAX_TIMEOUT_MS` only raise the limit, and a call's `timeout` of up to 2 hours replaces the 10 minutes of a `claude -p` run (docs: code.claude.com/docs/en/tools-reference.md → "Time limit for background commands"). On 2026-11-18 a worker waited 95 minutes in an `until` loop whose `pgrep -f` matched its own command line.
"""
import pathlib
from loop_effects.child_shells import Shell
from loop_effects.effects import Effects, Streaming, say
# Seconds between the checks, counted from the last line the worker printed.
CHECK_EVERY = 300
# Seconds a final result must have stood, and a shell's output must have been quiet, before a shell is stopped.
SETTLED = 180
RECORD = "lingering-workers.md"
def stopped_shells(proc: Streaming, settled_at: float | None, record: pathlib.Path, effects: Effects) -> None:
"""Stops the shells of a claude that printed its final result at settled_at and is still alive, and says so on the terminal and in record. Stops nothing where the claude is still working, or a shell writes output or holds a run-heavy slot."""
now = effects.now()
if settled_at is None or now - settled_at <= SETTLED:
return
shells = proc.shells()
if any(now - shell.written <= SETTLED or shell.holds_slot for shell in shells):
return
for shell in shells:
proc.stop(shell)
line = stop_line(shell, now)
say(line, effects, stdout=True)
record.parent.mkdir(parents=True, exist_ok=True)
with record.open("a") as handle:
handle.write(f"- {effects.local_time(now)} {line}\n")
def stop_line(shell: Shell, now: float) -> str:
return f"stopped pid {shell.pid} ({int(now - shell.started)} s old, quiet {int(now - shell.written)} s) that held a worker after its final result: {shell.command}"
+1 -1
View File
@@ -122,7 +122,7 @@ def work(cycle: Cycle, answered: QuotedAnswer | TypedAnswer | None, stepped: boo
if isinstance(answered, QuotedAnswer):
say(f"\n=== {prompts.QUOTE_SOURCES} answered: {answered.question} ===\n{answered.answer}", cycle.effects, stdout=True)
# worker_start runs inside the lambda: a failed attempt may have opened the session.
if stop := retried("worker", lambda: run_worker(cycle.root, worker_start(cycle, answered, stepped), worker_environment(cycle), cycle.effects), cycle.effects):
if stop := retried("worker", lambda: run_worker(cycle.root, worker_start(cycle, answered, stepped), worker_environment(cycle), cycle.project_archive, cycle.effects), cycle.effects):
return stop
path = transcript_file(cycle.session)
return WorkerRan(path) if path is not None else Stop(1, f"claude left no transcript for session {cycle.session.id}")
+11 -2
View File
@@ -1,4 +1,7 @@
import pathlib
from claude_code.entries import AssistantTurn, ends_stream, entry_from, hit_limit, json_object, limit_said, rejected_until
from loop_claude.cycle.lingering_workers import CHECK_EVERY, RECORD, stopped_shells
from loop_effects.effects import Effects, say
from failure import Failure
from claude_code.reply import Limited
@@ -15,16 +18,22 @@ AUTO_COMPACT_WINDOW = "CLAUDE_CODE_AUTO_COMPACT_WINDOW"
COMPACT_WINDOW = 200_000
def run_worker(root: str, args: list[str], env: dict[str, str], effects: Effects) -> Failure | Limited | None:
def run_worker(root: str, args: list[str], env: dict[str, str], project_archive: pathlib.Path, effects: Effects) -> Failure | Limited | None:
wait_for_memory(effects)
try:
proc = effects.stream_claude(args, root, env)
except OSError as failure:
return Failure(f"could not start claude: {failure.strerror}")
result: dict | None = None
# When the last line the worker printed, if it is a result: a worker that prints on is working.
settled_at: float | None = None
resets: list[int] = []
for raw in proc.lines():
for raw in proc.lines(CHECK_EVERY):
if raw is None:
stopped_shells(proc, settled_at, project_archive / RECORD, effects)
continue
event = json_object(raw) or {}
settled_at = effects.now() if ends_stream(event) else None
if ends_stream(event):
result = event
if (resets_at := rejected_until(event)) is not None:
+5
View File
@@ -48,6 +48,11 @@ failure, keeps its window, with its last lines, and its chunk keeps its items; t
claude session is named "A <key> loop-claude <window>", so a rerun resumes a chunk left open, and is
deleted when the chunk finishes. A fresh worker is told the items the project's other chunks hold,
running or stopped: each one's own and those on its latest `Chunk items:` line, and leaves them out.
A worker whose claude printed its final result and still waits on a background shell that has
written nothing for 3 minutes loses that shell, found at the next 5-minute check. A shell that
writes, or holds a run-heavy slot, keeps the worker alive. Each stop is printed and appended to
lingering-workers.md in the project's transcript archive
(bin/loop_claude/cycle/lingering_workers.py).
A question only the human can answer is an interrupt (bin/loop_claude/cycle/interrupt.py).
loop-claude prints the question in the chunk's window and rings its bell. Until it is answered the
+67 -7
View File
@@ -30,7 +30,7 @@ from claude_code import entries, reply, session
from failure import Failure
from loop_claude.verdicts import Finished, Interrupt, Judged, QuotedAnswer, TakenIn, Unfinished
from git_cli.git_repo import git
from loop_claude.cycle import interrupt, judge, prompts, retry, run, spend, states, worker
from loop_claude.cycle import interrupt, judge, lingering_workers, prompts, retry, run, spend, states, worker
from claude_code.reply import Limited
from loop_claude.loop import main, reload, routing_check, this_loop, usage
from machine_load import memory
@@ -43,6 +43,7 @@ from model_routing.ladder import ProjectRung, Rung
from model_routing.ladder_file import read_ladder, SHIPPED
from model_routing.route import Route, route_from, WorkerLine
from loop_claude import routing
from loop_effects.child_shells import Shell
from loop_claude.stop import Stop
from loop_claude import tmux
from claude_code.tokens import Tokens, tokens_of
@@ -212,12 +213,26 @@ class Worker:
class Streamed:
def __init__(self, code, *events):
"""A started claude. An event of None is an idle stretch of CHECK_EVERY seconds, which advances `clock` by that; `held` are the shells it has, and `stopped` collects the ones stopped."""
def __init__(self, code, *events, clock=None, held=()):
self.code = code
self.events = events or ({"type": "result"},)
self.clock = clock
self.held = list(held)
self.stopped = []
def lines(self):
return [json.dumps(event) + "\n" for event in self.events]
def lines(self, idle):
for event in self.events:
if event is None:
self.clock[0] += idle
yield None if event is None else json.dumps(event) + "\n"
def shells(self):
return self.held
def stop(self, shell):
self.stopped.append(shell)
def wait(self):
return self.code
@@ -740,7 +755,7 @@ class Limits(unittest.TestCase):
allowed = {"type": "rate_limit_event", "rate_limit_info": {"status": "allowed", "resetsAt": NOW + 9000}}
limited = {"type": "result", "api_error_status": 429, "result": "You've hit your session limit"}
fake = FakeEffects(streams=[Streamed(1, rejected[1], allowed, rejected[0], limited)])
self.assertEqual(worker.run_worker("/r", [], {}, fake.effects()), Limited(NOW + 600, "You've hit your session limit"))
self.assertEqual(worker.run_worker("/r", [], {}, pathlib.Path("/archive"), fake.effects()), Limited(NOW + 600, "You've hit your session limit"))
def test_a_limit_outranks_the_exit_code_and_its_message_fits_one_line(self):
def limited(**said):
@@ -762,18 +777,63 @@ class Limits(unittest.TestCase):
self.assertEqual(entries.rejected_until({"type": "rate_limit_event", "rate_limit_info": info}), want)
class LingeringWorkers(unittest.TestCase):
RESULT = {"type": "result"}
def worker_with(self, *events, held=()):
"""Runs a worker whose claude prints events and holds shells `held`; returns the claude, what loop-claude said and the record's text."""
clock = [NOW]
streamed = Streamed(0, *events, clock=clock, held=held)
fake = FakeEffects(streams=[streamed])
effects = dataclasses.replace(fake.effects(), now=lambda: clock[0])
with tempfile.TemporaryDirectory() as folder, contextlib.redirect_stdout(io.StringIO()) as said:
worker.run_worker("/r", [], {}, pathlib.Path(folder), effects)
record = pathlib.Path(folder, lingering_workers.RECORD)
return streamed, said.getvalue(), record.read_text() if record.exists() else ""
def shell(self, written=NOW - 1000, holds_slot=False):
return Shell(pid=7, started=NOW - 1000, command="until false; do sleep 15; done", written=written, holds_slot=holds_slot)
def test_a_worker_that_lingers_after_its_result_loses_its_quiet_shell_and_the_stop_is_said_and_recorded(self):
shell = self.shell()
streamed, said, record = self.worker_with(self.RESULT, None, held=[shell])
self.assertEqual(streamed.stopped, [shell])
line = f"stopped pid 7 (1300 s old, quiet 1300 s) that held a worker after its final result: {shell.command}"
self.assertEqual(said, f"<{NOW + 300}> {line}\n")
self.assertEqual(record, f"- <{NOW + 300}> {line}\n")
def test_nothing_is_stopped_where_the_worker_printed_after_its_result_or_no_check_came_due(self):
for events in ((self.RESULT, {"type": "assistant", "message": {"content": []}}, None), (self.RESULT,), (None,), ({"type": "assistant", "message": {"content": []}}, None)):
with self.subTest(events=events):
streamed, said, record = self.worker_with(*events, held=[self.shell()])
self.assertEqual((streamed.stopped, said, record), ([], "", ""))
def test_nothing_is_stopped_while_a_shell_writes_output_or_holds_a_heavy_slot(self):
for busy in (self.shell(written=NOW + 200), self.shell(holds_slot=True)):
with self.subTest(busy=busy):
streamed, _, _ = self.worker_with(self.RESULT, None, held=[self.shell(), busy])
self.assertEqual(streamed.stopped, [])
def test_a_check_before_the_result_has_stood_long_enough_stops_nothing(self):
fake = FakeEffects()
effects = dataclasses.replace(fake.effects(), now=lambda: NOW + lingering_workers.SETTLED)
proc = Streamed(0, held=[self.shell()])
lingering_workers.stopped_shells(proc, NOW, pathlib.Path("/unused"), effects)
self.assertEqual(proc.stopped, [])
class MemoryWait(unittest.TestCase):
def test_a_worker_starts_only_once_a_quarter_of_memory_is_available(self):
fake = FakeEffects(free_memory=[memory.Memory(available=3, total=16), memory.Memory(available=3, total=16), memory.Memory(available=4, total=16)], streams=[Streamed(0)])
with contextlib.redirect_stderr(io.StringIO()) as said:
self.assertIsNone(worker.run_worker("/r", [], {}, fake.effects()))
self.assertIsNone(worker.run_worker("/r", [], {}, pathlib.Path("/archive"), fake.effects()))
self.assertEqual([seam for seam, _ in fake.calls], ["memory", "sleep", "memory", "sleep", "memory", "stream_claude"])
self.assertEqual(fake.made("sleep"), [worker.MEMORY_POLL] * 2)
self.assertEqual(said.getvalue().count("waiting"), 1, said.getvalue())
def test_a_worker_starts_at_once_where_memory_cannot_be_read(self):
fake = FakeEffects(free_memory=[None], streams=[Streamed(0)])
self.assertIsNone(worker.run_worker("/r", [], {}, fake.effects()))
self.assertIsNone(worker.run_worker("/r", [], {}, pathlib.Path("/archive"), fake.effects()))
self.assertEqual([seam for seam, _ in fake.calls], ["memory", "stream_claude"])
+93
View File
@@ -0,0 +1,93 @@
"""The shells a running claude has started, read from /proc (Linux), and how to stop one."""
import dataclasses
import os
import signal
import stat
# The command names of a shell claude starts for a Bash call.
SHELL_NAMES = frozenset({"bash", "dash", "sh", "zsh"})
# How much of a shell's command line a Shell keeps.
COMMAND_CHARS = 200
@dataclasses.dataclass(frozen=True)
class Shell:
pid: int
# Epoch seconds at which the shell started.
started: float
command: str
# Epoch seconds of the last write to the file the shell's output goes to; its start where the output goes to no file.
written: float
# Whether the shell or one of its descendants is run-heavy, which holds a slot of the machine's heavy work or waits for one.
holds_slot: bool
def parent_and_start(pid: int) -> tuple[int, int, str] | None:
"""The parent pid, the start time in clock ticks since boot and the command name of a process; None where it is gone."""
try:
with open(f"/proc/{pid}/stat") as handle:
text = handle.read()
except OSError:
return None
# The name sits in parentheses and may hold spaces or parentheses itself, so split at the last one.
name = text[text.index("(") + 1 : text.rindex(")")]
fields = text[text.rindex(")") + 2 :].split()
return int(fields[1]), int(fields[19]), name
def children_of(pid: int) -> list[int]:
return [int(entry) for entry in os.listdir("/proc") if entry.isdigit() and (found := parent_and_start(int(entry))) and found[0] == pid]
def command_of(pid: int) -> str:
try:
with open(f"/proc/{pid}/cmdline", "rb") as handle:
return handle.read().replace(b"\0", b" ").decode(errors="replace").strip()
except OSError:
return ""
def descendants(pid: int) -> list[int]:
"""Every process below pid, nearest first."""
found = children_of(pid)
for child in list(found):
found += descendants(child)
return found
def boot_time() -> float:
with open("/proc/stat") as handle:
return next(float(line.split()[1]) for line in handle if line.startswith("btime"))
def output_written(pid: int, started: float) -> float:
"""When the file that pid's standard output goes to was last written; started where that is a pipe or a terminal, whose writes leave no time."""
try:
info = os.stat(f"/proc/{pid}/fd/1")
except OSError:
return started
return info.st_mtime if stat.S_ISREG(info.st_mode) else started
def shells_of(claude_pid: int) -> list[Shell]:
"""The shells that are children of claude_pid."""
booted = boot_time()
ticks = os.sysconf("SC_CLK_TCK")
shells = []
for pid in children_of(claude_pid):
found = parent_and_start(pid)
if found is None or found[2] not in SHELL_NAMES:
continue
started = booted + found[1] / ticks
below = [pid, *descendants(pid)]
shells.append(Shell(pid, started, command_of(pid)[:COMMAND_CHARS], output_written(pid, started), any("run-heavy" in command_of(each) for each in below)))
return shells
def stop(shell: Shell) -> None:
"""Ends the shell and everything below it, so the loop it ran doesn't outlive it."""
for pid in [*reversed(descendants(shell.pid)), shell.pid]:
try:
os.kill(pid, signal.SIGTERM)
except OSError:
pass
+36 -4
View File
@@ -5,6 +5,7 @@ A change here, such as to how input reads a line, changes every tool that runs o
import dataclasses
import os
import pathlib
import select
import signal
import subprocess
import sys
@@ -13,6 +14,7 @@ import time
import typing
from notifier.notify import Backend, BACKENDS, notify
from loop_effects import child_shells
from failure import Failure
from git_cli import git_repo
from machine_load.memory import Memory, read_memory
@@ -25,7 +27,17 @@ PYTHON_TIMEOUT = 600
class Streaming(typing.Protocol):
"""A started claude whose stream-json lines arrive as it works."""
def lines(self) -> typing.Iterable[str]: ...
def lines(self, idle: float) -> typing.Iterable[str | None]:
"""Each line as it arrives, and None whenever idle seconds pass without one."""
...
def shells(self) -> list[child_shells.Shell]:
"""The shells that are children of the claude."""
...
def stop(self, shell: child_shells.Shell) -> None:
"""Ends the shell and everything below it."""
...
def wait(self) -> int: ...
@@ -59,11 +71,31 @@ class Effects:
class StreamingClaude:
def __init__(self, args: list[str], cwd: str, env: dict[str, str]) -> None:
self.proc = subprocess.Popen(["claude", *args], cwd=cwd, env={**os.environ, **env}, encoding="utf-8", errors="replace", stdin=subprocess.DEVNULL, stdout=subprocess.PIPE)
self.proc = subprocess.Popen(["claude", *args], cwd=cwd, env={**os.environ, **env}, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, bufsize=0)
def lines(self) -> typing.Iterable[str]:
def lines(self, idle: float) -> typing.Iterable[str | None]:
# Reads the pipe's file descriptor itself, since select cannot see what a buffered reader holds, and a thread blocked in a read keeps the pipe from closing when a Ctrl-C ends the loop.
pending = b""
with self.proc:
yield from self.proc.stdout or ()
fd = self.proc.stdout.fileno() if self.proc.stdout else -1
while True:
if not select.select([fd], [], [], idle)[0]:
yield None
continue
chunk = os.read(fd, 65536)
if not chunk:
break
*complete, pending = (pending + chunk).split(b"\n")
for line in complete:
yield line.decode("utf-8", errors="replace") + "\n"
if pending:
yield pending.decode("utf-8", errors="replace")
def shells(self) -> list[child_shells.Shell]:
return child_shells.shells_of(self.proc.pid)
def stop(self, shell: child_shells.Shell) -> None:
child_shells.stop(shell)
def wait(self) -> int:
return self.proc.wait()
+23
View File
@@ -4,6 +4,7 @@ import io
import os
import pathlib
import signal
import subprocess
import sys
import tempfile
import time
@@ -12,6 +13,7 @@ import unittest.mock
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
from loop_effects import child_shells
from loop_effects.effects import run_python, say, system_effects
NOW = 1790000000
@@ -134,3 +136,24 @@ class PythonEffect(unittest.TestCase):
done = run_python(["-c", "pass"], gone, 60)
self.assertEqual(done.returncode, 1)
self.assertIn(gone, done.stderr)
class ChildShells(unittest.TestCase):
def test_reads_a_child_shell_with_its_command_and_a_heavy_slot_and_stops_it_with_what_it_started(self):
quiet = subprocess.Popen(["bash", "-c", "sleep 60; true"])
heavy = subprocess.Popen(["bash", "-c", "sleep 60; true # run-heavy"])
other = subprocess.Popen(["sleep", "60"])
try:
time.sleep(0.3)
shells = {shell.pid: shell for shell in child_shells.shells_of(os.getpid())}
self.assertEqual(sorted(shells), sorted([quiet.pid, heavy.pid]))
self.assertEqual(shells[quiet.pid].command, "bash -c sleep 60; true")
self.assertEqual((shells[quiet.pid].holds_slot, shells[heavy.pid].holds_slot), (False, True))
self.assertLess(abs(shells[quiet.pid].started - time.time()), 5)
child_shells.stop(shells[quiet.pid])
self.assertEqual(quiet.wait(timeout=5), -signal.SIGTERM)
self.assertIsNone(other.poll())
finally:
for proc in (quiet, heavy, other):
proc.kill()
proc.wait()