Keep a worker's shells that run a busy tool, name the shell's real command and fix the review's prose findings

This commit is contained in:
2026-11-18 17:20:13 +01:00
parent 72ffb9f748
commit abae3a0559
8 changed files with 96 additions and 50 deletions
+1 -1
View File
@@ -183,7 +183,7 @@ Install puts only `loop-claude` on `PATH`; call the others by their path, such a
- `deploy-release [--cut] [<checkout>]` moves `~/.claude`, or the checkout named, to the newest release, as Update says; with `--cut`, run in the development clone, it first cuts the next release, as Release says.
- `judge-replay <model> [--offline]` replays loop-claude's judge on archived verdicts of the projects run on the model. It prints what each rung a review climbs to misses, and the lowest rung below those that returns every verdict, as each candidate above it does. With `--offline` it doesn't call a judge: it counts what a "?" rule, a decision table and a classifier trained on the archive decide of each verdict's kind from the worker's final message, and how many interrupts each misses. `judge-replay --help` states both replays.
- `larv-review <command>` holds larv-review's mechanical rules: the chunk's review folder, the paired ruling's base and vote count, each pass's round count and the opening of its round commit's title, the scoring run's trigger and setup; the comprehension history's format, its latest places and its count of how often seats agree; and each reviewer's prompt header. `larv-review --help` states them. `settings.json` → `hooks` runs `larv-review reviewer-context` as each subagent starts. A reviewer under `agents/` whose frontmatter has `omitClaudeMd: true` starts without `CLAUDE.md`, and the command hands it that file's "Shared goal", "Prose" and "Vocabulary" sections.
- `loop-claude <task>` runs a project's next chunks, each in its own tmux window, cycle after cycle; `loop-claude --help` states its contract. A worker that printed its final result and waits on a background shell that stays quiet for 3 minutes loses that shell, as `bin/loop_claude/cycle/lingering_workers.py` says.
- `loop-claude <task>` runs a project's next chunks, each in its own tmux window, cycle after cycle; `loop-claude --help` states its contract.
- `markdown-section <file> <fragment>` prints the section of a markdown file under the first heading that holds the fragment, to the next heading of its level or a higher one, instead of a `grep -n` and a guessed `sed -n` range; `markdown-section --help` says what it does where several headings match.
- `metric-bench [history]` benches cheap static metrics of a project's Python against the places its comprehension history names as hardest; `metric-bench --help` states the metrics and the scores.
- `mutate-changes [--base <ref>] [--prefix <shell text>] [--jobs <n>]` mutation-tests the lines a branch changes and prints each mutant its tests don't kill; `mutate-changes --help` says which tool it runs for each language.
+10 -7
View File
@@ -1,13 +1,16 @@
"""Ends a worker whose claude -p printed its final result and then waits on a background shell that never ends.
"""Ends a worker whose claude -p printed its final result and then waits on background shells that have gone quiet.
Claude Code's own time limit for a background command cannot do this: `BASH_DEFAULT_TIMEOUT_MS` and `BASH_MAX_TIMEOUT_MS` only raise the limit, and a call's `timeout` of up to 2 hours replaces the 10 minutes of a `claude -p` run (docs: code.claude.com/docs/en/tools-reference.md → "Time limit for background commands"). On 2026-11-18 a worker waited 95 minutes in an `until` loop whose `pgrep -f` matched its own command line.
The stops go to RECORD in the project's transcript archive rather than to the judge's records: the judge reads a worker's transcript, so the archive is where the pattern can be counted when it becomes a rule-hook ban (item 421).
"""
import pathlib
from loop_effects.effects import Effects, say, Streaming
from loop_effects.child_shells import Shell
from loop_effects.effects import Effects, Streaming, say
from text_files.utc_stamp import utc
# Seconds between the checks, counted from the last line the worker printed.
# Seconds without a line from the worker before a check, so a check is 5 minutes after its last line.
CHECK_EVERY = 300
# Seconds a final result must have stood, and a shell's output must have been quiet, before a shell is stopped.
SETTLED = 180
@@ -15,12 +18,12 @@ RECORD = "lingering-workers.md"
def stopped_shells(proc: Streaming, settled_at: float | None, record: pathlib.Path, effects: Effects) -> None:
"""Stops the shells of a claude that printed its final result at settled_at and is still alive, and says so on the terminal and in record. Stops nothing where the claude is still working, or a shell writes output or holds a run-heavy slot."""
"""Stops the shells of a claude whose final result came at settled_at and that is still alive, saying each stop on the terminal and in record. A claude that is still working (settled_at is None) or whose result came SETTLED seconds ago or less keeps its shells, and so does one with a shell that wrote output within SETTLED seconds or runs a busy tool, as `Shell.runs_busy_tool` says."""
now = effects.now()
if settled_at is None or now - settled_at <= SETTLED:
return
shells = proc.shells()
if any(now - shell.written <= SETTLED or shell.holds_slot for shell in shells):
if any(now - shell.written <= SETTLED or shell.runs_busy_tool for shell in shells):
return
for shell in shells:
proc.stop(shell)
@@ -28,8 +31,8 @@ def stopped_shells(proc: Streaming, settled_at: float | None, record: pathlib.Pa
say(line, effects, stdout=True)
record.parent.mkdir(parents=True, exist_ok=True)
with record.open("a") as handle:
handle.write(f"- {effects.local_time(now)} {line}\n")
handle.write(f"- {utc(now)} {line}\n")
def stop_line(shell: Shell, now: float) -> str:
return f"stopped pid {shell.pid} ({int(now - shell.started)} s old, quiet {int(now - shell.written)} s) that held a worker after its final result: {shell.command}"
return f"stopped shell {shell.pid}, which held a worker after its final result; it had run {int(now - shell.started)} s and written nothing for {int(now - shell.written)} s: {shell.command}"
+1 -1
View File
@@ -25,7 +25,7 @@ def run_worker(root: str, args: list[str], env: dict[str, str], project_archive:
except OSError as failure:
return Failure(f"could not start claude: {failure.strerror}")
result: dict | None = None
# When the last line the worker printed, if it is a result: a worker that prints on is working.
# When the worker printed its latest result; None while its latest line is something else, since a worker that prints on is working.
settled_at: float | None = None
resets: list[int] = []
for raw in proc.lines(CHECK_EVERY):
+7 -5
View File
@@ -48,11 +48,6 @@ failure, keeps its window, with its last lines, and its chunk keeps its items; t
claude session is named "A <key> loop-claude <window>", so a rerun resumes a chunk left open, and is
deleted when the chunk finishes. A fresh worker is told the items the project's other chunks hold,
running or stopped: each one's own and those on its latest `Chunk items:` line, and leaves them out.
A worker whose claude printed its final result and still waits on a background shell that has
written nothing for 3 minutes loses that shell, found at the next 5-minute check. A shell that
writes, or holds a run-heavy slot, keeps the worker alive. Each stop is printed and appended to
lingering-workers.md in the project's transcript archive
(bin/loop_claude/cycle/lingering_workers.py).
A question only the human can answer is an interrupt (bin/loop_claude/cycle/interrupt.py).
loop-claude prints the question in the chunk's window and rings its bell. Until it is answered the
@@ -83,6 +78,13 @@ transcript is copied into a folder per chunk under $LOOP_CLAUDE_TRANSCRIPTS/<key
then each verdict, quoted answer, rejected judge reply, step-up and stop, each opened by its UTC
time. The layout, and when a copy is replaced, are in bin/transcript_archive/__init__.py.
A worker's claude can print its final result and then keep waiting on a background shell that
never ends. Every 5 minutes without output from that claude, loop-claude stops each of its shells
and what runs below them, once the result is over 3 minutes old and no shell has written output in
the last 3 minutes or runs run-heavy or chunk-pr. It prints each stop and appends it, with the UTC
time, to lingering-workers.md at the project's folder in that archive
(bin/loop_claude/cycle/lingering_workers.py).
Before each worker starts, loop-claude waits until at least a quarter of the machine's memory is
available (bin/machine_load/memory.py).
+9 -9
View File
@@ -791,30 +791,30 @@ class LingeringWorkers(unittest.TestCase):
record = pathlib.Path(folder, lingering_workers.RECORD)
return streamed, said.getvalue(), record.read_text() if record.exists() else ""
def shell(self, written=NOW - 1000, holds_slot=False):
return Shell(pid=7, started=NOW - 1000, command="until false; do sleep 15; done", written=written, holds_slot=holds_slot)
def shell(self, written=NOW - 1000, runs_busy_tool=False):
return Shell(pid=7, started=NOW - 1000, command="until false; do sleep 15; done", written=written, runs_busy_tool=runs_busy_tool)
def test_a_worker_that_lingers_after_its_result_loses_its_quiet_shell_and_the_stop_is_said_and_recorded(self):
def test_a_worker_that_lingers_after_its_result_loses_its_quiet_shell_and_the_stop_is_said_and_recorded_in_utc(self):
shell = self.shell()
streamed, said, record = self.worker_with(self.RESULT, None, held=[shell])
self.assertEqual(streamed.stopped, [shell])
line = f"stopped pid 7 (1300 s old, quiet 1300 s) that held a worker after its final result: {shell.command}"
line = f"stopped shell 7, which held a worker after its final result; it had run 1300 s and written nothing for 1300 s: {shell.command}"
self.assertEqual(said, f"<{NOW + 300}> {line}\n")
self.assertEqual(record, f"- <{NOW + 300}> {line}\n")
self.assertEqual(record, f"- {utc(NOW + 300)} {line}\n")
def test_nothing_is_stopped_where_the_worker_printed_after_its_result_or_no_check_came_due(self):
def test_a_worker_keeps_its_shells_where_it_printed_after_its_result_or_printed_none_or_no_check_came_due(self):
for events in ((self.RESULT, {"type": "assistant", "message": {"content": []}}, None), (self.RESULT,), (None,), ({"type": "assistant", "message": {"content": []}}, None)):
with self.subTest(events=events):
streamed, said, record = self.worker_with(*events, held=[self.shell()])
self.assertEqual((streamed.stopped, said, record), ([], "", ""))
def test_nothing_is_stopped_while_a_shell_writes_output_or_holds_a_heavy_slot(self):
for busy in (self.shell(written=NOW + 200), self.shell(holds_slot=True)):
def test_a_worker_keeps_every_shell_while_one_of_them_writes_output_or_runs_a_busy_tool(self):
for busy in (self.shell(written=NOW + 200), self.shell(runs_busy_tool=True)):
with self.subTest(busy=busy):
streamed, _, _ = self.worker_with(self.RESULT, None, held=[self.shell(), busy])
self.assertEqual(streamed.stopped, [])
def test_a_check_before_the_result_has_stood_long_enough_stops_nothing(self):
def test_a_worker_keeps_its_shells_until_its_result_has_stood_longer_than_the_settled_time(self):
fake = FakeEffects()
effects = dataclasses.replace(fake.effects(), now=lambda: NOW + lingering_workers.SETTLED)
proc = Streamed(0, held=[self.shell()])
+30 -14
View File
@@ -1,4 +1,7 @@
"""The shells a running claude has started, read from /proc (Linux), and how to stop one."""
"""The shells a running claude has started, read from /proc (Linux), and how to stop one.
Claude Code starts each Bash call as `bash -c "source <snapshot> && … && eval '<the command>'"` and writes its output to a file under /tmp/claude-<uid>/.
"""
import dataclasses
import os
import signal
@@ -6,8 +9,9 @@ import stat
# The command names of a shell claude starts for a Bash call.
SHELL_NAMES = frozenset({"bash", "dash", "sh", "zsh"})
# How much of a shell's command line a Shell keeps.
COMMAND_CHARS = 200
COMMAND_CHARS = 300
# Tools whose process below a shell shows work that may go quiet for long: run-heavy holds a slot of the machine's heavy work or waits for one, and chunk-pr waits for a merge's approval and checks.
BUSY_TOOLS = frozenset({"chunk-pr", "run-heavy"})
@dataclasses.dataclass(frozen=True)
@@ -15,11 +19,12 @@ class Shell:
pid: int
# Epoch seconds at which the shell started.
started: float
# What the shell runs, without the snapshot Claude Code sources before it.
command: str
# Epoch seconds of the last write to the file the shell's output goes to; its start where the output goes to no file.
# Epoch seconds of the last write to the file the shell's output goes to; the shell's start where the output goes to no file, which the check cannot read.
written: float
# Whether the shell or one of its descendants is run-heavy, which holds a slot of the machine's heavy work or waits for one.
holds_slot: bool
# Whether a process below the shell runs one of BUSY_TOOLS.
runs_busy_tool: bool
def parent_and_start(pid: int) -> tuple[int, int, str] | None:
@@ -39,16 +44,16 @@ def children_of(pid: int) -> list[int]:
return [int(entry) for entry in os.listdir("/proc") if entry.isdigit() and (found := parent_and_start(int(entry))) and found[0] == pid]
def command_of(pid: int) -> str:
def argv_of(pid: int) -> list[str]:
try:
with open(f"/proc/{pid}/cmdline", "rb") as handle:
return handle.read().replace(b"\0", b" ").decode(errors="replace").strip()
return handle.read().decode(errors="replace").rstrip("\0").split("\0")
except OSError:
return ""
return []
def descendants(pid: int) -> list[int]:
"""Every process below pid, nearest first."""
"""Every process below pid, each after its parent."""
found = children_of(pid)
for child in list(found):
found += descendants(child)
@@ -61,7 +66,7 @@ def boot_time() -> float:
def output_written(pid: int, started: float) -> float:
"""When the file that pid's standard output goes to was last written; started where that is a pipe or a terminal, whose writes leave no time."""
"""When the file that pid's standard output goes to was last written; started where it goes to a pipe or a terminal."""
try:
info = os.stat(f"/proc/{pid}/fd/1")
except OSError:
@@ -69,6 +74,18 @@ def output_written(pid: int, started: float) -> float:
return info.st_mtime if stat.S_ISREG(info.st_mode) else started
def command_of(shell_argv: list[str]) -> str:
"""What a shell runs: the text of its command line between Claude Code's `eval '` and the `' < /dev/null` that ends it, or the whole line where it holds none."""
line = " ".join(shell_argv)
_, found, command = line.partition(" && eval ")
return (command.split(" < /dev/null", 1)[0].removeprefix("'").removesuffix("'") if found else line)[:COMMAND_CHARS]
def runs_busy_tool(pid: int) -> bool:
"""Whether a process below pid has one of BUSY_TOOLS as its program or its script, as `python3 /x/chunk-pr`. A command line that merely names one, such as a `pgrep -f chunk-pr`, doesn't count."""
return any(os.path.basename(name) in BUSY_TOOLS for each in descendants(pid) for name in argv_of(each)[:2])
def shells_of(claude_pid: int) -> list[Shell]:
"""The shells that are children of claude_pid."""
booted = boot_time()
@@ -79,13 +96,12 @@ def shells_of(claude_pid: int) -> list[Shell]:
if found is None or found[2] not in SHELL_NAMES:
continue
started = booted + found[1] / ticks
below = [pid, *descendants(pid)]
shells.append(Shell(pid, started, command_of(pid)[:COMMAND_CHARS], output_written(pid, started), any("run-heavy" in command_of(each) for each in below)))
shells.append(Shell(pid, started, command_of(argv_of(pid)), output_written(pid, started), runs_busy_tool(pid)))
return shells
def stop(shell: Shell) -> None:
"""Ends the shell and everything below it, so the loop it ran doesn't outlive it."""
"""Ends the shell and every process below it, so that a loop the shell runs doesn't outlive it."""
for pid in [*reversed(descendants(shell.pid)), shell.pid]:
try:
os.kill(pid, signal.SIGTERM)
+2 -1
View File
@@ -74,7 +74,8 @@ class StreamingClaude:
self.proc = subprocess.Popen(["claude", *args], cwd=cwd, env={**os.environ, **env}, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, bufsize=0)
def lines(self, idle: float) -> typing.Iterable[str | None]:
# Reads the pipe's file descriptor itself, since select cannot see what a buffered reader holds, and a thread blocked in a read keeps the pipe from closing when a Ctrl-C ends the loop.
# Reads the pipe's file descriptor itself. select cannot see what a buffered reader holds.
# A thread blocked in a read would keep the pipe from closing when a Ctrl-C ends the loop.
pending = b""
with self.proc:
fd = self.proc.stdout.fileno() if self.proc.stdout else -1
+36 -12
View File
@@ -139,21 +139,45 @@ class PythonEffect(unittest.TestCase):
class ChildShells(unittest.TestCase):
def test_reads_a_child_shell_with_its_command_and_a_heavy_slot_and_stops_it_with_what_it_started(self):
quiet = subprocess.Popen(["bash", "-c", "sleep 60; true"])
heavy = subprocess.Popen(["bash", "-c", "sleep 60; true # run-heavy"])
def test_reads_a_child_shells_command_and_whether_a_busy_tool_runs_below_it(self):
with tempfile.TemporaryDirectory() as folder:
tool = pathlib.Path(folder, "chunk-pr")
tool.write_text("#!/bin/sh\nsleep 60\n")
tool.chmod(0o755)
# Like the shell of a loop that only names the tool, as the one that held a worker on 2026-11-18 did.
naming = subprocess.Popen(["bash", "-c", "source /dev/null && eval 'until false; do sleep 60; done # chunk-pr merge'"])
busy = subprocess.Popen(["bash", "-c", f"{tool} merge; true"])
other = subprocess.Popen(["sleep", "60"])
try:
time.sleep(0.5)
shells = {shell.pid: shell for shell in child_shells.shells_of(os.getpid())}
self.assertEqual(sorted(shells), sorted([naming.pid, busy.pid]))
self.assertEqual(shells[naming.pid].command, "until false; do sleep 60; done # chunk-pr merge")
self.assertEqual((shells[naming.pid].runs_busy_tool, shells[busy.pid].runs_busy_tool), (False, True))
self.assertLess(abs(shells[naming.pid].started - time.time()), 5)
finally:
for proc in (naming, busy, other):
proc.kill()
proc.wait()
def test_stops_a_shell_and_the_process_it_started_and_no_other(self):
shell = subprocess.Popen(["bash", "-c", "sleep 60; true"])
other = subprocess.Popen(["sleep", "60"])
try:
time.sleep(0.3)
shells = {shell.pid: shell for shell in child_shells.shells_of(os.getpid())}
self.assertEqual(sorted(shells), sorted([quiet.pid, heavy.pid]))
self.assertEqual(shells[quiet.pid].command, "bash -c sleep 60; true")
self.assertEqual((shells[quiet.pid].holds_slot, shells[heavy.pid].holds_slot), (False, True))
self.assertLess(abs(shells[quiet.pid].started - time.time()), 5)
child_shells.stop(shells[quiet.pid])
self.assertEqual(quiet.wait(timeout=5), -signal.SIGTERM)
time.sleep(0.5)
found = child_shells.shells_of(os.getpid())
[stopping] = [each for each in found if each.pid == shell.pid]
below = child_shells.descendants(shell.pid)
self.assertEqual(len(below), 1)
child_shells.stop(stopping)
self.assertEqual(shell.wait(timeout=5), -signal.SIGTERM)
for _ in range(50):
if not os.path.exists(f"/proc/{below[0]}"):
break
time.sleep(0.1)
self.assertFalse(os.path.exists(f"/proc/{below[0]}"))
self.assertIsNone(other.poll())
finally:
for proc in (quiet, heavy, other):
for proc in (shell, other):
proc.kill()
proc.wait()