Keep a worker's shells that run a busy tool, name the shell's real command and fix the review's prose findings
This commit is contained in:
@@ -183,7 +183,7 @@ Install puts only `loop-claude` on `PATH`; call the others by their path, such a
|
||||
- `deploy-release [--cut] [<checkout>]` moves `~/.claude`, or the checkout named, to the newest release, as Update says; with `--cut`, run in the development clone, it first cuts the next release, as Release says.
|
||||
- `judge-replay <model> [--offline]` replays loop-claude's judge on archived verdicts of the projects run on the model. It prints what each rung a review climbs to misses, and the lowest rung below those that returns every verdict, as each candidate above it does. With `--offline` it doesn't call a judge: it counts what a "?" rule, a decision table and a classifier trained on the archive decide of each verdict's kind from the worker's final message, and how many interrupts each misses. `judge-replay --help` states both replays.
|
||||
- `larv-review <command>` holds larv-review's mechanical rules: the chunk's review folder, the paired ruling's base and vote count, each pass's round count and the opening of its round commit's title, the scoring run's trigger and setup; the comprehension history's format, its latest places and its count of how often seats agree; and each reviewer's prompt header. `larv-review --help` states them. `settings.json` → `hooks` runs `larv-review reviewer-context` as each subagent starts. A reviewer under `agents/` whose frontmatter has `omitClaudeMd: true` starts without `CLAUDE.md`, and the command hands it that file's "Shared goal", "Prose" and "Vocabulary" sections.
|
||||
- `loop-claude <task>` runs a project's next chunks, each in its own tmux window, cycle after cycle; `loop-claude --help` states its contract. A worker that printed its final result and waits on a background shell that stays quiet for 3 minutes loses that shell, as `bin/loop_claude/cycle/lingering_workers.py` says.
|
||||
- `loop-claude <task>` runs a project's next chunks, each in its own tmux window, cycle after cycle; `loop-claude --help` states its contract.
|
||||
- `markdown-section <file> <fragment>` prints the section of a markdown file under the first heading that holds the fragment, to the next heading of its level or a higher one, instead of a `grep -n` and a guessed `sed -n` range; `markdown-section --help` says what it does where several headings match.
|
||||
- `metric-bench [history]` benches cheap static metrics of a project's Python against the places its comprehension history names as hardest; `metric-bench --help` states the metrics and the scores.
|
||||
- `mutate-changes [--base <ref>] [--prefix <shell text>] [--jobs <n>]` mutation-tests the lines a branch changes and prints each mutant its tests don't kill; `mutate-changes --help` says which tool it runs for each language.
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
"""Ends a worker whose claude -p printed its final result and then waits on a background shell that never ends.
|
||||
"""Ends a worker whose claude -p printed its final result and then waits on background shells that have gone quiet.
|
||||
|
||||
Claude Code's own time limit for a background command cannot do this: `BASH_DEFAULT_TIMEOUT_MS` and `BASH_MAX_TIMEOUT_MS` only raise the limit, and a call's `timeout` of up to 2 hours replaces the 10 minutes of a `claude -p` run (docs: code.claude.com/docs/en/tools-reference.md → "Time limit for background commands"). On 2026-11-18 a worker waited 95 minutes in an `until` loop whose `pgrep -f` matched its own command line.
|
||||
|
||||
The stops go to RECORD in the project's transcript archive rather than to the judge's records: the judge reads a worker's transcript, so the archive is where the pattern can be counted when it becomes a rule-hook ban (item 421).
|
||||
"""
|
||||
import pathlib
|
||||
|
||||
from loop_effects.effects import Effects, say, Streaming
|
||||
from loop_effects.child_shells import Shell
|
||||
from loop_effects.effects import Effects, Streaming, say
|
||||
from text_files.utc_stamp import utc
|
||||
|
||||
# Seconds between the checks, counted from the last line the worker printed.
|
||||
# Seconds without a line from the worker before a check, so a check is 5 minutes after its last line.
|
||||
CHECK_EVERY = 300
|
||||
# Seconds a final result must have stood, and a shell's output must have been quiet, before a shell is stopped.
|
||||
SETTLED = 180
|
||||
@@ -15,12 +18,12 @@ RECORD = "lingering-workers.md"
|
||||
|
||||
|
||||
def stopped_shells(proc: Streaming, settled_at: float | None, record: pathlib.Path, effects: Effects) -> None:
|
||||
"""Stops the shells of a claude that printed its final result at settled_at and is still alive, and says so on the terminal and in record. Stops nothing where the claude is still working, or a shell writes output or holds a run-heavy slot."""
|
||||
"""Stops the shells of a claude whose final result came at settled_at and that is still alive, saying each stop on the terminal and in record. A claude that is still working (settled_at is None) or whose result came SETTLED seconds ago or less keeps its shells, and so does one with a shell that wrote output within SETTLED seconds or runs a busy tool, as `Shell.runs_busy_tool` says."""
|
||||
now = effects.now()
|
||||
if settled_at is None or now - settled_at <= SETTLED:
|
||||
return
|
||||
shells = proc.shells()
|
||||
if any(now - shell.written <= SETTLED or shell.holds_slot for shell in shells):
|
||||
if any(now - shell.written <= SETTLED or shell.runs_busy_tool for shell in shells):
|
||||
return
|
||||
for shell in shells:
|
||||
proc.stop(shell)
|
||||
@@ -28,8 +31,8 @@ def stopped_shells(proc: Streaming, settled_at: float | None, record: pathlib.Pa
|
||||
say(line, effects, stdout=True)
|
||||
record.parent.mkdir(parents=True, exist_ok=True)
|
||||
with record.open("a") as handle:
|
||||
handle.write(f"- {effects.local_time(now)} {line}\n")
|
||||
handle.write(f"- {utc(now)} {line}\n")
|
||||
|
||||
|
||||
def stop_line(shell: Shell, now: float) -> str:
|
||||
return f"stopped pid {shell.pid} ({int(now - shell.started)} s old, quiet {int(now - shell.written)} s) that held a worker after its final result: {shell.command}"
|
||||
return f"stopped shell {shell.pid}, which held a worker after its final result; it had run {int(now - shell.started)} s and written nothing for {int(now - shell.written)} s: {shell.command}"
|
||||
|
||||
@@ -25,7 +25,7 @@ def run_worker(root: str, args: list[str], env: dict[str, str], project_archive:
|
||||
except OSError as failure:
|
||||
return Failure(f"could not start claude: {failure.strerror}")
|
||||
result: dict | None = None
|
||||
# When the last line the worker printed, if it is a result: a worker that prints on is working.
|
||||
# When the worker printed its latest result; None while its latest line is something else, since a worker that prints on is working.
|
||||
settled_at: float | None = None
|
||||
resets: list[int] = []
|
||||
for raw in proc.lines(CHECK_EVERY):
|
||||
|
||||
@@ -48,11 +48,6 @@ failure, keeps its window, with its last lines, and its chunk keeps its items; t
|
||||
claude session is named "A <key> loop-claude <window>", so a rerun resumes a chunk left open, and is
|
||||
deleted when the chunk finishes. A fresh worker is told the items the project's other chunks hold,
|
||||
running or stopped: each one's own and those on its latest `Chunk items:` line, and leaves them out.
|
||||
A worker whose claude printed its final result and still waits on a background shell that has
|
||||
written nothing for 3 minutes loses that shell, found at the next 5-minute check. A shell that
|
||||
writes, or holds a run-heavy slot, keeps the worker alive. Each stop is printed and appended to
|
||||
lingering-workers.md in the project's transcript archive
|
||||
(bin/loop_claude/cycle/lingering_workers.py).
|
||||
|
||||
A question only the human can answer is an interrupt (bin/loop_claude/cycle/interrupt.py).
|
||||
loop-claude prints the question in the chunk's window and rings its bell. Until it is answered the
|
||||
@@ -83,6 +78,13 @@ transcript is copied into a folder per chunk under $LOOP_CLAUDE_TRANSCRIPTS/<key
|
||||
then each verdict, quoted answer, rejected judge reply, step-up and stop, each opened by its UTC
|
||||
time. The layout, and when a copy is replaced, are in bin/transcript_archive/__init__.py.
|
||||
|
||||
A worker's claude can print its final result and then keep waiting on a background shell that
|
||||
never ends. Every 5 minutes without output from that claude, loop-claude stops each of its shells
|
||||
and what runs below them, once the result is over 3 minutes old and no shell has written output in
|
||||
the last 3 minutes or runs run-heavy or chunk-pr. It prints each stop and appends it, with the UTC
|
||||
time, to lingering-workers.md at the project's folder in that archive
|
||||
(bin/loop_claude/cycle/lingering_workers.py).
|
||||
|
||||
Before each worker starts, loop-claude waits until at least a quarter of the machine's memory is
|
||||
available (bin/machine_load/memory.py).
|
||||
|
||||
|
||||
@@ -791,30 +791,30 @@ class LingeringWorkers(unittest.TestCase):
|
||||
record = pathlib.Path(folder, lingering_workers.RECORD)
|
||||
return streamed, said.getvalue(), record.read_text() if record.exists() else ""
|
||||
|
||||
def shell(self, written=NOW - 1000, holds_slot=False):
|
||||
return Shell(pid=7, started=NOW - 1000, command="until false; do sleep 15; done", written=written, holds_slot=holds_slot)
|
||||
def shell(self, written=NOW - 1000, runs_busy_tool=False):
|
||||
return Shell(pid=7, started=NOW - 1000, command="until false; do sleep 15; done", written=written, runs_busy_tool=runs_busy_tool)
|
||||
|
||||
def test_a_worker_that_lingers_after_its_result_loses_its_quiet_shell_and_the_stop_is_said_and_recorded(self):
|
||||
def test_a_worker_that_lingers_after_its_result_loses_its_quiet_shell_and_the_stop_is_said_and_recorded_in_utc(self):
|
||||
shell = self.shell()
|
||||
streamed, said, record = self.worker_with(self.RESULT, None, held=[shell])
|
||||
self.assertEqual(streamed.stopped, [shell])
|
||||
line = f"stopped pid 7 (1300 s old, quiet 1300 s) that held a worker after its final result: {shell.command}"
|
||||
line = f"stopped shell 7, which held a worker after its final result; it had run 1300 s and written nothing for 1300 s: {shell.command}"
|
||||
self.assertEqual(said, f"<{NOW + 300}> {line}\n")
|
||||
self.assertEqual(record, f"- <{NOW + 300}> {line}\n")
|
||||
self.assertEqual(record, f"- {utc(NOW + 300)} {line}\n")
|
||||
|
||||
def test_nothing_is_stopped_where_the_worker_printed_after_its_result_or_no_check_came_due(self):
|
||||
def test_a_worker_keeps_its_shells_where_it_printed_after_its_result_or_printed_none_or_no_check_came_due(self):
|
||||
for events in ((self.RESULT, {"type": "assistant", "message": {"content": []}}, None), (self.RESULT,), (None,), ({"type": "assistant", "message": {"content": []}}, None)):
|
||||
with self.subTest(events=events):
|
||||
streamed, said, record = self.worker_with(*events, held=[self.shell()])
|
||||
self.assertEqual((streamed.stopped, said, record), ([], "", ""))
|
||||
|
||||
def test_nothing_is_stopped_while_a_shell_writes_output_or_holds_a_heavy_slot(self):
|
||||
for busy in (self.shell(written=NOW + 200), self.shell(holds_slot=True)):
|
||||
def test_a_worker_keeps_every_shell_while_one_of_them_writes_output_or_runs_a_busy_tool(self):
|
||||
for busy in (self.shell(written=NOW + 200), self.shell(runs_busy_tool=True)):
|
||||
with self.subTest(busy=busy):
|
||||
streamed, _, _ = self.worker_with(self.RESULT, None, held=[self.shell(), busy])
|
||||
self.assertEqual(streamed.stopped, [])
|
||||
|
||||
def test_a_check_before_the_result_has_stood_long_enough_stops_nothing(self):
|
||||
def test_a_worker_keeps_its_shells_until_its_result_has_stood_longer_than_the_settled_time(self):
|
||||
fake = FakeEffects()
|
||||
effects = dataclasses.replace(fake.effects(), now=lambda: NOW + lingering_workers.SETTLED)
|
||||
proc = Streamed(0, held=[self.shell()])
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
"""The shells a running claude has started, read from /proc (Linux), and how to stop one."""
|
||||
"""The shells a running claude has started, read from /proc (Linux), and how to stop one.
|
||||
|
||||
Claude Code starts each Bash call as `bash -c "source <snapshot> && … && eval '<the command>'"` and writes its output to a file under /tmp/claude-<uid>/.
|
||||
"""
|
||||
import dataclasses
|
||||
import os
|
||||
import signal
|
||||
@@ -6,8 +9,9 @@ import stat
|
||||
|
||||
# The command names of a shell claude starts for a Bash call.
|
||||
SHELL_NAMES = frozenset({"bash", "dash", "sh", "zsh"})
|
||||
# How much of a shell's command line a Shell keeps.
|
||||
COMMAND_CHARS = 200
|
||||
COMMAND_CHARS = 300
|
||||
# Tools whose process below a shell shows work that may go quiet for long: run-heavy holds a slot of the machine's heavy work or waits for one, and chunk-pr waits for a merge's approval and checks.
|
||||
BUSY_TOOLS = frozenset({"chunk-pr", "run-heavy"})
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
@@ -15,11 +19,12 @@ class Shell:
|
||||
pid: int
|
||||
# Epoch seconds at which the shell started.
|
||||
started: float
|
||||
# What the shell runs, without the snapshot Claude Code sources before it.
|
||||
command: str
|
||||
# Epoch seconds of the last write to the file the shell's output goes to; its start where the output goes to no file.
|
||||
# Epoch seconds of the last write to the file the shell's output goes to; the shell's start where the output goes to no file, which the check cannot read.
|
||||
written: float
|
||||
# Whether the shell or one of its descendants is run-heavy, which holds a slot of the machine's heavy work or waits for one.
|
||||
holds_slot: bool
|
||||
# Whether a process below the shell runs one of BUSY_TOOLS.
|
||||
runs_busy_tool: bool
|
||||
|
||||
|
||||
def parent_and_start(pid: int) -> tuple[int, int, str] | None:
|
||||
@@ -39,16 +44,16 @@ def children_of(pid: int) -> list[int]:
|
||||
return [int(entry) for entry in os.listdir("/proc") if entry.isdigit() and (found := parent_and_start(int(entry))) and found[0] == pid]
|
||||
|
||||
|
||||
def command_of(pid: int) -> str:
|
||||
def argv_of(pid: int) -> list[str]:
|
||||
try:
|
||||
with open(f"/proc/{pid}/cmdline", "rb") as handle:
|
||||
return handle.read().replace(b"\0", b" ").decode(errors="replace").strip()
|
||||
return handle.read().decode(errors="replace").rstrip("\0").split("\0")
|
||||
except OSError:
|
||||
return ""
|
||||
return []
|
||||
|
||||
|
||||
def descendants(pid: int) -> list[int]:
|
||||
"""Every process below pid, nearest first."""
|
||||
"""Every process below pid, each after its parent."""
|
||||
found = children_of(pid)
|
||||
for child in list(found):
|
||||
found += descendants(child)
|
||||
@@ -61,7 +66,7 @@ def boot_time() -> float:
|
||||
|
||||
|
||||
def output_written(pid: int, started: float) -> float:
|
||||
"""When the file that pid's standard output goes to was last written; started where that is a pipe or a terminal, whose writes leave no time."""
|
||||
"""When the file that pid's standard output goes to was last written; started where it goes to a pipe or a terminal."""
|
||||
try:
|
||||
info = os.stat(f"/proc/{pid}/fd/1")
|
||||
except OSError:
|
||||
@@ -69,6 +74,18 @@ def output_written(pid: int, started: float) -> float:
|
||||
return info.st_mtime if stat.S_ISREG(info.st_mode) else started
|
||||
|
||||
|
||||
def command_of(shell_argv: list[str]) -> str:
|
||||
"""What a shell runs: the text of its command line between Claude Code's `eval '` and the `' < /dev/null` that ends it, or the whole line where it holds none."""
|
||||
line = " ".join(shell_argv)
|
||||
_, found, command = line.partition(" && eval ")
|
||||
return (command.split(" < /dev/null", 1)[0].removeprefix("'").removesuffix("'") if found else line)[:COMMAND_CHARS]
|
||||
|
||||
|
||||
def runs_busy_tool(pid: int) -> bool:
|
||||
"""Whether a process below pid has one of BUSY_TOOLS as its program or its script, as `python3 /x/chunk-pr`. A command line that merely names one, such as a `pgrep -f chunk-pr`, doesn't count."""
|
||||
return any(os.path.basename(name) in BUSY_TOOLS for each in descendants(pid) for name in argv_of(each)[:2])
|
||||
|
||||
|
||||
def shells_of(claude_pid: int) -> list[Shell]:
|
||||
"""The shells that are children of claude_pid."""
|
||||
booted = boot_time()
|
||||
@@ -79,13 +96,12 @@ def shells_of(claude_pid: int) -> list[Shell]:
|
||||
if found is None or found[2] not in SHELL_NAMES:
|
||||
continue
|
||||
started = booted + found[1] / ticks
|
||||
below = [pid, *descendants(pid)]
|
||||
shells.append(Shell(pid, started, command_of(pid)[:COMMAND_CHARS], output_written(pid, started), any("run-heavy" in command_of(each) for each in below)))
|
||||
shells.append(Shell(pid, started, command_of(argv_of(pid)), output_written(pid, started), runs_busy_tool(pid)))
|
||||
return shells
|
||||
|
||||
|
||||
def stop(shell: Shell) -> None:
|
||||
"""Ends the shell and everything below it, so the loop it ran doesn't outlive it."""
|
||||
"""Ends the shell and every process below it, so that a loop the shell runs doesn't outlive it."""
|
||||
for pid in [*reversed(descendants(shell.pid)), shell.pid]:
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
|
||||
@@ -74,7 +74,8 @@ class StreamingClaude:
|
||||
self.proc = subprocess.Popen(["claude", *args], cwd=cwd, env={**os.environ, **env}, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, bufsize=0)
|
||||
|
||||
def lines(self, idle: float) -> typing.Iterable[str | None]:
|
||||
# Reads the pipe's file descriptor itself, since select cannot see what a buffered reader holds, and a thread blocked in a read keeps the pipe from closing when a Ctrl-C ends the loop.
|
||||
# Reads the pipe's file descriptor itself. select cannot see what a buffered reader holds.
|
||||
# A thread blocked in a read would keep the pipe from closing when a Ctrl-C ends the loop.
|
||||
pending = b""
|
||||
with self.proc:
|
||||
fd = self.proc.stdout.fileno() if self.proc.stdout else -1
|
||||
|
||||
@@ -139,21 +139,45 @@ class PythonEffect(unittest.TestCase):
|
||||
|
||||
|
||||
class ChildShells(unittest.TestCase):
|
||||
def test_reads_a_child_shell_with_its_command_and_a_heavy_slot_and_stops_it_with_what_it_started(self):
|
||||
quiet = subprocess.Popen(["bash", "-c", "sleep 60; true"])
|
||||
heavy = subprocess.Popen(["bash", "-c", "sleep 60; true # run-heavy"])
|
||||
def test_reads_a_child_shells_command_and_whether_a_busy_tool_runs_below_it(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
tool = pathlib.Path(folder, "chunk-pr")
|
||||
tool.write_text("#!/bin/sh\nsleep 60\n")
|
||||
tool.chmod(0o755)
|
||||
# Like the shell of a loop that only names the tool, as the one that held a worker on 2026-11-18 did.
|
||||
naming = subprocess.Popen(["bash", "-c", "source /dev/null && eval 'until false; do sleep 60; done # chunk-pr merge'"])
|
||||
busy = subprocess.Popen(["bash", "-c", f"{tool} merge; true"])
|
||||
other = subprocess.Popen(["sleep", "60"])
|
||||
try:
|
||||
time.sleep(0.5)
|
||||
shells = {shell.pid: shell for shell in child_shells.shells_of(os.getpid())}
|
||||
self.assertEqual(sorted(shells), sorted([naming.pid, busy.pid]))
|
||||
self.assertEqual(shells[naming.pid].command, "until false; do sleep 60; done # chunk-pr merge")
|
||||
self.assertEqual((shells[naming.pid].runs_busy_tool, shells[busy.pid].runs_busy_tool), (False, True))
|
||||
self.assertLess(abs(shells[naming.pid].started - time.time()), 5)
|
||||
finally:
|
||||
for proc in (naming, busy, other):
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
|
||||
def test_stops_a_shell_and_the_process_it_started_and_no_other(self):
|
||||
shell = subprocess.Popen(["bash", "-c", "sleep 60; true"])
|
||||
other = subprocess.Popen(["sleep", "60"])
|
||||
try:
|
||||
time.sleep(0.3)
|
||||
shells = {shell.pid: shell for shell in child_shells.shells_of(os.getpid())}
|
||||
self.assertEqual(sorted(shells), sorted([quiet.pid, heavy.pid]))
|
||||
self.assertEqual(shells[quiet.pid].command, "bash -c sleep 60; true")
|
||||
self.assertEqual((shells[quiet.pid].holds_slot, shells[heavy.pid].holds_slot), (False, True))
|
||||
self.assertLess(abs(shells[quiet.pid].started - time.time()), 5)
|
||||
child_shells.stop(shells[quiet.pid])
|
||||
self.assertEqual(quiet.wait(timeout=5), -signal.SIGTERM)
|
||||
time.sleep(0.5)
|
||||
found = child_shells.shells_of(os.getpid())
|
||||
[stopping] = [each for each in found if each.pid == shell.pid]
|
||||
below = child_shells.descendants(shell.pid)
|
||||
self.assertEqual(len(below), 1)
|
||||
child_shells.stop(stopping)
|
||||
self.assertEqual(shell.wait(timeout=5), -signal.SIGTERM)
|
||||
for _ in range(50):
|
||||
if not os.path.exists(f"/proc/{below[0]}"):
|
||||
break
|
||||
time.sleep(0.1)
|
||||
self.assertFalse(os.path.exists(f"/proc/{below[0]}"))
|
||||
self.assertIsNone(other.poll())
|
||||
finally:
|
||||
for proc in (quiet, heavy, other):
|
||||
for proc in (shell, other):
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
|
||||
Reference in New Issue
Block a user