End a worker that lingers after its final result by stopping the quiet background shells it left #611
@@ -2,6 +2,7 @@
|
||||
|
||||
## Unreleased
|
||||
|
||||
- loop-claude stops the background shells that hold a worker after its final result, once they have been quiet for a few minutes, and records each stop
|
||||
- Add `do-epic-e2e`, a command that runs `do-epic` end to end against a sandbox epic on a real Jira site and deletes the epic afterwards.
|
||||
- `do-epic <EPIC-KEY>` plans the epic before it starts chunks: it files a sub-task per repo under an open issue that changes several repos, labels each other unlabelled issue with where it is worked, asking on the terminal where the repo rule leaves that unclear, and puts the scored issues it filed in order of score.
|
||||
- loop-claude compacts a waiting worker's session 10 minutes before its hour-long prompt cache expires, where no answer to an interrupt has come and the context holds at least 60k tokens, so the resume doesn't write the whole context again.
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Ends a worker whose claude -p printed its final result and then waits on background shells that have gone quiet.
|
||||
|
||||
Claude Code's own time limit for a background command cannot do this: `BASH_DEFAULT_TIMEOUT_MS` and `BASH_MAX_TIMEOUT_MS` only raise the limit, and a call's `timeout` of up to 2 hours replaces the 10 minutes of a `claude -p` run (docs: code.claude.com/docs/en/tools-reference.md → "Time limit for background commands"). On 2026-11-18 a worker waited 95 minutes in an `until` loop whose `pgrep -f` matched its own command line.
|
||||
|
||||
The stops go to RECORD in the project's transcript archive rather than to the judge's records: the judge reads a worker's transcript, so the archive is where the pattern can be counted when it becomes a rule-hook ban (item 421).
|
||||
"""
|
||||
import pathlib
|
||||
|
||||
from loop_effects.effects import Effects, say, Streaming
|
||||
from loop_effects.child_shells import Shell
|
||||
from text_files.utc_stamp import utc
|
||||
|
||||
# Seconds without a line from the worker before a check, so a check is 5 minutes after its last line.
|
||||
CHECK_EVERY = 300
|
||||
# Seconds a shell's output must have been quiet before the shell is stopped.
|
||||
QUIET = 180
|
||||
RECORD = "lingering-workers.md"
|
||||
|
||||
|
||||
def stopped_shells(proc: Streaming, printed_result: bool, record: pathlib.Path, effects: Effects) -> None:
|
||||
"""Stops the shells of a claude that is still alive CHECK_EVERY seconds after it printed its final result (printed_result), saying each stop on the terminal and in record. A claude whose latest line is not a result is working and keeps its shells, and so does one with a shell that wrote output within QUIET seconds or runs a busy tool, as `Shell.runs_busy_tool` says. A subagent's shells are children of the claude too, so they count like its own: a subagent that prints keeps them, and one inside a single quiet call loses it."""
|
||||
if not printed_result:
|
||||
return
|
||||
now = effects.now()
|
||||
shells = proc.shells()
|
||||
if any(now - shell.written <= QUIET or shell.runs_busy_tool for shell in shells):
|
||||
return
|
||||
for shell in shells:
|
||||
proc.stop(shell)
|
||||
line = stop_line(shell, now)
|
||||
say(line, effects, stdout=True)
|
||||
try:
|
||||
record.parent.mkdir(parents=True, exist_ok=True)
|
||||
with record.open("a") as handle:
|
||||
handle.write(f"- {utc(now)} {line}\n")
|
||||
except OSError as failure:
|
||||
say(f"could not record the stop in {record}: {failure.strerror}", effects)
|
||||
|
||||
|
||||
def stop_line(shell: Shell, now: float) -> str:
|
||||
return f"stopped shell {shell.pid}, which held a worker after its final result; it had run {int(now - shell.started)} s and written nothing for {int(now - shell.written)} s: {shell.command}"
|
||||
@@ -122,7 +122,7 @@ def work(cycle: Cycle, answered: QuotedAnswer | TypedAnswer | None, stepped: boo
|
||||
if isinstance(answered, QuotedAnswer):
|
||||
say(f"\n=== {cycle.prompts.quote_sources} answered: {answered.question} ===\n{answered.answer}", cycle.effects, stdout=True)
|
||||
# worker_start runs inside the lambda: a failed attempt may have opened the session.
|
||||
if stop := retried("worker", lambda: run_worker(cycle.root, worker_start(cycle, answered, stepped), worker_environment(cycle), cycle.effects), cycle.effects):
|
||||
if stop := retried("worker", lambda: run_worker(cycle.root, worker_start(cycle, answered, stepped), worker_environment(cycle), cycle.project_archive, cycle.effects), cycle.effects):
|
||||
return stop
|
||||
path = transcript_file(cycle.session)
|
||||
return WorkerRan(path) if path is not None else Stop(1, f"claude left no transcript for session {cycle.session.id}")
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import pathlib
|
||||
|
||||
from claude_code.entries import AssistantTurn, ends_stream, entry_from, hit_limit, json_object, limit_said, rejected_until
|
||||
from agent_cycle.lingering_workers import CHECK_EVERY, RECORD, stopped_shells
|
||||
from loop_effects.effects import Effects, say
|
||||
from failure import Failure
|
||||
from claude_code.reply import Limited
|
||||
@@ -15,22 +18,29 @@ AUTO_COMPACT_WINDOW = "CLAUDE_CODE_AUTO_COMPACT_WINDOW"
|
||||
COMPACT_WINDOW = 200_000
|
||||
|
||||
|
||||
def run_worker(root: str, args: list[str], env: dict[str, str], effects: Effects) -> Failure | Limited | None:
|
||||
def run_worker(root: str, args: list[str], env: dict[str, str], project_archive: pathlib.Path, effects: Effects) -> Failure | Limited | None:
|
||||
wait_for_memory(effects)
|
||||
return run_claude(root, args, env, effects)
|
||||
return run_claude(root, args, env, effects, project_archive / RECORD)
|
||||
|
||||
|
||||
def run_claude(root: str, args: list[str], env: dict[str, str], effects: Effects) -> Failure | Limited | None:
|
||||
"""Runs a headless claude at once, printing what it says, and returns how it failed, or None."""
|
||||
def run_claude(root: str, args: list[str], env: dict[str, str], effects: Effects, record: pathlib.Path | None = None) -> Failure | Limited | None:
|
||||
"""Runs a headless claude at once, printing what it says, and returns how it failed, or None. Where record is given, a claude that lingers after its final result loses its quiet shells, as `lingering_workers.stopped_shells` says, and each stop is appended to record."""
|
||||
try:
|
||||
proc = effects.stream_claude(args, root, env)
|
||||
except OSError as failure:
|
||||
return Failure(f"could not start claude: {failure.strerror}")
|
||||
result: dict | None = None
|
||||
# Whether the worker's latest line is its result; a worker that prints on is working.
|
||||
printed_result = False
|
||||
resets: list[int] = []
|
||||
for raw in proc.lines():
|
||||
for raw in proc.lines(CHECK_EVERY):
|
||||
if raw is None:
|
||||
if record is not None:
|
||||
stopped_shells(proc, printed_result, record, effects)
|
||||
continue
|
||||
event = json_object(raw) or {}
|
||||
if ends_stream(event):
|
||||
printed_result = ends_stream(event)
|
||||
if printed_result:
|
||||
result = event
|
||||
if (resets_at := rejected_until(event)) is not None:
|
||||
resets.append(resets_at)
|
||||
|
||||
@@ -81,6 +81,13 @@ transcript is copied into a folder per chunk under $LOOP_CLAUDE_TRANSCRIPTS/<key
|
||||
then each verdict, quoted answer, rejected judge reply, step-up and stop, each opened by its UTC
|
||||
time. The layout, and when a copy is replaced, are in bin/transcript_archive/__init__.py.
|
||||
|
||||
A worker's claude can print its final result and then keep waiting on a background shell that
|
||||
never ends. Every 5 minutes without output from that claude after its final result, loop-claude
|
||||
stops each of its shells and what runs below them, unless a shell has written output in the last 3
|
||||
minutes, runs chunk-pr or run-heavy, or holds a heavy slot. It prints each stop and appends it, with
|
||||
the UTC time, to lingering-workers.md at the project's folder in that archive
|
||||
(bin/agent_cycle/lingering_workers.py).
|
||||
|
||||
Before each worker starts, loop-claude waits until at least a quarter of the machine's memory is
|
||||
available (bin/machine_load/memory.py).
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ from claude_code import entries, reply, session
|
||||
from failure import Failure
|
||||
from agent_cycle.verdicts import Finished, Interrupt, Judged, QuotedAnswer, TakenIn, Unfinished
|
||||
from git_cli.git_repo import git
|
||||
from agent_cycle import interrupt, judge, retry, run, spend, states, worker, worker_prompts
|
||||
from agent_cycle import interrupt, judge, lingering_workers, retry, run, spend, states, worker, worker_prompts
|
||||
from claude_code.reply import Limited
|
||||
import loop_claude.routed_item
|
||||
from loop_claude.loop import main, reload, routing_check, this_loop, usage
|
||||
@@ -47,6 +47,7 @@ from loop_claude.cycle import prompts
|
||||
from model_routing.ladder_file import read_ladder, SHIPPED
|
||||
from model_routing.route import Route, route_from, WorkerLine
|
||||
from agent_cycle import routing
|
||||
from loop_effects.child_shells import Shell
|
||||
from agent_cycle.stop import Stop
|
||||
from agent_cycle import tmux
|
||||
from claude_code.tokens import Tokens, tokens_of
|
||||
@@ -232,12 +233,26 @@ class Worker:
|
||||
|
||||
|
||||
class Streamed:
|
||||
def __init__(self, code, *events):
|
||||
"""A started claude. An event of None is an idle stretch of CHECK_EVERY seconds, which advances `clock` by that; `held` are the shells it has, and `stopped` collects the ones stopped."""
|
||||
|
||||
def __init__(self, code, *events, clock=None, held=()):
|
||||
self.code = code
|
||||
self.events = events or ({"type": "result"},)
|
||||
self.clock = clock
|
||||
self.held = list(held)
|
||||
self.stopped = []
|
||||
|
||||
def lines(self):
|
||||
return [json.dumps(event) + "\n" for event in self.events]
|
||||
def lines(self, idle):
|
||||
for event in self.events:
|
||||
if event is None:
|
||||
self.clock[0] += idle
|
||||
yield None if event is None else json.dumps(event) + "\n"
|
||||
|
||||
def shells(self):
|
||||
return self.held
|
||||
|
||||
def stop(self, shell):
|
||||
self.stopped.append(shell)
|
||||
|
||||
def wait(self):
|
||||
return self.code
|
||||
@@ -780,7 +795,7 @@ class Limits(unittest.TestCase):
|
||||
allowed = {"type": "rate_limit_event", "rate_limit_info": {"status": "allowed", "resetsAt": NOW + 9000}}
|
||||
limited = {"type": "result", "api_error_status": 429, "result": "You've hit your session limit"}
|
||||
fake = FakeEffects(streams=[Streamed(1, rejected[1], allowed, rejected[0], limited)])
|
||||
self.assertEqual(worker.run_worker("/r", [], {}, fake.effects()), Limited(NOW + 600, "You've hit your session limit"))
|
||||
self.assertEqual(worker.run_worker("/r", [], {}, pathlib.Path("/archive"), fake.effects()), Limited(NOW + 600, "You've hit your session limit"))
|
||||
|
||||
def test_a_limit_outranks_the_exit_code_and_its_message_fits_one_line(self):
|
||||
def limited(**said):
|
||||
@@ -802,18 +817,85 @@ class Limits(unittest.TestCase):
|
||||
self.assertEqual(entries.rejected_until({"type": "rate_limit_event", "rate_limit_info": info}), want)
|
||||
|
||||
|
||||
class LingeringWorkers(unittest.TestCase):
|
||||
RESULT = {"type": "result"}
|
||||
|
||||
def worker_with(self, *events, held=()):
|
||||
"""Runs a worker whose claude prints events and holds shells `held`; returns the claude, what loop-claude said and the record's text."""
|
||||
clock = [NOW]
|
||||
streamed = Streamed(0, *events, clock=clock, held=held)
|
||||
fake = FakeEffects(streams=[streamed])
|
||||
effects = dataclasses.replace(fake.effects(), now=lambda: clock[0])
|
||||
with tempfile.TemporaryDirectory() as folder, contextlib.redirect_stdout(io.StringIO()) as said:
|
||||
# The archive folder of a project's first stop doesn't exist yet.
|
||||
archive = pathlib.Path(folder, "transcripts", "project")
|
||||
worker.run_worker("/r", [], {}, archive, effects)
|
||||
record = archive / lingering_workers.RECORD
|
||||
return streamed, said.getvalue(), record.read_text() if record.exists() else ""
|
||||
|
||||
def shell(self, written=NOW - 1000, runs_busy_tool=False):
|
||||
return Shell(pid=7, started=NOW - 1000, command="until false; do sleep 15; done", written=written, runs_busy_tool=runs_busy_tool)
|
||||
|
||||
def test_a_worker_that_lingers_after_its_result_loses_its_quiet_shell_and_the_stop_is_said_and_recorded_in_utc(self):
|
||||
shell = self.shell()
|
||||
streamed, said, record = self.worker_with(self.RESULT, None, held=[shell])
|
||||
self.assertEqual(streamed.stopped, [shell])
|
||||
line = f"stopped shell 7, which held a worker after its final result; it had run 1300 s and written nothing for 1300 s: {shell.command}"
|
||||
self.assertEqual(said, f"<{NOW + 300}> {line}\n")
|
||||
self.assertEqual(record, f"- {utc(NOW + 300)} {line}\n")
|
||||
|
||||
def test_a_worker_keeps_its_shells_where_it_printed_after_its_result_or_printed_none_or_no_check_came_due(self):
|
||||
for events in ((self.RESULT, {"type": "assistant", "message": {"content": []}}, None), (self.RESULT,), (None,), ({"type": "assistant", "message": {"content": []}}, None)):
|
||||
with self.subTest(events=events):
|
||||
streamed, said, record = self.worker_with(*events, held=[self.shell()])
|
||||
self.assertEqual((streamed.stopped, said, record), ([], "", ""))
|
||||
|
||||
def test_a_worker_keeps_every_shell_while_one_of_them_writes_output_or_runs_a_busy_tool(self):
|
||||
for busy in (self.shell(written=NOW + 200), self.shell(runs_busy_tool=True)):
|
||||
with self.subTest(busy=busy):
|
||||
streamed, _, _ = self.worker_with(self.RESULT, None, held=[self.shell(), busy])
|
||||
self.assertEqual(streamed.stopped, [])
|
||||
|
||||
def test_a_worker_keeps_a_shell_that_wrote_within_the_quiet_time_and_stops_one_that_wrote_before_it(self):
|
||||
# The check comes 300 s after the result, so a shell written at NOW + 300 - QUIET is the latest that is still kept.
|
||||
kept = self.shell(written=NOW + 300 - lingering_workers.QUIET)
|
||||
stopped = self.shell(written=NOW + 300 - lingering_workers.QUIET - 1)
|
||||
self.assertEqual(self.worker_with(self.RESULT, None, held=[kept])[0].stopped, [])
|
||||
self.assertEqual(self.worker_with(self.RESULT, None, held=[stopped])[0].stopped, [stopped])
|
||||
|
||||
def test_a_worker_reads_on_past_a_check_that_found_it_still_working(self):
|
||||
clock = [NOW]
|
||||
effects = dataclasses.replace(FakeEffects(streams=[Streamed(0, None, self.RESULT, clock=clock)]).effects(), now=lambda: clock[0])
|
||||
self.assertIsNone(worker.run_worker("/r", [], {}, pathlib.Path("/archive"), effects))
|
||||
|
||||
def test_a_stop_that_cannot_be_recorded_is_said_on_stderr_and_every_shell_is_still_stopped(self):
|
||||
clock = [NOW]
|
||||
held = [self.shell(), self.shell()]
|
||||
streamed = Streamed(0, self.RESULT, None, clock=clock, held=held)
|
||||
effects = dataclasses.replace(FakeEffects(streams=[streamed]).effects(), now=lambda: clock[0])
|
||||
with tempfile.NamedTemporaryFile() as blocker, contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()) as failed:
|
||||
worker.run_worker("/r", [], {}, pathlib.Path(blocker.name), effects)
|
||||
self.assertEqual(streamed.stopped, held)
|
||||
self.assertIn("could not record the stop", failed.getvalue())
|
||||
|
||||
def test_every_stop_of_a_check_is_recorded_though_the_archive_folder_exists_after_the_first(self):
|
||||
held = [self.shell(), self.shell()]
|
||||
_, said, record = self.worker_with(self.RESULT, None, held=held)
|
||||
self.assertEqual(record.count("stopped shell 7"), 2)
|
||||
|
||||
|
||||
class MemoryWait(unittest.TestCase):
|
||||
def test_a_worker_starts_only_once_a_quarter_of_memory_is_available(self):
|
||||
fake = FakeEffects(free_memory=[memory.Memory(available=3, total=16), memory.Memory(available=3, total=16), memory.Memory(available=4, total=16)], streams=[Streamed(0)])
|
||||
with contextlib.redirect_stderr(io.StringIO()) as said:
|
||||
self.assertIsNone(worker.run_worker("/r", [], {}, fake.effects()))
|
||||
self.assertIsNone(worker.run_worker("/r", [], {}, pathlib.Path("/archive"), fake.effects()))
|
||||
self.assertEqual([seam for seam, _ in fake.calls], ["memory", "sleep", "memory", "sleep", "memory", "stream_claude"])
|
||||
self.assertEqual(fake.made("sleep"), [worker.MEMORY_POLL] * 2)
|
||||
self.assertEqual(said.getvalue().count("waiting"), 1, said.getvalue())
|
||||
|
||||
def test_a_worker_starts_at_once_where_memory_cannot_be_read(self):
|
||||
fake = FakeEffects(free_memory=[None], streams=[Streamed(0)])
|
||||
self.assertIsNone(worker.run_worker("/r", [], {}, fake.effects()))
|
||||
self.assertIsNone(worker.run_worker("/r", [], {}, pathlib.Path("/archive"), fake.effects()))
|
||||
self.assertEqual([seam for seam, _ in fake.calls], ["memory", "stream_claude"])
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
"""The shells a running claude has started, read from /proc (Linux), and how to stop one.
|
||||
|
||||
Claude Code starts each Bash call as `bash -c "source <snapshot> && … && eval '<the command>'"` and writes its output to a file under /tmp/claude-<uid>/.
|
||||
"""
|
||||
import dataclasses
|
||||
import os
|
||||
import pathlib
|
||||
import signal
|
||||
import stat
|
||||
|
||||
from machine_load.slots import slot_folder
|
||||
|
||||
# The command names of a shell claude starts for a Bash call.
|
||||
SHELL_NAMES = frozenset({"bash", "dash", "sh", "zsh"})
|
||||
COMMAND_CHARS = 300
|
||||
# Scripts whose process below a shell shows work that may go quiet for long: run-heavy while it waits for a slot, and chunk-pr, which waits for a merge's approval and checks.
|
||||
BUSY_TOOLS = frozenset({"chunk-pr", "run-heavy"})
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Shell:
|
||||
pid: int
|
||||
# Epoch seconds at which the shell started.
|
||||
started: float
|
||||
# What the shell runs, without the snapshot Claude Code sources before it.
|
||||
command: str
|
||||
# Epoch seconds of the last write to the file the shell's output goes to; the shell's start where the output goes to no file, which the check cannot read.
|
||||
written: float
|
||||
# Whether a process below the shell runs one of BUSY_TOOLS or holds a heavy slot, a file of slot_folder that run-heavy leaves open in the command it runs.
|
||||
runs_busy_tool: bool
|
||||
|
||||
|
||||
def parent_and_start(pid: int) -> tuple[int, int, str] | None:
|
||||
"""The parent pid, the start time in clock ticks since boot and the command name of a process; None where it is gone."""
|
||||
try:
|
||||
with open(f"/proc/{pid}/stat") as handle:
|
||||
text = handle.read()
|
||||
except OSError:
|
||||
return None
|
||||
# The name sits in parentheses and may hold spaces or parentheses itself, so split at the last one.
|
||||
name = text[text.index("(") + 1 : text.rindex(")")]
|
||||
fields = text[text.rindex(")") + 2 :].split()
|
||||
return int(fields[1]), int(fields[19]), name
|
||||
|
||||
|
||||
def children_of(pid: int) -> list[int]:
|
||||
return [int(entry) for entry in os.listdir("/proc") if entry.isdigit() and (found := parent_and_start(int(entry))) and found[0] == pid]
|
||||
|
||||
|
||||
def argv_of(pid: int) -> list[str]:
|
||||
try:
|
||||
with open(f"/proc/{pid}/cmdline", "rb") as handle:
|
||||
return handle.read().decode(errors="replace").rstrip("\0").split("\0")
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
|
||||
def descendants(pid: int) -> list[int]:
|
||||
"""Every process below pid, each after its parent."""
|
||||
found = children_of(pid)
|
||||
for child in list(found):
|
||||
found += descendants(child)
|
||||
return found
|
||||
|
||||
|
||||
def boot_time() -> float:
|
||||
with open("/proc/stat") as handle:
|
||||
return next(float(line.split()[1]) for line in handle if line.startswith("btime"))
|
||||
|
||||
|
||||
def output_written(pid: int, started: float) -> float:
|
||||
"""When the file that pid's standard output goes to was last written; started where it goes to a pipe or a terminal."""
|
||||
try:
|
||||
info = os.stat(f"/proc/{pid}/fd/1")
|
||||
except OSError:
|
||||
return started
|
||||
return info.st_mtime if stat.S_ISREG(info.st_mode) else started
|
||||
|
||||
|
||||
def command_of(shell_argv: list[str]) -> str:
|
||||
"""What a shell runs: the text of its command line after Claude Code's `eval '`, up to the last ` < /dev/null`, which a call that began in the foreground lacks; the whole line where it holds no `eval`."""
|
||||
line = " ".join(shell_argv)
|
||||
_, found, command = line.partition(" && eval ")
|
||||
return (command.rsplit(" < /dev/null", 1)[0].removeprefix("'").removesuffix("'") if found else line)[:COMMAND_CHARS]
|
||||
|
||||
|
||||
def holds_slot(pid: int, slots: pathlib.Path) -> bool:
|
||||
"""Whether pid has a file of the heavy slots' folder open. run-heavy replaces itself with its command once it holds a slot, and the command keeps the slot's file open."""
|
||||
try:
|
||||
return any(os.readlink(f"/proc/{pid}/fd/{fd}").startswith(f"{slots}/") for fd in os.listdir(f"/proc/{pid}/fd"))
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def runs_busy_tool(pid: int, slots: pathlib.Path) -> bool:
|
||||
"""Whether a process below pid holds a heavy slot or has one of BUSY_TOOLS as its program or its script, as `python3 /x/chunk-pr`. A command line that merely names a tool, such as a `pgrep -f chunk-pr`, doesn't count."""
|
||||
return any(holds_slot(each, slots) or any(os.path.basename(name) in BUSY_TOOLS for name in argv_of(each)[:2]) for each in descendants(pid))
|
||||
|
||||
|
||||
def shells_of(claude_pid: int) -> list[Shell]:
|
||||
"""The shells that are children of claude_pid; none where /proc doesn't exist, as off Linux."""
|
||||
if not os.path.isdir("/proc"):
|
||||
return []
|
||||
slots = slot_folder(os.environ, pathlib.Path.home())
|
||||
booted = boot_time()
|
||||
ticks = os.sysconf("SC_CLK_TCK")
|
||||
shells = []
|
||||
for pid in children_of(claude_pid):
|
||||
found = parent_and_start(pid)
|
||||
if found is None or found[2] not in SHELL_NAMES:
|
||||
continue
|
||||
started = booted + found[1] / ticks
|
||||
shells.append(Shell(pid, started, command_of(argv_of(pid)), output_written(pid, started), runs_busy_tool(pid, slots)))
|
||||
return shells
|
||||
|
||||
|
||||
def stop(shell: Shell) -> None:
|
||||
"""Ends the shell and every process below it, so that a loop the shell runs doesn't outlive it."""
|
||||
for pid in [*reversed(descendants(shell.pid)), shell.pid]:
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except OSError:
|
||||
pass
|
||||
@@ -5,6 +5,7 @@ A change here, such as to how input reads a line, changes every tool that runs o
|
||||
import dataclasses
|
||||
import os
|
||||
import pathlib
|
||||
import select
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -14,6 +15,7 @@ import time
|
||||
import typing
|
||||
|
||||
from notifier.notify import Backend, BACKENDS, notify
|
||||
from loop_effects import child_shells
|
||||
from failure import Failure
|
||||
from git_cli import git_repo
|
||||
from machine_load.memory import Memory, read_memory
|
||||
@@ -26,7 +28,17 @@ PYTHON_TIMEOUT = 600
|
||||
class Streaming(typing.Protocol):
|
||||
"""A started claude whose stream-json lines arrive as it works."""
|
||||
|
||||
def lines(self) -> typing.Iterable[str]: ...
|
||||
def lines(self, idle: float) -> typing.Iterable[str | None]:
|
||||
"""Each line as it arrives, and None whenever idle seconds pass without one."""
|
||||
...
|
||||
|
||||
def shells(self) -> list[child_shells.Shell]:
|
||||
"""The shells that are children of the claude."""
|
||||
...
|
||||
|
||||
def stop(self, shell: child_shells.Shell) -> None:
|
||||
"""Ends the shell and everything below it."""
|
||||
...
|
||||
|
||||
def wait(self) -> int: ...
|
||||
|
||||
@@ -62,11 +74,32 @@ class Effects:
|
||||
|
||||
class StreamingClaude:
|
||||
def __init__(self, args: list[str], cwd: str, env: dict[str, str]) -> None:
|
||||
self.proc = subprocess.Popen(["claude", *args], cwd=cwd, env={**os.environ, **env}, encoding="utf-8", errors="replace", stdin=subprocess.DEVNULL, stdout=subprocess.PIPE)
|
||||
self.proc = subprocess.Popen(["claude", *args], cwd=cwd, env={**os.environ, **env}, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, bufsize=0)
|
||||
|
||||
def lines(self) -> typing.Iterable[str]:
|
||||
def lines(self, idle: float) -> typing.Iterable[str | None]:
|
||||
# Reads the pipe's file descriptor itself. select cannot see what a buffered reader holds.
|
||||
# A thread blocked in a read would keep the pipe from closing when a Ctrl-C ends the loop.
|
||||
pending = b""
|
||||
with self.proc:
|
||||
yield from self.proc.stdout or ()
|
||||
fd = self.proc.stdout.fileno() if self.proc.stdout else -1
|
||||
while True:
|
||||
if not select.select([fd], [], [], idle)[0]:
|
||||
yield None
|
||||
continue
|
||||
chunk = os.read(fd, 65536)
|
||||
if not chunk:
|
||||
break
|
||||
*complete, pending = (pending + chunk).split(b"\n")
|
||||
for line in complete:
|
||||
yield line.decode("utf-8", errors="replace") + "\n"
|
||||
if pending:
|
||||
yield pending.decode("utf-8", errors="replace")
|
||||
|
||||
def shells(self) -> list[child_shells.Shell]:
|
||||
return child_shells.shells_of(self.proc.pid)
|
||||
|
||||
def stop(self, shell: child_shells.Shell) -> None:
|
||||
child_shells.stop(shell)
|
||||
|
||||
def wait(self) -> int:
|
||||
return self.proc.wait()
|
||||
|
||||
@@ -4,6 +4,7 @@ import io
|
||||
import os
|
||||
import pathlib
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
@@ -13,7 +14,8 @@ import unittest.mock
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
|
||||
|
||||
from loop_effects.effects import run_python, say, start_timer, system_effects
|
||||
from loop_effects import child_shells
|
||||
from loop_effects.effects import run_python, say, start_timer, StreamingClaude, system_effects
|
||||
|
||||
NOW = 1790000000
|
||||
# How a line said at NOW opens, with clocked's local_time.
|
||||
@@ -25,6 +27,15 @@ def clocked():
|
||||
return dataclasses.replace(system_effects(), local_time=lambda seconds: f"<{seconds}>", now=lambda: NOW)
|
||||
|
||||
|
||||
def wait_until(condition):
|
||||
"""Polls up to 5 s for condition, so a test doesn't depend on how fast a process starts."""
|
||||
for _ in range(100):
|
||||
if condition():
|
||||
return
|
||||
time.sleep(0.05)
|
||||
raise AssertionError("a process did not reach its state in time")
|
||||
|
||||
|
||||
class Said(unittest.TestCase):
|
||||
def test_opens_a_line_with_the_local_time_of_the_effects_clock(self):
|
||||
with contextlib.redirect_stderr(io.StringIO()) as stderr, contextlib.redirect_stdout(io.StringIO()) as stdout:
|
||||
@@ -137,6 +148,139 @@ class PythonEffect(unittest.TestCase):
|
||||
self.assertIn(gone, done.stderr)
|
||||
|
||||
|
||||
class ChildShells(unittest.TestCase):
|
||||
def started(self, command, **options):
|
||||
"""A shell running command, once a process runs below it."""
|
||||
proc = subprocess.Popen(["bash", "-c", command], **options)
|
||||
self.addCleanup(lambda: (proc.kill(), proc.wait()))
|
||||
wait_until(lambda: child_shells.descendants(proc.pid))
|
||||
return proc
|
||||
|
||||
def shell_of(self, proc):
|
||||
[found] = [each for each in child_shells.shells_of(os.getpid()) if each.pid == proc.pid]
|
||||
return found
|
||||
|
||||
def test_reads_a_child_shells_command_and_whether_a_busy_tool_runs_below_it(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
tool = pathlib.Path(folder, "chunk-pr")
|
||||
tool.write_text("#!/bin/sh\nsleep 60\n")
|
||||
tool.chmod(0o755)
|
||||
# Like the shell of a loop that only names the tool, as the one that held a worker on 2026-11-18 did.
|
||||
naming = self.started("source /dev/null && eval 'until false; do sleep 60; done # chunk-pr merge' < /dev/null && pwd")
|
||||
busy = self.started(f"{tool} merge; true")
|
||||
self.assertEqual(self.shell_of(naming).command, "until false; do sleep 60; done # chunk-pr merge")
|
||||
self.assertEqual((self.shell_of(naming).runs_busy_tool, self.shell_of(busy).runs_busy_tool), (False, True))
|
||||
self.assertLess(abs(self.shell_of(naming).started - time.time()), 5)
|
||||
|
||||
def test_a_shell_below_which_a_process_holds_a_heavy_slot_file_runs_a_busy_tool(self):
|
||||
# run-heavy replaces itself with its command, which keeps the slot's file open.
|
||||
with tempfile.TemporaryDirectory() as state, unittest.mock.patch.dict(os.environ, {"XDG_STATE_HOME": state}):
|
||||
slot = pathlib.Path(state, "run-heavy", "slot-0")
|
||||
slot.parent.mkdir()
|
||||
holding = self.started(f"exec 9>{slot}; sleep 60; true")
|
||||
plain = self.started("sleep 60; true")
|
||||
self.assertEqual((self.shell_of(holding).runs_busy_tool, self.shell_of(plain).runs_busy_tool), (True, False))
|
||||
|
||||
def test_a_shell_written_output_is_the_modified_time_of_its_output_file_and_else_its_start(self):
|
||||
with tempfile.NamedTemporaryFile() as output:
|
||||
os.utime(output.name, (1000, 1000))
|
||||
to_file = self.started("sleep 60; true", stdout=output)
|
||||
to_pipe = self.started("sleep 60; true", stdout=subprocess.PIPE)
|
||||
self.assertEqual(self.shell_of(to_file).written, 1000)
|
||||
self.assertEqual(self.shell_of(to_pipe).written, self.shell_of(to_pipe).started)
|
||||
|
||||
def test_stops_a_shell_and_the_process_it_started_and_no_other(self):
|
||||
shell = self.started("sleep 60; true")
|
||||
other = subprocess.Popen(["sleep", "60"])
|
||||
self.addCleanup(lambda: (other.kill(), other.wait()))
|
||||
[below] = child_shells.descendants(shell.pid)
|
||||
child_shells.stop(self.shell_of(shell))
|
||||
self.assertEqual(shell.wait(timeout=5), -signal.SIGTERM)
|
||||
wait_until(lambda: not os.path.exists(f"/proc/{below}"))
|
||||
self.assertIsNone(other.poll())
|
||||
|
||||
def test_reads_the_parent_and_name_of_a_process_whose_name_holds_parentheses(self):
|
||||
with open("/proc/self/comm") as handle:
|
||||
before = handle.read().strip()
|
||||
self.addCleanup(lambda: pathlib.Path("/proc/self/comm").write_text(before))
|
||||
pathlib.Path("/proc/self/comm").write_text("a) (b")
|
||||
parent, _, name = child_shells.parent_and_start(os.getpid())
|
||||
self.assertEqual((parent, name), (os.getppid(), "a) (b"))
|
||||
|
||||
def test_reads_every_argument_of_a_process_including_bytes_that_are_not_utf8(self):
|
||||
proc = subprocess.Popen(["sh", "-c", "sleep 60; true", "\udcffX"])
|
||||
self.addCleanup(lambda: (proc.kill(), proc.wait()))
|
||||
wait_until(lambda: child_shells.argv_of(proc.pid)[:2] == ["sh", "-c"])
|
||||
self.assertEqual(child_shells.argv_of(proc.pid), ["sh", "-c", "sleep 60; true", "\ufffdX"])
|
||||
self.assertEqual(child_shells.argv_of(2**30), [])
|
||||
|
||||
def test_finds_a_process_below_a_process_below_a_shell(self):
|
||||
shell = self.started("bash -c 'sleep 60; true'; true")
|
||||
wait_until(lambda: len(child_shells.descendants(shell.pid)) == 2)
|
||||
|
||||
def test_a_process_that_is_gone_holds_no_slot_and_a_vanished_child_does_not_hide_the_shells_after_it(self):
|
||||
self.assertFalse(child_shells.holds_slot(2**30, pathlib.Path("/")))
|
||||
shell = self.started("sleep 60; true")
|
||||
real = child_shells.children_of
|
||||
with unittest.mock.patch.object(child_shells, "children_of", side_effect=lambda pid: [2**30, shell.pid] if pid == os.getpid() else real(pid)):
|
||||
self.assertEqual([each.pid for each in child_shells.shells_of(os.getpid())], [shell.pid])
|
||||
|
||||
def test_command_of_takes_the_text_of_the_last_eval_wrapper_and_else_the_whole_line(self):
|
||||
wrapped = ["/bin/bash", "-c", "source /s && eval 'echo a && eval b < /dev/null' < /dev/null && pwd -P >| /t"]
|
||||
self.assertEqual(child_shells.command_of(wrapped), "echo a && eval b < /dev/null")
|
||||
self.assertEqual(child_shells.command_of(["sleep", "60"]), "sleep 60")
|
||||
self.assertEqual(child_shells.command_of(["bash", "-c", "x" * 400]), "bash -c " + "x" * 292)
|
||||
|
||||
|
||||
class StreamingClaudeLines(unittest.TestCase):
|
||||
def claude(self, script, args=()):
|
||||
"""A StreamingClaude whose `claude` is a shell script."""
|
||||
folder = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(folder.cleanup)
|
||||
tool = pathlib.Path(folder.name, "claude")
|
||||
tool.write_text("#!/bin/sh\n" + script)
|
||||
tool.chmod(0o755)
|
||||
streaming = StreamingClaude(list(args), folder.name, {"PATH": f"{folder.name}:{os.environ['PATH']}", "MARK": "marked"})
|
||||
self.addCleanup(lambda: (streaming.proc.kill(), streaming.proc.wait()))
|
||||
return streaming, folder.name
|
||||
|
||||
def test_yields_each_line_with_its_newline_and_the_last_one_without_and_replaces_bytes_that_are_not_utf8(self):
|
||||
streaming, _ = self.claude("printf 'one\\n\\377two\\n\\377-tail'")
|
||||
self.assertEqual(list(streaming.lines(5)), ["one\n", "\ufffdtwo\n", "\ufffd-tail"])
|
||||
self.assertEqual(streaming.wait(), 0)
|
||||
|
||||
def test_runs_claude_in_the_folder_with_the_arguments_and_the_environment_and_without_input(self):
|
||||
# Fills the test's own standard input, so that a claude that inherited it would read data.
|
||||
read, write = os.pipe()
|
||||
os.write(write, b"data\n")
|
||||
saved = os.dup(0)
|
||||
os.dup2(read, 0)
|
||||
try:
|
||||
streaming, folder = self.claude('echo "$PWD $MARK $*"; if read -r line; then echo data; else echo eof; fi', ["-p", "hi"])
|
||||
finally:
|
||||
os.dup2(saved, 0)
|
||||
for each in (read, write, saved):
|
||||
os.close(each)
|
||||
self.assertEqual(list(streaming.lines(5)), [f"{os.path.realpath(folder)} marked -p hi\n", "eof\n"])
|
||||
|
||||
def test_yields_none_whenever_the_idle_time_passes_without_a_line(self):
|
||||
streaming, _ = self.claude("sleep 1; echo late")
|
||||
seen = list(streaming.lines(0.1))
|
||||
self.assertEqual([each for each in seen if each is not None], ["late\n"])
|
||||
self.assertIn(None, seen[: seen.index("late\n")])
|
||||
|
||||
def test_lists_the_shells_below_claude_and_stops_one(self):
|
||||
streaming, _ = self.claude("bash -c 'sleep 60; true' &\necho ready\nwait")
|
||||
lines = streaming.lines(5)
|
||||
self.assertEqual(next(lines), "ready\n")
|
||||
wait_until(streaming.shells)
|
||||
[shell] = streaming.shells()
|
||||
self.assertEqual(shell.command, "bash -c sleep 60; true")
|
||||
streaming.stop(shell)
|
||||
self.assertEqual(streaming.wait(), 0)
|
||||
|
||||
|
||||
|
||||
class Timer(unittest.TestCase):
|
||||
def test_runs_the_action_beside_the_caller_once_the_delay_has_passed(self):
|
||||
ran = threading.Event()
|
||||
|
||||
@@ -12,8 +12,7 @@
|
||||
|
||||
| ID | Release | Blocked by | Exempt | Item | L | E | I | R | S | Moves | Score |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 420 | 0.12.0 | | | **End a worker that printed its final result and lingers, by killing the background shells it left, after a few minutes, and print each kill with its evidence on loop-claude's terminal.** | 2 | 2 | 2 | 4 | 3 | 1.5 by 6 | 2.0 |
|
||||
| 422 | 0.12.0 | 420 | | **Check every 5 minutes whether a tmux window of the loop looks stuck by other signs than a lingering worker, investigate it with a headless claude, and kill only the process the investigation names.** | 3 | 2 | 3 | 6 | 5 | 1.5 by 4 | 0.8 |
|
||||
| 422 | 0.12.0 | | | **Check every 5 minutes whether a tmux window of the loop looks stuck by other signs than a lingering worker, investigate it with a headless claude, and kill only the process the investigation names.** | 3 | 2 | 3 | 6 | 5 | 1.5 by 4 | 0.8 |
|
||||
| 445 | 0.13.0 | | | **Refuse a plan that holds an item a `Finish item <ID>` commit already removed, so a merge resolution cannot bring a finished item back.** | 2 | 2 | 2 | 4 | 3 | 1.4 by 3, 1.1 by 2 | 1.8 |
|
||||
| 365 | 0.13.0 | | | **Have `score-outcomes` read the finished and dropped items of a plan outside the checkout, from the project repo's history or from Jira's issue changelogs.** | 3 | 2 | 2 | 5 | 4 | 1.6 by 2, 1.1 by 2 | 1.2 |
|
||||
| 368 | 0.13.0 | | | **Call a release review due, call a release ready to cut, and tell whether a chunk owes a scoring run, for a plan outside the checkout, from the project repo's history or from Jira.** | 3 | 2 | 2 | 5 | 4 | 1.6 by 2, 1.1 by 2 | 1.2 |
|
||||
@@ -69,22 +68,6 @@
|
||||
|
||||
## Details
|
||||
|
||||
### 420. End a worker that printed its final result and lingers, by killing the background shells it left, after a few minutes, and print each kill with its evidence on loop-claude's terminal.
|
||||
|
||||
On 2026-11-18 the worker of the item 402 chunk merged its pull request at about 11:47, printed its summary and then waited 95 minutes: it had started `until [ -s /tmp/merge-402.log ] && ! pgrep -f "chunk-pr merg"; do sleep 15; done` in the background, and `pgrep -f` matched the loop's own command line, so the condition never held. Its `claude -p` stayed alive ("Waiting for background work to finish"), `loop-claude` waited for it to exit, and the window's slot started no new chunk until the human found it and killed the shell.
|
||||
|
||||
Every 5 minutes loop-claude checks each worker it runs: when the worker's last transcript entry is its final result and its `claude -p` is still alive more than 3 minutes later, it kills the background shells that are children of that process, and the worker then exits. Nothing else is killed: no worker that is still working, no process of another session, and no window with a sign short of this one. A worker whose shell is still writing output or holds a `run-heavy` slot is left alone. Each kill is printed on loop-claude's terminal with the process, its age and its command, and appended to the judge's records so the pattern can become a rule-hook ban (item 421). Item 422 handles the vaguer signs of a stuck window.
|
||||
|
||||
The check lives in the main run beside the other per-cycle work; item 401 moves that code into a package, and a conflict with it is cheaper than waiting for it.
|
||||
|
||||
Before building the check, try the platform's own bound: Claude Code documents `BASH_MAX_TIMEOUT_MS` and `BASH_DEFAULT_TIMEOUT_MS` as the time limit of a background command, 10 minutes by default in `claude -p` (docs: code.claude.com/docs/en/headless.md and env-vars). Nothing in this repo sets them, yet the lingering shell ran 95 minutes on Claude Code 2.1.327, so first find out why the limit did not apply (an explicit timeout in the worker's call, or a gap in the docs). Hand the workers a cap through those variables where that bounds the wait, and build the lingering-worker check only for what the cap leaves.
|
||||
|
||||
Touches:
|
||||
|
||||
- `bin/loop_claude/loop/main.py`: `run_loop` (small)
|
||||
- `bin/loop_claude/loop/lingering_workers.py`
|
||||
- `README.md`: `## Tools` (small)
|
||||
|
||||
### 422. Check every 5 minutes whether a tmux window of the loop looks stuck by other signs than a lingering worker, investigate it with a headless claude, and kill only the process the investigation names.
|
||||
|
||||
Item 420 ends a worker that printed its final result and lingers. A window can also stall earlier: a test run that hangs on a shell ignoring SIGINT (item 402's first full run), a pane with no output while its process tree does no work, or a background shell older than the chunk's usual length.
|
||||
|
||||
Reference in New Issue
Block a user