Answer stability round 1: 1 🔴, 4 🟡 (1 misread); keep a shell whose command holds a heavy slot, and say a failed record write on stderr

This commit is contained in:
2026-11-18 20:12:40 +01:00
parent abae3a0559
commit a7439c2e00
6 changed files with 108 additions and 66 deletions
+13 -10
View File
@@ -12,26 +12,29 @@ from text_files.utc_stamp import utc
# Seconds without a line from the worker before a check, so a check is 5 minutes after its last line.
CHECK_EVERY = 300
# Seconds a final result must have stood, and a shell's output must have been quiet, before a shell is stopped.
SETTLED = 180
# Seconds a shell's output must have been quiet before the shell is stopped.
QUIET = 180
RECORD = "lingering-workers.md"
def stopped_shells(proc: Streaming, settled_at: float | None, record: pathlib.Path, effects: Effects) -> None:
"""Stops the shells of a claude whose final result came at settled_at and that is still alive, saying each stop on the terminal and in record. A claude that is still working (settled_at is None) or whose result came SETTLED seconds ago or less keeps its shells, and so does one with a shell that wrote output within SETTLED seconds or runs a busy tool, as `Shell.runs_busy_tool` says."""
now = effects.now()
if settled_at is None or now - settled_at <= SETTLED:
def stopped_shells(proc: Streaming, printed_result: bool, record: pathlib.Path, effects: Effects) -> None:
"""Stops the shells of a claude that is still alive CHECK_EVERY seconds after it printed its final result (printed_result), saying each stop on the terminal and in record. A claude whose latest line is not a result is working and keeps its shells, and so does one with a shell that wrote output within QUIET seconds or runs a busy tool, as `Shell.runs_busy_tool` says. A shell a background subagent started is not one of the claude's children, so it is never stopped."""
if not printed_result:
return
now = effects.now()
shells = proc.shells()
if any(now - shell.written <= SETTLED or shell.runs_busy_tool for shell in shells):
if any(now - shell.written <= QUIET or shell.runs_busy_tool for shell in shells):
return
for shell in shells:
proc.stop(shell)
line = stop_line(shell, now)
say(line, effects, stdout=True)
record.parent.mkdir(parents=True, exist_ok=True)
with record.open("a") as handle:
handle.write(f"- {utc(now)} {line}\n")
try:
record.parent.mkdir(parents=True, exist_ok=True)
with record.open("a") as handle:
handle.write(f"- {utc(now)} {line}\n")
except OSError as failure:
say(f"could not record the stop in {record}: {failure.strerror}", effects)
def stop_line(shell: Shell, now: float) -> str:
+5 -5
View File
@@ -25,16 +25,16 @@ def run_worker(root: str, args: list[str], env: dict[str, str], project_archive:
except OSError as failure:
return Failure(f"could not start claude: {failure.strerror}")
result: dict | None = None
# When the worker printed its latest result; None while its latest line is something else, since a worker that prints on is working.
settled_at: float | None = None
# Whether the worker's latest line is its result; a worker that prints on is working.
printed_result = False
resets: list[int] = []
for raw in proc.lines(CHECK_EVERY):
if raw is None:
stopped_shells(proc, settled_at, project_archive / RECORD, effects)
stopped_shells(proc, printed_result, project_archive / RECORD, effects)
continue
event = json_object(raw) or {}
settled_at = effects.now() if ends_stream(event) else None
if ends_stream(event):
printed_result = ends_stream(event)
if printed_result:
result = event
if (resets_at := rejected_until(event)) is not None:
resets.append(resets_at)
+4 -4
View File
@@ -79,10 +79,10 @@ then each verdict, quoted answer, rejected judge reply, step-up and stop, each o
time. The layout, and when a copy is replaced, are in bin/transcript_archive/__init__.py.
A worker's claude can print its final result and then keep waiting on a background shell that
never ends. Every 5 minutes without output from that claude, loop-claude stops each of its shells
and what runs below them, once the result is over 3 minutes old and no shell has written output in
the last 3 minutes or runs run-heavy or chunk-pr. It prints each stop and appends it, with the UTC
time, to lingering-workers.md at the project's folder in that archive
never ends. Every 5 minutes without output from that claude after its final result, loop-claude
stops each of its shells and what runs below them, unless a shell has written output in the last 3
minutes, runs chunk-pr or run-heavy, or holds a heavy slot. It prints each stop and appends it, with
the UTC time, to lingering-workers.md at the project's folder in that archive
(bin/loop_claude/cycle/lingering_workers.py).
Before each worker starts, loop-claude waits until at least a quarter of the machine's memory is
+16 -6
View File
@@ -814,12 +814,22 @@ class LingeringWorkers(unittest.TestCase):
streamed, _, _ = self.worker_with(self.RESULT, None, held=[self.shell(), busy])
self.assertEqual(streamed.stopped, [])
def test_a_worker_keeps_its_shells_until_its_result_has_stood_longer_than_the_settled_time(self):
fake = FakeEffects()
effects = dataclasses.replace(fake.effects(), now=lambda: NOW + lingering_workers.SETTLED)
proc = Streamed(0, held=[self.shell()])
lingering_workers.stopped_shells(proc, NOW, pathlib.Path("/unused"), effects)
self.assertEqual(proc.stopped, [])
def test_a_worker_keeps_a_shell_that_wrote_within_the_quiet_time_and_stops_one_that_wrote_before_it(self):
# The check comes 300 s after the result, so a shell written at NOW + 300 - QUIET is the latest that is still kept.
kept = self.shell(written=NOW + 300 - lingering_workers.QUIET)
stopped = self.shell(written=NOW + 300 - lingering_workers.QUIET - 1)
self.assertEqual(self.worker_with(self.RESULT, None, held=[kept])[0].stopped, [])
self.assertEqual(self.worker_with(self.RESULT, None, held=[stopped])[0].stopped, [stopped])
def test_a_stop_that_cannot_be_recorded_is_said_on_stderr_and_every_shell_is_still_stopped(self):
clock = [NOW]
held = [self.shell(), self.shell()]
streamed = Streamed(0, self.RESULT, None, clock=clock, held=held)
effects = dataclasses.replace(FakeEffects(streams=[streamed]).effects(), now=lambda: clock[0])
with tempfile.NamedTemporaryFile() as blocker, contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()) as failed:
worker.run_worker("/r", [], {}, pathlib.Path(blocker.name), effects)
self.assertEqual(streamed.stopped, held)
self.assertIn("could not record the stop", failed.getvalue())
class MemoryWait(unittest.TestCase):
+22 -8
View File
@@ -4,13 +4,16 @@ Claude Code starts each Bash call as `bash -c "source <snapshot> && … && eval
"""
import dataclasses
import os
import pathlib
import signal
import stat
from machine_load.slots import slot_folder
# The command names of a shell claude starts for a Bash call.
SHELL_NAMES = frozenset({"bash", "dash", "sh", "zsh"})
COMMAND_CHARS = 300
# Tools whose process below a shell shows work that may go quiet for long: run-heavy holds a slot of the machine's heavy work or waits for one, and chunk-pr waits for a merge's approval and checks.
# Scripts whose process below a shell shows work that may go quiet for long: run-heavy while it waits for a slot, and chunk-pr, which waits for a merge's approval and checks.
BUSY_TOOLS = frozenset({"chunk-pr", "run-heavy"})
@@ -23,7 +26,7 @@ class Shell:
command: str
# Epoch seconds of the last write to the file the shell's output goes to; the shell's start where the output goes to no file, which the check cannot read.
written: float
# Whether a process below the shell runs one of BUSY_TOOLS.
# Whether a process below the shell runs one of BUSY_TOOLS or holds a heavy slot, a file of slot_folder that run-heavy leaves open in the command it runs.
runs_busy_tool: bool
@@ -75,19 +78,30 @@ def output_written(pid: int, started: float) -> float:
def command_of(shell_argv: list[str]) -> str:
"""What a shell runs: the text of its command line between Claude Code's `eval '` and the `' < /dev/null` that ends it, or the whole line where it holds none."""
"""What a shell runs: the text of its command line after Claude Code's `eval '`, up to the first ` < /dev/null`, which a call that began in the foreground lacks; the whole line where it holds no `eval`."""
line = " ".join(shell_argv)
_, found, command = line.partition(" && eval ")
return (command.split(" < /dev/null", 1)[0].removeprefix("'").removesuffix("'") if found else line)[:COMMAND_CHARS]
def runs_busy_tool(pid: int) -> bool:
"""Whether a process below pid has one of BUSY_TOOLS as its program or its script, as `python3 /x/chunk-pr`. A command line that merely names one, such as a `pgrep -f chunk-pr`, doesn't count."""
return any(os.path.basename(name) in BUSY_TOOLS for each in descendants(pid) for name in argv_of(each)[:2])
def holds_slot(pid: int, slots: pathlib.Path) -> bool:
"""Whether pid has a file of the heavy slots' folder open. run-heavy replaces itself with its command once it holds a slot, and the command keeps the slot's file open."""
try:
return any(os.readlink(f"/proc/{pid}/fd/{fd}").startswith(f"{slots}/") for fd in os.listdir(f"/proc/{pid}/fd"))
except OSError:
return False
def runs_busy_tool(pid: int, slots: pathlib.Path) -> bool:
"""Whether a process below pid holds a heavy slot or has one of BUSY_TOOLS as its program or its script, as `python3 /x/chunk-pr`. A command line that merely names a tool, such as a `pgrep -f chunk-pr`, doesn't count."""
return any(holds_slot(each, slots) or any(os.path.basename(name) in BUSY_TOOLS for name in argv_of(each)[:2]) for each in descendants(pid))
def shells_of(claude_pid: int) -> list[Shell]:
"""The shells that are children of claude_pid."""
"""The shells that are children of claude_pid; none where /proc doesn't exist, as off Linux."""
if not os.path.isdir("/proc"):
return []
slots = slot_folder(os.environ, pathlib.Path.home())
booted = boot_time()
ticks = os.sysconf("SC_CLK_TCK")
shells = []
@@ -96,7 +110,7 @@ def shells_of(claude_pid: int) -> list[Shell]:
if found is None or found[2] not in SHELL_NAMES:
continue
started = booted + found[1] / ticks
shells.append(Shell(pid, started, command_of(argv_of(pid)), output_written(pid, started), runs_busy_tool(pid)))
shells.append(Shell(pid, started, command_of(argv_of(pid)), output_written(pid, started), runs_busy_tool(pid, slots)))
return shells
+48 -33
View File
@@ -139,45 +139,60 @@ class PythonEffect(unittest.TestCase):
class ChildShells(unittest.TestCase):
def wait_until(self, condition):
"""Polls up to 5 s for condition, so a test doesn't depend on how fast a process starts."""
for _ in range(100):
if condition():
return
time.sleep(0.05)
self.fail("a process did not reach its state in time")
def started(self, command, **options):
"""A shell running command, once a process runs below it."""
proc = subprocess.Popen(["bash", "-c", command], **options)
self.addCleanup(lambda: (proc.kill(), proc.wait()))
self.wait_until(lambda: child_shells.descendants(proc.pid))
return proc
def shell_of(self, proc):
[found] = [each for each in child_shells.shells_of(os.getpid()) if each.pid == proc.pid]
return found
def test_reads_a_child_shells_command_and_whether_a_busy_tool_runs_below_it(self):
with tempfile.TemporaryDirectory() as folder:
tool = pathlib.Path(folder, "chunk-pr")
tool.write_text("#!/bin/sh\nsleep 60\n")
tool.chmod(0o755)
# Like the shell of a loop that only names the tool, as the one that held a worker on 2026-11-18 did.
naming = subprocess.Popen(["bash", "-c", "source /dev/null && eval 'until false; do sleep 60; done # chunk-pr merge'"])
busy = subprocess.Popen(["bash", "-c", f"{tool} merge; true"])
other = subprocess.Popen(["sleep", "60"])
try:
time.sleep(0.5)
shells = {shell.pid: shell for shell in child_shells.shells_of(os.getpid())}
self.assertEqual(sorted(shells), sorted([naming.pid, busy.pid]))
self.assertEqual(shells[naming.pid].command, "until false; do sleep 60; done # chunk-pr merge")
self.assertEqual((shells[naming.pid].runs_busy_tool, shells[busy.pid].runs_busy_tool), (False, True))
self.assertLess(abs(shells[naming.pid].started - time.time()), 5)
finally:
for proc in (naming, busy, other):
proc.kill()
proc.wait()
naming = self.started("source /dev/null && eval 'until false; do sleep 60; done # chunk-pr merge' < /dev/null && pwd")
busy = self.started(f"{tool} merge; true")
self.assertEqual(self.shell_of(naming).command, "until false; do sleep 60; done # chunk-pr merge")
self.assertEqual((self.shell_of(naming).runs_busy_tool, self.shell_of(busy).runs_busy_tool), (False, True))
self.assertLess(abs(self.shell_of(naming).started - time.time()), 5)
def test_a_shell_below_which_a_process_holds_a_heavy_slot_file_runs_a_busy_tool(self):
# run-heavy replaces itself with its command, which keeps the slot's file open.
with tempfile.TemporaryDirectory() as state, unittest.mock.patch.dict(os.environ, {"XDG_STATE_HOME": state}):
slot = pathlib.Path(state, "run-heavy", "slot-0")
slot.parent.mkdir()
holding = self.started(f"exec 9>{slot}; sleep 60; true")
plain = self.started("sleep 60; true")
self.assertEqual((self.shell_of(holding).runs_busy_tool, self.shell_of(plain).runs_busy_tool), (True, False))
def test_a_shell_written_output_is_the_modified_time_of_its_output_file_and_else_its_start(self):
with tempfile.NamedTemporaryFile() as output:
os.utime(output.name, (1000, 1000))
to_file = self.started("sleep 60; true", stdout=output)
to_pipe = self.started("sleep 60; true", stdout=subprocess.PIPE)
self.assertEqual(self.shell_of(to_file).written, 1000)
self.assertEqual(self.shell_of(to_pipe).written, self.shell_of(to_pipe).started)
def test_stops_a_shell_and_the_process_it_started_and_no_other(self):
shell = subprocess.Popen(["bash", "-c", "sleep 60; true"])
shell = self.started("sleep 60; true")
other = subprocess.Popen(["sleep", "60"])
try:
time.sleep(0.5)
found = child_shells.shells_of(os.getpid())
[stopping] = [each for each in found if each.pid == shell.pid]
below = child_shells.descendants(shell.pid)
self.assertEqual(len(below), 1)
child_shells.stop(stopping)
self.assertEqual(shell.wait(timeout=5), -signal.SIGTERM)
for _ in range(50):
if not os.path.exists(f"/proc/{below[0]}"):
break
time.sleep(0.1)
self.assertFalse(os.path.exists(f"/proc/{below[0]}"))
self.assertIsNone(other.poll())
finally:
for proc in (shell, other):
proc.kill()
proc.wait()
self.addCleanup(lambda: (other.kill(), other.wait()))
[below] = child_shells.descendants(shell.pid)
child_shells.stop(self.shell_of(shell))
self.assertEqual(shell.wait(timeout=5), -signal.SIGTERM)
self.wait_until(lambda: not os.path.exists(f"/proc/{below}"))
self.assertIsNone(other.poll())