Answer stability round 1: 1 🔴, 4 🟡 (1 misread); keep a shell whose command holds a heavy slot, and say a failed record write on stderr
This commit is contained in:
@@ -12,26 +12,29 @@ from text_files.utc_stamp import utc
|
||||
|
||||
# Seconds without a line from the worker before a check, so a check is 5 minutes after its last line.
|
||||
CHECK_EVERY = 300
|
||||
# Seconds a final result must have stood, and a shell's output must have been quiet, before a shell is stopped.
|
||||
SETTLED = 180
|
||||
# Seconds a shell's output must have been quiet before the shell is stopped.
|
||||
QUIET = 180
|
||||
RECORD = "lingering-workers.md"
|
||||
|
||||
|
||||
def stopped_shells(proc: Streaming, settled_at: float | None, record: pathlib.Path, effects: Effects) -> None:
|
||||
"""Stops the shells of a claude whose final result came at settled_at and that is still alive, saying each stop on the terminal and in record. A claude that is still working (settled_at is None) or whose result came SETTLED seconds ago or less keeps its shells, and so does one with a shell that wrote output within SETTLED seconds or runs a busy tool, as `Shell.runs_busy_tool` says."""
|
||||
now = effects.now()
|
||||
if settled_at is None or now - settled_at <= SETTLED:
|
||||
def stopped_shells(proc: Streaming, printed_result: bool, record: pathlib.Path, effects: Effects) -> None:
|
||||
"""Stops the shells of a claude that is still alive CHECK_EVERY seconds after it printed its final result (printed_result), saying each stop on the terminal and in record. A claude whose latest line is not a result is working and keeps its shells, and so does one with a shell that wrote output within QUIET seconds or runs a busy tool, as `Shell.runs_busy_tool` says. A shell a background subagent started is not one of the claude's children, so it is never stopped."""
|
||||
if not printed_result:
|
||||
return
|
||||
now = effects.now()
|
||||
shells = proc.shells()
|
||||
if any(now - shell.written <= SETTLED or shell.runs_busy_tool for shell in shells):
|
||||
if any(now - shell.written <= QUIET or shell.runs_busy_tool for shell in shells):
|
||||
return
|
||||
for shell in shells:
|
||||
proc.stop(shell)
|
||||
line = stop_line(shell, now)
|
||||
say(line, effects, stdout=True)
|
||||
record.parent.mkdir(parents=True, exist_ok=True)
|
||||
with record.open("a") as handle:
|
||||
handle.write(f"- {utc(now)} {line}\n")
|
||||
try:
|
||||
record.parent.mkdir(parents=True, exist_ok=True)
|
||||
with record.open("a") as handle:
|
||||
handle.write(f"- {utc(now)} {line}\n")
|
||||
except OSError as failure:
|
||||
say(f"could not record the stop in {record}: {failure.strerror}", effects)
|
||||
|
||||
|
||||
def stop_line(shell: Shell, now: float) -> str:
|
||||
|
||||
@@ -25,16 +25,16 @@ def run_worker(root: str, args: list[str], env: dict[str, str], project_archive:
|
||||
except OSError as failure:
|
||||
return Failure(f"could not start claude: {failure.strerror}")
|
||||
result: dict | None = None
|
||||
# When the worker printed its latest result; None while its latest line is something else, since a worker that prints on is working.
|
||||
settled_at: float | None = None
|
||||
# Whether the worker's latest line is its result; a worker that prints on is working.
|
||||
printed_result = False
|
||||
resets: list[int] = []
|
||||
for raw in proc.lines(CHECK_EVERY):
|
||||
if raw is None:
|
||||
stopped_shells(proc, settled_at, project_archive / RECORD, effects)
|
||||
stopped_shells(proc, printed_result, project_archive / RECORD, effects)
|
||||
continue
|
||||
event = json_object(raw) or {}
|
||||
settled_at = effects.now() if ends_stream(event) else None
|
||||
if ends_stream(event):
|
||||
printed_result = ends_stream(event)
|
||||
if printed_result:
|
||||
result = event
|
||||
if (resets_at := rejected_until(event)) is not None:
|
||||
resets.append(resets_at)
|
||||
|
||||
@@ -79,10 +79,10 @@ then each verdict, quoted answer, rejected judge reply, step-up and stop, each o
|
||||
time. The layout, and when a copy is replaced, are in bin/transcript_archive/__init__.py.
|
||||
|
||||
A worker's claude can print its final result and then keep waiting on a background shell that
|
||||
never ends. Every 5 minutes without output from that claude, loop-claude stops each of its shells
|
||||
and what runs below them, once the result is over 3 minutes old and no shell has written output in
|
||||
the last 3 minutes or runs run-heavy or chunk-pr. It prints each stop and appends it, with the UTC
|
||||
time, to lingering-workers.md at the project's folder in that archive
|
||||
never ends. Every 5 minutes without output from that claude after its final result, loop-claude
|
||||
stops each of its shells and what runs below them, unless a shell has written output in the last 3
|
||||
minutes, runs chunk-pr or run-heavy, or holds a heavy slot. It prints each stop and appends it, with
|
||||
the UTC time, to lingering-workers.md at the project's folder in that archive
|
||||
(bin/loop_claude/cycle/lingering_workers.py).
|
||||
|
||||
Before each worker starts, loop-claude waits until at least a quarter of the machine's memory is
|
||||
|
||||
@@ -814,12 +814,22 @@ class LingeringWorkers(unittest.TestCase):
|
||||
streamed, _, _ = self.worker_with(self.RESULT, None, held=[self.shell(), busy])
|
||||
self.assertEqual(streamed.stopped, [])
|
||||
|
||||
def test_a_worker_keeps_its_shells_until_its_result_has_stood_longer_than_the_settled_time(self):
|
||||
fake = FakeEffects()
|
||||
effects = dataclasses.replace(fake.effects(), now=lambda: NOW + lingering_workers.SETTLED)
|
||||
proc = Streamed(0, held=[self.shell()])
|
||||
lingering_workers.stopped_shells(proc, NOW, pathlib.Path("/unused"), effects)
|
||||
self.assertEqual(proc.stopped, [])
|
||||
def test_a_worker_keeps_a_shell_that_wrote_within_the_quiet_time_and_stops_one_that_wrote_before_it(self):
|
||||
# The check comes 300 s after the result, so a shell written at NOW + 300 - QUIET is the latest that is still kept.
|
||||
kept = self.shell(written=NOW + 300 - lingering_workers.QUIET)
|
||||
stopped = self.shell(written=NOW + 300 - lingering_workers.QUIET - 1)
|
||||
self.assertEqual(self.worker_with(self.RESULT, None, held=[kept])[0].stopped, [])
|
||||
self.assertEqual(self.worker_with(self.RESULT, None, held=[stopped])[0].stopped, [stopped])
|
||||
|
||||
def test_a_stop_that_cannot_be_recorded_is_said_on_stderr_and_every_shell_is_still_stopped(self):
|
||||
clock = [NOW]
|
||||
held = [self.shell(), self.shell()]
|
||||
streamed = Streamed(0, self.RESULT, None, clock=clock, held=held)
|
||||
effects = dataclasses.replace(FakeEffects(streams=[streamed]).effects(), now=lambda: clock[0])
|
||||
with tempfile.NamedTemporaryFile() as blocker, contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()) as failed:
|
||||
worker.run_worker("/r", [], {}, pathlib.Path(blocker.name), effects)
|
||||
self.assertEqual(streamed.stopped, held)
|
||||
self.assertIn("could not record the stop", failed.getvalue())
|
||||
|
||||
|
||||
class MemoryWait(unittest.TestCase):
|
||||
|
||||
@@ -4,13 +4,16 @@ Claude Code starts each Bash call as `bash -c "source <snapshot> && … && eval
|
||||
"""
|
||||
import dataclasses
|
||||
import os
|
||||
import pathlib
|
||||
import signal
|
||||
import stat
|
||||
|
||||
from machine_load.slots import slot_folder
|
||||
|
||||
# The command names of a shell claude starts for a Bash call.
|
||||
SHELL_NAMES = frozenset({"bash", "dash", "sh", "zsh"})
|
||||
COMMAND_CHARS = 300
|
||||
# Tools whose process below a shell shows work that may go quiet for long: run-heavy holds a slot of the machine's heavy work or waits for one, and chunk-pr waits for a merge's approval and checks.
|
||||
# Scripts whose process below a shell shows work that may go quiet for long: run-heavy while it waits for a slot, and chunk-pr, which waits for a merge's approval and checks.
|
||||
BUSY_TOOLS = frozenset({"chunk-pr", "run-heavy"})
|
||||
|
||||
|
||||
@@ -23,7 +26,7 @@ class Shell:
|
||||
command: str
|
||||
# Epoch seconds of the last write to the file the shell's output goes to; the shell's start where the output goes to no file, which the check cannot read.
|
||||
written: float
|
||||
# Whether a process below the shell runs one of BUSY_TOOLS.
|
||||
# Whether a process below the shell runs one of BUSY_TOOLS or holds a heavy slot, a file of slot_folder that run-heavy leaves open in the command it runs.
|
||||
runs_busy_tool: bool
|
||||
|
||||
|
||||
@@ -75,19 +78,30 @@ def output_written(pid: int, started: float) -> float:
|
||||
|
||||
|
||||
def command_of(shell_argv: list[str]) -> str:
|
||||
"""What a shell runs: the text of its command line between Claude Code's `eval '` and the `' < /dev/null` that ends it, or the whole line where it holds none."""
|
||||
"""What a shell runs: the text of its command line after Claude Code's `eval '`, up to the first ` < /dev/null`, which a call that began in the foreground lacks; the whole line where it holds no `eval`."""
|
||||
line = " ".join(shell_argv)
|
||||
_, found, command = line.partition(" && eval ")
|
||||
return (command.split(" < /dev/null", 1)[0].removeprefix("'").removesuffix("'") if found else line)[:COMMAND_CHARS]
|
||||
|
||||
|
||||
def runs_busy_tool(pid: int) -> bool:
|
||||
"""Whether a process below pid has one of BUSY_TOOLS as its program or its script, as `python3 /x/chunk-pr`. A command line that merely names one, such as a `pgrep -f chunk-pr`, doesn't count."""
|
||||
return any(os.path.basename(name) in BUSY_TOOLS for each in descendants(pid) for name in argv_of(each)[:2])
|
||||
def holds_slot(pid: int, slots: pathlib.Path) -> bool:
|
||||
"""Whether pid has a file of the heavy slots' folder open. run-heavy replaces itself with its command once it holds a slot, and the command keeps the slot's file open."""
|
||||
try:
|
||||
return any(os.readlink(f"/proc/{pid}/fd/{fd}").startswith(f"{slots}/") for fd in os.listdir(f"/proc/{pid}/fd"))
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def runs_busy_tool(pid: int, slots: pathlib.Path) -> bool:
|
||||
"""Whether a process below pid holds a heavy slot or has one of BUSY_TOOLS as its program or its script, as `python3 /x/chunk-pr`. A command line that merely names a tool, such as a `pgrep -f chunk-pr`, doesn't count."""
|
||||
return any(holds_slot(each, slots) or any(os.path.basename(name) in BUSY_TOOLS for name in argv_of(each)[:2]) for each in descendants(pid))
|
||||
|
||||
|
||||
def shells_of(claude_pid: int) -> list[Shell]:
|
||||
"""The shells that are children of claude_pid."""
|
||||
"""The shells that are children of claude_pid; none where /proc doesn't exist, as off Linux."""
|
||||
if not os.path.isdir("/proc"):
|
||||
return []
|
||||
slots = slot_folder(os.environ, pathlib.Path.home())
|
||||
booted = boot_time()
|
||||
ticks = os.sysconf("SC_CLK_TCK")
|
||||
shells = []
|
||||
@@ -96,7 +110,7 @@ def shells_of(claude_pid: int) -> list[Shell]:
|
||||
if found is None or found[2] not in SHELL_NAMES:
|
||||
continue
|
||||
started = booted + found[1] / ticks
|
||||
shells.append(Shell(pid, started, command_of(argv_of(pid)), output_written(pid, started), runs_busy_tool(pid)))
|
||||
shells.append(Shell(pid, started, command_of(argv_of(pid)), output_written(pid, started), runs_busy_tool(pid, slots)))
|
||||
return shells
|
||||
|
||||
|
||||
|
||||
@@ -139,45 +139,60 @@ class PythonEffect(unittest.TestCase):
|
||||
|
||||
|
||||
class ChildShells(unittest.TestCase):
|
||||
def wait_until(self, condition):
|
||||
"""Polls up to 5 s for condition, so a test doesn't depend on how fast a process starts."""
|
||||
for _ in range(100):
|
||||
if condition():
|
||||
return
|
||||
time.sleep(0.05)
|
||||
self.fail("a process did not reach its state in time")
|
||||
|
||||
def started(self, command, **options):
|
||||
"""A shell running command, once a process runs below it."""
|
||||
proc = subprocess.Popen(["bash", "-c", command], **options)
|
||||
self.addCleanup(lambda: (proc.kill(), proc.wait()))
|
||||
self.wait_until(lambda: child_shells.descendants(proc.pid))
|
||||
return proc
|
||||
|
||||
def shell_of(self, proc):
|
||||
[found] = [each for each in child_shells.shells_of(os.getpid()) if each.pid == proc.pid]
|
||||
return found
|
||||
|
||||
def test_reads_a_child_shells_command_and_whether_a_busy_tool_runs_below_it(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
tool = pathlib.Path(folder, "chunk-pr")
|
||||
tool.write_text("#!/bin/sh\nsleep 60\n")
|
||||
tool.chmod(0o755)
|
||||
# Like the shell of a loop that only names the tool, as the one that held a worker on 2026-11-18 did.
|
||||
naming = subprocess.Popen(["bash", "-c", "source /dev/null && eval 'until false; do sleep 60; done # chunk-pr merge'"])
|
||||
busy = subprocess.Popen(["bash", "-c", f"{tool} merge; true"])
|
||||
other = subprocess.Popen(["sleep", "60"])
|
||||
try:
|
||||
time.sleep(0.5)
|
||||
shells = {shell.pid: shell for shell in child_shells.shells_of(os.getpid())}
|
||||
self.assertEqual(sorted(shells), sorted([naming.pid, busy.pid]))
|
||||
self.assertEqual(shells[naming.pid].command, "until false; do sleep 60; done # chunk-pr merge")
|
||||
self.assertEqual((shells[naming.pid].runs_busy_tool, shells[busy.pid].runs_busy_tool), (False, True))
|
||||
self.assertLess(abs(shells[naming.pid].started - time.time()), 5)
|
||||
finally:
|
||||
for proc in (naming, busy, other):
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
naming = self.started("source /dev/null && eval 'until false; do sleep 60; done # chunk-pr merge' < /dev/null && pwd")
|
||||
busy = self.started(f"{tool} merge; true")
|
||||
self.assertEqual(self.shell_of(naming).command, "until false; do sleep 60; done # chunk-pr merge")
|
||||
self.assertEqual((self.shell_of(naming).runs_busy_tool, self.shell_of(busy).runs_busy_tool), (False, True))
|
||||
self.assertLess(abs(self.shell_of(naming).started - time.time()), 5)
|
||||
|
||||
def test_a_shell_below_which_a_process_holds_a_heavy_slot_file_runs_a_busy_tool(self):
|
||||
# run-heavy replaces itself with its command, which keeps the slot's file open.
|
||||
with tempfile.TemporaryDirectory() as state, unittest.mock.patch.dict(os.environ, {"XDG_STATE_HOME": state}):
|
||||
slot = pathlib.Path(state, "run-heavy", "slot-0")
|
||||
slot.parent.mkdir()
|
||||
holding = self.started(f"exec 9>{slot}; sleep 60; true")
|
||||
plain = self.started("sleep 60; true")
|
||||
self.assertEqual((self.shell_of(holding).runs_busy_tool, self.shell_of(plain).runs_busy_tool), (True, False))
|
||||
|
||||
def test_a_shell_written_output_is_the_modified_time_of_its_output_file_and_else_its_start(self):
|
||||
with tempfile.NamedTemporaryFile() as output:
|
||||
os.utime(output.name, (1000, 1000))
|
||||
to_file = self.started("sleep 60; true", stdout=output)
|
||||
to_pipe = self.started("sleep 60; true", stdout=subprocess.PIPE)
|
||||
self.assertEqual(self.shell_of(to_file).written, 1000)
|
||||
self.assertEqual(self.shell_of(to_pipe).written, self.shell_of(to_pipe).started)
|
||||
|
||||
def test_stops_a_shell_and_the_process_it_started_and_no_other(self):
|
||||
shell = subprocess.Popen(["bash", "-c", "sleep 60; true"])
|
||||
shell = self.started("sleep 60; true")
|
||||
other = subprocess.Popen(["sleep", "60"])
|
||||
try:
|
||||
time.sleep(0.5)
|
||||
found = child_shells.shells_of(os.getpid())
|
||||
[stopping] = [each for each in found if each.pid == shell.pid]
|
||||
below = child_shells.descendants(shell.pid)
|
||||
self.assertEqual(len(below), 1)
|
||||
child_shells.stop(stopping)
|
||||
self.assertEqual(shell.wait(timeout=5), -signal.SIGTERM)
|
||||
for _ in range(50):
|
||||
if not os.path.exists(f"/proc/{below[0]}"):
|
||||
break
|
||||
time.sleep(0.1)
|
||||
self.assertFalse(os.path.exists(f"/proc/{below[0]}"))
|
||||
self.assertIsNone(other.poll())
|
||||
finally:
|
||||
for proc in (shell, other):
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
self.addCleanup(lambda: (other.kill(), other.wait()))
|
||||
[below] = child_shells.descendants(shell.pid)
|
||||
child_shells.stop(self.shell_of(shell))
|
||||
self.assertEqual(shell.wait(timeout=5), -signal.SIGTERM)
|
||||
self.wait_until(lambda: not os.path.exists(f"/proc/{below}"))
|
||||
self.assertIsNone(other.poll())
|
||||
|
||||
Reference in New Issue
Block a user