Hold off resuming while the host is short on memory or swapping

resume-stalled-sessions.py

@@ -22,6 +22,8 @@ from zoneinfo import ZoneInfo
 # Where `claude` installs itself, and what a scheduler's minimal PATH omits.
 CLAUDE_FALLBACK_DIRS = ("~/.local/bin", "/usr/local/bin", "/opt/homebrew/bin")
 
+MEMINFO = Path("/proc/meminfo")
+
 DEFAULT_PROMPT = (
     "Continue where you left off. Your previous run was interrupted by a "
     "transient failure (usage limit, API outage, or network), which has cleared."
@@ -86,6 +88,37 @@ def log(message):
     print(f"{datetime.now().astimezone().isoformat(timespec='seconds')} {message}", flush=True)
 
 
+def memory_strain(min_available_mb, max_swap_used_pct, meminfo=MEMINFO):
+    """Why the host cannot take another session, or None when it can.
+
+    Also None where the host does not report memory this way: a guard that
+    cannot read the numbers must not be the thing that stops recovery.
+    """
+    try:
+        lines = meminfo.read_text().splitlines()
+    except OSError:
+        return None
+    fields = {}
+    for line in lines:
+        key, _, rest = line.partition(":")
+        amount = rest.split()
+        if amount and amount[0].isdigit():
+            fields[key] = int(amount[0])
+
+    if "MemAvailable" not in fields:
+        return None
+    available_mb = fields["MemAvailable"] // 1024
+    if available_mb < min_available_mb:
+        return f"{available_mb} MB available, need {min_available_mb}"
+
+    swap_total = fields.get("SwapTotal", 0)
+    if swap_total:
+        used_pct = round(100 * (swap_total - fields.get("SwapFree", 0)) / swap_total)
+        if used_pct > max_swap_used_pct:
+            return f"swap {used_pct}% used, max {max_swap_used_pct}%"
+    return None
+
+
 def resolve_claude(name):
     """Absolute path to the Claude Code binary, or None if it cannot be found."""
     found = shutil.which(name)
@@ -318,6 +351,16 @@ def main():
         default=int(os.environ.get("WATCHDOG_MAX_PER_RUN", "0")),
         help="resumes per invocation, 0 for every eligible session (default: 0)",
     )
+    parser.add_argument(
+        "--max-swap-used-pct", type=int,
+        default=int(os.environ.get("WATCHDOG_MAX_SWAP_USED_PCT", "50")),
+        help="swap fill above which the host is already trading, so no resume (default: 50)",
+    )
+    parser.add_argument(
+        "--min-available-mb", type=int,
+        default=int(os.environ.get("WATCHDOG_MIN_AVAILABLE_MB", "2048")),
+        help="memory a resume needs the host to have spare, in MB (default: 2048)",
+    )
     parser.add_argument("--prompt", default=os.environ.get("WATCHDOG_PROMPT", DEFAULT_PROMPT))
     args = parser.parse_args()
 
@@ -347,6 +390,11 @@ def main():
         log(f"circuit breaker: {len(recent)} resumes in the last 24h (max {args.max_per_day})")
         return 0
 
+    strain = memory_strain(args.min_available_mb, args.max_swap_used_pct)
+    if strain:
+        log(f"holding off: {strain}")
+        return 0
+
     sessions = active_sessions(claude_bin)
     if sessions is None:
         return 1
@@ -358,6 +406,12 @@ def main():
         tried = ledger["attempts"].get(job["session_id"], 0)
         if tried >= args.max_attempts:
             continue
+        # A session launched seconds ago has not grown into its memory yet, so
+        # charge every resume of this run for the headroom it is about to take.
+        strain = memory_strain(args.min_available_mb * (resumed + 1), args.max_swap_used_pct)
+        if strain:
+            log(f"holding off: {strain}")
+            break
         if not resume(job, claude_bin, args.prompt, args.dry_run):
             continue
         ledger["attempts"][job["session_id"]] = tried + 1