commit 891b28d28c7179b97bb8028fed3f0d65bee28b1a Author: lilleman Date: Wed Jul 29 22:31:03 2026 +0200 Initial import of the session watchdog diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..e0f6088 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,17 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_style = tab +insert_final_newline = true +trim_trailing_whitespace = true + +# PEP 8. +[*.py] +indent_size = 4 +indent_style = space + +# A leading tab starts a code block in Markdown. +[*.md] +indent_style = space diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1ed4f47 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.log +.lock +ledger.json diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..02d5c6a --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Larv IT AB + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b6781b7 --- /dev/null +++ b/README.md @@ -0,0 +1,204 @@ +# Claude Code session watchdog + +Resumes Claude Code **background** sessions that died on a usage limit, an API +outage, or a network failure. + +## Why it lives outside Claude Code + +Claude Code has no auto-resume: on a usage limit it blocks further requests +until the reset time and the process exits. Nothing scheduled *inside* a +session (a 15-minute self-check, a `/loop`, a scheduled task) can rescue it, +because there is no process left to fire the schedule. Recovery has to come +from outside, so this is a systemd user timer. + +It also costs nothing while everything is healthy — it reads state files on +disk rather than waking sessions up. An in-session poll would re-send the +session's full context on every tick, which draws down the very limit it is +trying to survive. + +## How it decides + +Every 15 minutes it runs `claude agents --json` and, for each **background** +session, resumes it only when all of these hold: + +1. **The process is gone** — no `pid`, or the pid no longer exists. +2. **`~/.claude/jobs//state.json` says `"state": "blocked"`.** +3. **`detail` names a recoverable cause** — session/weekly/Opus limit, usage or + rate limit, API error, overload, or a network/connection failure. +4. **`detail` does not name a human** — "input needed", "waiting for", + permission, approval, or awaiting your direction/decision. A session parked + on your decision must stay parked, and this veto wins even when the same + detail also names a limit. +5. **Any stated reset time has passed.** `detail` carries it verbatim, e.g. + `You've hit your session limit · resets 7pm (Europe/Stockholm)`, so the + watchdog waits for that moment instead of guessing. Day-less times are + anchored to the `state.json` mtime; weekday forms like `resets Mon 12:00am` + are handled too. The result is clamped to the limit window — 5h for a + session or Opus limit, 7 days for a weekly one — because `state.json` is + sometimes rewritten *after* the reset already passed, which would otherwise + push the parsed time-of-day a full day into the future and strand the + session. Blocks with no stated reset (API outage, network) are retried on + the next tick, with no wait. +6. **Attempt caps are not exhausted** — see the circuit breakers below. + +It resumes with `claude --bg --resume --name ""` +from the session's own recorded working directory, so the conversation, its +context and its name all carry over. Oldest block is resumed first. + +The resume registers a **new** job id, leaving the original as a dead husk that +is permanently `blocked` yet still listed as active. The watchdog therefore +runs `claude stop ` after a successful resume. Without that the husk +would be picked up again on a later tick and resumed into a *second* session +working the same conversation — two agents racing on the same git worktree. + +### Circuit breakers + +Three limits stop a broken session, or a still-ongoing outage, from spawning an +endless chain of blocked jobs: + +| Limit | Default | Meaning | +| ----------------- | ------- | ------------------------------------------------ | +| `--max-per-run` | 0 | Resumes per tick; 0 means every eligible session | +| `--max-per-day` | 50 | Resumes across all sessions in a rolling 24h | +| `--max-attempts` | 1 | Resumes of any one session, ever | + +`--max-attempts 1` is not a one-shot recovery: if the resumed session stalls +again it is a *new* session id and qualifies on its own, so an outage spanning +several reset windows still gets retried. What the cap prevents is one +conversation being resumed into several concurrent copies. A resume that fails +to launch at all records no attempt and is retried on the next tick. + +`--max-per-day` is the one guard that cannot be derived from anything else. A +usage-limit block is self-pacing, because the stated reset time is respected — +but a *network or API* block states no reset, so it is retried every tick, and +each retry re-sends the session's whole context. During a multi-hour outage +that is the one path that can burn a lot of tokens for nothing. Set +`--max-per-run` to a small number instead if you would rather stagger. + +## What it deliberately does not do + +- **Interactive sessions** — you are sitting there; you can press enter. +- **Sessions waiting on a human** (condition 4 above). +- **Sessions you stopped** (`"state": "stopped"`) or that finished (`"done"`). +- **Anything whose working directory has disappeared** (deleted worktree) — + logged and skipped. + +## Install + +Needs Claude Code (for `claude agents --json` and `claude --bg --resume`) and +Python 3.9+ — standard library only. + +Clone into `~/.claude/session-watchdog`. The unit file expects it there, and the +watchdog then travels with the rest of `~/.claude`: + +```bash +git clone https://gitea.larvit.se/larvit/claude-session-watchdog.git ~/.claude/session-watchdog +chmod +x ~/.claude/session-watchdog/resume-stalled-sessions.py +mkdir -p ~/.config/systemd/user +ln -sf ~/.claude/session-watchdog/claude-session-watchdog.service ~/.config/systemd/user/ +ln -sf ~/.claude/session-watchdog/claude-session-watchdog.timer ~/.config/systemd/user/ +systemctl --user daemon-reload +systemctl --user enable --now claude-session-watchdog.timer +``` + +Cloned somewhere else? Point `ExecStart=` at it — that path is the only thing +tying the two together. `ledger.json` is written under `--claude-home` +regardless of where the script lives. + +To keep it running while you are logged out (background jobs outlive your +shell, so this is usually what you want): + +```bash +sudo loginctl enable-linger "$USER" +``` + +### cron (Alpine, macOS, anything without systemd) + +The script itself is portable — only the scheduling is not. Add a cron entry +with `crontab -e`: + +``` +7,22,37,52 * * * * PATH=/usr/local/bin:/usr/bin:/usr/sbin:/bin:/sbin /usr/bin/flock -n $HOME/.claude/session-watchdog/.lock $HOME/.claude/session-watchdog/resume-stalled-sessions.py >> $HOME/.claude/session-watchdog/watchdog.log 2>&1 +``` + +Both prefixes matter, and neither is cosmetic: + +- **`PATH`** — cron's default omits `/usr/local/bin`, which is where `claude` + usually is. Without it the watchdog fails *silently*: it logs `nothing to + resume` and looks perfectly healthy while resuming nothing, forever. Use + `CLAUDE_BIN=/full/path/to/claude` instead if you prefer. Verify with + `env -i HOME=$HOME PATH=/usr/bin:/bin ~/.claude/session-watchdog/resume-stalled-sessions.py --dry-run` + — that reproduces cron's environment, so it fails the same way cron would. +- **`flock`** — systemd's `Type=oneshot` refuses to run a second copy while the + first is going; cron happily overlaps them. Two concurrent runs read the same + `ledger.json` before either writes it, so both resume the same session. On a + host without `flock` (macOS), use a launchd agent with `StartInterval` set to + `900` instead — launchd also refuses to overlap a run with itself. + +Use absolute paths rather than `$HOME` if your cron does not export `HOME` +(busybox crond does). On Alpine, user crontabs live in `/etc/crontabs/` +and `crontab -e` writes there via `/var/spool/cron/crontabs`, so nothing needs +root. + +## Verify and observe + +```bash +# What would it do right now? Resumes nothing. +~/.claude/session-watchdog/resume-stalled-sessions.py --dry-run + +# Timer health and next fire time +systemctl --user list-timers claude-session-watchdog.timer + +# What it has done +journalctl --user -u claude-session-watchdog.service --since today +``` + +Under cron there is no journal, so the redirect in the crontab entry is the +record: + +```bash +crontab -l # is it scheduled? +tail -f ~/.claude/session-watchdog/watchdog.log # what it has done +``` + +One `nothing to resume` line per tick, so the log grows by roughly 8 KB a day. +Truncate it whenever you like — nothing reads it back. + +A stalled session that has been resumed shows up as a new job in +`claude agents`; `claude logs ` shows it picking the work back up. + +## Configuration + +Every option is a CLI flag with an environment-variable fallback — nothing is +hardcoded. To make one permanent, put it in the unit's `[Service]` block, or on +the crontab line next to `PATH`. + +| Flag | Env var | Default | +| ----------------- | ------------------------- | -------------------- | +| `--claude-bin` | `CLAUDE_BIN` | `claude` | +| `--claude-home` | `CLAUDE_HOME` | `~/.claude` | +| `--max-attempts` | `WATCHDOG_MAX_ATTEMPTS` | `1` | +| `--max-per-day` | `WATCHDOG_MAX_PER_DAY` | `50` | +| `--max-per-run` | `WATCHDOG_MAX_PER_RUN` | `0` (all) | +| `--prompt` | `WATCHDOG_PROMPT` | see `--help` | +| `--dry-run` | — | off | + +## Files + +| Path | Role | +| ----------------------------------- | ------------------------------------------------- | +| `resume-stalled-sessions.py` | The watchdog | +| `claude-session-watchdog.service` | systemd unit that runs it once | +| `claude-session-watchdog.timer` | Fires at `:07 :22 :37 :52` (off the `:00` crowd) | +| `~/.claude/session-watchdog/ledger.json` | Per-session attempt counts and resume timestamps | + +`ledger.json` is generated. Delete it to forget all history; edit the +`attempts` map to give a specific session another chance. + +## Uninstall + +```bash +systemctl --user disable --now claude-session-watchdog.timer +rm ~/.config/systemd/user/claude-session-watchdog.{service,timer} +systemctl --user daemon-reload +``` diff --git a/claude-session-watchdog.service b/claude-session-watchdog.service new file mode 100644 index 0000000..8ee3d11 --- /dev/null +++ b/claude-session-watchdog.service @@ -0,0 +1,7 @@ +[Unit] +Description=Resume Claude Code background sessions stalled on usage limits or outages + +[Service] +Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin +ExecStart=%h/.claude/session-watchdog/resume-stalled-sessions.py +Type=oneshot diff --git a/claude-session-watchdog.timer b/claude-session-watchdog.timer new file mode 100644 index 0000000..3b2f9ac --- /dev/null +++ b/claude-session-watchdog.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Check every 15 minutes for stalled Claude Code background sessions + +[Timer] +AccuracySec=1min +OnCalendar=*:07/15 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/resume-stalled-sessions.py b/resume-stalled-sessions.py new file mode 100755 index 0000000..ffd15e8 --- /dev/null +++ b/resume-stalled-sessions.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +"""Resume Claude Code background sessions that stalled on a recoverable error. + +Claude Code blocks and exits when it hits a usage limit, an API outage, or a +network failure. The process is gone, so nothing scheduled inside the session +can revive it. This runs outside every session and resumes the ones whose +cause has cleared. +""" + +import argparse +import json +import os +import re +import subprocess +import sys +from datetime import datetime, timedelta +from pathlib import Path +from zoneinfo import ZoneInfo + +DEFAULT_PROMPT = ( + "Continue where you left off. Your previous run was interrupted by a " + "transient failure (usage limit, API outage, or network), which has cleared." +) + +RECOVERABLE = re.compile( + r"hit your (session|weekly|opus) limit" + r"|usage limit" + r"|rate.?limit" + r"|api error" + r"|overloaded" + r"|connection (error|reset|closed|failure)" + r"|network error" + r"|fetch failed" + r"|socket hang up" + r"|etimedout|econnreset|econnrefused|enotfound", + re.IGNORECASE, +) + +# A session parked on a human decision must never be auto-resumed. Vetoes even +# when the same detail also names a limit. +NEEDS_HUMAN = re.compile( + r"input needed|waiting for|permission|approval" + r"|awaiting (your )?(direction|decision|input|answer|go.?ahead)" + r"|needs? (your )?(direction|decision|input)", + re.IGNORECASE, +) + +LIMIT_BLOCK = re.compile(r"(session|weekly|opus|usage|rate.?) ?limit", re.IGNORECASE) +WEEKLY_LIMIT = re.compile(r"weekly limit", re.IGNORECASE) +SESSION_WINDOW = timedelta(hours=5) +WEEKLY_WINDOW = timedelta(days=7) + +RESETS_AT = re.compile( + r"resets\s+(?:(?Pmon|tue|wed|thu|fri|sat|sun)[a-z]*\s+)?" + r"(?P\d{1,2})(?::(?P\d{2}))?\s*(?Pam|pm)?" + r"(?:\s*\((?P[A-Za-z_]+/[A-Za-z_]+)\))?", + re.IGNORECASE, +) + +WEEKDAYS = {"mon": 0, "tue": 1, "wed": 2, "thu": 3, "fri": 4, "sat": 5, "sun": 6} + + +def log(message): + print(f"{datetime.now().astimezone().isoformat(timespec='seconds')} {message}", flush=True) + + +def active_sessions(claude_bin): + """Sessions Claude Code still considers active (completed ones are excluded).""" + try: + done = subprocess.run( + [claude_bin, "agents", "--json"], + capture_output=True, text=True, timeout=120, check=True, + ) + except (OSError, subprocess.SubprocessError) as err: + log(f"error: could not list sessions: {err}") + return [] + try: + return json.loads(done.stdout) + except json.JSONDecodeError as err: + log(f"error: unparseable session list: {err}") + return [] + + +def pid_alive(pid): + if not isinstance(pid, int): + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def parse_reset(detail, blocked_at): + """The reset moment named in `detail`, or None. `blocked_at` anchors day-less times.""" + found = RESETS_AT.search(detail) + if not found: + return None + hour = int(found.group("hour")) + minute = int(found.group("minute") or 0) + meridiem = (found.group("meridiem") or "").lower() + if meridiem == "pm" and hour != 12: + hour += 12 + elif meridiem == "am" and hour == 12: + hour = 0 + if hour > 23 or minute > 59: + return None + + zone = None + if found.group("tz"): + try: + zone = ZoneInfo(found.group("tz")) + except Exception: + zone = None + + anchor = blocked_at.astimezone(zone) if zone else blocked_at + target = anchor.replace(hour=hour, minute=minute, second=0, microsecond=0) + day = (found.group("dow") or "").lower()[:3] + if day in WEEKDAYS: + target += timedelta(days=(WEEKDAYS[day] - target.weekday()) % 7) + if target < anchor: + target += timedelta(days=7) + elif target < anchor: + target += timedelta(days=1) + return target + + +def reset_moment(detail, blocked_at): + """When the block should have lifted, or None if it can be retried at once. + + A stated time is clamped to the limit window: state.json is sometimes + rewritten after the reset already happened, which would otherwise push the + parsed time-of-day a full day into the future. + """ + if not LIMIT_BLOCK.search(detail): + return None + window = WEEKLY_WINDOW if WEEKLY_LIMIT.search(detail) else SESSION_WINDOW + stated = parse_reset(detail, blocked_at) + bound = blocked_at + window + return min(stated, bound) if stated else bound + + +def load_ledger(path): + try: + ledger = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + ledger = {} + ledger.setdefault("attempts", {}) + ledger.setdefault("resumes", []) + return ledger + + +def save_ledger(path, ledger): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(ledger, indent=2, sort_keys=True)) + + +def candidates(sessions, jobs_dir, now): + """Stalled background sessions, oldest block first.""" + found = [] + for session in sessions: + job_id, session_id = session.get("id"), session.get("sessionId") + if session.get("kind") != "background" or not job_id or not session_id: + continue + if pid_alive(session.get("pid")): + continue + + state_file = jobs_dir / job_id / "state.json" + try: + state = json.loads(state_file.read_text()) + except (OSError, json.JSONDecodeError): + continue + if state.get("state") != "blocked": + continue + + detail = str(state.get("detail") or "") + if NEEDS_HUMAN.search(detail) or not RECOVERABLE.search(detail): + continue + + blocked_at = datetime.fromtimestamp(state_file.stat().st_mtime).astimezone() + reset_at = reset_moment(detail, blocked_at) + if reset_at and reset_at > now: + log(f"{job_id}: waiting for reset at {reset_at.isoformat(timespec='minutes')}") + continue + + found.append({ + "cwd": session.get("cwd"), + "detail": detail, + "id": job_id, + "blocked_at": blocked_at, + "name": session.get("name"), + "session_id": session_id, + }) + return sorted(found, key=lambda job: job["blocked_at"]) + + +def resume(job, claude_bin, prompt, dry_run): + cwd = job["cwd"] + if not cwd or not Path(cwd).is_dir(): + log(f"{job['id']}: skipped, working directory is gone ({cwd})") + return False + if dry_run: + log(f"{job['id']}: would resume '{job['name']}' in {cwd} — {job['detail']}") + return False + command = [claude_bin, "--bg", "--resume", job["session_id"]] + if job["name"]: + command += ["--name", job["name"]] + try: + done = subprocess.run( + command + [prompt], + capture_output=True, cwd=cwd, text=True, timeout=180, check=True, + ) + except (OSError, subprocess.SubprocessError) as err: + log(f"{job['id']}: resume failed: {err}") + return False + log(f"{job['id']}: resumed '{job['name']}' — {job['detail']}") + + # The husk stays 'blocked' and active forever otherwise, and would be + # resumed again into a duplicate session working the same conversation. + try: + subprocess.run( + [claude_bin, "stop", job["id"]], + capture_output=True, text=True, timeout=60, check=True, + ) + except (OSError, subprocess.SubprocessError) as err: + log(f"{job['id']}: could not retire stale entry: {err}") + return True + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--claude-bin", default=os.environ.get("CLAUDE_BIN", "claude")) + parser.add_argument( + "--claude-home", + type=Path, + default=Path(os.environ.get("CLAUDE_HOME", Path.home() / ".claude")), + ) + parser.add_argument("--dry-run", action="store_true", help="report candidates, resume nothing") + parser.add_argument( + "--max-attempts", type=int, + default=int(os.environ.get("WATCHDOG_MAX_ATTEMPTS", "1")), + help="resumes of any one session, ever (default: 1; its successor is " + "eligible on its own if it stalls again)", + ) + parser.add_argument( + "--max-per-day", type=int, + default=int(os.environ.get("WATCHDOG_MAX_PER_DAY", "50")), + help="runaway backstop across all sessions per 24h (default: 50)", + ) + parser.add_argument( + "--max-per-run", type=int, + default=int(os.environ.get("WATCHDOG_MAX_PER_RUN", "0")), + help="resumes per invocation, 0 for every eligible session (default: 0)", + ) + parser.add_argument("--prompt", default=os.environ.get("WATCHDOG_PROMPT", DEFAULT_PROMPT)) + args = parser.parse_args() + + now = datetime.now().astimezone() + ledger_path = args.claude_home / "session-watchdog" / "ledger.json" + ledger = load_ledger(ledger_path) + + recent = [at for at in ledger["resumes"] if now.timestamp() - at < 86400] + budget = args.max_per_day - len(recent) + if args.max_per_run > 0: + budget = min(budget, args.max_per_run) + if budget <= 0: + log(f"circuit breaker: {len(recent)} resumes in the last 24h (max {args.max_per_day})") + return 0 + + sessions = active_sessions(args.claude_bin) + resumed = 0 + for job in candidates(sessions, args.claude_home / "jobs", now): + if resumed >= budget: + break + tried = ledger["attempts"].get(job["session_id"], 0) + if tried >= args.max_attempts: + continue + if not resume(job, args.claude_bin, args.prompt, args.dry_run): + continue + ledger["attempts"][job["session_id"]] = tried + 1 + ledger["resumes"] = recent + [now.timestamp()] + recent = ledger["resumes"] + save_ledger(ledger_path, ledger) + resumed += 1 + + if not resumed: + log("nothing to resume") + return 0 + + +if __name__ == "__main__": + sys.exit(main())