Merge pull request 'Resume spend-limit and usage-credit blocks, retried hourly' (#1) from spend-limit into main

Reviewed-on: #1

README.md

@@ -23,12 +23,18 @@ session, resumes it only when all of these hold:
 
 1. **The process is gone** — no `pid`, or the pid no longer exists.
 2. **`~/.claude/jobs/<id>/state.json` says `"state": "blocked"`.**
-3. **`detail` names a recoverable cause** — session/weekly/Opus limit, usage or
+3. **`detail` names a recoverable cause** — any quota Claude Code reports as
-   rate limit, API error, overload, or a network/connection failure.
+   `You've hit your <limit>`: session, weekly, Opus, Sonnet, Fable 5, fast,
+   usage, rate, and the ones that cost money rather than time (monthly or
+   individual spend limit, usage credit limit, out of usage credits or extra
+   usage). Plus API errors, overload, and network/connection failures.
 4. **`detail` does not name a human** — "input needed", "waiting for",
    permission, approval, or awaiting your direction/decision. A session parked
    on your decision must stay parked, and this veto wins even when the same
-   detail also names a limit.
+   detail also names a limit. It also covers entitlements only an admin can
+   change, which no amount of waiting clears: a seat type that excludes usage
+   credits, an allocation disabled by your admin, a group limit set to $0, a
+   service disabled for your org.
 5. **Any stated reset time has passed.** `detail` carries it verbatim, e.g.
    `You've hit your session limit · resets 7pm (Europe/Stockholm)`, so the
    watchdog waits for that moment instead of guessing. Day-less times are
@@ -39,6 +45,11 @@ session, resumes it only when all of these hold:
    push the parsed time-of-day a full day into the future and strand the
    session. Blocks with no stated reset (API outage, network) are retried on
    the next tick, with no wait.
+
+   A **spend** block states no reset because none exists: it clears when you
+   raise the cap, when the month rolls over, or when the subscription window
+   puts the session back on included quota. So it is neither retried every tick
+   nor parked for 5h — it is retried hourly until one of those happens.
 6. **Attempt caps are not exhausted** — see the circuit breakers below.
 
 It resumes with `claude --bg --resume <sessionId> --name "<original name>"`
@@ -75,6 +86,12 @@ each retry re-sends the session's whole context. During a multi-hour outage
 that is the one path that can burn a lot of tokens for nothing. Set
 `--max-per-run` to a small number instead if you would rather stagger.
 
+A *spend* block is the other open-ended one: if you never raise the cap it
+retries hourly until the month turns over, one job per session per hour. Those
+retries are rejected before the model sees them, so they cost no tokens — but
+they do spend the daily budget, so raise `--max-per-day` if you routinely have
+several sessions blocked at once.
+
 ## What it deliberately does not do
 
 - **Interactive sessions** — you are sitting there; you can press enter.
@@ -86,7 +103,7 @@ that is the one path that can burn a lot of tokens for nothing. Set
 ## Install
 
 Needs Claude Code (for `claude agents --json` and `claude --bg --resume`) and
-Python 3.9+ — standard library only.
+Python 3.9+ — standard library only, 3.14+ to run the tests.
 
 Clone anywhere — `~/.claude/session-watchdog` just keeps it travelling with the
 rest of `~/.claude`. Then add one cron entry with `crontab -e`:
@@ -123,6 +140,9 @@ root.
 ## Verify and observe
 
 ```bash
+# Does it still classify every block correctly?
+python3 ~/.claude/session-watchdog/test-resume-stalled-sessions.py
+
 # What would it do right now? Resumes nothing.
 ~/.claude/session-watchdog/resume-stalled-sessions.py --dry-run
 
@@ -159,6 +179,7 @@ hardcoded. To make one permanent, add it to the crontab line.
 | Path                                       | Role                                       |
 | ------------------------------------------ | ------------------------------------------ |
 | `resume-stalled-sessions.py`               | The watchdog — the only file it needs      |
+| `test-resume-stalled-sessions.py`          | Its tests, on real Claude Code block texts |
 | `~/.claude/session-watchdog/ledger.json`   | Attempt counts and resume timestamps       |
 | `~/.claude/session-watchdog/.lock`         | Held for the duration of a run             |
 

resume-stalled-sessions.py

@@ -27,11 +27,17 @@ DEFAULT_PROMPT = (
     "transient failure (usage limit, API outage, or network), which has cleared."
 )
 
-RECOVERABLE = re.compile(
+# Every quota Claude Code can report as "You've hit your <limit> · <how to clear
-    r"hit your (session|weekly|opus) limit"
+# it>", including the ones that cost money rather than time.
-    r"|usage limit"
+QUOTA_BLOCK = re.compile(
-    r"|rate.?limit"
+    r"(session|weekly|opus|sonnet|fable \d+|fast|spend|rate.?|usage( credit)?) ?limit"
-    r"|api error"
+    r"|out of (usage|extra usage)"
+    r"|usage credits",
+    re.IGNORECASE,
+)
+
+TRANSIENT_BLOCK = re.compile(
+    r"api (error|unavailable)"
     r"|overloaded"
     r"|connection (error|reset|closed|failure)"
     r"|network error"
@@ -41,18 +47,29 @@ RECOVERABLE = re.compile(
     re.IGNORECASE,
 )
 
-# A session parked on a human decision must never be auto-resumed. Vetoes even
+# A session parked on a human decision must never be auto-resumed, and an
-# when the same detail also names a limit.
+# entitlement someone else controls never clears by waiting. Vetoes even when
+# the same detail also names a limit.
 NEEDS_HUMAN = re.compile(
     r"input needed|waiting for|permission|approval"
     r"|awaiting (your )?(direction|decision|input|answer|go.?ahead)"
-    r"|needs? (your )?(direction|decision|input)",
+    r"|needs? (your )?(direction|decision|input)"
+    r"|seat type does ?n.t include"
+    r"|disabled by your admin"
+    r"|disabled for your org"
+    r"|limit is set to \$0",
     re.IGNORECASE,
 )
 
-LIMIT_BLOCK = re.compile(r"(session|weekly|opus|usage|rate.?) ?limit", re.IGNORECASE)
+# Money, not time: cleared by raising the cap, by the month rolling over, or by
+# the subscription window putting the session back on included quota. None of
+# those announce themselves, so poll rather than wait out a window.
+SPEND_BLOCK = re.compile(
+    r"spend limit|usage credit|out of (usage|extra usage)|usage credits", re.IGNORECASE
+)
 WEEKLY_LIMIT = re.compile(r"weekly limit", re.IGNORECASE)
 SESSION_WINDOW = timedelta(hours=5)
+SPEND_WINDOW = timedelta(hours=1)
 WEEKLY_WINDOW = timedelta(days=7)
 
 RESETS_AT = re.compile(
@@ -160,6 +177,13 @@ def parse_reset(detail, blocked_at):
     return target
 
 
+def resumable(detail):
+    """True when `detail` names a cause that clears without anyone acting."""
+    if NEEDS_HUMAN.search(detail):
+        return False
+    return bool(QUOTA_BLOCK.search(detail) or TRANSIENT_BLOCK.search(detail))
+
+
 def reset_moment(detail, blocked_at):
     """When the block should have lifted, or None if it can be retried at once.
 
@@ -167,9 +191,14 @@ def reset_moment(detail, blocked_at):
     rewritten after the reset already happened, which would otherwise push the
     parsed time-of-day a full day into the future.
     """
-    if not LIMIT_BLOCK.search(detail):
+    if not QUOTA_BLOCK.search(detail):
         return None
-    window = WEEKLY_WINDOW if WEEKLY_LIMIT.search(detail) else SESSION_WINDOW
+    if WEEKLY_LIMIT.search(detail):
+        window = WEEKLY_WINDOW
+    elif SPEND_BLOCK.search(detail):
+        window = SPEND_WINDOW
+    else:
+        window = SESSION_WINDOW
     stated = parse_reset(detail, blocked_at)
     bound = blocked_at + window
     return min(stated, bound) if stated else bound
@@ -209,7 +238,7 @@ def candidates(sessions, jobs_dir, now):
             continue
 
         detail = str(state.get("detail") or "")
-        if NEEDS_HUMAN.search(detail) or not RECOVERABLE.search(detail):
+        if not resumable(detail):
             continue
 
         blocked_at = datetime.fromtimestamp(state_file.stat().st_mtime).astimezone()

test-resume-stalled-sessions.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""Tests for the watchdog: python3 test-resume-stalled-sessions.py"""
+
+import json
+import os
+import tempfile
+import unittest
+import uuid
+from datetime import datetime, timedelta
+from importlib import util
+from pathlib import Path
+
+spec = util.spec_from_file_location("watchdog", Path(__file__).with_name("resume-stalled-sessions.py"))
+watchdog = util.module_from_spec(spec)
+spec.loader.exec_module(watchdog)
+
+# Verbatim from Claude Code's own limit banners, so a wording it really emits
+# cannot silently stop matching.
+CLEARS_ON_ITS_OWN = [
+    "You've hit your session limit · resets 7pm (Europe/Stockholm)",
+    "You've hit your weekly limit · resets Mon 12:00am",
+    "You've hit your Opus limit · resets 7pm",
+    "You've hit your Sonnet limit · resets 7pm",
+    "You've hit your Fable 5 limit · resets 7pm",
+    "You've hit your fast limit · resets in 12m",
+    "You've hit your monthly spend limit · raise it at claude.ai/settings/usage?from=cc_cli_limit_message",
+    "You've hit your individual spend limit · run /usage-credits to raise it, or visit claude.ai/admin-settings/usage",
+    "You've hit your org's monthly spend limit · ask your admin to raise it at claude.ai/settings/usage",
+    "You've hit your usage credit limit · raise it at claude.ai/settings/usage",
+    "You're out of usage credits",
+    "You're out of extra usage",
+    "Your org is out of usage · add funds to continue",
+    "Fable 5 requires usage credits",
+    "usage limit reached — check plan",
+    "rate limited — wait and retry",
+    "API overloaded — wait and retry",
+    "API unavailable — retry",
+    "API error — see detail",
+    "fetch failed",
+    "socket hang up",
+    "ECONNRESET",
+]
+
+# An entitlement someone else controls, or a decision left to the user: no wait
+# clears either, so resuming only burns the daily budget.
+STAYS_PARKED = [
+    "Your seat type doesn't include usage credits",
+    "Your seat type doesn't include extra usage",
+    "Your usage allocation has been disabled by your admin",
+    "Your group's usage limit is set to $0",
+    "This service is disabled for your org",
+    "login required — run /login",
+    "request too large — /compact or trim",
+    "invalid API request — see detail",
+    "input needed: which branch should I target?",
+    "waiting for the migration to be approved",
+    "needs your decision on the schema",
+    "You've hit your session limit · waiting for your go-ahead",
+]
+
+
+class Resumable(unittest.TestCase):
+    def test_recoverable_causes(self):
+        for detail in CLEARS_ON_ITS_OWN:
+            with self.subTest(detail=detail):
+                self.assertTrue(watchdog.resumable(detail))
+
+    def test_human_or_permanent_causes(self):
+        for detail in STAYS_PARKED:
+            with self.subTest(detail=detail):
+                self.assertFalse(watchdog.resumable(detail))
+
+
+class ResetMoment(unittest.TestCase):
+    def setUp(self):
+        self.blocked_at = datetime(2026, 8, 4, 9, 0).astimezone()
+
+    def test_spend_and_credit_blocks_retry_hourly(self):
+        for detail in [
+            "You've hit your monthly spend limit · raise it at claude.ai/settings/usage?from=cc_cli_limit_message",
+            "You've hit your usage credit limit · raise it at claude.ai/settings/usage",
+            "You're out of usage credits",
+            "Your org is out of usage · add funds to continue",
+        ]:
+            with self.subTest(detail=detail):
+                self.assertEqual(
+                    watchdog.reset_moment(detail, self.blocked_at),
+                    self.blocked_at + timedelta(hours=1),
+                )
+
+    def test_stated_reset_wins_when_it_falls_inside_the_window(self):
+        detail = "You've hit your session limit · resets 11am"
+        self.assertEqual(
+            watchdog.reset_moment(detail, self.blocked_at),
+            self.blocked_at.replace(hour=11),
+        )
+
+    def test_unstated_reset_falls_back_to_the_limit_window(self):
+        self.assertEqual(
+            watchdog.reset_moment("You've hit your session limit", self.blocked_at),
+            self.blocked_at + watchdog.SESSION_WINDOW,
+        )
+        self.assertEqual(
+            watchdog.reset_moment("You've hit your weekly limit", self.blocked_at),
+            self.blocked_at + watchdog.WEEKLY_WINDOW,
+        )
+
+    def test_transient_failures_retry_at_once(self):
+        self.assertIsNone(watchdog.reset_moment("API error — see detail", self.blocked_at))
+
+
+class Candidates(unittest.TestCase):
+    """The whole path: a spend-limited background job with no process left."""
+
+    SPEND = "You've hit your monthly spend limit · raise it at claude.ai/settings/usage?from=cc_cli_limit_message"
+
+    def setUp(self):
+        temp = tempfile.TemporaryDirectory()
+        self.addCleanup(temp.cleanup)
+        self.jobs = Path(temp.name)
+        self.now = datetime.now().astimezone()
+
+    def blocked_job(self, detail, blocked_ago):
+        job_id, session_id = uuid.uuid7().hex[:8], str(uuid.uuid7())
+        state = self.jobs / job_id / "state.json"
+        state.parent.mkdir()
+        state.write_text(json.dumps({"detail": detail, "state": "blocked"}))
+        stamp = (self.now - blocked_ago).timestamp()
+        os.utime(state, (stamp, stamp))
+        return {"cwd": str(self.jobs), "id": job_id, "kind": "background", "sessionId": session_id}
+
+    def test_spend_block_is_picked_up_once_its_hour_is_up(self):
+        session = self.blocked_job(self.SPEND, timedelta(hours=2))
+        found = watchdog.candidates([session], self.jobs, self.now)
+        self.assertEqual([job["id"] for job in found], [session["id"]])
+
+    def test_fresh_spend_block_waits(self):
+        session = self.blocked_job(self.SPEND, timedelta(minutes=10))
+        self.assertEqual(watchdog.candidates([session], self.jobs, self.now), [])
+
+
+if __name__ == "__main__":
+    unittest.main()