From ad9d51d39988af3d714f80da1541d507d5ac0032 Mon Sep 17 00:00:00 2001 From: lilleman Date: Tue, 4 Aug 2026 09:40:53 +0200 Subject: [PATCH] Resume spend-limit and usage-credit blocks, retried hourly --- README.md | 29 ++++++- resume-stalled-sessions.py | 53 +++++++++--- test-resume-stalled-sessions.py | 143 ++++++++++++++++++++++++++++++++ 3 files changed, 209 insertions(+), 16 deletions(-) create mode 100644 test-resume-stalled-sessions.py diff --git a/README.md b/README.md index d6424e2..12563fc 100644 --- a/README.md +++ b/README.md @@ -23,12 +23,18 @@ session, resumes it only when all of these hold: 1. **The process is gone** — no `pid`, or the pid no longer exists. 2. **`~/.claude/jobs//state.json` says `"state": "blocked"`.** -3. **`detail` names a recoverable cause** — session/weekly/Opus limit, usage or - rate limit, API error, overload, or a network/connection failure. +3. **`detail` names a recoverable cause** — any quota Claude Code reports as + `You've hit your `: session, weekly, Opus, Sonnet, Fable 5, fast, + usage, rate, and the ones that cost money rather than time (monthly or + individual spend limit, usage credit limit, out of usage credits or extra + usage). Plus API errors, overload, and network/connection failures. 4. **`detail` does not name a human** — "input needed", "waiting for", permission, approval, or awaiting your direction/decision. A session parked on your decision must stay parked, and this veto wins even when the same - detail also names a limit. + detail also names a limit. It also covers entitlements only an admin can + change, which no amount of waiting clears: a seat type that excludes usage + credits, an allocation disabled by your admin, a group limit set to $0, a + service disabled for your org. 5. **Any stated reset time has passed.** `detail` carries it verbatim, e.g. `You've hit your session limit · resets 7pm (Europe/Stockholm)`, so the watchdog waits for that moment instead of guessing. Day-less times are @@ -39,6 +45,11 @@ session, resumes it only when all of these hold: push the parsed time-of-day a full day into the future and strand the session. Blocks with no stated reset (API outage, network) are retried on the next tick, with no wait. + + A **spend** block states no reset because none exists: it clears when you + raise the cap, when the month rolls over, or when the subscription window + puts the session back on included quota. So it is neither retried every tick + nor parked for 5h — it is retried hourly until one of those happens. 6. **Attempt caps are not exhausted** — see the circuit breakers below. It resumes with `claude --bg --resume --name ""` @@ -75,6 +86,12 @@ each retry re-sends the session's whole context. During a multi-hour outage that is the one path that can burn a lot of tokens for nothing. Set `--max-per-run` to a small number instead if you would rather stagger. +A *spend* block is the other open-ended one: if you never raise the cap it +retries hourly until the month turns over, one job per session per hour. Those +retries are rejected before the model sees them, so they cost no tokens — but +they do spend the daily budget, so raise `--max-per-day` if you routinely have +several sessions blocked at once. + ## What it deliberately does not do - **Interactive sessions** — you are sitting there; you can press enter. @@ -86,7 +103,7 @@ that is the one path that can burn a lot of tokens for nothing. Set ## Install Needs Claude Code (for `claude agents --json` and `claude --bg --resume`) and -Python 3.9+ — standard library only. +Python 3.9+ — standard library only, 3.14+ to run the tests. Clone anywhere — `~/.claude/session-watchdog` just keeps it travelling with the rest of `~/.claude`. Then add one cron entry with `crontab -e`: @@ -123,6 +140,9 @@ root. ## Verify and observe ```bash +# Does it still classify every block correctly? +python3 ~/.claude/session-watchdog/test-resume-stalled-sessions.py + # What would it do right now? Resumes nothing. ~/.claude/session-watchdog/resume-stalled-sessions.py --dry-run @@ -159,6 +179,7 @@ hardcoded. To make one permanent, add it to the crontab line. | Path | Role | | ------------------------------------------ | ------------------------------------------ | | `resume-stalled-sessions.py` | The watchdog — the only file it needs | +| `test-resume-stalled-sessions.py` | Its tests, on real Claude Code block texts | | `~/.claude/session-watchdog/ledger.json` | Attempt counts and resume timestamps | | `~/.claude/session-watchdog/.lock` | Held for the duration of a run | diff --git a/resume-stalled-sessions.py b/resume-stalled-sessions.py index f1cc350..6ef3f49 100755 --- a/resume-stalled-sessions.py +++ b/resume-stalled-sessions.py @@ -27,11 +27,17 @@ DEFAULT_PROMPT = ( "transient failure (usage limit, API outage, or network), which has cleared." ) -RECOVERABLE = re.compile( - r"hit your (session|weekly|opus) limit" - r"|usage limit" - r"|rate.?limit" - r"|api error" +# Every quota Claude Code can report as "You've hit your · ", including the ones that cost money rather than time. +QUOTA_BLOCK = re.compile( + r"(session|weekly|opus|sonnet|fable \d+|fast|spend|rate.?|usage( credit)?) ?limit" + r"|out of (usage|extra usage)" + r"|usage credits", + re.IGNORECASE, +) + +TRANSIENT_BLOCK = re.compile( + r"api (error|unavailable)" r"|overloaded" r"|connection (error|reset|closed|failure)" r"|network error" @@ -41,18 +47,29 @@ RECOVERABLE = re.compile( re.IGNORECASE, ) -# A session parked on a human decision must never be auto-resumed. Vetoes even -# when the same detail also names a limit. +# A session parked on a human decision must never be auto-resumed, and an +# entitlement someone else controls never clears by waiting. Vetoes even when +# the same detail also names a limit. NEEDS_HUMAN = re.compile( r"input needed|waiting for|permission|approval" r"|awaiting (your )?(direction|decision|input|answer|go.?ahead)" - r"|needs? (your )?(direction|decision|input)", + r"|needs? (your )?(direction|decision|input)" + r"|seat type does ?n.t include" + r"|disabled by your admin" + r"|disabled for your org" + r"|limit is set to \$0", re.IGNORECASE, ) -LIMIT_BLOCK = re.compile(r"(session|weekly|opus|usage|rate.?) ?limit", re.IGNORECASE) +# Money, not time: cleared by raising the cap, by the month rolling over, or by +# the subscription window putting the session back on included quota. None of +# those announce themselves, so poll rather than wait out a window. +SPEND_BLOCK = re.compile( + r"spend limit|usage credit|out of (usage|extra usage)|usage credits", re.IGNORECASE +) WEEKLY_LIMIT = re.compile(r"weekly limit", re.IGNORECASE) SESSION_WINDOW = timedelta(hours=5) +SPEND_WINDOW = timedelta(hours=1) WEEKLY_WINDOW = timedelta(days=7) RESETS_AT = re.compile( @@ -160,6 +177,13 @@ def parse_reset(detail, blocked_at): return target +def resumable(detail): + """True when `detail` names a cause that clears without anyone acting.""" + if NEEDS_HUMAN.search(detail): + return False + return bool(QUOTA_BLOCK.search(detail) or TRANSIENT_BLOCK.search(detail)) + + def reset_moment(detail, blocked_at): """When the block should have lifted, or None if it can be retried at once. @@ -167,9 +191,14 @@ def reset_moment(detail, blocked_at): rewritten after the reset already happened, which would otherwise push the parsed time-of-day a full day into the future. """ - if not LIMIT_BLOCK.search(detail): + if not QUOTA_BLOCK.search(detail): return None - window = WEEKLY_WINDOW if WEEKLY_LIMIT.search(detail) else SESSION_WINDOW + if WEEKLY_LIMIT.search(detail): + window = WEEKLY_WINDOW + elif SPEND_BLOCK.search(detail): + window = SPEND_WINDOW + else: + window = SESSION_WINDOW stated = parse_reset(detail, blocked_at) bound = blocked_at + window return min(stated, bound) if stated else bound @@ -209,7 +238,7 @@ def candidates(sessions, jobs_dir, now): continue detail = str(state.get("detail") or "") - if NEEDS_HUMAN.search(detail) or not RECOVERABLE.search(detail): + if not resumable(detail): continue blocked_at = datetime.fromtimestamp(state_file.stat().st_mtime).astimezone() diff --git a/test-resume-stalled-sessions.py b/test-resume-stalled-sessions.py new file mode 100644 index 0000000..ab7028d --- /dev/null +++ b/test-resume-stalled-sessions.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Tests for the watchdog: python3 test-resume-stalled-sessions.py""" + +import json +import os +import tempfile +import unittest +import uuid +from datetime import datetime, timedelta +from importlib import util +from pathlib import Path + +spec = util.spec_from_file_location("watchdog", Path(__file__).with_name("resume-stalled-sessions.py")) +watchdog = util.module_from_spec(spec) +spec.loader.exec_module(watchdog) + +# Verbatim from Claude Code's own limit banners, so a wording it really emits +# cannot silently stop matching. +CLEARS_ON_ITS_OWN = [ + "You've hit your session limit · resets 7pm (Europe/Stockholm)", + "You've hit your weekly limit · resets Mon 12:00am", + "You've hit your Opus limit · resets 7pm", + "You've hit your Sonnet limit · resets 7pm", + "You've hit your Fable 5 limit · resets 7pm", + "You've hit your fast limit · resets in 12m", + "You've hit your monthly spend limit · raise it at claude.ai/settings/usage?from=cc_cli_limit_message", + "You've hit your individual spend limit · run /usage-credits to raise it, or visit claude.ai/admin-settings/usage", + "You've hit your org's monthly spend limit · ask your admin to raise it at claude.ai/settings/usage", + "You've hit your usage credit limit · raise it at claude.ai/settings/usage", + "You're out of usage credits", + "You're out of extra usage", + "Your org is out of usage · add funds to continue", + "Fable 5 requires usage credits", + "usage limit reached — check plan", + "rate limited — wait and retry", + "API overloaded — wait and retry", + "API unavailable — retry", + "API error — see detail", + "fetch failed", + "socket hang up", + "ECONNRESET", +] + +# An entitlement someone else controls, or a decision left to the user: no wait +# clears either, so resuming only burns the daily budget. +STAYS_PARKED = [ + "Your seat type doesn't include usage credits", + "Your seat type doesn't include extra usage", + "Your usage allocation has been disabled by your admin", + "Your group's usage limit is set to $0", + "This service is disabled for your org", + "login required — run /login", + "request too large — /compact or trim", + "invalid API request — see detail", + "input needed: which branch should I target?", + "waiting for the migration to be approved", + "needs your decision on the schema", + "You've hit your session limit · waiting for your go-ahead", +] + + +class Resumable(unittest.TestCase): + def test_recoverable_causes(self): + for detail in CLEARS_ON_ITS_OWN: + with self.subTest(detail=detail): + self.assertTrue(watchdog.resumable(detail)) + + def test_human_or_permanent_causes(self): + for detail in STAYS_PARKED: + with self.subTest(detail=detail): + self.assertFalse(watchdog.resumable(detail)) + + +class ResetMoment(unittest.TestCase): + def setUp(self): + self.blocked_at = datetime(2026, 8, 4, 9, 0).astimezone() + + def test_spend_and_credit_blocks_retry_hourly(self): + for detail in [ + "You've hit your monthly spend limit · raise it at claude.ai/settings/usage?from=cc_cli_limit_message", + "You've hit your usage credit limit · raise it at claude.ai/settings/usage", + "You're out of usage credits", + "Your org is out of usage · add funds to continue", + ]: + with self.subTest(detail=detail): + self.assertEqual( + watchdog.reset_moment(detail, self.blocked_at), + self.blocked_at + timedelta(hours=1), + ) + + def test_stated_reset_wins_when_it_falls_inside_the_window(self): + detail = "You've hit your session limit · resets 11am" + self.assertEqual( + watchdog.reset_moment(detail, self.blocked_at), + self.blocked_at.replace(hour=11), + ) + + def test_unstated_reset_falls_back_to_the_limit_window(self): + self.assertEqual( + watchdog.reset_moment("You've hit your session limit", self.blocked_at), + self.blocked_at + watchdog.SESSION_WINDOW, + ) + self.assertEqual( + watchdog.reset_moment("You've hit your weekly limit", self.blocked_at), + self.blocked_at + watchdog.WEEKLY_WINDOW, + ) + + def test_transient_failures_retry_at_once(self): + self.assertIsNone(watchdog.reset_moment("API error — see detail", self.blocked_at)) + + +class Candidates(unittest.TestCase): + """The whole path: a spend-limited background job with no process left.""" + + SPEND = "You've hit your monthly spend limit · raise it at claude.ai/settings/usage?from=cc_cli_limit_message" + + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.jobs = Path(temp.name) + self.now = datetime.now().astimezone() + + def blocked_job(self, detail, blocked_ago): + job_id, session_id = uuid.uuid7().hex[:8], str(uuid.uuid7()) + state = self.jobs / job_id / "state.json" + state.parent.mkdir() + state.write_text(json.dumps({"detail": detail, "state": "blocked"})) + stamp = (self.now - blocked_ago).timestamp() + os.utime(state, (stamp, stamp)) + return {"cwd": str(self.jobs), "id": job_id, "kind": "background", "sessionId": session_id} + + def test_spend_block_is_picked_up_once_its_hour_is_up(self): + session = self.blocked_job(self.SPEND, timedelta(hours=2)) + found = watchdog.candidates([session], self.jobs, self.now) + self.assertEqual([job["id"] for job in found], [session["id"]]) + + def test_fresh_spend_block_waits(self): + session = self.blocked_job(self.SPEND, timedelta(minutes=10)) + self.assertEqual(watchdog.candidates([session], self.jobs, self.now), []) + + +if __name__ == "__main__": + unittest.main()