From 0398f9f190ad7cb6e38e68fa506ef0a4453b03a5 Mon Sep 17 00:00:00 2001 From: Fimeg Date: Mon, 17 Aug 2026 23:29:43 -0400 Subject: [PATCH] claude: don't fall back onto a login that cannot speak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rotation picked the spare on has_credential — a string check — and took one whose token had expired four days earlier; every call after 21:51 came back 401 revoked, reading as a dead subscription. A candidate now has to authenticate, and the window it left is recorded so the same round-robin can wrap back to it once the window returns. --- src/bridge/claude_subscription.rs | 171 +++++++++++++++++++++++++++--- 1 file changed, 157 insertions(+), 14 deletions(-) diff --git a/src/bridge/claude_subscription.rs b/src/bridge/claude_subscription.rs index 3762fbc..f5d7890 100644 --- a/src/bridge/claude_subscription.rs +++ b/src/bridge/claude_subscription.rs @@ -53,6 +53,10 @@ const TOKEN_REFRESH_BUFFER: Duration = Duration::from_secs(5 * 60); /// Longest `retry-after` still worth waiting out on the account already holding /// the cached prefix, instead of moving to another login. const BURST_RETRY_CEILING_SECS: u64 = 60; +/// How long a spent login waits when the response says nothing about its window. +/// One wasted call a quarter-hour is cheaper than parking a live login for the +/// five hours the window actually runs. +const DEFAULT_WINDOW_COOLDOWN: Duration = Duration::from_secs(15 * 60); const DEFAULT_MAX_TOKENS: u32 = 8192; /// Models that think by default cap thinking *plus* response text with /// `max_tokens`, so 8192 truncates mid-answer. @@ -132,6 +136,9 @@ struct TokenState { refresh_token: Option, expires_at_ms: Option, source_path: Option, + /// When this login's spent quota window comes back. Read from the response + /// that reported it spent; until then the login is not tried. + resumes_at_ms: Option, } impl TokenState { @@ -140,6 +147,16 @@ impl TokenState { live(&self.access_token) || live(&self.refresh_token) } + /// Worth trying: something to present, and no spent window still running. + fn is_available(&self, now: u64) -> bool { + self.has_credential() && self.resumes_at_ms.is_none_or(|at| now >= at) + } + + fn access_is_live(&self, now: u64) -> bool { + self.access_token.as_deref().is_some_and(|t| !t.is_empty()) + && self.expires_at_ms.is_some_and(|exp| exp > now) + } + fn label(&self) -> String { match &self.source_path { Some(p) => p.display().to_string(), @@ -327,15 +344,19 @@ impl ClaudeSubscriptionProvider { Ok((access, active)) } - /// Move off the account at `from`, which has nothing left to spend. + /// Move off the account at `from`, which answered `status` and is not worth + /// trying again until `resumes_at`. /// /// Sticky by design: the next login is used until it too is exhausted, /// rather than alternating. Switching accounts abandons a prompt cache the /// transcript has already paid for — measured 2026-08-12 at ~125k cached /// tokens a turn — so a swap has to be worth a full re-bill, and only an - /// exhausted window is. - async fn rotate_account(&self, from: usize) -> Option { + /// exhausted window is — which is also why nothing walks back to a better + /// login mid-conversation. The way home is this same round-robin: it wraps + /// to the primary, and takes it once its recorded window has passed. + async fn rotate_account(&self, from: usize, status: u16, resumes_at: u64) -> Option { let mut accounts = self.accounts.lock().await; + accounts.tokens[from].resumes_at_ms = Some(resumes_at); let count = accounts.tokens.len(); if count < 2 { return None; @@ -345,21 +366,41 @@ impl ClaudeSubscriptionProvider { if accounts.active != from { return Some(accounts.active); } + let now = now_ms(); for step in 1..count { let candidate = (from + step) % count; adopt_file_if_changed(&mut accounts.tokens[candidate]); - if accounts.tokens[candidate].has_credential() { - info!( - "claude login {} is out of quota; continuing as {}", - accounts.tokens[from].label(), - accounts.tokens[candidate].label(), - ); - accounts.active = candidate; - return Some(candidate); + if !accounts.tokens[candidate].is_available(now) { + continue; } + let label = accounts.tokens[candidate].label(); + if !self.can_authenticate(&mut accounts.tokens[candidate]).await { + warn!("claude login {label} cannot authenticate; skipping"); + continue; + } + info!( + "claude login {} answered {status}; continuing as {label}", + accounts.tokens[from].label(), + ); + accounts.active = candidate; + return Some(candidate); } None } + + /// Can this login still speak — an unexpired token, or a refresh the + /// endpoint accepts? + /// + /// `has_credential` only asks whether a string is present. On 2026-08-17 + /// that sent the provider onto a spare whose token had expired four days + /// earlier: every call after it 401'd as *revoked*, which reads as the + /// account being gone rather than the spare being dead. + async fn can_authenticate(&self, token: &mut TokenState) -> bool { + token.access_is_live(now_ms()) + || refresh_token(&self.http, &self.user_agent, token) + .await + .is_ok() + } } #[async_trait] @@ -500,13 +541,22 @@ impl LlmProvider for ClaudeSubscriptionProvider { } let retry_after = retry_after_secs(resp.headers()); + let resumes_at = window_resumes_at(resp.headers(), retry_after); let body_text = resp.text().await.unwrap_or_default(); // A spent quota window does not clear inside a retry loop — the only // thing that answers is another login. Retry it at once: the wait is // hours, not milliseconds. - if is_exhausted_window(status.as_u16(), retry_after) - && self.rotate_account(account).await.is_some() + // + // A login that cannot authenticate moves for the opposite reason: + // nothing is lost by leaving, because the cached prefix sits behind + // the same token that just failed. + if (is_exhausted_window(status.as_u16(), retry_after) + || is_auth_failure(status.as_u16())) + && self + .rotate_account(account, status.as_u16(), resumes_at) + .await + .is_some() { strain.push(InferenceStrain::Transient { attempt, @@ -588,7 +638,11 @@ fn adopt_file_if_changed(token: &mut TokenState) { credential_fingerprint(&fresh), credential_fingerprint(token), ); + // The quota belongs to the account, not to the token it was spent through: + // a refresh mid-window must not read as a fresh window. + let resumes = token.resumes_at_ms; *token = fresh; + token.resumes_at_ms = resumes; } /// Short, stable id for a credential — enough to see in a log that souveraine @@ -618,7 +672,8 @@ fn credential_fingerprint(token: &TokenState) -> String { /// the rotated refresh token we picked up alongside it. fn reread_credentials(current: &TokenState) -> Option { let path = current.source_path.as_ref()?; - let rotated = load_credentials(path).ok()?; + let mut rotated = load_credentials(path).ok()?; + rotated.resumes_at_ms = current.resumes_at_ms; let is_newer = rotated .access_token .as_deref() @@ -779,6 +834,7 @@ fn load_credentials(path: &Path) -> Result { .map(str::to_string), expires_at_ms: oauth.get("expiresAt").and_then(Value::as_u64), source_path: Some(path.to_path_buf()), + resumes_at_ms: None, }) } @@ -1385,6 +1441,27 @@ fn is_exhausted_window(status: u16, retry_after: Option) -> bool { status == 429 && retry_after.is_none_or(|secs| secs > BURST_RETRY_CEILING_SECS) } +/// This login cannot speak, whatever it is holding. +fn is_auth_failure(status: u16) -> bool { + matches!(status, 401 | 403) +} + +/// When the login that produced this response is worth trying again, in ms. +/// +/// These 429s carry no `retry-after`; the unified rate-limit headers carry the +/// reset as epoch seconds, and are the only precise answer on the wire. +fn window_resumes_at(headers: &reqwest::header::HeaderMap, retry_after: Option) -> u64 { + let now = now_ms(); + headers + .get("anthropic-ratelimit-unified-reset") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.trim().parse::().ok()) + .map(|secs| secs * 1000) + .filter(|at| *at > now) + .or_else(|| retry_after.map(|secs| now + secs * 1000)) + .unwrap_or(now + DEFAULT_WINDOW_COOLDOWN.as_millis() as u64) +} + fn reset_hint(retry_after: Option) -> String { match retry_after { Some(secs) if secs >= 60 => format!(" (resets in ~{}m)", secs / 60), @@ -1806,6 +1883,72 @@ mod tests { assert!(!is_exhausted_window(500, Some(3600))); } + /// 2026-08-17T21:51: the primary spent its window, rotation took the spare + /// on `has_credential` alone — a login whose token had expired on the 13th — + /// and every call after it came back 401 *revoked*, which reads as the + /// subscription being gone rather than the spare being dead. Presence of a + /// string is not the question. + #[test] + fn an_expired_spare_is_not_a_fallback() { + let dir = tempfile::tempdir().unwrap(); + let dead = creds_at(dir.path(), "expired-on-the-13th", now_ms() - 4 * 86_400_000); + assert!(dead.has_credential()); + assert!(!dead.access_is_live(now_ms())); + + let other = tempfile::tempdir().unwrap(); + let live = creds_at(other.path(), "good", now_ms() + 8 * 3_600_000); + assert!(live.access_is_live(now_ms())); + } + + /// A login parked on a spent window is skipped until the window returns — + /// and the window survives the refresh that rotates its token, because the + /// quota belongs to the account, not to the string it was spent through. + #[test] + fn a_parked_login_stays_parked_until_its_window_returns() { + let dir = tempfile::tempdir().unwrap(); + let mut spent = creds_at(dir.path(), "account-a", now_ms() + 8 * 3_600_000); + spent.resumes_at_ms = Some(now_ms() + 3_600_000); + assert!(!spent.is_available(now_ms())); + assert!(spent.is_available(now_ms() + 3_600_001)); + + creds_at(dir.path(), "account-a-refreshed", now_ms() + 8 * 3_600_000); + adopt_file_if_changed(&mut spent); + assert_eq!(spent.access_token.as_deref(), Some("account-a-refreshed")); + assert!(!spent.is_available(now_ms())); + } + + #[test] + fn the_window_reset_is_read_from_the_header_that_carries_it() { + let mut headers = reqwest::header::HeaderMap::new(); + let reset_secs = now_ms() / 1000 + 3600; + headers.insert( + "anthropic-ratelimit-unified-reset", + reqwest::header::HeaderValue::from_str(&reset_secs.to_string()).unwrap(), + ); + let at = window_resumes_at(&headers, None); + assert!(at.abs_diff(now_ms() + 3_600_000) < 2_000, "{at}"); + + // No header: `retry-after`, then a cooldown short enough that one wasted + // call is the whole cost of guessing. + let empty = reqwest::header::HeaderMap::new(); + let after = window_resumes_at(&empty, Some(120)); + assert!(after.abs_diff(now_ms() + 120_000) < 2_000, "{after}"); + let blind = window_resumes_at(&empty, None); + assert!(blind.abs_diff(now_ms() + 900_000) < 2_000, "{blind}"); + } + + /// 429 moves because the window is gone; 401 moves because the cached prefix + /// is unreachable behind the token that just failed. Everything else stays + /// put — a swap costs the prefix a re-bill. + #[test] + fn only_a_dead_login_and_a_spent_one_are_worth_leaving() { + assert!(is_auth_failure(401)); + assert!(is_auth_failure(403)); + for code in [400, 404, 429, 500, 529] { + assert!(!is_auth_failure(code), "{code} is not an auth failure"); + } + } + #[test] fn the_bail_says_when_the_quota_comes_back() { assert_eq!(reset_hint(Some(7200)), " (resets in ~120m)");