claude: don't fall back onto a login that cannot speak
Rotation picked the spare on has_credential — a string check — and took one whose token had expired four days earlier; every call after 21:51 came back 401 revoked, reading as a dead subscription. A candidate now has to authenticate, and the window it left is recorded so the same round-robin can wrap back to it once the window returns.
This commit is contained in:
parent
04ceac79bf
commit
0398f9f190
1 changed files with 157 additions and 14 deletions
|
|
@ -53,6 +53,10 @@ const TOKEN_REFRESH_BUFFER: Duration = Duration::from_secs(5 * 60);
|
|||
/// Longest `retry-after` still worth waiting out on the account already holding
|
||||
/// the cached prefix, instead of moving to another login.
|
||||
const BURST_RETRY_CEILING_SECS: u64 = 60;
|
||||
/// How long a spent login waits when the response says nothing about its window.
|
||||
/// One wasted call a quarter-hour is cheaper than parking a live login for the
|
||||
/// five hours the window actually runs.
|
||||
const DEFAULT_WINDOW_COOLDOWN: Duration = Duration::from_secs(15 * 60);
|
||||
const DEFAULT_MAX_TOKENS: u32 = 8192;
|
||||
/// Models that think by default cap thinking *plus* response text with
|
||||
/// `max_tokens`, so 8192 truncates mid-answer.
|
||||
|
|
@ -132,6 +136,9 @@ struct TokenState {
|
|||
refresh_token: Option<String>,
|
||||
expires_at_ms: Option<u64>,
|
||||
source_path: Option<PathBuf>,
|
||||
/// When this login's spent quota window comes back. Read from the response
|
||||
/// that reported it spent; until then the login is not tried.
|
||||
resumes_at_ms: Option<u64>,
|
||||
}
|
||||
|
||||
impl TokenState {
|
||||
|
|
@ -140,6 +147,16 @@ impl TokenState {
|
|||
live(&self.access_token) || live(&self.refresh_token)
|
||||
}
|
||||
|
||||
/// Worth trying: something to present, and no spent window still running.
|
||||
fn is_available(&self, now: u64) -> bool {
|
||||
self.has_credential() && self.resumes_at_ms.is_none_or(|at| now >= at)
|
||||
}
|
||||
|
||||
fn access_is_live(&self, now: u64) -> bool {
|
||||
self.access_token.as_deref().is_some_and(|t| !t.is_empty())
|
||||
&& self.expires_at_ms.is_some_and(|exp| exp > now)
|
||||
}
|
||||
|
||||
fn label(&self) -> String {
|
||||
match &self.source_path {
|
||||
Some(p) => p.display().to_string(),
|
||||
|
|
@ -327,15 +344,19 @@ impl ClaudeSubscriptionProvider {
|
|||
Ok((access, active))
|
||||
}
|
||||
|
||||
/// Move off the account at `from`, which has nothing left to spend.
|
||||
/// Move off the account at `from`, which answered `status` and is not worth
|
||||
/// trying again until `resumes_at`.
|
||||
///
|
||||
/// Sticky by design: the next login is used until it too is exhausted,
|
||||
/// rather than alternating. Switching accounts abandons a prompt cache the
|
||||
/// transcript has already paid for — measured 2026-08-12 at ~125k cached
|
||||
/// tokens a turn — so a swap has to be worth a full re-bill, and only an
|
||||
/// exhausted window is.
|
||||
async fn rotate_account(&self, from: usize) -> Option<usize> {
|
||||
/// exhausted window is — which is also why nothing walks back to a better
|
||||
/// login mid-conversation. The way home is this same round-robin: it wraps
|
||||
/// to the primary, and takes it once its recorded window has passed.
|
||||
async fn rotate_account(&self, from: usize, status: u16, resumes_at: u64) -> Option<usize> {
|
||||
let mut accounts = self.accounts.lock().await;
|
||||
accounts.tokens[from].resumes_at_ms = Some(resumes_at);
|
||||
let count = accounts.tokens.len();
|
||||
if count < 2 {
|
||||
return None;
|
||||
|
|
@ -345,21 +366,41 @@ impl ClaudeSubscriptionProvider {
|
|||
if accounts.active != from {
|
||||
return Some(accounts.active);
|
||||
}
|
||||
let now = now_ms();
|
||||
for step in 1..count {
|
||||
let candidate = (from + step) % count;
|
||||
adopt_file_if_changed(&mut accounts.tokens[candidate]);
|
||||
if accounts.tokens[candidate].has_credential() {
|
||||
info!(
|
||||
"claude login {} is out of quota; continuing as {}",
|
||||
accounts.tokens[from].label(),
|
||||
accounts.tokens[candidate].label(),
|
||||
);
|
||||
accounts.active = candidate;
|
||||
return Some(candidate);
|
||||
if !accounts.tokens[candidate].is_available(now) {
|
||||
continue;
|
||||
}
|
||||
let label = accounts.tokens[candidate].label();
|
||||
if !self.can_authenticate(&mut accounts.tokens[candidate]).await {
|
||||
warn!("claude login {label} cannot authenticate; skipping");
|
||||
continue;
|
||||
}
|
||||
info!(
|
||||
"claude login {} answered {status}; continuing as {label}",
|
||||
accounts.tokens[from].label(),
|
||||
);
|
||||
accounts.active = candidate;
|
||||
return Some(candidate);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Can this login still speak — an unexpired token, or a refresh the
|
||||
/// endpoint accepts?
|
||||
///
|
||||
/// `has_credential` only asks whether a string is present. On 2026-08-17
|
||||
/// that sent the provider onto a spare whose token had expired four days
|
||||
/// earlier: every call after it 401'd as *revoked*, which reads as the
|
||||
/// account being gone rather than the spare being dead.
|
||||
async fn can_authenticate(&self, token: &mut TokenState) -> bool {
|
||||
token.access_is_live(now_ms())
|
||||
|| refresh_token(&self.http, &self.user_agent, token)
|
||||
.await
|
||||
.is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
|
@ -500,13 +541,22 @@ impl LlmProvider for ClaudeSubscriptionProvider {
|
|||
}
|
||||
|
||||
let retry_after = retry_after_secs(resp.headers());
|
||||
let resumes_at = window_resumes_at(resp.headers(), retry_after);
|
||||
let body_text = resp.text().await.unwrap_or_default();
|
||||
|
||||
// A spent quota window does not clear inside a retry loop — the only
|
||||
// thing that answers is another login. Retry it at once: the wait is
|
||||
// hours, not milliseconds.
|
||||
if is_exhausted_window(status.as_u16(), retry_after)
|
||||
&& self.rotate_account(account).await.is_some()
|
||||
//
|
||||
// A login that cannot authenticate moves for the opposite reason:
|
||||
// nothing is lost by leaving, because the cached prefix sits behind
|
||||
// the same token that just failed.
|
||||
if (is_exhausted_window(status.as_u16(), retry_after)
|
||||
|| is_auth_failure(status.as_u16()))
|
||||
&& self
|
||||
.rotate_account(account, status.as_u16(), resumes_at)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
strain.push(InferenceStrain::Transient {
|
||||
attempt,
|
||||
|
|
@ -588,7 +638,11 @@ fn adopt_file_if_changed(token: &mut TokenState) {
|
|||
credential_fingerprint(&fresh),
|
||||
credential_fingerprint(token),
|
||||
);
|
||||
// The quota belongs to the account, not to the token it was spent through:
|
||||
// a refresh mid-window must not read as a fresh window.
|
||||
let resumes = token.resumes_at_ms;
|
||||
*token = fresh;
|
||||
token.resumes_at_ms = resumes;
|
||||
}
|
||||
|
||||
/// Short, stable id for a credential — enough to see in a log that souveraine
|
||||
|
|
@ -618,7 +672,8 @@ fn credential_fingerprint(token: &TokenState) -> String {
|
|||
/// the rotated refresh token we picked up alongside it.
|
||||
fn reread_credentials(current: &TokenState) -> Option<TokenState> {
|
||||
let path = current.source_path.as_ref()?;
|
||||
let rotated = load_credentials(path).ok()?;
|
||||
let mut rotated = load_credentials(path).ok()?;
|
||||
rotated.resumes_at_ms = current.resumes_at_ms;
|
||||
let is_newer = rotated
|
||||
.access_token
|
||||
.as_deref()
|
||||
|
|
@ -779,6 +834,7 @@ fn load_credentials(path: &Path) -> Result<TokenState> {
|
|||
.map(str::to_string),
|
||||
expires_at_ms: oauth.get("expiresAt").and_then(Value::as_u64),
|
||||
source_path: Some(path.to_path_buf()),
|
||||
resumes_at_ms: None,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -1385,6 +1441,27 @@ fn is_exhausted_window(status: u16, retry_after: Option<u64>) -> bool {
|
|||
status == 429 && retry_after.is_none_or(|secs| secs > BURST_RETRY_CEILING_SECS)
|
||||
}
|
||||
|
||||
/// This login cannot speak, whatever it is holding.
|
||||
fn is_auth_failure(status: u16) -> bool {
|
||||
matches!(status, 401 | 403)
|
||||
}
|
||||
|
||||
/// When the login that produced this response is worth trying again, in ms.
|
||||
///
|
||||
/// These 429s carry no `retry-after`; the unified rate-limit headers carry the
|
||||
/// reset as epoch seconds, and are the only precise answer on the wire.
|
||||
fn window_resumes_at(headers: &reqwest::header::HeaderMap, retry_after: Option<u64>) -> u64 {
|
||||
let now = now_ms();
|
||||
headers
|
||||
.get("anthropic-ratelimit-unified-reset")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.trim().parse::<u64>().ok())
|
||||
.map(|secs| secs * 1000)
|
||||
.filter(|at| *at > now)
|
||||
.or_else(|| retry_after.map(|secs| now + secs * 1000))
|
||||
.unwrap_or(now + DEFAULT_WINDOW_COOLDOWN.as_millis() as u64)
|
||||
}
|
||||
|
||||
fn reset_hint(retry_after: Option<u64>) -> String {
|
||||
match retry_after {
|
||||
Some(secs) if secs >= 60 => format!(" (resets in ~{}m)", secs / 60),
|
||||
|
|
@ -1806,6 +1883,72 @@ mod tests {
|
|||
assert!(!is_exhausted_window(500, Some(3600)));
|
||||
}
|
||||
|
||||
/// 2026-08-17T21:51: the primary spent its window, rotation took the spare
|
||||
/// on `has_credential` alone — a login whose token had expired on the 13th —
|
||||
/// and every call after it came back 401 *revoked*, which reads as the
|
||||
/// subscription being gone rather than the spare being dead. Presence of a
|
||||
/// string is not the question.
|
||||
#[test]
|
||||
fn an_expired_spare_is_not_a_fallback() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dead = creds_at(dir.path(), "expired-on-the-13th", now_ms() - 4 * 86_400_000);
|
||||
assert!(dead.has_credential());
|
||||
assert!(!dead.access_is_live(now_ms()));
|
||||
|
||||
let other = tempfile::tempdir().unwrap();
|
||||
let live = creds_at(other.path(), "good", now_ms() + 8 * 3_600_000);
|
||||
assert!(live.access_is_live(now_ms()));
|
||||
}
|
||||
|
||||
/// A login parked on a spent window is skipped until the window returns —
|
||||
/// and the window survives the refresh that rotates its token, because the
|
||||
/// quota belongs to the account, not to the string it was spent through.
|
||||
#[test]
|
||||
fn a_parked_login_stays_parked_until_its_window_returns() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut spent = creds_at(dir.path(), "account-a", now_ms() + 8 * 3_600_000);
|
||||
spent.resumes_at_ms = Some(now_ms() + 3_600_000);
|
||||
assert!(!spent.is_available(now_ms()));
|
||||
assert!(spent.is_available(now_ms() + 3_600_001));
|
||||
|
||||
creds_at(dir.path(), "account-a-refreshed", now_ms() + 8 * 3_600_000);
|
||||
adopt_file_if_changed(&mut spent);
|
||||
assert_eq!(spent.access_token.as_deref(), Some("account-a-refreshed"));
|
||||
assert!(!spent.is_available(now_ms()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_window_reset_is_read_from_the_header_that_carries_it() {
|
||||
let mut headers = reqwest::header::HeaderMap::new();
|
||||
let reset_secs = now_ms() / 1000 + 3600;
|
||||
headers.insert(
|
||||
"anthropic-ratelimit-unified-reset",
|
||||
reqwest::header::HeaderValue::from_str(&reset_secs.to_string()).unwrap(),
|
||||
);
|
||||
let at = window_resumes_at(&headers, None);
|
||||
assert!(at.abs_diff(now_ms() + 3_600_000) < 2_000, "{at}");
|
||||
|
||||
// No header: `retry-after`, then a cooldown short enough that one wasted
|
||||
// call is the whole cost of guessing.
|
||||
let empty = reqwest::header::HeaderMap::new();
|
||||
let after = window_resumes_at(&empty, Some(120));
|
||||
assert!(after.abs_diff(now_ms() + 120_000) < 2_000, "{after}");
|
||||
let blind = window_resumes_at(&empty, None);
|
||||
assert!(blind.abs_diff(now_ms() + 900_000) < 2_000, "{blind}");
|
||||
}
|
||||
|
||||
/// 429 moves because the window is gone; 401 moves because the cached prefix
|
||||
/// is unreachable behind the token that just failed. Everything else stays
|
||||
/// put — a swap costs the prefix a re-bill.
|
||||
#[test]
|
||||
fn only_a_dead_login_and_a_spent_one_are_worth_leaving() {
|
||||
assert!(is_auth_failure(401));
|
||||
assert!(is_auth_failure(403));
|
||||
for code in [400, 404, 429, 500, 529] {
|
||||
assert!(!is_auth_failure(code), "{code} is not an auth failure");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_bail_says_when_the_quota_comes_back() {
|
||||
assert_eq!(reset_hint(Some(7200)), " (resets in ~120m)");
|
||||
|
|
|
|||
Loading…
Reference in a new issue