Watch
1
0
Fork
You've already forked souveraine
0

claude: don't fall back onto a login that cannot speak

Rotation picked the spare on has_credential — a string check — and took
one whose token had expired four days earlier; every call after 21:51
came back 401 revoked, reading as a dead subscription. A candidate now
has to authenticate, and the window it left is recorded so the same
round-robin can wrap back to it once the window returns.
This commit is contained in:
Fimeg 2026-08-17 23:29:43 -04:00
commit 0398f9f190

View file

@ -53,6 +53,10 @@ const TOKEN_REFRESH_BUFFER: Duration = Duration::from_secs(5 * 60);
/// Longest `retry-after` still worth waiting out on the account already holding
/// the cached prefix, instead of moving to another login.
const BURST_RETRY_CEILING_SECS: u64 = 60;
/// How long a spent login waits when the response says nothing about its window.
/// One wasted call a quarter-hour is cheaper than parking a live login for the
/// five hours the window actually runs.
const DEFAULT_WINDOW_COOLDOWN: Duration = Duration::from_secs(15 * 60);
const DEFAULT_MAX_TOKENS: u32 = 8192;
/// Models that think by default cap thinking *plus* response text with
/// `max_tokens`, so 8192 truncates mid-answer.
@ -132,6 +136,9 @@ struct TokenState {
refresh_token: Option<String>,
expires_at_ms: Option<u64>,
source_path: Option<PathBuf>,
/// When this login's spent quota window comes back. Read from the response
/// that reported it spent; until then the login is not tried.
resumes_at_ms: Option<u64>,
}
impl TokenState {
@ -140,6 +147,16 @@ impl TokenState {
live(&self.access_token) || live(&self.refresh_token)
}
/// Worth trying: something to present, and no spent window still running.
fn is_available(&self, now: u64) -> bool {
self.has_credential() && self.resumes_at_ms.is_none_or(|at| now >= at)
}
fn access_is_live(&self, now: u64) -> bool {
self.access_token.as_deref().is_some_and(|t| !t.is_empty())
&& self.expires_at_ms.is_some_and(|exp| exp > now)
}
fn label(&self) -> String {
match &self.source_path {
Some(p) => p.display().to_string(),
@ -327,15 +344,19 @@ impl ClaudeSubscriptionProvider {
Ok((access, active))
}
/// Move off the account at `from`, which has nothing left to spend.
/// Move off the account at `from`, which answered `status` and is not worth
/// trying again until `resumes_at`.
///
/// Sticky by design: the next login is used until it too is exhausted,
/// rather than alternating. Switching accounts abandons a prompt cache the
/// transcript has already paid for — measured 2026-08-12 at ~125k cached
/// tokens a turn — so a swap has to be worth a full re-bill, and only an
/// exhausted window is.
async fn rotate_account(&self, from: usize) -> Option<usize> {
/// exhausted window is — which is also why nothing walks back to a better
/// login mid-conversation. The way home is this same round-robin: it wraps
/// to the primary, and takes it once its recorded window has passed.
async fn rotate_account(&self, from: usize, status: u16, resumes_at: u64) -> Option<usize> {
let mut accounts = self.accounts.lock().await;
accounts.tokens[from].resumes_at_ms = Some(resumes_at);
let count = accounts.tokens.len();
if count < 2 {
return None;
@ -345,21 +366,41 @@ impl ClaudeSubscriptionProvider {
if accounts.active != from {
return Some(accounts.active);
}
let now = now_ms();
for step in 1..count {
let candidate = (from + step) % count;
adopt_file_if_changed(&mut accounts.tokens[candidate]);
if accounts.tokens[candidate].has_credential() {
info!(
"claude login {} is out of quota; continuing as {}",
accounts.tokens[from].label(),
accounts.tokens[candidate].label(),
);
accounts.active = candidate;
return Some(candidate);
if !accounts.tokens[candidate].is_available(now) {
continue;
}
let label = accounts.tokens[candidate].label();
if !self.can_authenticate(&mut accounts.tokens[candidate]).await {
warn!("claude login {label} cannot authenticate; skipping");
continue;
}
info!(
"claude login {} answered {status}; continuing as {label}",
accounts.tokens[from].label(),
);
accounts.active = candidate;
return Some(candidate);
}
None
}
/// Can this login still speak — an unexpired token, or a refresh the
/// endpoint accepts?
///
/// `has_credential` only asks whether a string is present. On 2026-08-17
/// that sent the provider onto a spare whose token had expired four days
/// earlier: every call after it 401'd as *revoked*, which reads as the
/// account being gone rather than the spare being dead.
async fn can_authenticate(&self, token: &mut TokenState) -> bool {
token.access_is_live(now_ms())
|| refresh_token(&self.http, &self.user_agent, token)
.await
.is_ok()
}
}
#[async_trait]
@ -500,13 +541,22 @@ impl LlmProvider for ClaudeSubscriptionProvider {
}
let retry_after = retry_after_secs(resp.headers());
let resumes_at = window_resumes_at(resp.headers(), retry_after);
let body_text = resp.text().await.unwrap_or_default();
// A spent quota window does not clear inside a retry loop — the only
// thing that answers is another login. Retry it at once: the wait is
// hours, not milliseconds.
if is_exhausted_window(status.as_u16(), retry_after)
&& self.rotate_account(account).await.is_some()
//
// A login that cannot authenticate moves for the opposite reason:
// nothing is lost by leaving, because the cached prefix sits behind
// the same token that just failed.
if (is_exhausted_window(status.as_u16(), retry_after)
|| is_auth_failure(status.as_u16()))
&& self
.rotate_account(account, status.as_u16(), resumes_at)
.await
.is_some()
{
strain.push(InferenceStrain::Transient {
attempt,
@ -588,7 +638,11 @@ fn adopt_file_if_changed(token: &mut TokenState) {
credential_fingerprint(&fresh),
credential_fingerprint(token),
);
// The quota belongs to the account, not to the token it was spent through:
// a refresh mid-window must not read as a fresh window.
let resumes = token.resumes_at_ms;
*token = fresh;
token.resumes_at_ms = resumes;
}
/// Short, stable id for a credential — enough to see in a log that souveraine
@ -618,7 +672,8 @@ fn credential_fingerprint(token: &TokenState) -> String {
/// the rotated refresh token we picked up alongside it.
fn reread_credentials(current: &TokenState) -> Option<TokenState> {
let path = current.source_path.as_ref()?;
let rotated = load_credentials(path).ok()?;
let mut rotated = load_credentials(path).ok()?;
rotated.resumes_at_ms = current.resumes_at_ms;
let is_newer = rotated
.access_token
.as_deref()
@ -779,6 +834,7 @@ fn load_credentials(path: &Path) -> Result<TokenState> {
.map(str::to_string),
expires_at_ms: oauth.get("expiresAt").and_then(Value::as_u64),
source_path: Some(path.to_path_buf()),
resumes_at_ms: None,
})
}
@ -1385,6 +1441,27 @@ fn is_exhausted_window(status: u16, retry_after: Option<u64>) -> bool {
status == 429 && retry_after.is_none_or(|secs| secs > BURST_RETRY_CEILING_SECS)
}
/// This login cannot speak, whatever it is holding.
fn is_auth_failure(status: u16) -> bool {
matches!(status, 401 | 403)
}
/// When the login that produced this response is worth trying again, in ms.
///
/// These 429s carry no `retry-after`; the unified rate-limit headers carry the
/// reset as epoch seconds, and are the only precise answer on the wire.
fn window_resumes_at(headers: &reqwest::header::HeaderMap, retry_after: Option<u64>) -> u64 {
let now = now_ms();
headers
.get("anthropic-ratelimit-unified-reset")
.and_then(|v| v.to_str().ok())
.and_then(|v| v.trim().parse::<u64>().ok())
.map(|secs| secs * 1000)
.filter(|at| *at > now)
.or_else(|| retry_after.map(|secs| now + secs * 1000))
.unwrap_or(now + DEFAULT_WINDOW_COOLDOWN.as_millis() as u64)
}
fn reset_hint(retry_after: Option<u64>) -> String {
match retry_after {
Some(secs) if secs >= 60 => format!(" (resets in ~{}m)", secs / 60),
@ -1806,6 +1883,72 @@ mod tests {
assert!(!is_exhausted_window(500, Some(3600)));
}
/// 2026-08-17T21:51: the primary spent its window, rotation took the spare
/// on `has_credential` alone — a login whose token had expired on the 13th —
/// and every call after it came back 401 *revoked*, which reads as the
/// subscription being gone rather than the spare being dead. Presence of a
/// string is not the question.
#[test]
fn an_expired_spare_is_not_a_fallback() {
let dir = tempfile::tempdir().unwrap();
let dead = creds_at(dir.path(), "expired-on-the-13th", now_ms() - 4 * 86_400_000);
assert!(dead.has_credential());
assert!(!dead.access_is_live(now_ms()));
let other = tempfile::tempdir().unwrap();
let live = creds_at(other.path(), "good", now_ms() + 8 * 3_600_000);
assert!(live.access_is_live(now_ms()));
}
/// A login parked on a spent window is skipped until the window returns —
/// and the window survives the refresh that rotates its token, because the
/// quota belongs to the account, not to the string it was spent through.
#[test]
fn a_parked_login_stays_parked_until_its_window_returns() {
let dir = tempfile::tempdir().unwrap();
let mut spent = creds_at(dir.path(), "account-a", now_ms() + 8 * 3_600_000);
spent.resumes_at_ms = Some(now_ms() + 3_600_000);
assert!(!spent.is_available(now_ms()));
assert!(spent.is_available(now_ms() + 3_600_001));
creds_at(dir.path(), "account-a-refreshed", now_ms() + 8 * 3_600_000);
adopt_file_if_changed(&mut spent);
assert_eq!(spent.access_token.as_deref(), Some("account-a-refreshed"));
assert!(!spent.is_available(now_ms()));
}
#[test]
fn the_window_reset_is_read_from_the_header_that_carries_it() {
let mut headers = reqwest::header::HeaderMap::new();
let reset_secs = now_ms() / 1000 + 3600;
headers.insert(
"anthropic-ratelimit-unified-reset",
reqwest::header::HeaderValue::from_str(&reset_secs.to_string()).unwrap(),
);
let at = window_resumes_at(&headers, None);
assert!(at.abs_diff(now_ms() + 3_600_000) < 2_000, "{at}");
// No header: `retry-after`, then a cooldown short enough that one wasted
// call is the whole cost of guessing.
let empty = reqwest::header::HeaderMap::new();
let after = window_resumes_at(&empty, Some(120));
assert!(after.abs_diff(now_ms() + 120_000) < 2_000, "{after}");
let blind = window_resumes_at(&empty, None);
assert!(blind.abs_diff(now_ms() + 900_000) < 2_000, "{blind}");
}
/// 429 moves because the window is gone; 401 moves because the cached prefix
/// is unreachable behind the token that just failed. Everything else stays
/// put — a swap costs the prefix a re-bill.
#[test]
fn only_a_dead_login_and_a_spent_one_are_worth_leaving() {
assert!(is_auth_failure(401));
assert!(is_auth_failure(403));
for code in [400, 404, 429, 500, 529] {
assert!(!is_auth_failure(code), "{code} is not an auth failure");
}
}
#[test]
fn the_bail_says_when_the_quota_comes_back() {
assert_eq!(reset_hint(Some(7200)), " (resets in ~120m)");