Watch
1
0
Fork
You've already forked souveraine
0

provider-balance: distinguish a subscription cap from bankruptcy, ship the tests

The classifier's tests existed only as a throwaway heredoc, so they could not
be re-run and protected nothing. They are now in the file behind --selftest,
built from HTTP bodies captured verbatim rather than written from memory.

Cap and rate wording is now checked before the bankruptcy markers. OpenCode Go
is a subscription with $12/5h, $30/week and $60/month ceilings; hitting one
means wait, not pay, and a cap message that also says 'billing' would otherwise
be reported as an empty account. Verified safe: none of the three real
bankruptcy bodies contain any cap or rate wording.

The Go cap wording itself is unverified and labelled as such in the source
rather than presented as observed.

Gate proven able to reject: removing one marker fails 2/9 and exits 1.
This commit is contained in:
Fimeg 2026-08-13 10:13:35 -04:00
commit a2648543b2

View file

@ -11,6 +11,9 @@ every case the first thing that told us was a failed subconscious pass:
* deepseek -- 402 "Insufficient Balance", total_balance -0.04
* opencode zen -- 401 CreditsError
Note the codes disagree completely: 429, 402, 401, all for "you are out of
money". Classifying on status code cannot work; we classify on wording.
Balance exhaustion is invisible until something dies. Absence of complaint
reads as health. This is the instrument that speaks first.
@ -33,6 +36,7 @@ Usage
provider-balance.py # probe role-backing providers (default)
provider-balance.py --all # probe every configured provider
provider-balance.py --json # machine-readable
provider-balance.py --selftest # prove the classifier can reject
"""
from __future__ import annotations
@ -49,6 +53,7 @@ TIMEOUT = 45
OK = "OK"
NO_BALANCE = "NO_BALANCE"
RATE_LIMITED = "RATE_LIMITED"
CAPPED = "CAPPED"
AUTH = "AUTH"
UNREACHABLE = "UNREACHABLE"
SKIPPED = "SKIPPED"
@ -66,9 +71,29 @@ BROKE_MARKERS = (
"payment required",
)
# A time-window cap is NOT bankruptcy. OpenCode Go is a $10/month
# subscription with $12/5h, $30/week and $60/month ceilings; hitting one means
# "wait", not "pay". These are checked BEFORE the broke markers because a cap
# message may well also mention billing, and misreporting a throttle as
# bankruptcy would send someone to a payment page for no reason.
#
# Ordering is safe against every real bankruptcy body we hold -- none of them
# contain any of the wording below. --selftest asserts exactly that.
#
# VERIFIED wording: "FreeUsageLimitError" (opencode zen free tier, captured
# 2026-08-13). UNVERIFIED: the Go 5h/weekly/monthly cap body -- no one has
# spent $12 in five hours yet, so the remaining markers are a best guess and
# are marked as such rather than presented as observed.
CAP_MARKERS = (
"freeusagelimiterror",
"usagelimiterror",
"usage limit",
"limit reached",
"limit exceeded",
)
RATE_MARKERS = (
"rate limit",
"freeusagelimiterror",
"too many requests",
)
@ -107,10 +132,14 @@ def classify(code: int, body: str) -> str:
low = body.lower()
if code == 0:
return UNREACHABLE
if any(m in low for m in BROKE_MARKERS):
return NO_BALANCE
# Cap and rate wording first: both are recoverable-by-waiting, and a cap
# message that also says "billing" must not be reported as bankruptcy.
if any(m in low for m in CAP_MARKERS):
return CAPPED
if any(m in low for m in RATE_MARKERS):
return RATE_LIMITED
if any(m in low for m in BROKE_MARKERS):
return NO_BALANCE
if code == 200:
return OK
if code in (401, 403):
@ -179,13 +208,72 @@ def roles(cfg: dict) -> dict[str, str]:
return out
def selftest() -> int:
"""Prove the classifier can reject, using real captured bodies.
Every string below was copied out of an actual HTTP response during
2026-08-12/13. Nothing here is paraphrased -- a fixture I wrote from
memory would only test my memory. The negative cases matter most: a
context overflow and an overload are NOT bankruptcy, and reporting them
as such would send someone to a billing page over a full conversation.
"""
cases = [
# (label, http code, body, expected)
("deepseek 402 bankrupt", 402,
'{"error":{"message":"Insufficient Balance","type":"unknown_error",'
'"param":null,"code":"invalid_request_error"}}', NO_BALANCE),
("zai 429 code 1113 bankrupt", 429,
'{"error":{"code":"1113","message":"Insufficient balance or no '
'resource package. Please recharge."}}', NO_BALANCE),
("opencode zen 401 bankrupt", 401,
'{"type":"error","error":{"type":"CreditsError","message":'
'"Insufficient balance. Manage your billing here: '
'https://opencode.ai/workspace/wrk_01KZ/billing"}}', NO_BALANCE),
("opencode zen free tier capped", 429,
'{"type":"error","error":{"type":"FreeUsageLimitError","message":'
'"Free usage limit reached."}}', CAPPED),
("opencode go healthy", 200,
'{"id":"49dd810d","object":"chat.completion","model":'
'"deepseek-v4-flash","choices":[{"message":{"content":"alive"}}],'
'"cost":"0"}', OK),
# --- negatives: these must NOT read as bankruptcy ---
("context overflow is not bankruptcy", 400,
'{"error":{"message":"This model\'s maximum context length is '
'1048576 tokens. However, you requested 1377662 tokens (1377662 in '
'the messages, 0 in the completion). Please reduce the length of '
'the messages or completion.","type":"invalid_request_error"}}',
UNREACHABLE),
("anthropic 529 overload is not bankruptcy", 529,
'{"type":"error","error":{"type":"overloaded_error",'
'"message":"Overloaded"}}', UNREACHABLE),
("timeout is unreachable", 0, "timed out", UNREACHABLE),
("bad key is auth, not bankruptcy", 401,
'{"error":{"message":"Invalid API key","type":"authentication_error"}}',
AUTH),
]
failed = 0
for label, code, body, expected in cases:
got = classify(code, body)
ok = got == expected
failed += not ok
print(f" {'PASS' if ok else 'FAIL'} {label:<42} -> {got}"
+ ("" if ok else f" (expected {expected})"))
print(f"\n{len(cases) - failed}/{len(cases)} passed")
return 1 if failed else 0
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--all", action="store_true",
help="probe every provider, not just role-backing ones")
ap.add_argument("--json", action="store_true")
ap.add_argument("--selftest", action="store_true",
help="run the classifier against real captured bodies")
args = ap.parse_args()
if args.selftest:
return selftest()
try:
with CONFIG.open("rb") as fh:
cfg = tomllib.load(fh)