provider-balance: distinguish a subscription cap from bankruptcy, ship the tests
The classifier's tests existed only as a throwaway heredoc, so they could not be re-run and protected nothing. They are now in the file behind --selftest, built from HTTP bodies captured verbatim rather than written from memory. Cap and rate wording is now checked before the bankruptcy markers. OpenCode Go is a subscription with $12/5h, $30/week and $60/month ceilings; hitting one means wait, not pay, and a cap message that also says 'billing' would otherwise be reported as an empty account. Verified safe: none of the three real bankruptcy bodies contain any cap or rate wording. The Go cap wording itself is unverified and labelled as such in the source rather than presented as observed. Gate proven able to reject: removing one marker fails 2/9 and exits 1.
This commit is contained in:
parent
6b9a415aac
commit
a2648543b2
1 changed files with 91 additions and 3 deletions
|
|
@ -11,6 +11,9 @@ every case the first thing that told us was a failed subconscious pass:
|
|||
* deepseek -- 402 "Insufficient Balance", total_balance -0.04
|
||||
* opencode zen -- 401 CreditsError
|
||||
|
||||
Note the codes disagree completely: 429, 402, 401, all for "you are out of
|
||||
money". Classifying on status code cannot work; we classify on wording.
|
||||
|
||||
Balance exhaustion is invisible until something dies. Absence of complaint
|
||||
reads as health. This is the instrument that speaks first.
|
||||
|
||||
|
|
@ -33,6 +36,7 @@ Usage
|
|||
provider-balance.py # probe role-backing providers (default)
|
||||
provider-balance.py --all # probe every configured provider
|
||||
provider-balance.py --json # machine-readable
|
||||
provider-balance.py --selftest # prove the classifier can reject
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
|
@ -49,6 +53,7 @@ TIMEOUT = 45
|
|||
OK = "OK"
|
||||
NO_BALANCE = "NO_BALANCE"
|
||||
RATE_LIMITED = "RATE_LIMITED"
|
||||
CAPPED = "CAPPED"
|
||||
AUTH = "AUTH"
|
||||
UNREACHABLE = "UNREACHABLE"
|
||||
SKIPPED = "SKIPPED"
|
||||
|
|
@ -66,9 +71,29 @@ BROKE_MARKERS = (
|
|||
"payment required",
|
||||
)
|
||||
|
||||
# A time-window cap is NOT bankruptcy. OpenCode Go is a $10/month
|
||||
# subscription with $12/5h, $30/week and $60/month ceilings; hitting one means
|
||||
# "wait", not "pay". These are checked BEFORE the broke markers because a cap
|
||||
# message may well also mention billing, and misreporting a throttle as
|
||||
# bankruptcy would send someone to a payment page for no reason.
|
||||
#
|
||||
# Ordering is safe against every real bankruptcy body we hold -- none of them
|
||||
# contain any of the wording below. --selftest asserts exactly that.
|
||||
#
|
||||
# VERIFIED wording: "FreeUsageLimitError" (opencode zen free tier, captured
|
||||
# 2026-08-13). UNVERIFIED: the Go 5h/weekly/monthly cap body -- no one has
|
||||
# spent $12 in five hours yet, so the remaining markers are a best guess and
|
||||
# are marked as such rather than presented as observed.
|
||||
CAP_MARKERS = (
|
||||
"freeusagelimiterror",
|
||||
"usagelimiterror",
|
||||
"usage limit",
|
||||
"limit reached",
|
||||
"limit exceeded",
|
||||
)
|
||||
|
||||
RATE_MARKERS = (
|
||||
"rate limit",
|
||||
"freeusagelimiterror",
|
||||
"too many requests",
|
||||
)
|
||||
|
||||
|
|
@ -107,10 +132,14 @@ def classify(code: int, body: str) -> str:
|
|||
low = body.lower()
|
||||
if code == 0:
|
||||
return UNREACHABLE
|
||||
if any(m in low for m in BROKE_MARKERS):
|
||||
return NO_BALANCE
|
||||
# Cap and rate wording first: both are recoverable-by-waiting, and a cap
|
||||
# message that also says "billing" must not be reported as bankruptcy.
|
||||
if any(m in low for m in CAP_MARKERS):
|
||||
return CAPPED
|
||||
if any(m in low for m in RATE_MARKERS):
|
||||
return RATE_LIMITED
|
||||
if any(m in low for m in BROKE_MARKERS):
|
||||
return NO_BALANCE
|
||||
if code == 200:
|
||||
return OK
|
||||
if code in (401, 403):
|
||||
|
|
@ -179,13 +208,72 @@ def roles(cfg: dict) -> dict[str, str]:
|
|||
return out
|
||||
|
||||
|
||||
def selftest() -> int:
|
||||
"""Prove the classifier can reject, using real captured bodies.
|
||||
|
||||
Every string below was copied out of an actual HTTP response during
|
||||
2026-08-12/13. Nothing here is paraphrased -- a fixture I wrote from
|
||||
memory would only test my memory. The negative cases matter most: a
|
||||
context overflow and an overload are NOT bankruptcy, and reporting them
|
||||
as such would send someone to a billing page over a full conversation.
|
||||
"""
|
||||
cases = [
|
||||
# (label, http code, body, expected)
|
||||
("deepseek 402 bankrupt", 402,
|
||||
'{"error":{"message":"Insufficient Balance","type":"unknown_error",'
|
||||
'"param":null,"code":"invalid_request_error"}}', NO_BALANCE),
|
||||
("zai 429 code 1113 bankrupt", 429,
|
||||
'{"error":{"code":"1113","message":"Insufficient balance or no '
|
||||
'resource package. Please recharge."}}', NO_BALANCE),
|
||||
("opencode zen 401 bankrupt", 401,
|
||||
'{"type":"error","error":{"type":"CreditsError","message":'
|
||||
'"Insufficient balance. Manage your billing here: '
|
||||
'https://opencode.ai/workspace/wrk_01KZ/billing"}}', NO_BALANCE),
|
||||
("opencode zen free tier capped", 429,
|
||||
'{"type":"error","error":{"type":"FreeUsageLimitError","message":'
|
||||
'"Free usage limit reached."}}', CAPPED),
|
||||
("opencode go healthy", 200,
|
||||
'{"id":"49dd810d","object":"chat.completion","model":'
|
||||
'"deepseek-v4-flash","choices":[{"message":{"content":"alive"}}],'
|
||||
'"cost":"0"}', OK),
|
||||
# --- negatives: these must NOT read as bankruptcy ---
|
||||
("context overflow is not bankruptcy", 400,
|
||||
'{"error":{"message":"This model\'s maximum context length is '
|
||||
'1048576 tokens. However, you requested 1377662 tokens (1377662 in '
|
||||
'the messages, 0 in the completion). Please reduce the length of '
|
||||
'the messages or completion.","type":"invalid_request_error"}}',
|
||||
UNREACHABLE),
|
||||
("anthropic 529 overload is not bankruptcy", 529,
|
||||
'{"type":"error","error":{"type":"overloaded_error",'
|
||||
'"message":"Overloaded"}}', UNREACHABLE),
|
||||
("timeout is unreachable", 0, "timed out", UNREACHABLE),
|
||||
("bad key is auth, not bankruptcy", 401,
|
||||
'{"error":{"message":"Invalid API key","type":"authentication_error"}}',
|
||||
AUTH),
|
||||
]
|
||||
failed = 0
|
||||
for label, code, body, expected in cases:
|
||||
got = classify(code, body)
|
||||
ok = got == expected
|
||||
failed += not ok
|
||||
print(f" {'PASS' if ok else 'FAIL'} {label:<42} -> {got}"
|
||||
+ ("" if ok else f" (expected {expected})"))
|
||||
print(f"\n{len(cases) - failed}/{len(cases)} passed")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--all", action="store_true",
|
||||
help="probe every provider, not just role-backing ones")
|
||||
ap.add_argument("--json", action="store_true")
|
||||
ap.add_argument("--selftest", action="store_true",
|
||||
help="run the classifier against real captured bodies")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.selftest:
|
||||
return selftest()
|
||||
|
||||
try:
|
||||
with CONFIG.open("rb") as fh:
|
||||
cfg = tomllib.load(fh)
|
||||
|
|
|
|||
Loading…
Reference in a new issue