Fimeg/souveraine
Watch
1
0
Fork
You've already forked souveraine
 0
Code Releases Activity

claude: cache the prefix the bridge was re-billing every turn

Browse source 
No cache_control was ever set, so tools, system and the whole transcript
were charged in full on every round. Breakpoints now sit on the tail of
each stable span; message content is always a block array so the marked
tail cannot change the prefix bytes as it moves.

Mid-conversation system notes — the turn loop's pulse and interjections,
the subagent round warnings — were hoisted into the top-level system
field, rewriting the front of the prefix on the round they fired. They
stay where they happened now. Usage carries the cache split; input_tokens
alone understates the prompt by whatever was served cheaply.
This commit is contained in:
Fimeg 2026-08-10 13:33:00 -04:00
commit a10ef5e3ed
3 changed files with 284 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/bridge/bifrost.rs
View file

@ -314,12 +314,19 @@ pub struct ToolCallFunction {
pub arguments: String,
}
#[derive(Debug, Clone, Deserialize)]
#[derive(Debug, Clone, Default, Deserialize)]
pub struct Usage {
pub prompt_tokens: u32,
pub completion_tokens: u32,
#[serde(default)]
pub total_tokens: u32,
/// Prefix served from cache. Anthropic reports the uncached remainder as
/// `input_tokens`, so these two are the only way to tell a cache hit from a
/// short prompt — without them there is no way to know caching works.
#[serde(default)]
pub cache_read_tokens: u32,
#[serde(default)]
pub cache_write_tokens: u32,
}
/// Stream chunk from Bifrost (OpenAI SSE format)

297
src/bridge/claude_subscription.rs
View file

@ -83,6 +83,20 @@ fn thinks_by_default(model: &str) -> bool {
THINKS.iter().any(|m| model.starts_with(m))
}
/// Models that accept a `role: "system"` turn inside `messages`. Everywhere
/// else a mid-conversation note has to be folded into the user turn, which
/// costs the operator role but keeps the cached prefix intact either way.
/// Sonnet 5 is deliberately absent — it 400s on the role.
fn supports_mid_conversation_system(model: &str) -> bool {
const SUPPORTED: &[&str] = &[
"claude-opus-5",
"claude-opus-4-8",
"claude-fable-5",
"claude-mythos-5",
];
SUPPORTED.iter().any(|m| model.starts_with(m))
}
/// Direct Claude Code subscription inference provider.
pub struct ClaudeSubscriptionProvider {
name: String,
@ -190,7 +204,8 @@ impl ClaudeSubscriptionProvider {
/// Translate an OpenAI chat-completion request to a wire-shaped Anthropic
/// `/v1/messages` body.
fn shape_body(&self, model: &str, request: &ChatCompletionRequest) -> Result<Vec<u8>> {
let (messages, system_text, first_user_text) = translate_messages(&request.messages);
let (messages, system_text, first_user_text) =
translate_messages(&request.messages, supports_mid_conversation_system(model));
let default_max_tokens = if thinks_by_default(model) {
THINKING_MAX_TOKENS
} else {
@ -225,6 +240,7 @@ impl ClaudeSubscriptionProvider {
let obj = body.as_object_mut().unwrap();
inject_attribution(obj, &self.cc_version, &first_user_text);
inject_metadata(obj, self);
mark_cache_breakpoints(obj);
Ok(serde_json::to_vec(&body)?)
}
@ -663,15 +679,33 @@ fn compute_fingerprint(first_user_text: &str, version: &str) -> String {
// ── OpenAI ↔ Anthropic translation =========================================
/// Translate OpenAI messages into Anthropic messages + a joined system string.
fn translate_messages(openai: &[Message]) -> (Vec<Value>, Option<String>, String) {
///
/// `mid_conv_system` decides where a system note that arrives *after* the
/// conversation has started lands. The top-level `system` field renders at the
/// very front of the prefix, so hoisting the turn loop's pulses, interjections
/// and subagent-awareness notes into it rewrites the front of the prompt on the
/// round they fire and re-bills tools, system and the whole transcript. They
/// stay where they happened instead.
fn translate_messages(
openai: &[Message],
mid_conv_system: bool,
) -> (Vec<Value>, Option<String>, String) {
let mut system_parts: Vec<String> = Vec::new();
let mut merged: Vec<(String, Vec<Value>)> = Vec::new();
for msg in openai {
match msg.role.as_str() {
"system" => {
if !msg.content.is_empty() {
system_parts.push(msg.content.as_text());
if msg.content.is_empty() {
continue;
}
let text = msg.content.as_text();
let conversation_started = merged.iter().any(|(role, _)| role == "user");
if !conversation_started {
system_parts.push(text);
} else {
let role = if mid_conv_system { "system" } else { "user" };
push_merge(&mut merged, role, json!({ "type": "text", "text": text }));
}
}
"tool" => {
@ -735,24 +769,24 @@ fn translate_messages(openai: &[Message]) -> (Vec<Value>, Option<String>, String
while merged.first().is_some_and(|(role, _)| role == "assistant") {
merged.remove(0);
}
let merged = normalize_system_turns(merged);
let messages: Vec<Value> = merged
.into_iter()
.map(|(role, blocks)| {
let content = if blocks.len() == 1
&& blocks[0].get("type").and_then(Value::as_str) == Some("text")
{
Value::String(
blocks[0]
.get("text")
.and_then(Value::as_str)
.unwrap_or("")
.to_string(),
)
} else {
Value::Array(blocks)
};
json!({ "role": role, "content": content })
if role == "system" {
let text = blocks
.iter()
.filter_map(|b| b.get("text").and_then(Value::as_str))
.collect::<Vec<_>>()
.join("\n\n");
return json!({ "role": "system", "content": text });
}
// Content is always a block array, never the equivalent bare
// string. Cache breakpoints attach to a block, and a shape that
// flipped between the two as the marked tail moved would change the
// prefix bytes every round and miss every read.
json!({ "role": role, "content": Value::Array(blocks) })
})
.collect();
@ -771,6 +805,75 @@ fn translate_messages(openai: &[Message]) -> (Vec<Value>, Option<String>, String
(messages, system, first_user_text)
}
/// A `role: "system"` turn is only legal between a user turn and an assistant
/// turn, or as the last entry. Demote every other one into the user turn ahead
/// of it — still after the cached prefix, just without the operator role.
fn normalize_system_turns(merged: Vec<(String, Vec<Value>)>) -> Vec<(String, Vec<Value>)> {
let mut out: Vec<(String, Vec<Value>)> = Vec::with_capacity(merged.len());
for (i, (role, blocks)) in merged.iter().enumerate() {
let legal = role != "system"
|| (out.last().is_some_and(|(r, _)| r == "user")
&& merged
.get(i + 1)
.is_none_or(|(r, _)| r.as_str() == "assistant"));
let role = if legal { role.as_str() } else { "user" };
match out.last_mut() {
Some(last) if last.0 == role => last.1.extend(blocks.iter().cloned()),
_ => out.push((role.to_string(), blocks.clone())),
}
}
out
}
/// Place the four breakpoints a request is allowed.
///
/// Caching is a prefix match rendered `tools` → `system` → `messages`. The tool
/// schemas are byte-identical across every conversation and every agent, and
/// the system prompt is fixed for the life of one conversation, so each gets a
/// breakpoint of its own — editing the prompt then still reads the tools. The
/// remaining two ride the tail of the transcript so round N+1 of the tool loop
/// reads round N's prefix instead of paying for it again.
fn mark_cache_breakpoints(body: &mut Map<String, Value>) {
fn mark(block: &mut Value) {
if let Some(obj) = block.as_object_mut() {
obj.insert("cache_control".to_string(), json!({ "type": "ephemeral" }));
}
}
fn mark_last(body: &mut Map<String, Value>, key: &str) {
if let Some(last) = body
.get_mut(key)
.and_then(Value::as_array_mut)
.and_then(|blocks| blocks.last_mut())
{
mark(last);
}
}
mark_last(body, "tools");
mark_last(body, "system");
let Some(messages) = body.get_mut("messages").and_then(Value::as_array_mut) else {
return;
};
let mut marked = 0;
for message in messages.iter_mut().rev() {
if marked == 2 {
break;
}
// A system turn carries a bare string — no block to hang a breakpoint
// on, and it is the volatile tail anyway.
let Some(last) = message
.get_mut("content")
.and_then(Value::as_array_mut)
.and_then(|blocks| blocks.last_mut())
else {
continue;
};
mark(last);
marked += 1;
}
}
fn push_merge(merged: &mut Vec<(String, Vec<Value>)>, role: &str, block: Value) {
if let Some(last) = merged.last_mut() {
if last.0 == role {
@ -847,12 +950,22 @@ fn parse_completion(text: &str) -> Result<CompletionResult> {
.or(Some("stop".to_string()));
let usage = value.get("usage").map(|u| {
let input = u.get("input_tokens").and_then(Value::as_u64).unwrap_or(0) as u32;
let output = u.get("output_tokens").and_then(Value::as_u64).unwrap_or(0) as u32;
let field = |key: &str| u.get(key).and_then(Value::as_u64).unwrap_or(0) as u32;
// `input_tokens` is only the part of the prompt that missed cache, so
// it understates the context by however much was served cheaply.
// `prompt_tokens` carries the real prompt size; the split rides
// alongside it.
let uncached = field("input_tokens");
let cache_read = field("cache_read_input_tokens");
let cache_write = field("cache_creation_input_tokens");
let output = field("output_tokens");
let input = uncached + cache_read + cache_write;
Usage {
prompt_tokens: input,
completion_tokens: output,
total_tokens: input + output,
cache_read_tokens: cache_read,
cache_write_tokens: cache_write,
}
});
@ -1002,7 +1115,7 @@ mod tests {
Message::tool_result("call_1", "read", "example-host"),
Message::text("user", "thanks"),
];
let (messages, system, first_user) = translate_messages(&msgs);
let (messages, system, first_user) = translate_messages(&msgs, true);
assert!(system.is_none());
let roles: Vec<&str> = messages
.iter()
@ -1027,10 +1140,150 @@ mod tests {
Message::text("system", "you are souvie"),
Message::text("user", "hi"),
];
let (_, system, _) = translate_messages(&msgs);
let (_, system, _) = translate_messages(&msgs, true);
assert_eq!(system.as_deref(), Some("you are souvie"));
}
/// The turn loop's pulse and the subagent's round warnings arrive as system
/// messages mid-conversation. Hoisting them into the top-level field would
/// rewrite the front of the prefix on the round they fire.
#[test]
fn mid_conversation_system_note_stays_out_of_the_system_field() {
let msgs = vec![
Message::text("system", "you are souvie"),
Message::text("user", "hi"),
Message::text("system", "[pulse] 4m elapsed"),
];
let (messages, system, _) = translate_messages(&msgs, true);
assert_eq!(system.as_deref(), Some("you are souvie"));
let last = messages.last().unwrap();
assert_eq!(last["role"], "system");
assert_eq!(last["content"], "[pulse] 4m elapsed");
}
#[test]
fn mid_conversation_note_folds_into_the_user_turn_when_unsupported() {
let msgs = vec![
Message::text("user", "hi"),
Message::text("system", "[pulse] 4m elapsed"),
];
let (messages, system, _) = translate_messages(&msgs, false);
assert!(system.is_none());
assert_eq!(messages.len(), 1);
assert_eq!(messages[0]["role"], "user");
let blocks = messages[0]["content"].as_array().unwrap();
assert_eq!(blocks.len(), 2);
assert_eq!(blocks[1]["text"], "[pulse] 4m elapsed");
}
/// A system turn may only sit between a user turn and an assistant turn, or
/// last. One landing anywhere else is demoted rather than sent and 400ed.
#[test]
fn demotes_a_system_turn_that_cannot_sit_where_it_landed() {
let msgs = vec![
Message::text("user", "hi"),
Message::text("assistant", "hello"),
Message::text("system", "[migraine] stop"),
Message::text("user", "still there?"),
];
let (messages, _, _) = translate_messages(&msgs, true);
let roles: Vec<&str> = messages
.iter()
.map(|m| m["role"].as_str().unwrap())
.collect();
assert_eq!(roles, vec!["user", "assistant", "user"]);
let folded = messages[2]["content"].as_array().unwrap();
assert_eq!(folded[0]["text"], "[migraine] stop");
assert_eq!(folded[1]["text"], "still there?");
}
#[test]
fn message_content_is_always_a_block_array() {
let msgs = vec![Message::text("user", "hi")];
let (messages, _, _) = translate_messages(&msgs, true);
assert!(messages[0]["content"].is_array());
}
/// Every request gets at most the four breakpoints the API allows, and they
/// land on the tail of each stable span: tools, system, transcript.
#[test]
fn places_breakpoints_on_tools_system_and_the_transcript_tail() {
let mut body = json!({
"tools": [{ "name": "read" }, { "name": "bash" }],
"system": "you are souvie",
"messages": [
{ "role": "user", "content": [{ "type": "text", "text": "one" }] },
{ "role": "assistant", "content": [{ "type": "text", "text": "two" }] },
{ "role": "user", "content": [{ "type": "text", "text": "three" }] },
],
})
.as_object_mut()
.unwrap()
.clone();
inject_attribution(&mut body, "2.1.196", "one");
mark_cache_breakpoints(&mut body);
let marked = |v: &Value| v.get("cache_control").is_some();
let tools = body["tools"].as_array().unwrap();
assert!(!marked(&tools[0]) && marked(&tools[1]));
let system = body["system"].as_array().unwrap();
assert!(!marked(&system[0]) && marked(system.last().unwrap()));
let messages = body["messages"].as_array().unwrap();
let tails: Vec<bool> = messages
.iter()
.map(|m| marked(m["content"].as_array().unwrap().last().unwrap()))
.collect();
assert_eq!(tails, vec![false, true, true]);
let total = serde_json::to_string(&body)
.unwrap()
.matches("cache_control")
.count();
assert_eq!(total, 4);
}
/// A system turn carries a bare string, so it has no block to mark — the
/// breakpoint has to skip past it to the last turn that does.
#[test]
fn breakpoints_skip_a_trailing_system_turn() {
let mut body = json!({
"system": ["you are souvie"],
"messages": [
{ "role": "user", "content": [{ "type": "text", "text": "one" }] },
{ "role": "system", "content": "[pulse] 4m elapsed" },
],
})
.as_object_mut()
.unwrap()
.clone();
mark_cache_breakpoints(&mut body);
let messages = body["messages"].as_array().unwrap();
assert!(messages[1].get("cache_control").is_none());
assert!(messages[0]["content"].as_array().unwrap()[0]
.get("cache_control")
.is_some());
}
#[test]
fn usage_counts_the_cached_prefix_as_prompt_tokens() {
let raw = r#"{
"id":"msg_3","stop_reason":"end_turn",
"content":[{"type":"text","text":"pong"}],
"usage":{
"input_tokens":12,
"cache_read_input_tokens":9000,
"cache_creation_input_tokens":300,
"output_tokens":4
}
}"#;
let usage = parse_completion(raw).unwrap().usage.unwrap();
assert_eq!(usage.prompt_tokens, 9312);
assert_eq!(usage.cache_read_tokens, 9000);
assert_eq!(usage.cache_write_tokens, 300);
assert_eq!(usage.total_tokens, 9316);
}
#[test]
fn parses_anthropic_text_response() {
let raw = r#"{

1
src/bridge/providers/responses.rs
View file

@ -244,5 +244,6 @@ fn parse_usage(u: &Value) -> Option<Usage> {
prompt_tokens: prompt,
completion_tokens: completion,
total_tokens: prompt + completion,
..Usage::default()
})
}