Skip to content

fix(desktop): enable Anthropic prompt caching for macOS chat #7951

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Git-on-my-level wants to merge 2 commits into
base: main
Choose a base branch
from
+158 −5
Open

fix(desktop): enable Anthropic prompt caching for macOS chat #7951

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
de2491f
fix(desktop): enable Anthropic prompt caching for macOS chat
Codex-Agent Jun 14, 2026
311431a
fix: skip caching empty/whitespace system prompts (Codex P2)
Codex-Agent Jun 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion desktop/macos/Backend-Rust/src/models/chat_completions.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ pub struct AnthropicRequest {
pub max_tokens: u64,
pub messages: Vec<AnthropicMessage>,
#[serde(skip_serializing_if = "Option::is_none")]
pub system: Option<String>,
pub system: Option<Vec<AnthropicSystemContentBlock>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub temperature: Option<f64>,
pub stream: bool,
Expand All @@ -180,6 +180,20 @@ pub struct AnthropicMessage {
pub content: serde_json::Value,
}

#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct AnthropicSystemContentBlock {
#[serde(rename = "type")]
pub block_type: String,
pub text: String,
pub cache_control: AnthropicCacheControl,
}

#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct AnthropicCacheControl {
#[serde(rename = "type")]
pub cache_type: String,
}
Comment on lines +183 to +195

@greptile-apps greptile-apps Bot Jun 14, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 The block_type and cache_type fields are plain String, so a typo (e.g. "Ephemeral" or "Text") compiles cleanly but produces a 400 from Anthropic at runtime. Since these fields are discriminants with a fixed, known set of valid values, typed enums would catch mistakes at compile time with zero runtime cost.

Suggested change
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct AnthropicSystemContentBlock {
#[serde(rename = "type")]
pub block_type: String,
pub text: String,
pub cache_control: AnthropicCacheControl,
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct AnthropicCacheControl {
#[serde(rename = "type")]
pub cache_type: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum AnthropicContentBlockType {
Text,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum AnthropicCacheControlType {
Ephemeral,
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct AnthropicSystemContentBlock {
#[serde(rename = "type")]
pub block_type: AnthropicContentBlockType,
pub text: String,
pub cache_control: AnthropicCacheControl,
}
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct AnthropicCacheControl {
#[serde(rename = "type")]
pub cache_type: AnthropicCacheControlType,
}


#[derive(Debug, Clone, Serialize)]
pub struct AnthropicTool {
pub name: String,
Expand Down
147 changes: 143 additions & 4 deletions desktop/macos/Backend-Rust/src/routes/chat_completions.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,20 @@ fn translate_request(
model: upstream_model.to_string(),
max_tokens,
messages: anthropic_messages,
system: system_prompt,
system: system_prompt.and_then(|text| {
let text = text.trim().to_string();
if text.is_empty() {
None
} else {
Some(vec![AnthropicSystemContentBlock {
block_type: "text".to_string(),
text,
cache_control: AnthropicCacheControl {
cache_type: "ephemeral".to_string(),
},
}])
}
}),
temperature: req.temperature,
stream: req.stream,
tools: if is_tool_choice_none { None } else { anthropic_tools },
Expand Down Expand Up @@ -1175,7 +1188,16 @@ mod tests {

let result = translate_request(&req, "claude-sonnet-4-6").unwrap();
assert_eq!(result.model, "claude-sonnet-4-6");
assert_eq!(result.system, Some("You are helpful.".to_string()));
assert_eq!(
result.system,
Some(vec![AnthropicSystemContentBlock {
block_type: "text".to_string(),
text: "You are helpful.".to_string(),
cache_control: AnthropicCacheControl {
cache_type: "ephemeral".to_string(),
},
}])
);
assert_eq!(result.messages.len(), 1); // only user message, system extracted
assert_eq!(result.messages[0].role, "user");
assert_eq!(result.max_tokens, 1024);
Expand Down Expand Up @@ -1263,11 +1285,128 @@ mod tests {
};

let result = translate_request(&req, "claude-sonnet-4-6").unwrap();
assert_eq!(result.system, Some("You are terse.".to_string()));
assert_eq!(result.messages.len(), 1, "developer msg must be extracted, not forwarded");
assert_eq!(
result.system,
Some(vec![AnthropicSystemContentBlock {
block_type: "text".to_string(),
text: "You are terse.".to_string(),
cache_control: AnthropicCacheControl {
cache_type: "ephemeral".to_string(),
},
}])
);
assert_eq!(
result.messages.len(),
1,
"developer msg must be extracted, not forwarded"
);
assert_eq!(result.messages[0].role, "user");
}

#[test]
fn test_translate_request_system_prompt_uses_cache_control_blocks() {
let req = ChatCompletionRequest {
model: "omi-sonnet".to_string(),
messages: vec![
ChatMessage {
role: "system".to_string(),
content: Some(json!("You are helpful.")),
name: None,
tool_calls: None,
tool_call_id: None,
},
ChatMessage {
role: "user".to_string(),
content: Some(json!("Hello")),
name: None,
tool_calls: None,
tool_call_id: None,
},
],
stream: false,
temperature: None,
max_tokens: None,
max_completion_tokens: None,
tools: None,
tool_choice: None,
};

let result = translate_request(&req, "claude-sonnet-4-6").unwrap();
let json = serde_json::to_value(&result).unwrap();

assert_eq!(
json["system"],
json!([{
"type": "text",
"text": "You are helpful.",
"cache_control": {"type": "ephemeral"}
}])
);
}

#[test]
fn test_translate_request_without_system_prompt_omits_system() {
let req = ChatCompletionRequest {
model: "omi-sonnet".to_string(),
messages: vec![ChatMessage {
role: "user".to_string(),
content: Some(json!("Hello")),
name: None,
tool_calls: None,
tool_call_id: None,
}],
stream: false,
temperature: None,
max_tokens: None,
max_completion_tokens: None,
tools: None,
tool_choice: None,
};

let result = translate_request(&req, "claude-sonnet-4-6").unwrap();
let json = serde_json::to_value(&result).unwrap();

assert!(result.system.is_none());
assert!(json.get("system").is_none());
}

#[test]
fn test_translate_request_empty_system_prompt_omits_system() {
// Empty or whitespace-only system prompts must NOT be sent as cached blocks
// (Anthropic rejects empty cached text blocks with 400).
for content in [Some(json!("")), Some(json!(" ")), None] {
let req = ChatCompletionRequest {
model: "omi-sonnet".to_string(),
messages: vec![ChatMessage {
role: "system".to_string(),
content: content.clone(),
name: None,
tool_calls: None,
tool_call_id: None,
}, ChatMessage {
role: "user".to_string(),
content: Some(json!("Hello")),
name: None,
tool_calls: None,
tool_call_id: None,
}],
stream: false,
temperature: None,
max_tokens: None,
max_completion_tokens: None,
tools: None,
tool_choice: None,
};

let result = translate_request(&req, "claude-sonnet-4-6").unwrap();
assert!(
result.system.is_none(),
"empty/whitespace system prompt must omit system field, got: {:?}",
result.system
);
}
}

#[test]
fn test_translate_request_max_completion_tokens_preferred() {
// OpenAI renamed `max_tokens` → `max_completion_tokens` for reasoning
Expand Down
Loading