feat: Implement cache management for Anthropic API with eligibility tracking

gh-4325: Enhance cache management for Anthropic API by introudicing per-message TTL and configurable content block usage optimization.
Signed-off-by: Austin Dase <[email protected]>

Author: adase11

models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java

@@ -42,7 +42,9 @@
 import org.springframework.ai.anthropic.api.AnthropicApi.ContentBlock.Source;
 import org.springframework.ai.anthropic.api.AnthropicApi.ContentBlock.Type;
 import org.springframework.ai.anthropic.api.AnthropicApi.Role;
-import org.springframework.ai.anthropic.api.AnthropicCacheStrategy;
+import org.springframework.ai.anthropic.api.AnthropicCacheOptions;
+import org.springframework.ai.anthropic.api.AnthropicCacheTtl;
+import org.springframework.ai.anthropic.api.utils.CacheEligibilityResolver;
 import org.springframework.ai.chat.messages.AssistantMessage;
 import org.springframework.ai.chat.messages.Message;
 import org.springframework.ai.chat.messages.MessageType;
@@ -93,6 +95,7 @@
  * @author Alexandros Pappas
  * @author Jonghoon Park
  * @author Soby Chacko
+ * @author Austin Dase
  * @since 1.0.0
  */
 public class AnthropicChatModel implements ChatModel {
@@ -463,11 +466,9 @@ Prompt buildRequestPrompt(Prompt prompt) {
 			requestOptions.setToolContext(ToolCallingChatOptions.mergeToolContext(runtimeOptions.getToolContext(),
 					this.defaultOptions.getToolContext()));
 
-			// Merge cache strategy and TTL (also @JsonIgnore fields)
-			requestOptions.setCacheStrategy(runtimeOptions.getCacheStrategy() != null
-					? runtimeOptions.getCacheStrategy() : this.defaultOptions.getCacheStrategy());
-			requestOptions.setCacheTtl(runtimeOptions.getCacheTtl() != null ? runtimeOptions.getCacheTtl()
-					: this.defaultOptions.getCacheTtl());
+			// Merge cache options that are Json-ignored
+			requestOptions.setCacheOptions(runtimeOptions.getCacheOptions() != null ? runtimeOptions.getCacheOptions()
+					: this.defaultOptions.getCacheOptions());
 		}
 		else {
 			requestOptions.setHttpHeaders(this.defaultOptions.getHttpHeaders());
@@ -498,41 +499,23 @@ ChatCompletionRequest createRequest(Prompt prompt, boolean stream) {
 		AnthropicChatOptions requestOptions = null;
 		if (prompt.getOptions() instanceof AnthropicChatOptions) {
 			requestOptions = (AnthropicChatOptions) prompt.getOptions();
-			logger.debug("DEBUGINFO: Found AnthropicChatOptions - cacheStrategy: {}, cacheTtl: {}",
-					requestOptions.getCacheStrategy(), requestOptions.getCacheTtl());
+			logger.debug("DEBUGINFO: Found AnthropicChatOptions - cacheOptions {}", requestOptions.getCacheOptions());
 		}
 		else {
 			logger.debug("DEBUGINFO: Options is NOT AnthropicChatOptions, it's: {}",
 					prompt.getOptions() != null ? prompt.getOptions().getClass().getName() : "null");
 		}
 
-		AnthropicCacheStrategy strategy = requestOptions != null ? requestOptions.getCacheStrategy()
-				: AnthropicCacheStrategy.NONE;
-		String cacheTtl = requestOptions != null ? requestOptions.getCacheTtl() : "5m";
+		AnthropicCacheOptions cacheOptions = requestOptions != null ? requestOptions.getCacheOptions()
+				: AnthropicCacheOptions.DISABLED;
 
-		logger.debug("Cache strategy: {}, TTL: {}", strategy, cacheTtl);
+		CacheEligibilityResolver cacheEligibilityResolver = CacheEligibilityResolver.from(cacheOptions);
 
-		// Track how many breakpoints we've used (max 4)
-		CacheBreakpointTracker breakpointsUsed = new CacheBreakpointTracker();
-		ChatCompletionRequest.CacheControl cacheControl = null;
-
-		if (strategy != AnthropicCacheStrategy.NONE) {
-			// Create cache control with TTL if specified, otherwise use default 5m
-			if (cacheTtl != null && !cacheTtl.equals("5m")) {
-				cacheControl = new ChatCompletionRequest.CacheControl("ephemeral", cacheTtl);
-				logger.debug("Created cache control with TTL: type={}, ttl={}", "ephemeral", cacheTtl);
-			}
-			else {
-				cacheControl = new ChatCompletionRequest.CacheControl("ephemeral");
-				logger.debug("Created cache control with default TTL: type={}, ttl={}", "ephemeral", "5m");
-			}
-		}
+		// Process system - as array if caching, string otherwise
+		Object systemContent = buildSystemContent(prompt, cacheEligibilityResolver);
 
 		// Build messages WITHOUT blanket cache control - strategic placement only
-		List<AnthropicMessage> userMessages = buildMessages(prompt, strategy, cacheControl, breakpointsUsed);
-
-		// Process system - as array if caching, string otherwise
-		Object systemContent = buildSystemContent(prompt, strategy, cacheControl, breakpointsUsed);
+		List<AnthropicMessage> userMessages = buildMessages(prompt, cacheEligibilityResolver);
 
 		// Build base request
 		ChatCompletionRequest request = new ChatCompletionRequest(this.defaultOptions.getModel(), userMessages,
@@ -547,16 +530,13 @@ ChatCompletionRequest createRequest(Prompt prompt, boolean stream) {
 			List<AnthropicApi.Tool> tools = getFunctionTools(toolDefinitions);
 
 			// Apply caching to tools if strategy includes them
-			if ((strategy == AnthropicCacheStrategy.SYSTEM_AND_TOOLS
-					|| strategy == AnthropicCacheStrategy.CONVERSATION_HISTORY) && breakpointsUsed.canUse()) {
-				tools = addCacheToLastTool(tools, cacheControl, breakpointsUsed);
-			}
+			tools = addCacheToLastTool(tools, cacheEligibilityResolver);
 
 			request = ChatCompletionRequest.from(request).tools(tools).build();
 		}
 
 		// Add beta header for 1-hour TTL if needed
-		if ("1h".equals(cacheTtl) && requestOptions != null) {
+		if (cacheOptions.getMessageTypeTtl().containsValue(AnthropicCacheTtl.ONE_HOUR)) {
 			Map<String, String> headers = new HashMap<>(requestOptions.getHttpHeaders());
 			headers.put("anthropic-beta", AnthropicApi.BETA_EXTENDED_CACHE_TTL);
 			requestOptions.setHttpHeaders(headers);
@@ -565,6 +545,25 @@ ChatCompletionRequest createRequest(Prompt prompt, boolean stream) {
 		return request;
 	}
 
+	private static ContentBlock cacheAwareContentBlock(ContentBlock contentBlock, MessageType messageType,
+			CacheEligibilityResolver cacheEligibilityResolver) {
+		String basisForLength = switch (contentBlock.type()) {
+			case TEXT, TEXT_DELTA -> contentBlock.text();
+			case TOOL_RESULT -> contentBlock.content();
+			case TOOL_USE -> JsonParser.toJson(contentBlock.input());
+			case THINKING, THINKING_DELTA -> contentBlock.thinking();
+			case REDACTED_THINKING -> contentBlock.data();
+			default -> null;
+		};
+
+		ChatCompletionRequest.CacheControl cacheControl = cacheEligibilityResolver.resolve(messageType, basisForLength);
+		if (cacheControl == null) {
+			return contentBlock;
+		}
+		cacheEligibilityResolver.useCacheBlock();
+		return ContentBlock.from(contentBlock).cacheControl(cacheControl).build();
+	}
+
 	private List<AnthropicApi.Tool> getFunctionTools(List<ToolDefinition> toolDefinitions) {
 		return toolDefinitions.stream().map(toolDefinition -> {
 			var name = toolDefinition.name();
@@ -579,8 +578,7 @@ private List<AnthropicApi.Tool> getFunctionTools(List<ToolDefinition> toolDefini
 	 * Build messages strategically, applying cache control only where specified by the
 	 * strategy.
 	 */
-	private List<AnthropicMessage> buildMessages(Prompt prompt, AnthropicCacheStrategy strategy,
-			ChatCompletionRequest.CacheControl cacheControl, CacheBreakpointTracker breakpointsUsed) {
+	private List<AnthropicMessage> buildMessages(Prompt prompt, CacheEligibilityResolver cacheEligibilityResolver) {
 
 		List<Message> allMessages = prompt.getInstructions()
 			.stream()
@@ -589,7 +587,7 @@ private List<AnthropicMessage> buildMessages(Prompt prompt, AnthropicCacheStrate
 
 		// Find the last user message (current question) for CONVERSATION_HISTORY strategy
 		int lastUserIndex = -1;
-		if (strategy == AnthropicCacheStrategy.CONVERSATION_HISTORY) {
+		if (cacheEligibilityResolver.isCachingEnabled()) {
 			for (int i = allMessages.size() - 1; i >= 0; i--) {
 				if (allMessages.get(i).getMessageType() == MessageType.USER) {
 					lastUserIndex = i;
@@ -601,30 +599,18 @@ private List<AnthropicMessage> buildMessages(Prompt prompt, AnthropicCacheStrate
 		List<AnthropicMessage> result = new ArrayList<>();
 		for (int i = 0; i < allMessages.size(); i++) {
 			Message message = allMessages.get(i);
-			boolean shouldApplyCache = false;
-
-			// Apply cache to history tail (message before current question) for
-			// CONVERSATION_HISTORY
-			if (strategy == AnthropicCacheStrategy.CONVERSATION_HISTORY && breakpointsUsed.canUse()) {
-				if (lastUserIndex > 0) {
-					// Cache the message immediately before the last user message
-					// (multi-turn conversation)
-					shouldApplyCache = (i == lastUserIndex - 1);
-				}
-				if (shouldApplyCache) {
-					breakpointsUsed.use();
-				}
-			}
-
-			if (message.getMessageType() == MessageType.USER) {
-				List<ContentBlock> contents = new ArrayList<>();
-
-				// Apply cache control strategically, not to all user messages
-				if (shouldApplyCache && cacheControl != null) {
-					contents.add(new ContentBlock(message.getText(), cacheControl));
+			MessageType messageType = message.getMessageType();
+			if (messageType == MessageType.USER) {
+				List<ContentBlock> contentBlocks = new ArrayList<>();
+				String content = message.getText();
+				boolean isLastUserMessage = lastUserIndex == i;
+				ContentBlock contentBlock = new ContentBlock(content);
+				if (isLastUserMessage) {
+					// Never cache the latest user message
+					contentBlocks.add(contentBlock);
 				}
 				else {
-					contents.add(new ContentBlock(message.getText()));
+					contentBlocks.add(cacheAwareContentBlock(contentBlock, messageType, cacheEligibilityResolver));
 				}
 
 				if (message instanceof UserMessage userMessage) {
@@ -634,30 +620,33 @@ private List<AnthropicMessage> buildMessages(Prompt prompt, AnthropicCacheStrate
 							var source = getSourceByMedia(media);
 							return new ContentBlock(contentBlockType, source);
 						}).toList();
-						contents.addAll(mediaContent);
+						contentBlocks.addAll(mediaContent);
 					}
 				}
-				result.add(new AnthropicMessage(contents, Role.valueOf(message.getMessageType().name())));
+				result.add(new AnthropicMessage(contentBlocks, Role.valueOf(message.getMessageType().name())));
 			}
-			else if (message.getMessageType() == MessageType.ASSISTANT) {
+			else if (messageType == MessageType.ASSISTANT) {
 				AssistantMessage assistantMessage = (AssistantMessage) message;
 				List<ContentBlock> contentBlocks = new ArrayList<>();
 				if (StringUtils.hasText(message.getText())) {
-					contentBlocks.add(new ContentBlock(message.getText()));
+					ContentBlock contentBlock = new ContentBlock(message.getText());
+					contentBlocks.add(cacheAwareContentBlock(contentBlock, messageType, cacheEligibilityResolver));
 				}
 				if (!CollectionUtils.isEmpty(assistantMessage.getToolCalls())) {
 					for (AssistantMessage.ToolCall toolCall : assistantMessage.getToolCalls()) {
-						contentBlocks.add(new ContentBlock(Type.TOOL_USE, toolCall.id(), toolCall.name(),
-								ModelOptionsUtils.jsonToMap(toolCall.arguments())));
+						ContentBlock contentBlock = new ContentBlock(Type.TOOL_USE, toolCall.id(), toolCall.name(),
+								ModelOptionsUtils.jsonToMap(toolCall.arguments()));
+						contentBlocks.add(cacheAwareContentBlock(contentBlock, messageType, cacheEligibilityResolver));
 					}
 				}
 				result.add(new AnthropicMessage(contentBlocks, Role.ASSISTANT));
 			}
-			else if (message.getMessageType() == MessageType.TOOL) {
+			else if (messageType == MessageType.TOOL) {
 				List<ContentBlock> toolResponses = ((ToolResponseMessage) message).getResponses()
 					.stream()
 					.map(toolResponse -> new ContentBlock(Type.TOOL_RESULT, toolResponse.id(),
 							toolResponse.responseData()))
+					.map(contentBlock -> cacheAwareContentBlock(contentBlock, messageType, cacheEligibilityResolver))
 					.toList();
 				result.add(new AnthropicMessage(toolResponses, Role.USER));
 			}
@@ -671,8 +660,7 @@ else if (message.getMessageType() == MessageType.TOOL) {
 	/**
 	 * Build system content - as array if caching, string otherwise.
 	 */
-	private Object buildSystemContent(Prompt prompt, AnthropicCacheStrategy strategy,
-			ChatCompletionRequest.CacheControl cacheControl, CacheBreakpointTracker breakpointsUsed) {
+	private Object buildSystemContent(Prompt prompt, CacheEligibilityResolver cacheEligibilityResolver) {
 
 		String systemText = prompt.getInstructions()
 			.stream()
@@ -685,15 +673,9 @@ private Object buildSystemContent(Prompt prompt, AnthropicCacheStrategy strategy
 		}
 
 		// Use array format when caching system
-		if ((strategy == AnthropicCacheStrategy.SYSTEM_ONLY || strategy == AnthropicCacheStrategy.SYSTEM_AND_TOOLS
-				|| strategy == AnthropicCacheStrategy.CONVERSATION_HISTORY) && breakpointsUsed.canUse()
-				&& cacheControl != null) {
-
-			logger.debug("Applying cache control to system message - strategy: {}, cacheControl: {}", strategy,
-					cacheControl);
-			List<ContentBlock> systemBlocks = List.of(new ContentBlock(systemText, cacheControl));
-			breakpointsUsed.use();
-			return systemBlocks;
+		if (cacheEligibilityResolver.isCachingEnabled()) {
+			return List
+				.of(cacheAwareContentBlock(new ContentBlock(systemText), MessageType.SYSTEM, cacheEligibilityResolver));
 		}
 
 		// Use string format when not caching (backward compatible)
@@ -704,9 +686,11 @@ private Object buildSystemContent(Prompt prompt, AnthropicCacheStrategy strategy
 	 * Add cache control to the last tool for deterministic caching.
 	 */
 	private List<AnthropicApi.Tool> addCacheToLastTool(List<AnthropicApi.Tool> tools,
-			ChatCompletionRequest.CacheControl cacheControl, CacheBreakpointTracker breakpointsUsed) {
+			CacheEligibilityResolver cacheEligibilityResolver) {
+
+		ChatCompletionRequest.CacheControl cacheControl = cacheEligibilityResolver.resolveToolCacheControl();
 
-		if (tools == null || tools.isEmpty() || !breakpointsUsed.canUse() || cacheControl == null) {
+		if (cacheControl == null || tools == null || tools.isEmpty()) {
 			return tools;
 		}
 
@@ -716,7 +700,7 @@ private List<AnthropicApi.Tool> addCacheToLastTool(List<AnthropicApi.Tool> tools
 			if (i == tools.size() - 1) {
 				// Add cache control to last tool
 				tool = new AnthropicApi.Tool(tool.name(), tool.description(), tool.inputSchema(), cacheControl);
-				breakpointsUsed.use();
+				cacheEligibilityResolver.useCacheBlock();
 			}
 			modifiedTools.add(tool);
 		}
@@ -804,36 +788,4 @@ public AnthropicChatModel build() {
 
 	}
 
-	/**
-	 * Tracks cache breakpoints used (max 4 allowed by Anthropic). Non-static to ensure
-	 * each request has its own instance.
-	 */
-	private class CacheBreakpointTracker {
-
-		private int count = 0;
-
-		private boolean hasWarned = false;
-
-		public boolean canUse() {
-			return this.count < 4;
-		}
-
-		public void use() {
-			if (this.count < 4) {
-				this.count++;
-			}
-			else if (!this.hasWarned) {
-				logger.warn(
-						"Anthropic cache breakpoint limit (4) reached. Additional cache_control directives will be ignored. "
-								+ "Consider using fewer cache strategies or simpler content structure.");
-				this.hasWarned = true;
-			}
-		}
-
-		public int getCount() {
-			return this.count;
-		}
-
-	}
-
 }