Merge pull request #201 from krissetto/dont-continue-loop-after-max-tokens

Stop request loop when going over max_tokens

Author: dgageot

pkg/model/provider/dmr/client.go

@@ -186,9 +186,15 @@ func mergeRuntimeFlagsPreferUser(derived, user []string) (out, warnings []string
 }
 
 func convertMessages(messages []chat.Message) []openai.ChatCompletionMessage {
-	openaiMessages := make([]openai.ChatCompletionMessage, len(messages))
+	openaiMessages := make([]openai.ChatCompletionMessage, 0, len(messages))
 	for i := range messages {
 		msg := &messages[i]
+
+		// Skip invalid assistant messages upfront. This can happen if the model is out of tokens (max_tokens reached)
+		if msg.Role == chat.MessageRoleAssistant && len(msg.ToolCalls) == 0 && len(msg.MultiContent) == 0 && strings.TrimSpace(msg.Content) == "" {
+			continue
+		}
+
 		openaiMessage := openai.ChatCompletionMessage{
 			Role: string(msg.Role),
 			Name: msg.Name,
@@ -225,7 +231,7 @@ func convertMessages(messages []chat.Message) []openai.ChatCompletionMessage {
 			openaiMessage.ToolCallID = msg.ToolCallID
 		}
 
-		openaiMessages[i] = openaiMessage
+		openaiMessages = append(openaiMessages, openaiMessage)
 	}
 
 	var mergedMessages []openai.ChatCompletionMessage

pkg/model/provider/oaistream/adapter.go

@@ -66,7 +66,7 @@ func (a *StreamAdapter) Recv() (chat.MessageStreamResponse, error) {
 	// Convert the choices
 	for i := range openaiResponse.Choices {
 		choice := &openaiResponse.Choices[i]
-		if a.trackUsage && choice.FinishReason == openai.FinishReasonStop {
+		if a.trackUsage && (choice.FinishReason == openai.FinishReasonStop || choice.FinishReason == openai.FinishReasonLength) {
 			choice.FinishReason = openai.FinishReasonNull
 		}
 

pkg/model/provider/openai/client.go

@@ -139,9 +139,15 @@ func convertMultiContent(multiContent []chat.MessagePart) []openai.ChatMessagePa
 
 // convertMessages converts chat.ChatCompletionMessage to openai.ChatCompletionMessage
 func convertMessages(messages []chat.Message) []openai.ChatCompletionMessage {
-	openaiMessages := make([]openai.ChatCompletionMessage, len(messages))
+	openaiMessages := make([]openai.ChatCompletionMessage, 0, len(messages))
 	for i := range messages {
 		msg := &messages[i]
+
+		// Skip invalid assistant messages upfront. This can happen if the model is out of tokens (max_tokens reached)
+		if msg.Role == chat.MessageRoleAssistant && len(msg.ToolCalls) == 0 && len(msg.MultiContent) == 0 && strings.TrimSpace(msg.Content) == "" {
+			continue
+		}
+
 		openaiMessage := openai.ChatCompletionMessage{
 			Role: string(msg.Role),
 			Name: msg.Name,
@@ -178,7 +184,7 @@ func convertMessages(messages []chat.Message) []openai.ChatCompletionMessage {
 			openaiMessage.ToolCallID = msg.ToolCallID
 		}
 
-		openaiMessages[i] = openaiMessage
+		openaiMessages = append(openaiMessages, openaiMessage)
 	}
 	return openaiMessages
 }

pkg/runtime/runtime.go

@@ -246,17 +246,22 @@ func (r *runtime) RunStream(ctx context.Context, sess *session.Session) <-chan E
 			streamSpan.End()
 			slog.Debug("Stream processed", "agent", a.Name(), "tool_calls", len(calls), "content_length", len(content), "stopped", stopped)
 
-			// Add assistant message to conversation history
-			assistantMessage := chat.Message{
-				Role:             chat.MessageRoleAssistant,
-				Content:          content,
-				ReasoningContent: reasoningContent,
-				ToolCalls:        calls,
-				CreatedAt:        time.Now().Format(time.RFC3339),
-			}
+			// Add assistant message to conversation history, but skip empty assistant messages
+			// Providers reject assistant messages that have neither content nor tool calls.
+			if strings.TrimSpace(content) != "" || len(calls) > 0 {
+				assistantMessage := chat.Message{
+					Role:             chat.MessageRoleAssistant,
+					Content:          content,
+					ReasoningContent: reasoningContent,
+					ToolCalls:        calls,
+					CreatedAt:        time.Now().Format(time.RFC3339),
+				}
 
-			sess.AddMessage(session.NewAgentMessage(a, &assistantMessage))
-			slog.Debug("Added assistant message to session", "agent", a.Name(), "total_messages", len(sess.GetAllMessages()))
+				sess.AddMessage(session.NewAgentMessage(a, &assistantMessage))
+				slog.Debug("Added assistant message to session", "agent", a.Name(), "total_messages", len(sess.GetAllMessages()))
+			} else {
+				slog.Debug("Skipping empty assistant message (no content and no tool calls)", "agent", a.Name())
+			}
 
 			contextLimit := 0
 			if m != nil {
@@ -371,7 +376,7 @@ func (r *runtime) handleStream(ctx context.Context, stream chat.MessageStream, a
 			continue
 		}
 		choice := response.Choices[0]
-		if choice.FinishReason == chat.FinishReasonStop {
+		if choice.FinishReason == chat.FinishReasonStop || choice.FinishReason == chat.FinishReasonLength {
 			return toolCalls, fullContent.String(), fullReasoningContent.String(), true, nil
 		}
 
@@ -438,7 +443,10 @@ func (r *runtime) handleStream(ctx context.Context, stream chat.MessageStream, a
 		}
 	}
 
-	return toolCalls, fullContent.String(), fullReasoningContent.String(), false, nil
+	// If the stream completed without producing any content or tool calls, likely because of a token limit, stop to avoid breaking the request loop
+	// NOTE(krissetto): this can likely be removed once compaction works properly with all providers (aka dmr)
+	stoppedDueToNoOutput := fullContent.Len() == 0 && len(toolCalls) == 0
+	return toolCalls, fullContent.String(), fullReasoningContent.String(), stoppedDueToNoOutput, nil
 }
 
 // processToolCalls handles the execution of tool calls for an agent