fix(ai): keep multimodal parts structured in otel gen_ai.input.messages (#1527)

captureContent flattened image/audio/video/document parts to '[image]'
placeholders, dropping the media reference from traces. Array content now
serializes to OTel GenAI semconv parts: text (redacted), uri for URL
sources, file for provider handles. Inline data and data: URLs keep the
placeholder. Span events stay flat strings.

Closes #1525

Author: tombeckenham

.changeset/otel-structured-multimodal-parts.md

@@ -0,0 +1,5 @@
+---
+'@tanstack/ai': patch
+---
+
+`otelMiddleware` with `captureContent: true` now keeps multimodal parts structured in `gen_ai.input.messages` (OTel GenAI part shapes). URL media becomes a `uri` part and provider file handles become a `file` part, so traces show what the model looked at. Inline base64 data still records a `[image]`-style placeholder.

docs/advanced/otel.md

@@ -142,7 +142,14 @@ If `redact` throws, the middleware writes the literal sentinel `"[redaction_fail
 
 Accumulated assistant text (the `gen_ai.choice` event) is capped at `maxContentLength` characters (default `100 000`); longer completions are truncated with a trailing `"…"` marker.
 
-Multimodal content (images, audio, video, documents) is represented as placeholder strings (`[image]`, `[audio]`, ...) to preserve message order without dumping binary data onto spans. Use `onSpanEnd` if you need richer multimodal capture.
+Multimodal messages (images, audio, video, documents) keep their parts in `gen_ai.input.messages`, in the OTel GenAI part shapes:
+
+- Text: `{ "type": "text", "content": "..." }`. `redact` runs on it.
+- URL source: `{ "type": "uri", "modality": "image", "uri": "https://...", "mime_type": "image/png" }`.
+- Provider file handle: `{ "type": "file", "modality": "image", "file_id": "..." }`.
+- Inline base64 data or a `data:` URL: a `[image]` text placeholder, so the bytes do not go onto the span.
+
+Span events stay flat strings, with the same placeholders for every media part (`look at this [image]`).
 
 Prompt/system/user message events fire from `onConfig` at the start of every iteration, which means the full conversation history (as the adapter will re-send it) is re-emitted on each iteration span. This mirrors what the provider actually sees on the wire.
 

packages/ai/src/middlewares/otel.ts

@@ -221,6 +221,51 @@ function serializeContent(content: unknown): string {
   return parts.join(' ')
 }
 
+type InputPart =
+  | { type: 'text'; content: string }
+  | { type: 'uri'; modality: string; uri: string; mime_type?: string }
+  | { type: 'file'; modality: string; file_id: string; mime_type?: string }
+
+/**
+ * Structured form of `ContentPart[]` for `gen_ai.input.messages`, using the
+ * OTel GenAI semconv part shapes. URL and file-handle media keep their
+ * reference; inline bytes (and `data:` URLs) stay a `[type]` placeholder so
+ * they never blow attribute size limits. `redact` runs on text parts only.
+ */
+function serializeParts(
+  content: Array<unknown>,
+  redact: (text: string) => string,
+): Array<InputPart> {
+  const parts: Array<InputPart> = []
+  for (const part of content) {
+    if (!part || typeof part !== 'object') continue
+    const p = part as {
+      type?: string
+      text?: string
+      content?: string
+      source?: { type?: string; value?: string; mimeType?: string }
+    }
+    if (p.type === 'text') {
+      parts.push({
+        type: 'text',
+        content: redact((p.text ?? p.content ?? '').toString()),
+      })
+      continue
+    }
+    const modality = p.type ?? 'unknown'
+    const { type, value, mimeType } = p.source ?? {}
+    const mime = mimeType ? { mime_type: mimeType } : {}
+    if (type === 'url' && value && !value.startsWith('data:')) {
+      parts.push({ type: 'uri', modality, uri: value, ...mime })
+    } else if (type === 'file' && value) {
+      parts.push({ type: 'file', modality, file_id: value, ...mime })
+    } else {
+      parts.push({ type: 'text', content: `[${modality}]` })
+    }
+  }
+  return parts
+}
+
 function messageEventName(role: string): string {
   switch (role) {
     case 'user':
@@ -581,14 +626,25 @@ export function otelMiddleware(
           // Also emit the current GenAI-semconv attribute form
           // (`gen_ai.input.messages`) — backends like PostHog read prompt
           // content from this attribute, not from span events.
-          const inputMessages: Array<{ role: string; content: string }> = []
+          // Multimodal messages keep their parts structured so image / audio /
+          // video / document references survive into the trace (#1525).
+          const inputMessages: Array<{
+            role: string
+            content: string | Array<InputPart>
+          }> = []
           for (const sys of systemPromptContents) {
             inputMessages.push({
               role: 'system',
               content: redactContent(sys),
             })
           }
           for (const m of config.messages) {
+            if (Array.isArray(m.content)) {
+              const parts = serializeParts(m.content, redactContent)
+              if (parts.length === 0) continue
+              inputMessages.push({ role: m.role, content: parts })
+              continue
+            }
             const body = serializeContent(m.content)
             if (body.length === 0) continue
             inputMessages.push({

packages/ai/tests/middlewares/otel.test.ts

@@ -965,6 +965,64 @@ describe('otelMiddleware — captureContent', () => {
     expect(userEvt.attributes!['content']).toBe('look at this [image]')
   })
 
+  it('keeps multimodal parts structured in gen_ai.input.messages', async () => {
+    const { tracer, spans } = createFakeTracer()
+    const mw = otelMiddleware({
+      tracer,
+      captureContent: true,
+      redact: (s) => s.replace('secret', '***'),
+    })
+    const ctx = makeCtx()
+
+    await runToIterationStart(mw, ctx, {
+      messages: [
+        {
+          role: 'user',
+          content: [
+            { type: 'text', content: 'describe secret' },
+            {
+              type: 'image',
+              source: {
+                type: 'url',
+                value: 'https://x.test/a.png',
+                mimeType: 'image/png',
+              },
+            },
+            { type: 'audio', source: { type: 'file', value: 'file_123' } },
+            {
+              type: 'video',
+              source: { type: 'data', value: 'AAAA', mimeType: 'video/mp4' },
+            },
+            {
+              type: 'image',
+              source: { type: 'url', value: 'data:image/png;base64,AAAA' },
+            },
+          ],
+        },
+      ],
+    })
+
+    expect(
+      JSON.parse(spans[1]!.attributes['gen_ai.input.messages'] as string),
+    ).toEqual([
+      {
+        role: 'user',
+        content: [
+          { type: 'text', content: 'describe ***' },
+          {
+            type: 'uri',
+            modality: 'image',
+            uri: 'https://x.test/a.png',
+            mime_type: 'image/png',
+          },
+          { type: 'file', modality: 'audio', file_id: 'file_123' },
+          { type: 'text', content: '[video]' },
+          { type: 'text', content: '[image]' },
+        ],
+      },
+    ])
+  })
+
   it('emits redaction sentinel and never raw content when redact throws', async () => {
     const { tracer, spans } = createFakeTracer()
     const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})

testing/e2e/src/routes/api.otel-usage.ts

@@ -25,6 +25,9 @@ const weatherTool = toolDefinition({
  *   `completion_tokens_details.reasoning_tokens`.
  * - `provider: 'openrouter'` → `/openrouter-cost` mount, whose trailing usage
  *   chunk carries `cost` / `cost_details`.
+ * - `provider: 'multimodal'` → the `/openai-usage-details` mount with an
+ *   image part and `captureContent: true`, so the spec can check the image
+ *   URL survives into `gen_ai.input.messages` (#1525).
  *
  * The spec asserts the corresponding `gen_ai.usage.*` / `tanstack.ai.usage.*`
  * attributes land on the iteration and root spans.
@@ -63,14 +66,34 @@ export const Route = createFileRoute('/api/otel-usage')({
           for await (const _chunk of chat({
             ...createChatOptions({ adapter }),
             messages: [
-              {
-                role: 'user',
-                content:
-                  provider === 'tool-loop' ? '[with-tool] run test' : 'hi',
-              },
+              provider === 'multimodal'
+                ? {
+                    role: 'user',
+                    content: [
+                      { type: 'text', content: 'describe this' },
+                      {
+                        type: 'image',
+                        source: {
+                          type: 'url',
+                          value: 'https://example.com/cat.png',
+                          mimeType: 'image/png',
+                        },
+                      },
+                    ],
+                  }
+                : {
+                    role: 'user',
+                    content:
+                      provider === 'tool-loop' ? '[with-tool] run test' : 'hi',
+                  },
             ],
             ...(provider === 'tool-loop' ? { tools: [weatherTool] } : {}),
-            middleware: [otelMiddleware({ tracer })],
+            middleware: [
+              otelMiddleware({
+                tracer,
+                captureContent: provider === 'multimodal',
+              }),
+            ],
           })) {
             // Drain — the assertions live on the captured spans.
           }

testing/e2e/tests/middleware.spec.ts

@@ -413,6 +413,38 @@ test.describe('Middleware Lifecycle', () => {
     })
   })
 
+  test('otel middleware keeps image parts structured in gen_ai.input.messages', async ({
+    request,
+  }) => {
+    // #1525: captureContent used to flatten image parts to "[image]". The
+    // URL reference must survive as an OTel semconv `uri` part.
+    const res = await request.post('/api/otel-usage', {
+      data: { provider: 'multimodal' },
+    })
+    expect(res.ok()).toBe(true)
+    const { ok, error, spans } = await res.json()
+    expect(error ?? null).toBeNull()
+    expect(ok).toBe(true)
+
+    const iterationSpan = spans.find((s: any) => s.kind === SpanKind.CLIENT)
+    expect(
+      JSON.parse(iterationSpan.attributes['gen_ai.input.messages']),
+    ).toEqual([
+      {
+        role: 'user',
+        content: [
+          { type: 'text', content: 'describe this' },
+          {
+            type: 'uri',
+            modality: 'image',
+            uri: 'https://example.com/cat.png',
+            mime_type: 'image/png',
+          },
+        ],
+      },
+    ])
+  })
+
   test('otel middleware emits provider-reported cost on spans', async ({
     request,
   }) => {