feat: add multi modal tool result support for mcp tool

src/__tests__/mcp.test.ts

@@ -4,6 +4,7 @@ import type { Transport } from '@modelcontextprotocol/sdk/shared/transport.js'
 import { McpClient } from '../mcp.js'
 import { McpTool } from '../tools/mcp-tool.js'
 import { JsonBlock, type TextBlock, type ToolResultBlock } from '../types/messages.js'
+import { ImageBlock } from '../types/media.js'
 import type { LocalAgent } from '../types/agent.js'
 import type { ToolContext } from '../tools/tool.js'
 import { context, propagation, trace, TraceFlags } from '@opentelemetry/api'
@@ -370,5 +371,97 @@ describe('MCP Integration', () => {
       expect(result.status).toBe('error')
       expect((result.content[0] as TextBlock).text).toContain('missing content array')
     })
+
+    it('maps MCP image content to ImageBlock', async () => {
+      // "iVBOR..." is a minimal base64 PNG prefix
+      const base64Data = 'iVBORw0KGgoAAAANSUhEUg=='
+      vi.mocked(mockClientWrapper.callTool).mockResolvedValue({
+        content: [{ type: 'image', data: base64Data, mimeType: 'image/png' }],
+      })
+
+      const result = await runTool<ToolResultBlock>(tool.stream(toolContext))
+
+      expect(result.status).toBe('success')
+      expect(result.content).toHaveLength(1)
+      const imageBlock = result.content[0] as ImageBlock
+      expect(imageBlock).toBeInstanceOf(ImageBlock)
+      expect(imageBlock.format).toBe('png')
+      expect(imageBlock.source.type).toBe('imageSourceBytes')
+    })
+
+    it('falls back to JsonBlock for unsupported image mime type', async () => {
+      vi.mocked(mockClientWrapper.callTool).mockResolvedValue({
+        content: [{ type: 'image', data: 'abc123', mimeType: 'image/bmp' }],
+      })
+
+      const result = await runTool<ToolResultBlock>(tool.stream(toolContext))
+
+      expect(result.content[0]).toBeInstanceOf(JsonBlock)
+    })
+
+    it('falls back to JsonBlock for image content missing data', async () => {
+      vi.mocked(mockClientWrapper.callTool).mockResolvedValue({
+        content: [{ type: 'image', mimeType: 'image/png' }],
+      })
+
+      const result = await runTool<ToolResultBlock>(tool.stream(toolContext))
+
+      expect(result.content[0]).toBeInstanceOf(JsonBlock)
+    })
+
+    it('maps MCP text resource to TextBlock', async () => {
+      vi.mocked(mockClientWrapper.callTool).mockResolvedValue({
+        content: [
+          { type: 'resource', resource: { uri: 'file:///doc.txt', text: 'hello world', mimeType: 'text/plain' } },
+        ],
+      })
+
+      const result = await runTool<ToolResultBlock>(tool.stream(toolContext))
+
+      expect(result.status).toBe('success')
+      expect((result.content[0] as TextBlock).text).toBe('hello world')
+    })
+
+    it('maps MCP blob resource with image mime type to ImageBlock', async () => {
+      const base64Data = 'iVBORw0KGgoAAAANSUhEUg=='
+      vi.mocked(mockClientWrapper.callTool).mockResolvedValue({
+        content: [{ type: 'resource', resource: { uri: 'file:///img.png', blob: base64Data, mimeType: 'image/jpeg' } }],
+      })
+
+      const result = await runTool<ToolResultBlock>(tool.stream(toolContext))
+
+      expect(result.content[0]).toBeInstanceOf(ImageBlock)
+      expect((result.content[0] as ImageBlock).format).toBe('jpeg')
+    })
+
+    it('falls back to JsonBlock for blob resource with non-image mime type', async () => {
+      vi.mocked(mockClientWrapper.callTool).mockResolvedValue({
+        content: [
+          { type: 'resource', resource: { uri: 'file:///doc.pdf', blob: 'abc123', mimeType: 'application/pdf' } },
+        ],
+      })
+
+      const result = await runTool<ToolResultBlock>(tool.stream(toolContext))
+
+      expect(result.content[0]).toBeInstanceOf(JsonBlock)
+    })
+
+    it('handles mixed content types in a single result', async () => {
+      const base64Data = 'iVBORw0KGgoAAAANSUhEUg=='
+      vi.mocked(mockClientWrapper.callTool).mockResolvedValue({
+        content: [
+          { type: 'text', text: 'Here is the image:' },
+          { type: 'image', data: base64Data, mimeType: 'image/png' },
+          { type: 'resource', resource: { uri: 'file:///notes.txt', text: 'Some notes' } },
+        ],
+      })
+
+      const result = await runTool<ToolResultBlock>(tool.stream(toolContext))
+
+      expect(result.content).toHaveLength(3)
+      expect((result.content[0] as TextBlock).text).toBe('Here is the image:')
+      expect(result.content[1]).toBeInstanceOf(ImageBlock)
+      expect((result.content[2] as TextBlock).text).toBe('Some notes')
+    })
   })
 })

src/tools/mcp-tool.ts

@@ -1,8 +1,13 @@
 import { createErrorResult, Tool, type ToolContext, type ToolStreamGenerator } from './tool.js'
 import type { ToolSpec } from './types.js'
 import type { JSONSchema, JSONValue } from '../types/json.js'
-import { JsonBlock, TextBlock, ToolResultBlock } from '../types/messages.js'
+import { JsonBlock, TextBlock, ToolResultBlock, type ToolResultContent } from '../types/messages.js'
+import { ImageBlock } from '../types/media.js'
+import { decodeBase64 } from '../types/media.js'
+import { toMediaFormat } from '../mime.js'
+import type { ImageFormat } from '../mime.js'
 import type { McpClient } from '../mcp.js'
+import { logger } from '../logging/logger.js'
 
 export interface McpToolConfig {
   name: string
@@ -48,13 +53,14 @@ export class McpTool extends Tool {
         throw new Error('Invalid tool result from MCP Client: missing content array')
       }
 
-      const content = rawResult.content.map((item: unknown) => {
-        if (this._isMcpTextContent(item)) {
-          return new TextBlock(item.text)
-        }
+      const content: ToolResultContent[] = []
 
-        return new JsonBlock({ json: item as JSONValue })
-      })
+      for (const item of rawResult.content) {
+        const block = this._mapMcpContent(item)
+        if (block) {
+          content.push(block)
+        }
+      }
 
       if (content.length === 0) {
         content.push(new TextBlock('Tool execution completed successfully with no output.'))
@@ -70,6 +76,100 @@ export class McpTool extends Tool {
     }
   }
 
+  /**
+   * Maps a single MCP content item to an SDK ToolResultContent block.
+   *
+   * @param item - MCP content item from tool result
+   * @returns Mapped content block, or undefined if the content type is unsupported
+   */
+  private _mapMcpContent(item: unknown): ToolResultContent | undefined {
+    if (!item || typeof item !== 'object') {
+      return new JsonBlock({ json: item as JSONValue })
+    }
+
+    const record = item as Record<string, unknown>
+
+    switch (record.type) {
+      case 'text':
+        if (typeof record.text === 'string') {
+          return new TextBlock(record.text)
+        }
+        return new JsonBlock({ json: item as JSONValue })
+
+      case 'image':
+        return this._mapMcpImageContent(record)
+
+      case 'resource':
+        return this._mapMcpEmbeddedResource(record)
+
+      default:
+        return new JsonBlock({ json: item as JSONValue })
+    }
+  }
+
+  /**
+   * Maps an MCP image content item to an ImageBlock.
+   *
+   * @param record - MCP image content with data (base64) and mimeType
+   * @returns ImageBlock or TextBlock fallback if format is unsupported
+   */
+  private _mapMcpImageContent(record: Record<string, unknown>): ToolResultContent {
+    const data = record.data
+    const mimeType = record.mimeType
+
+    if (typeof data !== 'string' || typeof mimeType !== 'string') {
+      logger.warn('mcp image content missing data or mimeType, falling back to json')
+      return new JsonBlock({ json: record as JSONValue })
+    }
+
+    const format = toMediaFormat(mimeType)
+    if (!format || !this._isImageFormat(format)) {
+      logger.warn(`mime_type=<${mimeType}> | unsupported mcp image mime type, falling back to json`)
+      return new JsonBlock({ json: record as JSONValue })
+    }
+
+    return new ImageBlock({
+      format,
+      source: { bytes: decodeBase64(data) },
+    })
+  }
+
+  /**
+   * Maps an MCP embedded resource to an SDK content block.
+   * Text resources become TextBlock, blob resources with image MIME types become ImageBlock.
+   *
+   * @param record - MCP embedded resource content
+   * @returns Mapped content block or undefined if unsupported
+   */
+  private _mapMcpEmbeddedResource(record: Record<string, unknown>): ToolResultContent {
+    const resource = record.resource
+    if (!resource || typeof resource !== 'object') {
+      return new JsonBlock({ json: record as JSONValue })
+    }
+
+    const res = resource as Record<string, unknown>
+
+    // Text resource
+    if (typeof res.text === 'string') {
+      return new TextBlock(res.text)
+    }
+
+    // Blob resource
+    if (typeof res.blob === 'string' && typeof res.mimeType === 'string') {
+      const format = toMediaFormat(res.mimeType)
+      if (format && this._isImageFormat(format)) {
+        return new ImageBlock({
+          format,
+          source: { bytes: decodeBase64(res.blob) },
+        })
+      }
+      // Non-image blob: fall back to json
+      logger.warn(`mime_type=<${res.mimeType}> | unsupported mcp resource blob mime type, falling back to json`)
+    }
+
+    return new JsonBlock({ json: record as JSONValue })
+  }
+
   /**
    * Type Guard: Checks if value matches the expected MCP SDK result shape.
    * \{ content: unknown[]; isError?: boolean \}
@@ -86,16 +186,9 @@ export class McpTool extends Tool {
   }
 
   /**
-   * Type Guard: Checks if an item is a Text content block.
-   * \{ type: 'text'; text: string \}
+   * Type Guard: Checks if a media format is a supported image format.
    */
-  private _isMcpTextContent(value: unknown): value is { type: 'text'; text: string } {
-    if (typeof value !== 'object' || value === null) {
-      return false
-    }
-
-    const record = value as Record<string, unknown>
-
-    return record.type === 'text' && typeof record.text === 'string'
+  private _isImageFormat(format: string): format is ImageFormat {
+    return ['png', 'jpg', 'jpeg', 'gif', 'webp'].includes(format)
   }
 }