diff --git a/.changeset/github-issue-offer.md b/.changeset/github-issue-offer.md new file mode 100644 index 000000000..a780d5e73 --- /dev/null +++ b/.changeset/github-issue-offer.md @@ -0,0 +1,4 @@ +--- +--- + +Add GitHub issue offer for open-source agent test failures diff --git a/.github/workflows/check-testable-snippets.yml b/.github/workflows/check-testable-snippets.yml index 7dea6c929..e3ef8c278 100644 --- a/.github/workflows/check-testable-snippets.yml +++ b/.github/workflows/check-testable-snippets.yml @@ -26,7 +26,7 @@ jobs: if [ -s changed_files.txt ]; then echo "๐Ÿ“‹ Checking documentation changes for testable snippets..." - node scripts/check-testable-snippets.js + node scripts/check-testable-snippets.cjs else echo "โœ“ No documentation files changed" fi diff --git a/docs/media-buy/advanced-topics/testing.mdx b/docs/media-buy/advanced-topics/testing.mdx index 03a196df9..3eb46502c 100644 --- a/docs/media-buy/advanced-topics/testing.mdx +++ b/docs/media-buy/advanced-topics/testing.mdx @@ -50,7 +50,90 @@ AdCP provides a **public test agent** with free credentials for development and ## Protocol Compliance Testing -Use the [AdCP Protocol Test Harness](https://testing.adcontextprotocol.org) to validate your implementation's compliance with the AdCP specification. This interactive tool allows you to test all AdCP tasks and verify correct behavior across different scenarios. +### Testing via Addie + +The easiest way to test your AdCP agent is to ask Addie in Slack: + +> "Hey Addie, test my sales agent at https://sales.example.com" + +Addie can run comprehensive E2E tests including: + +**Standard Scenarios:** +- **health_check** - Verify agent responds +- **discovery** - Test `get_products`, `list_creative_formats`, `list_authorized_properties` +- **create_media_buy** - Discovery + create a test campaign +- **full_sales_flow** - Complete lifecycle: create โ†’ update โ†’ delivery +- **creative_sync** - Test `sync_creatives` flow +- **creative_inline** - Test inline creatives in `create_media_buy` +- **pricing_models** - Analyze pricing options across channels + +**Edge Case Scenarios:** +- **error_handling** - Verify proper discriminated union error responses +- **validation** - Test rejection of invalid inputs (negative budgets, invalid enums) +- **pricing_edge_cases** - Test auction vs fixed pricing, min_spend requirements, bid_price handling +- **temporal_validation** - Test date/time ordering, ISO 8601 format validation + +**Behavioral Analysis:** +- **behavior_analysis** - Analyze agent characteristics: authentication requirements, brand_manifest requirements, brief relevance filtering, channel filtering behavior + +By default tests run in dry-run mode. For real testing, ask Addie to run without dry-run. + +### Sales Agent Compliance Checklist + +Use this checklist to verify your sales agent implementation covers all required features: + +**Core Discovery (Required)** +- [ ] `get_products` - Returns products with pricing_options, format_ids, delivery_type +- [ ] `list_creative_formats` - Returns supported formats and creative agents +- [ ] `list_authorized_properties` - Returns publisher domains (if applicable) + +**Media Buy Lifecycle (Required)** +- [ ] `create_media_buy` - Accepts packages with product_id, pricing_option_id, budget +- [ ] `update_media_buy` - Supports PATCH semantics for budget, pacing, targeting +- [ ] `get_media_buy_delivery` - Returns impressions, spend, status + +**Creative Management (Required for most channels)** +- [ ] `sync_creatives` - Upsert creatives with per-item action tracking +- [ ] `list_creatives` - Query creative library with filtering +- [ ] Support inline creatives in `create_media_buy` +- [ ] Support creative references (`creative_ids`) + +**Pricing Models (as applicable)** +- [ ] CPM - Cost per thousand impressions +- [ ] vCPM - Viewable CPM (MRC standard) +- [ ] CPCV - Cost per completed view +- [ ] CPC - Cost per click +- [ ] CPP - Cost per rating point (TV/radio) +- [ ] Flat rate - Fixed cost sponsorships +- [ ] Auction pricing - Support bid_price when is_fixed=false + +**Creative Types** +- [ ] Static creatives (image, video assets) +- [ ] Reference creatives (creative_ids to existing library) +- [ ] Generative creatives (manifest-based) +- [ ] Parameterized creatives (with substitution) + +**Response Patterns** +- [ ] Discriminated union responses (success XOR errors) +- [ ] Schema-compliant responses (validate against JSON schemas) +- [ ] Async operations return status: submitted/working/completed +- [ ] Per-item errors in batch operations (e.g., sync_creatives) + +**Testing Support** +- [ ] `X-Dry-Run` header support +- [ ] `X-Test-Session-ID` for parallel test isolation +- [ ] `X-Mock-Time` for time simulation + +**Edge Case Validation (Required)** +- [ ] Reject negative budget values +- [ ] Reject invalid pacing enum values +- [ ] Reject end_time before start_time +- [ ] Reject invalid ISO 8601 date formats +- [ ] Return proper error for non-existent product_id +- [ ] Require bid_price for auction pricing options +- [ ] Reject budget below min_spend_per_package +- [ ] Reject creative weight > 100 +- [ ] Return discriminated union error responses (success XOR errors, never both) ## Testing Modes diff --git a/package-lock.json b/package-lock.json index 088ded2ff..73acf2c97 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,7 @@ "name": "adcontextprotocol", "version": "2.5.1", "dependencies": { - "@adcp/client": "^3.4.0", + "@adcp/client": "^3.5.0", "@anthropic-ai/sdk": "^0.71.2", "@modelcontextprotocol/sdk": "^1.24.3", "@mozilla/readability": "^0.6.0", @@ -87,9 +87,9 @@ } }, "node_modules/@adcp/client": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/@adcp/client/-/client-3.4.0.tgz", - "integrity": "sha512-DRC/sib4y05Slg3KP8um2cpxc4GRKtKTub7/hQMuX4CKNB76PxEdGed/Lubiq2CoaBRnygHBIeo0gsDX55tcdA==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/@adcp/client/-/client-3.5.0.tgz", + "integrity": "sha512-pcCUMIztHY3tVlTOecw6rh8vUmKUyZ1VJ0DNDX17LuvqnGhhFvwmoTbUlhvn47pWlSFu5vhK7TBGFVeO0p0drQ==", "license": "MIT", "dependencies": { "better-sqlite3": "^12.4.1", diff --git a/package.json b/package.json index 93738cfdb..b316bcf71 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "verify-version-sync": "node scripts/verify-version-sync.cjs" }, "dependencies": { - "@adcp/client": "^3.4.0", + "@adcp/client": "^3.5.0", "@anthropic-ai/sdk": "^0.71.2", "@modelcontextprotocol/sdk": "^1.24.3", "@mozilla/readability": "^0.6.0", diff --git a/server/public/chat.html b/server/public/chat.html index 4574c0c42..ac1a92415 100644 --- a/server/public/chat.html +++ b/server/public/chat.html @@ -177,6 +177,55 @@ color: white; } + /* Image support */ + .message-content img { + max-width: 100%; + height: auto; + border-radius: 8px; + margin: 8px 0; + display: block; + } + + .message-content img:hover { + cursor: pointer; + } + + /* iframe support for creative previews */ + .message-content .creative-preview-container { + position: relative; + width: 100%; + margin: 12px 0; + border-radius: 8px; + overflow: hidden; + border: 1px solid var(--color-border); + background: var(--color-bg-subtle); + } + + .message-content .creative-preview-container iframe { + width: 100%; + min-height: 200px; + border: none; + display: block; + } + + .message-content .creative-preview-label { + font-size: 11px; + color: var(--color-text-muted); + padding: 4px 8px; + background: var(--color-bg-card); + border-top: 1px solid var(--color-border); + } + + /* Inline HTML creative container */ + .message-content .creative-html-container { + margin: 12px 0; + border-radius: 8px; + overflow: hidden; + border: 1px solid var(--color-border); + background: white; + padding: 16px; + } + .message-content ul, .message-content ol { margin: 8px 0; padding-left: 20px; @@ -623,8 +672,8 @@

Hi! I'm Addie

+ -
@@ -856,7 +905,7 @@

Hi! I'm Addie

gfm: true, // GitHub flavored markdown }); - // Use marked's built-in renderer with custom link handling + // Use marked's built-in renderer with custom link and image handling const renderer = new marked.Renderer(); renderer.link = function(href, title, text) { // Handle marked v17+ which passes an object @@ -870,9 +919,46 @@

Hi! I'm Addie

return `${text}`; }; + // Custom image renderer + renderer.image = function(href, title, text) { + // Handle marked v17+ which passes an object + if (typeof href === 'object') { + const img = href; + href = img.href; + title = img.title; + text = img.text; + } + const titleAttr = title ? ` title="${title}"` : ''; + const altAttr = text ? ` alt="${text}"` : ' alt="Image"'; + // Make images clickable to open in new tab + return ``; + }; + return marked.parse(text, { renderer }); } + // Render creative preview (iframe or HTML) + function renderCreativePreview(previewUrl, label) { + if (!previewUrl) return ''; + const safeLabel = label ? label.replace(//g, '>') : 'Creative Preview'; + return ` +
+ +
${safeLabel}
+
+ `; + } + + // Render inline HTML creative + function renderCreativeHtml(html, label) { + if (!html) return ''; + return ` +
+ ${html} +
+ `; + } + // Add message to chat function addMessage(content, role, messageId = null) { // Hide welcome message @@ -1291,9 +1377,33 @@

Hi! I'm Addie

} }); + // Check for prompt in query string (e.g., ?prompt=Try%20AdCP) + function checkQueryPrompt() { + const params = new URLSearchParams(window.location.search); + const prompt = params.get('prompt'); + if (prompt && prompt.trim()) { + // Wait for Addie to be ready before sending + const waitForReady = setInterval(() => { + if (isReady) { + clearInterval(waitForReady); + chatInput.value = prompt.trim(); + autoResize(); + updateSendButton(); + sendMessage(); + // Clean URL without reloading + window.history.replaceState({}, '', window.location.pathname); + } + }, 100); + // Timeout after 10 seconds + setTimeout(() => clearInterval(waitForReady), 10000); + } + } + // Initialize checkStatus(); checkImpersonation(); + // Check for prompt in URL after status check + setTimeout(checkQueryPrompt, 500); // Check status periodically setInterval(checkStatus, 30000); diff --git a/server/public/index.html b/server/public/index.html index b99cffdf8..2854aba21 100644 --- a/server/public/index.html +++ b/server/public/index.html @@ -256,7 +256,7 @@ -

AdCP: The Open Standard for Agentic Advertising

From brief to buy, helping agents advertise anywhere: from CTV to chat, from tiny blog to the World Cup.

๐ŸŽฏ
Built for outcomes
Buy the way you want to grow
๐Ÿค–
Built for agents
Supports MCP and A2A protocols
๐ŸŒ
Built for everyone
A diverse ecosystem of tech and content
Start BuildingTry It LiveView on GitHub
v2.5.0 ReleasedDeveloper experience and API refinement: type safety, batch previews (5-10x faster), schema versioning, and more!Read the release notes โ†’

Why we built AdCP

The advertising ecosystem is fragmented. Every platform has its own API, its own workflow, its own reporting format. Media buyers and agencies waste countless hours navigating this complexity.

The Integration Problem

Each new platform requires custom integration work. APIs change, documentation varies, and maintenance never ends. Teams spend more time on plumbing than on strategy.

The Discovery Problem

Inventory is scattered across platforms with different taxonomies and targeting options. Finding the right audiences means learning multiple systems and manually comparing options.

The Automation Problem

AI agents and automation tools can't easily interact with advertising platforms. Each integration is bespoke, limiting the potential of AI-powered workflows.

We believe there's a better way. A single protocol that any platform can implement and any tool can use. An open standard that makes advertising technology work together, not against each other.

Agentic Advertising

Managed by Agentic Advertising

AdCP is stewarded by Agentic Advertising (AAO), an industry trade association advancing open standards for AI-powered advertising. AAO brings together publishers, platforms, agencies, and technology providers to shape the future of the ecosystem.

Become a Founding MemberLearn About AAO

Limited time: Founding member pricing ends March 31, 2026

One protocol. Every platform. Total control.

AdCP is the open standard that unifies advertising workflows across all platforms.
Think of it as the USB-C of advertising technology.

Before AdCP

  • 15+ different platform APIs
  • Months of custom integration
  • Manual data reconciliation
  • Fragmented reporting
  • Vendor lock-in

With AdCP

  • One unified interface
  • Deploy in days
  • Automated workflows
  • Consolidated analytics
  • Complete flexibility

See the difference

Traditional Workflow

1. Log into Platform A
2. Search for audiences (30 min)
3. Export to spreadsheet
4. Log into Platform B
5. Manually recreate targeting
6. Wait for approval (2 days)
7. Repeat for 10 more platforms...

AdCP Workflow

"Find sports enthusiasts with
high purchase intent, compare
prices across all platforms,
and activate the best option."

โœ“ Done in minutes

Everything you need, production-ready

AdCP v2.5.0 includes a complete suite of capabilities for modern advertising workflows.

๐Ÿ›’ Media Buy Protocol

Complete campaign lifecycle management with 9 core tasks:

  • get_products - Discover inventory with natural language
  • create_media_buy - Launch campaigns across platforms
  • get_media_buy_delivery - Real-time performance metrics
  • Plus sync, update, feedback, and more

๐ŸŽจ Creative Protocol

AI-powered creative generation and management:

  • build_creative - Generate creatives from briefs
  • preview_creative - Visual preview generation
  • list_creative_formats - Discover format specs
  • Standard formats library included

๐Ÿ“Š Signals Protocol

First-party data integration:

  • get_signals - Discover available signals
  • activate_signal - Activate for campaigns
  • Privacy-first audience building
  • Platform-agnostic data sharing

โšก Protocol Features

Enterprise-ready infrastructure:

  • MCP & A2A protocol support
  • Async workflows with webhooks
  • Human-in-the-loop approval
  • JSON Schema validation

How AdCP works

Built on the Model Context Protocol (MCP), AdCP provides a unified interface for advertising operations across any platform.

1

Discovery

Use natural language to describe your target audience. AdCP searches across all connected platforms to find matching inventory and audiences.

"Find sports enthusiasts interested in running gear"
2

Comparison

Get standardized results from all platforms in a consistent format. Compare pricing, reach, and targeting capabilities side by side.

Platform A: $12 CPM โ€ข 2.3M reach
Platform B: $18 CPM โ€ข 4.1M reach
Platform C: $9 CPM โ€ข 1.8M reach
3

Activation

Launch campaigns across multiple platforms with a single command. AdCP handles the technical details while maintaining platform-specific optimizations.

"Activate on Platform B with $10,000 budget"
4

Management

Monitor performance, adjust budgets, and generate reports across all platforms from one interface. Set up automated rules and alerts.

"Show performance metrics for all active campaigns"

Ready to join the revolution?

Whether you're a platform provider or an advertiser, AdCP is your path to the future of advertising.

Platform Providers

Make your inventory accessible to every AI assistant and automation platform.

  • Enable AI-powered workflows for your inventory
  • Simplify integration with a standard protocol
  • Reach new customers through automation platforms
Explore the Spec

Open source โ€ข MIT licensed

Advertisers & Agencies

Start using natural language to manage campaigns across all platforms.

  • Manage campaigns with natural language
  • Access unified analytics across platforms
  • Build on open standards, avoid vendor lock-in
Start Building Today

Documentation & guides

Join the conversation

AdCP is an open standard developed in collaboration with the advertising community. We're building this together, and your input matters.

Open Development

All development happens in the open on GitHub. Watch progress, submit issues, and contribute code.

Working Group

Join monthly meetings to discuss protocol evolution, implementation challenges, and future directions.

Implementation Support

Get help implementing AdCP for your platform or building tools that use the protocol.

Star on GitHubJoin Slack Community
+

AdCP: The Open Standard for Agentic Advertising

From brief to buy, helping agents advertise anywhere: from CTV to chat, from tiny blog to the World Cup.

๐ŸŽฏ
Built for outcomes
Buy the way you want to grow
๐Ÿค–
Built for agents
Supports MCP and A2A protocols
๐ŸŒ
Built for everyone
A diverse ecosystem of tech and content
Start BuildingTest with AddieView on GitHub
v2.5.0 ReleasedDeveloper experience and API refinement: type safety, batch previews (5-10x faster), schema versioning, and more!Read the release notes โ†’

Why we built AdCP

The advertising ecosystem is fragmented. Every platform has its own API, its own workflow, its own reporting format. Media buyers and agencies waste countless hours navigating this complexity.

The Integration Problem

Each new platform requires custom integration work. APIs change, documentation varies, and maintenance never ends. Teams spend more time on plumbing than on strategy.

The Discovery Problem

Inventory is scattered across platforms with different taxonomies and targeting options. Finding the right audiences means learning multiple systems and manually comparing options.

The Automation Problem

AI agents and automation tools can't easily interact with advertising platforms. Each integration is bespoke, limiting the potential of AI-powered workflows.

We believe there's a better way. A single protocol that any platform can implement and any tool can use. An open standard that makes advertising technology work together, not against each other.

Agentic Advertising

Managed by Agentic Advertising

AdCP is stewarded by Agentic Advertising (AAO), an industry trade association advancing open standards for AI-powered advertising. AAO brings together publishers, platforms, agencies, and technology providers to shape the future of the ecosystem.

Become a Founding MemberLearn About AAO

Limited time: Founding member pricing ends March 31, 2026

One protocol. Every platform. Total control.

AdCP is the open standard that unifies advertising workflows across all platforms.
Think of it as the USB-C of advertising technology.

Before AdCP

  • 15+ different platform APIs
  • Months of custom integration
  • Manual data reconciliation
  • Fragmented reporting
  • Vendor lock-in

With AdCP

  • One unified interface
  • Deploy in days
  • Automated workflows
  • Consolidated analytics
  • Complete flexibility

See the difference

Traditional Workflow

1. Log into Platform A
2. Search for audiences (30 min)
3. Export to spreadsheet
4. Log into Platform B
5. Manually recreate targeting
6. Wait for approval (2 days)
7. Repeat for 10 more platforms...

AdCP Workflow

"Find sports enthusiasts with
high purchase intent, compare
prices across all platforms,
and activate the best option."

โœ“ Done in minutes

Everything you need, production-ready

AdCP v2.5.0 includes a complete suite of capabilities for modern advertising workflows.

๐Ÿ›’ Media Buy Protocol

Complete campaign lifecycle management with 9 core tasks:

  • get_products - Discover inventory with natural language
  • create_media_buy - Launch campaigns across platforms
  • get_media_buy_delivery - Real-time performance metrics
  • Plus sync, update, feedback, and more

๐ŸŽจ Creative Protocol

AI-powered creative generation and management:

  • build_creative - Generate creatives from briefs
  • preview_creative - Visual preview generation
  • list_creative_formats - Discover format specs
  • Standard formats library included

๐Ÿ“Š Signals Protocol

First-party data integration:

  • get_signals - Discover available signals
  • activate_signal - Activate for campaigns
  • Privacy-first audience building
  • Platform-agnostic data sharing

โšก Protocol Features

Enterprise-ready infrastructure:

  • MCP & A2A protocol support
  • Async workflows with webhooks
  • Human-in-the-loop approval
  • JSON Schema validation

How AdCP works

Built on the Model Context Protocol (MCP), AdCP provides a unified interface for advertising operations across any platform.

1

Discovery

Use natural language to describe your target audience. AdCP searches across all connected platforms to find matching inventory and audiences.

"Find sports enthusiasts interested in running gear"
2

Comparison

Get standardized results from all platforms in a consistent format. Compare pricing, reach, and targeting capabilities side by side.

Platform A: $12 CPM โ€ข 2.3M reach
Platform B: $18 CPM โ€ข 4.1M reach
Platform C: $9 CPM โ€ข 1.8M reach
3

Activation

Launch campaigns across multiple platforms with a single command. AdCP handles the technical details while maintaining platform-specific optimizations.

"Activate on Platform B with $10,000 budget"
4

Management

Monitor performance, adjust budgets, and generate reports across all platforms from one interface. Set up automated rules and alerts.

"Show performance metrics for all active campaigns"

Ready to join the revolution?

Whether you're a platform provider or an advertiser, AdCP is your path to the future of advertising.

Platform Providers

Make your inventory accessible to every AI assistant and automation platform.

  • Enable AI-powered workflows for your inventory
  • Simplify integration with a standard protocol
  • Reach new customers through automation platforms
Explore the Spec

Open source โ€ข MIT licensed

Advertisers & Agencies

Start using natural language to manage campaigns across all platforms.

  • Manage campaigns with natural language
  • Access unified analytics across platforms
  • Build on open standards, avoid vendor lock-in
Start Building Today

Documentation & guides

Join the conversation

AdCP is an open standard developed in collaboration with the advertising community. We're building this together, and your input matters.

Open Development

All development happens in the open on GitHub. Watch progress, submit issues, and contribute code.

Working Group

Join monthly meetings to discuss protocol evolution, implementation challenges, and future directions.

Implementation Support

Get help implementing AdCP for your platform or building tools that use the protocol.

Star on GitHubJoin Slack Community
diff --git a/server/src/addie/mcp/member-tools.ts b/server/src/addie/mcp/member-tools.ts index 4214c7d1d..803ac1f30 100644 --- a/server/src/addie/mcp/member-tools.ts +++ b/server/src/addie/mcp/member-tools.ts @@ -15,8 +15,71 @@ import { logger } from '../../logger.js'; import type { AddieTool } from '../types.js'; import { AdAgentsManager } from '../../adagents-manager.js'; import type { MemberContext } from '../member-context.js'; +import { + runAgentTests, + formatTestResults, + setAgentTesterLogger, + createTestClient, + type TestScenario, + type TestOptions, +} from '@adcp/client/testing'; +import { AgentContextDatabase } from '../../db/agent-context-db.js'; const adagentsManager = new AdAgentsManager(); +const agentContextDb = new AgentContextDatabase(); + +/** + * Known open-source agents and their GitHub repositories. + * Used to offer GitHub issue links when tests fail on these agents. + * Keys must be lowercase (hostnames are case-insensitive). + */ +const KNOWN_OPEN_SOURCE_AGENTS: Record = { + 'test-agent.adcontextprotocol.org': { + org: 'adcontextprotocol', + repo: 'salesagent', + name: 'AdCP Reference Sales Agent', + }, + 'wonderstruck.sales-agent.scope3.com': { + org: 'adcontextprotocol', + repo: 'salesagent', + name: 'Wonderstruck (Scope3 Sales Agent)', + }, + 'creative.adcontextprotocol.org': { + org: 'adcontextprotocol', + repo: 'creative-agent', + name: 'AdCP Reference Creative Agent', + }, +}; + +/** + * Extract hostname from an agent URL for matching against known agents + */ +function getAgentHostname(agentUrl: string): string | null { + try { + const url = new URL(agentUrl); + return url.hostname; + } catch { + return null; + } +} + +/** + * Check if an agent URL is a known open-source agent + */ +function getOpenSourceAgentInfo(agentUrl: string): { org: string; repo: string; name: string } | null { + const hostname = getAgentHostname(agentUrl); + if (!hostname) return null; + // Normalize to lowercase for case-insensitive matching + return KNOWN_OPEN_SOURCE_AGENTS[hostname.toLowerCase()] || null; +} + +// Configure the agent tester to use our pino logger +setAgentTesterLogger({ + info: (ctx, msg) => logger.info(ctx, msg), + error: (ctx, msg) => logger.error(ctx, msg), + warn: (ctx, msg) => logger.warn(ctx, msg), + debug: (ctx, msg) => logger.debug(ctx, msg), +}); /** * Tool definitions for member-related operations @@ -292,6 +355,162 @@ export const MEMBER_TOOLS: AddieTool[] = [ required: ['agent_url'], }, }, + { + name: 'test_adcp_agent', + description: + 'Run end-to-end tests against an AdCP agent to verify it works correctly. Tests the full workflow: discover products, create media buys, sync creatives, etc. By default runs in dry-run mode - set dry_run=false for real testing. Use this when users want to test their agent implementation, verify compliance, or debug issues. This replaces the testing.adcontextprotocol.org harness.', + usage_hints: 'use for "test my agent", "run the full flow", "verify my sales agent works", "test against test-agent", "test creative sync", "test pricing models"', + input_schema: { + type: 'object', + properties: { + agent_url: { + type: 'string', + description: 'The agent URL to test (e.g., "https://sales.example.com" or "https://test-agent.adcontextprotocol.org")', + }, + scenario: { + type: 'string', + enum: [ + 'health_check', + 'discovery', + 'create_media_buy', + 'full_sales_flow', + 'creative_sync', + 'creative_inline', + 'creative_reference', + 'pricing_models', + 'creative_flow', + 'signals_flow', + 'error_handling', + 'validation', + 'pricing_edge_cases', + 'temporal_validation', + 'behavior_analysis', + 'response_consistency', + ], + description: 'Test scenario: health_check (agent responds), discovery (products/formats/properties), create_media_buy (discovery + create), full_sales_flow (create + update + delivery), creative_sync (sync_creatives flow), creative_inline (inline creatives in create_media_buy), creative_reference (reference existing creatives), pricing_models (analyze pricing options), creative_flow (creative agents), signals_flow (signals agents), error_handling (proper error responses), validation (invalid input rejection), pricing_edge_cases (auction vs fixed, min spend), temporal_validation (date ordering, format), behavior_analysis (auth requirements, brief relevance, filtering behavior), response_consistency (schema errors, pagination bugs, data mismatches)', + }, + brief: { + type: 'string', + description: 'Optional custom brief for product discovery (default: generic tech brand brief)', + }, + budget: { + type: 'number', + description: 'Budget for test media buy in dollars (default: 1000)', + }, + dry_run: { + type: 'boolean', + description: 'Whether to run in dry-run mode (default: true). Set to false for real testing that creates actual media buys.', + }, + channels: { + type: 'array', + items: { type: 'string' }, + description: 'Specific channels to test (e.g., ["display", "video", "ctv"]). If not specified, tests all channels the agent supports.', + }, + pricing_models: { + type: 'array', + items: { type: 'string' }, + description: 'Specific pricing models to test (e.g., ["cpm", "cpcv"]). If not specified, uses first available.', + }, + auth_token: { + type: 'string', + description: 'Bearer token for agents that require authentication. For test-agent.adcontextprotocol.org, use the published test credentials.', + }, + }, + required: ['agent_url'], + }, + }, + { + name: 'call_adcp_tool', + description: + 'Call any AdCP task on an agent and return the raw response. Use this for custom testing, exploring agent capabilities, or when you need to call a specific task with specific parameters. This is lower-level than test_adcp_agent - use it when you need precise control over the request/response.', + usage_hints: 'use for "call get_products on my agent", "try create_media_buy with these parameters", "what does list_creative_formats return", exploring agent responses, debugging specific calls', + input_schema: { + type: 'object', + properties: { + agent_url: { + type: 'string', + description: 'The agent URL to call (e.g., "https://sales.example.com/mcp")', + }, + task: { + type: 'string', + description: 'The AdCP task to call (e.g., "get_products", "create_media_buy", "list_creative_formats", "sync_creatives")', + }, + params: { + type: 'object', + description: 'Parameters to pass to the task. Structure depends on the task being called.', + }, + auth_token: { + type: 'string', + description: 'Bearer token for agents that require authentication. Will auto-lookup saved token if not provided.', + }, + dry_run: { + type: 'boolean', + description: 'Whether to include X-Dry-Run header (default: true for safety)', + }, + }, + required: ['agent_url', 'task'], + }, + }, + + // ============================================ + // AGENT CONTEXT MANAGEMENT + // ============================================ + { + name: 'save_agent', + description: + 'Save an agent URL to the organization\'s context. Optionally store an auth token securely (encrypted, never shown in conversations). Use this when users want to save their agent for easy testing later, or when they provide an auth token.', + usage_hints: 'use for "save my agent", "remember this agent URL", "store my auth token"', + input_schema: { + type: 'object', + properties: { + agent_url: { + type: 'string', + description: 'The agent URL to save (e.g., "https://sales.example.com/mcp")', + }, + agent_name: { + type: 'string', + description: 'Friendly name for the agent (e.g., "Production Sales Agent")', + }, + auth_token: { + type: 'string', + description: 'Optional auth token to store securely. Will be encrypted and never shown again.', + }, + protocol: { + type: 'string', + enum: ['mcp', 'a2a'], + description: 'Protocol type (default: mcp)', + }, + }, + required: ['agent_url'], + }, + }, + { + name: 'list_saved_agents', + description: + 'List all agents saved for this organization. Shows agent URLs, names, types, and whether they have auth tokens stored (but never shows the actual tokens). Use this when users ask "what agents do I have saved?" or want to see their configured agents.', + usage_hints: 'use for "show my agents", "what agents are saved?", "list our agents"', + input_schema: { + type: 'object', + properties: {}, + required: [], + }, + }, + { + name: 'remove_saved_agent', + description: + 'Remove a saved agent and its stored auth token. Use this when users want to delete or forget an agent configuration.', + usage_hints: 'use for "remove my agent", "delete the agent", "forget this agent"', + input_schema: { + type: 'object', + properties: { + agent_url: { + type: 'string', + description: 'The agent URL to remove', + }, + }, + required: ['agent_url'], + }, + }, // ============================================ // GITHUB ISSUE DRAFTING @@ -333,12 +552,14 @@ export const MEMBER_TOOLS: AddieTool[] = [ /** * Base URL for internal API calls * Uses BASE_URL env var in production, falls back to localhost for development + * Note: PORT takes precedence over CONDUCTOR_PORT for internal calls (inside Docker, PORT=8080) */ function getBaseUrl(): string { if (process.env.BASE_URL) { return process.env.BASE_URL; } - const port = process.env.CONDUCTOR_PORT || process.env.PORT || '3000'; + // PORT is the internal server port (8080 in Docker), CONDUCTOR_PORT is external mapping + const port = process.env.PORT || process.env.CONDUCTOR_PORT || '3000'; return `http://localhost:${port}`; } @@ -1020,6 +1241,191 @@ export function createMemberToolHandlers( return response; }); + // ============================================ + // E2E AGENT TESTING + // ============================================ + handlers.set('test_adcp_agent', async (input) => { + const agentUrl = input.agent_url as string; + const scenario = (input.scenario as TestScenario) || 'discovery'; + const brief = input.brief as string | undefined; + const budget = input.budget as number | undefined; + const dryRun = input.dry_run as boolean | undefined; + const channels = input.channels as string[] | undefined; + const pricingModels = input.pricing_models as string[] | undefined; + let authToken = input.auth_token as string | undefined; + + // Auto-lookup saved token if user didn't provide one and has org context + let usingSavedToken = false; + const organizationId = memberContext?.organization?.workos_organization_id; + if (!authToken && organizationId) { + try { + const savedToken = await agentContextDb.getAuthTokenByOrgAndUrl( + organizationId, + agentUrl + ); + if (savedToken) { + authToken = savedToken; + usingSavedToken = true; + logger.info({ agentUrl }, 'Using saved auth token for agent test'); + } + } catch (error) { + // Non-fatal - continue without saved token + logger.debug({ error, agentUrl }, 'Could not lookup saved token'); + } + } + + const options: TestOptions = { + test_session_id: `addie-test-${Date.now()}`, + dry_run: dryRun, // undefined means default to true + }; + if (brief) options.brief = brief; + if (budget) options.budget = budget; + if (channels) options.channels = channels; + if (pricingModels) options.pricing_models = pricingModels; + if (authToken) options.auth = { type: 'bearer', token: authToken }; + + try { + const result = await runAgentTests(agentUrl, scenario, options); + + // If user is authenticated and agent test succeeded, update the saved context + if (organizationId) { + try { + const context = await agentContextDb.getByOrgAndUrl( + organizationId, + agentUrl + ); + if (context && result.agent_profile) { + // Update with discovered tools and test results + const tools = result.agent_profile.tools || []; + await agentContextDb.update(context.id, { + tools_discovered: tools, + agent_type: agentContextDb.inferAgentType(tools), + last_test_scenario: scenario, + last_test_passed: result.overall_passed, + last_test_summary: result.summary, + }); + + // Record test history + await agentContextDb.recordTest({ + agent_context_id: context.id, + scenario, + overall_passed: result.overall_passed, + steps_passed: result.steps.filter((s) => s.passed).length, + steps_failed: result.steps.filter((s) => !s.passed).length, + total_duration_ms: result.total_duration_ms, + summary: result.summary, + dry_run: options.dry_run !== false, + brief: options.brief, + triggered_by: 'user', + user_id: memberContext?.workos_user?.workos_user_id, + steps_json: result.steps, + agent_profile_json: result.agent_profile, + }); + } + } catch (error) { + // Non-fatal - test still ran + logger.debug({ error }, 'Could not update agent context after test'); + } + } + + let output = formatTestResults(result); + if (usingSavedToken) { + output = `_Using saved credentials for this agent._\n\n` + output; + } + + // If tests failed on a known open-source agent, offer to help file a GitHub issue + const failedSteps = result.steps.filter((s) => !s.passed); + if (failedSteps.length > 0) { + const openSourceInfo = getOpenSourceAgentInfo(agentUrl); + if (openSourceInfo) { + output += `\n---\n\n`; + output += `๐Ÿ’ก **This is an open-source agent** (${openSourceInfo.name})\n\n`; + output += `Since ${failedSteps.length} test step(s) failed, would you like me to help you report this issue?\n`; + output += `I can draft a GitHub issue for the \`${openSourceInfo.org}/${openSourceInfo.repo}\` repository with all the relevant details.\n\n`; + output += `Just say "yes, file an issue" or "help me report this bug" and I'll create a pre-filled GitHub link for you.`; + } + } + + return output; + } catch (error) { + logger.error({ error, agentUrl, scenario }, 'Addie: test_adcp_agent failed'); + return `Failed to test agent ${agentUrl}: ${error instanceof Error ? error.message : 'Unknown error'}`; + } + }); + + handlers.set('call_adcp_tool', async (input) => { + const agentUrl = input.agent_url as string; + const task = input.task as string; + const params = (input.params as Record) || {}; + const dryRun = input.dry_run as boolean | undefined; + let authToken = input.auth_token as string | undefined; + + // Auto-lookup saved token if user didn't provide one and has org context + let usingSavedToken = false; + const organizationId = memberContext?.organization?.workos_organization_id; + if (!authToken && organizationId) { + try { + const savedToken = await agentContextDb.getAuthTokenByOrgAndUrl(organizationId, agentUrl); + if (savedToken) { + authToken = savedToken; + usingSavedToken = true; + logger.info({ agentUrl, task }, 'Using saved auth token for call_adcp_tool'); + } + } catch (error) { + logger.debug({ error, agentUrl }, 'Could not lookup saved token'); + } + } + + try { + // Create a test client for the agent + const testOptions: TestOptions = { + test_session_id: `addie-call-${Date.now()}`, + dry_run: dryRun, // undefined defaults to true in createTestClient + }; + if (authToken) { + testOptions.auth = { type: 'bearer', token: authToken }; + } + + const client = createTestClient(agentUrl, 'mcp', testOptions); + const startTime = Date.now(); + + // Execute the task + const result = await client.executeTask(task, params); + const duration = Date.now() - startTime; + + // Format response + let output = `## AdCP Task Result\n\n`; + output += `**Agent:** ${agentUrl}\n`; + output += `**Task:** \`${task}\`\n`; + output += `**Duration:** ${duration}ms\n`; + output += `**Mode:** ${dryRun !== false ? '๐Ÿงช Dry Run' : '๐Ÿ”ด Live'}\n`; + if (usingSavedToken) { + output += `**Auth:** Using saved credentials\n`; + } + output += `\n`; + + if (result.success) { + output += `### โœ… Success\n\n`; + output += '```json\n'; + output += JSON.stringify(result.data, null, 2); + output += '\n```\n'; + } else { + output += `### โŒ Error\n\n`; + output += `**Error:** ${result.error || 'Unknown error'}\n`; + if (result.data) { + output += '\n**Response data:**\n```json\n'; + output += JSON.stringify(result.data, null, 2); + output += '\n```\n'; + } + } + + return output; + } catch (error) { + logger.error({ error, agentUrl, task }, 'Addie: call_adcp_tool failed'); + return `Failed to call ${task} on ${agentUrl}: ${error instanceof Error ? error.message : 'Unknown error'}`; + } + }); + // ============================================ // GITHUB ISSUE DRAFTING // ============================================ @@ -1080,5 +1486,172 @@ export function createMemberToolHandlers( return response; }); + // ============================================ + // AGENT CONTEXT MANAGEMENT + // ============================================ + handlers.set('save_agent', async (input) => { + // Require authenticated user with organization + if (!memberContext?.workos_user?.workos_user_id) { + return 'You need to be logged in to save agents. Please log in at https://agenticadvertising.org/dashboard first.'; + } + + const saveOrgId = memberContext.organization?.workos_organization_id; + if (!saveOrgId) { + return 'Your account is not associated with an organization. Please contact support.'; + } + + const agentUrl = input.agent_url as string; + const agentName = input.agent_name as string | undefined; + const authToken = input.auth_token as string | undefined; + const protocol = (input.protocol as 'mcp' | 'a2a') || 'mcp'; + + try { + // Check if agent already exists for this org + let context = await agentContextDb.getByOrgAndUrl(saveOrgId, agentUrl); + + if (context) { + // Update existing context + if (agentName) { + await agentContextDb.update(context.id, { agent_name: agentName, protocol }); + } + if (authToken) { + await agentContextDb.saveAuthToken(context.id, authToken); + } + // Refresh context + context = await agentContextDb.getById(context.id); + + let response = `โœ… Updated saved agent: **${context?.agent_name || agentUrl}**\n\n`; + if (authToken) { + response += `๐Ÿ” Auth token saved securely (hint: ${context?.auth_token_hint})\n`; + response += `_The token is encrypted and will never be shown again._\n`; + } + return response; + } + + // Create new context + context = await agentContextDb.create({ + organization_id: saveOrgId, + agent_url: agentUrl, + agent_name: agentName, + protocol, + created_by: memberContext.workos_user.workos_user_id, + }); + + // Save auth token if provided + if (authToken) { + await agentContextDb.saveAuthToken(context.id, authToken); + context = await agentContextDb.getById(context.id); + } + + let response = `โœ… Saved agent: **${context?.agent_name || agentUrl}**\n\n`; + response += `**URL:** ${agentUrl}\n`; + response += `**Protocol:** ${protocol.toUpperCase()}\n`; + if (authToken) { + response += `\n๐Ÿ” Auth token saved securely (hint: ${context?.auth_token_hint})\n`; + response += `_The token is encrypted and will never be shown again._\n`; + } + response += `\nWhen you test this agent, I'll automatically use the saved credentials.`; + + return response; + } catch (error) { + logger.error({ error, agentUrl }, 'Addie: save_agent failed'); + return `Failed to save agent: ${error instanceof Error ? error.message : 'Unknown error'}`; + } + }); + + handlers.set('list_saved_agents', async () => { + // Require authenticated user with organization + if (!memberContext?.workos_user?.workos_user_id) { + return 'You need to be logged in to list saved agents. Please log in at https://agenticadvertising.org/dashboard first.'; + } + + const listOrgId = memberContext.organization?.workos_organization_id; + if (!listOrgId) { + return 'Your account is not associated with an organization. Please contact support.'; + } + + try { + const agents = await agentContextDb.getByOrganization(listOrgId); + + if (agents.length === 0) { + return 'No agents saved yet. Use `save_agent` to save an agent URL for easy testing.'; + } + + let response = `## Your Saved Agents\n\n`; + + for (const agent of agents) { + const name = agent.agent_name || 'Unnamed Agent'; + const type = agent.agent_type !== 'unknown' ? ` (${agent.agent_type})` : ''; + const hasToken = agent.has_auth_token ? `๐Ÿ” ${agent.auth_token_hint}` : '๐Ÿ”“ No token'; + + response += `### ${name}${type}\n`; + response += `**URL:** ${agent.agent_url}\n`; + response += `**Protocol:** ${agent.protocol.toUpperCase()}\n`; + response += `**Auth:** ${hasToken}\n`; + + if (agent.tools_discovered && agent.tools_discovered.length > 0) { + response += `**Tools:** ${agent.tools_discovered.slice(0, 5).join(', ')}`; + if (agent.tools_discovered.length > 5) { + response += ` (+${agent.tools_discovered.length - 5} more)`; + } + response += `\n`; + } + + if (agent.last_tested_at) { + const lastTest = new Date(agent.last_tested_at).toLocaleDateString(); + const status = agent.last_test_passed ? 'โœ…' : 'โŒ'; + response += `**Last Test:** ${status} ${agent.last_test_scenario} (${lastTest})\n`; + response += `**Total Tests:** ${agent.total_tests_run}\n`; + } + + response += `\n`; + } + + return response; + } catch (error) { + logger.error({ error }, 'Addie: list_saved_agents failed'); + return `Failed to list agents: ${error instanceof Error ? error.message : 'Unknown error'}`; + } + }); + + handlers.set('remove_saved_agent', async (input) => { + // Require authenticated user with organization + if (!memberContext?.workos_user?.workos_user_id) { + return 'You need to be logged in to remove saved agents. Please log in at https://agenticadvertising.org/dashboard first.'; + } + + const removeOrgId = memberContext.organization?.workos_organization_id; + if (!removeOrgId) { + return 'Your account is not associated with an organization. Please contact support.'; + } + + const agentUrl = input.agent_url as string; + + try { + // Find the agent + const context = await agentContextDb.getByOrgAndUrl(removeOrgId, agentUrl); + + if (!context) { + return `No saved agent found with URL: ${agentUrl}\n\nUse \`list_saved_agents\` to see your saved agents.`; + } + + const agentName = context.agent_name || agentUrl; + + // Delete it + await agentContextDb.delete(context.id); + + let response = `โœ… Removed saved agent: **${agentName}**\n\n`; + if (context.has_auth_token) { + response += `๐Ÿ” The stored auth token has been permanently deleted.\n`; + } + response += `All test history for this agent has also been removed.`; + + return response; + } catch (error) { + logger.error({ error, agentUrl }, 'Addie: remove_saved_agent failed'); + return `Failed to remove agent: ${error instanceof Error ? error.message : 'Unknown error'}`; + } + }); + return handlers; } diff --git a/server/src/db/agent-context-db.ts b/server/src/db/agent-context-db.ts new file mode 100644 index 000000000..0f7c84134 --- /dev/null +++ b/server/src/db/agent-context-db.ts @@ -0,0 +1,534 @@ +import { query } from './client.js'; +import crypto from 'crypto'; + +// ===================================================== +// TYPES +// ===================================================== + +export type AgentType = 'sales' | 'creative' | 'signals' | 'unknown'; +export type Protocol = 'mcp' | 'a2a'; + +export interface AgentContext { + id: string; + organization_id: string; + agent_url: string; + agent_name: string | null; + agent_type: AgentType; + protocol: Protocol; + // Token info (never expose actual token!) + has_auth_token: boolean; + auth_token_hint: string | null; + // Discovery cache + tools_discovered: string[] | null; + last_discovered_at: Date | null; + // Test history + last_test_scenario: string | null; + last_test_passed: boolean | null; + last_test_summary: string | null; + last_tested_at: Date | null; + total_tests_run: number; + // Metadata + created_at: Date; + updated_at: Date; + created_by: string | null; +} + +export interface AgentTestHistory { + id: string; + agent_context_id: string; + scenario: string; + overall_passed: boolean; + steps_passed: number; + steps_failed: number; + total_duration_ms: number | null; + summary: string | null; + dry_run: boolean; + brief: string | null; + triggered_by: string | null; + user_id: string | null; + steps_json: any; + agent_profile_json: any; + started_at: Date; + completed_at: Date | null; +} + +export interface CreateAgentContextInput { + organization_id: string; + agent_url: string; + agent_name?: string; + agent_type?: AgentType; + protocol?: Protocol; + created_by?: string; +} + +export interface UpdateAgentContextInput { + agent_name?: string; + agent_type?: AgentType; + protocol?: Protocol; + tools_discovered?: string[]; + last_test_scenario?: string; + last_test_passed?: boolean; + last_test_summary?: string; +} + +export interface RecordTestInput { + agent_context_id: string; + scenario: string; + overall_passed: boolean; + steps_passed: number; + steps_failed: number; + total_duration_ms?: number; + summary?: string; + dry_run?: boolean; + brief?: string; + triggered_by?: string; + user_id?: string; + steps_json?: any; + agent_profile_json?: any; +} + +// ===================================================== +// ENCRYPTION HELPERS +// ===================================================== + +// Encryption key derivation (in production, use a proper KMS) +// For now, derive key from a secret + org ID +const ENCRYPTION_SECRET = process.env.AGENT_TOKEN_ENCRYPTION_SECRET || 'dev-secret-change-in-production'; + +function deriveKey(organizationId: string): Buffer { + return crypto.pbkdf2Sync(ENCRYPTION_SECRET, organizationId, 100000, 32, 'sha256'); +} + +function encryptToken(token: string, organizationId: string): { encrypted: string; iv: string } { + const key = deriveKey(organizationId); + const iv = crypto.randomBytes(16); + const cipher = crypto.createCipheriv('aes-256-gcm', key, iv); + + let encrypted = cipher.update(token, 'utf8', 'base64'); + encrypted += cipher.final('base64'); + + // Append auth tag + const authTag = cipher.getAuthTag(); + encrypted += ':' + authTag.toString('base64'); + + return { + encrypted, + iv: iv.toString('base64'), + }; +} + +function decryptToken(encrypted: string, iv: string, organizationId: string): string { + const key = deriveKey(organizationId); + const ivBuffer = Buffer.from(iv, 'base64'); + + // Split encrypted data and auth tag + const [encryptedData, authTagBase64] = encrypted.split(':'); + const authTag = Buffer.from(authTagBase64, 'base64'); + + const decipher = crypto.createDecipheriv('aes-256-gcm', key, ivBuffer); + decipher.setAuthTag(authTag); + + let decrypted = decipher.update(encryptedData, 'base64', 'utf8'); + decrypted += decipher.final('utf8'); + + return decrypted; +} + +function getTokenHint(token: string): string { + if (token.length <= 4) return '****'; + return '****' + token.slice(-4); +} + +// ===================================================== +// AGENT CONTEXT DATABASE +// ===================================================== + +export class AgentContextDatabase { + /** + * Get all agent contexts for an organization + */ + async getByOrganization(organizationId: string): Promise { + const result = await query( + `SELECT + id, + organization_id, + agent_url, + agent_name, + agent_type, + protocol, + auth_token_encrypted IS NOT NULL as has_auth_token, + auth_token_hint, + tools_discovered, + last_discovered_at, + last_test_scenario, + last_test_passed, + last_test_summary, + last_tested_at, + total_tests_run, + created_at, + updated_at, + created_by + FROM agent_contexts + WHERE organization_id = $1 + ORDER BY updated_at DESC`, + [organizationId] + ); + return result.rows; + } + + /** + * Get a specific agent context by ID + */ + async getById(id: string): Promise { + const result = await query( + `SELECT + id, + organization_id, + agent_url, + agent_name, + agent_type, + protocol, + auth_token_encrypted IS NOT NULL as has_auth_token, + auth_token_hint, + tools_discovered, + last_discovered_at, + last_test_scenario, + last_test_passed, + last_test_summary, + last_tested_at, + total_tests_run, + created_at, + updated_at, + created_by + FROM agent_contexts + WHERE id = $1`, + [id] + ); + return result.rows[0] || null; + } + + /** + * Get agent context by organization and URL + */ + async getByOrgAndUrl(organizationId: string, agentUrl: string): Promise { + const result = await query( + `SELECT + id, + organization_id, + agent_url, + agent_name, + agent_type, + protocol, + auth_token_encrypted IS NOT NULL as has_auth_token, + auth_token_hint, + tools_discovered, + last_discovered_at, + last_test_scenario, + last_test_passed, + last_test_summary, + last_tested_at, + total_tests_run, + created_at, + updated_at, + created_by + FROM agent_contexts + WHERE organization_id = $1 AND agent_url = $2`, + [organizationId, agentUrl] + ); + return result.rows[0] || null; + } + + /** + * Create a new agent context + */ + async create(input: CreateAgentContextInput): Promise { + const result = await query( + `INSERT INTO agent_contexts ( + organization_id, + agent_url, + agent_name, + agent_type, + protocol, + created_by + ) VALUES ($1, $2, $3, $4, $5, $6) + RETURNING + id, + organization_id, + agent_url, + agent_name, + agent_type, + protocol, + FALSE as has_auth_token, + auth_token_hint, + tools_discovered, + last_discovered_at, + last_test_scenario, + last_test_passed, + last_test_summary, + last_tested_at, + total_tests_run, + created_at, + updated_at, + created_by`, + [ + input.organization_id, + input.agent_url, + input.agent_name || null, + input.agent_type || 'unknown', + input.protocol || 'mcp', + input.created_by || null, + ] + ); + return result.rows[0]; + } + + /** + * Update an agent context + */ + async update(id: string, input: UpdateAgentContextInput): Promise { + const updates: string[] = []; + const values: any[] = []; + let paramIndex = 1; + + if (input.agent_name !== undefined) { + updates.push(`agent_name = $${paramIndex++}`); + values.push(input.agent_name); + } + if (input.agent_type !== undefined) { + updates.push(`agent_type = $${paramIndex++}`); + values.push(input.agent_type); + } + if (input.protocol !== undefined) { + updates.push(`protocol = $${paramIndex++}`); + values.push(input.protocol); + } + if (input.tools_discovered !== undefined) { + updates.push(`tools_discovered = $${paramIndex++}`); + updates.push(`last_discovered_at = NOW()`); + values.push(input.tools_discovered); + } + if (input.last_test_scenario !== undefined) { + updates.push(`last_test_scenario = $${paramIndex++}`); + values.push(input.last_test_scenario); + } + if (input.last_test_passed !== undefined) { + updates.push(`last_test_passed = $${paramIndex++}`); + values.push(input.last_test_passed); + } + if (input.last_test_summary !== undefined) { + updates.push(`last_test_summary = $${paramIndex++}`); + values.push(input.last_test_summary); + } + + if (updates.length === 0) { + return this.getById(id); + } + + updates.push('updated_at = NOW()'); + values.push(id); + + const result = await query( + `UPDATE agent_contexts + SET ${updates.join(', ')} + WHERE id = $${paramIndex} + RETURNING + id, + organization_id, + agent_url, + agent_name, + agent_type, + protocol, + auth_token_encrypted IS NOT NULL as has_auth_token, + auth_token_hint, + tools_discovered, + last_discovered_at, + last_test_scenario, + last_test_passed, + last_test_summary, + last_tested_at, + total_tests_run, + created_at, + updated_at, + created_by`, + values + ); + return result.rows[0] || null; + } + + /** + * Save an auth token (encrypted) + * IMPORTANT: Token is encrypted and never returned in queries + */ + async saveAuthToken(id: string, token: string): Promise { + // Get the org ID for key derivation + const context = await this.getById(id); + if (!context) { + throw new Error(`Agent context ${id} not found`); + } + + const { encrypted, iv } = encryptToken(token, context.organization_id); + const hint = getTokenHint(token); + + await query( + `UPDATE agent_contexts + SET + auth_token_encrypted = $1, + auth_token_iv = $2, + auth_token_hint = $3, + updated_at = NOW() + WHERE id = $4`, + [encrypted, iv, hint, id] + ); + } + + /** + * Get decrypted auth token (for internal use only - NEVER expose to users) + * Returns null if no token stored + */ + async getAuthToken(id: string): Promise { + const result = await query( + `SELECT organization_id, auth_token_encrypted, auth_token_iv + FROM agent_contexts + WHERE id = $1`, + [id] + ); + + const row = result.rows[0]; + if (!row || !row.auth_token_encrypted || !row.auth_token_iv) { + return null; + } + + return decryptToken(row.auth_token_encrypted, row.auth_token_iv, row.organization_id); + } + + /** + * Get auth token by org and URL (for test_adcp_agent tool) + */ + async getAuthTokenByOrgAndUrl(organizationId: string, agentUrl: string): Promise { + const result = await query( + `SELECT id, auth_token_encrypted, auth_token_iv + FROM agent_contexts + WHERE organization_id = $1 AND agent_url = $2`, + [organizationId, agentUrl] + ); + + const row = result.rows[0]; + if (!row || !row.auth_token_encrypted || !row.auth_token_iv) { + return null; + } + + return decryptToken(row.auth_token_encrypted, row.auth_token_iv, organizationId); + } + + /** + * Remove auth token + */ + async removeAuthToken(id: string): Promise { + await query( + `UPDATE agent_contexts + SET + auth_token_encrypted = NULL, + auth_token_iv = NULL, + auth_token_hint = NULL, + updated_at = NOW() + WHERE id = $1`, + [id] + ); + } + + /** + * Delete an agent context + */ + async delete(id: string): Promise { + const result = await query('DELETE FROM agent_contexts WHERE id = $1', [id]); + return (result.rowCount ?? 0) > 0; + } + + /** + * Record a test run + */ + async recordTest(input: RecordTestInput): Promise { + // Update the agent context + await query( + `UPDATE agent_contexts + SET + last_test_scenario = $1, + last_test_passed = $2, + last_test_summary = $3, + last_tested_at = NOW(), + total_tests_run = total_tests_run + 1, + updated_at = NOW() + WHERE id = $4`, + [input.scenario, input.overall_passed, input.summary || null, input.agent_context_id] + ); + + // Insert history record + const result = await query( + `INSERT INTO agent_test_history ( + agent_context_id, + scenario, + overall_passed, + steps_passed, + steps_failed, + total_duration_ms, + summary, + dry_run, + brief, + triggered_by, + user_id, + steps_json, + agent_profile_json, + completed_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, NOW()) + RETURNING *`, + [ + input.agent_context_id, + input.scenario, + input.overall_passed, + input.steps_passed, + input.steps_failed, + input.total_duration_ms || null, + input.summary || null, + input.dry_run ?? true, + input.brief || null, + input.triggered_by || null, + input.user_id || null, + input.steps_json ? JSON.stringify(input.steps_json) : null, + input.agent_profile_json ? JSON.stringify(input.agent_profile_json) : null, + ] + ); + + return result.rows[0]; + } + + /** + * Get test history for an agent + */ + async getTestHistory(agentContextId: string, limit: number = 20): Promise { + const result = await query( + `SELECT * + FROM agent_test_history + WHERE agent_context_id = $1 + ORDER BY started_at DESC + LIMIT $2`, + [agentContextId, limit] + ); + return result.rows; + } + + /** + * Infer agent type from discovered tools + */ + inferAgentType(tools: string[]): AgentType { + if (tools.includes('get_products') || tools.includes('create_media_buy')) { + return 'sales'; + } + if (tools.includes('list_creative_formats') && !tools.includes('get_products')) { + return 'creative'; + } + if (tools.includes('get_signals') || tools.includes('activate_signal')) { + return 'signals'; + } + return 'unknown'; + } +} diff --git a/server/src/db/migrations/109_agent_contexts.sql b/server/src/db/migrations/109_agent_contexts.sql new file mode 100644 index 000000000..e80b6af7f --- /dev/null +++ b/server/src/db/migrations/109_agent_contexts.sql @@ -0,0 +1,136 @@ +-- Migration: 079_agent_contexts.sql +-- Agent Testing Context System +-- +-- Stores agent URLs that users are working on, their test history, +-- and securely stored auth tokens (encrypted). + +-- ===================================================== +-- AGENT CONTEXTS TABLE +-- ===================================================== +-- Tracks agents that organizations are developing/testing + +CREATE TABLE IF NOT EXISTS agent_contexts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- Organization scope + organization_id TEXT NOT NULL REFERENCES organizations(workos_organization_id), + + -- Agent identification + agent_url TEXT NOT NULL, -- The agent's MCP/A2A endpoint + agent_name TEXT, -- Friendly name like "Our Production Sales Agent" + agent_type TEXT DEFAULT 'unknown' -- 'sales' | 'creative' | 'signals' | 'unknown' + CHECK (agent_type IN ('sales', 'creative', 'signals', 'unknown')), + protocol TEXT DEFAULT 'mcp' -- 'mcp' | 'a2a' + CHECK (protocol IN ('mcp', 'a2a')), + + -- Secure token storage + -- Token is encrypted with AES-256-GCM using org-specific derived key + -- NEVER expose the actual token in responses or logs + auth_token_encrypted TEXT, -- Encrypted token (base64) + auth_token_iv TEXT, -- Initialization vector (base64) + auth_token_hint TEXT, -- Last 4 chars for display: "****ABCD" + + -- Discovery cache (updated after each test) + tools_discovered TEXT[], -- ['get_products', 'create_media_buy', ...] + last_discovered_at TIMESTAMPTZ, + + -- Test history + last_test_scenario TEXT, -- Most recent scenario run + last_test_passed BOOLEAN, -- Did it pass? + last_test_summary TEXT, -- Brief summary + last_tested_at TIMESTAMPTZ, + total_tests_run INTEGER DEFAULT 0, + + -- Metadata + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + created_by TEXT, -- WorkOS user ID who added it + + -- One agent URL per organization + UNIQUE(organization_id, agent_url) +); + +-- Indexes +CREATE INDEX IF NOT EXISTS idx_agent_contexts_org ON agent_contexts(organization_id); +CREATE INDEX IF NOT EXISTS idx_agent_contexts_type ON agent_contexts(agent_type); +CREATE INDEX IF NOT EXISTS idx_agent_contexts_updated ON agent_contexts(updated_at DESC); + +COMMENT ON TABLE agent_contexts IS 'Agent URLs and test history for each organization'; +COMMENT ON COLUMN agent_contexts.auth_token_encrypted IS 'AES-256-GCM encrypted auth token - NEVER expose'; +COMMENT ON COLUMN agent_contexts.auth_token_hint IS 'Last 4 chars of token for display (e.g., ****ABCD)'; +COMMENT ON COLUMN agent_contexts.tools_discovered IS 'Cached list of tools from last discovery'; + +-- ===================================================== +-- AGENT TEST HISTORY TABLE +-- ===================================================== +-- Detailed history of test runs (for debugging and analysis) + +CREATE TABLE IF NOT EXISTS agent_test_history ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + + -- Link to agent context + agent_context_id UUID NOT NULL REFERENCES agent_contexts(id) ON DELETE CASCADE, + + -- Test details + scenario TEXT NOT NULL, + overall_passed BOOLEAN NOT NULL, + steps_passed INTEGER NOT NULL DEFAULT 0, + steps_failed INTEGER NOT NULL DEFAULT 0, + total_duration_ms INTEGER, + summary TEXT, + + -- Options used + dry_run BOOLEAN DEFAULT TRUE, + brief TEXT, -- Custom brief if provided + + -- Who ran it + triggered_by TEXT, -- 'user' | 'scheduled' | 'api' + user_id TEXT, -- WorkOS user ID if user-triggered + + -- Results (stored as JSON for flexibility) + steps_json JSONB, -- Full step results + agent_profile_json JSONB, -- Discovered agent profile + + -- Timestamps + started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ +); + +-- Indexes +CREATE INDEX IF NOT EXISTS idx_test_history_agent ON agent_test_history(agent_context_id); +CREATE INDEX IF NOT EXISTS idx_test_history_started ON agent_test_history(started_at DESC); +CREATE INDEX IF NOT EXISTS idx_test_history_scenario ON agent_test_history(scenario); +CREATE INDEX IF NOT EXISTS idx_test_history_passed ON agent_test_history(overall_passed); + +COMMENT ON TABLE agent_test_history IS 'Detailed test run history for debugging and analysis'; +COMMENT ON COLUMN agent_test_history.steps_json IS 'Full TestStepResult[] as JSON'; + +-- ===================================================== +-- VIEW: AGENT CONTEXT SUMMARY +-- ===================================================== +-- Summary view for displaying agent contexts to users + +CREATE OR REPLACE VIEW agent_context_summary AS +SELECT + ac.id, + ac.organization_id, + ac.agent_url, + ac.agent_name, + ac.agent_type, + ac.protocol, + ac.auth_token_hint, + ac.auth_token_encrypted IS NOT NULL as has_auth_token, + ac.tools_discovered, + ac.last_test_scenario, + ac.last_test_passed, + ac.last_test_summary, + ac.last_tested_at, + ac.total_tests_run, + ac.created_at, + ac.updated_at, + -- Aggregated stats from history + (SELECT COUNT(*) FROM agent_test_history h WHERE h.agent_context_id = ac.id) as history_count, + (SELECT COUNT(*) FROM agent_test_history h WHERE h.agent_context_id = ac.id AND h.overall_passed) as history_passed_count +FROM agent_contexts ac; + +COMMENT ON VIEW agent_context_summary IS 'Agent contexts with token visibility hidden and stats aggregated'; diff --git a/server/tsconfig.json b/server/tsconfig.json index 37c5ef22c..9e6050d72 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -3,7 +3,7 @@ "target": "ES2022", "module": "ES2022", "lib": ["ES2022"], - "moduleResolution": "node", + "moduleResolution": "bundler", "outDir": "../dist", "rootDir": "./src", "strict": true,