diff --git a/packages/botonic-core/src/models/ai-agents.ts b/packages/botonic-core/src/models/ai-agents.ts index f2b8defb81..eb93e94489 100644 --- a/packages/botonic-core/src/models/ai-agents.ts +++ b/packages/botonic-core/src/models/ai-agents.ts @@ -115,14 +115,45 @@ export interface HubtypeUserMessage { content: string } -export interface AiAgentArgs { +export enum AiAgentType { + Worker = 'worker', + Router = 'router', + Manager = 'manager', +} + +export type AiAgentArgs = + | AiAgentWorkerArgs + | AIAgentRouterArgs + | AIAgentManagerArgs + +export type AiAgentBaseArgs = { + type: AiAgentType name: string instructions: string model: string verbosity: VerbosityLevel - activeTools?: { name: string }[] inputGuardrailRules?: GuardrailRule[] - sourceIds?: string[] previousHubtypeMessages?: HubtypeAssistantMessage[] outputMessagesSchemas?: z.ZodObject[] } + +export interface AiAgentWorkerArgs extends AiAgentBaseArgs { + type: AiAgentType.Worker + activeTools: { name: string }[] + sourceIds: string[] +} + +interface AIAgentDataWithDescription extends AiAgentWorkerArgs { + description: string +} + +export interface AIAgentRouterArgs extends AiAgentBaseArgs { + type: AiAgentType.Router + agents: AIAgentDataWithDescription[] +} + +export interface AIAgentManagerArgs extends AiAgentBaseArgs { + type: AiAgentType.Manager + agents: AIAgentDataWithDescription[] + activeTools: { name: string }[] +} diff --git a/packages/botonic-plugin-ai-agents/src/agent-builder.ts b/packages/botonic-plugin-ai-agents/src/agent-builder.ts index 3b17cdc3d3..9d3505e3fa 100644 --- a/packages/botonic-plugin-ai-agents/src/agent-builder.ts +++ b/packages/botonic-plugin-ai-agents/src/agent-builder.ts @@ -3,16 +3,25 @@ import { Agent, type AgentOutputType, type InputGuardrail, + type ModelSettings, } from '@openai/agents' import type { z } from 'zod' import { OPENAI_PROVIDER } from './constants' import type { DebugLogger } from './debug-logger' -import { createInputGuardrail } from './guardrails' +import { createInputGuardrails } from './guardrails' import type { GuardrailTrackingContext } from './guardrails/input' import type { LLMConfig } from './llm-config' -import { getOutputSchema, type OutputSchema } from './structured-output' -import { mandatoryTools, retrieveKnowledge } from './tools' +import { + getOutputInstructions, + getOutputSchema, + type OutputSchema, +} from './structured-output' +import { + createRetrieveKnowledge, + mandatoryTools, + RETRIEVE_KNOWLEDGE_TOOL_NAME, +} from './tools' import type { AIAgent, Context, GuardrailRule, Tool } from './types' interface AIAgentBuilderOptions< @@ -43,6 +52,8 @@ export class AIAgentBuilder< private inputGuardrails: InputGuardrail[] public llmConfig: LLMConfig private logger: DebugLogger + private inputGuardrailRules: GuardrailRule[] + private guardrailTrackingContext: GuardrailTrackingContext constructor(options: AIAgentBuilderOptions) { this.name = options.name @@ -56,33 +67,32 @@ export class AIAgentBuilder< this.inputGuardrails = [] this.llmConfig = options.llmConfig this.logger = options.logger - if (options.inputGuardrailRules.length > 0) { - const inputGuardrail = createInputGuardrail( - options.inputGuardrailRules, - options.llmConfig, - options.guardrailTrackingContext - ) - this.inputGuardrails.push(inputGuardrail) - } + this.inputGuardrailRules = options.inputGuardrailRules + this.guardrailTrackingContext = options.guardrailTrackingContext } - build(): AIAgent { - // When using standard OpenAI API, we need to specify the model - // Azure OpenAI uses deployment name instead - + async build(): Promise> { + // When using standard OpenAI API, we need to specify the model. + // Azure OpenAI uses deployment name instead. const model = this.llmConfig.modelName - const hasRetrieveKnowledge = this.tools.includes(retrieveKnowledge) + const resolvedModel = await this.llmConfig.getModel() + const hasRetrieveKnowledge = this.tools.some( + tool => tool.name === RETRIEVE_KNOWLEDGE_TOOL_NAME + ) + const modelSettings = this.getAgentModelSettings(hasRetrieveKnowledge) + + this.inputGuardrails = await createInputGuardrails( + this.inputGuardrailRules, + this.llmConfig, + this.guardrailTrackingContext + ) this.logger.logModelSettings({ provider: OPENAI_PROVIDER, model, - reasoning: this.llmConfig.modelSettings.reasoning as - | { effort: string } - | undefined, - text: this.llmConfig.modelSettings.text as - | { verbosity: string } - | undefined, - toolChoice: this.llmConfig.modelSettings.toolChoice as string | undefined, + reasoning: modelSettings.reasoning as { effort: string } | undefined, + text: modelSettings.text as { verbosity: string } | undefined, + toolChoice: modelSettings.toolChoice as string | undefined, hasRetrieveKnowledge, }) @@ -91,7 +101,8 @@ export class AIAgentBuilder< AgentOutputType >({ name: this.name, - model, + model: resolvedModel, + modelSettings, instructions: this.instructions, tools: this.tools, outputType: getOutputSchema(this.externalOutputMessagesSchemas), @@ -100,6 +111,23 @@ export class AIAgentBuilder< }) } + private getAgentModelSettings(hasRetrieveKnowledge: boolean): ModelSettings { + const modelSettings: ModelSettings = { ...this.llmConfig.modelSettings } + if (this.llmConfig.modelSettings.reasoning) { + modelSettings.reasoning = { ...this.llmConfig.modelSettings.reasoning } + } + if (this.llmConfig.modelSettings.text) { + modelSettings.text = { ...this.llmConfig.modelSettings.text } + } + + if (hasRetrieveKnowledge) { + // && this.llmConfig.modelName.includes('gpt-4')) { + modelSettings.toolChoice = RETRIEVE_KNOWLEDGE_TOOL_NAME + } + + return modelSettings + } + private addExtraInstructions( initialInstructions: string, contactInfo: ContactInfo[], @@ -109,7 +137,7 @@ export class AIAgentBuilder< const metadataInstructions = this.getMetadataInstructions() const contactInfoInstructions = this.getContactInfoInstructions(contactInfo) const campaignInstructions = this.getCampaignInstructions(campaignsContext) - const outputInstructions = this.getOutputInstructions() + const outputInstructions = getOutputInstructions() return `${instructions}\n\n${metadataInstructions}\n\n${contactInfoInstructions}\n\n${campaignInstructions}\n\n${outputInstructions}` } @@ -155,28 +183,13 @@ export class AIAgentBuilder< .join('\n') } - private getOutputInstructions(): string { - const example = { - messages: [ - { - type: 'text', - content: { - text: 'Hello, how can I help you today?', - }, - }, - ], - } - const output = `Return a JSON that follows the output schema provided. Never return multiple output schemas concatenated by a line break.\n\n${JSON.stringify(example)}\n` - return `\n${output}\n` - } - private addHubtypeTools( tools: Tool[], sourceIds: string[] ): Tool[] { const hubtypeTools: Tool[] = [...mandatoryTools] if (sourceIds.length > 0) { - hubtypeTools.push(retrieveKnowledge) + hubtypeTools.push(createRetrieveKnowledge(sourceIds)) } return [...hubtypeTools, ...tools] } diff --git a/packages/botonic-plugin-ai-agents/src/agent-manager-builder.ts b/packages/botonic-plugin-ai-agents/src/agent-manager-builder.ts new file mode 100644 index 0000000000..fc18cec06e --- /dev/null +++ b/packages/botonic-plugin-ai-agents/src/agent-manager-builder.ts @@ -0,0 +1,99 @@ +import type { CampaignV2, ContactInfo, ResolvedPlugins } from '@botonic/core' +import { + Agent, + type AgentOutputType, + type Handoff, + type ModelSettings, +} from '@openai/agents' +import type { z } from 'zod' + +import { createInputGuardrails } from './guardrails' +import type { GuardrailTrackingContext } from './guardrails/input' +import type { LLMConfig } from './llm-config' +import { + getOutputInstructions, + getOutputSchema, + type OutputSchema, +} from './structured-output' +import type { AIAgent, Context, GuardrailRule, Tool } from './types' + +interface AIAgentManagerBuilderOptions< + TPlugins extends ResolvedPlugins = ResolvedPlugins, + TExtraData = unknown, +> { + name: string + instructions: string + tools: Tool[] + campaignsContext?: CampaignV2[] + contactInfo: ContactInfo[] + llmConfig: LLMConfig + inputGuardrailRules: GuardrailRule[] + outputMessagesSchemas?: z.ZodObject[] + guardrailTrackingContext: GuardrailTrackingContext +} + +export class AIAgentManagerBuilder< + TPlugins extends ResolvedPlugins = ResolvedPlugins, + TExtraData = unknown, +> { + private name: string + private instructions: string + private tools: Tool[] + private campaignsContext?: CampaignV2[] + private contactInfo: ContactInfo[] + private llmConfig: LLMConfig + private handoffs: Handoff< + Context, + AgentOutputType + >[] + private inputGuardrailRules: GuardrailRule[] + private outputMessagesSchemas: z.ZodObject[] + private guardrailTrackingContext: GuardrailTrackingContext + + constructor(options: AIAgentManagerBuilderOptions) { + this.name = options.name + this.instructions = options.instructions + this.tools = options.tools + this.campaignsContext = options.campaignsContext + this.contactInfo = options.contactInfo + this.llmConfig = options.llmConfig + this.inputGuardrailRules = options.inputGuardrailRules + this.outputMessagesSchemas = options.outputMessagesSchemas || [] + this.guardrailTrackingContext = options.guardrailTrackingContext + } + + async build(): Promise> { + const inputGuardrails = await createInputGuardrails( + this.inputGuardrailRules, + this.llmConfig, + this.guardrailTrackingContext + ) + const modelSettings = this.getAgentModelSettings() + const resolvedModel = await this.llmConfig.getModel() + + return new Agent< + Context, + AgentOutputType + >({ + name: this.name, + model: resolvedModel, + modelSettings, + instructions: `${this.instructions}\n\n${getOutputInstructions()}`, + tools: this.tools, + outputType: getOutputSchema(this.outputMessagesSchemas), + inputGuardrails, + outputGuardrails: [], + }) + } + + private getAgentModelSettings(): ModelSettings { + const modelSettings: ModelSettings = { ...this.llmConfig.modelSettings } + if (this.llmConfig.modelSettings.reasoning) { + modelSettings.reasoning = { ...this.llmConfig.modelSettings.reasoning } + } + if (this.llmConfig.modelSettings.text) { + modelSettings.text = { ...this.llmConfig.modelSettings.text } + } + return modelSettings + } +} diff --git a/packages/botonic-plugin-ai-agents/src/agent-router-builder.ts b/packages/botonic-plugin-ai-agents/src/agent-router-builder.ts new file mode 100644 index 0000000000..e85cb0d4f4 --- /dev/null +++ b/packages/botonic-plugin-ai-agents/src/agent-router-builder.ts @@ -0,0 +1,94 @@ +import type { ResolvedPlugins } from '@botonic/core' +import { + Agent, + type AgentOutputType, + type Handoff, + type ModelSettings, +} from '@openai/agents' +import { RECOMMENDED_PROMPT_PREFIX } from '@openai/agents-core/extensions' +import type { z } from 'zod' + +import { createInputGuardrails } from './guardrails' +import type { GuardrailTrackingContext } from './guardrails/input' +import type { LLMConfig } from './llm-config' +import { + getOutputInstructions, + getOutputSchema, + type OutputSchema, +} from './structured-output' +import type { AIAgent, Context, GuardrailRule } from './types' + +interface AIAgentRouterBuilderOptions< + TPlugins extends ResolvedPlugins = ResolvedPlugins, + TExtraData = unknown, +> { + name: string + instructions: string + llmConfig: LLMConfig + handoffs: Handoff< + Context, + AgentOutputType + >[] + inputGuardrailRules: GuardrailRule[] + outputMessagesSchemas?: z.ZodObject[] + guardrailTrackingContext: GuardrailTrackingContext +} + +export class AIAgentRouterBuilder< + TPlugins extends ResolvedPlugins = ResolvedPlugins, + TExtraData = unknown, +> { + private name: string + private instructions: string + private llmConfig: LLMConfig + private handoffs: Handoff< + Context, + AgentOutputType + >[] + private inputGuardrailRules: GuardrailRule[] + private outputMessagesSchemas: z.ZodObject[] + private guardrailTrackingContext: GuardrailTrackingContext + + constructor(options: AIAgentRouterBuilderOptions) { + this.name = options.name + this.instructions = options.instructions + this.llmConfig = options.llmConfig + this.handoffs = options.handoffs + this.inputGuardrailRules = options.inputGuardrailRules + this.outputMessagesSchemas = options.outputMessagesSchemas || [] + this.guardrailTrackingContext = options.guardrailTrackingContext + } + + async build(): Promise> { + const inputGuardrails = await createInputGuardrails( + this.inputGuardrailRules, + this.llmConfig, + this.guardrailTrackingContext + ) + const modelSettings = this.getAgentModelSettings() + + // Agent.create is typed as Agent; we run with Context. + const agent = Agent.create({ + name: this.name, + model: await this.llmConfig.getModel(), + modelSettings, + instructions: `${RECOMMENDED_PROMPT_PREFIX}${this.instructions}\n\n${getOutputInstructions()}`, + handoffs: this.handoffs, + outputType: getOutputSchema(this.outputMessagesSchemas), + inputGuardrails, + }) as AIAgent + + return agent + } + + private getAgentModelSettings(): ModelSettings { + const modelSettings: ModelSettings = { ...this.llmConfig.modelSettings } + if (this.llmConfig.modelSettings.reasoning) { + modelSettings.reasoning = { ...this.llmConfig.modelSettings.reasoning } + } + if (this.llmConfig.modelSettings.text) { + modelSettings.text = { ...this.llmConfig.modelSettings.text } + } + return modelSettings + } +} diff --git a/packages/botonic-plugin-ai-agents/src/debug-logger.ts b/packages/botonic-plugin-ai-agents/src/debug-logger.ts index 8a9ec09dbe..a7783fbd44 100644 --- a/packages/botonic-plugin-ai-agents/src/debug-logger.ts +++ b/packages/botonic-plugin-ai-agents/src/debug-logger.ts @@ -1,4 +1,8 @@ -import type { AiAgentArgs, ToolExecution } from '@botonic/core' +import { + type AiAgentArgs, + AiAgentType, + type ToolExecution, +} from '@botonic/core' import type { ModelSettings } from '@openai/agents' import { OPENAI_PROVIDER } from './constants' import type { AgenticInputMessage, MemoryOptions, RunResult } from './types' @@ -66,9 +70,11 @@ class EnabledDebugLogger implements DebugLogger { console.log(`${PREFIX} === AI Agent Debug Info ===`) console.log(`${PREFIX} Agent Name: ${aiAgentArgs.name}`) console.log(`${PREFIX} Active Tools: ${JSON.stringify(toolNames)}`) - console.log( - `${PREFIX} Source IDs: ${JSON.stringify(aiAgentArgs.sourceIds || [])}` - ) + if (aiAgentArgs.type === AiAgentType.Worker) { + console.log( + `${PREFIX} Source IDs: ${JSON.stringify(aiAgentArgs.sourceIds || [])}` + ) + } console.log(`${PREFIX} Message History Count: ${messages.length}`) console.log( `${PREFIX} Input Guardrail Rules: ${aiAgentArgs.inputGuardrailRules?.length || 0}` diff --git a/packages/botonic-plugin-ai-agents/src/guardrails/input.ts b/packages/botonic-plugin-ai-agents/src/guardrails/input.ts index a65c09f5d6..6b11f50d30 100644 --- a/packages/botonic-plugin-ai-agents/src/guardrails/input.ts +++ b/packages/botonic-plugin-ai-agents/src/guardrails/input.ts @@ -1,6 +1,7 @@ import { Agent, type InputGuardrail, + type ModelSettings, Runner, type UserMessageItem, } from '@openai/agents' @@ -18,20 +19,41 @@ export interface GuardrailTrackingContext { inferenceId: string } -export function createInputGuardrail( +export async function createInputGuardrails( rules: GuardrailRule[], llmConfig: LLMConfig, trackingContext: GuardrailTrackingContext -): InputGuardrail { +): Promise { + if (rules.length === 0) { + return [] + } + + return [await buildInputGuardrail(rules, llmConfig, trackingContext)] +} + +async function buildInputGuardrail( + rules: GuardrailRule[], + llmConfig: LLMConfig, + trackingContext: GuardrailTrackingContext +): Promise { const outputType = z.object( Object.fromEntries( rules.map(rule => [rule.name, z.boolean().describe(rule.description)]) ) ) + const modelSettings: ModelSettings = { ...llmConfig.modelSettings } + if (llmConfig.modelSettings.reasoning) { + modelSettings.reasoning = { ...llmConfig.modelSettings.reasoning } + } + if (llmConfig.modelSettings.text) { + modelSettings.text = { ...llmConfig.modelSettings.text } + } + delete modelSettings.toolChoice const agent = new Agent({ name: 'InputGuardrail', - model: llmConfig.modelName, + model: await llmConfig.getModel(), + modelSettings, instructions: 'Check if the user triggers some of the following guardrails.', outputType, @@ -41,12 +63,7 @@ export function createInputGuardrail( name: 'InputGuardrail', execute: async ({ input, context }) => { const lastMessage = input[input.length - 1] as UserMessageItem - const modelProvider = llmConfig.modelProvider - const modelSettings = llmConfig.modelSettings - modelSettings.toolChoice = undefined const runner = new Runner({ - modelSettings, - modelProvider, tracingDisabled: true, }) const startTime = Date.now() @@ -105,7 +122,7 @@ async function sendGuardrailLlmRunTracking( product_name: TrackProductName.AI_AGENT, deployment_name: llmConfig.modelName, model_name: - (response.providerData?.['model'] as string | undefined) ?? + (response.providerData?.model as string | undefined) ?? llmConfig.modelName, feature: TrackFeature.AI_AGENT_GUARDRAIL, api_version: apiVersion, diff --git a/packages/botonic-plugin-ai-agents/src/index.ts b/packages/botonic-plugin-ai-agents/src/index.ts index dfe7a0b755..5dcec403fc 100644 --- a/packages/botonic-plugin-ai-agents/src/index.ts +++ b/packages/botonic-plugin-ai-agents/src/index.ts @@ -1,13 +1,21 @@ -import type { - AiAgentArgs, - BotContext, - HubtypeAssistantMessage, - Plugin, - ResolvedPlugins, +import { + type AIAgentManagerArgs, + type AIAgentRouterArgs, + type AiAgentArgs, + AiAgentType, + type AiAgentWorkerArgs, + type BotContext, + type HubtypeAssistantMessage, + type Plugin, + type ResolvedPlugins, } from '@botonic/core' -import { setTracingDisabled, tool } from '@openai/agents' +import { handoff, setTracingDisabled, tool } from '@openai/agents' import { v7 as uuidv7 } from 'uuid' +import type { ZodObject } from 'zod' + import { AIAgentBuilder } from './agent-builder' +import { AIAgentManagerBuilder } from './agent-manager-builder' +import { AIAgentRouterBuilder } from './agent-router-builder' import { DEFAULT_MAX_RETRIES, DEFAULT_TIMEOUT_16_SECONDS, @@ -17,6 +25,8 @@ import { import { createDebugLogger, type DebugLogger } from './debug-logger' import { LLMConfig } from './llm-config' import { AIAgentRunner } from './runner' +import { AIAgentManagerRunner } from './runner-manager' +import { AIAgentRouterRunner } from './runner-router' import { HubtypeApiClient } from './services/hubtype-api-client' import type { AgenticInputMessage, @@ -77,84 +87,42 @@ export default class BotonicPluginAiAgents< botContext: BotContext, aiAgentArgs: AiAgentArgs ): Promise { - try { - const authToken = isProd - ? botContext.session._access_token - : this.authToken - if (!authToken) { - throw new Error('Auth token is required') - } - - const inferenceId = uuidv7() - - // Create client for OpenAI/Azure OpenAI - const llmConfig = new LLMConfig( - this.maxRetries, - this.timeout, - aiAgentArgs.model, - aiAgentArgs.verbosity - ) + const authToken = isProd ? botContext.session._access_token : this.authToken + if (!authToken) { + throw new Error('Auth token is required') + } - // Build tools - const tools = this.buildTools( - aiAgentArgs.activeTools?.map(tool => tool.name) || [] - ) + const inferenceId = uuidv7() - // Build agent - const agent = new AIAgentBuilder({ - name: aiAgentArgs.name, - instructions: aiAgentArgs.instructions, - tools: tools, - contactInfo: botContext.session.user.contact_info || [], - inputGuardrailRules: aiAgentArgs.inputGuardrailRules || [], - sourceIds: aiAgentArgs.sourceIds || [], - outputMessagesSchemas: aiAgentArgs.outputMessagesSchemas || [], - campaignsContext: botContext.input.context?.campaigns_v2, - logger: this.logger, - llmConfig, - guardrailTrackingContext: { - botId: botContext.session.bot.id, - isTest: botContext.session.is_test_integration, + try { + if (aiAgentArgs.type === AiAgentType.Worker) { + return await this.executeWorkerAIAgent( + botContext, + aiAgentArgs, authToken, - inferenceId, - }, - }).build() - - // Get messages - const messages = await this.getMessages( - botContext, - authToken, - aiAgentArgs.previousHubtypeMessages || [] - ) + inferenceId + ) + } - // Build context - const context: Context = { - authToken, - sourceIds: aiAgentArgs.sourceIds || [], - knowledgeUsed: { - query: '', - sourceIds: [], - chunksIds: [], - chunkTexts: [], - }, - request: botContext, + if (aiAgentArgs.type === AiAgentType.Router) { + return await this.executeRouterAIAgent( + botContext, + aiAgentArgs, + authToken, + inferenceId + ) } - // Log agent debug info - this.logger.logAgentDebugInfo( - aiAgentArgs, - tools.map(t => t.name), - messages - ) + if (aiAgentArgs.type === AiAgentType.Manager) { + return await this.executeManagerAIAgent( + botContext, + aiAgentArgs, + authToken, + inferenceId + ) + } - // Run agent - const runner = new AIAgentRunner( - agent, - llmConfig, - inferenceId, - this.logger - ) - return await runner.run(messages, context) + throw new Error('Invalid agent type') } catch (error) { console.error('error plugin returns undefined', error) return { @@ -169,6 +137,268 @@ export default class BotonicPluginAiAgents< } } + private async executeWorkerAIAgent( + botContext: BotContext, + aiAgentArgs: AiAgentWorkerArgs, + authToken: string, + inferenceId: string + ) { + const llmConfig = new LLMConfig( + this.maxRetries, + this.timeout, + aiAgentArgs.model, + aiAgentArgs.verbosity + ) + + // Get LLM config, tools and agent + const { tools, agent } = await this.getAIAgentWorkerAndTools( + botContext, + aiAgentArgs, + aiAgentArgs.outputMessagesSchemas || [], + authToken, + inferenceId, + llmConfig + ) + + // Get messages + const messages = await this.getMessages( + botContext, + authToken, + aiAgentArgs.previousHubtypeMessages || [] + ) + + // Build context + const context: Context = { + authToken, + knowledgeUsed: { + query: '', + sourceIds: [], + chunksIds: [], + chunkTexts: [], + }, + request: botContext, + } + + // Log agent debug info + this.logger.logAgentDebugInfo( + aiAgentArgs, + tools.map(t => t.name), + messages + ) + + // Run agent + const runner = new AIAgentRunner( + agent, + llmConfig, + inferenceId, + this.logger + ) + return await runner.run(messages, context) + } + + private async executeRouterAIAgent( + botContext: BotContext, + aiAgentArgs: AIAgentRouterArgs, + authToken: string, + inferenceId: string + ) { + const { agents, name, instructions } = aiAgentArgs + + const llmConfig = new LLMConfig( + this.maxRetries, + this.timeout, + aiAgentArgs.model, + aiAgentArgs.verbosity + ) + + const handoffAgents = await Promise.all( + agents.map(async aiAgentData => { + const { agent } = await this.getAIAgentWorkerAndTools( + botContext, + aiAgentData, + aiAgentArgs.outputMessagesSchemas || [], + authToken, + inferenceId, + llmConfig + ) + return handoff(agent, { + toolNameOverride: aiAgentData.name, + toolDescriptionOverride: aiAgentData.description, + // TODO: Review if is possible use onHandoff action to track the handoff + // onHandoff: result => { + // console.log('onHandoff', aiAgentData.name, result) + // }, + // TODO: when onHandoff function is defined, we need to provide inputType + // inputType: ????, + // isEnabled: (context: RunContext) => { + // return true + // }, + }) + }) + ) + + const agentRouter = await new AIAgentRouterBuilder({ + name, + instructions, + llmConfig, + handoffs: handoffAgents, + inputGuardrailRules: aiAgentArgs.inputGuardrailRules || [], + outputMessagesSchemas: aiAgentArgs.outputMessagesSchemas || [], + guardrailTrackingContext: { + botId: botContext.session.bot.id, + isTest: botContext.session.is_test_integration, + authToken, + inferenceId, + }, + }).build() + + // Get messages + const messages = await this.getMessages( + botContext, + authToken, + aiAgentArgs.previousHubtypeMessages || [] + ) + + // Build context + const context: Context = { + authToken, + knowledgeUsed: { + query: '', + sourceIds: [], + chunksIds: [], + chunkTexts: [], + }, + request: botContext, + } + + // Run agent + const runner = new AIAgentRouterRunner( + agentRouter, + llmConfig, + inferenceId, + this.logger + ) + + return await runner.run(messages, context) + } + + private async executeManagerAIAgent( + botContext: BotContext, + aiAgentArgs: AIAgentManagerArgs, + authToken: string, + inferenceId: string + ) { + const { agents, name, instructions } = aiAgentArgs + + const llmConfig = new LLMConfig( + this.maxRetries, + this.timeout, + aiAgentArgs.model, + aiAgentArgs.verbosity + ) + + const agentsAsTools = await Promise.all( + agents.map(async aiAgentData => { + const { agent } = await this.getAIAgentWorkerAndTools( + botContext, + aiAgentData, + aiAgentArgs.outputMessagesSchemas || [], + authToken, + inferenceId, + llmConfig + ) + return agent.asTool({ + toolName: aiAgentData.name, + toolDescription: aiAgentData.description, + }) + }) + ) + + const tools = [...agentsAsTools, ...this.buildTools(aiAgentArgs)] + + console.log('Manager tools', tools) + + // TODO: Join tools with agents as tools + const agentManager = await new AIAgentManagerBuilder({ + name, + instructions, + tools, + contactInfo: botContext.session.user.contact_info || [], + inputGuardrailRules: aiAgentArgs.inputGuardrailRules || [], + guardrailTrackingContext: { + botId: botContext.session.bot.id, + isTest: botContext.session.is_test_integration, + authToken, + inferenceId, + }, + outputMessagesSchemas: aiAgentArgs.outputMessagesSchemas || [], + llmConfig, + }).build() + + const messages = await this.getMessages( + botContext, + authToken, + aiAgentArgs.previousHubtypeMessages || [] + ) + + const context: Context = { + authToken, + knowledgeUsed: { + query: '', + sourceIds: [], + chunksIds: [], + chunkTexts: [], + }, + request: botContext, + } + + const runner = new AIAgentManagerRunner( + agentManager, + llmConfig, + inferenceId, + this.logger + ) + + return await runner.run(messages, context) + } + + private async getAIAgentWorkerAndTools( + botContext: BotContext, + aiAgentArgs: AiAgentArgs, + outputMessagesSchemas: ZodObject[], + authToken: string, + inferenceId: string, + llmConfig: LLMConfig + ) { + // Build tools + const tools = this.buildTools(aiAgentArgs) + + // Build agent + const sourceIds = + aiAgentArgs.type === AiAgentType.Worker ? aiAgentArgs.sourceIds : [] + const agentBuilder = new AIAgentBuilder({ + name: aiAgentArgs.name, + instructions: aiAgentArgs.instructions, + tools: tools, + contactInfo: botContext.session.user.contact_info || [], + inputGuardrailRules: aiAgentArgs.inputGuardrailRules || [], + sourceIds, + outputMessagesSchemas: outputMessagesSchemas || [], + campaignsContext: botContext.input.context?.campaigns_v2, + logger: this.logger, + llmConfig, + guardrailTrackingContext: { + botId: botContext.session.bot.id, + isTest: botContext.session.is_test_integration, + authToken, + inferenceId, + }, + }) + const agent = await agentBuilder.build() + + return { agent, tools } + } + private async getMessages( botContext: BotContext, authToken: string, @@ -192,7 +422,10 @@ export default class BotonicPluginAiAgents< return result.messages } - private buildTools(activeToolNames: string[]): Tool[] { + private buildTools(aiAgentArgs: AiAgentArgs): Tool[] { + const activeTools = + aiAgentArgs.type === AiAgentType.Router ? [] : aiAgentArgs.activeTools + const activeToolNames = activeTools.map(tool => tool.name) const availableTools = this.toolDefinitions.filter(tool => activeToolNames.includes(tool.name) ) diff --git a/packages/botonic-plugin-ai-agents/src/llm-config.ts b/packages/botonic-plugin-ai-agents/src/llm-config.ts index 8cf4a91e7a..ef483f0145 100644 --- a/packages/botonic-plugin-ai-agents/src/llm-config.ts +++ b/packages/botonic-plugin-ai-agents/src/llm-config.ts @@ -1,5 +1,6 @@ import type { VerbosityLevel } from '@botonic/core' import { + type Model, type ModelProvider, type ModelSettings, OpenAIProvider, @@ -35,6 +36,10 @@ export class LLMConfig { this.modelSettings = this.getModelSettings(modelName, verbosity) } + async getModel(): Promise { + return await this.modelProvider.getModel(this.modelName) + } + private getModelProvider(): ModelProvider { const client = this.getClient() return new OpenAIProvider({ diff --git a/packages/botonic-plugin-ai-agents/src/runner-manager.ts b/packages/botonic-plugin-ai-agents/src/runner-manager.ts new file mode 100644 index 0000000000..fc72a8d4ba --- /dev/null +++ b/packages/botonic-plugin-ai-agents/src/runner-manager.ts @@ -0,0 +1,96 @@ +import type { AgenticOutputMessage, ResolvedPlugins } from '@botonic/core' +import { InputGuardrailTripwireTriggered, Runner } from '@openai/agents' +import type { DebugLogger } from './debug-logger' +import type { LLMConfig } from './llm-config' +import type { AIAgentRunnerResult } from './runner' +import type { AgenticInputMessage, AIAgent, Context, RunResult } from './types' + +export class AIAgentManagerRunner< + TPlugins extends ResolvedPlugins = ResolvedPlugins, + TExtraData = any, +> { + private agent: AIAgent + private llmConfig: LLMConfig + private inferenceId: string + private logger: DebugLogger + + constructor( + agent: AIAgent, + llmConfig: LLMConfig, + inferenceId: string, // TODO: Use it for tracking + logger: DebugLogger + ) { + this.agent = agent + this.llmConfig = llmConfig + this.inferenceId = inferenceId + this.logger = logger + } + + async run( + messages: AgenticInputMessage[], + context: Context + ): Promise { + const startTime = Date.now() + + this.logger.logRunnerStart( + this.llmConfig.modelName, + this.llmConfig.modelSettings + ) + + try { + const runner = new Runner({ + tracingDisabled: true, + }) + const result = (await runner.run(this.agent, messages, { + context, + })) as AIAgentRunnerResult + + // const endTime = Date.now() + + // await this.sendLlmRunTracking(result, context, startTime, endTime) + + console.log('AIAgentManagerRunner result', result) + const outputMessages = result.finalOutput?.messages || [] + const hasExit = + outputMessages.length === 0 || + outputMessages.some(message => message.type === 'exit') + + const runResult: RunResult = { + messages: hasExit + ? [] + : (outputMessages.filter( + message => message.type !== 'exit' + ) as AgenticOutputMessage[]), + toolsExecuted: [], + exit: hasExit, + memoryLength: messages.length, + error: false, + inputGuardrailsTriggered: [], + outputGuardrailsTriggered: [], + } + + this.logger.logRunResult(runResult, startTime) + + return runResult + } catch (error) { + console.error('AIAgentManagerRunner error', error) + if (error instanceof InputGuardrailTripwireTriggered) { + const runResult: RunResult = { + messages: [], + memoryLength: 0, + toolsExecuted: [], + exit: true, + error: false, + inputGuardrailsTriggered: error.result.output.outputInfo, + outputGuardrailsTriggered: [], + } + + this.logger.logGuardrailTriggered() + + return runResult + } + + throw error + } + } +} diff --git a/packages/botonic-plugin-ai-agents/src/runner-router.ts b/packages/botonic-plugin-ai-agents/src/runner-router.ts new file mode 100644 index 0000000000..38ab3efbf3 --- /dev/null +++ b/packages/botonic-plugin-ai-agents/src/runner-router.ts @@ -0,0 +1,97 @@ +import type { AgenticOutputMessage, ResolvedPlugins } from '@botonic/core' +import { InputGuardrailTripwireTriggered, Runner } from '@openai/agents' +import type { DebugLogger } from './debug-logger' +import type { LLMConfig } from './llm-config' +import type { AIAgentRunnerResult } from './runner' +import type { AgenticInputMessage, AIAgent, Context, RunResult } from './types' + +export class AIAgentRouterRunner< + TPlugins extends ResolvedPlugins = ResolvedPlugins, + TExtraData = any, +> { + private agent: AIAgent + private llmConfig: LLMConfig + private inferenceId: string + private logger: DebugLogger + + constructor( + agent: AIAgent, + llmConfig: LLMConfig, + inferenceId: string, // TODO: Use it for tracking + logger: DebugLogger + ) { + this.agent = agent + this.llmConfig = llmConfig + this.inferenceId = inferenceId + this.logger = logger + } + + async run( + messages: AgenticInputMessage[], + context: Context + ): Promise { + const startTime = Date.now() + + this.logger.logRunnerStart( + this.llmConfig.modelName, + this.llmConfig.modelSettings + ) + + try { + const runner = new Runner({ + tracingDisabled: true, + }) + const result = (await runner.run(this.agent, messages, { + context, + })) as AIAgentRunnerResult + + // const endTime = Date.now() + + // await this.sendLlmRunTracking(result, context, startTime, endTime) + + console.log('AIAgentRouterRunner result', result) + console.log('CURRENT_AGENT: ', result.state?._currentAgent?.name) + const outputMessages = result.finalOutput?.messages || [] + const hasExit = + outputMessages.length === 0 || + outputMessages.some(message => message.type === 'exit') + + const runResult: RunResult = { + messages: hasExit + ? [] + : (outputMessages.filter( + message => message.type !== 'exit' + ) as AgenticOutputMessage[]), + toolsExecuted: [], + exit: hasExit, + memoryLength: messages.length, + error: false, + inputGuardrailsTriggered: [], + outputGuardrailsTriggered: [], + } + + this.logger.logRunResult(runResult, startTime) + + return runResult + } catch (error) { + console.error('AIAgentRouterRunner error', error) + if (error instanceof InputGuardrailTripwireTriggered) { + const runResult: RunResult = { + messages: [], + memoryLength: 0, + toolsExecuted: [], + exit: true, + error: false, + inputGuardrailsTriggered: error.result.output.outputInfo, + outputGuardrailsTriggered: [], + } + + this.logger.logGuardrailTriggered() + + return runResult + } + + throw error + } + } +} diff --git a/packages/botonic-plugin-ai-agents/src/runner.ts b/packages/botonic-plugin-ai-agents/src/runner.ts index 5d43bb12a8..c575e107fa 100644 --- a/packages/botonic-plugin-ai-agents/src/runner.ts +++ b/packages/botonic-plugin-ai-agents/src/runner.ts @@ -9,12 +9,12 @@ import { RunToolCallItem, RunToolCallOutputItem, } from '@openai/agents' -import { isProd, OPENAI_PROVIDER } from './constants' +import { isProd } from './constants' import type { DebugLogger } from './debug-logger' import { getApiVersion, type LLMConfig } from './llm-config' import { HubtypeApiClient } from './services/hubtype-api-client' import { TrackFeature, TrackProductName } from './services/types' -import { retrieveKnowledge } from './tools' +import { RETRIEVE_KNOWLEDGE_TOOL_NAME } from './tools' import type { AgenticInputMessage, AgenticOutputMessage, @@ -26,7 +26,7 @@ import type { // Minimal interface matching the properties we actually use from Runner.run() result // This bypasses strict type checking while maintaining type safety for accessed properties -interface AIAgentRunnerResult { +export interface AIAgentRunnerResult { finalOutput?: { messages?: OutputMessage[] } @@ -47,12 +47,12 @@ export class AIAgentRunner< constructor( agent: AIAgent, - openAiClient: LLMConfig, + llmConfig: LLMConfig, inferenceId: string, logger: DebugLogger ) { this.agent = agent - this.llmConfig = openAiClient + this.llmConfig = llmConfig this.inferenceId = inferenceId this.logger = logger } @@ -69,17 +69,7 @@ export class AIAgentRunner< ) try { - const modelProvider = this.llmConfig.modelProvider - const modelSettings = this.llmConfig.modelSettings - - const hasRetrieveKnowledge = this.agent.tools.includes(retrieveKnowledge) - if (hasRetrieveKnowledge && OPENAI_PROVIDER === 'azure') { - modelSettings.toolChoice = retrieveKnowledge.name - } - const runner = new Runner({ - modelSettings, - modelProvider, tracingDisabled: true, }) // Type assertion to bypass strict type checking - the actual return type from runner.run() @@ -87,7 +77,6 @@ export class AIAgentRunner< const result = (await runner.run(this.agent, messages, { context, })) as AIAgentRunnerResult - const endTime = Date.now() await this.sendLlmRunTracking(result, context, startTime, endTime) @@ -166,7 +155,7 @@ export class AIAgentRunner< product_name: TrackProductName.AI_AGENT, deployment_name: this.llmConfig.modelName, model_name: - (response.providerData?.['model'] as string | undefined) ?? + (response.providerData?.model as string | undefined) ?? this.llmConfig.modelName, feature: TrackFeature.AI_AGENT_RUN, api_version: apiVersion, @@ -262,7 +251,7 @@ export class AIAgentRunner< toolResults, } - if (toolName === retrieveKnowledge.name) { + if (toolName === RETRIEVE_KNOWLEDGE_TOOL_NAME) { return { ...toolExecution, knowledgebaseSourcesIds: context.knowledgeUsed.sourceIds, diff --git a/packages/botonic-plugin-ai-agents/src/structured-output/index.ts b/packages/botonic-plugin-ai-agents/src/structured-output/index.ts index ff0cf7df30..f6d5a35129 100644 --- a/packages/botonic-plugin-ai-agents/src/structured-output/index.ts +++ b/packages/botonic-plugin-ai-agents/src/structured-output/index.ts @@ -32,3 +32,18 @@ export function getOutputSchema( ), }) } + +export function getOutputInstructions(): string { + const example = { + messages: [ + { + type: 'text', + content: { + text: 'Hello, how can I help you today?', + }, + }, + ], + } + const output = `Return a JSON that follows the output schema provided. Never return multiple output schemas concatenated by a line break.\n\n${JSON.stringify(example)}\n` + return `\n${output}\n` +} diff --git a/packages/botonic-plugin-ai-agents/src/tools/index.ts b/packages/botonic-plugin-ai-agents/src/tools/index.ts index d5a8d28e0f..cdfbfea366 100644 --- a/packages/botonic-plugin-ai-agents/src/tools/index.ts +++ b/packages/botonic-plugin-ai-agents/src/tools/index.ts @@ -1,5 +1,8 @@ import type { Tool } from '../types' -export { retrieveKnowledge } from './retrieve-knowledge' +export { + createRetrieveKnowledge, + RETRIEVE_KNOWLEDGE_TOOL_NAME, +} from './retrieve-knowledge' export const mandatoryTools: Tool[] = [] diff --git a/packages/botonic-plugin-ai-agents/src/tools/retrieve-knowledge.ts b/packages/botonic-plugin-ai-agents/src/tools/retrieve-knowledge.ts index 80a62bab20..713db649ce 100644 --- a/packages/botonic-plugin-ai-agents/src/tools/retrieve-knowledge.ts +++ b/packages/botonic-plugin-ai-agents/src/tools/retrieve-knowledge.ts @@ -4,34 +4,37 @@ import { z } from 'zod' import { HubtypeApiClient } from '../services/hubtype-api-client' import type { Context } from '../types' -export const retrieveKnowledge = tool({ - name: 'retrieve_knowledge', - description: - 'Consult the knowledge base for information before answering. Use this tool to make sure the information you provide is faithful.', - parameters: z.object({ - query: z.string().describe('The query to search the knowledge base for'), - }), - execute: async ( - input: { query: string }, - runContext?: RunContext - ): Promise => { - const context = runContext?.context - const query = input.query - if (!context) { - throw new Error('Context is required') - } - const sourceIds = context.sourceIds - const client = new HubtypeApiClient(context.authToken) - const chunks = await client.retrieveSimilarChunks(query, sourceIds) - const chunkTexts = chunks.map(chunk => chunk.text) +export const RETRIEVE_KNOWLEDGE_TOOL_NAME = 'retrieve_knowledge' - context.knowledgeUsed = { - query, - sourceIds, - chunksIds: chunks.map(chunk => chunk.id), - chunkTexts, - } +export const createRetrieveKnowledge = (sourceIds: string[]) => + tool({ + name: RETRIEVE_KNOWLEDGE_TOOL_NAME, + description: + 'Consult the knowledge base for information before answering. Use this tool to make sure the information you provide is faithful.', + parameters: z.object({ + query: z.string().describe('The query to search the knowledge base for'), + }), + execute: async ( + input: { query: string }, + runContext?: RunContext + ): Promise => { + const context = runContext?.context + const query = input.query + if (!context) { + throw new Error('Context is required') + } + const client = new HubtypeApiClient(context.authToken) + const chunks = await client.retrieveSimilarChunks(query, sourceIds) + const chunksIds = chunks.map(chunk => chunk.id) + const chunkTexts = chunks.map(chunk => chunk.text) - return chunkTexts - }, -}) + context.knowledgeUsed = { + query, + sourceIds, + chunksIds, + chunkTexts, + } + + return chunkTexts + }, + }) diff --git a/packages/botonic-plugin-ai-agents/src/types.ts b/packages/botonic-plugin-ai-agents/src/types.ts index a9be427be9..137a0d6afb 100644 --- a/packages/botonic-plugin-ai-agents/src/types.ts +++ b/packages/botonic-plugin-ai-agents/src/types.ts @@ -23,7 +23,6 @@ export interface Context< TExtraData = any, > { authToken: string - sourceIds: string[] knowledgeUsed: { query: string sourceIds: string[] diff --git a/packages/botonic-plugin-ai-agents/tests/agent-builder.test.ts b/packages/botonic-plugin-ai-agents/tests/agent-builder.test.ts index 5afa8de05a..774f7a6d02 100644 --- a/packages/botonic-plugin-ai-agents/tests/agent-builder.test.ts +++ b/packages/botonic-plugin-ai-agents/tests/agent-builder.test.ts @@ -34,11 +34,13 @@ jest.mock('@openai/agents', () => ({ })) jest.mock('../src/tools', () => ({ - mandatoryTools: [], - retrieveKnowledge: { + createRetrieveKnowledge: jest.fn((sourceIds: string[]) => ({ name: 'retrieve_knowledge', description: 'Consult the knowledge base for information before answering.', - }, + sourceIds, + })), + mandatoryTools: [], + RETRIEVE_KNOWLEDGE_TOOL_NAME: 'retrieve_knowledge', })) // Mock constants - can be overridden per test @@ -63,6 +65,7 @@ const mockGuardrailTrackingContext: GuardrailTrackingContext = { } // Mock LLMConfig for tests (builder uses modelName and modelSettings for logging) +const resolvedModel = { id: 'resolved-model' } const mockLlmConfig = { modelName: 'gpt-4.1-mini', modelSettings: { @@ -71,6 +74,7 @@ const mockLlmConfig = { toolChoice: undefined as string | undefined, }, modelProvider: {}, + getModel: jest.fn().mockResolvedValue(resolvedModel), } as unknown as LLMConfig describe('AIAgentBuilder', () => { @@ -134,8 +138,8 @@ describe('AIAgentBuilder', () => { afterEach(() => { jest.restoreAllMocks() }) - it('should initialize correctly with name, instructions and tools', () => { - const aiAgent = new AIAgentBuilder({ + it('should initialize correctly with name, instructions and tools', async () => { + const aiAgent = await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -168,6 +172,12 @@ describe('AIAgentBuilder', () => { expect(aiAgent.name).toBe(agentName) expect(aiAgent.instructions).toBe(expectedInstructions) expect(aiAgent.tools).toHaveLength(3) // 2 custom tools + 1 retrieveKnowledge tool + expect(aiAgent.tools[0]).toEqual( + expect.objectContaining({ + name: 'retrieve_knowledge', + sourceIds, + }) + ) }) describe('Structured Output Schema Validation', () => { @@ -264,8 +274,8 @@ describe('AIAgentBuilder', () => { }) describe('Campaign context handling', () => { - it('should NOT include campaign_context when campaignsContext is undefined', () => { - const aiAgent = new AIAgentBuilder({ + it('should NOT include campaign_context when campaignsContext is undefined', async () => { + const aiAgent = await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -281,7 +291,7 @@ describe('AIAgentBuilder', () => { expect(aiAgent.instructions).not.toContain(' { + it('should NOT include campaign_context when agent_context is undefined', async () => { const campaignWithoutContext = [ { id: '1234-5678-9012-3456', @@ -290,7 +300,7 @@ describe('AIAgentBuilder', () => { }, ] - const aiAgent = new AIAgentBuilder({ + const aiAgent = await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -306,7 +316,7 @@ describe('AIAgentBuilder', () => { expect(aiAgent.instructions).not.toContain(' { + it('should NOT include campaign_context when agent_context is empty string', async () => { const campaignWithEmptyContext = [ { id: '1234-5678-9012-3456', @@ -315,7 +325,7 @@ describe('AIAgentBuilder', () => { }, ] - const aiAgent = new AIAgentBuilder({ + const aiAgent = await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -332,7 +342,7 @@ describe('AIAgentBuilder', () => { expect(aiAgent.instructions).not.toContain(' { + it('should include campaign_context when agent_context has content', async () => { const campaignWithContext = [ { id: '1234-5678-9012-3456', @@ -341,7 +351,7 @@ describe('AIAgentBuilder', () => { }, ] - const aiAgent = new AIAgentBuilder({ + const aiAgent = await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -362,8 +372,8 @@ describe('AIAgentBuilder', () => { }) describe('outputMessagesSchemas handling', () => { - it('should build with only base schemas when outputMessagesSchemas is not provided', () => { - new AIAgentBuilder({ + it('should build with only base schemas when outputMessagesSchemas is not provided', async () => { + await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -395,7 +405,7 @@ describe('AIAgentBuilder', () => { expect(outputType.safeParse(invalidCustomMessage).success).toBe(false) }) - it('should include custom schemas when outputMessagesSchemas is provided', () => { + it('should include custom schemas when outputMessagesSchemas is provided', async () => { const customVideoSchema = z.object({ type: z.enum(['customVideo']), content: z.object({ @@ -404,7 +414,7 @@ describe('AIAgentBuilder', () => { }), }) - new AIAgentBuilder({ + await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -437,7 +447,7 @@ describe('AIAgentBuilder', () => { expect(outputType.safeParse(validBaseMessage).success).toBe(true) }) - it('should include multiple custom schemas when provided', () => { + it('should include multiple custom schemas when provided', async () => { const customVideoSchema = z.object({ type: z.enum(['customVideo']), content: z.object({ @@ -452,7 +462,7 @@ describe('AIAgentBuilder', () => { }), }) - new AIAgentBuilder({ + await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -493,7 +503,7 @@ describe('AIAgentBuilder', () => { expect(outputType.safeParse(validImageMessage).success).toBe(true) }) - it('should reject invalid custom message when custom schemas are provided', () => { + it('should reject invalid custom message when custom schemas are provided', async () => { const customVideoSchema = z.object({ type: z.enum(['customVideo']), content: z.object({ @@ -501,7 +511,7 @@ describe('AIAgentBuilder', () => { }), }) - new AIAgentBuilder({ + await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -529,8 +539,8 @@ describe('AIAgentBuilder', () => { expect(outputType.safeParse(invalidMessage).success).toBe(false) }) - it('should produce same schema as OutputSchema when empty array is provided', () => { - new AIAgentBuilder({ + it('should produce same schema as OutputSchema when empty array is provided', async () => { + await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -572,9 +582,8 @@ describe('AIAgentBuilder', () => { }) describe('Provider logic (openai vs azure)', () => { - it('should configure modelSettings for azure provider with retrieveKnowledge tool', () => { - // Default OPENAI_PROVIDER is 'azure' from constants - const aiAgent = new AIAgentBuilder({ + it('should configure toolChoice for gpt-4 models with retrieveKnowledge tool', async () => { + await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -587,17 +596,55 @@ describe('AIAgentBuilder', () => { guardrailTrackingContext: mockGuardrailTrackingContext, }).build() - // When using azure provider with retrieveKnowledge, logModelSettings is called expect(mockLogger.logModelSettings).toHaveBeenCalledWith( expect.objectContaining({ provider: 'azure', + toolChoice: 'retrieve_knowledge', hasRetrieveKnowledge: true, }) ) + expect(capturedAgentConfig.modelSettings.toolChoice).toBe( + 'retrieve_knowledge' + ) }) - it('should NOT set toolChoice when sourceIds is empty (no retrieveKnowledge)', () => { - const aiAgent = new AIAgentBuilder({ + it('should set toolChoice for non gpt-4 models with retrieveKnowledge', async () => { + const nonGpt4LlmConfig = { + ...mockLlmConfig, + modelName: 'gpt-5-mini', + modelSettings: { + reasoning: { effort: 'none' as const }, + text: { verbosity: 'medium' as const }, + toolChoice: undefined as string | undefined, + }, + } as unknown as LLMConfig + + await new AIAgentBuilder({ + name: agentName, + instructions: agentInstructions, + llmConfig: nonGpt4LlmConfig, + tools: agentCustomTools, + contactInfo, + inputGuardrailRules: [], + sourceIds: ['source-1'], + campaignsContext: undefined, + logger: mockLogger, + guardrailTrackingContext: mockGuardrailTrackingContext, + }).build() + + expect(mockLogger.logModelSettings).toHaveBeenCalledWith( + expect.objectContaining({ + toolChoice: 'retrieve_knowledge', + hasRetrieveKnowledge: true, + }) + ) + expect(capturedAgentConfig.modelSettings.toolChoice).toBe( + 'retrieve_knowledge' + ) + }) + + it('should NOT set toolChoice when sourceIds is empty (no retrieveKnowledge)', async () => { + await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -617,9 +664,9 @@ describe('AIAgentBuilder', () => { ) }) - it('should set model (deployment name) for azure provider', () => { + it('should set resolved model for azure provider', async () => { // Default OPENAI_PROVIDER is 'azure' - const aiAgent = new AIAgentBuilder({ + await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -633,13 +680,12 @@ describe('AIAgentBuilder', () => { }).build() expect(capturedAgentConfig).toBeDefined() - // Azure uses deployment name as model - expect(capturedAgentConfig.model).toBe('gpt-4.1-mini') + expect(capturedAgentConfig.model).toBe(resolvedModel) }) - it('should set reasoning and text settings for azure provider (same as openai)', () => { + it('should set reasoning and text settings for azure provider (same as openai)', async () => { // Default OPENAI_PROVIDER is 'azure' - const aiAgent = new AIAgentBuilder({ + await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -660,6 +706,10 @@ describe('AIAgentBuilder', () => { text: { verbosity: 'medium' }, }) ) + expect(capturedAgentConfig.modelSettings).toMatchObject({ + reasoning: { effort: 'none' }, + text: { verbosity: 'medium' }, + }) }) }) }) @@ -701,8 +751,8 @@ describe('AIAgentBuilder - OpenAI Provider', () => { mockConstants.OPENAI_PROVIDER = 'azure' }) - it('should set reasoning setting with effort: none for openai provider', () => { - new AIAgentBuilder({ + it('should set reasoning setting with effort: none for openai provider', async () => { + await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -723,8 +773,8 @@ describe('AIAgentBuilder - OpenAI Provider', () => { ) }) - it('should set text setting with verbosity: medium for openai provider', () => { - new AIAgentBuilder({ + it('should set text setting with verbosity: medium for openai provider', async () => { + await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -744,8 +794,8 @@ describe('AIAgentBuilder - OpenAI Provider', () => { ) }) - it('should set model to OPENAI_MODEL for openai provider', () => { - new AIAgentBuilder({ + it('should set resolved model for openai provider', async () => { + await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -759,11 +809,11 @@ describe('AIAgentBuilder - OpenAI Provider', () => { }).build() expect(capturedAgentConfig).toBeDefined() - expect(capturedAgentConfig.model).toBe('gpt-4.1-mini') + expect(capturedAgentConfig.model).toBe(resolvedModel) }) - it('should NOT set toolChoice for openai provider even with retrieveKnowledge', () => { - new AIAgentBuilder({ + it('should set toolChoice for gpt-4 models even with openai provider', async () => { + await new AIAgentBuilder({ name: agentName, instructions: agentInstructions, llmConfig: mockLlmConfig, @@ -779,8 +829,12 @@ describe('AIAgentBuilder - OpenAI Provider', () => { expect(mockLogger.logModelSettings).toHaveBeenCalledWith( expect.objectContaining({ provider: 'openai', + toolChoice: 'retrieve_knowledge', hasRetrieveKnowledge: true, }) ) + expect(capturedAgentConfig.modelSettings.toolChoice).toBe( + 'retrieve_knowledge' + ) }) }) diff --git a/packages/botonic-plugin-ai-agents/tests/debug-logger.test.ts b/packages/botonic-plugin-ai-agents/tests/debug-logger.test.ts index ac36dd2336..fec3664ccc 100644 --- a/packages/botonic-plugin-ai-agents/tests/debug-logger.test.ts +++ b/packages/botonic-plugin-ai-agents/tests/debug-logger.test.ts @@ -1,4 +1,4 @@ -import { VerbosityLevel } from '@botonic/core' +import { type AiAgentArgs, AiAgentType, VerbosityLevel } from '@botonic/core' import { createDebugLogger, type DebugLogger } from '../src/debug-logger' describe('DebugLogger', () => { @@ -77,11 +77,13 @@ describe('DebugLogger', () => { }) it('should log agent debug info', () => { - const aiAgentArgs = { + const aiAgentArgs: AiAgentArgs = { + type: AiAgentType.Worker, name: 'TestAgent', instructions: 'Test instructions', model: 'gpt-4.1-mini', verbosity: VerbosityLevel.Medium, + activeTools: [], sourceIds: ['source1'], inputGuardrailRules: [], } @@ -217,10 +219,14 @@ describe('DebugLogger', () => { }) logger.logAgentDebugInfo( { + type: AiAgentType.Worker, name: 'Test', instructions: '', model: 'gpt-4.1-mini', verbosity: VerbosityLevel.Low, + activeTools: [], + sourceIds: [], + inputGuardrailRules: [], }, [], [] diff --git a/packages/botonic-plugin-ai-agents/tests/guardrails/input.test.ts b/packages/botonic-plugin-ai-agents/tests/guardrails/input.test.ts index 2e29e7d124..76a996068a 100644 --- a/packages/botonic-plugin-ai-agents/tests/guardrails/input.test.ts +++ b/packages/botonic-plugin-ai-agents/tests/guardrails/input.test.ts @@ -2,7 +2,7 @@ import { Agent, type RunContext, Runner, type Usage } from '@openai/agents' import { - createInputGuardrail, + createInputGuardrails, type GuardrailTrackingContext, } from '../../src/guardrails/input' import type { LLMConfig } from '../../src/llm-config' @@ -10,17 +10,27 @@ import type { GuardrailRule } from '../../src/types' const mockRunnerRun = jest.fn() const mockTrackLlmRuns = jest.fn().mockResolvedValue(undefined) +let capturedAgentConfig: any = null +let capturedRunnerConfig: any = null // Mock OpenAI Agent and Runner jest.mock('@openai/agents', () => ({ - Agent: jest.fn().mockImplementation(config => ({ - name: config.name, - instructions: config.instructions, - outputType: config.outputType, - })), - Runner: jest.fn().mockImplementation(() => ({ - run: mockRunnerRun, - })), + Agent: jest.fn().mockImplementation(config => { + capturedAgentConfig = config + return { + name: config.name, + instructions: config.instructions, + outputType: config.outputType, + model: config.model, + modelSettings: config.modelSettings, + } + }), + Runner: jest.fn().mockImplementation(config => { + capturedRunnerConfig = config + return { + run: mockRunnerRun, + } + }), })) jest.mock('../../src/services/hubtype-api-client', () => ({ @@ -35,7 +45,7 @@ jest.mock('../../src/constants', () => ({ AZURE_OPENAI_API_VERSION: '2025-01-01-preview', })) -describe('createInputGuardrail', () => { +describe('createInputGuardrails', () => { const mockRules: GuardrailRule[] = [ { name: 'is_offensive', @@ -76,8 +86,13 @@ describe('createInputGuardrail', () => { const mockLlmConfig = { modelName: 'gpt-4.1-mini', - modelSettings: { temperature: 0, text: { verbosity: 'medium' } }, + modelSettings: { + temperature: 0, + text: { verbosity: 'medium' }, + toolChoice: 'retrieve_knowledge', + }, modelProvider: {}, + getModel: jest.fn().mockResolvedValue({ id: 'guardrail-model' }), } as unknown as LLMConfig const mockTrackingContext: GuardrailTrackingContext = { @@ -89,11 +104,13 @@ describe('createInputGuardrail', () => { beforeEach(() => { jest.clearAllMocks() + capturedAgentConfig = null + capturedRunnerConfig = null jest.requireMock('../../src/constants').isProd = false }) - it('should create a guardrail with the correct configuration', () => { - const guardrail = createInputGuardrail( + it('should create a guardrail with the correct configuration', async () => { + const [guardrail] = await createInputGuardrails( mockRules, mockLlmConfig, mockTrackingContext @@ -102,11 +119,27 @@ describe('createInputGuardrail', () => { expect(guardrail.name).toBe('InputGuardrail') expect(Agent).toHaveBeenCalledWith({ name: 'InputGuardrail', - model: mockLlmConfig.modelName, + model: { id: 'guardrail-model' }, + modelSettings: { + temperature: 0, + text: { verbosity: 'medium' }, + }, instructions: 'Check if the user triggers some of the following guardrails.', outputType: expect.any(Object), }) + expect(capturedAgentConfig.modelSettings).not.toHaveProperty('toolChoice') + }) + + it('should return no guardrails when no rules are configured', async () => { + const guardrails = await createInputGuardrails( + [], + mockLlmConfig, + mockTrackingContext + ) + + expect(guardrails).toEqual([]) + expect(Agent).not.toHaveBeenCalled() }) it('should return triggered guardrails when rules are violated', async () => { @@ -118,7 +151,7 @@ describe('createInputGuardrail', () => { } mockRunnerRun.mockResolvedValue(mockAgentOutput) - const guardrail = createInputGuardrail( + const [guardrail] = await createInputGuardrails( mockRules, mockLlmConfig, mockTrackingContext @@ -148,6 +181,9 @@ describe('createInputGuardrail', () => { ], { context: mockRunContext } ) + expect(capturedRunnerConfig).toEqual({ tracingDisabled: true }) + expect(capturedRunnerConfig).not.toHaveProperty('modelSettings') + expect(capturedRunnerConfig).not.toHaveProperty('modelProvider') }) it('should return no triggered guardrails when no rules are violated', async () => { @@ -159,7 +195,7 @@ describe('createInputGuardrail', () => { } mockRunnerRun.mockResolvedValue(mockAgentOutput) - const guardrail = createInputGuardrail( + const [guardrail] = await createInputGuardrails( mockRules, mockLlmConfig, mockTrackingContext @@ -187,7 +223,7 @@ describe('createInputGuardrail', () => { } mockRunnerRun.mockResolvedValue(mockAgentOutput) - const guardrail = createInputGuardrail( + const [guardrail] = await createInputGuardrails( mockRules, mockLlmConfig, mockTrackingContext @@ -219,7 +255,7 @@ describe('createInputGuardrail', () => { } mockRunnerRun.mockResolvedValue(mockAgentOutput) - const guardrail = createInputGuardrail( + const [guardrail] = await createInputGuardrails( mockRules, mockLlmConfig, mockTrackingContext diff --git a/packages/botonic-plugin-ai-agents/tests/index.test.ts b/packages/botonic-plugin-ai-agents/tests/index.test.ts index 5f7b7e9997..4ca02b4ab5 100644 --- a/packages/botonic-plugin-ai-agents/tests/index.test.ts +++ b/packages/botonic-plugin-ai-agents/tests/index.test.ts @@ -1,5 +1,7 @@ import { + type AIAgentRouterArgs, type AiAgentArgs, + AiAgentType, type BotContext, INPUT, PROVIDER, @@ -16,17 +18,75 @@ import { } from '@jest/globals' import BotonicPluginAiAgents from '../src/index' +import { LLMConfig as MockedLLMConfig } from '../src/llm-config' // Store the captured AIAgentBuilder arguments // eslint-disable-next-line @typescript-eslint/no-explicit-any let capturedBuilderArgs: any = null +type MockLlmConfig = { + modelName: string + modelSettings: { temperature: number } + modelProvider: Record + getModel: () => Promise<{ id: string }> +} +type MockRouterBuilderArgs = { + name: string + instructions: string + llmConfig: MockLlmConfig + handoffs: unknown[] + inputGuardrailRules: unknown[] + outputMessagesSchemas: unknown[] + guardrailTrackingContext: unknown +} +let capturedRouterBuilderArgs: MockRouterBuilderArgs | null = null +type MockAgentConfig = { + name: string + instructions?: string + model?: unknown + modelSettings?: unknown + handoffs?: unknown + inputGuardrails?: { name: string }[] +} +type MockAgentInstance = MockAgentConfig + +jest.mock('@openai/agents', () => { + const create = jest.fn((config: MockAgentConfig): MockAgentInstance => { + return { + name: config.name, + instructions: config.instructions, + model: config.model, + modelSettings: config.modelSettings, + handoffs: config.handoffs, + inputGuardrails: config.inputGuardrails, + } + }) + const AgentMock = Object.assign( + jest.fn( + (config: MockAgentConfig): MockAgentInstance => ({ + name: config.name, + instructions: config.instructions, + model: config.model, + modelSettings: config.modelSettings, + }) + ), + { create } + ) + + return { + Agent: AgentMock, + handoff: jest.fn().mockImplementation(agent => ({ agent })), + setTracingDisabled: jest.fn(), + tool: jest.fn().mockImplementation(config => config), + } +}) // Mock LLMConfig to avoid actual OpenAI/Azure setup jest.mock('../src/llm-config', () => ({ - LLMConfig: jest.fn().mockImplementation(() => ({ - modelName: 'gpt-4.1-mini', - modelSettings: {}, + LLMConfig: jest.fn().mockImplementation((_maxRetries, _timeout, model) => ({ + modelName: model, + modelSettings: { temperature: 0 }, modelProvider: {}, + getModel: jest.fn(async () => ({ id: `resolved-${model}` })), })), })) @@ -36,11 +96,28 @@ jest.mock('../src/agent-builder', () => ({ AIAgentBuilder: jest.fn().mockImplementation((args: any) => { capturedBuilderArgs = args return { - build: jest.fn().mockReturnValue({ + build: jest.fn(async () => ({ name: args.name, instructions: args.instructions, + model: { id: `resolved-${args.llmConfig.modelName}` }, + modelSettings: args.llmConfig.modelSettings, tools: args.tools || [], - }), + })), + } + }), +})) + +jest.mock('../src/agent-router-builder', () => ({ + AIAgentRouterBuilder: jest.fn().mockImplementation((args: unknown) => { + const routerBuilderArgs = args as MockRouterBuilderArgs + capturedRouterBuilderArgs = routerBuilderArgs + return { + build: jest.fn(async () => ({ + name: routerBuilderArgs.name, + instructions: routerBuilderArgs.instructions, + modelSettings: routerBuilderArgs.llmConfig.modelSettings, + handoffs: routerBuilderArgs.handoffs, + })), } }), })) @@ -60,6 +137,20 @@ jest.mock('../src/runner', () => ({ })), })) +jest.mock('../src/runner-router', () => ({ + AIAgentRouterRunner: jest.fn().mockImplementation(() => ({ + run: jest.fn().mockResolvedValue({ + messages: [], + toolsExecuted: [], + memoryLength: 0, + exit: false, + error: false, + inputGuardrailsTriggered: [], + outputGuardrailsTriggered: [], + } as never), + })), +})) + // Mock HubtypeApiClient to avoid actual API calls jest.mock('../src/services/hubtype-api-client', () => ({ HubtypeApiClient: jest.fn().mockImplementation(() => ({ @@ -108,6 +199,7 @@ describe('BotonicPluginAiAgents - Campaign Context Integration', () => { }) const mockAiAgentArgs: AiAgentArgs = { + type: AiAgentType.Worker, name: 'Test Agent', instructions: 'Test instructions', model: 'gpt-4.1-mini', @@ -120,6 +212,7 @@ describe('BotonicPluginAiAgents - Campaign Context Integration', () => { beforeEach(() => { jest.clearAllMocks() capturedBuilderArgs = null + capturedRouterBuilderArgs = null // Set NODE_ENV to non-production to use authToken from options process.env.NODE_ENV = 'test' }) @@ -230,6 +323,7 @@ describe('BotonicPluginAiAgents - Campaign Context Integration', () => { const request = createMockRequest() const customAiAgentArgs: AiAgentArgs = { + type: AiAgentType.Worker, name: 'Custom Agent', instructions: 'Custom instructions for the agent', model: 'gpt-4.1-mini', @@ -254,6 +348,122 @@ describe('BotonicPluginAiAgents - Campaign Context Integration', () => { ]) }) + it('should pass router configuration to AIAgentRouterBuilder', async () => { + const plugin = new BotonicPluginAiAgents({ + authToken: 'test-auth-token', + }) + + const request = createMockRequest() + const routerArgs: AIAgentRouterArgs = { + type: AiAgentType.Router, + name: 'Router Agent', + instructions: 'Route the conversation to the right worker', + model: 'gpt-4.1-mini', + verbosity: VerbosityLevel.High, + inputGuardrailRules: [ + { + name: 'is_offensive', + description: 'Check for offensive content', + }, + ], + agents: [ + { + type: AiAgentType.Worker, + name: 'Support Worker', + description: 'Handles support questions', + instructions: 'Answer support questions', + model: 'gpt-4.1-mini', + verbosity: VerbosityLevel.Medium, + activeTools: [], + sourceIds: [], + inputGuardrailRules: [], + }, + ], + } + + await plugin.getInference(request, routerArgs) + + const routerBuilderArgs = capturedRouterBuilderArgs + if (!routerBuilderArgs) { + throw new Error('Router builder was not created') + } + expect(routerBuilderArgs.name).toBe('Router Agent') + expect(routerBuilderArgs.instructions).toBe( + 'Route the conversation to the right worker' + ) + expect(routerBuilderArgs.llmConfig).toMatchObject({ + modelName: 'gpt-4.1-mini', + modelSettings: { temperature: 0 }, + }) + expect(MockedLLMConfig).toHaveBeenCalledWith( + 2, + 16000, + 'gpt-4.1-mini', + VerbosityLevel.High + ) + expect(routerBuilderArgs.inputGuardrailRules).toEqual([ + { + name: 'is_offensive', + description: 'Check for offensive content', + }, + ]) + expect(routerBuilderArgs.outputMessagesSchemas).toEqual([]) + expect(routerBuilderArgs.guardrailTrackingContext).toEqual({ + botId: 'bot-123', + isTest: false, + authToken: 'test-auth-token', + inferenceId: expect.any(String), + }) + expect(routerBuilderArgs.handoffs).toEqual([ + expect.objectContaining({ + agent: expect.objectContaining({ + name: 'Support Worker', + }), + }), + ]) + }) + + it('should pass router worker sourceIds to the handoff agent builder', async () => { + const plugin = new BotonicPluginAiAgents({ + authToken: 'test-auth-token', + }) + + const request = createMockRequest() + const routerArgs: AIAgentRouterArgs = { + type: AiAgentType.Router, + name: 'Router Agent', + instructions: 'Route the conversation to the right worker', + model: 'gpt-4.1-mini', + verbosity: VerbosityLevel.Medium, + agents: [ + { + type: AiAgentType.Worker, + name: 'Knowledge Worker', + description: 'Handles knowledge questions', + instructions: 'Answer with knowledge sources', + model: 'gpt-4.1-mini', + verbosity: VerbosityLevel.Medium, + activeTools: [], + sourceIds: ['source-1', 'source-2'], + inputGuardrailRules: [], + }, + ], + } + + await plugin.getInference(request, routerArgs) + + expect(capturedBuilderArgs).toBeDefined() + expect(capturedBuilderArgs.name).toBe('Knowledge Worker') + expect(capturedBuilderArgs.sourceIds).toEqual(['source-1', 'source-2']) + expect(capturedRouterBuilderArgs?.handoffs).toEqual([ + expect.objectContaining({ + agent: expect.objectContaining({ + name: 'Knowledge Worker', + }), + }), + ]) + }) + it('should pass contact_info from session.user', async () => { const plugin = new BotonicPluginAiAgents({ authToken: 'test-auth-token', diff --git a/packages/botonic-plugin-ai-agents/tests/llm-config.test.ts b/packages/botonic-plugin-ai-agents/tests/llm-config.test.ts index 01803aefac..d1a3fd076c 100644 --- a/packages/botonic-plugin-ai-agents/tests/llm-config.test.ts +++ b/packages/botonic-plugin-ai-agents/tests/llm-config.test.ts @@ -8,6 +8,7 @@ const DEFAULT_TIMEOUT = 16000 let capturedOpenAIConfig: Record | null = null let capturedAzureConfig: Record | null = null +const mockResolvedModel = { id: 'resolved-model' } jest.mock('openai', () => ({ __esModule: true, @@ -24,7 +25,10 @@ jest.mock('openai', () => ({ })) jest.mock('@openai/agents', () => ({ - OpenAIProvider: jest.fn().mockImplementation(() => ({ type: 'provider' })), + OpenAIProvider: jest.fn().mockImplementation(() => ({ + type: 'provider', + getModel: jest.fn().mockResolvedValue(mockResolvedModel), + })), })) // var so the variable is hoisted and assignable when the mock factory runs (Jest hoists mocks) @@ -180,4 +184,20 @@ describe('LLMConfig', () => { expect(capturedAzureConfig?.apiVersion).toBe('2025-01-01-preview') }) }) + + describe('getModel', () => { + it('should resolve model from the configured provider and model name', async () => { + mockConstants.OPENAI_PROVIDER = 'azure' + + const config = new LLMConfig( + DEFAULT_MAX_RETRIES, + DEFAULT_TIMEOUT, + 'gpt-4.1-mini', + VerbosityLevel.Medium + ) + + await expect(config.getModel()).resolves.toBe(mockResolvedModel) + expect(config.modelProvider.getModel).toHaveBeenCalledWith('gpt-4.1-mini') + }) + }) }) diff --git a/packages/botonic-plugin-ai-agents/tests/retrieve-knowledge.test.ts b/packages/botonic-plugin-ai-agents/tests/retrieve-knowledge.test.ts new file mode 100644 index 0000000000..cd46dddb52 --- /dev/null +++ b/packages/botonic-plugin-ai-agents/tests/retrieve-knowledge.test.ts @@ -0,0 +1,65 @@ +import { createRetrieveKnowledge } from '../src/tools/retrieve-knowledge' +import type { Context } from '../src/types' + +const mockRetrieveSimilarChunks = jest.fn() + +type RetrieveKnowledgeTool = { + execute: ( + input: { query: string }, + runContext: { context: Context } + ) => Promise +} + +jest.mock('@openai/agents', () => ({ + tool: jest.fn(config => config), +})) + +jest.mock('../src/services/hubtype-api-client', () => ({ + HubtypeApiClient: jest.fn().mockImplementation(() => ({ + retrieveSimilarChunks: mockRetrieveSimilarChunks, + })), +})) + +const buildContext = (): Context => + ({ + authToken: 'test-token', + knowledgeUsed: { + query: '', + sourceIds: [], + chunksIds: [], + chunkTexts: [], + }, + request: {}, + }) as unknown as Context + +describe('createRetrieveKnowledge', () => { + beforeEach(() => { + jest.clearAllMocks() + mockRetrieveSimilarChunks.mockResolvedValue([ + { id: 'chunk-1', text: 'Knowledge chunk' }, + ] as never) + }) + + it('uses configured sourceIds when runtime context has none', async () => { + const retrieveKnowledge = createRetrieveKnowledge([ + 'source-1', + ]) as unknown as RetrieveKnowledgeTool + const context = buildContext() + + const result = await retrieveKnowledge.execute( + { query: 'shipping policy' }, + { context } + ) + + expect(mockRetrieveSimilarChunks).toHaveBeenCalledWith('shipping policy', [ + 'source-1', + ]) + expect(context.knowledgeUsed).toEqual({ + query: 'shipping policy', + sourceIds: ['source-1'], + chunksIds: ['chunk-1'], + chunkTexts: ['Knowledge chunk'], + }) + expect(result).toEqual(['Knowledge chunk']) + }) +}) diff --git a/packages/botonic-plugin-ai-agents/tests/router-agent-builder.test.ts b/packages/botonic-plugin-ai-agents/tests/router-agent-builder.test.ts new file mode 100644 index 0000000000..ac82c3364f --- /dev/null +++ b/packages/botonic-plugin-ai-agents/tests/router-agent-builder.test.ts @@ -0,0 +1,168 @@ +import type { AgentOutputType, Handoff, ModelSettings } from '@openai/agents' +import { z } from 'zod' + +import type { GuardrailTrackingContext } from '../src/guardrails/input' +import type { LLMConfig } from '../src/llm-config' +import type { OutputSchema } from '../src/structured-output' +import type { Context, GuardrailRule } from '../src/types' + +type MockOutputType = { + safeParse: (value: unknown) => { success: boolean } +} +type MockAgentConfig = { + name: string + instructions?: string + model?: unknown + modelSettings?: ModelSettings + outputType?: MockOutputType + handoffs?: unknown + inputGuardrails?: { name: string }[] +} + +let capturedAgentConfig: MockAgentConfig | null = null + +jest.mock('@openai/agents', () => ({ + Agent: { + create: jest.fn((config: MockAgentConfig) => { + capturedAgentConfig = config + return config + }), + }, +})) + +const mockResolvedModel = { id: 'resolved-gpt-4.1-mini' } +const mockModelSettings: ModelSettings = { + temperature: 1, + reasoning: { effort: 'none' }, + text: { verbosity: 'medium' }, +} +const mockLlmConfig = { + modelName: 'gpt-4.1-mini', + modelSettings: mockModelSettings, + modelProvider: {}, + getModel: jest.fn().mockResolvedValue(mockResolvedModel), +} as unknown as LLMConfig + +const mockInputGuardrails = [{ name: 'InputGuardrail' }] +const mockCreateInputGuardrails = jest + .fn() + .mockResolvedValue(mockInputGuardrails as never) + +jest.mock('../src/guardrails', () => ({ + createInputGuardrails: mockCreateInputGuardrails, +})) + +import { AIAgentRouterBuilder } from '../src/agent-router-builder' + +describe('AIAgentRouterBuilder', () => { + const handoffs = [{ agentName: 'Support Worker' }] as unknown as Handoff< + Context, + AgentOutputType + >[] + const inputGuardrailRules: GuardrailRule[] = [ + { name: 'is_offensive', description: 'Check for offensive content' }, + ] + const guardrailTrackingContext: GuardrailTrackingContext = { + botId: 'test-bot-id', + isTest: false, + authToken: 'test-auth-token', + inferenceId: 'test-inference-id', + } + + beforeEach(() => { + jest.clearAllMocks() + capturedAgentConfig = null + }) + + it('should build a router agent with handoffs, guardrails and structured output', async () => { + const builder = new AIAgentRouterBuilder({ + name: 'Router Agent', + instructions: 'Route the conversation to the right worker', + llmConfig: mockLlmConfig, + handoffs, + inputGuardrailRules, + outputMessagesSchemas: [], + guardrailTrackingContext, + }) + + const agent = await builder.build() + + expect(mockCreateInputGuardrails).toHaveBeenCalledWith( + inputGuardrailRules, + mockLlmConfig, + guardrailTrackingContext + ) + expect(agent).toBe(capturedAgentConfig) + + const agentConfig = capturedAgentConfig + if (!agentConfig?.outputType) { + throw new Error('Router agent was not created with outputType') + } + + expect(agentConfig.name).toBe('Router Agent') + expect(agentConfig.model).toBe(mockResolvedModel) + expect(agentConfig.handoffs).toBe(handoffs) + expect(agentConfig.inputGuardrails).toBe(mockInputGuardrails) + expect(agentConfig.instructions).toContain( + 'Route the conversation to the right worker' + ) + expect(agentConfig.instructions).toContain('') + expect( + agentConfig.outputType.safeParse({ + messages: [{ type: 'text', content: { text: 'Hi' } }], + }).success + ).toBe(true) + }) + + it('should include external output message schemas in the router output type', async () => { + const customMessageSchema = z.object({ + type: z.literal('custom'), + content: z.object({ value: z.string() }), + }) + const builder = new AIAgentRouterBuilder({ + name: 'Router Agent', + instructions: 'Route the conversation to the right worker', + llmConfig: mockLlmConfig, + handoffs, + inputGuardrailRules: [], + outputMessagesSchemas: [customMessageSchema], + guardrailTrackingContext, + }) + + await builder.build() + + const outputType = capturedAgentConfig?.outputType + if (!outputType) { + throw new Error('Router agent was not created with outputType') + } + + expect( + outputType.safeParse({ + messages: [{ type: 'custom', content: { value: 'extra' } }], + }).success + ).toBe(true) + }) + + it('should copy nested router model settings before passing them to the agent', async () => { + const builder = new AIAgentRouterBuilder({ + name: 'Router Agent', + instructions: 'Route the conversation to the right worker', + llmConfig: mockLlmConfig, + handoffs, + inputGuardrailRules: [], + outputMessagesSchemas: [], + guardrailTrackingContext, + }) + + await builder.build() + + expect(capturedAgentConfig?.modelSettings).toEqual(mockModelSettings) + expect(capturedAgentConfig?.modelSettings).not.toBe(mockModelSettings) + expect(capturedAgentConfig?.modelSettings?.reasoning).not.toBe( + mockModelSettings.reasoning + ) + expect(capturedAgentConfig?.modelSettings?.text).not.toBe( + mockModelSettings.text + ) + }) +}) diff --git a/packages/botonic-plugin-ai-agents/tests/runner-router.test.ts b/packages/botonic-plugin-ai-agents/tests/runner-router.test.ts new file mode 100644 index 0000000000..0d79a1ad6d --- /dev/null +++ b/packages/botonic-plugin-ai-agents/tests/runner-router.test.ts @@ -0,0 +1,199 @@ +import type { DebugLogger } from '../src/debug-logger' +import type { LLMConfig } from '../src/llm-config' +import { AIAgentRouterRunner } from '../src/runner-router' +import type { AgenticInputMessage, AIAgent, Context } from '../src/types' + +const mockRunnerRunImpl = jest.fn() +let capturedRunnerConfig: any = null + +jest.mock('@openai/agents', () => { + class MockInputGuardrailTripwireTriggered extends Error { + result: any + constructor(result: any) { + super('InputGuardrailTripwireTriggered') + this.result = result + } + } + + const MockRunner = jest.fn().mockImplementation((config: any) => { + capturedRunnerConfig = config + return { + run: mockRunnerRunImpl, + } + }) + + return { + Runner: MockRunner, + InputGuardrailTripwireTriggered: MockInputGuardrailTripwireTriggered, + } +}) + +const mockLogger: DebugLogger = { + logInitialConfig: jest.fn(), + logAgentDebugInfo: jest.fn(), + logModelSettings: jest.fn(), + logRunnerStart: jest.fn(), + logRunResult: jest.fn(), + logGuardrailTriggered: jest.fn(), + logRunnerError: jest.fn(), + logToolExecution: jest.fn(), +} + +const mockLlmConfig = { + modelName: 'gpt-4.1-mini', + modelSettings: { temperature: 0 }, + modelProvider: {}, +} as unknown as LLMConfig + +const mockAgent = { + name: 'RouterAgent', + tools: [], + modelSettings: { temperature: 0 }, +} as unknown as AIAgent + +const mockContext = { + authToken: 'test-token', + knowledgeUsed: { + query: '', + sourceIds: [], + chunksIds: [], + chunkTexts: [], + }, + request: { + session: { + bot: { id: 'test-bot-id' }, + is_test_integration: false, + }, + }, +} as unknown as Context + +const sampleMessages: AgenticInputMessage[] = [ + { role: 'user', content: 'Hello' } as any, +] + +describe('AIAgentRouterRunner', () => { + beforeEach(() => { + jest.clearAllMocks() + capturedRunnerConfig = null + }) + + it('should create Runner with execution settings only', async () => { + mockRunnerRunImpl.mockResolvedValueOnce({ + finalOutput: { + messages: [{ type: 'text', content: { text: 'Hi' } }], + }, + state: { _currentAgent: { name: 'RouterAgent' } }, + }) + + const runner = new AIAgentRouterRunner( + mockAgent, + mockLlmConfig, + 'test-inference-id', + mockLogger + ) + + const result = await runner.run(sampleMessages, mockContext) + + expect(capturedRunnerConfig).toEqual({ tracingDisabled: true }) + expect(capturedRunnerConfig).not.toHaveProperty('modelSettings') + expect(capturedRunnerConfig).not.toHaveProperty('modelProvider') + expect(result.messages).toEqual([{ type: 'text', content: { text: 'Hi' } }]) + expect(result.exit).toBe(false) + }) + + it('should return all direct router messages when there is no exit message', async () => { + const messages = [ + { type: 'text', content: { text: 'Hi' } }, + { type: 'text', content: { text: 'How can I help?' } }, + ] + mockRunnerRunImpl.mockResolvedValueOnce({ + finalOutput: { messages }, + state: { _currentAgent: { name: 'RouterAgent' } }, + }) + + const runner = new AIAgentRouterRunner( + mockAgent, + mockLlmConfig, + 'test-inference-id', + mockLogger + ) + + const result = await runner.run(sampleMessages, mockContext) + + expect(result.messages).toEqual(messages) + expect(result.exit).toBe(false) + expect(result.memoryLength).toBe(sampleMessages.length) + expect(result.toolsExecuted).toEqual([]) + expect(result.error).toBe(false) + }) + + it('should exit when finalOutput is missing', async () => { + mockRunnerRunImpl.mockResolvedValueOnce({ + state: { _currentAgent: { name: 'RouterAgent' } }, + }) + + const runner = new AIAgentRouterRunner( + mockAgent, + mockLlmConfig, + 'test-inference-id', + mockLogger + ) + + const result = await runner.run(sampleMessages, mockContext) + + expect(result.messages).toEqual([]) + expect(result.exit).toBe(true) + expect(result.memoryLength).toBe(sampleMessages.length) + expect(result.toolsExecuted).toEqual([]) + expect(result.error).toBe(false) + }) + + it('should exit when finalOutput has no messages', async () => { + mockRunnerRunImpl.mockResolvedValueOnce({ + finalOutput: { messages: [] }, + state: { _currentAgent: { name: 'RouterAgent' } }, + }) + + const runner = new AIAgentRouterRunner( + mockAgent, + mockLlmConfig, + 'test-inference-id', + mockLogger + ) + + const result = await runner.run(sampleMessages, mockContext) + + expect(result.messages).toEqual([]) + expect(result.exit).toBe(true) + expect(result.memoryLength).toBe(sampleMessages.length) + expect(result.toolsExecuted).toEqual([]) + expect(result.error).toBe(false) + }) + + it('should exit and drop messages when an exit message is present', async () => { + mockRunnerRunImpl.mockResolvedValueOnce({ + finalOutput: { + messages: [ + { type: 'text', content: { text: 'Goodbye' } }, + { type: 'exit' }, + ], + }, + state: { _currentAgent: { name: 'RouterAgent' } }, + }) + + const runner = new AIAgentRouterRunner( + mockAgent, + mockLlmConfig, + 'test-inference-id', + mockLogger + ) + + const result = await runner.run(sampleMessages, mockContext) + + expect(result.messages).toEqual([]) + expect(result.exit).toBe(true) + expect(result.memoryLength).toBe(sampleMessages.length) + expect(result.toolsExecuted).toEqual([]) + expect(result.error).toBe(false) + }) +}) diff --git a/packages/botonic-plugin-ai-agents/tests/runner.test.ts b/packages/botonic-plugin-ai-agents/tests/runner.test.ts index 138f85cb08..3adceee01b 100644 --- a/packages/botonic-plugin-ai-agents/tests/runner.test.ts +++ b/packages/botonic-plugin-ai-agents/tests/runner.test.ts @@ -71,7 +71,7 @@ const mockRetrieveKnowledge = { jest.mock('../src/tools', () => ({ mandatoryTools: [], - retrieveKnowledge: mockRetrieveKnowledge, + RETRIEVE_KNOWLEDGE_TOOL_NAME: 'retrieve_knowledge', })) const mockConstants = { @@ -111,17 +111,20 @@ function buildMockLlmConfig(provider: 'openai' | 'azure' = 'azure'): LLMConfig { } as unknown as LLMConfig } -function buildMockAgent(includeRetrieveKnowledge = false): AIAgent { +function buildMockAgent( + includeRetrieveKnowledge = false, + modelSettings: Record = {} +): AIAgent { return { name: 'TestAgent', tools: includeRetrieveKnowledge ? [mockRetrieveKnowledge] : [], + modelSettings, } as unknown as AIAgent } function buildMockContext(): Context { return { authToken: 'test-token', - sourceIds: [], knowledgeUsed: { query: '', sourceIds: ['src-1'], @@ -377,17 +380,19 @@ describe('AIAgentRunner', () => { // ── run() – provider logic ─────────────────────────────────────────────── describe('run() – provider logic', () => { - it('should set toolChoice to retrieve_knowledge for azure provider when agent has that tool', async () => { + it('should not mutate llmConfig toolChoice for azure provider when agent has retrieve_knowledge', async () => { mockConstants.OPENAI_PROVIDER = 'azure' mockRunnerRunImpl.mockResolvedValueOnce(makeTextRunnerResult()) const llmConfig = buildMockLlmConfig('azure') - await createRunner(buildMockAgent(true), llmConfig).run( + const agent = buildMockAgent(true, { toolChoice: 'retrieve_knowledge' }) + await createRunner(agent, llmConfig).run( sampleMessages, buildMockContext() ) - expect(llmConfig.modelSettings.toolChoice).toBe('retrieve_knowledge') + expect(agent.modelSettings.toolChoice).toBe('retrieve_knowledge') + expect(llmConfig.modelSettings.toolChoice).toBeUndefined() }) it('should NOT set toolChoice for openai provider even with retrieve_knowledge tool', async () => { @@ -416,7 +421,7 @@ describe('AIAgentRunner', () => { expect(llmConfig.modelSettings.toolChoice).toBeUndefined() }) - it('should pass modelProvider and modelSettings to Runner', async () => { + it('should create Runner with execution settings only', async () => { mockRunnerRunImpl.mockResolvedValueOnce(makeTextRunnerResult()) const llmConfig = buildMockLlmConfig() @@ -425,11 +430,11 @@ describe('AIAgentRunner', () => { buildMockContext() ) - expect(capturedRunnerConfig).toMatchObject({ - modelSettings: llmConfig.modelSettings, - modelProvider: llmConfig.modelProvider, + expect(capturedRunnerConfig).toEqual({ tracingDisabled: true, }) + expect(capturedRunnerConfig).not.toHaveProperty('modelSettings') + expect(capturedRunnerConfig).not.toHaveProperty('modelProvider') }) }) diff --git a/packages/botonic-plugin-flow-builder/src/action/ai-agent-from-user-input.ts b/packages/botonic-plugin-flow-builder/src/action/ai-agent-from-user-input.ts index 21685c5310..b9d3417a22 100644 --- a/packages/botonic-plugin-flow-builder/src/action/ai-agent-from-user-input.ts +++ b/packages/botonic-plugin-flow-builder/src/action/ai-agent-from-user-input.ts @@ -17,18 +17,33 @@ export async function getContentsByAiAgentFromUserInput({ await flowBuilderPlugin.getContentsByNode(startNodeAiAgentFlow) const splitContents = splitAiAgentContents(contents) + if (!splitContents) { return [] } - const { aiAgentContent, contentsBeforeAiAgent } = splitContents - const aiAgentResponse = await aiAgentContent.resolveAIAgentResponse( - request, - contentsBeforeAiAgent - ) + if ('aiAgentRouterContent' in splitContents) { + const { aiAgentRouterContent, contentsBeforeAiAgentRouter } = splitContents + const aiAgentResponse = await aiAgentRouterContent.resolveAIAgentResponse( + request, + contentsBeforeAiAgentRouter + ) - if (!aiAgentResponse || aiAgentResponse.exit) { - return [] + if (!aiAgentResponse || aiAgentResponse.exit) { + return [] + } + } + + if ('aiAgentContent' in splitContents) { + const { aiAgentContent, contentsBeforeAiAgent } = splitContents + const aiAgentResponse = await aiAgentContent.resolveAIAgentResponse( + request, + contentsBeforeAiAgent + ) + + if (!aiAgentResponse || aiAgentResponse.exit) { + return [] + } } return contents diff --git a/packages/botonic-plugin-flow-builder/src/action/index.tsx b/packages/botonic-plugin-flow-builder/src/action/index.tsx index 988b0495bb..2b0161ba10 100644 --- a/packages/botonic-plugin-flow-builder/src/action/index.tsx +++ b/packages/botonic-plugin-flow-builder/src/action/index.tsx @@ -7,7 +7,12 @@ import { } from '@botonic/react' import React from 'react' -import { FlowAiAgent, type FlowContent } from '../content-fields' +import { + FlowAiAgent, + FlowAiAgentManager, + FlowAiAgentRouter, + type FlowContent, +} from '../content-fields' import { splitAiAgentContents } from '../utils/ai-agent' import { getFlowBuilderActionContext } from './context' import { getContentsByFirstInteraction } from './first-interaction' @@ -55,21 +60,59 @@ export class FlowBuilderAction extends React.Component { contents: FlowContent[] ) { for (const content of contents) { - if (content instanceof FlowAiAgent) { - const splitContents = splitAiAgentContents(contents) - if (!splitContents) { - continue - } - const { contentsBeforeAiAgent } = splitContents - await content.processContent(botContext, contentsBeforeAiAgent) - } else { - await content.processContent(botContext) + if (FlowBuilderAction.isAiAgentContent(content)) { + await FlowBuilderAction.processAiAgentContent(botContext, contents) + continue } + + await content.processContent(botContext) } return contents } + private static isAiAgentContent(content: FlowContent): boolean { + return ( + content instanceof FlowAiAgent || + content instanceof FlowAiAgentRouter || + content instanceof FlowAiAgentManager + ) + } + + // TODO: Refactor this to be more generic and reusable + private static async processAiAgentContent( + botContext: BotContext, + contents: FlowContent[] + ) { + const splitContents = splitAiAgentContents(contents) + if (!splitContents) { + return + } + + if ('aiAgentRouterContent' in splitContents) { + const { aiAgentRouterContent, contentsBeforeAiAgentRouter } = + splitContents + await aiAgentRouterContent.processContent( + botContext, + contentsBeforeAiAgentRouter + ) + } + + if ('aiAgentManagerContent' in splitContents) { + const { aiAgentManagerContent, contentsBeforeAiAgentManager } = + splitContents + await aiAgentManagerContent.processContent( + botContext, + contentsBeforeAiAgentManager + ) + } + + if ('aiAgentContent' in splitContents) { + const { aiAgentContent, contentsBeforeAiAgent } = splitContents + await aiAgentContent.processContent(botContext, contentsBeforeAiAgent) + } + } + protected getWebchatSettingsParams(botContext: BotContext): { shouldSendWebchatSettings: boolean webchatSettingsParams?: WebchatSettingsProps diff --git a/packages/botonic-plugin-flow-builder/src/content-fields/flow-ai-agent-manager.tsx b/packages/botonic-plugin-flow-builder/src/content-fields/flow-ai-agent-manager.tsx new file mode 100644 index 0000000000..ba9a339e13 --- /dev/null +++ b/packages/botonic-plugin-flow-builder/src/content-fields/flow-ai-agent-manager.tsx @@ -0,0 +1,242 @@ +import { + type AgenticOutputMessage, + AiAgentType, + type BotContext, + type GuardrailRule, + type HubtypeAssistantMessage, + type InferenceResponse, + VerbosityLevel, +} from '@botonic/core' +import type { FlowBuilderApi } from '../api' +import { + type FlowBuilderContentMessage, + FlowBuilderContentSchema, +} from '../structured-output/flow-builder-content' +import { getCommonFlowContentEventArgsForContentId } from '../tracking' +import { HubtypeAssistantContent } from '../utils/ai-agent' +import { getFlowBuilderPlugin } from '../utils/get-flow-builder-plugin' +import { ContentFieldsBase } from './content-fields-base' +import { FlowAiAgent } from './flow-ai-agent' +import type { AiAgentWithNameAndDescription } from './flow-ai-agent-router' +import type { + HtAiAgentManagerNode, + HtAiAgentNode, + HtInputGuardrailRule, + HtNodeWithContent, +} from './hubtype-fields' +import { FlowCarousel, type FlowContent, FlowText } from './index' + +export class FlowAiAgentManager extends ContentFieldsBase { + public name: string = '' + public instructions: string = '' + public model: string = '' + public verbosity: VerbosityLevel = VerbosityLevel.Medium + public activeTools?: { name: string }[] + public agents: AiAgentWithNameAndDescription[] = [] + public inputGuardrailRules: HtInputGuardrailRule[] = [] + + public aiAgentResponse?: InferenceResponse + public messages: AgenticOutputMessage[] = [] + public jsxElements: JSX.Element[] = [] + + static fromHubtypeCMS( + component: HtAiAgentManagerNode, + cmsApi: FlowBuilderApi + ): FlowAiAgentManager { + const newAiAgentManager = new FlowAiAgentManager(component.id) + newAiAgentManager.name = component.code + newAiAgentManager.instructions = component.content.instructions + newAiAgentManager.model = component.content.model + newAiAgentManager.verbosity = component.content.verbosity + newAiAgentManager.activeTools = component.content.active_tools + newAiAgentManager.agents = component.content.agent_slots.map(agentSlot => { + const agentNode = cmsApi.getNodeById(agentSlot.target.id) + const aiAgent = FlowAiAgent.fromHubtypeCMS(agentNode) + return { + agent: aiAgent, + description: agentSlot.description || '', + name: agentSlot.name || '', + } + }) + newAiAgentManager.inputGuardrailRules = + component.content.input_guardrail_rules || [] + return newAiAgentManager + } + + async resolveAIAgentResponse( + botContext: BotContext, + previousContents?: FlowContent[] + ): Promise | undefined> { + const aiAgentResponse = await this.getAIAgentResponse( + botContext, + previousContents + ) + + if (aiAgentResponse) { + this.aiAgentResponse = aiAgentResponse + await this.trackAiAgentResponse(botContext) + this.messages = aiAgentResponse.messages + } + + return aiAgentResponse + } + + async getAIAgentResponse( + botContext: BotContext, + previousContents?: FlowContent[] + ): Promise | undefined> { + const previousHubtypeContents: HubtypeAssistantMessage[] = + previousContents?.map(content => { + return { + role: 'assistant', + content: HubtypeAssistantContent.adapt(content), + } + }) || [] + + const flowBuilderPlugin = getFlowBuilderPlugin(botContext.plugins) + const aiAgentResponse = await flowBuilderPlugin.getAiAgentResponse?.( + botContext, + { + type: AiAgentType.Manager, + name: this.name, + instructions: this.instructions, + model: this.model, + verbosity: VerbosityLevel.Medium, + activeTools: this.activeTools ?? [], + agents: this.agents.map(({ agent, description, name }) => ({ + type: AiAgentType.Worker, + name, + description, + instructions: agent.instructions, + model: agent.model, + verbosity: agent.verbosity, + activeTools: agent.activeTools ?? [], + inputGuardrailRules: this.getActiveInputGuardrailRules( + agent.inputGuardrailRules + ), + sourceIds: agent.sources?.map(s => s.id) ?? [], + })), + inputGuardrailRules: this.getActiveInputGuardrailRules( + this.inputGuardrailRules + ), + outputMessagesSchemas: [FlowBuilderContentSchema], + previousHubtypeMessages: previousHubtypeContents, + } + ) + + console.log('FlowAiAgentManager aiAgentResponse', { + aiAgentResponse, + }) + + return aiAgentResponse + } + + async trackFlow(_botContext: BotContext): Promise { + return + } + + async trackAiAgentResponse(botContext: BotContext) { + const { flowThreadId, flowId, flowName, flowNodeId } = + getCommonFlowContentEventArgsForContentId(botContext, this.id) + + const event = { + action: 'AiAgentManager', + flowThreadId: flowThreadId, + flowId: flowId, + flowName: flowName, + flowNodeId: flowNodeId, + flowNodeContentId: this.name, + flowNodeIsMeaningful: true, + toolsExecuted: this.aiAgentResponse?.toolsExecuted ?? [], + memoryLength: this.aiAgentResponse?.memoryLength ?? 0, + inputMessageId: botContext.input.message_id!, + exit: this.aiAgentResponse?.exit ?? true, + inputGuardrailsTriggered: + this.aiAgentResponse?.inputGuardrailsTriggered ?? [], + outputGuardrailsTriggered: [], //aiAgentResponse.outputGuardrailsTriggered, + error: this.aiAgentResponse?.error ?? false, + } + const { action, ...eventArgs } = event + + // await trackEvent(botContext, action, eventArgs) + console.log('trackAiAgentResponse', { + action, + eventArgs, + }) + } + + async getFlowContentsByContentId( + botContext: BotContext, + contentId: string + ): Promise { + const flowBuilderPlugin = getFlowBuilderPlugin(botContext.plugins) + const cmsApi = flowBuilderPlugin.cmsApi + const node = cmsApi.getNodeByContentID(contentId) + const flowContents = await flowBuilderPlugin.getContentsByNode( + node as HtNodeWithContent + ) + + return flowContents + } + + async messagesToBotonicJSXElements(botContext: BotContext): Promise { + for (const message of this.messages) { + if ( + message.type === 'text' || + message.type === 'textWithButtons' || + message.type === 'botExecutor' + ) { + this.jsxElements.push(FlowText.fromAIAgent(this.id, message)) + } + + if (message.type === 'carousel') { + this.jsxElements.push( + FlowCarousel.fromAIAgent(this.id, message, botContext) + ) + } + + if (message.type === 'flowBuilderContent') { + const flowContents = await this.getFlowContentsByContentId( + botContext, + message.contentId + ) + for (const content of flowContents) { + await content.processContent(botContext) + this.jsxElements.push(content.toBotonic(botContext)) + } + } + } + return + } + + private getActiveInputGuardrailRules( + inputGuardrailRules: HtInputGuardrailRule[] + ): GuardrailRule[] { + return ( + inputGuardrailRules + ?.filter(rule => rule.is_active) + ?.map(rule => ({ + name: rule.name, + description: rule.description, + })) || [] + ) + } + + async processContent( + botContext: BotContext, + previousContents?: FlowContent[] + ): Promise { + if (this.messages.length === 0) { + await this.resolveAIAgentResponse(botContext, previousContents) + } + if (this.jsxElements.length === 0) { + await this.filterContent(botContext, this) + await this.messagesToBotonicJSXElements(botContext) + } + return + } + + toBotonic(): JSX.Element { + return <>{this.jsxElements} + } +} diff --git a/packages/botonic-plugin-flow-builder/src/content-fields/flow-ai-agent-router.tsx b/packages/botonic-plugin-flow-builder/src/content-fields/flow-ai-agent-router.tsx new file mode 100644 index 0000000000..6514986f28 --- /dev/null +++ b/packages/botonic-plugin-flow-builder/src/content-fields/flow-ai-agent-router.tsx @@ -0,0 +1,245 @@ +import { + type AgenticOutputMessage, + AiAgentType, + type BotContext, + type GuardrailRule, + type HubtypeAssistantMessage, + type InferenceResponse, + VerbosityLevel, +} from '@botonic/core' +import type { FlowBuilderApi } from '../api' +import { + type FlowBuilderContentMessage, + FlowBuilderContentSchema, +} from '../structured-output/flow-builder-content' +import { getCommonFlowContentEventArgsForContentId } from '../tracking' +import { HubtypeAssistantContent } from '../utils/ai-agent' +import { getFlowBuilderPlugin } from '../utils/get-flow-builder-plugin' +import { ContentFieldsBase } from './content-fields-base' +import { FlowAiAgent } from './flow-ai-agent' +import type { + HtAiAgentNode, + HtAiAgentRouterNode, + HtInputGuardrailRule, + HtNodeWithContent, +} from './hubtype-fields' +import { FlowCarousel, type FlowContent, FlowText } from './index' + +export interface AiAgentWithNameAndDescription { + agent: FlowAiAgent + description: string + name: string +} + +export class FlowAiAgentRouter extends ContentFieldsBase { + public name: string = '' + public instructions: string = '' + public model: string = '' + public verbosity: VerbosityLevel = VerbosityLevel.Medium + public agents: AiAgentWithNameAndDescription[] = [] + public inputGuardrailRules: HtInputGuardrailRule[] = [] + + public aiAgentResponse?: InferenceResponse + public messages: AgenticOutputMessage[] = [] + public jsxElements: JSX.Element[] = [] + + static fromHubtypeCMS( + component: HtAiAgentRouterNode, + cmsApi: FlowBuilderApi + ): FlowAiAgentRouter { + const newAiAgentRouter = new FlowAiAgentRouter(component.id) + newAiAgentRouter.name = component.code + newAiAgentRouter.instructions = component.content.instructions + newAiAgentRouter.model = component.content.model + newAiAgentRouter.verbosity = component.content.verbosity + newAiAgentRouter.agents = component.content.agent_slots.map(agentSlot => { + const agentNode = cmsApi.getNodeById(agentSlot.target.id) + const aiAgent = FlowAiAgent.fromHubtypeCMS(agentNode) + return { + agent: aiAgent, + description: agentSlot.description || '', + name: agentSlot.name || '', + } + }) + newAiAgentRouter.inputGuardrailRules = + component.content.input_guardrail_rules || [] + return newAiAgentRouter + } + + async resolveAIAgentResponse( + botContext: BotContext, + previousContents?: FlowContent[] + ): Promise | undefined> { + const aiAgentResponse = await this.getAIAgentResponse( + botContext, + previousContents + ) + + if (aiAgentResponse) { + this.aiAgentResponse = aiAgentResponse + await this.trackAiAgentResponse(botContext) + this.messages = aiAgentResponse.messages + } + + return aiAgentResponse + } + + async getAIAgentResponse( + botContext: BotContext, + previousContents?: FlowContent[] + ): Promise | undefined> { + const previousHubtypeContents: HubtypeAssistantMessage[] = + previousContents?.map(content => { + return { + role: 'assistant', + content: HubtypeAssistantContent.adapt(content), + } + }) || [] + + const flowBuilderPlugin = getFlowBuilderPlugin(botContext.plugins) + const aiAgentResponse = await flowBuilderPlugin.getAiAgentResponse?.( + botContext, + { + type: AiAgentType.Router, + name: this.name, + instructions: this.instructions, + model: this.model, + verbosity: this.verbosity, + agents: this.agents.map(({ agent, description, name }) => ({ + type: AiAgentType.Worker, + name, + description, + instructions: agent.instructions, + model: agent.model, + verbosity: agent.verbosity, + activeTools: agent.activeTools ?? [], + inputGuardrailRules: this.getActiveInputGuardrailRules( + agent.inputGuardrailRules + ), + sourceIds: agent.sources?.map(s => s.id) ?? [], + })), + inputGuardrailRules: this.getActiveInputGuardrailRules( + this.inputGuardrailRules + ), + outputMessagesSchemas: [FlowBuilderContentSchema], + previousHubtypeMessages: previousHubtypeContents, + } + ) + + console.log('FlowAiAgentRouter aiAgentResponse', { + aiAgentResponse, + }) + + return aiAgentResponse + } + + async trackFlow(_botContext: BotContext): Promise { + return + } + + async trackAiAgentResponse(botContext: BotContext) { + const { flowThreadId, flowId, flowName, flowNodeId } = + getCommonFlowContentEventArgsForContentId(botContext, this.id) + + // TODO: Create a new endpoint for AIAgentRouter + const event = { + action: 'AIAgentRouter', + flowThreadId: flowThreadId, + flowId: flowId, + flowName: flowName, + flowNodeId: flowNodeId, + flowNodeContentId: this.name, + flowNodeIsMeaningful: true, + toolsExecuted: this.aiAgentResponse?.toolsExecuted ?? [], + memoryLength: this.aiAgentResponse?.memoryLength ?? 0, + inputMessageId: botContext.input.message_id!, + exit: this.aiAgentResponse?.exit ?? true, + inputGuardrailsTriggered: + this.aiAgentResponse?.inputGuardrailsTriggered ?? [], + outputGuardrailsTriggered: [], //aiAgentResponse.outputGuardrailsTriggered, + } + + const { action, ...eventArgs } = event + + console.log('trackAiAgentResponse', { + action, + eventArgs, + }) + // await trackEvent(botContext, action, eventArgs) + } + + async getFlowContentsByContentId( + botContext: BotContext, + contentId: string + ): Promise { + const flowBuilderPlugin = getFlowBuilderPlugin(botContext.plugins) + const cmsApi = flowBuilderPlugin.cmsApi + const node = cmsApi.getNodeByContentID(contentId) + const flowContents = await flowBuilderPlugin.getContentsByNode( + node as HtNodeWithContent + ) + + return flowContents + } + + async messagesToBotonicJSXElements(botContext: BotContext): Promise { + for (const message of this.messages) { + if ( + message.type === 'text' || + message.type === 'textWithButtons' || + message.type === 'botExecutor' + ) { + this.jsxElements.push(FlowText.fromAIAgent(this.id, message)) + } + + if (message.type === 'carousel') { + this.jsxElements.push( + FlowCarousel.fromAIAgent(this.id, message, botContext) + ) + } + + if (message.type === 'flowBuilderContent') { + const flowContents = await this.getFlowContentsByContentId( + botContext, + message.contentId + ) + for (const content of flowContents) { + await content.processContent(botContext) + this.jsxElements.push(content.toBotonic(botContext)) + } + } + } + return + } + + private getActiveInputGuardrailRules( + inputGuardrailRules: HtInputGuardrailRule[] + ): GuardrailRule[] { + return ( + inputGuardrailRules + ?.filter(rule => rule.is_active) + ?.map(rule => ({ + name: rule.name, + description: rule.description, + })) || [] + ) + } + + async processContent( + botContext: BotContext, + previousContents?: FlowContent[] + ): Promise { + if (this.messages.length === 0) { + await this.resolveAIAgentResponse(botContext, previousContents) + } + if (this.jsxElements.length === 0) { + await this.filterContent(botContext, this) + await this.messagesToBotonicJSXElements(botContext) + } + return + } + + toBotonic(): JSX.Element { + return <>{this.jsxElements} + } +} diff --git a/packages/botonic-plugin-flow-builder/src/content-fields/flow-ai-agent.tsx b/packages/botonic-plugin-flow-builder/src/content-fields/flow-ai-agent.tsx index b57a90ba6c..0b9f033291 100644 --- a/packages/botonic-plugin-flow-builder/src/content-fields/flow-ai-agent.tsx +++ b/packages/botonic-plugin-flow-builder/src/content-fields/flow-ai-agent.tsx @@ -1,5 +1,6 @@ import { type AgenticOutputMessage, + AiAgentType, type BotContext, EventAction, type EventAiAgent, @@ -87,26 +88,23 @@ export class FlowAiAgent extends ContentFieldsBase { } }) || [] - const activeInputGuardrailRules: GuardrailRule[] = + const activeInputGuardrailRules = this.getActiveInputGuardrailRules( this.inputGuardrailRules - ?.filter(rule => rule.is_active) - ?.map(rule => ({ - name: rule.name, - description: rule.description, - })) || [] + ) const flowBuilderPlugin = getFlowBuilderPlugin(botContext.plugins) const aiAgentResponse = await flowBuilderPlugin.getAiAgentResponse?.( botContext, { + type: AiAgentType.Worker, name: this.name, instructions: this.instructions, model: this.model, verbosity: this.verbosity, - activeTools: this.activeTools, + activeTools: this.activeTools ?? [], inputGuardrailRules: activeInputGuardrailRules, - sourceIds: this.sources?.map(source => source.id), + sourceIds: this.sources?.map(source => source.id) ?? [], outputMessagesSchemas: [FlowBuilderContentSchema], previousHubtypeMessages: previousHubtypeContents, } @@ -188,6 +186,19 @@ export class FlowAiAgent extends ContentFieldsBase { return } + private getActiveInputGuardrailRules( + inputGuardrailRules: HtInputGuardrailRule[] + ): GuardrailRule[] { + return ( + inputGuardrailRules + ?.filter(rule => rule.is_active) + ?.map(rule => ({ + name: rule.name, + description: rule.description, + })) || [] + ) + } + async processContent( botContext: BotContext, previousContents?: FlowContent[] diff --git a/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/ai-agent-manager.ts b/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/ai-agent-manager.ts new file mode 100644 index 0000000000..fba1c46a42 --- /dev/null +++ b/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/ai-agent-manager.ts @@ -0,0 +1,17 @@ +import type { VerbosityLevel } from '@botonic/core' +import type { HtInputGuardrailRule } from './ai-agent' +import type { HtAiAgentSlotNode } from './ai-agent-router' +import type { HtBaseNode } from './common' +import type { HtNodeWithContentType } from './node-types' + +export interface HtAiAgentManagerNode extends HtBaseNode { + type: HtNodeWithContentType.AI_AGENT_MANAGER + content: { + instructions: string + model: string + verbosity: VerbosityLevel + active_tools?: { name: string }[] + agent_slots: HtAiAgentSlotNode[] + input_guardrail_rules?: HtInputGuardrailRule[] + } +} diff --git a/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/ai-agent-router.ts b/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/ai-agent-router.ts new file mode 100644 index 0000000000..c964687316 --- /dev/null +++ b/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/ai-agent-router.ts @@ -0,0 +1,22 @@ +import type { VerbosityLevel } from '@botonic/core' +import type { HtInputGuardrailRule } from './ai-agent' +import type { HtBaseNode, HtNodeLink } from './common' +import type { HtNodeWithContentType } from './node-types' + +export interface HtAiAgentSlotNode { + id: string + target: HtNodeLink + name?: string + description?: string +} + +export interface HtAiAgentRouterNode extends HtBaseNode { + type: HtNodeWithContentType.AI_AGENT_ROUTER + content: { + instructions: string + model: string + verbosity: VerbosityLevel + agent_slots: HtAiAgentSlotNode[] + input_guardrail_rules?: HtInputGuardrailRule[] + } +} diff --git a/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/index.ts b/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/index.ts index 71d7e67bfc..0723d026a5 100644 --- a/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/index.ts +++ b/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/index.ts @@ -1,3 +1,6 @@ +export * from './ai-agent' +export * from './ai-agent-manager' +export * from './ai-agent-router' export * from './bot-action' export * from './button' export * from './capture-user-input' diff --git a/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/node-types.ts b/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/node-types.ts index f87cfa1b4c..8ed7ca86f1 100644 --- a/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/node-types.ts +++ b/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/node-types.ts @@ -14,6 +14,8 @@ export enum HtNodeWithContentType { KNOWLEDGE_BASE = 'knowledge-base', BOT_ACTION = 'bot-action', AI_AGENT = 'ai-agent', + AI_AGENT_ROUTER = 'ai-agent-router', + AI_AGENT_MANAGER = 'ai-agent-manager', RATING = 'rating', WEBVIEW = 'webview', GO_TO_FLOW = 'go-to-flow', diff --git a/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/nodes.ts b/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/nodes.ts index 1a875e146c..d3d4109135 100644 --- a/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/nodes.ts +++ b/packages/botonic-plugin-flow-builder/src/content-fields/hubtype-fields/nodes.ts @@ -1,4 +1,6 @@ import type { HtAiAgentNode } from './ai-agent' +import type { HtAiAgentManagerNode } from './ai-agent-manager' +import type { HtAiAgentRouterNode } from './ai-agent-router' import type { HtBotActionNode } from './bot-action' import type { HtCaptureUserInputNode } from './capture-user-input' import type { HtCarouselNode } from './carousel' @@ -39,6 +41,8 @@ export type HtNodeWithContent = | HtKnowledgeBaseNode | HtBotActionNode | HtAiAgentNode + | HtAiAgentRouterNode + | HtAiAgentManagerNode | HtRatingNode | HtWebviewNode | HtGoToFlow diff --git a/packages/botonic-plugin-flow-builder/src/content-fields/index.ts b/packages/botonic-plugin-flow-builder/src/content-fields/index.ts index 262f4e4ae0..1701366482 100644 --- a/packages/botonic-plugin-flow-builder/src/content-fields/index.ts +++ b/packages/botonic-plugin-flow-builder/src/content-fields/index.ts @@ -1,4 +1,6 @@ import { FlowAiAgent } from './flow-ai-agent' +import { FlowAiAgentManager } from './flow-ai-agent-manager' +import { FlowAiAgentRouter } from './flow-ai-agent-router' import { FlowBotAction } from './flow-bot-action' import { FlowCaptureUserInput } from './flow-capture-user-input' import { FlowCarousel } from './flow-carousel' @@ -24,6 +26,8 @@ export { ContentFieldsBase } from './content-fields-base' export { FlowButton } from './flow-button' export { FlowElement } from './flow-element' export { + FlowAiAgentRouter, + FlowAiAgentManager, FlowAiAgent, FlowBotAction, FlowCaptureUserInput, @@ -63,5 +67,7 @@ export type FlowContent = | FlowCustomConditional | FlowGoToFlow | FlowCaptureUserInput + | FlowAiAgentRouter + | FlowAiAgentManager export { DISABLED_MEMORY_LENGTH } diff --git a/packages/botonic-plugin-flow-builder/src/flow-factory.ts b/packages/botonic-plugin-flow-builder/src/flow-factory.ts index 5f8ad3b815..1bb9997e44 100644 --- a/packages/botonic-plugin-flow-builder/src/flow-factory.ts +++ b/packages/botonic-plugin-flow-builder/src/flow-factory.ts @@ -3,6 +3,8 @@ import type { ActionRequest } from '@botonic/react' import type { FlowBuilderApi } from './api' import { FlowAiAgent, + FlowAiAgentManager, + FlowAiAgentRouter, FlowBotAction, FlowCarousel, FlowChannelConditional, @@ -98,6 +100,12 @@ export class FlowFactory { case HtNodeWithContentType.CAPTURE_USER_INPUT: return FlowCaptureUserInput.fromHubtypeCMS(hubtypeContent) + case HtNodeWithContentType.AI_AGENT_ROUTER: + return FlowAiAgentRouter.fromHubtypeCMS(hubtypeContent, this.cmsApi) + + case HtNodeWithContentType.AI_AGENT_MANAGER: + return FlowAiAgentManager.fromHubtypeCMS(hubtypeContent, this.cmsApi) + default: return undefined } diff --git a/packages/botonic-plugin-flow-builder/src/utils/ai-agent.ts b/packages/botonic-plugin-flow-builder/src/utils/ai-agent.ts index 05da0a2dbf..6a268f4937 100644 --- a/packages/botonic-plugin-flow-builder/src/utils/ai-agent.ts +++ b/packages/botonic-plugin-flow-builder/src/utils/ai-agent.ts @@ -1,5 +1,6 @@ import { WhatsappCTAUrlHeaderType } from '@botonic/react' - +import { FlowAiAgentManager } from '../content-fields/flow-ai-agent-manager' +import { FlowAiAgentRouter } from '../content-fields/flow-ai-agent-router' import type { FlowButton } from '../content-fields/flow-button' import { HtButtonStyle } from '../content-fields/hubtype-fields' import { @@ -14,14 +15,45 @@ import { FlowWhatsappTemplate, } from '../content-fields/index' -interface AiAgentContentAndContentsBeforeAiAgent { - aiAgentContent: FlowAiAgent - contentsBeforeAiAgent: FlowContent[] -} +type AiAgentContentAndContentsBeforeAiAgent = + | { + aiAgentContent: FlowAiAgent + contentsBeforeAiAgent: FlowContent[] + } + | { + aiAgentRouterContent: FlowAiAgentRouter + contentsBeforeAiAgentRouter: FlowContent[] + } + | { + aiAgentManagerContent: FlowAiAgentManager + contentsBeforeAiAgentManager: FlowContent[] + } export function splitAiAgentContents( contents: FlowContent[] ): AiAgentContentAndContentsBeforeAiAgent | undefined { + const aiAgentRouterIndex = contents.findIndex( + content => content instanceof FlowAiAgentRouter + ) + if (aiAgentRouterIndex >= 0) { + return { + aiAgentRouterContent: contents[aiAgentRouterIndex] as FlowAiAgentRouter, + contentsBeforeAiAgentRouter: contents.slice(0, aiAgentRouterIndex), + } + } + + const aiAgentManagerIndex = contents.findIndex( + content => content instanceof FlowAiAgentManager + ) + if (aiAgentManagerIndex >= 0) { + return { + aiAgentManagerContent: contents[ + aiAgentManagerIndex + ] as FlowAiAgentManager, + contentsBeforeAiAgentManager: contents.slice(0, aiAgentManagerIndex), + } + } + const aiAgentIndex = contents.findIndex( content => content instanceof FlowAiAgent )