diff --git a/src/Observability/Hosting/Extensions/ChannelAdapterExtensions.cs b/src/Observability/Hosting/Extensions/ChannelAdapterExtensions.cs new file mode 100644 index 00000000..f63776f9 --- /dev/null +++ b/src/Observability/Hosting/Extensions/ChannelAdapterExtensions.cs @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using Microsoft.Agents.A365.Observability.Hosting.Middleware; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts; +using Microsoft.Agents.Builder; + +namespace Microsoft.Agents.A365.Observability.Hosting.Extensions +{ + /// + /// Extension methods for adding observability middleware to the channel adapter. + /// + public static class ChannelAdapterExtensions + { + /// + /// Adds the to the adapter's pipeline. + /// + /// The channel adapter to add the middleware to. + /// The updated channel adapter. + /// + /// + /// This method adds the to the adapter's pipeline, + /// which creates InputScope and OutputScope spans for incoming and outgoing activities respectively. + /// + /// + /// + /// // In your adapter configuration: + /// adapter.UseA365Observability(); + /// + /// + /// + public static IChannelAdapter UseA365Observability(this IChannelAdapter adapter) + { + if (adapter == null) + { + throw new ArgumentNullException(nameof(adapter)); + } + + return adapter.Use(new ObservabilityMiddleware()); + } + + /// + /// Adds the to the adapter's pipeline with custom resolvers. + /// + /// The channel adapter to add the middleware to. + /// Optional resolver to extract agent details from the turn context. + /// Optional resolver to extract caller details from the turn context. + /// The updated channel adapter. + /// + /// + /// This method adds the to the adapter's pipeline with custom resolvers. + /// Use this overload when you need to provide custom logic for extracting agent and caller details from the turn context. + /// + /// + /// + /// // In your adapter configuration: + /// adapter.UseA365Observability( + /// agentDetailsResolver: turnContext => new AgentDetails( + /// agentId: "my-agent-id", + /// agentName: "My Agent"), + /// callerDetailsResolver: turnContext => new CallerDetails( + /// callerId: turnContext.Activity?.From?.Id ?? "unknown", + /// callerName: turnContext.Activity?.From?.Name ?? "Unknown", + /// callerUpn: turnContext.Activity?.From?.Name ?? "unknown")); + /// + /// + /// + public static IChannelAdapter UseA365Observability( + this IChannelAdapter adapter, + Func? agentDetailsResolver = null, + Func? callerDetailsResolver = null) + { + if (adapter == null) + { + throw new ArgumentNullException(nameof(adapter)); + } + + return adapter.Use(new ObservabilityMiddleware(agentDetailsResolver, callerDetailsResolver)); + } + + /// + /// Adds the provided instance to the adapter's pipeline. + /// + /// The channel adapter to add the middleware to. + /// The observability middleware instance to add. + /// The updated channel adapter. + /// + /// + /// Use this method when you have already created an instance, + /// for example when using dependency injection. + /// + /// + /// + /// // When using DI: + /// var middleware = serviceProvider.GetRequiredService<ObservabilityMiddleware>(); + /// adapter.UseA365Observability(middleware); + /// + /// + /// + public static IChannelAdapter UseA365Observability( + this IChannelAdapter adapter, + ObservabilityMiddleware middleware) + { + if (adapter == null) + { + throw new ArgumentNullException(nameof(adapter)); + } + + if (middleware == null) + { + throw new ArgumentNullException(nameof(middleware)); + } + + return adapter.Use(middleware); + } + } +} diff --git a/src/Observability/Hosting/Microsoft.Agents.A365.Observability.Hosting.csproj b/src/Observability/Hosting/Microsoft.Agents.A365.Observability.Hosting.csproj index 271f6a1c..8ede3261 100644 --- a/src/Observability/Hosting/Microsoft.Agents.A365.Observability.Hosting.csproj +++ b/src/Observability/Hosting/Microsoft.Agents.A365.Observability.Hosting.csproj @@ -20,5 +20,6 @@ + diff --git a/src/Observability/Hosting/Middleware/ObservabilityMiddleware.cs b/src/Observability/Hosting/Middleware/ObservabilityMiddleware.cs new file mode 100644 index 00000000..33439518 --- /dev/null +++ b/src/Observability/Hosting/Middleware/ObservabilityMiddleware.cs @@ -0,0 +1,303 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Agents.A365.Observability.Hosting.Extensions; +using Microsoft.Agents.A365.Observability.Runtime.Common; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes; +using Microsoft.Agents.Builder; +using Microsoft.Agents.Core.Models; + +namespace Microsoft.Agents.A365.Observability.Hosting.Middleware +{ + /// + /// Middleware for logging incoming and outgoing activities to the Microsoft Agent 365 observability infrastructure. + /// Creates InputScope and OutputScope spans for incoming and outgoing activities respectively. + /// + public class ObservabilityMiddleware : IMiddleware + { + private readonly Func? _agentDetailsResolver; + private readonly Func? _callerDetailsResolver; + + /// + /// Initializes a new instance of the class. + /// + public ObservabilityMiddleware() + { + } + + /// + /// Initializes a new instance of the class with custom resolvers. + /// + /// Optional resolver to extract agent details from the turn context. + /// Optional resolver to extract caller details from the turn context. + public ObservabilityMiddleware( + Func? agentDetailsResolver = null, + Func? callerDetailsResolver = null) + { + _agentDetailsResolver = agentDetailsResolver; + _callerDetailsResolver = callerDetailsResolver; + } + + /// + /// Processes an incoming activity, creating observability spans for input and output. + /// + /// The context object for this turn. + /// The delegate to call to continue the Agent middleware pipeline. + /// A cancellation token that can be used by other objects + /// or threads to receive notice of cancellation. + /// A task that represents the work queued to execute. + public async Task OnTurnAsync(ITurnContext turnContext, NextDelegate next, CancellationToken cancellationToken = default) + { + if (turnContext == null) + { + throw new ArgumentNullException(nameof(turnContext)); + } + + var outputActivities = new List(); + + // Extract details from turn context + var agentDetails = ResolveAgentDetails(turnContext); + var tenantDetails = ResolveTenantDetails(turnContext); + var callerDetails = ResolveCallerDetails(turnContext); + var request = ResolveRequest(turnContext); + var conversationId = turnContext.Activity?.Conversation?.Id; + var sessionId = turnContext.Activity?.Conversation?.Id; + var sourceMetadata = ResolveSourceMetadata(turnContext); + var executionType = ResolveExecutionType(turnContext); + + // Set baggage context from turn context for downstream operations + using (new BaggageBuilder().FromTurnContext(turnContext).Build()) + { + // Create InputScope for incoming activity + using (var inputScope = CreateInputScope(turnContext, agentDetails, tenantDetails, request, callerDetails, conversationId, sessionId)) + { + // Inject observability context into turn context for downstream use + if (inputScope != null) + { + turnContext.InjectObservabilityContext(inputScope); + } + + // Hook up OnSendActivities to capture outgoing activities + turnContext.OnSendActivities(async (ctx, activities, nextSend) => + { + // Run the full pipeline first + var responses = await nextSend().ConfigureAwait(false); + + // Collect sent activities for output scope + foreach (var activity in activities) + { + if (activity != null) + { + outputActivities.Add(CloneActivity(activity)); + } + } + + return responses; + }); + + // Process Agent logic + await next(cancellationToken).ConfigureAwait(false); + } + } + + // Create OutputScope for outgoing activities after the turn completes + if (outputActivities.Count > 0) + { + var outputMessages = ExtractOutputMessages(outputActivities); + var response = outputMessages.Length > 0 ? new Response(string.Join(",", outputMessages)) : null; + + using (var outputScope = OutputScope.Start( + agentDetails: agentDetails, + tenantDetails: tenantDetails, + response: response, + callerDetails: callerDetails, + conversationId: conversationId, + sessionId: sessionId, + sourceMetadata: sourceMetadata, + executionType: executionType)) + { + outputScope.RecordOutputMessages(outputMessages); + } + } + } + + private InputScope? CreateInputScope( + ITurnContext turnContext, + AgentDetails agentDetails, + TenantDetails tenantDetails, + Request? request, + CallerDetails? callerDetails, + string? conversationId, + string? sessionId) + { + // Skip creating scope for ContinueConversation events (used to initialize middleware) + if (turnContext.Activity?.Type == ActivityTypes.Event && + turnContext.Activity?.Name == ActivityEventNames.ContinueConversation) + { + return null; + } + + var inputScope = InputScope.Start( + agentDetails: agentDetails, + tenantDetails: tenantDetails, + request: request, + callerDetails: callerDetails, + conversationId: conversationId, + sessionId: sessionId); + + // Record input message if present + var activityText = turnContext.Activity?.Text; + if (!string.IsNullOrEmpty(activityText)) + { + inputScope.RecordInputMessages(new[] { activityText! }); + } + + return inputScope; + } + + private AgentDetails ResolveAgentDetails(ITurnContext turnContext) + { + if (_agentDetailsResolver != null) + { + return _agentDetailsResolver(turnContext); + } + + // Extract from turn context + var activity = turnContext.Activity; + return new AgentDetails( + agentId: activity?.Recipient?.AgenticAppId ?? activity?.Recipient?.Id, + agentName: activity?.Recipient?.Name, + agentAUID: activity?.Recipient?.AgenticUserId ?? activity?.Recipient?.AadObjectId, + agentUPN: activity?.Recipient?.Name, + tenantId: activity?.Recipient?.TenantId); + } + + private TenantDetails ResolveTenantDetails(ITurnContext turnContext) + { + var tenantId = turnContext.Activity?.Recipient?.TenantId; + + // Try to extract from ChannelData if not available on recipient + if (string.IsNullOrWhiteSpace(tenantId) && turnContext.Activity?.ChannelData != null) + { + try + { + var channelDataJson = turnContext.Activity.ChannelData.ToString(); + if (!string.IsNullOrWhiteSpace(channelDataJson)) + { + using var doc = System.Text.Json.JsonDocument.Parse(channelDataJson); + if (doc.RootElement.TryGetProperty("tenant", out var tenantElem) && + tenantElem.TryGetProperty("id", out var idElem) && + idElem.ValueKind == System.Text.Json.JsonValueKind.String) + { + tenantId = idElem.GetString(); + } + } + } + catch + { + // Ignore parsing errors + } + } + + return Guid.TryParse(tenantId, out var guid) ? new TenantDetails(guid) : new TenantDetails(Guid.Empty); + } + + private CallerDetails? ResolveCallerDetails(ITurnContext turnContext) + { + if (_callerDetailsResolver != null) + { + return _callerDetailsResolver(turnContext); + } + + var from = turnContext.Activity?.From; + if (from == null || string.IsNullOrEmpty(from.Id)) + { + return null; + } + + return new CallerDetails( + callerId: from.Id, + callerName: from.Name ?? string.Empty, + callerUpn: from.Name ?? string.Empty, + tenantId: from.TenantId); + } + + private Request? ResolveRequest(ITurnContext turnContext) + { + var content = turnContext.Activity?.Text; + if (string.IsNullOrEmpty(content)) + { + return null; + } + + return new Request( + content: content!, + executionType: ResolveExecutionType(turnContext), + sourceMetadata: ResolveSourceMetadata(turnContext)); + } + + private SourceMetadata? ResolveSourceMetadata(ITurnContext turnContext) + { + var channelId = turnContext.Activity?.ChannelId; + if (channelId == null) + { + return null; + } + + return new SourceMetadata( + name: channelId.Channel, + description: channelId.SubChannel); + } + + private ExecutionType ResolveExecutionType(ITurnContext turnContext) + { + const string AgentRole = "agenticUser"; + + var isAgenticCaller = turnContext.Activity?.From?.AgenticUserId != null + || (turnContext.Activity?.From?.Role != null && + turnContext.Activity.From.Role.Equals(AgentRole, StringComparison.OrdinalIgnoreCase)); + + var isAgenticRecipient = turnContext.Activity?.Recipient?.AgenticUserId != null + || (turnContext.Activity?.Recipient?.Role != null && + turnContext.Activity.Recipient.Role.Equals(AgentRole, StringComparison.OrdinalIgnoreCase)); + + return isAgenticRecipient && isAgenticCaller + ? ExecutionType.Agent2Agent + : ExecutionType.HumanToAgent; + } + + private static IActivity CloneActivity(IActivity activity) + { + var cloned = activity.Clone(); + EnsureActivityHasId(cloned); + return cloned; + } + + private static void EnsureActivityHasId(IActivity activity) + { + if (activity != null && string.IsNullOrEmpty(activity.Id)) + { + activity.Id = $"g_{Guid.NewGuid()}"; + } + } + + private static string[] ExtractOutputMessages(List activities) + { + var messages = new List(); + foreach (var activity in activities) + { + var text = activity?.Text; + if (!string.IsNullOrEmpty(text)) + { + messages.Add(text!); + } + } + return messages.ToArray(); + } + } +} diff --git a/src/Observability/Hosting/ObservabilityBuilderExtensions.cs b/src/Observability/Hosting/ObservabilityBuilderExtensions.cs index 17b6fbd5..29f7830b 100644 --- a/src/Observability/Hosting/ObservabilityBuilderExtensions.cs +++ b/src/Observability/Hosting/ObservabilityBuilderExtensions.cs @@ -3,9 +3,13 @@ namespace Microsoft.Agents.A365.Observability.Hosting { using System; + using Microsoft.Agents.A365.Observability.Hosting.Middleware; using Microsoft.Agents.A365.Observability.Runtime; + using Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts; using Microsoft.Agents.A365.Observability.Runtime.Tracing.Exporters; + using Microsoft.Agents.Builder; using Microsoft.AspNetCore.Hosting; + using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; /// @@ -66,5 +70,72 @@ public static IHostBuilder AddA365Tracing( }); return builder; } + + /// + /// Adds the to the builder's service collection. + /// + /// The type of the application builder implementing . + /// The builder to configure. + /// The configured builder for method chaining. + /// + /// + /// This method registers the as a singleton service. + /// After calling this method, you need to add the middleware to your adapter's pipeline. + /// + /// + /// + /// // In your startup/program configuration: + /// builder.WithObservabilityMiddleware(); + /// + /// // Then add to your adapter: + /// adapter.Use(serviceProvider.GetRequiredService<ObservabilityMiddleware>()); + /// // Or use the extension method: + /// adapter.UseA365Observability(serviceProvider.GetRequiredService<ObservabilityMiddleware>()); + /// + /// + /// + public static TBuilder WithObservabilityMiddleware(this TBuilder builder) + where TBuilder : IHostApplicationBuilder + { + builder.Services.AddSingleton(); + return builder; + } + + /// + /// Adds the to the builder's service collection with custom resolvers. + /// + /// The type of the application builder implementing . + /// The builder to configure. + /// Optional resolver to extract agent details from the turn context. + /// Optional resolver to extract caller details from the turn context. + /// The configured builder for method chaining. + /// + /// + /// This method registers the as a singleton service with custom resolvers. + /// Use this overload when you need to provide custom logic for extracting agent and caller details from the turn context. + /// + /// + /// + /// // In your startup/program configuration: + /// builder.WithObservabilityMiddleware( + /// agentDetailsResolver: turnContext => new AgentDetails( + /// agentId: "my-agent-id", + /// agentName: "My Agent"), + /// callerDetailsResolver: turnContext => new CallerDetails( + /// callerId: turnContext.Activity?.From?.Id ?? "unknown", + /// callerName: turnContext.Activity?.From?.Name ?? "Unknown", + /// callerUpn: turnContext.Activity?.From?.Name ?? "unknown")); + /// + /// + /// + public static TBuilder WithObservabilityMiddleware( + this TBuilder builder, + Func? agentDetailsResolver = null, + Func? callerDetailsResolver = null) + where TBuilder : IHostApplicationBuilder + { + builder.Services.AddSingleton(sp => new ObservabilityMiddleware(agentDetailsResolver, callerDetailsResolver)); + return builder; + } } } diff --git a/src/Observability/Hosting/ObservabilityServiceCollectionExtensions.cs b/src/Observability/Hosting/ObservabilityServiceCollectionExtensions.cs index 7e3875c0..70c2b9d2 100644 --- a/src/Observability/Hosting/ObservabilityServiceCollectionExtensions.cs +++ b/src/Observability/Hosting/ObservabilityServiceCollectionExtensions.cs @@ -2,9 +2,9 @@ // Licensed under the MIT License. namespace Microsoft.Agents.A365.Observability.Hosting { - using Microsoft.Extensions.DependencyInjection; using Microsoft.Agents.A365.Observability.Hosting.Caching; using Microsoft.Agents.A365.Observability.Runtime.Tracing.Exporters; + using Microsoft.Extensions.DependencyInjection; /// /// Provides extension methods for configuring Microsoft Agent 365 SDK with OpenTelemetry tracing. diff --git a/src/Observability/Runtime/Tracing/Contracts/Response.cs b/src/Observability/Runtime/Tracing/Contracts/Response.cs new file mode 100644 index 00000000..b396e89d --- /dev/null +++ b/src/Observability/Runtime/Tracing/Contracts/Response.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts +{ + /// + /// Represents a response from an AI agent with telemetry context. + /// + public sealed class Response : IEquatable + { + /// + /// Initializes a new instance of the class. + /// + /// The payload content returned by the agent. + public Response(string content) + { + Content = content; + } + + /// + /// Gets the textual content of the response. + /// + public string Content { get; } + + /// + public bool Equals(Response? other) + { + if (other is null) + { + return false; + } + + return string.Equals(Content, other.Content, StringComparison.Ordinal); + } + + /// + public override bool Equals(object? obj) + { + return Equals(obj as Response); + } + + /// + public override int GetHashCode() + { + return Content != null ? StringComparer.Ordinal.GetHashCode(Content) : 0; + } + } +} diff --git a/src/Observability/Runtime/Tracing/Scopes/InputScope.cs b/src/Observability/Runtime/Tracing/Scopes/InputScope.cs new file mode 100644 index 00000000..c56cea9e --- /dev/null +++ b/src/Observability/Runtime/Tracing/Scopes/InputScope.cs @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts; + +namespace Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes +{ + /// + /// Provides OpenTelemetry tracing scope for AI agent input operations. + /// + public sealed class InputScope : OpenTelemetryScope + { + /// + /// The operation name for input tracing. + /// + public const string OperationName = "input_messages"; + + /// + /// Creates and starts a new scope for input tracing. + /// + /// Information about the agent receiving the input (service, version, identifiers). + /// Tenant context used for telemetry enrichment and correlation. + /// Optional request content containing input messages and execution context. + /// Optional details about the non-agentic caller. + /// Optional conversation or session correlation ID. + /// Optional session identifier. + /// Optional session description. + /// Optional parent Activity ID used to link this span to an upstream operation. + /// A new InputScope instance. + /// + /// + /// Certification Requirements: The following parameters must be set for the agent to pass certification requirements: + /// + /// + /// + /// + /// + /// + /// Learn more about certification requirements + /// + /// + public static InputScope Start( + AgentDetails agentDetails, + TenantDetails tenantDetails, + Request? request = null, + CallerDetails? callerDetails = null, + string? conversationId = null, + string? sessionId = null, + string? sessionDescription = null, + string? parentId = null) => new InputScope(agentDetails, tenantDetails, request, callerDetails, conversationId, sessionId, sessionDescription, parentId); + + private InputScope( + AgentDetails agentDetails, + TenantDetails tenantDetails, + Request? request, + CallerDetails? callerDetails, + string? conversationId, + string? sessionId, + string? sessionDescription, + string? parentId) + : base( + kind: ActivityKind.Client, + agentDetails: agentDetails, + tenantDetails: tenantDetails, + operationName: OperationName, + activityName: string.IsNullOrWhiteSpace(agentDetails.AgentName) + ? OperationName + : $"{OperationName} {agentDetails.AgentName}", + parentId: parentId, + conversationId: conversationId, + sourceMetadata: request?.SourceMetadata) + { + SetTagMaybe(OpenTelemetryConstants.SessionIdKey, sessionId); + SetTagMaybe(OpenTelemetryConstants.SessionDescriptionKey, sessionDescription); + SetTagMaybe(OpenTelemetryConstants.GenAiExecutionTypeKey, request?.ExecutionType?.ToString()); + + // Set caller details tags + if (callerDetails != null) + { + SetTagMaybe(OpenTelemetryConstants.GenAiCallerIdKey, callerDetails.CallerId); + SetTagMaybe(OpenTelemetryConstants.GenAiCallerUpnKey, callerDetails.CallerUpn); + SetTagMaybe(OpenTelemetryConstants.GenAiCallerNameKey, callerDetails.CallerName); + SetTagMaybe(OpenTelemetryConstants.GenAiCallerClientIpKey, callerDetails.CallerClientIP?.ToString()); + SetTagMaybe(OpenTelemetryConstants.GenAiCallerTenantIdKey, callerDetails.TenantId); + } + + // Set input messages + if (request?.Content != null) + { + SetTagMaybe(OpenTelemetryConstants.GenAiInputMessagesKey, request.Content); + } + } + + /// + /// Records the input messages for telemetry tracking. + /// + /// Array of input messages to record. + public void RecordInputMessages(string[] messages) + { + SetTagMaybe(OpenTelemetryConstants.GenAiInputMessagesKey, string.Join(",", messages)); + } + } +} diff --git a/src/Observability/Runtime/Tracing/Scopes/OutputScope.cs b/src/Observability/Runtime/Tracing/Scopes/OutputScope.cs new file mode 100644 index 00000000..66247c0c --- /dev/null +++ b/src/Observability/Runtime/Tracing/Scopes/OutputScope.cs @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts; + +namespace Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes +{ + /// + /// Provides OpenTelemetry tracing scope for AI agent output operations. + /// + public sealed class OutputScope : OpenTelemetryScope + { + /// + /// The operation name for output tracing. + /// + public const string OperationName = "output_messages"; + + /// + /// Creates and starts a new scope for output tracing. + /// + /// Information about the agent sending the output (service, version, identifiers). + /// Tenant context used for telemetry enrichment and correlation. + /// Optional response content containing output messages. + /// Optional details about the non-agentic caller. + /// Optional conversation or session correlation ID. + /// Optional session identifier. + /// Optional session description. + /// Optional parent Activity ID used to link this span to an upstream operation. + /// Optional metadata describing the source of the call for observability. + /// Optional execution type describing the request. + /// A new OutputScope instance. + /// + /// + /// Certification Requirements: The following parameters must be set for the agent to pass certification requirements: + /// + /// + /// + /// + /// + /// + /// Learn more about certification requirements + /// + /// + public static OutputScope Start( + AgentDetails agentDetails, + TenantDetails tenantDetails, + Response? response = null, + CallerDetails? callerDetails = null, + string? conversationId = null, + string? sessionId = null, + string? sessionDescription = null, + string? parentId = null, + SourceMetadata? sourceMetadata = null, + ExecutionType? executionType = null) => new OutputScope(agentDetails, tenantDetails, response, callerDetails, conversationId, sessionId, sessionDescription, parentId, sourceMetadata, executionType); + + private OutputScope( + AgentDetails agentDetails, + TenantDetails tenantDetails, + Response? response, + CallerDetails? callerDetails, + string? conversationId, + string? sessionId, + string? sessionDescription, + string? parentId, + SourceMetadata? sourceMetadata, + ExecutionType? executionType) + : base( + kind: ActivityKind.Client, + agentDetails: agentDetails, + tenantDetails: tenantDetails, + operationName: OperationName, + activityName: string.IsNullOrWhiteSpace(agentDetails.AgentName) + ? OperationName + : $"{OperationName} {agentDetails.AgentName}", + parentId: parentId, + conversationId: conversationId, + sourceMetadata: sourceMetadata) + { + SetTagMaybe(OpenTelemetryConstants.SessionIdKey, sessionId); + SetTagMaybe(OpenTelemetryConstants.SessionDescriptionKey, sessionDescription); + SetTagMaybe(OpenTelemetryConstants.GenAiExecutionTypeKey, executionType?.ToString()); + + // Set caller details tags + if (callerDetails != null) + { + SetTagMaybe(OpenTelemetryConstants.GenAiCallerIdKey, callerDetails.CallerId); + SetTagMaybe(OpenTelemetryConstants.GenAiCallerUpnKey, callerDetails.CallerUpn); + SetTagMaybe(OpenTelemetryConstants.GenAiCallerNameKey, callerDetails.CallerName); + SetTagMaybe(OpenTelemetryConstants.GenAiCallerClientIpKey, callerDetails.CallerClientIP?.ToString()); + SetTagMaybe(OpenTelemetryConstants.GenAiCallerTenantIdKey, callerDetails.TenantId); + } + + // Set output messages + if (response?.Content != null) + { + SetTagMaybe(OpenTelemetryConstants.GenAiOutputMessagesKey, response.Content); + } + } + + /// + /// Records the output messages for telemetry tracking. + /// + /// Array of output messages to record. + public void RecordOutputMessages(string[] messages) + { + SetTagMaybe(OpenTelemetryConstants.GenAiOutputMessagesKey, string.Join(",", messages)); + } + } +} diff --git a/src/Tests/Microsoft.Agents.A365.Observability.Runtime.IntegrationTests/Agent365ExporterE2ETests.cs b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.IntegrationTests/Agent365ExporterE2ETests.cs index f6d055b9..1cfb2053 100644 --- a/src/Tests/Microsoft.Agents.A365.Observability.Runtime.IntegrationTests/Agent365ExporterE2ETests.cs +++ b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.IntegrationTests/Agent365ExporterE2ETests.cs @@ -352,12 +352,26 @@ private async Task RunNestedScopes_AllExporterRequestsReceived(bool useAgentId) finishReasons: new[] { "stop" }, responseId: "response-nested"); + var callerDetails = new CallerDetails( + callerId: "caller-nested", + callerName: "Nested Caller", + callerUpn: "nested.caller@ztaittest12.onmicrosoft.com", + callerClientIP: IPAddress.Parse("192.168.1.1"), + tenantId: agentDetails.TenantId); + + var response = new Response("Nested output response content"); + // Act using (var agentScope = InvokeAgentScope.Start(invokeAgentDetails, tenantDetails, request)) { agentScope.RecordInputMessages(new[] { "Agent input" }); agentScope.RecordOutputMessages(new[] { "Agent output" }); + using (var inputScope = InputScope.Start(agentDetails, tenantDetails, request, callerDetails, conversationId: "conv-nested", sessionId: "session-nested")) + { + inputScope.RecordInputMessages(new[] { "Input scope message 1", "Input scope message 2" }); + } + using (var toolScope = ExecuteToolScope.Start(toolCallDetails, agentDetails, tenantDetails)) { toolScope.RecordResponse("Tool response"); @@ -372,6 +386,11 @@ private async Task RunNestedScopes_AllExporterRequestsReceived(bool useAgentId) inferenceScope.RecordFinishReasons(new[] { "stop" }); } } + + using (var outputScope = OutputScope.Start(agentDetails, tenantDetails, response, callerDetails, conversationId: "conv-nested", sessionId: "session-nested")) + { + outputScope.RecordOutputMessages(new[] { "Output scope message 1" }); + } } // Wait for up to 10 seconds for all spans to be exported @@ -401,7 +420,7 @@ private async Task RunNestedScopes_AllExporterRequestsReceived(bool useAgentId) allAgentTypes.Add(agentTypeTag); } } - allOperationNames.Should().Contain(new[] { "invoke_agent", "execute_tool", InferenceOperationType.Chat.ToString() }, "All three nested scopes should be exported, even if batched in fewer requests."); + allOperationNames.Should().Contain(new[] { "invoke_agent", "execute_tool", InferenceOperationType.Chat.ToString(), "input_messages", "output_messages" }, "All five nested scopes should be exported, even if batched in fewer requests."); allAgentTypes.Should().OnlyContain(t => t == agentType.ToString()); } diff --git a/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Tracing/Scopes/InputScopeTest.cs b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Tracing/Scopes/InputScopeTest.cs new file mode 100644 index 00000000..988ab856 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Tracing/Scopes/InputScopeTest.cs @@ -0,0 +1,300 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.Observability.Tests.Tracing.Scopes; + +using System; +using System.Diagnostics; +using System.Linq; +using System.Net; +using FluentAssertions; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts; +using static Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes.OpenTelemetryConstants; + +[TestClass] +public sealed class InputScopeTest : ActivityTest +{ + [TestMethod] + public void Start_SetsOperationName() + { + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start(Util.GetAgentDetails(), Util.GetTenantDetails()); + }); + + activity.ShouldHaveTag(GenAiOperationNameKey, InputScope.OperationName); + } + + [TestMethod] + public void Start_SetsActivityKindToClient() + { + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start(Util.GetAgentDetails(), Util.GetTenantDetails()); + }); + + activity.Kind.Should().Be(ActivityKind.Client); + } + + [TestMethod] + public void Start_SetsAgentDetails() + { + var agentDetails = new AgentDetails( + agentId: "agent-123", + agentName: "TestAgent", + agentDescription: "Test agent description", + agentAUID: "auid-456", + agentUPN: "agent@contoso.com", + agentBlueprintId: "blueprint-789", + agentType: AgentType.Foundry, + agentPlatformId: "platform-001"); + + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start(agentDetails, Util.GetTenantDetails()); + }); + + activity.ShouldHaveTag(GenAiAgentIdKey, "agent-123"); + activity.ShouldHaveTag(GenAiAgentNameKey, "TestAgent"); + activity.ShouldHaveTag(GenAiAgentDescriptionKey, "Test agent description"); + activity.ShouldHaveTag(GenAiAgentAUIDKey, "auid-456"); + activity.ShouldHaveTag(GenAiAgentUPNKey, "agent@contoso.com"); + activity.ShouldHaveTag(GenAiAgentBlueprintIdKey, "blueprint-789"); + activity.ShouldHaveTag(GenAiAgentTypeKey, AgentType.Foundry.ToString()); + activity.ShouldHaveTag(GenAiAgentPlatformIdKey, "platform-001"); + } + + [TestMethod] + public void Start_SetsTenantId() + { + var tenantId = Guid.NewGuid(); + var tenantDetails = new TenantDetails(tenantId); + + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start(Util.GetAgentDetails(), tenantDetails); + }); + + // Verify tenant.id tag is set by checking TagObjects (which includes non-string tags) + activity.TagObjects.Should().ContainKey(TenantIdKey, "Activity should have tag 'tenant.id'") + .WhoseValue.Should().Be(tenantId); + } + + [TestMethod] + public void Start_SetsSessionId() + { + const string sessionId = "session-123"; + + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + sessionId: sessionId); + }); + + activity.ShouldHaveTag(SessionIdKey, sessionId); + } + + [TestMethod] + public void Start_SetsSessionDescription() + { + const string sessionDescription = "Test session description"; + + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + sessionDescription: sessionDescription); + }); + + activity.ShouldHaveTag(SessionDescriptionKey, sessionDescription); + } + + [TestMethod] + public void Start_SetsConversationId() + { + const string conversationId = "conv-456"; + + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + conversationId: conversationId); + }); + + activity.ShouldHaveTag(GenAiConversationIdKey, conversationId); + } + + [TestMethod] + public void Start_SetsInputMessagesFromRequest() + { + const string requestContent = "This is the input message content"; + var request = new Request(requestContent); + + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + request: request); + }); + + activity.ShouldHaveTag(GenAiInputMessagesKey, requestContent); + } + + [TestMethod] + public void Start_SetsExecutionType() + { + var request = new Request("content", ExecutionType.HumanToAgent); + + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + request: request); + }); + + activity.ShouldHaveTag(GenAiExecutionTypeKey, ExecutionType.HumanToAgent.ToString()); + } + + [TestMethod] + public void Start_SetsCallerDetails() + { + var callerIp = IPAddress.Parse("192.168.1.100"); + var callerDetails = new CallerDetails( + callerId: "caller-001", + callerName: "Test Caller", + callerUpn: "test.caller@contoso.com", + callerClientIP: callerIp, + tenantId: "tenant-xyz"); + + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + callerDetails: callerDetails); + }); + + activity.ShouldHaveTag(GenAiCallerIdKey, "caller-001"); + activity.ShouldHaveTag(GenAiCallerNameKey, "Test Caller"); + activity.ShouldHaveTag(GenAiCallerUpnKey, "test.caller@contoso.com"); + activity.ShouldHaveTag(GenAiCallerClientIpKey, callerIp.ToString()); + activity.ShouldHaveTag(GenAiCallerTenantIdKey, "tenant-xyz"); + } + + [TestMethod] + public void Start_SetsChannelMetadataFromSourceMetadata() + { + var sourceMetadata = new SourceMetadata( + name: "TestChannel", + role: Role.Human, + description: "Test channel description"); + var request = new Request("content", sourceMetadata: sourceMetadata); + + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + request: request); + }); + + activity.ShouldHaveTag(GenAiChannelNameKey, "TestChannel"); + activity.ShouldHaveTag(GenAiChannelLinkKey, "Test channel description"); + } + + [TestMethod] + public void RecordInputMessages_SetsTag() + { + var messages = new[] { "Hello", "How are you?" }; + + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start(Util.GetAgentDetails(), Util.GetTenantDetails()); + scope.RecordInputMessages(messages); + }); + + activity.ShouldHaveTag(GenAiInputMessagesKey, string.Join(",", messages)); + } + + [TestMethod] + public void RecordError_SetsExpectedFields() + { + const string expected = "Test error"; + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start(Util.GetAgentDetails(), Util.GetTenantDetails()); + scope.RecordError(new Exception(expected)); + }); + + activity.ShouldBeError(expected); + } + + [TestMethod] + public void SetStartTime_SetsActivityStartTime() + { + var customStartTime = DateTimeOffset.UtcNow.AddMinutes(-5); + + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start(Util.GetAgentDetails(), Util.GetTenantDetails()); + scope.SetStartTime(customStartTime); + }); + + var startTime = new DateTimeOffset(activity.StartTimeUtc); + startTime.Should().BeCloseTo(customStartTime, TimeSpan.FromMilliseconds(100)); + } + + [TestMethod] + public void SetParentId_SetsActivityParentId() + { + var manualParentActivity = CreateActivity(); + var parentId = manualParentActivity.Id; + var parentSpanId = manualParentActivity.SpanId.ToString() ?? string.Empty; + + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + parentId: parentId); + }); + + activity.Should().NotBeNull(); + activity!.ParentSpanId.ToString().Should().Be(parentSpanId); + } + + [TestMethod] + public void ActivityName_UsesAgentName_WhenProvided() + { + var agentDetails = new AgentDetails( + agentId: "agent-123", + agentName: "MyTestAgent"); + + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start(agentDetails, Util.GetTenantDetails()); + }); + + activity.DisplayName.Should().Be("input_messages MyTestAgent"); + } + + [TestMethod] + public void ActivityName_UsesOperationName_WhenAgentNameIsNull() + { + var agentDetails = new AgentDetails(agentId: "agent-123"); + + var activity = ListenForActivity(() => + { + using var scope = InputScope.Start(agentDetails, Util.GetTenantDetails()); + }); + + activity.DisplayName.Should().Be(InputScope.OperationName); + } +} diff --git a/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Tracing/Scopes/OutputScopeTest.cs b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Tracing/Scopes/OutputScopeTest.cs new file mode 100644 index 00000000..dc773013 --- /dev/null +++ b/src/Tests/Microsoft.Agents.A365.Observability.Runtime.Tests/Tracing/Scopes/OutputScopeTest.cs @@ -0,0 +1,296 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.Agents.A365.Observability.Tests.Tracing.Scopes; + +using System; +using System.Diagnostics; +using System.Net; +using FluentAssertions; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes; +using Microsoft.Agents.A365.Observability.Runtime.Tracing.Contracts; +using static Microsoft.Agents.A365.Observability.Runtime.Tracing.Scopes.OpenTelemetryConstants; + +[TestClass] +public sealed class OutputScopeTest : ActivityTest +{ + [TestMethod] + public void Start_SetsOperationName() + { + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start(Util.GetAgentDetails(), Util.GetTenantDetails()); + }); + + activity.ShouldHaveTag(GenAiOperationNameKey, OutputScope.OperationName); + } + + [TestMethod] + public void Start_SetsActivityKindToClient() + { + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start(Util.GetAgentDetails(), Util.GetTenantDetails()); + }); + + activity.Kind.Should().Be(ActivityKind.Client); + } + + [TestMethod] + public void Start_SetsAgentDetails() + { + var agentDetails = new AgentDetails( + agentId: "agent-123", + agentName: "TestAgent", + agentDescription: "Test agent description", + agentAUID: "auid-456", + agentUPN: "agent@contoso.com", + agentBlueprintId: "blueprint-789", + agentType: AgentType.Foundry, + agentPlatformId: "platform-001"); + + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start(agentDetails, Util.GetTenantDetails()); + }); + + activity.ShouldHaveTag(GenAiAgentIdKey, "agent-123"); + activity.ShouldHaveTag(GenAiAgentNameKey, "TestAgent"); + activity.ShouldHaveTag(GenAiAgentDescriptionKey, "Test agent description"); + activity.ShouldHaveTag(GenAiAgentAUIDKey, "auid-456"); + activity.ShouldHaveTag(GenAiAgentUPNKey, "agent@contoso.com"); + activity.ShouldHaveTag(GenAiAgentBlueprintIdKey, "blueprint-789"); + activity.ShouldHaveTag(GenAiAgentTypeKey, AgentType.Foundry.ToString()); + activity.ShouldHaveTag(GenAiAgentPlatformIdKey, "platform-001"); + } + + [TestMethod] + public void Start_SetsTenantId() + { + var tenantId = Guid.NewGuid(); + var tenantDetails = new TenantDetails(tenantId); + + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start(Util.GetAgentDetails(), tenantDetails); + }); + + // Verify tenant.id tag is set by checking TagObjects (which includes non-string tags) + activity.TagObjects.Should().ContainKey(TenantIdKey, "Activity should have tag 'tenant.id'") + .WhoseValue.Should().Be(tenantId); + } + + [TestMethod] + public void Start_SetsSessionId() + { + const string sessionId = "session-123"; + + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + sessionId: sessionId); + }); + + activity.ShouldHaveTag(SessionIdKey, sessionId); + } + + [TestMethod] + public void Start_SetsSessionDescription() + { + const string sessionDescription = "Test session description"; + + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + sessionDescription: sessionDescription); + }); + + activity.ShouldHaveTag(SessionDescriptionKey, sessionDescription); + } + + [TestMethod] + public void Start_SetsConversationId() + { + const string conversationId = "conv-456"; + + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + conversationId: conversationId); + }); + + activity.ShouldHaveTag(GenAiConversationIdKey, conversationId); + } + + [TestMethod] + public void Start_SetsExecutionType() + { + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + executionType: ExecutionType.HumanToAgent); + }); + + activity.ShouldHaveTag(GenAiExecutionTypeKey, ExecutionType.HumanToAgent.ToString()); + } + + [TestMethod] + public void Start_SetsCallerDetails() + { + var callerIp = IPAddress.Parse("192.168.1.100"); + var callerDetails = new CallerDetails( + callerId: "caller-001", + callerName: "Test Caller", + callerUpn: "test.caller@contoso.com", + callerClientIP: callerIp, + tenantId: "tenant-xyz"); + + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + callerDetails: callerDetails); + }); + + activity.ShouldHaveTag(GenAiCallerIdKey, "caller-001"); + activity.ShouldHaveTag(GenAiCallerNameKey, "Test Caller"); + activity.ShouldHaveTag(GenAiCallerUpnKey, "test.caller@contoso.com"); + activity.ShouldHaveTag(GenAiCallerClientIpKey, callerIp.ToString()); + activity.ShouldHaveTag(GenAiCallerTenantIdKey, "tenant-xyz"); + } + + [TestMethod] + public void Start_SetsChannelMetadataFromSourceMetadata() + { + var sourceMetadata = new SourceMetadata( + name: "TestChannel", + role: Role.Human, + description: "Test channel description"); + + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + sourceMetadata: sourceMetadata); + }); + + activity.ShouldHaveTag(GenAiChannelNameKey, "TestChannel"); + activity.ShouldHaveTag(GenAiChannelLinkKey, "Test channel description"); + } + + [TestMethod] + public void Start_SetsOutputMessagesFromResponse() + { + const string responseContent = "This is the output message content"; + var response = new Response(responseContent); + + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + response: response); + }); + + activity.ShouldHaveTag(GenAiOutputMessagesKey, responseContent); + } + + [TestMethod] + public void RecordOutputMessages_SetsTag() + { + var messages = new[] { "Hello!", "Here is your response." }; + + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start(Util.GetAgentDetails(), Util.GetTenantDetails()); + scope.RecordOutputMessages(messages); + }); + + activity.ShouldHaveTag(GenAiOutputMessagesKey, string.Join(",", messages)); + } + + [TestMethod] + public void RecordError_SetsExpectedFields() + { + const string expected = "Test error"; + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start(Util.GetAgentDetails(), Util.GetTenantDetails()); + scope.RecordError(new Exception(expected)); + }); + + activity.ShouldBeError(expected); + } + + [TestMethod] + public void SetStartTime_SetsActivityStartTime() + { + var customStartTime = DateTimeOffset.UtcNow.AddMinutes(-5); + + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start(Util.GetAgentDetails(), Util.GetTenantDetails()); + scope.SetStartTime(customStartTime); + }); + + var startTime = new DateTimeOffset(activity.StartTimeUtc); + startTime.Should().BeCloseTo(customStartTime, TimeSpan.FromMilliseconds(100)); + } + + [TestMethod] + public void SetParentId_SetsActivityParentId() + { + var manualParentActivity = CreateActivity(); + var parentId = manualParentActivity.Id; + var parentSpanId = manualParentActivity.SpanId.ToString() ?? string.Empty; + + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start( + Util.GetAgentDetails(), + Util.GetTenantDetails(), + parentId: parentId); + }); + + activity.Should().NotBeNull(); + activity!.ParentSpanId.ToString().Should().Be(parentSpanId); + } + + [TestMethod] + public void ActivityName_UsesAgentName_WhenProvided() + { + var agentDetails = new AgentDetails( + agentId: "agent-123", + agentName: "MyTestAgent"); + + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start(agentDetails, Util.GetTenantDetails()); + }); + + activity.DisplayName.Should().Be("output_messages MyTestAgent"); + } + + [TestMethod] + public void ActivityName_UsesOperationName_WhenAgentNameIsNull() + { + var agentDetails = new AgentDetails(agentId: "agent-123"); + + var activity = ListenForActivity(() => + { + using var scope = OutputScope.Start(agentDetails, Util.GetTenantDetails()); + }); + + activity.DisplayName.Should().Be(OutputScope.OperationName); + } +}