diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/execute_tool_scope.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/execute_tool_scope.py index 6a05728d..656f7ef1 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/execute_tool_scope.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/execute_tool_scope.py @@ -5,6 +5,8 @@ from .constants import ( EXECUTE_TOOL_OPERATION_NAME, GEN_AI_EVENT_CONTENT, + GEN_AI_EXECUTION_SOURCE_DESCRIPTION_KEY, + GEN_AI_EXECUTION_SOURCE_NAME_KEY, GEN_AI_TOOL_ARGS_KEY, GEN_AI_TOOL_CALL_ID_KEY, GEN_AI_TOOL_DESCRIPTION_KEY, @@ -14,6 +16,7 @@ SERVER_PORT_KEY, ) from .opentelemetry_scope import OpenTelemetryScope +from .request import Request from .tenant_details import TenantDetails from .tool_call_details import ToolCallDetails @@ -26,6 +29,7 @@ def start( details: ToolCallDetails, agent_details: AgentDetails, tenant_details: TenantDetails, + request: Request | None = None, ) -> "ExecuteToolScope": """Creates and starts a new scope for tool execution tracing. @@ -33,17 +37,19 @@ def start( details: The details of the tool call agent_details: The details of the agent making the call tenant_details: The details of the tenant + request: Optional request details for additional context Returns: A new ExecuteToolScope instance """ - return ExecuteToolScope(details, agent_details, tenant_details) + return ExecuteToolScope(details, agent_details, tenant_details, request) def __init__( self, details: ToolCallDetails, agent_details: AgentDetails, tenant_details: TenantDetails, + request: Request | None = None, ): """Initialize the tool execution scope. @@ -51,6 +57,7 @@ def __init__( details: The details of the tool call agent_details: The details of the agent making the call tenant_details: The details of the tenant + request: Optional request details for additional context """ super().__init__( kind="Internal", @@ -79,6 +86,13 @@ def __init__( if endpoint.port and endpoint.port != 443: self.set_tag_maybe(SERVER_PORT_KEY, endpoint.port) + # Set request metadata if provided + if request and request.source_metadata: + self.set_tag_maybe(GEN_AI_EXECUTION_SOURCE_NAME_KEY, request.source_metadata.name) + self.set_tag_maybe( + GEN_AI_EXECUTION_SOURCE_DESCRIPTION_KEY, request.source_metadata.description + ) + def record_response(self, response: str) -> None: """Records response information for telemetry tracking. diff --git a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_scope.py b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_scope.py index 7274a1b6..f3e11bdb 100644 --- a/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_scope.py +++ b/libraries/microsoft-agents-a365-observability-core/microsoft_agents_a365/observability/core/inference_scope.py @@ -5,6 +5,8 @@ from .agent_details import AgentDetails from .constants import ( + GEN_AI_EXECUTION_SOURCE_DESCRIPTION_KEY, + GEN_AI_EXECUTION_SOURCE_NAME_KEY, GEN_AI_INPUT_MESSAGES_KEY, GEN_AI_OPERATION_NAME_KEY, GEN_AI_OUTPUT_MESSAGES_KEY, @@ -90,6 +92,13 @@ def __init__( ) self.set_tag_maybe(GEN_AI_RESPONSE_ID_KEY, details.responseId) + # Set request metadata if provided + if request and request.source_metadata: + self.set_tag_maybe(GEN_AI_EXECUTION_SOURCE_NAME_KEY, request.source_metadata.name) + self.set_tag_maybe( + GEN_AI_EXECUTION_SOURCE_DESCRIPTION_KEY, request.source_metadata.description + ) + def record_input_messages(self, messages: List[str]) -> None: """Records the input messages for telemetry tracking. diff --git a/tests/observability/core/test_execute_tool_scope.py b/tests/observability/core/test_execute_tool_scope.py index 26f9900e..76e9c2b4 100644 --- a/tests/observability/core/test_execute_tool_scope.py +++ b/tests/observability/core/test_execute_tool_scope.py @@ -1,15 +1,29 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import os +from pathlib import Path +import sys import unittest +import pytest from microsoft_agents_a365.observability.core import ( AgentDetails, + ExecutionType, ExecuteToolScope, + Request, + SourceMetadata, TenantDetails, ToolCallDetails, configure, + get_tracer_provider, ) +from microsoft_agents_a365.observability.core.constants import ( + GEN_AI_EXECUTION_SOURCE_DESCRIPTION_KEY, + GEN_AI_EXECUTION_SOURCE_NAME_KEY, +) +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter class TestExecuteToolScope(unittest.TestCase): @@ -19,6 +33,8 @@ class TestExecuteToolScope(unittest.TestCase): def setUpClass(cls): """Set up test environment once for all tests.""" # Configure Microsoft Agent 365 for testing + os.environ["ENABLE_A365_OBSERVABILITY"] = "true" + configure( service_name="test-execute-tool-service", service_namespace="test-namespace", @@ -37,6 +53,19 @@ def setUpClass(cls): description="Get current weather information for a location", ) + def setUp(self): + super().setUp() + + # Set up tracer to capture spans + self.span_exporter = InMemorySpanExporter() + tracer_provider = get_tracer_provider() + tracer_provider.add_span_processor(SimpleSpanProcessor(self.span_exporter)) + + def tearDown(self): + super().tearDown() + + self.span_exporter.clear() + def test_record_response_method_exists(self): """Test that record_response method exists on ExecuteToolScope.""" scope = ExecuteToolScope.start(self.tool_details, self.agent_details, self.tenant_details) @@ -47,6 +76,49 @@ def test_record_response_method_exists(self): self.assertTrue(callable(scope.record_response)) scope.dispose() + def test_request_metadata_set_on_span(self): + """Test that request source metadata is set on span attributes.""" + request = Request( + content="Execute tool with request metadata", + execution_type=ExecutionType.AGENT_TO_AGENT, + session_id="session-xyz", + source_metadata=SourceMetadata(name="Channel 1", description="Link to channel"), + ) + + scope = ExecuteToolScope.start( + self.tool_details, self.agent_details, self.tenant_details, request + ) + + if scope is not None: + scope.dispose() + + finished_spans = self.span_exporter.get_finished_spans() + self.assertTrue(finished_spans, "Expected at least one span to be created") + + span = finished_spans[-1] + span_attributes = getattr(span, "attributes", {}) or {} + + self.assertIn( + GEN_AI_EXECUTION_SOURCE_NAME_KEY, + span_attributes, + "Expected source name to be set on span", + ) + self.assertEqual( + span_attributes[GEN_AI_EXECUTION_SOURCE_NAME_KEY], + request.source_metadata.name, + ) + + self.assertIn( + GEN_AI_EXECUTION_SOURCE_DESCRIPTION_KEY, + span_attributes, + "Expected source description to be set on span", + ) + self.assertEqual( + span_attributes[GEN_AI_EXECUTION_SOURCE_DESCRIPTION_KEY], + request.source_metadata.description, + ) + if __name__ == "__main__": - unittest.main(verbosity=2) + # Run pytest only on the current file + sys.exit(pytest.main([str(Path(__file__))] + sys.argv[1:])) diff --git a/tests/observability/core/test_inference_scope.py b/tests/observability/core/test_inference_scope.py index 6743287f..c7361bd0 100644 --- a/tests/observability/core/test_inference_scope.py +++ b/tests/observability/core/test_inference_scope.py @@ -1,7 +1,11 @@ -# Copyright (c) Microsoft. All rights reserved. - +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +import os +from pathlib import Path +import sys import unittest +import pytest from microsoft_agents_a365.observability.core import ( ExecutionType, @@ -9,10 +13,18 @@ InferenceOperationType, InferenceScope, Request, + SourceMetadata, TenantDetails, configure, + get_tracer_provider, ) from microsoft_agents_a365.observability.core.agent_details import AgentDetails +from microsoft_agents_a365.observability.core.constants import ( + GEN_AI_EXECUTION_SOURCE_DESCRIPTION_KEY, + GEN_AI_EXECUTION_SOURCE_NAME_KEY, +) +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter class TestInferenceScope(unittest.TestCase): @@ -22,6 +34,8 @@ class TestInferenceScope(unittest.TestCase): def setUpClass(cls): """Set up test environment once for all tests.""" # Configure Microsoft Agent 365 for testing + os.environ["ENABLE_A365_OBSERVABILITY"] = "true" + configure( service_name="test-inference-service", service_namespace="test-namespace", @@ -30,6 +44,20 @@ def setUpClass(cls): cls.agent_details = AgentDetails(agent_id="test-inference-agent") cls.tenant_details = TenantDetails(tenant_id="12345678-1234-5678-1234-567812345678") + def setUp(self): + super().setUp() + + # Set up tracer to capture spans + self.span_exporter = InMemorySpanExporter() + tracer_provider = get_tracer_provider() + tracer_provider.add_span_processor(SimpleSpanProcessor(self.span_exporter)) + # trace.set_tracer_provider(tracer_provider) + + def tearDown(self): + super().tearDown() + + self.span_exporter.clear() + def test_inference_operation_type_enum(self): """Test InferenceOperationType enum values.""" # Test enum values exist @@ -40,7 +68,9 @@ def test_inference_operation_type_enum(self): def test_inference_call_details_creation(self): """Test InferenceCallDetails creation with required fields.""" details = InferenceCallDetails( - operationName=InferenceOperationType.CHAT, model="gpt-4", providerName="openai" + operationName=InferenceOperationType.CHAT, + model="gpt-4", + providerName="openai", ) self.assertEqual(details.operationName, InferenceOperationType.CHAT) @@ -74,7 +104,9 @@ def test_inference_call_details_with_all_fields(self): def test_inference_scope_start_method(self): """Test InferenceScope.start() static method.""" details = InferenceCallDetails( - operationName=InferenceOperationType.CHAT, model="gpt-4", providerName="openai" + operationName=InferenceOperationType.CHAT, + model="gpt-4", + providerName="openai", ) scope = InferenceScope.start(details, self.agent_details, self.tenant_details) @@ -90,7 +122,9 @@ def test_inference_scope_start_method(self): def test_inference_scope_with_request(self): """Test InferenceScope with request parameter.""" details = InferenceCallDetails( - operationName=InferenceOperationType.CHAT, model="gpt-4", providerName="openai" + operationName=InferenceOperationType.CHAT, + model="gpt-4", + providerName="openai", ) request = Request( @@ -105,6 +139,52 @@ def test_inference_scope_with_request(self): if scope is not None: self.assertIsInstance(scope, InferenceScope) + def test_request_metadata_set_on_span(self): + """Test that request source metadata is set on span attributes.""" + details = InferenceCallDetails( + operationName=InferenceOperationType.CHAT, + model="gpt-4", + providerName="openai", + ) + + request = Request( + content="Inference request with source metadata", + execution_type=ExecutionType.AGENT_TO_AGENT, + session_id="session-meta", + source_metadata=SourceMetadata(name="Channel 1", description="Link to channel"), + ) + + scope = InferenceScope.start(details, self.agent_details, self.tenant_details, request) + + if scope is not None: + scope.dispose() + + finished_spans = self.span_exporter.get_finished_spans() + self.assertTrue(finished_spans, "Expected at least one span to be created") + + span = finished_spans[-1] + span_attributes = getattr(span, "attributes", {}) or {} + + self.assertIn( + GEN_AI_EXECUTION_SOURCE_NAME_KEY, + span_attributes, + "Expected source name to be set on span", + ) + self.assertEqual( + span_attributes[GEN_AI_EXECUTION_SOURCE_NAME_KEY], + request.source_metadata.name, + ) + + self.assertIn( + GEN_AI_EXECUTION_SOURCE_DESCRIPTION_KEY, + span_attributes, + "Expected source description to be set on span", + ) + self.assertEqual( + span_attributes[GEN_AI_EXECUTION_SOURCE_DESCRIPTION_KEY], + request.source_metadata.description, + ) + def test_inference_scope_context_manager(self): """Test InferenceScope as context manager.""" details = InferenceCallDetails( @@ -135,7 +215,9 @@ def test_inference_scope_context_manager(self): def test_inference_scope_dispose(self): """Test InferenceScope dispose method.""" details = InferenceCallDetails( - operationName=InferenceOperationType.CHAT, model="gpt-4", providerName="openai" + operationName=InferenceOperationType.CHAT, + model="gpt-4", + providerName="openai", ) scope = InferenceScope.start(details, self.agent_details, self.tenant_details) @@ -149,7 +231,9 @@ def test_inference_scope_dispose(self): def test_record_input_messages(self): """Test record_input_messages method.""" details = InferenceCallDetails( - operationName=InferenceOperationType.CHAT, model="gpt-4", providerName="openai" + operationName=InferenceOperationType.CHAT, + model="gpt-4", + providerName="openai", ) scope = InferenceScope.start(details, self.agent_details, self.tenant_details) @@ -164,7 +248,9 @@ def test_record_input_messages(self): def test_record_output_messages(self): """Test record_output_messages method.""" details = InferenceCallDetails( - operationName=InferenceOperationType.CHAT, model="gpt-4", providerName="openai" + operationName=InferenceOperationType.CHAT, + model="gpt-4", + providerName="openai", ) scope = InferenceScope.start(details, self.agent_details, self.tenant_details) @@ -179,7 +265,9 @@ def test_record_output_messages(self): def test_record_input_tokens(self): """Test record_input_tokens method.""" details = InferenceCallDetails( - operationName=InferenceOperationType.CHAT, model="gpt-4", providerName="openai" + operationName=InferenceOperationType.CHAT, + model="gpt-4", + providerName="openai", ) scope = InferenceScope.start(details, self.agent_details, self.tenant_details) @@ -193,7 +281,9 @@ def test_record_input_tokens(self): def test_record_output_tokens(self): """Test record_output_tokens method.""" details = InferenceCallDetails( - operationName=InferenceOperationType.CHAT, model="gpt-4", providerName="openai" + operationName=InferenceOperationType.CHAT, + model="gpt-4", + providerName="openai", ) scope = InferenceScope.start(details, self.agent_details, self.tenant_details) @@ -207,7 +297,9 @@ def test_record_output_tokens(self): def test_record_finish_reasons(self): """Test record_finish_reasons method.""" details = InferenceCallDetails( - operationName=InferenceOperationType.CHAT, model="gpt-4", providerName="openai" + operationName=InferenceOperationType.CHAT, + model="gpt-4", + providerName="openai", ) scope = InferenceScope.start(details, self.agent_details, self.tenant_details) @@ -222,7 +314,9 @@ def test_record_finish_reasons(self): def test_record_thought_process(self): """Test record_thought_process method.""" details = InferenceCallDetails( - operationName=InferenceOperationType.CHAT, model="gpt-4", providerName="openai" + operationName=InferenceOperationType.CHAT, + model="gpt-4", + providerName="openai", ) scope = InferenceScope.start(details, self.agent_details, self.tenant_details) @@ -236,5 +330,5 @@ def test_record_thought_process(self): if __name__ == "__main__": - # Run the tests - unittest.main(verbosity=2) + # Run pytest only on the current file + sys.exit(pytest.main([str(Path(__file__))] + sys.argv[1:])) diff --git a/tests/observability/core/test_invoke_agent_scope.py b/tests/observability/core/test_invoke_agent_scope.py index 924d469d..eec2b30e 100644 --- a/tests/observability/core/test_invoke_agent_scope.py +++ b/tests/observability/core/test_invoke_agent_scope.py @@ -1,7 +1,11 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import os +from pathlib import Path +import sys import unittest +import pytest from urllib.parse import urlparse from microsoft_agents_a365.observability.core import ( @@ -13,6 +17,7 @@ SourceMetadata, TenantDetails, configure, + get_tracer_provider, ) from microsoft_agents_a365.observability.core.constants import ( GEN_AI_CALLER_AGENT_USER_CLIENT_IP, @@ -22,8 +27,6 @@ GEN_AI_INPUT_MESSAGES_KEY, ) from microsoft_agents_a365.observability.core.models.caller_details import CallerDetails -from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter @@ -35,6 +38,8 @@ class TestInvokeAgentScope(unittest.TestCase): def setUpClass(cls): """Set up test environment once for all tests.""" # Configure Microsoft Agent 365 for testing + os.environ["ENABLE_A365_OBSERVABILITY"] = "true" + configure( service_name="test-invoke-agent-service", service_namespace="test-namespace", @@ -89,6 +94,19 @@ def setUpClass(cls): agent_client_ip="192.168.1.100", ) + def setUp(self): + super().setUp() + + # Set up tracer to capture spans + self.span_exporter = InMemorySpanExporter() + tracer_provider = get_tracer_provider() + tracer_provider.add_span_processor(SimpleSpanProcessor(self.span_exporter)) + + def tearDown(self): + super().tearDown() + + self.span_exporter.clear() + def test_record_response_method_exists(self): """Test that record_response method exists on InvokeAgentScope.""" scope = InvokeAgentScope.start(self.invoke_details, self.tenant_details) @@ -121,12 +139,6 @@ def test_record_output_messages_method_exists(self): def test_request_attributes_set_on_span(self): """Test that request parameters from mock data are available on span attributes.""" - # Set up tracer to capture spans - span_exporter = InMemorySpanExporter() - tracer_provider = TracerProvider() - tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) - trace.set_tracer_provider(tracer_provider) - # Create scope with request scope = InvokeAgentScope.start( invoke_agent_details=self.invoke_details, @@ -138,50 +150,49 @@ def test_request_attributes_set_on_span(self): scope.dispose() # Check if mock data parameters are available in span attributes - finished_spans = span_exporter.get_finished_spans() - - if finished_spans: - # Get attributes from the span - span = finished_spans[-1] - span_attributes = getattr(span, "attributes", {}) or {} - - # Verify mock data request parameters are in span attributes - # Check source channel name from mock data - if GEN_AI_EXECUTION_SOURCE_NAME_KEY in span_attributes: - self.assertEqual( - span_attributes[GEN_AI_EXECUTION_SOURCE_NAME_KEY], - self.source_metadata.name, # From cls.source_metadata.name - ) - - # Check source channel description from mock data - if GEN_AI_EXECUTION_SOURCE_DESCRIPTION_KEY in span_attributes: - self.assertEqual( - span_attributes[GEN_AI_EXECUTION_SOURCE_DESCRIPTION_KEY], - self.source_metadata.description, # From cls.source_metadata.description - ) - - # Check execution type from mock data - if GEN_AI_EXECUTION_TYPE_KEY in span_attributes: - self.assertEqual( - span_attributes[GEN_AI_EXECUTION_TYPE_KEY], - self.test_request.execution_type.value, # From cls.test_request.execution_type - ) - - # Check input messages contain request content from mock data - if GEN_AI_INPUT_MESSAGES_KEY in span_attributes: - input_messages = span_attributes[GEN_AI_INPUT_MESSAGES_KEY] - self.assertIn( - self.test_request.content, # From cls.test_request.content - input_messages, - ) + finished_spans = self.span_exporter.get_finished_spans() + self.assertTrue(finished_spans, "Expected at least one span to be created") + + # Get attributes from the span + span = finished_spans[-1] + span_attributes = getattr(span, "attributes", {}) or {} + + # Verify mock data request parameters are in span attributes + # Check source channel name from mock data + if GEN_AI_EXECUTION_SOURCE_NAME_KEY in span_attributes: + self.assertEqual( + span_attributes[GEN_AI_EXECUTION_SOURCE_NAME_KEY], + self.source_metadata.name, # From cls.source_metadata.name + ) + + # Check source channel description from mock data + if GEN_AI_EXECUTION_SOURCE_DESCRIPTION_KEY in span_attributes: + self.assertEqual( + span_attributes[GEN_AI_EXECUTION_SOURCE_DESCRIPTION_KEY], + self.source_metadata.description, # From cls.source_metadata.description + ) + + # Check execution type from mock data + if GEN_AI_EXECUTION_TYPE_KEY in span_attributes: + self.assertEqual( + span_attributes[GEN_AI_EXECUTION_TYPE_KEY], + self.test_request.execution_type.value, # From cls.test_request.execution_type + ) + + # Check input messages contain request content from mock data + if GEN_AI_INPUT_MESSAGES_KEY in span_attributes: + input_messages = span_attributes[GEN_AI_INPUT_MESSAGES_KEY] + self.assertIn( + self.test_request.content, # From cls.test_request.content + input_messages, + ) def test_caller_agent_client_ip_in_scope(self): """Test that caller agent client IP is properly handled when creating InvokeAgentScope.""" # Set up tracer to capture spans span_exporter = InMemorySpanExporter() - tracer_provider = TracerProvider() + tracer_provider = get_tracer_provider() tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) - trace.set_tracer_provider(tracer_provider) # Create scope with caller agent details that include client IP scope = InvokeAgentScope.start( @@ -209,4 +220,5 @@ def test_caller_agent_client_ip_in_scope(self): if __name__ == "__main__": - unittest.main(verbosity=2) + # Run pytest only on the current file + sys.exit(pytest.main([str(Path(__file__))] + sys.argv[1:]))