From dc7114fd552966be5563b85b1a86f2e00edc6a27 Mon Sep 17 00:00:00 2001 From: Howie Leung Date: Mon, 8 Dec 2025 11:57:53 -0800 Subject: [PATCH 1/8] clean up --- .../tests/samples/test_samples.py | 179 +++++++++++++++--- 1 file changed, 149 insertions(+), 30 deletions(-) diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py index a60773a0b691..b9d4991bde25 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py @@ -1,8 +1,7 @@ # pylint: disable=line-too-long,useless-suppression # ------------------------------------ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ +# Licensed------------------------- import csv, os, pytest, re, inspect, sys import importlib.util import unittest.mock as mock @@ -12,23 +11,141 @@ from test_base import servicePreparer, patched_open_crlf_to_lf from pytest import MonkeyPatch from azure.ai.projects import AIProjectClient +from azure.ai.projects.models import PromptAgentDefinition +from azure.identity import DefaultAzureCredential +from functools import wraps +from typing import Callable, Optional class SampleExecutor: - """Helper class for executing sample files with proper environment setup and credential mocking.""" + """Decorator for executing sample files with proper environment setup and credential mocking.""" + + def __init__(self, env_var_mapping_fn: Callable): + """ + Initialize the SampleExecutor decorator. + + Args: + env_var_mapping_fn: Function that returns the environment variable mapping + """ + self.env_var_mapping_fn = env_var_mapping_fn + self.agent = None + self.project_client = None + self.credential = None + + def __enter__(self): + """Context manager entry - creates agent for all tests.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager exit - deletes agent after all tests.""" + if self.agent and self.project_client: + try: + self.project_client.agents.delete_version( + agent_name=self.agent.name, + agent_version=self.agent.version + ) + print(f"Agent {self.agent.name} deleted") + except Exception as e: + print(f"Error deleting agent: {e}") + + if self.project_client: + self.project_client.close() + + + return False + + def setup_agent(self, test_instance: "AzureRecordedTestCase", endpoint: str, is_async: bool): + """ + Setup agent using context managers. + + Args: + test_instance: The test instance to get credentials from + endpoint: The Azure AI project endpoint + """ + # Get credential from test infrastructure + credential = test_instance.get_credential(AIProjectClient, is_async=is_async) + self.credential = credential + + # Create project client + self.project_client = AIProjectClient(endpoint=endpoint, credential=credential) + + # Create agent + self.agent = self.project_client.agents.create_version( + agent_name="TestToolsAgent", + definition=PromptAgentDefinition( + model="gpt-4o", + instructions="You are a helpful assistant for testing purposes.", + ), + ) + print(f"Agent {self.agent.name} (version {self.agent.version}) created") - def __init__( - self, test_instance: "AzureRecordedTestCase", sample_path: str, env_var_mapping: dict[str, str], **kwargs - ): + def __call__(self, fn): + """ + Wrap the test function to provide executor instance. + + Args: + fn: The test function to wrap + + Returns: + Wrapped function with executor injection + """ + if inspect.iscoroutinefunction(fn): + @wraps(fn) + async def _async_wrapper(test_instance, sample_path: str, **kwargs): + # Setup agent on first call + if self.agent is None: + endpoint = kwargs.get("azure_ai_projects_tests_project_endpoint", "") + self.setup_agent(test_instance, endpoint, True) + + env_var_mapping = self.env_var_mapping_fn(test_instance) + executor = _SampleExecutorInstance( + test_instance, + env_var_mapping, + self.agent, + **kwargs + ) + await fn(test_instance, sample_path, executor=executor, **kwargs) + + return _async_wrapper + else: + @wraps(fn) + def _sync_wrapper(test_instance, sample_path: str, **kwargs): + # Setup agent on first call + if self.agent is None: + endpoint = kwargs.get("azure_ai_projects_tests_project_endpoint", "") + self.setup_agent(test_instance, endpoint, False) + + env_var_mapping = self.env_var_mapping_fn(test_instance) + executor = _SampleExecutorInstance( + test_instance, + env_var_mapping, + self.agent, + **kwargs + ) + fn(test_instance, sample_path, executor=executor, **kwargs) + + return _sync_wrapper + + +class _SampleExecutorInstance: + """Internal class for executing sample files with proper environment setup and credential mocking.""" + + def __init__(self, test_instance: "AzureRecordedTestCase", env_var_mapping: dict[str, str], agent, **kwargs): self.test_instance = test_instance - self.sample_path = sample_path + self.env_var_mapping = env_var_mapping + self.agent = agent + self.kwargs = kwargs self.print_calls: list[str] = [] self._original_print = print + def _prepare_execution(self, sample_path: str): + """Prepare for sample execution by setting up environment and module.""" + self.sample_path = sample_path + # Prepare environment variables self.env_vars = {} - for sample_var, test_var in env_var_mapping.items(): - value = kwargs.pop(test_var, None) + for sample_var, test_var in self.env_var_mapping.items(): + value = self.kwargs.get(test_var, None) if value is not None: self.env_vars[sample_var] = value self.env_vars["AZURE_AI_MODEL_DEPLOYMENT_NAME"] = "gpt-4o" @@ -52,9 +169,10 @@ def _capture_print(self, *args, **kwargs): self.print_calls.append(" ".join(str(arg) for arg in args)) self._original_print(*args, **kwargs) - def execute(self): + def execute(self, sample_path: str): """Execute a synchronous sample with proper mocking and environment setup.""" - + self._prepare_execution(sample_path) + with ( MonkeyPatch.context() as mp, mock.patch("builtins.print", side_effect=self._capture_print), @@ -74,8 +192,10 @@ def execute(self): self._validate_output() - async def execute_async(self): + async def execute_async(self, sample_path: str): """Execute an asynchronous sample with proper mocking and environment setup.""" + self._prepare_execution(sample_path) + with ( MonkeyPatch.context() as mp, mock.patch("builtins.print", side_effect=self._capture_print), @@ -205,6 +325,17 @@ def _get_tools_sample_paths_async(): return samples +def _get_tools_sample_environment_variables_map(self) -> dict[str, str]: + return { + "AZURE_AI_PROJECT_ENDPOINT": "azure_ai_projects_tests_project_endpoint", + "AI_SEARCH_PROJECT_CONNECTION_ID": "azure_ai_projects_tests_ai_search_project_connection_id", + "AI_SEARCH_INDEX_NAME": "azure_ai_projects_tests_ai_search_index_name", + "AI_SEARCH_USER_INPUT": "azure_ai_projects_tests_ai_search_user_input", + "SHAREPOINT_USER_INPUT": "azure_ai_projects_tests_sharepoint_user_input", + "SHAREPOINT_PROJECT_CONNECTION_ID": "azure_ai_projects_tests_sharepoint_project_connection_id", + } + + class TestSamples(AzureRecordedTestCase): _samples_folder_path: str _results: dict[str, tuple[bool, str]] @@ -506,27 +637,15 @@ async def test_samples_async( @servicePreparer() @pytest.mark.parametrize("sample_path", _get_tools_sample_paths()) @SamplePathPasser() + @SampleExecutor(_get_tools_sample_environment_variables_map) @recorded_by_proxy(RecordedTransport.AZURE_CORE, RecordedTransport.HTTPX) - def test_tools_samples(self, sample_path: str, **kwargs) -> None: - env_var_mapping = self._get_sample_environment_variables_map() - executor = SampleExecutor(self, sample_path, env_var_mapping, **kwargs) - executor.execute() + def test_tools_samples(self, sample_path: str, executor: _SampleExecutorInstance, **kwargs) -> None: + executor.execute(sample_path) @servicePreparer() @pytest.mark.parametrize("sample_path", _get_tools_sample_paths_async()) @SamplePathPasser() + @SampleExecutor(_get_tools_sample_environment_variables_map) @recorded_by_proxy_async(RecordedTransport.AZURE_CORE, RecordedTransport.HTTPX) - async def test_tools_samples_async(self, sample_path: str, **kwargs) -> None: - env_var_mapping = self._get_sample_environment_variables_map() - executor = SampleExecutor(self, sample_path, env_var_mapping, **kwargs) - await executor.execute_async() - - def _get_sample_environment_variables_map(self) -> dict[str, str]: - return { - "AZURE_AI_PROJECT_ENDPOINT": "azure_ai_projects_tests_project_endpoint", - "AI_SEARCH_PROJECT_CONNECTION_ID": "azure_ai_projects_tests_ai_search_project_connection_id", - "AI_SEARCH_INDEX_NAME": "azure_ai_projects_tests_ai_search_index_name", - "AI_SEARCH_USER_INPUT": "azure_ai_projects_tests_ai_search_user_input", - "SHAREPOINT_USER_INPUT": "azure_ai_projects_tests_sharepoint_user_input", - "SHAREPOINT_PROJECT_CONNECTION_ID": "azure_ai_projects_tests_sharepoint_project_connection_id", - } + async def test_tools_samples_async(self, sample_path: str, executor: _SampleExecutorInstance, **kwargs) -> None: + await executor.execute_async(sample_path) \ No newline at end of file From 0d226245ea0dea8bf30698f36068ca5c134483ee Mon Sep 17 00:00:00 2001 From: Howie Leung Date: Tue, 9 Dec 2025 16:58:48 -0800 Subject: [PATCH 2/8] update --- .../tests/samples/test_samples.py | 165 ++++++++---------- 1 file changed, 71 insertions(+), 94 deletions(-) diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py index b9d4991bde25..5cf2d6bc9213 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py @@ -2,7 +2,7 @@ # ------------------------------------ # Copyright (c) Microsoft Corporation. # Licensed------------------------- -import csv, os, pytest, re, inspect, sys +import csv, os, pytest, re, inspect, sys, json import importlib.util import unittest.mock as mock from azure.core.exceptions import HttpResponseError @@ -11,11 +11,10 @@ from test_base import servicePreparer, patched_open_crlf_to_lf from pytest import MonkeyPatch from azure.ai.projects import AIProjectClient -from azure.ai.projects.models import PromptAgentDefinition from azure.identity import DefaultAzureCredential from functools import wraps -from typing import Callable, Optional - +from typing import Callable, Optional, Literal +from pydantic import BaseModel class SampleExecutor: """Decorator for executing sample files with proper environment setup and credential mocking.""" @@ -54,31 +53,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): return False - def setup_agent(self, test_instance: "AzureRecordedTestCase", endpoint: str, is_async: bool): - """ - Setup agent using context managers. - - Args: - test_instance: The test instance to get credentials from - endpoint: The Azure AI project endpoint - """ - # Get credential from test infrastructure - credential = test_instance.get_credential(AIProjectClient, is_async=is_async) - self.credential = credential - - # Create project client - self.project_client = AIProjectClient(endpoint=endpoint, credential=credential) - - # Create agent - self.agent = self.project_client.agents.create_version( - agent_name="TestToolsAgent", - definition=PromptAgentDefinition( - model="gpt-4o", - instructions="You are a helpful assistant for testing purposes.", - ), - ) - print(f"Agent {self.agent.name} (version {self.agent.version}) created") - def __call__(self, fn): """ Wrap the test function to provide executor instance. @@ -92,11 +66,6 @@ def __call__(self, fn): if inspect.iscoroutinefunction(fn): @wraps(fn) async def _async_wrapper(test_instance, sample_path: str, **kwargs): - # Setup agent on first call - if self.agent is None: - endpoint = kwargs.get("azure_ai_projects_tests_project_endpoint", "") - self.setup_agent(test_instance, endpoint, True) - env_var_mapping = self.env_var_mapping_fn(test_instance) executor = _SampleExecutorInstance( test_instance, @@ -110,11 +79,6 @@ async def _async_wrapper(test_instance, sample_path: str, **kwargs): else: @wraps(fn) def _sync_wrapper(test_instance, sample_path: str, **kwargs): - # Setup agent on first call - if self.agent is None: - endpoint = kwargs.get("azure_ai_projects_tests_project_endpoint", "") - self.setup_agent(test_instance, endpoint, False) - env_var_mapping = self.env_var_mapping_fn(test_instance) executor = _SampleExecutorInstance( test_instance, @@ -169,6 +133,23 @@ def _capture_print(self, *args, **kwargs): self.print_calls.append(" ".join(str(arg) for arg in args)) self._original_print(*args, **kwargs) + def _get_mock_credential(self, is_async: bool): + """Get a mock credential that supports context manager protocol.""" + credential_instance = self.test_instance.get_credential(AIProjectClient, is_async=is_async) + if is_async: + patch_target = "azure.identity.aio.DefaultAzureCredential" + else: + patch_target = "azure.identity.DefaultAzureCredential" + + # Create a mock that returns a context manager wrapping the credential + mock_credential_class = mock.MagicMock() + mock_credential_class.return_value.__enter__ = mock.MagicMock(return_value=credential_instance) + mock_credential_class.return_value.__exit__ = mock.MagicMock(return_value=None) + mock_credential_class.return_value.__aenter__ = mock.AsyncMock(return_value=credential_instance) + mock_credential_class.return_value.__aexit__ = mock.AsyncMock(return_value=None) + + return mock.patch(patch_target, new=mock_credential_class) + def execute(self, sample_path: str): """Execute a synchronous sample with proper mocking and environment setup.""" self._prepare_execution(sample_path) @@ -177,20 +158,15 @@ def execute(self, sample_path: str): MonkeyPatch.context() as mp, mock.patch("builtins.print", side_effect=self._capture_print), mock.patch("builtins.open", side_effect=patched_open_crlf_to_lf), - mock.patch("azure.identity.DefaultAzureCredential") as mock_credential, + self._get_mock_credential(is_async=False), ): for var_name, var_value in self.env_vars.items(): mp.setenv(var_name, var_value) - credential_instance = self.test_instance.get_credential(AIProjectClient, is_async=False) - credential_mock = mock.MagicMock() - credential_mock.__enter__.return_value = credential_instance - credential_mock.__exit__.return_value = False - mock_credential.return_value = credential_mock if self.spec.loader is None: raise ImportError(f"Could not load module {self.spec.name} from {self.sample_path}") self.spec.loader.exec_module(self.module) - self._validate_output() + self._validate_output() async def execute_async(self, sample_path: str): """Execute an asynchronous sample with proper mocking and environment setup.""" @@ -200,65 +176,66 @@ async def execute_async(self, sample_path: str): MonkeyPatch.context() as mp, mock.patch("builtins.print", side_effect=self._capture_print), mock.patch("builtins.open", side_effect=patched_open_crlf_to_lf), - mock.patch("azure.identity.aio.DefaultAzureCredential") as mock_credential, + self._get_mock_credential(is_async=True), + self._get_mock_credential(is_async=False), ): for var_name, var_value in self.env_vars.items(): mp.setenv(var_name, var_value) - # Create a mock credential that supports async context manager protocol - credential_instance = self.test_instance.get_credential(AIProjectClient, is_async=True) - credential_mock = mock.AsyncMock() - credential_mock.__aenter__.return_value = credential_instance - credential_mock.__aexit__.return_value = False - mock_credential.return_value = credential_mock if self.spec.loader is None: raise ImportError(f"Could not load module {self.spec.name} from {self.sample_path}") self.spec.loader.exec_module(self.module) await self.module.main() - self._validate_output() + self._validate_output() def _validate_output(self): - """ - Validates: - * Sample output contains the marker '==> Result: ' - * If the sample includes a comment '# Print result (should contain "")', - the result must include that keyword (case-insensitive) - * If no keyword is specified, the result must be at least 200 characters long - """ - # Find content after ==> Result: marker in print_calls array - result_line = None - for call in self.print_calls: - if call.startswith("==> Result:"): - result_line = call - break - - if not result_line: - assert False, "Expected to find '==> Result:' in print calls." - - # Extract content after ==> Result: - arrow_match = re.search(r"==> Result:(.*)", result_line, re.IGNORECASE | re.DOTALL) - if not arrow_match: - assert False, f"Expected to find '==> Result:' in line: {result_line}" - - content_after_arrow = arrow_match.group(1).strip() - - # Read the sample file to check for expected output comment - with open(self.sample_path) as f: - sample_code = f.read() - - # Verify pattern: # Print result (should contain '...') if exist - match = re.search(r"# Print result \(should contain ['\"](.+?)['\"]\)", sample_code) - if match: - # Decode Unicode escape sequences like \u00b0F to actual characters - expected_contain = match.group(1).encode().decode("unicode_escape") - assert ( - expected_contain.lower() in content_after_arrow.lower() - ), f"Expected to find '{expected_contain}' after '==> Result:', but got: {content_after_arrow}" - else: - result_len = len(content_after_arrow) - assert result_len > 200, f"Expected 200 characters after '==> Result:', but got {result_len} characters" - + class TestReport(BaseModel): + model_config = {"extra": "forbid"} + result: Literal["correct", "incorrect"] + reason: str + + with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"], credential=credential) as project_client, + project_client.get_openai_client() as openai_client, + ): + response = openai_client.responses.create( + model="gpt-4o", + instructions="""We just run Python code and captured content from print. +One entry of string array per print. +Validating the printed content to determine if correct or not: +Respond "incorrect" if any entries show: +- Error messages or exception text +- Empty or null results where data is expected +- Malformed or corrupted data +- HTTP error codes (4xx, 5xx) +- Timeout or connection errors +- Warning messages indicating failures +- Failure to retrieve or process data +- Statements saying documents/information didn't provide relevant data +- Statements saying unable to find/retrieve information +- Asking the user to specify, clarify, or provide more details +- Suggesting to use other tools or sources +- Asking follow-up questions to complete the task +- Indicating lack of knowledge or missing information +- Responses that defer answering or redirect the question +Respond with "correct" only if the result provides a complete, substantive answer with actual data/information. +Always respond with `reason` indicating the reason for the response.""", + text={ + "format": { + "type": "json_schema", + "name": "TestReport", + "schema": TestReport.model_json_schema(), + } + }, + input=str(self.print_calls), + ) + + test_report = json.loads(response.output_text) + + assert test_report["result"] == "correct", f"Error is identified: {test_report['reason']}" + print(f"Reason: {test_report['reason']}") class SamplePathPasser: def __call__(self, fn): From 288b2f271b43e1c592e9aa221de6a625aa75ebb1 Mon Sep 17 00:00:00 2001 From: Howie Leung Date: Wed, 10 Dec 2025 11:19:20 -0800 Subject: [PATCH 3/8] update --- .../samples/agents/tools/computer_use_util.py | 2 +- .../agents/tools/sample_agent_ai_search.py | 4 +- .../tools/sample_agent_code_interpreter.py | 3 +- .../sample_agent_code_interpreter_async.py | 3 +- .../agents/tools/sample_agent_file_search.py | 2 +- .../sample_agent_file_search_in_stream.py | 2 +- ...ample_agent_file_search_in_stream_async.py | 2 +- .../tools/sample_agent_function_tool.py | 3 +- .../tools/sample_agent_function_tool_async.py | 3 +- .../tools/sample_agent_image_generation.py | 3 +- .../sample_agent_image_generation_async.py | 3 +- .../samples/agents/tools/sample_agent_mcp.py | 3 +- .../agents/tools/sample_agent_mcp_async.py | 3 +- .../agents/tools/sample_agent_openapi.py | 3 +- .../agents/tools/sample_agent_sharepoint.py | 2 +- .../agents/tools/sample_agent_web_search.py | 2 +- sdk/ai/azure-ai-projects/tests/conftest.py | 11 +- .../tests/samples/test_samples.py | 200 ++++++++++-------- 18 files changed, 134 insertions(+), 120 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/computer_use_util.py b/sdk/ai/azure-ai-projects/samples/agents/tools/computer_use_util.py index 46e3cb6803e4..c1793779b0fe 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/computer_use_util.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/computer_use_util.py @@ -159,4 +159,4 @@ def print_final_output(response): final_output += getattr(part, "text", None) or getattr(part, "refusal", None) or "" + "\n" print(f"Final status: {response.status}") - print(f"==> Result: {final_output.strip()}") + print(f"Final result: {final_output.strip()}") diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_ai_search.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_ai_search.py index 0b227f224754..bfeb59dde66f 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_ai_search.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_ai_search.py @@ -102,13 +102,13 @@ for annotation in text_content.annotations: if annotation.type == "url_citation": print( - f"URL Citation: {annotation.url}, " + f"URL Citation: , " f"Start index: {annotation.start_index}, " f"End index: {annotation.end_index}" ) elif event.type == "response.completed": print(f"\nFollow-up completed!") - print(f"==> Result: {event.response.output_text}") + print(f"Agent response: {event.response.output_text}") print("\nCleaning up...") project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_code_interpreter.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_code_interpreter.py index 9cd10f63af2d..db84258efae8 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_code_interpreter.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_code_interpreter.py @@ -108,7 +108,6 @@ file_path = os.path.join(tempfile.gettempdir(), filename) with open(file_path, "wb") as f: f.write(file_content.read()) - # Print result (should contain "file") - print(f"==> Result: file, {file_path} downloaded successfully.") + print(f"File downloaded successfully: {file_path}") else: print("No file generated in response") diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_code_interpreter_async.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_code_interpreter_async.py index be88a1f9555c..b794599b70af 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_code_interpreter_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_code_interpreter_async.py @@ -112,8 +112,7 @@ async def main() -> None: file_path = os.path.join(tempfile.gettempdir(), filename) with open(file_path, "wb") as f: f.write(file_content.read()) - # Print result (should contain "file") - print(f"==> Result: file, {file_path} downloaded successfully.") + print(f"File downloaded successfully: {file_path}") else: print("No file generated in response") diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search.py index 78b69f7e0e33..9351a29a8977 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search.py @@ -78,7 +78,7 @@ input="Tell me about Contoso products", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - print(f"==> Result: {response.output_text}") + print(f"Agent response: {response.output_text}") print("\nCleaning up...") project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search_in_stream.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search_in_stream.py index 48728d150870..21c04ba22bf6 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search_in_stream.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search_in_stream.py @@ -140,7 +140,7 @@ print(f"File Citation - Filename: {annotation.filename}, File ID: {annotation.file_id}") elif event.type == "response.completed": print(f"\nFollow-up completed!") - print(f"==> Result: {event.response.output_text}") + print(f"Agent response: {event.response.output_text}") # Clean up resources print("\n" + "=" * 60) diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search_in_stream_async.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search_in_stream_async.py index 3c5e22d1b440..31280352ac7c 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search_in_stream_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search_in_stream_async.py @@ -145,7 +145,7 @@ async def main() -> None: print(f"File Citation - Filename: {annotation.filename}, File ID: {annotation.file_id}") elif event.type == "response.completed": print(f"\nFollow-up completed!") - print(f"==> Result: {event.response.output_text}") + print(f"Agent response: {event.response.output_text}") # Clean up resources print("\n" + "=" * 60) diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_function_tool.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_function_tool.py index 97d349bec751..cfa1dad9ba7f 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_function_tool.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_function_tool.py @@ -108,8 +108,7 @@ def get_horoscope(sign: str) -> str: extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - # Print result (should contain "Tuesday") - print(f"==> Result: {response.output_text}") + print(f"Agent response: {response.output_text}") print("\nCleaning up...") project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_function_tool_async.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_function_tool_async.py index 2ff5c44d108f..9a70726fb2cf 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_function_tool_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_function_tool_async.py @@ -109,8 +109,7 @@ async def main(): extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - # Print result (should contain "Tuesday") - print(f"==> Result: {response.output_text}") + print(f"Agent response: {response.output_text}") if __name__ == "__main__": diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_image_generation.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_image_generation.py index 2f9764f2ba95..d901af701df4 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_image_generation.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_image_generation.py @@ -96,5 +96,4 @@ f.write(base64.b64decode(image_data[0])) # [END download_image] - # Print result (should contain "file") - print(f"==> Result: Image downloaded and saved to file: {file_path}") + print(f"Image saved to: {file_path}") diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_image_generation_async.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_image_generation_async.py index 4202bee8a8b1..aeb2e099695c 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_image_generation_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_image_generation_async.py @@ -91,8 +91,7 @@ async def main(): with open(file_path, "wb") as f: f.write(base64.b64decode(image_data[0])) - # Print result (should contain "file") - print(f"==> Result: Image downloaded and saved to file: {file_path}") + print(f"Image saved to: {file_path}") if __name__ == "__main__": diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp.py index 4be58ad9c32a..17fcbe97df6f 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp.py @@ -95,8 +95,7 @@ extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - # Print result (should contain "Azure") - print(f"==> Result: {response.output_text}") + print(f"Agent response: {response.output_text}") # Clean up resources by deleting the agent version # This prevents accumulation of unused agent versions in your project diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp_async.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp_async.py index 9c1ae981c5b1..ddcbc3e4614c 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp_async.py @@ -99,8 +99,7 @@ async def main(): extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - # Print result (should contain "Azure") - print(f"==> Result: {response.output_text}") + print(f"Agent response: {response.output_text}") # Clean up resources by deleting the agent version # This prevents accumulation of unused agent versions in your project diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_openapi.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_openapi.py index 781056555036..a95740e2e53f 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_openapi.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_openapi.py @@ -76,8 +76,7 @@ input="Use the OpenAPI tool to print out, what is the weather in Seattle, WA today.", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - # Print result (should contain "\u00b0F") - print(f"==> Result: {response.output_text}") + print(f"Agent response: {response.output_text}") print("\nCleaning up...") project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_sharepoint.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_sharepoint.py index 71b6ae49168a..c41be0e489d2 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_sharepoint.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_sharepoint.py @@ -101,7 +101,7 @@ ) elif event.type == "response.completed": print(f"\nFollow-up completed!") - print(f"==> Result: {event.response.output_text}") + print(f"Agent response: {event.response.output_text}") print("Cleaning up...") project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_web_search.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_web_search.py index 3a08a869c62a..7ef45abec3dd 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_web_search.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_web_search.py @@ -65,7 +65,7 @@ input="Show me the latest London Underground service updates", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - print(f"==> Result: {response.output_text}") + print(f"Agent response: {response.output_text}") print("\nCleaning up...") project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) diff --git a/sdk/ai/azure-ai-projects/tests/conftest.py b/sdk/ai/azure-ai-projects/tests/conftest.py index ad54a729211e..321008ec533b 100644 --- a/sdk/ai/azure-ai-projects/tests/conftest.py +++ b/sdk/ai/azure-ai-projects/tests/conftest.py @@ -19,7 +19,7 @@ add_general_regex_sanitizer, add_body_key_sanitizer, add_remove_header_sanitizer, - add_general_string_sanitizer, + add_body_regex_sanitizer, ) if not load_dotenv(find_dotenv(), override=True): @@ -120,6 +120,15 @@ def sanitize_url_paths(): value="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/connections/connector-name", ) + # # Sanitize print output from sample validation to prevent replay failures when print statements change + # # Only targets the validation Responses API call by matching the unique input prefix + # add_body_regex_sanitizer( + add_body_key_sanitizer( + json_path="$.input", + value="sanitized-print-output", + regex=r"print contents array = .*", + ) + # Remove Stainless headers from OpenAI client requests, since they include platform and OS specific info, which we can't have in recorded requests. # Here is an example of all the `x-stainless` headers from a Responses call: # x-stainless-arch: other:amd64 diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py index 5cf2d6bc9213..9dbacb6a81a5 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py @@ -11,93 +11,69 @@ from test_base import servicePreparer, patched_open_crlf_to_lf from pytest import MonkeyPatch from azure.ai.projects import AIProjectClient +from azure.ai.projects.aio import AIProjectClient as AsyncAIProjectClient from azure.identity import DefaultAzureCredential +from azure.identity.aio import DefaultAzureCredential as AsyncDefaultAzureCredential from functools import wraps from typing import Callable, Optional, Literal from pydantic import BaseModel + class SampleExecutor: """Decorator for executing sample files with proper environment setup and credential mocking.""" def __init__(self, env_var_mapping_fn: Callable): """ Initialize the SampleExecutor decorator. - + Args: env_var_mapping_fn: Function that returns the environment variable mapping """ self.env_var_mapping_fn = env_var_mapping_fn - self.agent = None - self.project_client = None - self.credential = None - - def __enter__(self): - """Context manager entry - creates agent for all tests.""" - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """Context manager exit - deletes agent after all tests.""" - if self.agent and self.project_client: - try: - self.project_client.agents.delete_version( - agent_name=self.agent.name, - agent_version=self.agent.version - ) - print(f"Agent {self.agent.name} deleted") - except Exception as e: - print(f"Error deleting agent: {e}") - - if self.project_client: - self.project_client.close() - - - return False def __call__(self, fn): """ Wrap the test function to provide executor instance. - + Args: fn: The test function to wrap - + Returns: Wrapped function with executor injection """ if inspect.iscoroutinefunction(fn): + @wraps(fn) async def _async_wrapper(test_instance, sample_path: str, **kwargs): env_var_mapping = self.env_var_mapping_fn(test_instance) - executor = _SampleExecutorInstance( - test_instance, - env_var_mapping, - self.agent, - **kwargs - ) + executor = _SampleExecutorInstance(test_instance, env_var_mapping, **kwargs) await fn(test_instance, sample_path, executor=executor, **kwargs) - + return _async_wrapper else: + @wraps(fn) def _sync_wrapper(test_instance, sample_path: str, **kwargs): env_var_mapping = self.env_var_mapping_fn(test_instance) - executor = _SampleExecutorInstance( - test_instance, - env_var_mapping, - self.agent, - **kwargs - ) + executor = _SampleExecutorInstance(test_instance, env_var_mapping, **kwargs) fn(test_instance, sample_path, executor=executor, **kwargs) - + return _sync_wrapper class _SampleExecutorInstance: """Internal class for executing sample files with proper environment setup and credential mocking.""" - def __init__(self, test_instance: "AzureRecordedTestCase", env_var_mapping: dict[str, str], agent, **kwargs): + class TestReport(BaseModel): + """Schema for validation test report.""" + + model_config = {"extra": "forbid"} + correct: bool + reason: str + + def __init__(self, test_instance: "AzureRecordedTestCase", env_var_mapping: dict[str, str], **kwargs): self.test_instance = test_instance self.env_var_mapping = env_var_mapping - self.agent = agent self.kwargs = kwargs self.print_calls: list[str] = [] self._original_print = print @@ -105,7 +81,7 @@ def __init__(self, test_instance: "AzureRecordedTestCase", env_var_mapping: dict def _prepare_execution(self, sample_path: str): """Prepare for sample execution by setting up environment and module.""" self.sample_path = sample_path - + # Prepare environment variables self.env_vars = {} for sample_var, test_var in self.env_var_mapping.items(): @@ -140,72 +116,66 @@ def _get_mock_credential(self, is_async: bool): patch_target = "azure.identity.aio.DefaultAzureCredential" else: patch_target = "azure.identity.DefaultAzureCredential" - + # Create a mock that returns a context manager wrapping the credential mock_credential_class = mock.MagicMock() mock_credential_class.return_value.__enter__ = mock.MagicMock(return_value=credential_instance) mock_credential_class.return_value.__exit__ = mock.MagicMock(return_value=None) mock_credential_class.return_value.__aenter__ = mock.AsyncMock(return_value=credential_instance) mock_credential_class.return_value.__aexit__ = mock.AsyncMock(return_value=None) - + return mock.patch(patch_target, new=mock_credential_class) def execute(self, sample_path: str): """Execute a synchronous sample with proper mocking and environment setup.""" self._prepare_execution(sample_path) - + with ( MonkeyPatch.context() as mp, - mock.patch("builtins.print", side_effect=self._capture_print), - mock.patch("builtins.open", side_effect=patched_open_crlf_to_lf), self._get_mock_credential(is_async=False), ): for var_name, var_value in self.env_vars.items(): mp.setenv(var_name, var_value) - if self.spec.loader is None: - raise ImportError(f"Could not load module {self.spec.name} from {self.sample_path}") - self.spec.loader.exec_module(self.module) + + with ( + mock.patch("builtins.print", side_effect=self._capture_print), + mock.patch("builtins.open", side_effect=patched_open_crlf_to_lf), + ): + if self.spec.loader is None: + raise ImportError(f"Could not load module {self.spec.name} from {self.sample_path}") + self.spec.loader.exec_module(self.module) self._validate_output() async def execute_async(self, sample_path: str): """Execute an asynchronous sample with proper mocking and environment setup.""" self._prepare_execution(sample_path) - + with ( MonkeyPatch.context() as mp, - mock.patch("builtins.print", side_effect=self._capture_print), - mock.patch("builtins.open", side_effect=patched_open_crlf_to_lf), self._get_mock_credential(is_async=True), - self._get_mock_credential(is_async=False), ): for var_name, var_value in self.env_vars.items(): mp.setenv(var_name, var_value) - if self.spec.loader is None: - raise ImportError(f"Could not load module {self.spec.name} from {self.sample_path}") - self.spec.loader.exec_module(self.module) - await self.module.main() - - self._validate_output() - - def _validate_output(self): - class TestReport(BaseModel): - model_config = {"extra": "forbid"} - result: Literal["correct", "incorrect"] - reason: str - - with ( - DefaultAzureCredential() as credential, - AIProjectClient(endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"], credential=credential) as project_client, - project_client.get_openai_client() as openai_client, - ): - response = openai_client.responses.create( - model="gpt-4o", - instructions="""We just run Python code and captured content from print. -One entry of string array per print. + with ( + mock.patch("builtins.open", side_effect=patched_open_crlf_to_lf), + mock.patch("builtins.print", side_effect=self._capture_print), + ): + if self.spec.loader is None: + raise ImportError(f"Could not load module {self.spec.name} from {self.sample_path}") + self.spec.loader.exec_module(self.module) + await self.module.main() + + await self._validate_output_async() + + def _get_validation_request_params(self) -> dict: + """Get common parameters for validation request.""" + return { + "model": "gpt-4o", + "instructions": """We just run Python code and captured a Python array of print statements. Validating the printed content to determine if correct or not: -Respond "incorrect" if any entries show: +Respond false if any entries show: - Error messages or exception text - Empty or null results where data is expected - Malformed or corrupted data @@ -220,22 +190,66 @@ class TestReport(BaseModel): - Asking follow-up questions to complete the task - Indicating lack of knowledge or missing information - Responses that defer answering or redirect the question -Respond with "correct" only if the result provides a complete, substantive answer with actual data/information. +Respond with true only if the result provides a complete, substantive answer with actual data/information. Always respond with `reason` indicating the reason for the response.""", - text={ - "format": { - "type": "json_schema", - "name": "TestReport", - "schema": TestReport.model_json_schema(), - } - }, - input=str(self.print_calls), - ) + "text": { + "format": { + "type": "json_schema", + "name": "TestReport", + "schema": self.TestReport.model_json_schema(), + } + }, + # The input field is sanitized in recordings (see conftest.py) by matching the unique prefix + # "print contents array = ". This allows sample print statements to change without breaking playback. + # The instructions field is preserved as-is in recordings. If you modify the instructions, + # you must re-record the tests. + "input": f"print contents array = {self.print_calls}", + } + + def _assert_validation_result(self, test_report: dict) -> None: + """Assert validation result and print reason.""" + if not test_report["correct"]: + # Write print statements to log file in temp folder for debugging + import tempfile + from datetime import datetime + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = os.path.join(tempfile.gettempdir(), f"sample_validation_error_{timestamp}.log") + with open(log_file, "w") as f: + f.write(f"Sample: {self.sample_path}\n") + f.write(f"Validation Error: {test_report['reason']}\n\n") + f.write("Print Statements:\n") + f.write("=" * 80 + "\n") + for i, print_call in enumerate(self.print_calls, 1): + f.write(f"{i}. {print_call}\n") + print(f"\nValidation failed! Print statements logged to: {log_file}") + assert test_report["correct"], f"Error is identified: {test_report['reason']}" + print(f"Reason: {test_report['reason']}") + def _validate_output(self): + """Validate sample output using synchronous OpenAI client.""" + with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"], credential=credential) as project_client, + project_client.get_openai_client() as openai_client, + ): + response = openai_client.responses.create(**self._get_validation_request_params()) test_report = json.loads(response.output_text) + self._assert_validation_result(test_report) + + async def _validate_output_async(self): + """Validate sample output using asynchronous OpenAI client.""" + async with ( + AsyncDefaultAzureCredential() as credential, + AsyncAIProjectClient( + endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"], credential=credential + ) as project_client, + ): + async with project_client.get_openai_client() as openai_client: + response = await openai_client.responses.create(**self._get_validation_request_params()) + test_report = json.loads(response.output_text) + self._assert_validation_result(test_report) - assert test_report["result"] == "correct", f"Error is identified: {test_report['reason']}" - print(f"Reason: {test_report['reason']}") class SamplePathPasser: def __call__(self, fn): @@ -625,4 +639,4 @@ def test_tools_samples(self, sample_path: str, executor: _SampleExecutorInstance @SampleExecutor(_get_tools_sample_environment_variables_map) @recorded_by_proxy_async(RecordedTransport.AZURE_CORE, RecordedTransport.HTTPX) async def test_tools_samples_async(self, sample_path: str, executor: _SampleExecutorInstance, **kwargs) -> None: - await executor.execute_async(sample_path) \ No newline at end of file + await executor.execute_async(sample_path) From e83c7bd084b74884f9bb57e85312e8b8e51cbdbb Mon Sep 17 00:00:00 2001 From: Howie Leung Date: Wed, 10 Dec 2025 12:11:40 -0800 Subject: [PATCH 4/8] restore some code --- .../agents/tools/sample_agent_ai_search.py | 2 +- ...est_agent_code_interpreter_and_function.py | 1 + .../test_agent_file_search_and_function.py | 3 +- .../agents/tools/test_agent_ai_search.py | 1 + .../tools/test_agent_ai_search_async.py | 1 + sdk/ai/azure-ai-projects/tests/conftest.py | 5 +- .../tests/samples/test_samples.py | 400 ++---------------- 7 files changed, 47 insertions(+), 366 deletions(-) diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_ai_search.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_ai_search.py index bfeb59dde66f..263aee360715 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_ai_search.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_ai_search.py @@ -102,7 +102,7 @@ for annotation in text_content.annotations: if annotation.type == "url_citation": print( - f"URL Citation: , " + f"URL Citation: {annotation.url}, " f"Start index: {annotation.start_index}, " f"End index: {annotation.end_index}" ) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py index a51bc7b5514e..e4e42fe991f5 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py @@ -18,6 +18,7 @@ from azure.ai.projects.models import PromptAgentDefinition, CodeInterpreterTool, CodeInterpreterToolAuto, FunctionTool from openai.types.responses.response_input_param import FunctionCallOutput, ResponseInputParam + class TestAgentCodeInterpreterAndFunction(TestBase): """Tests for agents using Code Interpreter + Function Tool combination.""" diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py index ae6d9ad86c70..f95f7f21af1a 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py @@ -328,7 +328,8 @@ def calculate_sum(numbers): # Verify findings discuss the code (proves File Search was used) findings_lower = arguments["findings"].lower() assert any( - keyword in findings_lower for keyword in ["sum", "calculate", "function", "numbers", "list", "return"] + keyword in findings_lower + for keyword in ["sum", "calculate", "function", "numbers", "list", "return"] ), f"Expected findings to discuss the code content. Got: {arguments['findings'][:100]}" input_list.append( diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py index 5ee41aea715f..188b24806409 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py @@ -19,6 +19,7 @@ # The tests in this file rely on an existing Azure AI Search project connection that has been populated with the following document: # https://arxiv.org/pdf/2508.03680 + class TestAgentAISearch(TestBase): # Test questions with expected answers diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py index 71525b089398..6f050e553de1 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py @@ -20,6 +20,7 @@ # The tests in this file rely on an existing Azure AI Search project connection that has been populated with the following document: # https://arxiv.org/pdf/2508.03680 + class TestAgentAISearchAsync(TestBase): # Test questions with expected answers diff --git a/sdk/ai/azure-ai-projects/tests/conftest.py b/sdk/ai/azure-ai-projects/tests/conftest.py index 321008ec533b..a074884c8063 100644 --- a/sdk/ai/azure-ai-projects/tests/conftest.py +++ b/sdk/ai/azure-ai-projects/tests/conftest.py @@ -120,9 +120,8 @@ def sanitize_url_paths(): value="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/connections/connector-name", ) - # # Sanitize print output from sample validation to prevent replay failures when print statements change - # # Only targets the validation Responses API call by matching the unique input prefix - # add_body_regex_sanitizer( + # Sanitize print output from sample validation to prevent replay failures when print statements change + # Only targets the validation Responses API call by matching the unique input prefix add_body_key_sanitizer( json_path="$.input", value="sanitized-print-output", diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py index 9dbacb6a81a5..748845496144 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py @@ -1,7 +1,8 @@ # pylint: disable=line-too-long,useless-suppression # ------------------------------------ # Copyright (c) Microsoft Corporation. -# Licensed------------------------- +# Licensed under the MIT License. +# ------------------------------------ import csv, os, pytest, re, inspect, sys, json import importlib.util import unittest.mock as mock @@ -20,49 +21,7 @@ class SampleExecutor: - """Decorator for executing sample files with proper environment setup and credential mocking.""" - - def __init__(self, env_var_mapping_fn: Callable): - """ - Initialize the SampleExecutor decorator. - - Args: - env_var_mapping_fn: Function that returns the environment variable mapping - """ - self.env_var_mapping_fn = env_var_mapping_fn - - def __call__(self, fn): - """ - Wrap the test function to provide executor instance. - - Args: - fn: The test function to wrap - - Returns: - Wrapped function with executor injection - """ - if inspect.iscoroutinefunction(fn): - - @wraps(fn) - async def _async_wrapper(test_instance, sample_path: str, **kwargs): - env_var_mapping = self.env_var_mapping_fn(test_instance) - executor = _SampleExecutorInstance(test_instance, env_var_mapping, **kwargs) - await fn(test_instance, sample_path, executor=executor, **kwargs) - - return _async_wrapper - else: - - @wraps(fn) - def _sync_wrapper(test_instance, sample_path: str, **kwargs): - env_var_mapping = self.env_var_mapping_fn(test_instance) - executor = _SampleExecutorInstance(test_instance, env_var_mapping, **kwargs) - fn(test_instance, sample_path, executor=executor, **kwargs) - - return _sync_wrapper - - -class _SampleExecutorInstance: - """Internal class for executing sample files with proper environment setup and credential mocking.""" + """Helper class for executing sample files with proper environment setup and credential mocking.""" class TestReport(BaseModel): """Schema for validation test report.""" @@ -71,21 +30,18 @@ class TestReport(BaseModel): correct: bool reason: str - def __init__(self, test_instance: "AzureRecordedTestCase", env_var_mapping: dict[str, str], **kwargs): + def __init__( + self, test_instance: "AzureRecordedTestCase", sample_path: str, env_var_mapping: dict[str, str], **kwargs + ): self.test_instance = test_instance - self.env_var_mapping = env_var_mapping - self.kwargs = kwargs + self.sample_path = sample_path self.print_calls: list[str] = [] self._original_print = print - def _prepare_execution(self, sample_path: str): - """Prepare for sample execution by setting up environment and module.""" - self.sample_path = sample_path - # Prepare environment variables self.env_vars = {} - for sample_var, test_var in self.env_var_mapping.items(): - value = self.kwargs.get(test_var, None) + for sample_var, test_var in env_var_mapping.items(): + value = kwargs.pop(test_var, None) if value is not None: self.env_vars[sample_var] = value self.env_vars["AZURE_AI_MODEL_DEPLOYMENT_NAME"] = "gpt-4o" @@ -126,9 +82,8 @@ def _get_mock_credential(self, is_async: bool): return mock.patch(patch_target, new=mock_credential_class) - def execute(self, sample_path: str): + def execute(self): """Execute a synchronous sample with proper mocking and environment setup.""" - self._prepare_execution(sample_path) with ( MonkeyPatch.context() as mp, @@ -147,9 +102,8 @@ def execute(self, sample_path: str): self._validate_output() - async def execute_async(self, sample_path: str): + async def execute_async(self): """Execute an asynchronous sample with proper mocking and environment setup.""" - self._prepare_execution(sample_path) with ( MonkeyPatch.context() as mp, @@ -289,7 +243,11 @@ def _get_tools_sample_paths(): # Only include .py files, exclude __pycache__ and utility files if "sample_" in filename and "_async" not in filename and filename not in tools_samples_to_skip: sample_path = os.path.join(tools_folder, filename) - samples.append(pytest.param(sample_path, id=filename.replace(".py", ""))) + # Get relative path from samples folder and convert to test ID format + rel_path = os.path.relpath(sample_path, samples_folder_path) + # Remove 'samples\\' prefix and convert to forward slashes + test_id = rel_path.replace("samples\\", "").replace("\\", "/").replace(".py", "") + samples.append(pytest.param(sample_path, id=test_id)) return samples @@ -311,7 +269,11 @@ def _get_tools_sample_paths_async(): # Only include async .py files, exclude __pycache__ and utility files if "sample_" in filename and "_async" in filename and filename not in tools_samples_to_skip: sample_path = os.path.join(tools_folder, filename) - samples.append(pytest.param(sample_path, id=filename.replace(".py", ""))) + # Get relative path from samples folder and convert to test ID format + rel_path = os.path.relpath(sample_path, samples_folder_path) + # Remove 'samples\\' prefix and convert to forward slashes + test_id = rel_path.replace("samples\\", "").replace("\\", "/").replace(".py", "") + samples.append(pytest.param(sample_path, id=test_id)) return samples @@ -328,315 +290,31 @@ def _get_tools_sample_environment_variables_map(self) -> dict[str, str]: class TestSamples(AzureRecordedTestCase): - _samples_folder_path: str - _results: dict[str, tuple[bool, str]] - - """ - Test class for running all samples in the `/sdk/ai/azure-ai-projects/samples` folder. - - To run this test: - * 'cd' to the folder '/sdk/ai/azure-ai-projects' in your azure-sdk-for-python repo. - * set AZURE_AI_PROJECT_ENDPOINT= - Define your Microsoft Foundry project endpoint used by the test. - * set AZURE_AI_PROJECTS_CONSOLE_LOGGING=false - to make sure logging is not enabled in the test, to reduce console spew. - * Uncomment the two lines that start with "@pytest.mark.skip" below. - * Run: pytest tests\\samples\\test_samples.py::TestSamples - * Load the resulting report in Excel: tests\\samples\\samples_report.csv - """ - - @classmethod - def setup_class(cls): - current_path = os.path.abspath(__file__) - cls._samples_folder_path = os.path.join(current_path, os.pardir, os.pardir, os.pardir) - cls._results: dict[str, tuple[bool, str]] = {} - - @classmethod - def teardown_class(cls): - """ - Class-level teardown method that generates a report file named "samples_report.csv" after all tests have run. - - The report contains one line per sample run, with three columns: - 1. PASS or FAIL indicating the sample result. - 2. The name of the sample. - 3. The exception string summary if the sample failed, otherwise empty. - - The report is written to the same directory as this test file. - """ - report_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "samples_report.csv") - with open(report_path, mode="w", newline="") as file: - writer = csv.writer(file, quotechar='"', quoting=csv.QUOTE_ALL) # Ensures proper quoting - for test_name, (passed, exception_string) in cls._results.items(): - exception_message = f'"{exception_string.splitlines()[0]}"' if exception_string else "" - writer.writerow([f"{'PASS' if passed else 'FAIL'}", test_name, exception_message]) - - @classmethod - def _set_env_vars(cls, sample_name: str, **kwargs): - """ - Sets environment variables for a given sample run and prints them. - - Args: - sample_name (str): The name of the sample being executed. - **kwargs: Arbitrary keyword arguments representing environment variable names and their values. - """ - - print(f"\nRunning {sample_name} with environment variables: ", end="") - for key, value in kwargs.items(): - if value: - env_key = key.upper() - os.environ[env_key] = value - print(f"{env_key}={value} ", end="") - print("\n") - - @classmethod - def _run_sample(cls, sample_name: str) -> None: - """ - Executes a synchronous sample file and records the result. - - Args: - sample_name (str): The name of the sample file to execute. - - Raises: - Exception: Re-raises any exception encountered during execution of the sample file. - - Side Effects: - Updates the class-level _results dictionary with the execution status and error message (if any) - for the given sample. - Prints an error message to stdout if execution fails. - """ - - sample_path = os.path.normpath(os.path.join(TestSamples._samples_folder_path, sample_name)) - with open(sample_path) as f: - code = f.read() - try: - exec(code) - except HttpResponseError as exc: - exception_message = f"{exc.status_code}, {exc.reason}, {str(exc)}" - TestSamples._results[sample_name] = (False, exception_message) - print(f"=================> Error running sample {sample_path}: {exception_message}") - raise Exception from exc - except Exception as exc: - TestSamples._results[sample_name] = (False, str(exc)) - print(f"=================> Error running sample {sample_path}: {exc}") - raise Exception from exc - TestSamples._results[sample_name] = (True, "") - - @classmethod - async def _run_sample_async(cls, sample_name: str) -> None: - """ - Asynchronously runs a sample Python script specified by its file name. - - This method dynamically imports the sample module from the given file path, - executes its `main()` coroutine, and records the result. If an exception occurs - during execution, the error is logged and re-raised. - - Args: - sample_name (str): The name of the sample Python file to run (relative to the samples folder). - - Raises: - ImportError: If the sample module cannot be loaded. - Exception: If an error occurs during the execution of the sample's `main()` coroutine. - - Side Effects: - Updates the `_results` dictionary with the execution status and error message (if any). - Prints error messages to the console if execution fails. - """ - - sample_path = os.path.normpath(os.path.join(TestSamples._samples_folder_path, sample_name)) - # Dynamically import the module from the given path - module_name = os.path.splitext(os.path.basename(sample_path))[0] - spec = importlib.util.spec_from_file_location(module_name, sample_path) - if spec is None or spec.loader is None: - raise ImportError(f"Could not load module {module_name} from {sample_path}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - # Await the main() coroutine defined in the sample - try: - await module.main() - except HttpResponseError as exc: - exception_message = f"{exc.status_code}, {exc.reason}, {str(exc)}" - TestSamples._results[sample_name] = (False, exception_message) - print(f"=================> Error running sample {sample_path}: {exception_message}") - raise Exception from exc - except Exception as exc: - TestSamples._results[sample_name] = (False, str(exc)) - print(f"=================> Error running sample {sample_path}: {exc}") - raise Exception from exc - TestSamples._results[sample_name] = (True, "") - - @pytest.mark.parametrize( - "sample_name, model_deployment_name, connection_name, data_folder", - [ - ("samples\\agents\\sample_agents.py", "gpt-4o", "", ""), - ("samples\\connections\\sample_connections.py", "", "connection1", ""), - ("samples\\deployments\\sample_deployments.py", "DeepSeek-V3", "", ""), - ("samples\\datasets\\sample_datasets.py", "", "balapvbyostoragecanary", "samples\\datasets\\data_folder"), - ( - "samples\\datasets\\sample_datasets_download.py", - "", - "balapvbyostoragecanary", - "samples\\datasets\\data_folder", - ), - ("samples\\indexes\\sample_indexes.py", "", "", ""), - ( - "samples\\inference\\azure-ai-inference\\sample_chat_completions_with_azure_ai_inference_client.py", - "Phi-4", - "", - "", - ), - ( - "samples\\inference\\azure-ai-inference\\sample_chat_completions_with_azure_ai_inference_client_and_azure_monitor_tracing.py", - "Phi-4", - "", - "", - ), - ( - "samples\\inference\\azure-ai-inference\\sample_chat_completions_with_azure_ai_inference_client_and_console_tracing.py", - "Phi-4", - "", - "", - ), - ( - "samples\\inference\\azure-openai\\sample_chat_completions_with_azure_openai_client.py", - "gpt-4o", - "connection1", - "", - ), - ( - "samples\\inference\\azure-openai\\sample_responses_with_azure_openai_client.py", - "gpt-4o", - "connection1", - "", - ), - ( - "samples\\inference\\azure-openai\\sample_chat_completions_with_azure_openai_client_and_azure_monitor_tracing.py", - "gpt-4o", - "", - "", - ), - ( - "samples\\inference\\azure-openai\\sample_chat_completions_with_azure_openai_client_and_console_tracing.py", - "gpt-4o", - "", - "", - ), - ( - "samples\\inference\\azure-ai-inference\\sample_image_embeddings_with_azure_ai_inference_client.py", - "Cohere-embed-v3-english", - "", - "samples\\inference\\azure-ai-inference", - ), - ( - "samples\\inference\\azure-ai-inference\\sample_text_embeddings_with_azure_ai_inference_client.py", - "text-embedding-3-large", - "", - "", - ), - ("samples\\telemetry\\sample_telemetry.py", "", "", ""), - ], - ) - @pytest.mark.skip(reason="This test should only run manually on your local machine, with live service calls.") - def test_samples( - self, sample_name: str, model_deployment_name: str, connection_name: str, data_folder: str - ) -> None: - """ - Run all the synchronous sample code in the samples folder. If a sample throws an exception, which for example - happens when the service responds with an error, the test will fail. - - Before running this test, you need to define the following environment variables: - 1) AZURE_AI_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the overview page of your - Microsoft Foundry project. - """ - - self._set_env_vars( - sample_name, - **{ - "model_deployment_name": model_deployment_name, - "connection_name": connection_name, - "data_folder": data_folder, - }, - ) - TestSamples._run_sample(sample_name) - - @pytest.mark.parametrize( - "sample_name, model_deployment_name, connection_name, data_folder", - [ - ("samples\\agents\\sample_agents_async.py", "gpt-4o", "", ""), - ("samples\\connections\\sample_connections_async.py", "", "connection1", ""), - ( - "samples\\datasets\\sample_datasets_async.py", - "", - "balapvbyostoragecanary", - "samples\\datasets\\data_folder", - ), - ("samples\\deployments\\sample_deployments_async.py", "DeepSeek-V3", "", ""), - ("samples\\indexes\\sample_indexes_async.py", "", "", ""), - ( - "samples\\inference\\azure-ai-inference\\sample_chat_completions_with_azure_ai_inference_client_async.py", - "Phi-4", - "", - "", - ), - ( - "samples\\inference\\azure-openai\\sample_chat_completions_with_azure_openai_client_async.py", - "gpt-4o", - "connection1", - "", - ), - ( - "samples\\inference\\azure-openai\\sample_responses_with_azure_openai_client_async.py", - "gpt-4o", - "connection1", - "", - ), - ( - "samples\\inference\\azure-ai-inference\\sample_image_embeddings_with_azure_ai_inference_client_async.py", - "Cohere-embed-v3-english", - "", - "samples\\inference\\azure-ai-inference", - ), - ( - "samples\\inference\\azure-ai-inference\\sample_text_embeddings_with_azure_ai_inference_client_async.py", - "text-embedding-3-large", - "", - "", - ), - ("samples\\telemetry\\sample_telemetry_async.py", "", "", ""), - ], - ) - @pytest.mark.skip(reason="This test should only run manually on your local machine, with live service calls.") - async def test_samples_async( - self, sample_name: str, model_deployment_name: str, connection_name: str, data_folder: str - ) -> None: - """ - Run all the asynchronous sample code in the samples folder. If a sample throws an exception, which for example - happens when the service responds with an error, the test will fail. - - Before running this test, you need to define the following environment variables: - 1) AZURE_AI_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the overview page of your - Microsoft Foundry project. - """ - - self._set_env_vars( - sample_name, - **{ - "model_deployment_name": model_deployment_name, - "connection_name": connection_name, - "data_folder": data_folder, - }, - ) - await TestSamples._run_sample_async(sample_name) @servicePreparer() @pytest.mark.parametrize("sample_path", _get_tools_sample_paths()) @SamplePathPasser() - @SampleExecutor(_get_tools_sample_environment_variables_map) @recorded_by_proxy(RecordedTransport.AZURE_CORE, RecordedTransport.HTTPX) - def test_tools_samples(self, sample_path: str, executor: _SampleExecutorInstance, **kwargs) -> None: - executor.execute(sample_path) + def test_samples(self, sample_path: str, **kwargs) -> None: + env_var_mapping = self._get_sample_environment_variables_map() + executor = SampleExecutor(self, sample_path, env_var_mapping, **kwargs) + executor.execute() @servicePreparer() @pytest.mark.parametrize("sample_path", _get_tools_sample_paths_async()) @SamplePathPasser() - @SampleExecutor(_get_tools_sample_environment_variables_map) @recorded_by_proxy_async(RecordedTransport.AZURE_CORE, RecordedTransport.HTTPX) - async def test_tools_samples_async(self, sample_path: str, executor: _SampleExecutorInstance, **kwargs) -> None: - await executor.execute_async(sample_path) + async def test_samples_async(self, sample_path: str, **kwargs) -> None: + env_var_mapping = self._get_sample_environment_variables_map() + executor = SampleExecutor(self, sample_path, env_var_mapping, **kwargs) + await executor.execute_async() + + def _get_sample_environment_variables_map(self) -> dict[str, str]: + return { + "AZURE_AI_PROJECT_ENDPOINT": "azure_ai_projects_tests_project_endpoint", + "AI_SEARCH_PROJECT_CONNECTION_ID": "azure_ai_projects_tests_ai_search_project_connection_id", + "AI_SEARCH_INDEX_NAME": "azure_ai_projects_tests_ai_search_index_name", + "AI_SEARCH_USER_INPUT": "azure_ai_projects_tests_ai_search_user_input", + "SHAREPOINT_USER_INPUT": "azure_ai_projects_tests_sharepoint_user_input", + "SHAREPOINT_PROJECT_CONNECTION_ID": "azure_ai_projects_tests_sharepoint_project_connection_id", + } From 3cb71f3192386069934f2c3b61e57178aea13299 Mon Sep 17 00:00:00 2001 From: Howie Leung Date: Wed, 10 Dec 2025 12:26:51 -0800 Subject: [PATCH 5/8] clean up --- .../azure-ai-projects/tests/samples/test_samples.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py index 748845496144..4b61ecd29026 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py @@ -15,8 +15,6 @@ from azure.ai.projects.aio import AIProjectClient as AsyncAIProjectClient from azure.identity import DefaultAzureCredential from azure.identity.aio import DefaultAzureCredential as AsyncDefaultAzureCredential -from functools import wraps -from typing import Callable, Optional, Literal from pydantic import BaseModel @@ -278,17 +276,6 @@ def _get_tools_sample_paths_async(): return samples -def _get_tools_sample_environment_variables_map(self) -> dict[str, str]: - return { - "AZURE_AI_PROJECT_ENDPOINT": "azure_ai_projects_tests_project_endpoint", - "AI_SEARCH_PROJECT_CONNECTION_ID": "azure_ai_projects_tests_ai_search_project_connection_id", - "AI_SEARCH_INDEX_NAME": "azure_ai_projects_tests_ai_search_index_name", - "AI_SEARCH_USER_INPUT": "azure_ai_projects_tests_ai_search_user_input", - "SHAREPOINT_USER_INPUT": "azure_ai_projects_tests_sharepoint_user_input", - "SHAREPOINT_PROJECT_CONNECTION_ID": "azure_ai_projects_tests_sharepoint_project_connection_id", - } - - class TestSamples(AzureRecordedTestCase): @servicePreparer() From fa356ede2e5ce23011080943d2e3448f7a1b0d5b Mon Sep 17 00:00:00 2001 From: Howie Leung Date: Wed, 10 Dec 2025 15:23:08 -0800 Subject: [PATCH 6/8] update --- .../tests/samples/test_samples.py | 65 +++++++++++-------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py index 4b61ecd29026..501e2c358228 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py @@ -6,6 +6,9 @@ import csv, os, pytest, re, inspect, sys, json import importlib.util import unittest.mock as mock +from typing import cast +from azure.core.credentials import TokenCredential +from azure.core.credentials_async import AsyncTokenCredential from azure.core.exceptions import HttpResponseError from devtools_testutils.aio import recorded_by_proxy_async from devtools_testutils import AzureRecordedTestCase, recorded_by_proxy, RecordedTransport @@ -131,7 +134,6 @@ def _get_validation_request_params(self) -> dict: - Error messages or exception text - Empty or null results where data is expected - Malformed or corrupted data -- HTTP error codes (4xx, 5xx) - Timeout or connection errors - Warning messages indicating failures - Failure to retrieve or process data @@ -180,9 +182,11 @@ def _assert_validation_result(self, test_report: dict) -> None: def _validate_output(self): """Validate sample output using synchronous OpenAI client.""" + credential = self.test_instance.get_credential(AIProjectClient, is_async=False) with ( - DefaultAzureCredential() as credential, - AIProjectClient(endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"], credential=credential) as project_client, + AIProjectClient( + endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"], credential=cast(TokenCredential, credential) + ) as project_client, project_client.get_openai_client() as openai_client, ): response = openai_client.responses.create(**self._get_validation_request_params()) @@ -191,10 +195,10 @@ def _validate_output(self): async def _validate_output_async(self): """Validate sample output using asynchronous OpenAI client.""" + credential = self.test_instance.get_credential(AIProjectClient, is_async=True) async with ( - AsyncDefaultAzureCredential() as credential, AsyncAIProjectClient( - endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"], credential=credential + endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"], credential=cast(AsyncTokenCredential, credential) ) as project_client, ): async with project_client.get_openai_client() as openai_client: @@ -225,25 +229,27 @@ def _get_tools_sample_paths(): samples_folder_path = os.path.normpath(os.path.join(current_dir, os.pardir, os.pardir)) tools_folder = os.path.join(samples_folder_path, "samples", "agents", "tools") - tools_samples_to_skip = [ - "sample_agent_bing_custom_search.py", - "sample_agent_bing_grounding.py", - "sample_agent_browser_automation.py", - "sample_agent_fabric.py", - "sample_agent_mcp_with_project_connection.py", - "sample_agent_memory_search.py", - "sample_agent_openapi_with_project_connection.py", - "sample_agent_to_agent.py", + # Whitelist of samples to test + tools_samples_to_test = [ + "sample_agent_ai_search.py", + "sample_agent_code_interpreter.py", + "sample_agent_file_search.py", + "sample_agent_file_search_in_stream.py", + "sample_agent_function_tool.py", + "sample_agent_image_generation.py", + "sample_agent_mcp.py", + "sample_agent_openapi.py", + "sample_agent_sharepoint.py", + "sample_agent_web_search.py", ] samples = [] - for filename in sorted(os.listdir(tools_folder)): - # Only include .py files, exclude __pycache__ and utility files - if "sample_" in filename and "_async" not in filename and filename not in tools_samples_to_skip: - sample_path = os.path.join(tools_folder, filename) + for filename in tools_samples_to_test: + sample_path = os.path.join(tools_folder, filename) + if os.path.exists(sample_path): # Get relative path from samples folder and convert to test ID format rel_path = os.path.relpath(sample_path, samples_folder_path) - # Remove 'samples\\' prefix and convert to forward slashes + # Remove 'samples\' prefix and convert to forward slashes test_id = rel_path.replace("samples\\", "").replace("\\", "/").replace(".py", "") samples.append(pytest.param(sample_path, id=test_id)) @@ -256,20 +262,23 @@ def _get_tools_sample_paths_async(): samples_folder_path = os.path.normpath(os.path.join(current_dir, os.pardir, os.pardir)) tools_folder = os.path.join(samples_folder_path, "samples", "agents", "tools") - # Skip async samples that are not yet ready for testing - tools_samples_to_skip = [ - "sample_agent_mcp_with_project_connection_async.py", - "sample_agent_memory_search_async.py", + # Whitelist of async samples to test + tools_samples_to_test_async = [ + "sample_agent_code_interpreter_async.py", + "sample_agent_computer_use_async.py", + "sample_agent_file_search_in_stream_async.py", + "sample_agent_function_tool_async.py", + "sample_agent_image_generation_async.py", + "sample_agent_mcp_async.py", ] samples = [] - for filename in sorted(os.listdir(tools_folder)): - # Only include async .py files, exclude __pycache__ and utility files - if "sample_" in filename and "_async" in filename and filename not in tools_samples_to_skip: - sample_path = os.path.join(tools_folder, filename) + for filename in tools_samples_to_test_async: + sample_path = os.path.join(tools_folder, filename) + if os.path.exists(sample_path): # Get relative path from samples folder and convert to test ID format rel_path = os.path.relpath(sample_path, samples_folder_path) - # Remove 'samples\\' prefix and convert to forward slashes + # Remove 'samples\' prefix and convert to forward slashes test_id = rel_path.replace("samples\\", "").replace("\\", "/").replace(".py", "") samples.append(pytest.param(sample_path, id=test_id)) From 99386f819ff168ec37733beda3e1f160d1261f4b Mon Sep 17 00:00:00 2001 From: Howie Leung Date: Wed, 10 Dec 2025 18:36:46 -0800 Subject: [PATCH 7/8] fix recording --- sdk/ai/azure-ai-projects/assets.json | 2 +- .../tests/samples/test_samples.py | 14 ++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/sdk/ai/azure-ai-projects/assets.json b/sdk/ai/azure-ai-projects/assets.json index 4eaa940a0b11..32d58b6b191a 100644 --- a/sdk/ai/azure-ai-projects/assets.json +++ b/sdk/ai/azure-ai-projects/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "python", "TagPrefix": "python/ai/azure-ai-projects", - "Tag": "python/ai/azure-ai-projects_314598932e" + "Tag": "python/ai/azure-ai-projects_d44710c465" } diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py index 501e2c358228..1677c9a9d9f3 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py @@ -247,10 +247,7 @@ def _get_tools_sample_paths(): for filename in tools_samples_to_test: sample_path = os.path.join(tools_folder, filename) if os.path.exists(sample_path): - # Get relative path from samples folder and convert to test ID format - rel_path = os.path.relpath(sample_path, samples_folder_path) - # Remove 'samples\' prefix and convert to forward slashes - test_id = rel_path.replace("samples\\", "").replace("\\", "/").replace(".py", "") + test_id = filename.replace(".py", "") samples.append(pytest.param(sample_path, id=test_id)) return samples @@ -276,10 +273,7 @@ def _get_tools_sample_paths_async(): for filename in tools_samples_to_test_async: sample_path = os.path.join(tools_folder, filename) if os.path.exists(sample_path): - # Get relative path from samples folder and convert to test ID format - rel_path = os.path.relpath(sample_path, samples_folder_path) - # Remove 'samples\' prefix and convert to forward slashes - test_id = rel_path.replace("samples\\", "").replace("\\", "/").replace(".py", "") + test_id = filename.replace(".py", "") samples.append(pytest.param(sample_path, id=test_id)) return samples @@ -291,7 +285,7 @@ class TestSamples(AzureRecordedTestCase): @pytest.mark.parametrize("sample_path", _get_tools_sample_paths()) @SamplePathPasser() @recorded_by_proxy(RecordedTransport.AZURE_CORE, RecordedTransport.HTTPX) - def test_samples(self, sample_path: str, **kwargs) -> None: + def test_agent_tools_samples(self, sample_path: str, **kwargs) -> None: env_var_mapping = self._get_sample_environment_variables_map() executor = SampleExecutor(self, sample_path, env_var_mapping, **kwargs) executor.execute() @@ -300,7 +294,7 @@ def test_samples(self, sample_path: str, **kwargs) -> None: @pytest.mark.parametrize("sample_path", _get_tools_sample_paths_async()) @SamplePathPasser() @recorded_by_proxy_async(RecordedTransport.AZURE_CORE, RecordedTransport.HTTPX) - async def test_samples_async(self, sample_path: str, **kwargs) -> None: + async def test_agent_tools_samples_async(self, sample_path: str, **kwargs) -> None: env_var_mapping = self._get_sample_environment_variables_map() executor = SampleExecutor(self, sample_path, env_var_mapping, **kwargs) await executor.execute_async() From 65570abf19634f7160ba7bcb1037029d1343bfd1 Mon Sep 17 00:00:00 2001 From: Howie Leung Date: Wed, 10 Dec 2025 20:55:02 -0800 Subject: [PATCH 8/8] resolved comment --- sdk/ai/azure-ai-projects/assets.json | 2 +- .../tests/samples/test_samples.py | 145 ++++++++++++------ 2 files changed, 95 insertions(+), 52 deletions(-) diff --git a/sdk/ai/azure-ai-projects/assets.json b/sdk/ai/azure-ai-projects/assets.json index 32d58b6b191a..c5814cde3f02 100644 --- a/sdk/ai/azure-ai-projects/assets.json +++ b/sdk/ai/azure-ai-projects/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "python", "TagPrefix": "python/ai/azure-ai-projects", - "Tag": "python/ai/azure-ai-projects_d44710c465" + "Tag": "python/ai/azure-ai-projects_93d1dc0fe7" } diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py index 1677c9a9d9f3..edfa548ab38d 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py @@ -6,7 +6,7 @@ import csv, os, pytest, re, inspect, sys, json import importlib.util import unittest.mock as mock -from typing import cast +from typing import Union, cast, overload from azure.core.credentials import TokenCredential from azure.core.credentials_async import AsyncTokenCredential from azure.core.exceptions import HttpResponseError @@ -83,7 +83,7 @@ def _get_mock_credential(self, is_async: bool): return mock.patch(patch_target, new=mock_credential_class) - def execute(self): + def execute(self, enable_llm_validation: bool = True): """Execute a synchronous sample with proper mocking and environment setup.""" with ( @@ -101,9 +101,10 @@ def execute(self): raise ImportError(f"Could not load module {self.spec.name} from {self.sample_path}") self.spec.loader.exec_module(self.module) - self._validate_output() + if enable_llm_validation: + self._validate_output() - async def execute_async(self): + async def execute_async(self, enable_llm_validation: bool = True): """Execute an asynchronous sample with proper mocking and environment setup.""" with ( @@ -122,7 +123,8 @@ async def execute_async(self): self.spec.loader.exec_module(self.module) await self.module.main() - await self._validate_output_async() + if enable_llm_validation: + await self._validate_output_async() def _get_validation_request_params(self) -> dict: """Get common parameters for validation request.""" @@ -223,58 +225,73 @@ def _wrapper_sync(test_class, sample_path, **kwargs): return _wrapper_sync -def _get_tools_sample_paths(): - # Get the path to the samples folder - current_dir = os.path.dirname(os.path.abspath(__file__)) - samples_folder_path = os.path.normpath(os.path.join(current_dir, os.pardir, os.pardir)) - tools_folder = os.path.join(samples_folder_path, "samples", "agents", "tools") - - # Whitelist of samples to test - tools_samples_to_test = [ - "sample_agent_ai_search.py", - "sample_agent_code_interpreter.py", - "sample_agent_file_search.py", - "sample_agent_file_search_in_stream.py", - "sample_agent_function_tool.py", - "sample_agent_image_generation.py", - "sample_agent_mcp.py", - "sample_agent_openapi.py", - "sample_agent_sharepoint.py", - "sample_agent_web_search.py", - ] - samples = [] +@overload +def _get_sample_paths(sub_folder: str, *, samples_to_test: list[str]) -> list: + """Get sample paths for testing (whitelist mode). - for filename in tools_samples_to_test: - sample_path = os.path.join(tools_folder, filename) - if os.path.exists(sample_path): - test_id = filename.replace(".py", "") - samples.append(pytest.param(sample_path, id=test_id)) + Args: + sub_folder: Relative path to the samples subfolder (e.g., "agents/tools") + samples_to_test: Whitelist of sample filenames to include - return samples + Returns: + List of pytest.param objects with sample paths and test IDs + """ + ... + + +@overload +def _get_sample_paths(sub_folder: str, *, samples_to_skip: list[str], is_async: bool) -> list: + """Get sample paths for testing (blacklist mode). + + Args: + sub_folder: Relative path to the samples subfolder (e.g., "agents/tools") + samples_to_skip: Blacklist of sample filenames to exclude (auto-discovers all samples) + is_async: Whether to filter for async samples (_async.py suffix) + + Returns: + List of pytest.param objects with sample paths and test IDs + """ + ... -def _get_tools_sample_paths_async(): +def _get_sample_paths( + sub_folder: str, + *, + samples_to_skip: Union[list[str], None] = None, + is_async: Union[bool, None] = None, + samples_to_test: Union[list[str], None] = None, +) -> list: # Get the path to the samples folder current_dir = os.path.dirname(os.path.abspath(__file__)) samples_folder_path = os.path.normpath(os.path.join(current_dir, os.pardir, os.pardir)) - tools_folder = os.path.join(samples_folder_path, "samples", "agents", "tools") - - # Whitelist of async samples to test - tools_samples_to_test_async = [ - "sample_agent_code_interpreter_async.py", - "sample_agent_computer_use_async.py", - "sample_agent_file_search_in_stream_async.py", - "sample_agent_function_tool_async.py", - "sample_agent_image_generation_async.py", - "sample_agent_mcp_async.py", - ] - samples = [] + target_folder = os.path.join(samples_folder_path, "samples", *sub_folder.split("/")) + + if not os.path.exists(target_folder): + raise ValueError(f"Target folder does not exist: {target_folder}") + + # Discover all sample files in the folder + all_files = [f for f in os.listdir(target_folder) if f.startswith("sample_") and f.endswith(".py")] + + # Filter by async suffix only when using samples_to_skip + if samples_to_skip is not None and is_async is not None: + if is_async: + all_files = [f for f in all_files if f.endswith("_async.py")] + else: + all_files = [f for f in all_files if not f.endswith("_async.py")] - for filename in tools_samples_to_test_async: - sample_path = os.path.join(tools_folder, filename) - if os.path.exists(sample_path): - test_id = filename.replace(".py", "") - samples.append(pytest.param(sample_path, id=test_id)) + # Apply whitelist or blacklist + if samples_to_test is not None: + files_to_test = [f for f in all_files if f in samples_to_test] + else: # samples_to_skip is not None + assert samples_to_skip is not None + files_to_test = [f for f in all_files if f not in samples_to_skip] + + # Create pytest.param objects + samples = [] + for filename in sorted(files_to_test): + sample_path = os.path.join(target_folder, filename) + test_id = filename.replace(".py", "") + samples.append(pytest.param(sample_path, id=test_id)) return samples @@ -282,7 +299,23 @@ def _get_tools_sample_paths_async(): class TestSamples(AzureRecordedTestCase): @servicePreparer() - @pytest.mark.parametrize("sample_path", _get_tools_sample_paths()) + @pytest.mark.parametrize( + "sample_path", + _get_sample_paths( + "agents/tools", + samples_to_skip=[ + "sample_agent_bing_custom_search.py", + "sample_agent_bing_grounding.py", + "sample_agent_browser_automation.py", + "sample_agent_fabric.py", + "sample_agent_mcp_with_project_connection.py", + "sample_agent_memory_search.py", + "sample_agent_openapi_with_project_connection.py", + "sample_agent_to_agent.py", + ], + is_async=False, + ), + ) @SamplePathPasser() @recorded_by_proxy(RecordedTransport.AZURE_CORE, RecordedTransport.HTTPX) def test_agent_tools_samples(self, sample_path: str, **kwargs) -> None: @@ -291,7 +324,17 @@ def test_agent_tools_samples(self, sample_path: str, **kwargs) -> None: executor.execute() @servicePreparer() - @pytest.mark.parametrize("sample_path", _get_tools_sample_paths_async()) + @pytest.mark.parametrize( + "sample_path", + _get_sample_paths( + "agents/tools", + samples_to_skip=[ + "sample_agent_mcp_with_project_connection_async.py", + "sample_agent_memory_search_async.py", + ], + is_async=True, + ), + ) @SamplePathPasser() @recorded_by_proxy_async(RecordedTransport.AZURE_CORE, RecordedTransport.HTTPX) async def test_agent_tools_samples_async(self, sample_path: str, **kwargs) -> None: