Howie/validate sample by agent (#44345)

howieleung · web-flow · commit 7772fd6ecd4d · 2025-12-11T06:09:37.000-08:00
Use LLM to validate print content in agents tools samples.
diff --git a/sdk/ai/azure-ai-projects/assets.json b/sdk/ai/azure-ai-projects/assets.json
@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/ai/azure-ai-projects",
-  "Tag": "python/ai/azure-ai-projects_314598932e"
+  "Tag": "python/ai/azure-ai-projects_93d1dc0fe7"
 }
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/computer_use_util.py b/sdk/ai/azure-ai-projects/samples/agents/tools/computer_use_util.py
@@ -159,4 +159,4 @@ def print_final_output(response):
                 final_output += getattr(part, "text", None) or getattr(part, "refusal", None) or "" + "\n"
 
     print(f"Final status: {response.status}")
-    print(f"==> Result: {final_output.strip()}")
+    print(f"Final result: {final_output.strip()}")
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_ai_search.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_ai_search.py
@@ -108,7 +108,7 @@
                             )
         elif event.type == "response.completed":
             print(f"\nFollow-up completed!")
-            print(f"==> Result: {event.response.output_text}")
+            print(f"Agent response: {event.response.output_text}")
 
     print("\nCleaning up...")
     project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version)
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_code_interpreter.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_code_interpreter.py
@@ -108,7 +108,6 @@
         file_path = os.path.join(tempfile.gettempdir(), filename)
         with open(file_path, "wb") as f:
             f.write(file_content.read())
-        # Print result (should contain "file")
-        print(f"==> Result: file, {file_path} downloaded successfully.")
+        print(f"File downloaded successfully: {file_path}")
     else:
         print("No file generated in response")
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_code_interpreter_async.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_code_interpreter_async.py
@@ -112,8 +112,7 @@ async def main() -> None:
                 file_path = os.path.join(tempfile.gettempdir(), filename)
                 with open(file_path, "wb") as f:
                     f.write(file_content.read())
-                # Print result (should contain "file")
-                print(f"==> Result: file, {file_path} downloaded successfully.")
+                print(f"File downloaded successfully: {file_path}")
             else:
                 print("No file generated in response")
 
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search.py
@@ -78,7 +78,7 @@
         input="Tell me about Contoso products",
         extra_body={"agent": {"name": agent.name, "type": "agent_reference"}},
     )
-    print(f"==> Result: {response.output_text}")
+    print(f"Agent response: {response.output_text}")
     print("\nCleaning up...")
     project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version)
     print("Agent deleted")
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search_in_stream.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search_in_stream.py
@@ -140,7 +140,7 @@
                             print(f"File Citation - Filename: {annotation.filename}, File ID: {annotation.file_id}")
         elif event.type == "response.completed":
             print(f"\nFollow-up completed!")
-            print(f"==> Result: {event.response.output_text}")
+            print(f"Agent response: {event.response.output_text}")
 
     # Clean up resources
     print("\n" + "=" * 60)
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search_in_stream_async.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_file_search_in_stream_async.py
@@ -145,7 +145,7 @@ async def main() -> None:
                                 print(f"File Citation - Filename: {annotation.filename}, File ID: {annotation.file_id}")
             elif event.type == "response.completed":
                 print(f"\nFollow-up completed!")
-                print(f"==> Result: {event.response.output_text}")
+                print(f"Agent response: {event.response.output_text}")
 
         # Clean up resources
         print("\n" + "=" * 60)
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_function_tool.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_function_tool.py
@@ -108,8 +108,7 @@ def get_horoscope(sign: str) -> str:
         extra_body={"agent": {"name": agent.name, "type": "agent_reference"}},
     )
 
-    # Print result (should contain "Tuesday")
-    print(f"==> Result: {response.output_text}")
+    print(f"Agent response: {response.output_text}")
 
     print("\nCleaning up...")
     project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version)
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_function_tool_async.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_function_tool_async.py
@@ -109,8 +109,7 @@ async def main():
             extra_body={"agent": {"name": agent.name, "type": "agent_reference"}},
         )
 
-        # Print result (should contain "Tuesday")
-        print(f"==> Result: {response.output_text}")
+        print(f"Agent response: {response.output_text}")
 
 
 if __name__ == "__main__":
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_image_generation.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_image_generation.py
@@ -96,5 +96,4 @@
             f.write(base64.b64decode(image_data[0]))
 
         # [END download_image]
-        # Print result (should contain "file")
-        print(f"==> Result: Image downloaded and saved to file: {file_path}")
+        print(f"Image saved to: {file_path}")
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_image_generation_async.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_image_generation_async.py
@@ -91,8 +91,7 @@ async def main():
             with open(file_path, "wb") as f:
                 f.write(base64.b64decode(image_data[0]))
 
-            # Print result (should contain "file")
-            print(f"==> Result: Image downloaded and saved to file: {file_path}")
+            print(f"Image saved to: {file_path}")
 
 
 if __name__ == "__main__":
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp.py
@@ -95,8 +95,7 @@
         extra_body={"agent": {"name": agent.name, "type": "agent_reference"}},
     )
 
-    # Print result (should contain "Azure")
-    print(f"==> Result: {response.output_text}")
+    print(f"Agent response: {response.output_text}")
 
     # Clean up resources by deleting the agent version
     # This prevents accumulation of unused agent versions in your project
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp_async.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp_async.py
@@ -99,8 +99,7 @@ async def main():
             extra_body={"agent": {"name": agent.name, "type": "agent_reference"}},
         )
 
-        # Print result (should contain "Azure")
-        print(f"==> Result: {response.output_text}")
+        print(f"Agent response: {response.output_text}")
 
         # Clean up resources by deleting the agent version
         # This prevents accumulation of unused agent versions in your project
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_openapi.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_openapi.py
@@ -76,8 +76,7 @@
         input="Use the OpenAPI tool to print out, what is the weather in Seattle, WA today.",
         extra_body={"agent": {"name": agent.name, "type": "agent_reference"}},
     )
-    # Print result (should contain "\u00b0F")
-    print(f"==> Result: {response.output_text}")
+    print(f"Agent response: {response.output_text}")
 
     print("\nCleaning up...")
     project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version)
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_sharepoint.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_sharepoint.py
@@ -101,7 +101,7 @@
                             )
         elif event.type == "response.completed":
             print(f"\nFollow-up completed!")
-            print(f"==> Result: {event.response.output_text}")
+            print(f"Agent response: {event.response.output_text}")
 
     print("Cleaning up...")
     project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version)
diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_web_search.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_web_search.py
@@ -65,7 +65,7 @@
         input="Show me the latest London Underground service updates",
         extra_body={"agent": {"name": agent.name, "type": "agent_reference"}},
     )
-    print(f"==> Result: {response.output_text}")
+    print(f"Agent response: {response.output_text}")
 
     print("\nCleaning up...")
     project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version)
diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py
@@ -18,6 +18,7 @@
 from azure.ai.projects.models import PromptAgentDefinition, CodeInterpreterTool, CodeInterpreterToolAuto, FunctionTool
 from openai.types.responses.response_input_param import FunctionCallOutput, ResponseInputParam
 
+
 class TestAgentCodeInterpreterAndFunction(TestBase):
     """Tests for agents using Code Interpreter + Function Tool combination."""
 
diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py
@@ -328,7 +328,8 @@ def calculate_sum(numbers):
                 # Verify findings discuss the code (proves File Search was used)
                 findings_lower = arguments["findings"].lower()
                 assert any(
-                    keyword in findings_lower for keyword in ["sum", "calculate", "function", "numbers", "list", "return"]
+                    keyword in findings_lower
+                    for keyword in ["sum", "calculate", "function", "numbers", "list", "return"]
                 ), f"Expected findings to discuss the code content. Got: {arguments['findings'][:100]}"
 
                 input_list.append(
diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py
@@ -19,6 +19,7 @@
 # The tests in this file rely on an existing Azure AI Search project connection that has been populated with the following document:
 # https://arxiv.org/pdf/2508.03680
 
+
 class TestAgentAISearch(TestBase):
 
     # Test questions with expected answers
diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py
@@ -20,6 +20,7 @@
 # The tests in this file rely on an existing Azure AI Search project connection that has been populated with the following document:
 # https://arxiv.org/pdf/2508.03680
 
+
 class TestAgentAISearchAsync(TestBase):
 
     # Test questions with expected answers
diff --git a/sdk/ai/azure-ai-projects/tests/conftest.py b/sdk/ai/azure-ai-projects/tests/conftest.py
@@ -19,7 +19,7 @@
     add_general_regex_sanitizer,
     add_body_key_sanitizer,
     add_remove_header_sanitizer,
-    add_general_string_sanitizer,
+    add_body_regex_sanitizer,
 )
 
 if not load_dotenv(find_dotenv(), override=True):
@@ -120,6 +120,14 @@ def sanitize_url_paths():
         value="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/connections/connector-name",
     )
 
+    # Sanitize print output from sample validation to prevent replay failures when print statements change
+    # Only targets the validation Responses API call by matching the unique input prefix
+    add_body_key_sanitizer(
+        json_path="$.input",
+        value="sanitized-print-output",
+        regex=r"print contents array = .*",
+    )
+
     # Remove Stainless headers from OpenAI client requests, since they include platform and OS specific info, which we can't have in recorded requests.
     # Here is an example of all the `x-stainless` headers from a Responses call:
     #   x-stainless-arch: other:amd64
diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_samples.py

Original file line number	Diff line number	Diff line change
`@@ -2,5 +2,5 @@`
`2`	`2`	`"AssetsRepo": "Azure/azure-sdk-assets",`
`3`	`3`	`"AssetsRepoPrefixPath": "python",`
`4`	`4`	`"TagPrefix": "python/ai/azure-ai-projects",`
`5`		`- "Tag": "python/ai/azure-ai-projects_314598932e"`
	`5`	`+ "Tag": "python/ai/azure-ai-projects_93d1dc0fe7"`
`6`	`6`	`}`
Original file line number	Diff line number	Diff line change
`@@ -108,7 +108,7 @@`
`108`	`108`	`)`
`109`	`109`	`elif event.type == "response.completed":`
`110`	`110`	`print(f"\nFollow-up completed!")`
`111`		`- print(f"==> Result: {event.response.output_text}")`
	`111`	`+ print(f"Agent response: {event.response.output_text}")`
`112`	`112`
`113`	`113`	`print("\nCleaning up...")`
`114`	`114`	`project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version)`
Original file line number	Diff line number	Diff line change
`@@ -78,7 +78,7 @@`
`78`	`78`	`input="Tell me about Contoso products",`
`79`	`79`	`extra_body={"agent": {"name": agent.name, "type": "agent_reference"}},`
`80`	`80`	`)`
`81`		`- print(f"==> Result: {response.output_text}")`
	`81`	`+ print(f"Agent response: {response.output_text}")`
`82`	`82`	`print("\nCleaning up...")`
`83`	`83`	`project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version)`
`84`	`84`	`print("Agent deleted")`
Original file line number	Diff line number	Diff line change
`@@ -108,8 +108,7 @@ def get_horoscope(sign: str) -> str:`
`108`	`108`	`extra_body={"agent": {"name": agent.name, "type": "agent_reference"}},`
`109`	`109`	`)`
`110`	`110`
`111`		`- # Print result (should contain "Tuesday")`
`112`		`- print(f"==> Result: {response.output_text}")`
	`111`	`+ print(f"Agent response: {response.output_text}")`
`113`	`112`
`114`	`113`	`print("\nCleaning up...")`
`115`	`114`	`project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version)`
Original file line number	Diff line number	Diff line change
`@@ -109,8 +109,7 @@ async def main():`
`109`	`109`	`extra_body={"agent": {"name": agent.name, "type": "agent_reference"}},`
`110`	`110`	`)`
`111`	`111`
`112`		`- # Print result (should contain "Tuesday")`
`113`		`- print(f"==> Result: {response.output_text}")`
	`112`	`+ print(f"Agent response: {response.output_text}")`
`114`	`113`
`115`	`114`
`116`	`115`	`if __name__ == "__main__":`