chore: Gate Managed Agent evaluation to supported metrics.

PiperOrigin-RevId: 964769846

Author: vertex-sdk-bot

agentplatform/_genai/_evals_common.py

@@ -1564,13 +1564,63 @@ def _build_interaction_id_dataset(
     return types.EvaluationDataset(eval_cases=eval_cases)
 
 
+# Metrics supported for Managed Agent evaluation.
+_MANAGED_AGENT_SUPPORTED_METRICS = frozenset(
+    {
+        "safety_v1",
+        "final_response_quality_v1",
+        "multi_turn_task_success_v1",
+    }
+)
+
+
 def _has_interactions_data_source(
     eval_cases: list[types.EvalCase],
 ) -> bool:
     """Returns True if any EvalCase has interactions_data_source set."""
     return any(case.interactions_data_source is not None for case in eval_cases)
 
 
+def _validate_managed_agent_metrics(
+    agent: Optional[str],
+    metrics: Union[list[types.Metric], list[types.EvaluationRunMetric]],
+) -> None:
+    """Validates metrics are supported for Managed Agent evaluation.
+
+    When the ``agent`` parameter is a Gemini Agent resource name
+    (``projects/{p}/locations/{l}/agents/{id}``), only a subset of
+    metrics are supported for Preview. This function raises ValueError
+    if any unsupported metrics are requested.
+
+    Args:
+        agent: The agent resource name, or None.
+        metrics: The list of metrics to validate. Accepts either
+            ``types.Metric`` objects (which expose a ``name`` attribute)
+            or ``types.EvaluationRunMetric`` objects (which expose a
+            ``metric`` attribute holding the metric name string).
+
+    Raises:
+        ValueError: If any metric is not in the supported set.
+    """
+    if not agent or not _is_gemini_agent_resource(agent):
+        return
+
+    unsupported = []
+    for metric in metrics:
+        # EvaluationRunMetric uses `.metric` (str); types.Metric uses `.name`.
+        name = getattr(metric, "metric", None) or getattr(metric, "name", None)
+        if name:
+            name_lower = name.lower()
+            if name_lower not in _MANAGED_AGENT_SUPPORTED_METRICS:
+                unsupported.append(name_lower)
+    if unsupported:
+        raise ValueError(
+            f"Metrics {unsupported} are not supported for Managed Agent"
+            " evaluation. Supported metrics:"
+            f" {sorted(_MANAGED_AGENT_SUPPORTED_METRICS)}."
+        )
+
+
 def _resolve_interactions_to_eval_cases(
     api_client: BaseApiClient,
     eval_cases: list[types.EvalCase],
@@ -3037,6 +3087,9 @@ def _execute_evaluation(  # type: ignore[no-untyped-def]
 
     resolved_metrics = _resolve_metrics(metrics, api_client)
 
+    # Validate metrics are supported for Managed Agent evaluation.
+    _validate_managed_agent_metrics(agent, resolved_metrics)
+
     evaluation_run_config = _evals_metric_handlers.EvaluationRunConfig(
         evals_module=evals.Evals(api_client_=api_client),
         dataset=processed_eval_dataset,

agentplatform/_genai/evals.py

@@ -3596,6 +3596,10 @@ def create_evaluation_run(
 
         if isinstance(dataset, types.EvaluationDataset):
             _evals_utils._validate_dataset_agent_data(dataset, inference_configs)
+        # Validate metrics are supported for Managed Agent evaluation.
+        # Pass metrics directly; _validate_managed_agent_metrics handles both
+        # EvaluationRunMetric (.metric field) and Metric (.name field).
+        _evals_common._validate_managed_agent_metrics(agent, metrics)
         resolved_dataset = _evals_common._resolve_dataset(
             self._api_client, dataset, dest, parsed_agent_info
         )
@@ -6095,6 +6099,10 @@ async def create_evaluation_run(
 
         if isinstance(dataset, types.EvaluationDataset):
             _evals_utils._validate_dataset_agent_data(dataset, inference_configs)
+        # Validate metrics are supported for Managed Agent evaluation.
+        # Pass metrics directly; _validate_managed_agent_metrics handles both
+        # EvaluationRunMetric (.metric field) and Metric (.name field).
+        _evals_common._validate_managed_agent_metrics(agent, metrics)
         resolved_dataset = _evals_common._resolve_dataset(
             self._api_client, dataset, dest, parsed_agent_info
         )

tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py

@@ -952,7 +952,7 @@ def test_create_eval_run_with_gemini_agent(client):
         display_name="test_gemini_agent",
         dataset=types.EvaluationRunDataSource(evaluation_set=eval_set),
         dest=GCS_DEST,
-        metrics=[GENERAL_QUALITY_METRIC],
+        metrics=[FINAL_RESPONSE_QUALITY_METRIC],
         agent_info=types.evals.AgentInfo(name="gemini-agent"),
         agent=gemini_agent,
         user_simulator_config=types.evals.UserSimulatorConfig(max_turn=3),

tests/unit/agentplatform/genai/test_evals.py

@@ -10608,10 +10608,10 @@ def test_create_evaluation_run_passes_allow_cross_region_model(self):
                 ),
                 metrics=[
                     agentplatform_genai_types.EvaluationRunMetric(
-                        metric="general_quality_v1",
+                        metric="final_response_quality_v1",
                         metric_config=agentplatform_genai_types.UnifiedMetric(
                             predefined_metric_spec=genai_types.PredefinedMetricSpec(
-                                metric_spec_name="general_quality_v1",
+                                metric_spec_name="final_response_quality_v1",
                             )
                         ),
                     )
@@ -10650,10 +10650,10 @@ async def test_create_evaluation_run_async_passes_allow_cross_region_model(self)
                 ),
                 metrics=[
                     agentplatform_genai_types.EvaluationRunMetric(
-                        metric="general_quality_v1",
+                        metric="final_response_quality_v1",
                         metric_config=agentplatform_genai_types.UnifiedMetric(
                             predefined_metric_spec=genai_types.PredefinedMetricSpec(
-                                metric_spec_name="general_quality_v1",
+                                metric_spec_name="final_response_quality_v1",
                             )
                         ),
                     )
@@ -11767,6 +11767,70 @@ def test_non_sandbox_tool_before_user_input_not_merged(self):
         assert len(result.turns) == 2
 
 
+class TestValidateManagedAgentMetrics:
+    """Tests for _validate_managed_agent_metrics."""
+
+    MANAGED_AGENT = "projects/p/locations/global/agents/my-agent"
+    NON_MANAGED_AGENT = "projects/p/locations/global/reasoningEngines/123"
+
+    def _make_metric(self, name):
+        return agentplatform_genai_types.Metric(name=name)
+
+    def test_supported_metric_passes(self):
+        _evals_common._validate_managed_agent_metrics(
+            self.MANAGED_AGENT,
+            [self._make_metric("safety_v1")],
+        )
+
+    def test_multiple_supported_metrics_pass(self):
+        _evals_common._validate_managed_agent_metrics(
+            self.MANAGED_AGENT,
+            [
+                self._make_metric("safety_v1"),
+                self._make_metric("final_response_quality_v1"),
+                self._make_metric("multi_turn_task_success_v1"),
+            ],
+        )
+
+    def test_unsupported_metric_raises(self):
+        with pytest.raises(ValueError, match="not supported for Managed Agent"):
+            _evals_common._validate_managed_agent_metrics(
+                self.MANAGED_AGENT,
+                [self._make_metric("hallucination_v1")],
+            )
+
+    def test_unsupported_metric_lists_supported(self):
+        with pytest.raises(ValueError, match="multi_turn_task_success_v1"):
+            _evals_common._validate_managed_agent_metrics(
+                self.MANAGED_AGENT,
+                [self._make_metric("multi_turn_trajectory_quality_v1")],
+            )
+
+    def test_non_managed_agent_allows_any_metric(self):
+        # Non-Managed Agent (reasoning engine) should allow any metric.
+        _evals_common._validate_managed_agent_metrics(
+            self.NON_MANAGED_AGENT,
+            [self._make_metric("hallucination_v1")],
+        )
+
+    def test_no_agent_allows_any_metric(self):
+        # No agent parameter should allow any metric.
+        _evals_common._validate_managed_agent_metrics(
+            None,
+            [self._make_metric("hallucination_v1")],
+        )
+
+    def test_mixed_supported_and_unsupported_raises(self):
+        with pytest.raises(ValueError, match="not supported"):
+            _evals_common._validate_managed_agent_metrics(
+                self.MANAGED_AGENT,
+                [
+                    self._make_metric("safety_v1"),
+                    self._make_metric("hallucination_v1"),
+                ],
+            )
+
+
 class TestMergeTextPartsInAgentData:
     """Tests for _merge_text_parts_in_agent_data."""