From 0428fc0428be34a642127c4d11b7417e7501f682 Mon Sep 17 00:00:00 2001 From: A Vertex SDK engineer Date: Thu, 30 Jul 2026 17:13:10 -0700 Subject: [PATCH] chore: Gate Managed Agent evaluation to supported metrics. PiperOrigin-RevId: 956818196 --- agentplatform/_genai/_evals_common.py | 47 ++++++++++++++ agentplatform/_genai/evals.py | 10 +++ .../replays/test_create_evaluation_run.py | 4 +- tests/unit/agentplatform/genai/test_evals.py | 64 +++++++++++++++++++ 4 files changed, 123 insertions(+), 2 deletions(-) diff --git a/agentplatform/_genai/_evals_common.py b/agentplatform/_genai/_evals_common.py index c6eb338c04..6fc0d55f47 100644 --- a/agentplatform/_genai/_evals_common.py +++ b/agentplatform/_genai/_evals_common.py @@ -1564,6 +1564,14 @@ def _build_interaction_id_dataset( return types.EvaluationDataset(eval_cases=eval_cases) +# Metrics supported for Managed Agent evaluation. +_MANAGED_AGENT_SUPPORTED_METRICS = frozenset({ + "safety_v1", + "final_response_quality_v1", + "multi_turn_task_success_v1", +}) + + def _has_interactions_data_source( eval_cases: list[types.EvalCase], ) -> bool: @@ -1571,6 +1579,42 @@ def _has_interactions_data_source( return any(case.interactions_data_source is not None for case in eval_cases) +def _validate_managed_agent_metrics( + agent: Optional[str], + metrics: list[types.Metric], +) -> None: + """Validates metrics are supported for Managed Agent evaluation. + + When the ``agent`` parameter is a Gemini Agent resource name + (``projects/{p}/locations/{l}/agents/{id}``), only a subset of + metrics are supported for Preview. This function raises ValueError + if any unsupported metrics are requested. + + Args: + agent: The agent resource name, or None. + metrics: The list of metrics to validate. + + Raises: + ValueError: If any metric is not in the supported set. + """ + if not agent or not _is_gemini_agent_resource(agent): + return + + unsupported = [] + for metric in metrics: + name = getattr(metric, "name", None) + if name: + name_lower = name.lower() + if name_lower not in _MANAGED_AGENT_SUPPORTED_METRICS: + unsupported.append(name_lower) + if unsupported: + raise ValueError( + f"Metrics {unsupported} are not supported for Managed Agent" + " evaluation. Supported metrics:" + f" {sorted(_MANAGED_AGENT_SUPPORTED_METRICS)}." + ) + + def _resolve_interactions_to_eval_cases( api_client: BaseApiClient, eval_cases: list[types.EvalCase], @@ -3037,6 +3081,9 @@ def _execute_evaluation( # type: ignore[no-untyped-def] resolved_metrics = _resolve_metrics(metrics, api_client) + # Validate metrics are supported for Managed Agent evaluation. + _validate_managed_agent_metrics(agent, resolved_metrics) + evaluation_run_config = _evals_metric_handlers.EvaluationRunConfig( evals_module=evals.Evals(api_client_=api_client), dataset=processed_eval_dataset, diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index 4b600f7054..40a0a61fd2 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -3382,6 +3382,11 @@ def create_evaluation_run( if isinstance(dataset, types.EvaluationDataset): _evals_utils._validate_dataset_agent_data(dataset, inference_configs) + # Validate metrics are supported for Managed Agent evaluation. + _evals_common._validate_managed_agent_metrics( + agent, + _evals_common._resolve_metrics(metrics, self._api_client), + ) resolved_dataset = _evals_common._resolve_dataset( self._api_client, dataset, dest, parsed_agent_info ) @@ -5661,6 +5666,11 @@ async def create_evaluation_run( if isinstance(dataset, types.EvaluationDataset): _evals_utils._validate_dataset_agent_data(dataset, inference_configs) + # Validate metrics are supported for Managed Agent evaluation. + _evals_common._validate_managed_agent_metrics( + agent, + _evals_common._resolve_metrics(metrics, self._api_client), + ) resolved_dataset = _evals_common._resolve_dataset( self._api_client, dataset, dest, parsed_agent_info ) diff --git a/tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py b/tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py index 738a0b25b2..958d1cf70d 100644 --- a/tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py +++ b/tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py @@ -744,7 +744,7 @@ def test_create_eval_run_with_interactions_data_source(mock_uuid4, client): display_name="test_interactions_data_source", dataset=types.EvaluationDataset(eval_cases=[eval_case]), dest=GCS_DEST, - metrics=[GENERAL_QUALITY_METRIC], + metrics=[FINAL_RESPONSE_QUALITY_METRIC], ) assert isinstance(evaluation_run, types.EvaluationRun) assert evaluation_run.state == types.EvaluationRunState.PENDING @@ -927,7 +927,7 @@ def test_create_eval_run_with_gemini_agent(client): display_name="test_gemini_agent", dataset=types.EvaluationRunDataSource(evaluation_set=eval_set), dest=GCS_DEST, - metrics=[GENERAL_QUALITY_METRIC], + metrics=[FINAL_RESPONSE_QUALITY_METRIC], agent_info=types.evals.AgentInfo(name="gemini-agent"), agent=gemini_agent, user_simulator_config=types.evals.UserSimulatorConfig(max_turn=3), diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 39c5224f36..9d4d9fa8cd 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -11767,6 +11767,70 @@ def test_non_sandbox_tool_before_user_input_not_merged(self): assert len(result.turns) == 2 +class TestValidateManagedAgentMetrics: + """Tests for _validate_managed_agent_metrics.""" + + MANAGED_AGENT = "projects/p/locations/global/agents/my-agent" + NON_MANAGED_AGENT = "projects/p/locations/global/reasoningEngines/123" + + def _make_metric(self, name): + return agentplatform_genai_types.Metric(name=name) + + def test_supported_metric_passes(self): + _evals_common._validate_managed_agent_metrics( + self.MANAGED_AGENT, + [self._make_metric("safety_v1")], + ) + + def test_multiple_supported_metrics_pass(self): + _evals_common._validate_managed_agent_metrics( + self.MANAGED_AGENT, + [ + self._make_metric("safety_v1"), + self._make_metric("final_response_quality_v1"), + self._make_metric("multi_turn_task_success_v1"), + ], + ) + + def test_unsupported_metric_raises(self): + with pytest.raises(ValueError, match="not supported for Managed Agent"): + _evals_common._validate_managed_agent_metrics( + self.MANAGED_AGENT, + [self._make_metric("hallucination_v1")], + ) + + def test_unsupported_metric_lists_supported(self): + with pytest.raises(ValueError, match="multi_turn_task_success_v1"): + _evals_common._validate_managed_agent_metrics( + self.MANAGED_AGENT, + [self._make_metric("multi_turn_trajectory_quality_v1")], + ) + + def test_non_managed_agent_allows_any_metric(self): + # Non-Managed Agent (reasoning engine) should allow any metric. + _evals_common._validate_managed_agent_metrics( + self.NON_MANAGED_AGENT, + [self._make_metric("hallucination_v1")], + ) + + def test_no_agent_allows_any_metric(self): + # No agent parameter should allow any metric. + _evals_common._validate_managed_agent_metrics( + None, + [self._make_metric("hallucination_v1")], + ) + + def test_mixed_supported_and_unsupported_raises(self): + with pytest.raises(ValueError, match="not supported"): + _evals_common._validate_managed_agent_metrics( + self.MANAGED_AGENT, + [ + self._make_metric("safety_v1"), + self._make_metric("hallucination_v1"), + ], + ) + + class TestMergeTextPartsInAgentData: """Tests for _merge_text_parts_in_agent_data."""