diff --git a/agentplatform/_genai/_evals_common.py b/agentplatform/_genai/_evals_common.py index 4929d716ad..8a41bf0a25 100644 --- a/agentplatform/_genai/_evals_common.py +++ b/agentplatform/_genai/_evals_common.py @@ -2334,10 +2334,10 @@ async def _execute_local_agent_run_with_retry_async( if "session_inputs" in row.index and row.get("session_inputs") is not None: session_inputs = _get_session_inputs(row) user_id = session_inputs.user_id or str(uuid.uuid4()) - app_name = session_inputs.app_name or "local agent run" + app_name = session_inputs.app_name or "local_agent_run" else: user_id = str(uuid.uuid4()) - app_name = "local agent run" + app_name = "local_agent_run" session_id = str(uuid.uuid4()) session_service = InMemorySessionService() diff --git a/agentplatform/_genai/types/evals.py b/agentplatform/_genai/types/evals.py index b0baf1421a..d264a58c5e 100644 --- a/agentplatform/_genai/types/evals.py +++ b/agentplatform/_genai/types/evals.py @@ -84,15 +84,17 @@ def _get_tool_declarations_from_agent(agent: Any) -> genai_types.ToolListUnion: """ tool_declarations: genai_types.ToolListUnion = [] for tool in agent.tools: - # ADK tools (e.g. AgentTool) provide their own declaration via - # _get_declaration(). Use it when available to avoid calling - # typing.get_type_hints() on tool instances whose classes use - # `from __future__ import annotations`, which causes NameError. + # ADK tools (e.g. AgentTool, VertexAiSearchTool) own their declaration + # via _get_declaration(). A None result means the tool has no function + # declaration (e.g. built-in retrieval tools). In both cases, skip the + # plain-callable path, which calls typing.get_type_hints() on the + # instance and raises NameError for classes using + # `from __future__ import annotations`. if hasattr(tool, "_get_declaration") and callable(tool._get_declaration): declaration = tool._get_declaration() if declaration is not None: tool_declarations.append({"function_declarations": [declaration]}) - continue + continue tool_declarations.append( { diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 33f63a1a88..6570f9578f 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -3680,7 +3680,7 @@ def run_async_side_effect(*args, **kwargs): assert mock_session_service.call_count == 2 mock_runner.assert_called_with( agent=mock_agent_instance, - app_name="local agent run", + app_name="local_agent_run", session_service=mock_session_service.return_value, ) assert mock_runner.call_count == 2 @@ -3795,6 +3795,10 @@ def run_async_side_effect(*args, **kwargs): assert inference_result.candidate_name == "mock_agent" assert inference_result.gcs_source is None + def test_local_agent_run_default_app_name_is_valid(self): + """Default app_name must satisfy ADK 2.x's app name validation regex.""" + assert re.fullmatch(r"[a-zA-Z][a-zA-Z0-9_-]*", "local_agent_run") + def test_run_inference_with_litellm_string_prompt_format( self, mock_api_client_fixture, @@ -5850,8 +5854,8 @@ def my_plain_tool(query: str) -> str: ] mock_from_callable.assert_called_once_with(callable=my_plain_tool) - def test_load_from_agent_with_none_declaration_falls_back(self): - """Tests that tools returning None from _get_declaration fall back to from_callable.""" + def test_load_from_agent_with_none_declaration_is_skipped(self): + """Tools whose _get_declaration() returns None are skipped, not introspected.""" mock_tool = mock.Mock() mock_tool._get_declaration = mock.Mock(return_value=None) mock_tool.__name__ = "mock_tool" @@ -5867,19 +5871,38 @@ def test_load_from_agent_with_none_declaration_falls_back(self): with mock.patch.object( genai_types.FunctionDeclaration, "from_callable_with_api_option" ) as mock_from_callable: - mock_callable_declaration = mock.Mock(spec=genai_types.FunctionDeclaration) - mock_from_callable.return_value = mock_callable_declaration - agent_info = agentplatform_genai_types.evals.AgentInfo.load_from_agent( agent=mock_agent, ) - assert len(agent_info.agents["mock_agent"].tools) == 1 - assert agent_info.agents["mock_agent"].tools[0].function_declarations == [ - mock_callable_declaration - ] + assert agent_info.agents["mock_agent"].tools == [] mock_tool._get_declaration.assert_called_once() - mock_from_callable.assert_called_once_with(callable=mock_tool) + mock_from_callable.assert_not_called() + + def test_load_from_agent_none_declaration_skips_get_type_hints(self): + """A None-returning native tool must not trigger get_type_hints (NameError repro).""" + + class _BuiltinTool: + def _get_declaration(self): + return None + + def run(self, query: "Optional[str]" = None): # noqa: F821 + return query + + builtin_tool = _BuiltinTool() + + mock_agent = mock.Mock() + mock_agent.name = "mock_agent" + mock_agent.instruction = "mock instruction" + mock_agent.description = "mock description" + mock_agent.tools = [builtin_tool] + mock_agent.sub_agents = [] + + agent_info = agentplatform_genai_types.evals.AgentInfo.load_from_agent( + agent=mock_agent, + ) + + assert agent_info.agents["mock_agent"].tools == [] class TestValidateDatasetAgentData: diff --git a/vertexai/_genai/_evals_common.py b/vertexai/_genai/_evals_common.py index 73e3775f1f..a33e923049 100644 --- a/vertexai/_genai/_evals_common.py +++ b/vertexai/_genai/_evals_common.py @@ -2334,10 +2334,10 @@ async def _execute_local_agent_run_with_retry_async( if "session_inputs" in row.index and row.get("session_inputs") is not None: session_inputs = _get_session_inputs(row) user_id = session_inputs.user_id or str(uuid.uuid4()) - app_name = session_inputs.app_name or "local agent run" + app_name = session_inputs.app_name or "local_agent_run" else: user_id = str(uuid.uuid4()) - app_name = "local agent run" + app_name = "local_agent_run" session_id = str(uuid.uuid4()) session_service = InMemorySessionService() diff --git a/vertexai/_genai/types/evals.py b/vertexai/_genai/types/evals.py index ae45e08433..fefd7d3c0e 100644 --- a/vertexai/_genai/types/evals.py +++ b/vertexai/_genai/types/evals.py @@ -84,15 +84,17 @@ def _get_tool_declarations_from_agent(agent: Any) -> genai_types.ToolListUnion: """ tool_declarations: genai_types.ToolListUnion = [] for tool in agent.tools: - # ADK tools (e.g. AgentTool) provide their own declaration via - # _get_declaration(). Use it when available to avoid calling - # typing.get_type_hints() on tool instances whose classes use - # `from __future__ import annotations`, which causes NameError. + # ADK tools (e.g. AgentTool, VertexAiSearchTool) own their declaration + # via _get_declaration(). A None result means the tool has no function + # declaration (e.g. built-in retrieval tools). In both cases, skip the + # plain-callable path, which calls typing.get_type_hints() on the + # instance and raises NameError for classes using + # `from __future__ import annotations`. if hasattr(tool, "_get_declaration") and callable(tool._get_declaration): declaration = tool._get_declaration() if declaration is not None: tool_declarations.append({"function_declarations": [declaration]}) - continue + continue tool_declarations.append( {