Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 14 additions & 31 deletions agentplatform/_genai/_evals_builtin_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@

**If the server catalog changes, this SDK-side copy must be updated to match.**

This module also provides sandbox-detection helpers
(``SANDBOX_TOOL_NAMES``, ``is_sandbox_only_turn``) used by the display
path (``_evals_common._interaction_dict_to_agent_data``).
Sandbox orchestration tools (``provision_sandbox``, ``load_sandbox``) are
intentionally excluded from the tool catalog. They are infrastructure
initialization, not user-facing agent capabilities.
"""

from typing import Any, Optional
Expand Down Expand Up @@ -76,26 +76,12 @@
}


# Sandbox-environment orchestration tool declarations.
#
# Source of truth: interaction_converter.py, _SANDBOX_FUNCTION_DECLARATIONS
SANDBOX_DECLARATIONS: list[genai_types.FunctionDeclaration] = [
genai_types.FunctionDeclaration(
name="provision_sandbox",
description="Provisions a sandbox environment.",
),
genai_types.FunctionDeclaration(
name="load_sandbox",
description="Loads a previously provisioned sandbox environment.",
),
]


# Names of sandbox orchestration tools, derived from ``SANDBOX_DECLARATIONS``
# so there is a single source of truth.
SANDBOX_TOOL_NAMES: frozenset[str] = frozenset(
decl.name for decl in SANDBOX_DECLARATIONS if decl.name
)
# Sandbox-environment orchestration tools.
# Source of truth: interaction_converter.py, _SANDBOX_TOOL_NAMES
SANDBOX_TOOL_NAMES: frozenset[str] = frozenset({
"provision_sandbox",
"load_sandbox",
})


def is_sandbox_only_turn(
Expand Down Expand Up @@ -146,7 +132,6 @@ def is_sandbox_only_turn(

def agent_tools_to_config_tools(
agent_tools: Optional[list[Any]],
has_environment: bool = False,
) -> Optional[list[genai_types.Tool]]:
"""Maps Gemini Agents API tools to ``genai_types.Tool`` for display.

Expand All @@ -163,18 +148,19 @@ def agent_tools_to_config_tools(
* ``mcp_server`` is represented as a named declaration with a
human-readable label.
* Tools carrying explicit ``function_declarations`` are passed through.
* When ``has_environment`` is True, sandbox orchestration tools
(``provision_sandbox``, ``load_sandbox``) are appended.

Sandbox orchestration tools (``provision_sandbox``, ``load_sandbox``)
are intentionally excluded. They are infrastructure initialization,
not user-facing capabilities.

Args:
agent_tools: The ``tools`` list from a fetched Gemini agent dict.
has_environment: Whether the agent has a sandbox environment configured.

Returns:
A list of ``genai_types.Tool``, or ``None`` if there are no mappable
tools.
"""
if not agent_tools and not has_environment:
if not agent_tools:
return None
tools: list[genai_types.Tool] = []
for tool in agent_tools or []:
Expand Down Expand Up @@ -218,7 +204,4 @@ def agent_tools_to_config_tools(
elif remainder:
tools.append(genai_types.Tool.model_validate(remainder))

if has_environment:
tools.append(genai_types.Tool(function_declarations=list(SANDBOX_DECLARATIONS)))

return tools or None
6 changes: 1 addition & 5 deletions agentplatform/_genai/_evals_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -903,12 +903,8 @@ def _fetch_agent_config_dict(
instruction = agent_dict.get("system_instruction") or None
description = agent_dict.get("description") or None
agent_type = agent_dict.get("base_agent") or None
has_environment = bool(
agent_dict.get("environment_config")
or agent_dict.get("base_environment")
)
tools = _agent_tools_to_config_tools(
agent_dict.get("tools"), has_environment=has_environment
agent_dict.get("tools")
)
except Exception as e: # pylint: disable=broad-exception-caught
logger.warning(
Expand Down
51 changes: 26 additions & 25 deletions tests/unit/agentplatform/genai/test_evals.py
Original file line number Diff line number Diff line change
Expand Up @@ -12010,8 +12010,8 @@ def test_filesystem_expands_to_file_tools(self):
"move_file",
}

def test_environment_adds_sandbox_tools(self):
"""When agent has environment_config, sandbox tools are appended."""
def test_environment_does_not_add_sandbox_tools(self):
"""Sandbox tools should NOT appear even when agent has environment."""
agent_json = {
"tools": [{"type": "code_execution"}],
"environment_config": {"some_field": "value"},
Expand All @@ -12022,17 +12022,17 @@ def test_environment_adds_sandbox_tools(self):
mock_api_client,
"projects/p/locations/l/agents/a",
)
# code_execution + sandbox tool
assert len(result.tools) == 2
# Only code_execution, no sandbox tools.
assert len(result.tools) == 1
all_decl_names = {
fd.name
for t in result.tools
if t.function_declarations
for fd in t.function_declarations
}
assert "run_command" in all_decl_names
assert "provision_sandbox" in all_decl_names
assert "load_sandbox" in all_decl_names
assert "provision_sandbox" not in all_decl_names
assert "load_sandbox" not in all_decl_names

def test_mcp_server_kept_as_named_declaration(self):
"""mcp_server entries are kept as named declarations, not dropped."""
Expand Down Expand Up @@ -12062,17 +12062,23 @@ def test_mcp_server_kept_as_named_declaration(self):
def test_catalog_in_sync_with_server(self):
"""SDK catalog keys and function names match the server-side catalog.

The SDK-side BUILTIN_TOOL_DECLARATIONS and SANDBOX_DECLARATIONS in
_evals_builtin_tools are a display-only copy of the authoritative
server-side catalog in interaction_converter.py. This test imports
both and asserts that tool-type keys and declaration names stay in
sync. If this test fails, update _evals_builtin_tools.py to match.
The SDK-side BUILTIN_TOOL_DECLARATIONS in _evals_builtin_tools is a
display-only copy of the authoritative server-side catalog in
interaction_converter.py. This test imports both and asserts that
tool-type keys and declaration names stay in sync. If this test
fails, update _evals_builtin_tools.py to match.

Sandbox tool declarations (provision_sandbox, load_sandbox) are
intentionally excluded from both the SDK and server AgentConfig
tool catalogs, so no sync check is needed for them.
"""
# pylint: disable=g-import-not-at-top
try:
from cloud.ai.platform.evaluation.utils import interaction_converter
except ImportError:
pytest.skip("interaction_converter not available outside google3")
pytest.skip(
"interaction_converter not available outside google3"
)
# pylint: enable=g-import-not-at-top

# --- Built-in tool types: keys must match ---
Expand All @@ -12090,9 +12096,7 @@ def test_catalog_in_sync_with_server(self):
for tool_type in server_builtin_keys:
server_names = {
fd.name
for fd in interaction_converter._BUILTIN_TOOL_FUNCTION_DECLARATIONS[
tool_type
]
for fd in interaction_converter._BUILTIN_TOOL_FUNCTION_DECLARATIONS[tool_type]
}
sdk_names = {
fd.name
Expand All @@ -12104,17 +12108,14 @@ def test_catalog_in_sync_with_server(self):
f" SDK: {sorted(sdk_names)}"
)

# --- Sandbox declarations: names must match ---
server_sandbox_names = {
fd.name for fd in interaction_converter.sandbox_function_declarations()
}
sdk_sandbox_names = {
fd.name for fd in _evals_builtin_tools.SANDBOX_DECLARATIONS
}
assert sdk_sandbox_names == server_sandbox_names, (
f"SANDBOX_DECLARATIONS names out of sync.\n"
# --- Sandbox tool names: SDK names must match server names ---
server_sandbox_names = set(
interaction_converter._SANDBOX_TOOL_NAMES
)
assert _evals_builtin_tools.SANDBOX_TOOL_NAMES == server_sandbox_names, (
f"SANDBOX_TOOL_NAMES out of sync.\n"
f" Server: {sorted(server_sandbox_names)}\n"
f" SDK: {sorted(sdk_sandbox_names)}"
f" SDK: {sorted(_evals_builtin_tools.SANDBOX_TOOL_NAMES)}"
)


Expand Down
Loading