diff --git a/src/google/adk/cli/cli_eval.py b/src/google/adk/cli/cli_eval.py index f191f5efba..4063c34ba0 100644 --- a/src/google/adk/cli/cli_eval.py +++ b/src/google/adk/cli/cli_eval.py @@ -345,10 +345,10 @@ def get_eval_sets_manager( except ModuleNotFoundError as mnf: raise click.ClickException(MISSING_EVAL_DEPENDENCIES_MESSAGE) from mnf - if eval_storage_uri: + eval_storage = evals.resolve_eval_storage(eval_storage_uri, agents_dir) + if eval_storage.gcs_uri: gcs_eval_managers = evals.create_gcs_eval_managers_from_uri( - eval_storage_uri + eval_storage.gcs_uri ) return gcs_eval_managers.eval_sets_manager - else: - return LocalEvalSetsManager(agents_dir=agents_dir) + return LocalEvalSetsManager(agents_dir=eval_storage.local_dir) diff --git a/src/google/adk/cli/cli_tools_click.py b/src/google/adk/cli/cli_tools_click.py index c78c14668a..cdced14fac 100644 --- a/src/google/adk/cli/cli_tools_click.py +++ b/src/google/adk/cli/cli_tools_click.py @@ -1206,7 +1206,9 @@ def decorator(func): type=str, help=( "Optional. The evals storage URI to store agent evals," - " supported URIs: gs://." + " supported URIs: gs:// or file://. When" + " omitted, ADK_EVAL_STORAGE_URI then ADK_EVAL_STORAGE_DIR are" + " used." ), default=None, ) @@ -1355,6 +1357,7 @@ def cli_eval( from .cli_eval import get_app_or_root_agent from .cli_eval import parse_and_get_evals_to_run from .cli_eval import pretty_print_eval_result + from .utils import evals except ModuleNotFoundError as mnf: raise click.ClickException(_missing_eval_dependencies_message()) from mnf @@ -1364,16 +1367,17 @@ def cli_eval( eval_sets_manager = None eval_set_results_manager = None - if eval_storage_uri: - from .utils import evals - + eval_storage = evals.resolve_eval_storage(eval_storage_uri, agents_dir) + if eval_storage.gcs_uri: gcs_eval_managers = evals.create_gcs_eval_managers_from_uri( - eval_storage_uri + eval_storage.gcs_uri ) eval_sets_manager = gcs_eval_managers.eval_sets_manager eval_set_results_manager = gcs_eval_managers.eval_set_results_manager else: - eval_set_results_manager = LocalEvalSetResultsManager(agents_dir=agents_dir) + eval_set_results_manager = LocalEvalSetResultsManager( + agents_dir=eval_storage.local_dir + ) inference_requests = [] eval_set_file_or_id_to_evals = parse_and_get_evals_to_run( @@ -1440,8 +1444,8 @@ def cli_eval( # We assume that what we have are eval set ids instead. eval_sets_manager = ( eval_sets_manager - if eval_storage_uri - else LocalEvalSetsManager(agents_dir=agents_dir) + if eval_storage.gcs_uri + else LocalEvalSetsManager(agents_dir=eval_storage.local_dir) ) for eval_set_id_key, eval_case_ids in eval_set_file_or_id_to_evals.items(): @@ -2023,7 +2027,9 @@ def decorator(func): type=str, help=( "Optional. The evals storage URI to store agent evals," - " supported URIs: gs://." + " supported URIs: gs:// or file://. When" + " omitted, ADK_EVAL_STORAGE_URI then ADK_EVAL_STORAGE_DIR are" + " used." ), default=None, ) diff --git a/src/google/adk/cli/fast_api.py b/src/google/adk/cli/fast_api.py index 5b50858df9..2e4778e8cc 100644 --- a/src/google/adk/cli/fast_api.py +++ b/src/google/adk/cli/fast_api.py @@ -153,8 +153,10 @@ def get_fast_api_app( memory_service_uri: URI for the memory service. Uses local memory service if None. use_local_storage: Whether to use local storage for session and artifacts. - eval_storage_uri: URI for evaluation storage. If provided, uses GCS - managers. + eval_storage_uri: URI for evaluation storage. Supports ``gs://`` + for GCS and ``file://`` for a local directory. When omitted, + ``ADK_EVAL_STORAGE_URI`` then ``ADK_EVAL_STORAGE_DIR`` are used, and + local eval files default to ``agents_dir``. allow_origins: List of allowed origins for CORS. web: Whether to enable the web UI and serve its assets. a2a: Whether to enable Agent-to-Agent (A2A) protocol support. @@ -215,19 +217,22 @@ def get_fast_api_app( agents_dir = str(agents_path.parent) # Set up eval managers. - if eval_storage_uri: - from .utils import evals + from .utils import evals + eval_storage = evals.resolve_eval_storage(eval_storage_uri, agents_dir) + this_module = sys.modules[__name__] + if eval_storage.gcs_uri: gcs_eval_managers = evals.create_gcs_eval_managers_from_uri( - eval_storage_uri + eval_storage.gcs_uri ) eval_sets_manager = gcs_eval_managers.eval_sets_manager eval_set_results_manager = gcs_eval_managers.eval_set_results_manager else: - this_module = sys.modules[__name__] - eval_sets_manager = this_module.LocalEvalSetsManager(agents_dir=agents_dir) + eval_sets_manager = this_module.LocalEvalSetsManager( + agents_dir=eval_storage.local_dir + ) eval_set_results_manager = this_module.LocalEvalSetResultsManager( - agents_dir=agents_dir + agents_dir=eval_storage.local_dir ) # initialize Agent Loader if not passed as argument diff --git a/src/google/adk/cli/utils/evals.py b/src/google/adk/cli/utils/evals.py index 56c2035165..aae7ff870c 100644 --- a/src/google/adk/cli/utils/evals.py +++ b/src/google/adk/cli/utils/evals.py @@ -14,8 +14,13 @@ from __future__ import annotations +import logging import os +from pathlib import Path +from typing import NamedTuple from typing import TYPE_CHECKING +from urllib.parse import unquote +from urllib.parse import urlparse from pydantic import alias_generators from pydantic import BaseModel @@ -29,6 +34,11 @@ from ...evaluation.gcs_eval_set_results_manager import GcsEvalSetResultsManager from ...evaluation.gcs_eval_sets_manager import GcsEvalSetsManager +logger = logging.getLogger('google_adk.' + __name__) + +ADK_EVAL_STORAGE_URI_ENV = 'ADK_EVAL_STORAGE_URI' +ADK_EVAL_STORAGE_DIR_ENV = 'ADK_EVAL_STORAGE_DIR' + class GcsEvalManagers(BaseModel): model_config = ConfigDict( @@ -42,6 +52,122 @@ class GcsEvalManagers(BaseModel): eval_set_results_manager: 'GcsEvalSetResultsManager' +class ResolvedEvalStorage(NamedTuple): + """Where eval sets and results should be stored. + + Attributes: + gcs_uri: ``gs://`` URI when using Cloud Storage, otherwise None. + local_dir: Directory for the local eval managers. Unused when ``gcs_uri`` + is set. + """ + + gcs_uri: str | None + local_dir: str + + +def resolve_eval_storage_uri(eval_storage_uri: str | None) -> str | None: + """Resolves the eval storage URI from an argument or the environment. + + Precedence is the explicit ``eval_storage_uri``, then + ``ADK_EVAL_STORAGE_URI``, then ``ADK_EVAL_STORAGE_DIR`` converted to a + ``file://`` URI. + + Args: + eval_storage_uri: Explicit URI from a flag or ``get_fast_api_app``. + + Returns: + The URI to use, or None to store evals under ``agents_dir``. + """ + if eval_storage_uri: + return eval_storage_uri + + env_uri = os.environ.get(ADK_EVAL_STORAGE_URI_ENV) + if env_uri: + logger.info( + 'Using eval storage URI from %s: %s', ADK_EVAL_STORAGE_URI_ENV, env_uri + ) + return env_uri + + env_dir = os.environ.get(ADK_EVAL_STORAGE_DIR_ENV) + if env_dir: + file_uri = Path(os.path.abspath(os.path.expanduser(env_dir))).as_uri() + logger.info( + 'Using eval storage directory from %s: %s', + ADK_EVAL_STORAGE_DIR_ENV, + env_dir, + ) + return file_uri + + return None + + +def local_path_from_file_uri(eval_storage_uri: str) -> str: + """Returns the filesystem path for a ``file://`` eval storage URI. + + Args: + eval_storage_uri: A ``file://`` URI pointing at a local directory. + + Returns: + The decoded filesystem path. + + Raises: + ValueError: If the URI is not a ``file://`` URI. + """ + parsed = urlparse(eval_storage_uri) + if parsed.scheme != 'file': + raise ValueError( + f'Unsupported evals storage URI: {eval_storage_uri}. Supported URIs:' + ' gs://, file://' + ) + + path = unquote(parsed.path) + if os.name == 'nt': + if parsed.netloc and parsed.netloc.lower() != 'localhost': + return '\\\\' + parsed.netloc + path.replace('/', '\\') + if path.startswith('/') and len(path) >= 3 and path[2] == ':': + path = path[1:] + return path.replace('/', '\\') + + if parsed.netloc and parsed.netloc.lower() != 'localhost': + return '//' + parsed.netloc + path + return path + + +def prepare_local_eval_dir(path: str) -> str: + """Creates ``path`` if needed and returns it.""" + os.makedirs(path, exist_ok=True) + return path + + +def resolve_eval_storage( + eval_storage_uri: str | None, agents_dir: str +) -> ResolvedEvalStorage: + """Resolves GCS vs local eval storage from a URI, env vars, or agents_dir. + + Args: + eval_storage_uri: Explicit URI from a flag or ``get_fast_api_app``. + agents_dir: Fallback directory when no URI or env override is set. + + Returns: + A ``ResolvedEvalStorage`` with either a GCS URI or a local directory. + + Raises: + ValueError: If the resolved URI is neither ``gs://`` nor ``file://``. + """ + resolved_uri = resolve_eval_storage_uri(eval_storage_uri) + if not resolved_uri: + return ResolvedEvalStorage(gcs_uri=None, local_dir=agents_dir) + if resolved_uri.startswith('gs://'): + return ResolvedEvalStorage(gcs_uri=resolved_uri, local_dir=agents_dir) + if resolved_uri.startswith('file:'): + local_dir = prepare_local_eval_dir(local_path_from_file_uri(resolved_uri)) + return ResolvedEvalStorage(gcs_uri=None, local_dir=local_dir) + raise ValueError( + f'Unsupported evals storage URI: {resolved_uri}. Supported URIs:' + ' gs://, file://' + ) + + def convert_session_to_eval_invocations(session: Session) -> list[Invocation]: """Converts a session data into a list of Invocation. diff --git a/tests/unittests/cli/test_fast_api.py b/tests/unittests/cli/test_fast_api.py index a3fe28d35a..b010c05d50 100644 --- a/tests/unittests/cli/test_fast_api.py +++ b/tests/unittests/cli/test_fast_api.py @@ -554,6 +554,61 @@ def _create_test_client( return TestClient(app) +def test_get_fast_api_app_creates_file_eval_storage_dir( + tmp_path, + monkeypatch, + mock_session_service, + mock_artifact_service, + mock_memory_service, + mock_agent_loader, + mock_eval_sets_manager, + mock_eval_set_results_manager, +): + """file:// eval_storage_uri creates that directory for local eval managers.""" + monkeypatch.delenv("ADK_EVAL_STORAGE_URI", raising=False) + monkeypatch.delenv("ADK_EVAL_STORAGE_DIR", raising=False) + storage_dir = tmp_path / "adk_evals" + + _create_test_client( + mock_session_service, + mock_artifact_service, + mock_memory_service, + mock_agent_loader, + mock_eval_sets_manager, + mock_eval_set_results_manager, + eval_storage_uri=storage_dir.as_uri(), + ) + + assert storage_dir.is_dir() + + +def test_get_fast_api_app_honors_eval_storage_dir_env( + tmp_path, + monkeypatch, + mock_session_service, + mock_artifact_service, + mock_memory_service, + mock_agent_loader, + mock_eval_sets_manager, + mock_eval_set_results_manager, +): + """ADK_EVAL_STORAGE_DIR is used when eval_storage_uri is omitted.""" + monkeypatch.delenv("ADK_EVAL_STORAGE_URI", raising=False) + storage_dir = tmp_path / "from_env" + monkeypatch.setenv("ADK_EVAL_STORAGE_DIR", str(storage_dir)) + + _create_test_client( + mock_session_service, + mock_artifact_service, + mock_memory_service, + mock_agent_loader, + mock_eval_sets_manager, + mock_eval_set_results_manager, + ) + + assert storage_dir.is_dir() + + def test_agent_with_bigquery_analytics_plugin( tmp_path, mock_session_service, diff --git a/tests/unittests/cli/utils/test_cli_eval.py b/tests/unittests/cli/utils/test_cli_eval.py index 04f44699dd..1ed7b83a0f 100644 --- a/tests/unittests/cli/utils/test_cli_eval.py +++ b/tests/unittests/cli/utils/test_cli_eval.py @@ -86,6 +86,8 @@ def test_parse_and_get_evals_to_run_parses_eval_set_and_selectors( def test_get_eval_sets_manager_local(monkeypatch): mock_local_manager = mock.MagicMock() + monkeypatch.delenv("ADK_EVAL_STORAGE_URI", raising=False) + monkeypatch.delenv("ADK_EVAL_STORAGE_DIR", raising=False) monkeypatch.setattr( "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager", lambda *a, **k: mock_local_manager, @@ -115,6 +117,94 @@ def test_get_eval_sets_manager_gcs(monkeypatch): mock_create_gcs.assert_called_once_with("gs://bucket") +def test_get_eval_sets_manager_file_uri(monkeypatch, tmp_path): + """file:// eval storage uses a local manager rooted at that directory.""" + captured = {} + + def fake_local_manager(*, agents_dir): + captured["agents_dir"] = agents_dir + return mock.MagicMock(name="local-manager") + + monkeypatch.setattr( + "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager", + fake_local_manager, + ) + from google.adk.cli.cli_eval import get_eval_sets_manager + + storage_dir = tmp_path / "evals" + manager = get_eval_sets_manager( + eval_storage_uri=storage_dir.as_uri(), agents_dir="some/dir" + ) + + assert captured["agents_dir"] == str(storage_dir) + assert storage_dir.is_dir() + assert manager is not None + + +def test_get_eval_sets_manager_eval_storage_dir_env(monkeypatch, tmp_path): + """ADK_EVAL_STORAGE_DIR overrides agents_dir for local eval storage.""" + captured = {} + + def fake_local_manager(*, agents_dir): + captured["agents_dir"] = agents_dir + return mock.MagicMock(name="local-manager") + + monkeypatch.delenv("ADK_EVAL_STORAGE_URI", raising=False) + monkeypatch.setenv("ADK_EVAL_STORAGE_DIR", str(tmp_path / "from_env")) + monkeypatch.setattr( + "google.adk.evaluation.local_eval_sets_manager.LocalEvalSetsManager", + fake_local_manager, + ) + from google.adk.cli.cli_eval import get_eval_sets_manager + + get_eval_sets_manager(eval_storage_uri=None, agents_dir="some/dir") + + assert captured["agents_dir"] == str(tmp_path / "from_env") + assert (tmp_path / "from_env").is_dir() + + +def test_get_eval_sets_manager_explicit_uri_beats_env(monkeypatch, tmp_path): + """An explicit gs:// URI is used even when ADK_EVAL_STORAGE_DIR is set.""" + mock_gcs_manager = mock.MagicMock() + mock_create_gcs = mock.MagicMock() + mock_create_gcs.return_value = SimpleNamespace( + eval_sets_manager=mock_gcs_manager + ) + monkeypatch.setenv("ADK_EVAL_STORAGE_DIR", str(tmp_path / "from_env")) + monkeypatch.setattr( + "google.adk.cli.utils.evals.create_gcs_eval_managers_from_uri", + mock_create_gcs, + ) + from google.adk.cli.cli_eval import get_eval_sets_manager + + manager = get_eval_sets_manager( + eval_storage_uri="gs://bucket", agents_dir="some/dir" + ) + + assert manager == mock_gcs_manager + mock_create_gcs.assert_called_once_with("gs://bucket") + + +def test_get_eval_sets_manager_eval_storage_uri_env(monkeypatch): + """ADK_EVAL_STORAGE_URI selects GCS when no explicit URI is passed.""" + mock_gcs_manager = mock.MagicMock() + mock_create_gcs = mock.MagicMock() + mock_create_gcs.return_value = SimpleNamespace( + eval_sets_manager=mock_gcs_manager + ) + monkeypatch.setenv("ADK_EVAL_STORAGE_URI", "gs://from-env") + monkeypatch.setattr( + "google.adk.cli.utils.evals.create_gcs_eval_managers_from_uri", + mock_create_gcs, + ) + from google.adk.cli.cli_eval import get_eval_sets_manager + + manager = get_eval_sets_manager(eval_storage_uri=None, agents_dir="some/dir") + + assert manager == mock_gcs_manager + mock_create_gcs.assert_called_once_with("gs://from-env") + + @pytest.mark.asyncio async def test_get_root_agent_supports_root_agent(monkeypatch): root_agent = mock.MagicMock() diff --git a/tests/unittests/cli/utils/test_evals.py b/tests/unittests/cli/utils/test_evals.py index bfb1481700..a246d35732 100644 --- a/tests/unittests/cli/utils/test_evals.py +++ b/tests/unittests/cli/utils/test_evals.py @@ -15,11 +15,10 @@ """Tests for utilities in eval.""" import os +from pathlib import Path from unittest import mock from google.adk.cli.utils import evals -from google.adk.evaluation.gcs_eval_set_results_manager import GcsEvalSetResultsManager -from google.adk.evaluation.gcs_eval_sets_manager import GcsEvalSetsManager from google.adk.events.event import Event from google.adk.sessions.session import Session from google.genai import types @@ -38,12 +37,8 @@ def test_create_gcs_eval_managers_from_uri_success( mock_gcs_eval_sets_manager, mock_gcs_eval_set_results_manager ): - mock_gcs_eval_sets_manager.return_value = mock.MagicMock( - spec=GcsEvalSetsManager - ) - mock_gcs_eval_set_results_manager.return_value = mock.MagicMock( - spec=GcsEvalSetResultsManager - ) + mock_gcs_eval_sets_manager.return_value = mock.MagicMock() + mock_gcs_eval_set_results_manager.return_value = mock.MagicMock() managers = evals.create_gcs_eval_managers_from_uri('gs://test-bucket') @@ -66,6 +61,111 @@ def test_create_gcs_eval_managers_from_uri_failure(): evals.create_gcs_eval_managers_from_uri('unsupported-uri') +def test_resolve_eval_storage_uri_prefers_explicit_over_env(monkeypatch): + """An explicit URI wins over ADK_EVAL_STORAGE_URI and ADK_EVAL_STORAGE_DIR.""" + monkeypatch.setenv(evals.ADK_EVAL_STORAGE_URI_ENV, 'gs://from-env') + monkeypatch.setenv(evals.ADK_EVAL_STORAGE_DIR_ENV, '/tmp/from-dir') + + assert evals.resolve_eval_storage_uri('gs://explicit') == 'gs://explicit' + + +def test_resolve_eval_storage_uri_uses_uri_env(monkeypatch): + """ADK_EVAL_STORAGE_URI is used when no explicit URI is given.""" + monkeypatch.delenv(evals.ADK_EVAL_STORAGE_DIR_ENV, raising=False) + monkeypatch.setenv(evals.ADK_EVAL_STORAGE_URI_ENV, 'file:///tmp/from-uri') + + assert evals.resolve_eval_storage_uri(None) == 'file:///tmp/from-uri' + + +def test_resolve_eval_storage_uri_uses_dir_env(monkeypatch, tmp_path): + """ADK_EVAL_STORAGE_DIR becomes a file:// URI when no other URI is set.""" + monkeypatch.delenv(evals.ADK_EVAL_STORAGE_URI_ENV, raising=False) + storage_dir = tmp_path / 'evals' + monkeypatch.setenv(evals.ADK_EVAL_STORAGE_DIR_ENV, str(storage_dir)) + + expected = Path(os.path.abspath(str(storage_dir))).as_uri() + assert evals.resolve_eval_storage_uri(None) == expected + + +def test_resolve_eval_storage_uri_returns_none_without_overrides(monkeypatch): + """No URI or env override means callers should use agents_dir.""" + monkeypatch.delenv(evals.ADK_EVAL_STORAGE_URI_ENV, raising=False) + monkeypatch.delenv(evals.ADK_EVAL_STORAGE_DIR_ENV, raising=False) + + assert evals.resolve_eval_storage_uri(None) is None + + +def test_local_path_from_file_uri_decodes_posix_path(): + """file:///tmp/adk_evals maps to the local /tmp/adk_evals directory.""" + assert evals.local_path_from_file_uri('file:///tmp/adk_evals') == ( + '/tmp/adk_evals' + ) + + +def test_local_path_from_file_uri_rejects_non_file_scheme(): + with pytest.raises(ValueError, match='Unsupported evals storage URI'): + evals.local_path_from_file_uri('gs://bucket') + + +def test_resolve_eval_storage_defaults_to_agents_dir(monkeypatch): + """Without overrides, local evals stay in the agent directory.""" + monkeypatch.delenv(evals.ADK_EVAL_STORAGE_URI_ENV, raising=False) + monkeypatch.delenv(evals.ADK_EVAL_STORAGE_DIR_ENV, raising=False) + + storage = evals.resolve_eval_storage(None, 'some/agents') + + assert storage.gcs_uri is None + assert storage.local_dir == 'some/agents' + + +def test_resolve_eval_storage_file_uri_creates_directory(tmp_path): + """file:// URIs store evals in that directory, creating it if needed.""" + storage_dir = tmp_path / 'nested' / 'evals' + + storage = evals.resolve_eval_storage(storage_dir.as_uri(), 'some/agents') + + assert storage.gcs_uri is None + assert storage.local_dir == str(storage_dir) + assert storage_dir.is_dir() + + +def test_resolve_eval_storage_dir_env_creates_directory(monkeypatch, tmp_path): + """ADK_EVAL_STORAGE_DIR is created and used instead of agents_dir.""" + monkeypatch.delenv(evals.ADK_EVAL_STORAGE_URI_ENV, raising=False) + storage_dir = tmp_path / 'from_env' + monkeypatch.setenv(evals.ADK_EVAL_STORAGE_DIR_ENV, str(storage_dir)) + + storage = evals.resolve_eval_storage(None, 'some/agents') + + assert storage.gcs_uri is None + assert storage.local_dir == str(storage_dir) + assert storage_dir.is_dir() + + +def test_resolve_eval_storage_dir_env_expands_user_home(monkeypatch, tmp_path): + """~ in ADK_EVAL_STORAGE_DIR expands to the user home directory.""" + monkeypatch.delenv(evals.ADK_EVAL_STORAGE_URI_ENV, raising=False) + monkeypatch.setenv('HOME', str(tmp_path)) + monkeypatch.setenv(evals.ADK_EVAL_STORAGE_DIR_ENV, '~/evals') + + storage = evals.resolve_eval_storage(None, 'some/agents') + + assert storage.local_dir == str(tmp_path / 'evals') + assert (tmp_path / 'evals').is_dir() + + +def test_resolve_eval_storage_gcs_uri(): + storage = evals.resolve_eval_storage('gs://my-bucket', 'some/agents') + + assert storage.gcs_uri == 'gs://my-bucket' + assert storage.local_dir == 'some/agents' + + +def test_resolve_eval_storage_rejects_unknown_scheme(): + with pytest.raises(ValueError, match='Unsupported evals storage URI'): + evals.resolve_eval_storage('s3://bucket', 'some/agents') + + def _event(author: str, text: str, invocation_id: str) -> Event: return Event( author=author,