From ec343a2bdb09d8b148e9013de1258b9bee1b0ccb Mon Sep 17 00:00:00 2001 From: Hermes <6816583+digitallysavvy@users.noreply.github.com> Date: Wed, 23 Sep 2026 00:21:57 -0400 Subject: [PATCH] feat(agentkit): add Gemini TTS preview for v2.11.0 --- changelog.md | 7 + compat/agora-agent-server-sdk/pyproject.toml | 4 +- docs/guides/preview-endpoint.md | 30 ++++ pyproject.toml | 2 +- src/agora_agent/__init__.py | 2 + src/agora_agent/agentkit/__init__.py | 3 + src/agora_agent/agentkit/preview/__init__.py | 3 + src/agora_agent/agentkit/preview/client.py | 3 +- .../agentkit/preview/gemini_tts.py | 35 +++++ src/agora_agent/core/client_wrapper.py | 4 +- tests/custom/test_gemini_tts.py | 146 ++++++++++++++++++ 11 files changed, 233 insertions(+), 6 deletions(-) create mode 100644 src/agora_agent/agentkit/preview/gemini_tts.py create mode 100644 tests/custom/test_gemini_tts.py diff --git a/changelog.md b/changelog.md index acb87ae..d9149b9 100644 --- a/changelog.md +++ b/changelog.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/). +## [v2.11.0] — 2026-09-23 + +### Added + +- **Gemini TTS preview** — Added the Agent Kit Gemini TTS provider with the `gemini-3.8-flash-tts` model, default `Puck` voice, and optional natural-language `style`. Credentials serialize inside `tts.params`. +- **Session-scoped TTS routing** — Gemini TTS selects the preview endpoint with `agora-feature: gemini-live` for the full session lifecycle, including raw TTS configs, without changing the shared client's production route. Gemini ASR, Gemini Live, and OpenAI GPT Live retain their production routing. + ## [v2.10.0] — 2026-09-18 ### Added diff --git a/compat/agora-agent-server-sdk/pyproject.toml b/compat/agora-agent-server-sdk/pyproject.toml index 7618bee..f7e2451 100644 --- a/compat/agora-agent-server-sdk/pyproject.toml +++ b/compat/agora-agent-server-sdk/pyproject.toml @@ -3,7 +3,7 @@ name = "agora-agent-server-sdk" [tool.poetry] name = "agora-agent-server-sdk" -version = "v2.10.0" +version = "v2.11.0" description = "Compatibility shim for the renamed agora-agents package." readme = "README.md" authors = [] @@ -35,7 +35,7 @@ Repository = 'https://github.com/AgoraIO/agora-agents-python' [tool.poetry.dependencies] python = "^3.8" -agora-agents = ">=2.10.0,<3.0.0" +agora-agents = ">=2.11.0,<3.0.0" [build-system] requires = ["poetry-core"] diff --git a/docs/guides/preview-endpoint.md b/docs/guides/preview-endpoint.md index 3596932..dcb2a59 100644 --- a/docs/guides/preview-endpoint.md +++ b/docs/guides/preview-endpoint.md @@ -68,3 +68,33 @@ providers. `None` values are removed before the preview request is sent. Add routing tests for both synchronous and asynchronous sessions when registering a provider. Tests should verify the preview base URL, exact feature header, all lifecycle requests, caller-header precedence, and that the original client remains configured for production. + +## Gemini 3.8 Flash TTS preview + +`GeminiTTS` emits `tts.vendor = "gemini"` with `api_key`, `model`, `voice`, +and optional `style` inside `tts.params`. It defaults to `gemini-3.8-flash-tts` +and `Puck`. Model names are sent unchanged; there is no automatic fallback +or model rewriting. +Model strings remain open for preview rollout changes. Blank keys are rejected. + +AgentSession detects the TTS vendor from the resolved request body, including +handwritten configs, and uses the existing preview host with +`agora-feature: gemini-live` throughout the session lifecycle. Use the retained +session for stop/say/interrupt; the shared client remains on its normal route. +Gemini ASR alone still uses the production route. No sample-rate or avatar +compatibility is assumed by this preview provider. + +```python +from agora_agent.agentkit.preview import GeminiTTS, GeminiTTSModels + +agent.with_tts(GeminiTTS( + api_key=google_api_key, + model=GeminiTTSModels.FLASH_38, + voice="Puck", + style="warm and reassuring", +)) +``` + +Greeting audio for `gemini-3.8-flash-tts` was verified in the Next.js, Python, +and Go demos on 2026-09-22. A successful start response alone does not +establish that synthesis works; verify audio delivery when testing. diff --git a/pyproject.toml b/pyproject.toml index d0db875..d16773c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ dynamic = ["version"] [tool.poetry] name = "agora-agents" -version = "v2.10.0" +version = "v2.11.0" description = "" readme = "README.md" authors = [] diff --git a/src/agora_agent/__init__.py b/src/agora_agent/__init__.py index 9c97381..1aee5fb 100644 --- a/src/agora_agent/__init__.py +++ b/src/agora_agent/__init__.py @@ -65,6 +65,8 @@ MurfTTS, OpenAI, OpenAIRealtime, + GeminiTTS, + GeminiTTSModels, OpenAIGPTLive, OpenAISTT, OpenAITTS, diff --git a/src/agora_agent/agentkit/__init__.py b/src/agora_agent/agentkit/__init__.py index c45c514..8c364ca 100644 --- a/src/agora_agent/agentkit/__init__.py +++ b/src/agora_agent/agentkit/__init__.py @@ -155,6 +155,7 @@ OpenAITtsPresetModels, normalize_preset_input, ) +from .preview import GeminiTTS, GeminiTTSModels from .vendors import ( GEMINI_MLLM_DEFAULT_MODEL, AkoolAvatar, @@ -245,6 +246,8 @@ ) __all__ = [ + "GeminiTTS", + "GeminiTTSModels", "Agent", "AgentConfig", "AgentConfigUpdate", diff --git a/src/agora_agent/agentkit/preview/__init__.py b/src/agora_agent/agentkit/preview/__init__.py index db67a05..55b1675 100644 --- a/src/agora_agent/agentkit/preview/__init__.py +++ b/src/agora_agent/agentkit/preview/__init__.py @@ -13,6 +13,7 @@ create_preview_session_clients, required_preview_features, ) +from .gemini_tts import GeminiTTS, GeminiTTSModels from .vendors import ( GEMINI_MLLM_DEFAULT_MODEL, GEMINI_PREVIEW_MLLM_URL, @@ -26,6 +27,8 @@ ) __all__ = [ + "GeminiTTS", + "GeminiTTSModels", "PREVIEW_API_BASE_URL", "PREVIEW_FEATURE_HEADER", "GeminiSTTModels", diff --git a/src/agora_agent/agentkit/preview/client.py b/src/agora_agent/agentkit/preview/client.py index ecaa19f..e917b1c 100644 --- a/src/agora_agent/agentkit/preview/client.py +++ b/src/agora_agent/agentkit/preview/client.py @@ -37,7 +37,7 @@ class PreviewFeatures: vendors on the preview endpoint. """ - #: Deprecated compatibility value. Gemini Live now uses the production endpoint. + #: Gemini TTS preview gate. Gemini ASR and Live use production. GEMINI_LIVE = "gemini-live" LIVE_MODELS = "live-models" @@ -134,6 +134,7 @@ def create_preview_session_clients( #: ASR vendors served only by the preview endpoint. _PREVIEW_FEATURES_BY_CATEGORY: typing.Dict[str, typing.Dict[str, PreviewFeature]] = { "asr": {}, + "tts": {"gemini": PreviewFeatures.GEMINI_LIVE}, "mllm": {}, } diff --git a/src/agora_agent/agentkit/preview/gemini_tts.py b/src/agora_agent/agentkit/preview/gemini_tts.py new file mode 100644 index 0000000..fbf0282 --- /dev/null +++ b/src/agora_agent/agentkit/preview/gemini_tts.py @@ -0,0 +1,35 @@ +"""Gemini TTS preview; reuses the gemini-live session gate.""" + +from typing import Any, Dict, Optional + +from ..vendors.base import BaseTTS +from pydantic import ConfigDict, Field, field_validator + + +class GeminiTTSModels: + """Gemini 3.8 Flash TTS preview model.""" + + FLASH_38 = "gemini-3.8-flash-tts" + + +class GeminiTTS(BaseTTS): + """Preview-only TTS; model names are sent verbatim for rollout flexibility.""" + + model_config = ConfigDict(extra="forbid") + api_key: str = Field(..., repr=False) + model: str = GeminiTTSModels.FLASH_38 + voice: str = "Puck" + style: Optional[str] = None + + @field_validator("api_key", "model", "voice") + @classmethod + def require_nonblank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("GeminiTTS requires a nonblank value") + return value + + def to_config(self) -> Dict[str, Any]: + params: Dict[str, Any] = {"api_key": self.api_key, "model": self.model, "voice": self.voice} + if self.style is not None: + params["style"] = self.style + return {"vendor": "gemini", "params": params} diff --git a/src/agora_agent/core/client_wrapper.py b/src/agora_agent/core/client_wrapper.py index ac8bacd..8ce1e55 100644 --- a/src/agora_agent/core/client_wrapper.py +++ b/src/agora_agent/core/client_wrapper.py @@ -26,10 +26,10 @@ def __init__( def get_headers(self) -> typing.Dict[str, str]: headers: typing.Dict[str, str] = { - "User-Agent": "agora-agents/v2.10.0", + "User-Agent": "agora-agents/v2.11.0", "X-Fern-Language": "Python", "X-Fern-SDK-Name": "agora-agents", - "X-Fern-SDK-Version": "v2.10.0", + "X-Fern-SDK-Version": "v2.11.0", **(self.get_custom_headers() or {}), } headers["Authorization"] = httpx.BasicAuth(self._get_username(), self._get_password())._auth_header diff --git a/tests/custom/test_gemini_tts.py b/tests/custom/test_gemini_tts.py new file mode 100644 index 0000000..c3a491a --- /dev/null +++ b/tests/custom/test_gemini_tts.py @@ -0,0 +1,146 @@ +import json + +import httpx +import pytest + +from agora_agent import Agent, Agora, Area, AsyncAgora, Gemini, GeminiSTT, GeminiTTS, GeminiTTSModels +from agora_agent.agentkit.preview import PREVIEW_API_BASE_URL, required_preview_features + +MODELS = [GeminiTTSModels.FLASH_38] + + +def make_agent(client, model): + return ( + Agent(client=client) + .with_stt(GeminiSTT(api_key="test-key")) + .with_llm(Gemini(api_key="test-key", model="gemini-3.6-flash")) + .with_tts(GeminiTTS(api_key="test-key", model=model, voice="Puck", style="warm and reassuring")) + ) + + +def transport(requests): + def handle(request): + requests.append(request) + return httpx.Response(200, json={"agent_id": "agent-1", "data": {"list": []}}) + + return httpx.MockTransport(handle) + + +def assert_requests(requests, model): + assert len(requests) == 4 + assert json.loads(requests[0].content)["properties"]["tts"] == { + "vendor": "gemini", + "params": {"api_key": "test-key", "model": model, "voice": "Puck", "style": "warm and reassuring"}, + } + for request in requests: + assert str(request.url).startswith(PREVIEW_API_BASE_URL) + assert request.headers["agora-feature"] == "gemini-live" + assert request.headers["x-custom"] == "kept" + + +@pytest.mark.parametrize("model", MODELS) +def test_sync_lifecycle(model): + requests = [] + with httpx.Client(transport=transport(requests)) as http_client: + client = Agora( + area=Area.US, + app_id="0" * 32, + app_certificate="1" * 32, + httpx_client=http_client, + headers={"agora-feature": "", "x-custom": "kept"}, + ) + production_url = client.get_current_url() + session = make_agent(client, model).create_session(channel="test", agent_uid="1", remote_uids=["100"]) + session.start() + session.say("hello") + session.interrupt() + session.stop() + assert_requests(requests, model) + assert client.get_current_url() == production_url + assert client._client_wrapper.get_custom_headers()["agora-feature"] == "" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", MODELS) +async def test_async_lifecycle(model): + requests = [] + async with httpx.AsyncClient(transport=transport(requests)) as http_client: + client = AsyncAgora( + area=Area.US, + app_id="0" * 32, + app_certificate="1" * 32, + httpx_client=http_client, + headers={"agora-feature": "", "x-custom": "kept"}, + ) + session = make_agent(client, model).create_async_session(channel="test", agent_uid="1", remote_uids=["100"]) + await session.start() + await session.say("hello") + await session.interrupt() + await session.stop() + assert_requests(requests, model) + + +def test_defaults_and_raw_detection(): + assert GeminiTTS(api_key="test-key").to_config() == { + "vendor": "gemini", + "params": {"api_key": "test-key", "model": "gemini-3.8-flash-tts", "voice": "Puck"}, + } + assert required_preview_features({"tts": {"vendor": "gemini", "params": {"model": "future-model"}}}) == [ + "gemini-live" + ] + assert set(required_preview_features({"tts": {"vendor": "gemini"}, "mllm": {"vendor": "openai_gpt_live"}})) == { + "gemini-live", + } + assert required_preview_features({"asr": {"vendor": "gemini"}, "tts": {"vendor": "google"}}) == [] + + +@pytest.mark.parametrize("field", ["api_key", "model", "voice"]) +def test_rejects_blank_fields(field): + with pytest.raises(ValueError): + GeminiTTS(**{**{"api_key": "test-key"}, field: " "}) + + +@pytest.mark.parametrize("header", ["agora-feature", "Agora-Feature", "AGORA-FEATURE"]) +def test_preview_gate_survives_raw_per_call_headers(header): + requests = [] + with httpx.Client(transport=transport(requests)) as http_client: + client = Agora( + area=Area.US, + app_id="0" * 32, + app_certificate="1" * 32, + httpx_client=http_client, + headers={header: "caller-value"}, + ) + session = make_agent(client, MODELS[0]).create_session(channel="test", agent_uid="1", remote_uids=["100"]) + session.start() + headers = {header: "", "x-custom": "preserved"} + session.raw.get("0" * 32, "agent-1", request_options={"additional_headers": headers}) + for req in requests: + assert req.headers.get_list("agora-feature") == ["gemini-live"] + assert requests[-1].headers["x-custom"] == "preserved" + assert headers[header] == "" + assert client._client_wrapper.get_custom_headers()[header] == "caller-value" + client.agents.get("0" * 32, "agent-1") + assert not str(requests[-1].url).startswith(PREVIEW_API_BASE_URL) + assert requests[-1].headers["agora-feature"] == "caller-value" + + +@pytest.mark.asyncio +async def test_async_preview_gate_survives_raw_per_call_headers(): + requests = [] + async with httpx.AsyncClient(transport=transport(requests)) as http_client: + client = AsyncAgora( + area=Area.US, + app_id="0" * 32, + app_certificate="1" * 32, + httpx_client=http_client, + headers={"Agora-Feature": "caller-value"}, + ) + session = make_agent(client, MODELS[0]).create_async_session(channel="test", agent_uid="1", remote_uids=["100"]) + await session.start() + await session.raw.get("0" * 32, "agent-1", request_options={"additional_headers": {"AGORA-FEATURE": ""}}) + for req in requests: + assert req.headers.get_list("agora-feature") == ["gemini-live"] + await client.agents.get("0" * 32, "agent-1") + assert not str(requests[-1].url).startswith(PREVIEW_API_BASE_URL) + assert requests[-1].headers["agora-feature"] == "caller-value"