From 6cd65c1a25abeced3f7393640e73abce68decf5f Mon Sep 17 00:00:00 2001 From: Xiting Zhang Date: Tue, 20 Oct 2026 14:58:14 -0700 Subject: [PATCH 01/10] [azure-ai-projects] Add GPT Live WebSocket clients and samples Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/ai/azure-ai-projects/CHANGELOG.md | 7 + sdk/ai/azure-ai-projects/README.md | 40 +- sdk/ai/azure-ai-projects/api.md | 318 +++- sdk/ai/azure-ai-projects/api.metadata.yml | 2 +- .../azure/ai/projects/_live.py | 284 ++++ .../azure/ai/projects/_live_utils.py | 67 + .../azure/ai/projects/_realtime.py | 282 ++-- .../azure/ai/projects/aio/_live.py | 326 ++++ .../azure/ai/projects/aio/_realtime.py | 362 ++-- .../ai/projects/aio/operations/_patch.py | 19 + .../azure/ai/projects/operations/_patch.py | 13 + .../docs/GeneratePublicMethodsDoc.ps1 | 2 +- .../azure-ai-projects/docs/public-methods.md | 7 +- .../samples/agents/voice/live_sample_utils.py | 221 +++ ...ice_agent_live_audio_conversation_async.py | 269 +++ .../sample_voice_agent_live_connection.py | 84 + .../sample_voice_agent_live_function_tool.py | 166 ++ ...mple_voice_agent_live_text_conversation.py | 118 ++ ...oice_agent_live_text_conversation_async.py | 120 ++ .../tests/agents/test_live_client.py | 1486 +++++++++++++++++ .../tests/agents/test_realtime_client.py | 18 +- .../agents/test_realtime_client_async.py | 18 +- .../tests/samples/test_live_samples.py | 813 +++++++++ 23 files changed, 4799 insertions(+), 243 deletions(-) create mode 100644 sdk/ai/azure-ai-projects/azure/ai/projects/_live.py create mode 100644 sdk/ai/azure-ai-projects/azure/ai/projects/_live_utils.py create mode 100644 sdk/ai/azure-ai-projects/azure/ai/projects/aio/_live.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_connection.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_live_client.py create mode 100644 sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py diff --git a/sdk/ai/azure-ai-projects/CHANGELOG.md b/sdk/ai/azure-ai-projects/CHANGELOG.md index f4ac283d9f1a..3761928e5037 100644 --- a/sdk/ai/azure-ai-projects/CHANGELOG.md +++ b/sdk/ai/azure-ai-projects/CHANGELOG.md @@ -4,6 +4,7 @@ ### Features Added +* Added preview GPT Live support through `beta.voice_agents.live`, with synchronous and asynchronous WebSocket clients, typed Live events, session controls, audio streaming, and Responses-delegation tool input/output. * Added generated GPT Live voice-agent configuration, delegation, and client/server event models. * Added persisted Live conversation, item, and audio REST operations under `beta.voice_agents.live_conversations`. * Added image-generation model options and the `xhigh` and `max` quality values from the updated OpenAI TypeSpec dependency. @@ -12,6 +13,12 @@ * Preserved concrete Live/Realtime voice-agent types on agent readback without inserting an omitted `sub_protocol`. +### Sample updates + +* Added GPT Live connection, text conversation, asynchronous PCM audio streaming, and function-tool examples under `samples/agents/voice`. +* Updated GPT Live text examples with synchronous/asynchronous headless multi-turn conversations that reuse one session and preserve the default single-turn example. +* Fixed the GPT Live WAV sample to ignore silent/very-low-level PCM when detecting reply inactivity, reject silence-only replies, and preserve captured PCM in audio counts and optional WAV output. + ## 2.9.0 (2026-10-16) ### Features Added diff --git a/sdk/ai/azure-ai-projects/README.md b/sdk/ai/azure-ai-projects/README.md index 1660a07d9ebb..f00851117ed5 100644 --- a/sdk/ai/azure-ai-projects/README.md +++ b/sdk/ai/azure-ai-projects/README.md @@ -225,6 +225,41 @@ async with async_project_client.beta.voice_agents.realtime.connect( the peer connection, media, and SDP offer/answer exchange using `rtc.call.sdp.create` and `rtc.call.sdp.created` events. Selecting this value does not automatically create a WebRTC media client. +### Voice Agent GPT Live (preview) + +GPT Live is a separate protocol at `project_client.beta.voice_agents.live` (and the same namespace +on the asynchronous client). Create a `VoiceAgentLiveDefinition` using your GPT Live **deployment +name**; existing `VoiceAgentDefinition` and `.realtime` calls retain Realtime behavior. +Creating the preview definition through `project_client.agents` requires constructing the client +with `allow_preview=True`, as the samples do. The `.beta` namespace opts in automatically. + +Use `.live.connect(agent_name=..., agent_version=...)` as a context manager and receive +`session.started` before sending commands. Foundry starts the session from the saved definition; +do not send another `session.start`. Live uses `session.input_audio`, `session.instructions`, +`session.thinking`, and `session.commentary`, while `response.item.create` and `response.create` +require Responses delegation. Live has no Realtime `response.cancel`; cancel local receive tasks +normally or request `session.close` and receive `session.closed`, then close the connection. + +Live conversation REST operations use the emitted implementation without handwritten paging overrides. +The current emitter's REST pagers follow `last_id` without checking `has_more`; this known generation +limitation does not affect WebSocket sessions. + +Connection, text, PCM audio-streaming, and function-tool examples are in `samples/agents/voice` +in the [Samples][samples] folder. The optional `[voice]` dependencies are unchanged. + +The synchronous and asynchronous text examples accept quoted prompts for a headless multi-turn +conversation in one session. Omit prompts to run the original single-turn example: + +```powershell +python .\samples\agents\voice\sample_voice_agent_live_text_conversation.py "My name is Ada." "What is my name?" +python .\samples\agents\voice\sample_voice_agent_live_text_conversation_async.py "My name is Ada." "What is my name?" +``` + +These text examples print Responses backend text, not completion of the voice frontend's speech. +The WAV audio example uses a bounded speech-inactivity heuristic because Live has no speech-finished +event. Continued silent PCM does not keep it waiting forever, and silence-only output is not a +successful spoken reply. A long speech pause can still split a reply. + ## Client-side tracing See [Add client-side tracing to Foundry agents (preview)](https://learn.microsoft.com/azure/foundry/observability/how-to/trace-agent-client-side?tabs=python). @@ -291,7 +326,8 @@ openai_logger.setLevel(logging.DEBUG) openai_logger.propagate = False openai_logger.addHandler(handler) -# Optional: Voice Agent WebSocket transport logger. +# Optional: Voice Agent Realtime WebSocket transport logger. +# For GPT Live, use "azure.ai.projects.live" instead. voice_logger = logging.getLogger("azure.ai.projects.realtime") voice_logger.setLevel(logging.DEBUG) voice_logger.propagate = False @@ -302,7 +338,7 @@ voice_logger.addHandler(handler) # handler.setFormatter(formatter) ``` -The `azure` logger includes logs from `azure-ai-projects` and `azure-core`, including Voice Agent WebSocket transport logs. The dedicated OpenAI and Voice Agent loggers shown above can be used when you want only those transport logs. At `DEBUG` level, the Voice Agent logger records connection and close events and the type and byte count of each event sent or received. It does not log event payloads, URL query strings, request headers, or authentication tokens. +The `azure` logger includes logs from `azure-ai-projects` and `azure-core`, including Voice Agent WebSocket transport logs. The dedicated OpenAI and Voice Agent loggers shown above can be used when you want only those transport logs. The `azure.ai.projects.realtime` and `azure.ai.projects.live` loggers record connection and close events and the type and byte count of each event sent or received at `DEBUG` level. They do not log event payloads, URL query strings, request headers, or authentication tokens. By default, logs redact URL query values, some HTTP request and response headers (including `Authorization`), and request and response payloads. To create HTTP pipeline logs without redaction, add `logging_enable=True` to the client constructor: diff --git a/sdk/ai/azure-ai-projects/api.md b/sdk/ai/azure-ai-projects/api.md index 794ea298e6d7..3648cf565896 100644 --- a/sdk/ai/azure-ai-projects/api.md +++ b/sdk/ai/azure-ai-projects/api.md @@ -635,6 +635,68 @@ namespace azure.ai.projects.aio.operations ) -> SessionFileWriteResult: ... + class azure.ai.projects.aio.operations.AsyncBetaLive: + + def __init__( + self, + *args: Any, + **kwargs: Any + ) -> None: ... + + def connect( + self, + *, + agent_name: str, + agent_version: Optional[str] = ..., + api_version: Optional[str] = ..., + connection_url: Optional[str] = ..., + credential_scopes: Optional[list[str]] = ..., + extra_headers: Optional[Mapping[str, str]] = ..., + extra_query: Optional[Mapping[str, str]] = ..., + store: Optional[bool] = ..., + structured_inputs: Optional[Mapping[str, Any]] = ..., + **kwargs: Any + ) -> AsyncBetaLiveConnectionManager: ... + + + class azure.ai.projects.aio.operations.AsyncBetaLiveConnection(_BaseWebSocketConnection[LiveServerEvent, LiveClientEvent]): implements AsyncContextManager + property closed: bool # Read-only + + def __aiter__(self) -> AsyncIterator[LiveServerEvent]: ... + + def __init__( + self, + connection: ClientWebSocketResponse, + session: ClientSession + ) -> None: ... + + def __repr__(self) -> str: ... + + async def close( + self, + *, + code: int = 1000, + reason: str = "" + ) -> None: ... + + async def recv(self) -> LiveServerEvent: ... + + async def send(self, event: LiveClientEvent) -> None: ... + + + class azure.ai.projects.aio.operations.AsyncBetaLiveConnectionManager(_BaseWebSocketConnectionManager[AsyncBetaLiveConnection]): implements AsyncContextManager + + def __init__( + self, + *, + agent_version: Optional[str] = ..., + store: Optional[bool] = ..., + **kwargs: Any + ) -> None: ... + + async def enter(self) -> AsyncBetaLiveConnection: ... + + class azure.ai.projects.aio.operations.AsyncBetaRealtime: def __init__( @@ -659,7 +721,7 @@ namespace azure.ai.projects.aio.operations ) -> AsyncBetaRealtimeConnectionManager: ... - class azure.ai.projects.aio.operations.AsyncBetaRealtimeConnection: implements AsyncContextManager + class azure.ai.projects.aio.operations.AsyncBetaRealtimeConnection(_BaseWebSocketConnection[ServerEvent, ClientEvent]): implements AsyncContextManager property closed: bool # Read-only def __aiter__(self) -> AsyncIterator[ServerEvent]: ... @@ -684,7 +746,7 @@ namespace azure.ai.projects.aio.operations async def send(self, event: ClientEvent) -> None: ... - class azure.ai.projects.aio.operations.AsyncBetaRealtimeConnectionManager: implements AsyncContextManager + class azure.ai.projects.aio.operations.AsyncBetaRealtimeConnectionManager(_BaseWebSocketConnectionManager[AsyncBetaRealtimeConnection]): implements AsyncContextManager def __init__( self, @@ -2167,8 +2229,101 @@ namespace azure.ai.projects.aio.operations ) -> AsyncItemPaged[VoiceResponse]: ... + class azure.ai.projects.aio.operations.BetaVoiceAgentsLiveConversationsOperations: + + def __init__( + self, + *args, + **kwargs + ) -> None: ... + + @distributed_trace_async + async def delete( + self, + agent_name: str, + conversation_id: str, + **kwargs: Any + ) -> None: ... + + @distributed_trace_async + async def download_audio( + self, + agent_name: str, + conversation_id: str, + **kwargs: Any + ) -> AsyncIterator[bytes]: ... + + @distributed_trace_async + async def download_audio_item( + self, + agent_name: str, + conversation_id: str, + item_id: str, + **kwargs: Any + ) -> AsyncIterator[bytes]: ... + + @distributed_trace_async + async def get( + self, + agent_name: str, + conversation_id: str, + **kwargs: Any + ) -> VoiceAgentLiveConversation: ... + + @distributed_trace_async + async def get_audio( + self, + agent_name: str, + conversation_id: str, + **kwargs: Any + ) -> VoiceRecording: ... + + @distributed_trace_async + async def get_audio_item( + self, + agent_name: str, + conversation_id: str, + item_id: str, + **kwargs: Any + ) -> VoiceAudioItem: ... + + @distributed_trace_async + async def get_item( + self, + agent_name: str, + conversation_id: str, + item_id: str, + **kwargs: Any + ) -> VoiceAgentLiveConversationItem: ... + + @distributed_trace + def list( + self, + agent_name: str, + *, + before: Optional[str] = ..., + limit: Optional[int] = ..., + order: Optional[Union[str, PageOrder]] = ..., + **kwargs: Any + ) -> AsyncItemPaged[VoiceAgentLiveConversation]: ... + + @distributed_trace + def list_items( + self, + agent_name: str, + conversation_id: str, + *, + before: Optional[str] = ..., + limit: Optional[int] = ..., + order: Optional[Union[str, PageOrder]] = ..., + **kwargs: Any + ) -> AsyncItemPaged[VoiceAgentLiveConversationItem]: ... + + class azure.ai.projects.aio.operations.BetaVoiceAgentsOperations(GeneratedBetaVoiceAgentsOperations): conversations: BetaVoiceAgentsConversationsOperations + live: AsyncBetaLive + live_conversations: BetaVoiceAgentsLiveConversationsOperations realtime: AsyncBetaRealtime telephony: BetaVoiceAgentsTelephonyOperations @@ -20032,6 +20187,68 @@ namespace azure.ai.projects.operations ) -> ItemPaged[Insight]: ... + class azure.ai.projects.operations.BetaLive: + + def __init__( + self, + *args: Any, + **kwargs: Any + ) -> None: ... + + def connect( + self, + *, + agent_name: str, + agent_version: Optional[str] = ..., + api_version: Optional[str] = ..., + connection_url: Optional[str] = ..., + credential_scopes: Optional[list[str]] = ..., + extra_headers: Optional[Mapping[str, str]] = ..., + extra_query: Optional[Mapping[str, str]] = ..., + store: Optional[bool] = ..., + structured_inputs: Optional[Mapping[str, Any]] = ..., + **kwargs: Any + ) -> BetaLiveConnectionManager: ... + + + class azure.ai.projects.operations.BetaLiveConnection(_BaseWebSocketConnection[LiveServerEvent, LiveClientEvent]): implements ContextManager + property closed: bool # Read-only + + def __init__(self, connection: ClientConnection) -> None: ... + + def __iter__(self) -> Iterator[LiveServerEvent]: ... + + def __repr__(self) -> str: ... + + def close( + self, + *, + code: int = 1000, + reason: str = "" + ) -> None: ... + + def recv( + self, + *, + timeout: Optional[float] = ... + ) -> LiveServerEvent: ... + + def send(self, event: LiveClientEvent) -> None: ... + + + class azure.ai.projects.operations.BetaLiveConnectionManager(_BaseWebSocketConnectionManager[BetaLiveConnection]): implements ContextManager + + def __init__( + self, + *, + agent_version: Optional[str] = ..., + store: Optional[bool] = ..., + **kwargs: Any + ) -> None: ... + + def enter(self) -> BetaLiveConnection: ... + + class azure.ai.projects.operations.BetaMemoryStoresOperations(GenerateBetaMemoryStoresOperations): def __init__( @@ -20592,7 +20809,7 @@ namespace azure.ai.projects.operations ) -> BetaRealtimeConnectionManager: ... - class azure.ai.projects.operations.BetaRealtimeConnection: implements ContextManager + class azure.ai.projects.operations.BetaRealtimeConnection(_BaseWebSocketConnection[ServerEvent, ClientEvent]): implements ContextManager property closed: bool # Read-only def __init__(self, connection: ClientConnection) -> None: ... @@ -20617,7 +20834,7 @@ namespace azure.ai.projects.operations def send(self, event: ClientEvent) -> None: ... - class azure.ai.projects.operations.BetaRealtimeConnectionManager: implements ContextManager + class azure.ai.projects.operations.BetaRealtimeConnectionManager(_BaseWebSocketConnectionManager[BetaRealtimeConnection]): implements ContextManager def __init__( self, @@ -21183,8 +21400,101 @@ namespace azure.ai.projects.operations ) -> ItemPaged[VoiceResponse]: ... + class azure.ai.projects.operations.BetaVoiceAgentsLiveConversationsOperations: + + def __init__( + self, + *args, + **kwargs + ) -> None: ... + + @distributed_trace + def delete( + self, + agent_name: str, + conversation_id: str, + **kwargs: Any + ) -> None: ... + + @distributed_trace + def download_audio( + self, + agent_name: str, + conversation_id: str, + **kwargs: Any + ) -> Iterator[bytes]: ... + + @distributed_trace + def download_audio_item( + self, + agent_name: str, + conversation_id: str, + item_id: str, + **kwargs: Any + ) -> Iterator[bytes]: ... + + @distributed_trace + def get( + self, + agent_name: str, + conversation_id: str, + **kwargs: Any + ) -> VoiceAgentLiveConversation: ... + + @distributed_trace + def get_audio( + self, + agent_name: str, + conversation_id: str, + **kwargs: Any + ) -> VoiceRecording: ... + + @distributed_trace + def get_audio_item( + self, + agent_name: str, + conversation_id: str, + item_id: str, + **kwargs: Any + ) -> VoiceAudioItem: ... + + @distributed_trace + def get_item( + self, + agent_name: str, + conversation_id: str, + item_id: str, + **kwargs: Any + ) -> VoiceAgentLiveConversationItem: ... + + @distributed_trace + def list( + self, + agent_name: str, + *, + before: Optional[str] = ..., + limit: Optional[int] = ..., + order: Optional[Union[str, PageOrder]] = ..., + **kwargs: Any + ) -> ItemPaged[VoiceAgentLiveConversation]: ... + + @distributed_trace + def list_items( + self, + agent_name: str, + conversation_id: str, + *, + before: Optional[str] = ..., + limit: Optional[int] = ..., + order: Optional[Union[str, PageOrder]] = ..., + **kwargs: Any + ) -> ItemPaged[VoiceAgentLiveConversationItem]: ... + + class azure.ai.projects.operations.BetaVoiceAgentsOperations(GeneratedBetaVoiceAgentsOperations): conversations: BetaVoiceAgentsConversationsOperations + live: BetaLive + live_conversations: BetaVoiceAgentsLiveConversationsOperations realtime: BetaRealtime telephony: BetaVoiceAgentsTelephonyOperations diff --git a/sdk/ai/azure-ai-projects/api.metadata.yml b/sdk/ai/azure-ai-projects/api.metadata.yml index f13e30cf6939..7b106eba1f3f 100644 --- a/sdk/ai/azure-ai-projects/api.metadata.yml +++ b/sdk/ai/azure-ai-projects/api.metadata.yml @@ -1,4 +1,4 @@ -apiHash: e8c65b5d00eae3440a51d6626ac9a885a01f02fd53dfab0443c3bc54ea8eb5cd +apiHash: fe49747be0d1a22890bded4f0dcf7cc415a2631ede3f5490f900e270641e5ba3 packageVersion: 2.10.0 parserVersion: 0.3.31 pythonVersion: 3.12.10 diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/_live.py b/sdk/ai/azure-ai-projects/azure/ai/projects/_live.py new file mode 100644 index 000000000000..51bda73220db --- /dev/null +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/_live.py @@ -0,0 +1,284 @@ +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------- +"""Synchronous GPT Live WebSocket client, alongside the Realtime client.""" + +from __future__ import annotations + +import base64 +from typing import Any, Iterator, Mapping, Optional, TYPE_CHECKING, Union + +from . import models +from ._realtime import _BaseWebSocketConnection, _BaseWebSocketConnectionManager, _ConfigProvider +from ._live_utils import LiveClientEvent, LiveServerEvent, _LOGGER, _SERVER_EVENT_TYPES, _event, _live_query, _live_url + +if TYPE_CHECKING: + from websockets.sync.client import ClientConnection + +__all__ = ["BetaLive", "BetaLiveConnection", "BetaLiveConnectionManager", "LiveClientEvent", "LiveServerEvent"] + + +class _Resource: + def __init__(self, connection: BetaLiveConnection) -> None: + self._connection = connection + + +class TextResource(_Resource): + """Append instructions, silent thinking, or speakable commentary.""" + + def __init__(self, connection: BetaLiveConnection, event_model: type[models.LiveClientEvent]) -> None: + super().__init__(connection) + self._event_model = event_model + + def append(self, *, content: str, delegation_id: Optional[str], event_id: Optional[str] = None) -> None: + """Append text, optionally associated with a client delegation. + + :keyword str content: Text to append. + :keyword delegation_id: Delegation ID, or None for independent context. + :paramtype delegation_id: str or None + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + self._connection.send(_event(self._event_model, event_id, content=content, delegation_id=delegation_id)) + + +class InputAudioResource(_Resource): + """Stream primary-transport audio in the session's configured format.""" + + def append(self, *, audio: Union[str, bytes], event_id: Optional[str] = None) -> None: + """Send raw audio bytes or base64 without a container header. + + :keyword audio: Raw audio or an already base64-encoded string. + :paramtype audio: str or bytes + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + encoded = base64.b64encode(audio).decode("ascii") if isinstance(audio, bytes) else audio + self._connection.send(_event(models.LiveClientEventLiveInputAudioAppendEvent, event_id, audio=encoded)) + + def mute(self, *, event_id: Optional[str] = None) -> None: + """Mute input audio processing. + + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + self._connection.send(_event(models.LiveClientEventLiveInputAudioMuteParam, event_id)) + + def unmute(self, *, event_id: Optional[str] = None) -> None: + """Resume input audio processing. + + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + self._connection.send(_event(models.LiveClientEventLiveInputAudioUnmuteParam, event_id)) + + +class SessionResource(_Resource): + """Live session controls; startup is owned by Foundry, not the client.""" + + def __init__(self, connection: BetaLiveConnection) -> None: + super().__init__(connection) + self.input_audio = InputAudioResource(connection) + self.instructions = TextResource(connection, models.LiveClientEventLiveInstructionsAppendParam) + self.thinking = TextResource(connection, models.LiveClientEventLiveThinkingAppendParam) + self.commentary = TextResource(connection, models.LiveClientEventLiveCommentaryAppendParam) + + def update( + self, *, session: Union[models.LiveSessionUpdateParams, Mapping[str, Any]], event_id: Optional[str] = None + ) -> None: + """Update mutable delegation settings, not startup model/audio. + + :keyword session: Sparse session delegation settings. + :paramtype session: ~azure.ai.projects.models.LiveSessionUpdateParams or Mapping[str, Any] + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + self._connection.send(_event(models.LiveClientEventLiveSessionUpdateParam, event_id, session=session)) + + def close(self, *, event_id: Optional[str] = None) -> None: + """Request graceful session shutdown; receive session.closed before closing the socket. + + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + self._connection.send(_event(models.LiveClientEventLiveSessionCloseParam, event_id)) + + +class ResponseItemResource(_Resource): + """Append user messages or function results to a Responses delegation.""" + + def create(self, *, item: Mapping[str, Any], event_id: Optional[str] = None) -> None: + """Submit an item; requires Responses delegation. + + :keyword item: Responses input item, including function_call_output. + :paramtype item: Mapping[str, Any] + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + self._connection.send(_event(models.LiveClientEventLiveResponseItemCreateParam, event_id, item=dict(item))) + + +class ResponseResource(_Resource): + """Controls for the Responses backend, not Realtime response controls.""" + + def __init__(self, connection: BetaLiveConnection) -> None: + super().__init__(connection) + self.item = ResponseItemResource(connection) + + def create(self, *, event_id: Optional[str] = None) -> None: + """Request or continue a backend response after submitting tool results. + + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + self._connection.send(_event(models.LiveClientEventLiveResponseCreateParam, event_id)) + + +class BetaLiveConnection(_BaseWebSocketConnection[LiveServerEvent, LiveClientEvent]): + """An open Live connection. Server errors are returned as typed events. + + Local cancellation closes the transport through the context manager. Live + has no response.cancel command; session.close requests protocol shutdown. + """ + + _logger = _LOGGER + _server_event_types = _SERVER_EVENT_TYPES + _protocol_name = "Live" + + def __init__(self, connection: ClientConnection) -> None: + super().__init__(connection) + self.session = SessionResource(self) + self.response = ResponseResource(self) + + def __enter__(self) -> BetaLiveConnection: + return self + + def __iter__(self) -> Iterator[LiveServerEvent]: + return self._iter() + + def recv(self, *, timeout: Optional[float] = None) -> LiveServerEvent: + """Receive a generated Live event, or a mapping for an unknown event. + + :keyword timeout: Maximum receive wait in seconds; None waits indefinitely. + :paramtype timeout: float or None + :return: The server event. + :rtype: ~azure.ai.projects.operations.LiveServerEvent + :raises ConnectionResetError: On graceful or abnormal socket closure. + :raises TimeoutError: When the receive deadline expires. + """ + return super().recv(timeout=timeout) + + def send(self, event: LiveClientEvent) -> None: + """Send a generated Live event, a mapping, or a JSON string. + + :param event: The client event. + :type event: ~azure.ai.projects.operations.LiveClientEvent + :raises ValueError: If the supplied JSON string is invalid. + """ + super().send(event) + + +class BetaLiveConnectionManager(_BaseWebSocketConnectionManager[BetaLiveConnection]): + """Context manager for one Live session using the project's credential.""" + + _logger = _LOGGER + _subprotocol = None + _protocol_name = "Live" + + def __init__( + self, + *, + agent_version: Optional[str] = None, + store: Optional[bool] = None, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self._agent_version = agent_version + self._store = store + + def __enter__(self) -> BetaLiveConnection: + return self.enter() + + def enter(self) -> BetaLiveConnection: + """Open the Live connection without sending a second startup event. + + :return: The Live connection. + :rtype: ~azure.ai.projects.operations.BetaLiveConnection + """ + return super().enter() + + def _get_url(self) -> str: + return _live_url(self._endpoint, self._agent_name) + + def _get_query(self) -> dict[str, str]: + return _live_query( + self._api_version, self._agent_version, self._structured_inputs, self._store, self._extra_query + ) + + def _create_connection(self, connection: ClientConnection) -> BetaLiveConnection: + return BetaLiveConnection(connection) + + +class BetaLive: + """Live protocol entry point at client.beta.voice_agents.live. + + Foundry starts the session using the persisted Live agent definition. Receive + session.started before sending session commands; do not send session.start. + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + input_args = list(args) + client: _ConfigProvider = input_args.pop(0) if input_args else kwargs.pop("client") + self._config = client._config # pylint: disable=protected-access + + def connect( + self, + *, + agent_name: str, + agent_version: Optional[str] = None, + structured_inputs: Optional[Mapping[str, Any]] = None, + store: Optional[bool] = None, + connection_url: Optional[str] = None, + api_version: Optional[str] = None, + credential_scopes: Optional[list[str]] = None, + extra_query: Optional[Mapping[str, str]] = None, + extra_headers: Optional[Mapping[str, str]] = None, + **kwargs: Any, + ) -> BetaLiveConnectionManager: + """Open a Live WebSocket session; network I/O starts on context entry. + + :keyword str agent_name: Persisted Live agent name. + :keyword agent_version: Optional pinned agent version. + :paramtype agent_version: str or None + :keyword structured_inputs: Session template values. + :paramtype structured_inputs: Mapping[str, Any] or None + :keyword store: Optional conversation persistence override. + :paramtype store: bool or None + :keyword connection_url: Same-origin wss URL override. + :paramtype connection_url: str or None + :keyword api_version: API version override. + :paramtype api_version: str or None + :keyword credential_scopes: Token scope override. + :paramtype credential_scopes: list[str] or None + :keyword extra_query: Additional handshake query values. + :paramtype extra_query: Mapping[str, str] or None + :keyword extra_headers: Additional handshake headers. + :paramtype extra_headers: Mapping[str, str] or None + :return: A Live connection context manager. + :rtype: ~azure.ai.projects.operations.BetaLiveConnectionManager + """ + return BetaLiveConnectionManager( + endpoint=self._config.endpoint, + credential=self._config.credential, + credential_scopes=credential_scopes or self._config.credential_scopes, + api_version=api_version or self._config.api_version, + agent_name=agent_name, + agent_version=agent_version, + structured_inputs=structured_inputs, + store=store, + connection_url=connection_url, + extra_query=extra_query, + extra_headers=extra_headers, + **kwargs, + ) diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/_live_utils.py b/sdk/ai/azure-ai-projects/azure/ai/projects/_live_utils.py new file mode 100644 index 000000000000..6945be4c068e --- /dev/null +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/_live_utils.py @@ -0,0 +1,67 @@ +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------- +"""Shared protocol metadata for the handwritten Live clients. + +Foundry constructs session.start from the persisted agent definition. Live uses +its own event vocabulary; Realtime response.cancel and audio-buffer commands +must not be synthesized on this connection. +""" + +import json +import logging +from typing import Any, Mapping, Optional, Type, Union + +from . import models +from ._realtime import _to_ws_url, _USER_AGENT +from ._utils.model_base import Model, SdkJSONEncoder + +LiveClientEvent = Union[models.LiveClientEvent, str, Mapping[str, Any]] +LiveServerEvent = Union[models.LiveServerEvent, Mapping[str, Any]] +_LOGGER = logging.getLogger("azure.ai.projects.live") + + +def _server_event_types() -> dict[str, Type[Model]]: + result: dict[str, Type[Model]] = {} + for name, cls in models.LiveServerEvent.__mapping__.items(): + if isinstance(cls, type) and issubclass(cls, Model): + result[name] = cls + return result + + +_SERVER_EVENT_TYPES = _server_event_types() + + +def _live_url(endpoint: str, agent_name: str) -> str: + return _to_ws_url(endpoint, agent_name) + "/live/sessions" + + +def _live_query( + api_version: str, + agent_version: Optional[str], + structured_inputs: Optional[Mapping[str, Any]], + store: Optional[bool], + extra_query: Mapping[str, str], +) -> dict[str, str]: + result = {"api-version": api_version, "x-ms-client-sdk": _USER_AGENT} + result.update(extra_query) + if agent_version is not None: + result["x-agent-version-override"] = agent_version + if structured_inputs is not None: + if "structured_input" in result: + raise ValueError("structured_input and structured_inputs cannot be supplied together.") + result["structured_inputs"] = json.dumps(structured_inputs, cls=SdkJSONEncoder) + if "structured_inputs" in result and "structured_input" in result: + raise ValueError("structured_input and structured_inputs cannot be supplied together.") + if store is not None: + result["store"] = str(store).lower() + return result + + +def _event( + model: Type[models.LiveClientEvent], event_id: Optional[str] = None, **fields: Any +) -> models.LiveClientEvent: + if event_id is not None: + fields["event_id"] = event_id + return model(fields) diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py b/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py index d492b10b4d01..b11f64b00c31 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py @@ -24,6 +24,7 @@ from typing import ( Any, Dict, + Generic, Iterator, List, Mapping, @@ -31,6 +32,7 @@ Protocol, Tuple, Type, + TypeVar, TYPE_CHECKING, Union, cast, @@ -551,28 +553,22 @@ def cancel(self, *, response_id: Optional[str] = None, event_id: Optional[str] = ) -class BetaRealtimeConnection: # pylint: disable=too-many-instance-attributes - """An open realtime WebSocket connection to a voice agent. +_ServerEventT = TypeVar("_ServerEventT") +_ClientEventT = TypeVar("_ClientEventT") - Iterate over the connection to receive strongly-typed server events, and use the - sub-namespaces to send strongly-typed client events:: - with client.beta.voice_agents.realtime.connect(agent_name="my-agent") as conn: - for event in conn: - if event.type == RealtimeServerEventType.RESPONSE_DONE: - break - """ +class _BaseWebSocketConnection(Generic[_ServerEventT, _ClientEventT]): + """Shared synchronous WebSocket transport for the two voice protocols.""" + + _logger = _LOGGER + _server_event_types = _SERVER_EVENT_TYPES + _protocol_name = "realtime" def __init__(self, connection: "ClientConnection") -> None: self._connection = connection self._closed = False - self.session: SessionResource = SessionResource(self) - self.input_audio_buffer: InputAudioBufferResource = InputAudioBufferResource(self) - self.output_audio_buffer: OutputAudioBufferResource = OutputAudioBufferResource(self) - self.conversation: ConversationResource = ConversationResource(self) - self.response: ResponseResource = ResponseResource(self) - def __enter__(self) -> "BetaRealtimeConnection": + def __enter__(self) -> "_BaseWebSocketConnection[_ServerEventT, _ClientEventT]": return self def __exit__(self, *exc_details: Any) -> None: @@ -580,7 +576,7 @@ def __exit__(self, *exc_details: Any) -> None: def __repr__(self) -> str: state = "closed" if self.closed else "open" - return f"" + return f"<{type(self).__name__} [{state}]>" @property def closed(self) -> bool: @@ -590,76 +586,62 @@ def closed(self) -> bool: """ return self._closed - def __iter__(self) -> Iterator[ServerEvent]: + def __iter__(self) -> Iterator[_ServerEventT]: return self._iter() - def _iter(self) -> Iterator[ServerEvent]: + def _iter(self) -> Iterator[_ServerEventT]: while True: try: yield self.recv() except ConnectionResetError as exc: - # recv() below chains the *specific* websockets exception as the cause: a plain - # graceful closure has no cause (nothing went wrong), while an abnormal closure - # is chained from the ConnectionClosed that caused it. Only end iteration quietly - # for the former -- a `for event in conn:` caller must still see real failures - # (abnormal close codes, e.g. 1011) instead of silently observing end-of-stream. if exc.__cause__ is None: return raise - def recv(self, *, timeout: Optional[float] = None) -> ServerEvent: - """Receive and parse the next server event. - - Known event types are returned as their strongly-typed - ``VoiceAgentServerEventXxx`` model. Event types not (yet) represented by a - generated model are returned as a plain ``dict`` for forward compatibility. + def recv(self, *, timeout: Optional[float] = None) -> _ServerEventT: + """Receive a typed event, or a mapping for an unknown event. - :keyword timeout: Maximum time in seconds to wait for the next event. If ``None`` - (the default), block until an event is received. If no event arrives within - ``timeout`` seconds, raise :exc:`TimeoutError`. + :keyword timeout: Maximum receive wait in seconds; None waits indefinitely. :paramtype timeout: float or None - :return: The parsed server event. - :rtype: ~azure.ai.projects.ServerEvent - :raises ConnectionResetError: If the connection was closed by the server, gracefully or - otherwise. Iterating over the connection (``for event in conn:``) treats only a graceful - closure as end-of-stream and re-raises this for an abnormal one. - :raises TimeoutError: If ``timeout`` elapses before an event is received. + :return: The server event. + :rtype: object + :raises ConnectionResetError: On graceful or abnormal socket closure. + :raises TimeoutError: When the receive deadline expires. """ - from websockets.exceptions import ( # pylint: disable=import-outside-toplevel + from websockets.exceptions import ( ConnectionClosed, ConnectionClosedOK, - ) + ) # pylint: disable=import-outside-toplevel try: raw = self._connection.recv(timeout=timeout) except ConnectionClosedOK: self._closed = True - raise ConnectionResetError("The realtime connection was closed.") from None + raise ConnectionResetError(f"The {self._protocol_name} connection was closed.") from None except ConnectionClosed as exc: self._closed = True - raise ConnectionResetError(f"The realtime connection was closed abnormally: {exc}") from exc + raise ConnectionResetError(f"The {self._protocol_name} connection was closed abnormally: {exc}") from exc data = raw.decode("utf-8") if isinstance(raw, (bytes, bytearray)) else raw payload: Dict[str, Any] = json.loads(data) + if not isinstance(payload, dict): + raise ValueError("The WebSocket server event must be a JSON object.") event_type = payload.get("type") - _LOGGER.debug( + self._logger.debug( "WebSocket RECEIVE type=%s bytes=%d", event_type if isinstance(event_type, str) else "unknown", len(data.encode("utf-8")), ) - if not isinstance(event_type, str): - return payload - event_cls = _SERVER_EVENT_TYPES.get(event_type) - if event_cls is None: - return payload - return event_cls(payload) + event_cls = self._server_event_types.get(event_type) if isinstance(event_type, str) else None + return cast(_ServerEventT, event_cls(payload) if event_cls else payload) - def send(self, event: ClientEvent) -> None: - """Send a client event over the connection. + def send(self, event: _ClientEventT) -> None: + """Send a generated event, mapping, or JSON string. - :param event: A strongly-typed client event, a ready-made mapping, or a raw JSON string. - :type event: ~azure.ai.projects.ClientEvent or str - :raises ValueError: If ``event`` is a ``str`` that is not valid JSON. + :param event: The client event. + :type event: object + :raises ValueError: If the JSON string is invalid. """ + payload: str if isinstance(event, str): try: event_payload = json.loads(event) @@ -670,7 +652,7 @@ def send(self, event: ClientEvent) -> None: payload = json.dumps(event, cls=SdkJSONEncoder) event_payload = json.loads(payload) event_type = event_payload.get("type") if isinstance(event_payload, dict) else None - _LOGGER.debug( + self._logger.debug( "WebSocket SEND type=%s bytes=%d", event_type if isinstance(event_type, str) else "unknown", len(payload.encode("utf-8")), @@ -678,21 +660,107 @@ def send(self, event: ClientEvent) -> None: self._connection.send(payload) def close(self, *, code: int = 1000, reason: str = "") -> None: - """Close the connection. + """Close the socket once. - :keyword int code: The WebSocket close code. - :keyword str reason: The close reason. + :keyword int code: WebSocket close code. + :keyword str reason: WebSocket close reason. """ if self._closed: return - _LOGGER.debug("WebSocket CLOSE code=%d", code) + self._logger.debug("WebSocket CLOSE code=%d", code) try: self._connection.close(code=code, reason=reason) finally: self._closed = True -class BetaRealtimeConnectionManager: # pylint: disable=too-many-instance-attributes +class BetaRealtimeConnection( + _BaseWebSocketConnection[ServerEvent, ClientEvent] +): # pylint: disable=too-many-instance-attributes + """An open realtime WebSocket connection to a voice agent. + + Iterate over the connection to receive strongly-typed server events, and use the + sub-namespaces to send strongly-typed client events:: + + with client.beta.voice_agents.realtime.connect(agent_name="my-agent") as conn: + for event in conn: + if event.type == RealtimeServerEventType.RESPONSE_DONE: + break + """ + + def __init__(self, connection: "ClientConnection") -> None: + super().__init__(connection) + self.session: SessionResource = SessionResource(self) + self.input_audio_buffer: InputAudioBufferResource = InputAudioBufferResource(self) + self.output_audio_buffer: OutputAudioBufferResource = OutputAudioBufferResource(self) + self.conversation: ConversationResource = ConversationResource(self) + self.response: ResponseResource = ResponseResource(self) + + def __enter__(self) -> "BetaRealtimeConnection": + return self + + def __exit__(self, *exc_details: Any) -> None: + self.close() + + @property + def closed(self) -> bool: + """Whether the underlying WebSocket connection has been closed. + + :rtype: bool + """ + return self._closed + + def __iter__(self) -> Iterator[ServerEvent]: + return self._iter() + + def _iter(self) -> Iterator[ServerEvent]: + while True: + try: + yield self.recv() + except ConnectionResetError as exc: + # recv() below chains the *specific* websockets exception as the cause: a plain + # graceful closure has no cause (nothing went wrong), while an abnormal closure + # is chained from the ConnectionClosed that caused it. Only end iteration quietly + # for the former -- a `for event in conn:` caller must still see real failures + # (abnormal close codes, e.g. 1011) instead of silently observing end-of-stream. + if exc.__cause__ is None: + return + raise + + def recv(self, *, timeout: Optional[float] = None) -> ServerEvent: + """Receive and parse the next server event. + + Known event types are returned as their strongly-typed + ``VoiceAgentServerEventXxx`` model. Event types not (yet) represented by a + generated model are returned as a plain ``dict`` for forward compatibility. + + :keyword timeout: Maximum time in seconds to wait for the next event. If ``None`` + (the default), block until an event is received. If no event arrives within + ``timeout`` seconds, raise :exc:`TimeoutError`. + :paramtype timeout: float or None + :return: The parsed server event. + :rtype: ~azure.ai.projects.ServerEvent + :raises ConnectionResetError: If the connection was closed by the server, gracefully or + otherwise. Iterating over the connection (``for event in conn:``) treats only a graceful + closure as end-of-stream and re-raises this for an abnormal one. + :raises TimeoutError: If ``timeout`` elapses before an event is received. + """ + return super().recv(timeout=timeout) + + def send(self, event: ClientEvent) -> None: + """Send a client event over the connection. + + :param event: A strongly-typed client event, a ready-made mapping, or a raw JSON string. + :type event: ~azure.ai.projects.ClientEvent or str + :raises ValueError: If ``event`` is a ``str`` that is not valid JSON. + """ + super().send(event) + + +_ConnectionT = TypeVar("_ConnectionT", bound=_BaseWebSocketConnection) + + +class _BaseWebSocketConnectionManager(Generic[_ConnectionT]): # pylint: disable=too-many-instance-attributes """Context manager that opens a :class:`BetaRealtimeConnection`. Returned by :meth:`BetaRealtime.connect`; you normally use it as @@ -747,12 +815,16 @@ def __init__( # pylint: disable=too-many-arguments self._extra_query = dict(extra_query or {}) self._extra_headers = dict(extra_headers or {}) self._kwargs = kwargs - self._connection: Optional[BetaRealtimeConnection] = None + self._connection: Optional[_ConnectionT] = None - def __enter__(self) -> BetaRealtimeConnection: + _logger = _LOGGER + _subprotocol: Optional[str] = "realtime" + _protocol_name = "realtime" + + def __enter__(self) -> _ConnectionT: return self.enter() - def enter(self) -> BetaRealtimeConnection: # pylint: disable=too-many-locals + def enter(self) -> _ConnectionT: # pylint: disable=too-many-locals """Open the connection. :return: The live realtime connection. @@ -767,41 +839,30 @@ def enter(self) -> BetaRealtimeConnection: # pylint: disable=too-many-locals from websockets.typing import Subprotocol # pylint: disable=import-outside-toplevel except ImportError as exc: # pragma: no cover - dependency guard raise RuntimeError( - "The realtime client requires `websockets`. Install it with `pip install websockets`." + f"The {self._protocol_name} client requires `websockets`. Install it with `pip install websockets`." ) from exc # ``connection_url`` fully overrides the computed route (scheme/host/path). This is the # escape hatch used to reach a specific data-plane host/path directly. if self._connection_url is not None: _assert_trusted_connection_url(self._connection_url, self._endpoint) - url = self._connection_url or _to_ws_url(self._endpoint, self._agent_name) + url = self._connection_url or self._get_url() if self._transport is not None: url = _without_transport_query(url) if not url.startswith("wss://"): - raise ValueError("The realtime WebSocket URL must use wss:// to protect credentials in transit.") - - params: Dict[str, str] = {"api-version": self._api_version, "x-ms-client-sdk": _USER_AGENT} - if self._agent_session_id is not None: - params["agent_session_id"] = self._agent_session_id - if self._structured_inputs is not None: - # The service reads this from the `structured_input` query parameter (see the - # generated `build_beta_voice_agents_realtime_connect_voice_agent_request`), not a - # header -- it must be serialized and appended to the URL below, not sent as one. - params["structured_input"] = json.dumps(self._structured_inputs, cls=SdkJSONEncoder) - params.update(self._extra_query) - if self._transport is not None: - params["transport"] = self._transport + raise ValueError( + f"The {self._protocol_name} WebSocket URL must use wss:// to protect credentials in transit." + ) + params = self._get_query() if params: - # Preserve an existing query string on a `connection_url` override (for example a - # SAS-style `?sig=...`) instead of unconditionally appending a second `?`. delimiter = "&" if urlparse(url).query else "?" full_url = f"{url}{delimiter}{urlencode(params)}" else: full_url = url target_url = urlparse(url)._replace(query="", fragment="").geturl() - _LOGGER.debug("WebSocket CONNECT target=%s", target_url) + self._logger.debug("WebSocket CONNECT target=%s", target_url) token = self._credential.get_token(*self._credential_scopes) headers: Dict[str, str] = { @@ -810,27 +871,15 @@ def enter(self) -> BetaRealtimeConnection: # pylint: disable=too-many-locals } headers.update(self._extra_headers) if not _has_header_case_insensitive(headers, "User-Agent"): - # Only set our default if the caller didn't supply their own (in any casing) -- - # a plain dict merge would otherwise leave both as separate keys (HTTP header names - # are case-insensitive, but Python dict keys are not), sending two User-Agent-like - # headers instead of cleanly honoring the caller's override. headers["User-Agent"] = _USER_AGENT try: - # Force the "realtime" WebSocket subprotocol regardless of any caller-supplied - # override in ``self._kwargs``: the service requires this exact subprotocol, so - # silently accepting a different one here would just move the failure to a less - # clear error inside the handshake. Also disable ``websockets``' own - # ``user_agent_header`` default: unlike aiohttp, it is a wholly separate mechanism - # from ``additional_headers`` -- passing our own "User-Agent" there does not - # override it, so without this the connection would carry two distinct - # User-Agent-like values. ws_connect_kwargs = dict(self._kwargs) ws_connect_kwargs.pop("subprotocols", None) connection = _ws_connect( full_url, additional_headers=headers, - subprotocols=[Subprotocol("realtime")], + subprotocols=[Subprotocol(self._subprotocol)] if self._subprotocol else None, user_agent_header=None, **ws_connect_kwargs, ) @@ -838,19 +887,58 @@ def enter(self) -> BetaRealtimeConnection: # pylint: disable=too-many-locals if not isinstance(exc, Exception) or isinstance(exc, (ValueError, RuntimeError)): raise raise ConnectionError( - f"Failed to open the realtime WebSocket connection to voice agent " + f"Failed to open the {self._protocol_name} WebSocket connection to voice agent " f"'{self._agent_name}' at '{url}': {exc}" ) from exc - _LOGGER.debug("WebSocket CONNECTED target=%s", target_url) - self._connection = BetaRealtimeConnection(connection) + self._logger.debug("WebSocket CONNECTED target=%s", target_url) + self._connection = self._create_connection(connection) return self._connection + def _get_url(self) -> str: + return _to_ws_url(self._endpoint, self._agent_name) + + def _get_query(self) -> Dict[str, str]: + params: Dict[str, str] = {"api-version": self._api_version, "x-ms-client-sdk": _USER_AGENT} + if self._agent_session_id is not None: + params["agent_session_id"] = self._agent_session_id + if self._structured_inputs is not None: + # The service reads this from the `structured_input` query parameter (see the + # generated `build_beta_voice_agents_realtime_connect_voice_agent_request`), not a + # header -- it must be serialized and appended to the URL below, not sent as one. + params["structured_input"] = json.dumps(self._structured_inputs, cls=SdkJSONEncoder) + params.update(self._extra_query) + if self._transport is not None: + params["transport"] = self._transport + + return params + + def _create_connection(self, connection: "ClientConnection") -> _ConnectionT: + raise NotImplementedError + def __exit__(self, *exc_details: Any) -> None: if self._connection is not None: self._connection.close() self._connection = None +class BetaRealtimeConnectionManager(_BaseWebSocketConnectionManager[BetaRealtimeConnection]): + """Context manager for a synchronous Realtime connection.""" + + def __enter__(self) -> BetaRealtimeConnection: + return self.enter() + + def enter(self) -> BetaRealtimeConnection: + """Open the Realtime connection. + + :return: The Realtime connection. + :rtype: ~azure.ai.projects.operations.BetaRealtimeConnection + """ + return super().enter() + + def _create_connection(self, connection: "ClientConnection") -> BetaRealtimeConnection: + return BetaRealtimeConnection(connection) + + class BetaRealtime: # pylint: disable=too-few-public-methods """BetaRealtime streaming entry point, exposed as ``client.beta.voice_agents.realtime``. diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_live.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_live.py new file mode 100644 index 000000000000..aa1f9a515592 --- /dev/null +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_live.py @@ -0,0 +1,326 @@ +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------- +"""Asynchronous GPT Live WebSocket client, alongside the Realtime client.""" + +from __future__ import annotations + +import base64 +from typing import Any, AsyncIterator, Mapping, Optional, TYPE_CHECKING, Union + +from .. import models +from .._live_utils import LiveClientEvent, LiveServerEvent, _LOGGER, _SERVER_EVENT_TYPES, _event, _live_query, _live_url +from ._realtime import _BaseWebSocketConnection, _BaseWebSocketConnectionManager, _ConfigProvider + +if TYPE_CHECKING: + from aiohttp import ( # pylint: disable=networking-import-outside-azure-core-transport + ClientSession, + ClientWebSocketResponse, + ) + +__all__ = [ + "AsyncBetaLive", + "AsyncBetaLiveConnection", + "AsyncBetaLiveConnectionManager", + "LiveClientEvent", + "LiveServerEvent", +] + + +class _Resource: + def __init__(self, connection: AsyncBetaLiveConnection) -> None: + self._connection = connection + + +class TextResource(_Resource): + """Append instructions, silent thinking, or speakable commentary. + + :param connection: The parent Live connection. + :type connection: ~azure.ai.projects.aio.operations.AsyncBetaLiveConnection + :param event_model: The generated model for the text command. + :type event_model: type[~azure.ai.projects.models.LiveClientEvent] + """ + + def __init__(self, connection: AsyncBetaLiveConnection, event_model: type[models.LiveClientEvent]) -> None: + super().__init__(connection) + self._event_model = event_model + + async def append(self, *, content: str, delegation_id: Optional[str], event_id: Optional[str] = None) -> None: + """Append text, optionally associated with a client delegation. + + :keyword str content: Text to append. + :keyword delegation_id: Delegation ID, or None for independent context. + :paramtype delegation_id: str or None + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + await self._connection.send(_event(self._event_model, event_id, content=content, delegation_id=delegation_id)) + + +class InputAudioResource(_Resource): + """Stream primary-transport audio in the session's configured format.""" + + async def append(self, *, audio: Union[str, bytes], event_id: Optional[str] = None) -> None: + """Send raw audio bytes or base64 without a container header. + + :keyword audio: Raw audio or an already base64-encoded string. + :paramtype audio: str or bytes + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + encoded = base64.b64encode(audio).decode("ascii") if isinstance(audio, bytes) else audio + await self._connection.send(_event(models.LiveClientEventLiveInputAudioAppendEvent, event_id, audio=encoded)) + + async def mute(self, *, event_id: Optional[str] = None) -> None: + """Mute input audio processing. + + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + await self._connection.send(_event(models.LiveClientEventLiveInputAudioMuteParam, event_id)) + + async def unmute(self, *, event_id: Optional[str] = None) -> None: + """Resume input audio processing. + + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + await self._connection.send(_event(models.LiveClientEventLiveInputAudioUnmuteParam, event_id)) + + +class SessionResource(_Resource): + """Live session controls; startup is owned by Foundry, not the client. + + :param connection: The parent Live connection. + :type connection: ~azure.ai.projects.aio.operations.AsyncBetaLiveConnection + """ + + def __init__(self, connection: AsyncBetaLiveConnection) -> None: + super().__init__(connection) + self.input_audio = InputAudioResource(connection) + self.instructions = TextResource(connection, models.LiveClientEventLiveInstructionsAppendParam) + self.thinking = TextResource(connection, models.LiveClientEventLiveThinkingAppendParam) + self.commentary = TextResource(connection, models.LiveClientEventLiveCommentaryAppendParam) + + async def update( + self, *, session: Union[models.LiveSessionUpdateParams, Mapping[str, Any]], event_id: Optional[str] = None + ) -> None: + """Update mutable delegation settings, not startup model/audio. + + :keyword session: Sparse session delegation settings. + :paramtype session: ~azure.ai.projects.models.LiveSessionUpdateParams or Mapping[str, Any] + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + await self._connection.send(_event(models.LiveClientEventLiveSessionUpdateParam, event_id, session=session)) + + async def close(self, *, event_id: Optional[str] = None) -> None: + """Request graceful session shutdown; receive session.closed before closing the socket. + + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + await self._connection.send(_event(models.LiveClientEventLiveSessionCloseParam, event_id)) + + +class ResponseItemResource(_Resource): + """Append user messages or function results to a Responses delegation.""" + + async def create(self, *, item: Mapping[str, Any], event_id: Optional[str] = None) -> None: + """Submit an item; requires Responses delegation. + + :keyword item: Responses input item, including function_call_output. + :paramtype item: Mapping[str, Any] + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + await self._connection.send( + _event(models.LiveClientEventLiveResponseItemCreateParam, event_id, item=dict(item)) + ) + + +class ResponseResource(_Resource): + """Controls for the Responses backend, not Realtime response controls. + + :param connection: The parent Live connection. + :type connection: ~azure.ai.projects.aio.operations.AsyncBetaLiveConnection + """ + + def __init__(self, connection: AsyncBetaLiveConnection) -> None: + super().__init__(connection) + self.item = ResponseItemResource(connection) + + async def create(self, *, event_id: Optional[str] = None) -> None: + """Request or continue a backend response after submitting tool results. + + :keyword event_id: Optional client event identifier. + :paramtype event_id: str or None + """ + await self._connection.send(_event(models.LiveClientEventLiveResponseCreateParam, event_id)) + + +class AsyncBetaLiveConnection(_BaseWebSocketConnection[LiveServerEvent, LiveClientEvent]): + """An open Live connection. Server errors are returned as typed events. + + Local cancellation closes the transport through the context manager. Live + has no response.cancel command; session.close requests protocol shutdown. + + :param connection: The open WebSocket. + :type connection: ~aiohttp.ClientWebSocketResponse + :param session: The HTTP session that owns the WebSocket. + :type session: ~aiohttp.ClientSession + """ + + _logger = _LOGGER + _server_event_types = _SERVER_EVENT_TYPES + _protocol_name = "Live" + + def __init__(self, connection: ClientWebSocketResponse, session: ClientSession) -> None: + super().__init__(connection, session) + self.session = SessionResource(self) + self.response = ResponseResource(self) + + async def __aenter__(self) -> AsyncBetaLiveConnection: + return self + + def __aiter__(self) -> AsyncIterator[LiveServerEvent]: + return super().__aiter__() + + async def recv(self) -> LiveServerEvent: + """Receive a generated Live event, or a mapping for an unknown event. + + :return: The server event. + :rtype: ~azure.ai.projects.aio.operations.LiveServerEvent + :raises ConnectionResetError: On graceful or abnormal socket closure. + :raises ValueError: If the server event is not a JSON object. + """ + return await super().recv() + + async def send(self, event: LiveClientEvent) -> None: + """Send a generated Live event, a mapping, or a JSON string. + + :param event: The client event. + :type event: ~azure.ai.projects.aio.operations.LiveClientEvent + :raises ValueError: If the supplied JSON string is invalid. + """ + await super().send(event) + + +class AsyncBetaLiveConnectionManager(_BaseWebSocketConnectionManager[AsyncBetaLiveConnection]): + """Context manager for one Live session using the project's credential. + + :keyword agent_version: Optional pinned agent version. + :paramtype agent_version: str or None + :keyword store: Optional conversation persistence override. + :paramtype store: bool or None + """ + + _logger = _LOGGER + _subprotocol = None + _protocol_name = "Live" + + def __init__( + self, + *, + agent_version: Optional[str] = None, + store: Optional[bool] = None, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self._agent_version = agent_version + self._store = store + + async def __aenter__(self) -> AsyncBetaLiveConnection: + return await self.enter() + + async def enter(self) -> AsyncBetaLiveConnection: + """Open the Live connection without sending a second startup event. + + :return: The Live connection. + :rtype: ~azure.ai.projects.aio.operations.AsyncBetaLiveConnection + :raises RuntimeError: If ``aiohttp`` is not installed. + :raises ValueError: If the computed or supplied WebSocket URL does not use ``wss://``. + :raises ConnectionError: If the WebSocket upgrade handshake fails. + """ + return await super().enter() + + def _get_url(self) -> str: + return _live_url(self._endpoint, self._agent_name) + + def _get_query(self) -> dict[str, str]: + return _live_query( + self._api_version, self._agent_version, self._structured_inputs, self._store, self._extra_query + ) + + def _create_connection( + self, connection: ClientWebSocketResponse, session: ClientSession + ) -> AsyncBetaLiveConnection: + return AsyncBetaLiveConnection(connection, session) + + +class AsyncBetaLive: + """Live protocol entry point at client.beta.voice_agents.live. + + Foundry starts the session using the persisted Live agent definition. Receive + session.started before sending session commands; do not send session.start. + + :param args: The voice-agent sub-client whose shared configuration is reused. + :type args: ~azure.ai.projects.aio.operations.BetaVoiceAgentsOperations + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + input_args = list(args) + client: _ConfigProvider = input_args.pop(0) if input_args else kwargs.pop("client") + self._config = client._config # pylint: disable=protected-access + + def connect( + self, + *, + agent_name: str, + agent_version: Optional[str] = None, + structured_inputs: Optional[Mapping[str, Any]] = None, + store: Optional[bool] = None, + connection_url: Optional[str] = None, + api_version: Optional[str] = None, + credential_scopes: Optional[list[str]] = None, + extra_query: Optional[Mapping[str, str]] = None, + extra_headers: Optional[Mapping[str, str]] = None, + **kwargs: Any, + ) -> AsyncBetaLiveConnectionManager: + """Open a Live WebSocket session; network I/O starts on context entry. + + :keyword str agent_name: Persisted Live agent name. + :keyword agent_version: Optional pinned agent version. + :paramtype agent_version: str or None + :keyword structured_inputs: Session template values. + :paramtype structured_inputs: Mapping[str, Any] or None + :keyword store: Optional conversation persistence override. + :paramtype store: bool or None + :keyword connection_url: Same-origin wss URL override. + :paramtype connection_url: str or None + :keyword api_version: API version override. + :paramtype api_version: str or None + :keyword credential_scopes: Token scope override. + :paramtype credential_scopes: list[str] or None + :keyword extra_query: Additional handshake query values. + :paramtype extra_query: Mapping[str, str] or None + :keyword extra_headers: Additional handshake headers. + :paramtype extra_headers: Mapping[str, str] or None + :return: An asynchronous Live connection context manager. + :rtype: ~azure.ai.projects.aio.operations.AsyncBetaLiveConnectionManager + """ + return AsyncBetaLiveConnectionManager( + endpoint=self._config.endpoint, + credential=self._config.credential, + credential_scopes=credential_scopes or self._config.credential_scopes, + api_version=api_version or self._config.api_version, + agent_name=agent_name, + agent_version=agent_version, + structured_inputs=structured_inputs, + store=store, + connection_url=connection_url, + extra_query=extra_query, + extra_headers=extra_headers, + **kwargs, + ) diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py index 67ffc9df304c..d5c7d1a667f4 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py @@ -40,12 +40,14 @@ AsyncIterator, cast, Dict, + Generic, List, Mapping, Optional, Protocol, Tuple, Type, + TypeVar, TYPE_CHECKING, Union, ) @@ -573,31 +575,29 @@ class _AbnormalWebSocketClosure(Exception): from a normal end of stream, which chains no cause.""" -class AsyncBetaRealtimeConnection: # pylint: disable=too-many-instance-attributes - """An open realtime WebSocket connection to a voice agent. +_ServerEventT = TypeVar("_ServerEventT", bound=Union[_Model, Mapping[str, Any]]) +_ClientEventT = TypeVar("_ClientEventT", bound=Union[_Model, str, Mapping[str, Any]]) - Iterate over the connection to receive strongly-typed server events, and use the - sub-namespaces to send strongly-typed client events:: - async with client.beta.voice_agents.realtime.connect(agent_name="my-agent") as conn: - await conn.input_audio_buffer.append(audio=chunk) - await conn.input_audio_buffer.commit() - await conn.response.create() - async for event in conn: - if event.type == RealtimeServerEventType.RESPONSE_DONE: - break +class _BaseWebSocketConnection(Generic[_ServerEventT, _ClientEventT]): + """Shared asynchronous WebSocket transport for the two voice protocols. + + :param connection: The open WebSocket. + :type connection: ~aiohttp.ClientWebSocketResponse + :param session: The HTTP session that owns the WebSocket. + :type session: ~aiohttp.ClientSession """ + _logger = _LOGGER + _server_event_types = _SERVER_EVENT_TYPES + _protocol_name = "realtime" + def __init__(self, connection: "ClientWebSocketResponse", session: "ClientSession") -> None: self._connection = connection self._session = session - self.session: SessionResource = SessionResource(self) - self.input_audio_buffer: InputAudioBufferResource = InputAudioBufferResource(self) - self.output_audio_buffer: OutputAudioBufferResource = OutputAudioBufferResource(self) - self.conversation: ConversationResource = ConversationResource(self) - self.response: ResponseResource = ResponseResource(self) + self._closed = False - async def __aenter__(self) -> "AsyncBetaRealtimeConnection": + async def __aenter__(self) -> "_BaseWebSocketConnection[_ServerEventT, _ClientEventT]": return self async def __aexit__(self, *exc_details: Any) -> None: @@ -605,7 +605,7 @@ async def __aexit__(self, *exc_details: Any) -> None: def __repr__(self) -> str: state = "closed" if self.closed else "open" - return f"" + return f"<{type(self).__name__} [{state}]>" @property def closed(self) -> bool: @@ -615,36 +615,25 @@ def closed(self) -> bool: """ return self._connection.closed - def __aiter__(self) -> AsyncIterator[ServerEvent]: + def __aiter__(self) -> AsyncIterator[_ServerEventT]: return self._iter() - async def _iter(self) -> AsyncIterator[ServerEvent]: + async def _iter(self) -> AsyncIterator[_ServerEventT]: while True: try: yield await self.recv() except ConnectionResetError as exc: - # recv() below only chains a cause for a non-graceful closure or transport error - # (an abnormal close code, or the real exception behind a WSMsgType.ERROR); a - # graceful closure chains none. A `for event in conn:` caller must still see real - # failures instead of silently observing end-of-stream, so only the former ends - # iteration quietly. if exc.__cause__ is None: return raise - async def recv(self) -> ServerEvent: - """Receive and parse the next server event. + async def recv(self) -> _ServerEventT: + """Receive a typed event, or a mapping for an unknown event. - Known event types are returned as their strongly-typed - ``VoiceAgentServerEventXxx`` model. Event types not (yet) represented by a - generated model are returned as a plain ``dict`` for forward compatibility. - - :return: The parsed server event. - :rtype: ~azure.ai.projects.aio.ServerEvent - :raises ConnectionResetError: If the connection was closed by the server, gracefully or - otherwise, or if the transport reported an error. Iterating over the connection - (``async for event in conn:``) treats only a graceful closure as end-of-stream and - re-raises this for an abnormal one. + :return: The server event. + :rtype: object + :raises ConnectionResetError: On graceful or abnormal socket closure. + :raises ValueError: If the server event is not a JSON object. """ import aiohttp # pylint: disable=import-outside-toplevel @@ -659,35 +648,35 @@ async def recv(self) -> ServerEvent: code = self._connection.close_code if code not in (1000, 1001): raise ConnectionResetError( - f"The realtime connection was closed abnormally (code {code!r})." + f"The {self._protocol_name} connection was closed abnormally (code {code!r})." ) from _AbnormalWebSocketClosure(code) - raise ConnectionResetError("The realtime connection was closed.") + raise ConnectionResetError(f"The {self._protocol_name} connection was closed.") if msg.type == aiohttp.WSMsgType.ERROR: - raise ConnectionResetError( - "The realtime connection encountered an error." - ) from self._connection.exception() + error = self._connection.exception() + raise ConnectionResetError(f"The {self._protocol_name} connection encountered an error.") from ( + error if error is not None else _AbnormalWebSocketClosure() + ) raw = msg.data.decode("utf-8") if msg.type == aiohttp.WSMsgType.BINARY else msg.data payload: Dict[str, Any] = json.loads(raw) + if not isinstance(payload, dict): + raise ValueError("The WebSocket server event must be a JSON object.") event_type = payload.get("type") - _LOGGER.debug( + self._logger.debug( "WebSocket RECEIVE type=%s bytes=%d", event_type if isinstance(event_type, str) else "unknown", len(raw.encode("utf-8")), ) - if not isinstance(event_type, str): - return payload - event_cls = _SERVER_EVENT_TYPES.get(event_type) - if event_cls is None: - return payload - return event_cls(payload) + event_cls = self._server_event_types.get(event_type) if isinstance(event_type, str) else None + return cast(_ServerEventT, event_cls(payload) if event_cls else payload) - async def send(self, event: ClientEvent) -> None: - """Send a client event over the connection. + async def send(self, event: _ClientEventT) -> None: + """Send a generated event, mapping, or JSON string. - :param event: A strongly-typed client event, a ready-made mapping, or a raw JSON string. - :type event: ~azure.ai.projects.aio.ClientEvent or str - :raises ValueError: If ``event`` is a ``str`` that is not valid JSON. + :param event: The client event. + :type event: object + :raises ValueError: If the JSON string is invalid. """ + payload: str if isinstance(event, str): try: event_payload = json.loads(event) @@ -698,7 +687,7 @@ async def send(self, event: ClientEvent) -> None: payload = json.dumps(event, cls=SdkJSONEncoder) event_payload = json.loads(payload) event_type = event_payload.get("type") if isinstance(event_payload, dict) else None - _LOGGER.debug( + self._logger.debug( "WebSocket SEND type=%s bytes=%d", event_type if isinstance(event_type, str) else "unknown", len(payload.encode("utf-8")), @@ -706,23 +695,103 @@ async def send(self, event: ClientEvent) -> None: await self._connection.send_str(payload) async def close(self, *, code: int = 1000, reason: str = "") -> None: - """Close the connection and release the underlying HTTP session. + """Close the socket and its HTTP session once. - :keyword int code: The WebSocket close code. - :keyword str reason: The close reason. + :keyword int code: WebSocket close code. + :keyword str reason: WebSocket close reason. """ - _LOGGER.debug("WebSocket CLOSE code=%d", code) + if self._closed: + return + self._closed = True + self._logger.debug("WebSocket CLOSE code=%d", code) try: await self._connection.close(code=code, message=reason.encode("utf-8")) finally: await self._session.close() -class AsyncBetaRealtimeConnectionManager: # pylint: disable=too-many-instance-attributes - """Async context manager that opens an :class:`AsyncBetaRealtimeConnection`. +class AsyncBetaRealtimeConnection( # pylint: disable=too-many-instance-attributes + _BaseWebSocketConnection[ServerEvent, ClientEvent] +): + """An open realtime WebSocket connection to a voice agent. - Returned by :meth:`AsyncBetaRealtime.connect`; you normally use it as - ``async with client.beta.voice_agents.realtime.connect(...) as conn:``. + Iterate over the connection to receive strongly-typed server events, and use the + sub-namespaces to send strongly-typed client events:: + + async with client.beta.voice_agents.realtime.connect(agent_name="my-agent") as conn: + await conn.input_audio_buffer.append(audio=chunk) + await conn.input_audio_buffer.commit() + await conn.response.create() + async for event in conn: + if event.type == RealtimeServerEventType.RESPONSE_DONE: + break + """ + + def __init__(self, connection: "ClientWebSocketResponse", session: "ClientSession") -> None: + super().__init__(connection, session) + self.session: SessionResource = SessionResource(self) + self.input_audio_buffer: InputAudioBufferResource = InputAudioBufferResource(self) + self.output_audio_buffer: OutputAudioBufferResource = OutputAudioBufferResource(self) + self.conversation: ConversationResource = ConversationResource(self) + self.response: ResponseResource = ResponseResource(self) + + async def __aenter__(self) -> "AsyncBetaRealtimeConnection": + return self + + async def __aexit__(self, *exc_details: Any) -> None: + await super().__aexit__(*exc_details) + + @property + def closed(self) -> bool: + """Whether the underlying WebSocket connection has been closed. + + :rtype: bool + """ + return super().closed + + def __aiter__(self) -> AsyncIterator[ServerEvent]: + return super().__aiter__() + + async def recv(self) -> ServerEvent: + """Receive and parse the next server event. + + Known event types are returned as their strongly-typed + ``VoiceAgentServerEventXxx`` model. Event types not (yet) represented by a + generated model are returned as a plain ``dict`` for forward compatibility. + + :return: The parsed server event. + :rtype: ~azure.ai.projects.aio.ServerEvent + :raises ConnectionResetError: If the connection was closed by the server, gracefully or + otherwise, or if the transport reported an error. Iterating over the connection + (``async for event in conn:``) treats only a graceful closure as end-of-stream and + re-raises this for an abnormal one. + :raises ValueError: If the server event is not a JSON object. + """ + return await super().recv() + + async def send(self, event: ClientEvent) -> None: + """Send a client event over the connection. + + :param event: A strongly-typed client event, a ready-made mapping, or a raw JSON string. + :type event: ~azure.ai.projects.aio.ClientEvent or str + :raises ValueError: If ``event`` is a ``str`` that is not valid JSON. + """ + await super().send(event) + + async def close(self, *, code: int = 1000, reason: str = "") -> None: + """Close the connection and release the underlying HTTP session. + + :keyword int code: The WebSocket close code. + :keyword str reason: The close reason. + """ + await super().close(code=code, reason=reason) + + +_ConnectionT = TypeVar("_ConnectionT", bound=_BaseWebSocketConnection[Any, Any]) + + +class _BaseWebSocketConnectionManager(Generic[_ConnectionT]): # pylint: disable=too-many-instance-attributes + """Shared asynchronous voice protocol handshake and connection lifetime. :keyword str endpoint: The Foundry project endpoint. :keyword credential: The credential used for the WebSocket handshake. @@ -733,7 +802,7 @@ class AsyncBetaRealtimeConnectionManager: # pylint: disable=too-many-instance-a :keyword str agent_name: The name of the voice agent. :keyword agent_session_id: An optional session correlation identifier. :paramtype agent_session_id: str or None - :keyword transport: The optional connection transport. The SDK encodes its wire value internally. + :keyword transport: The optional Realtime connection transport. :paramtype transport: str or ~azure.ai.projects.models.VoiceAgentTransport or None :keyword structured_inputs: Optional structured prompt inputs. :paramtype structured_inputs: Mapping[str, Any] or None @@ -745,6 +814,10 @@ class AsyncBetaRealtimeConnectionManager: # pylint: disable=too-many-instance-a :paramtype extra_headers: Mapping[str, str] or None """ + _logger = _LOGGER + _subprotocol: Optional[str] = "realtime" + _protocol_name = "realtime" + def __init__( # pylint: disable=too-many-arguments self, *, @@ -773,12 +846,12 @@ def __init__( # pylint: disable=too-many-arguments self._extra_query = dict(extra_query or {}) self._extra_headers = dict(extra_headers or {}) self._kwargs = kwargs - self._connection: Optional[AsyncBetaRealtimeConnection] = None + self._connection: Optional[_ConnectionT] = None - async def __aenter__(self) -> AsyncBetaRealtimeConnection: + async def __aenter__(self) -> _ConnectionT: return await self.enter() - async def enter(self) -> AsyncBetaRealtimeConnection: # pylint: disable=too-many-locals + async def enter(self) -> _ConnectionT: # pylint: disable=too-many-locals """Open the connection. :return: The live realtime connection. @@ -792,74 +865,135 @@ async def enter(self) -> AsyncBetaRealtimeConnection: # pylint: disable=too-man import aiohttp # pylint: disable=import-outside-toplevel except ImportError as exc: # pragma: no cover - dependency guard raise RuntimeError( - "The realtime client requires `aiohttp`. Install it with `pip install aiohttp`." + f"The {self._protocol_name} client requires `aiohttp`. Install it with `pip install aiohttp`." ) from exc # ``connection_url`` fully overrides the computed route (scheme/host/path). This is the # escape hatch used to reach a specific data-plane host/path directly. if self._connection_url is not None: _assert_trusted_connection_url(self._connection_url, self._endpoint) - url = self._connection_url or _to_ws_url(self._endpoint, self._agent_name) + url = self._connection_url or self._get_url() if self._transport is not None: url = _without_transport_query(url) if not url.startswith("wss://"): - raise ValueError("The realtime WebSocket URL must use wss:// to protect credentials in transit.") + raise ValueError( + f"The {self._protocol_name} WebSocket URL must use wss:// to protect credentials in transit." + ) - params: Dict[str, str] = {"api-version": self._api_version, "x-ms-client-sdk": _USER_AGENT} - if self._agent_session_id is not None: - params["agent_session_id"] = self._agent_session_id - if self._structured_inputs is not None: - # The service reads this from the `structured_input` query parameter (see the - # generated `build_beta_voice_agents_realtime_connect_voice_agent_request`), not a - # header -- aiohttp appends `params` to the URL for us below. - params["structured_input"] = json.dumps(self._structured_inputs, cls=SdkJSONEncoder) - params.update(self._extra_query) - if self._transport is not None: - params["transport"] = self._transport + # Protocol-specific query values include the x-ms-client-sdk identification. + params = self._get_query() target_url = urlparse(url)._replace(query="", fragment="").geturl() - _LOGGER.debug("WebSocket CONNECT target=%s", target_url) - - token = await self._credential.get_token(*self._credential_scopes) - headers: Dict[str, str] = { - "Authorization": "Bearer " + token.token, - _FOUNDRY_FEATURES_HEADER_NAME: _VOICE_AGENT_FEATURE_HEADER, - } - headers.update(self._extra_headers) - if not _has_header_case_insensitive(headers, "User-Agent"): - # Only set our default if the caller didn't supply their own (in any casing) -- - # a plain dict merge would otherwise leave both as separate keys (HTTP header names - # are case-insensitive, but Python dict keys are not), sending two User-Agent-like - # headers instead of cleanly honoring the caller's override. - headers["User-Agent"] = _USER_AGENT + self._logger.debug("WebSocket CONNECT target=%s", target_url) session = aiohttp.ClientSession() try: - # Force the "realtime" WebSocket subprotocol regardless of any caller-supplied - # override in ``self._kwargs``: the service requires this exact subprotocol, so - # silently accepting a different one here would just move the failure to a less - # clear error inside aiohttp's handshake. + token = await self._credential.get_token(*self._credential_scopes) + headers: Dict[str, str] = { + "Authorization": "Bearer " + token.token, + _FOUNDRY_FEATURES_HEADER_NAME: _VOICE_AGENT_FEATURE_HEADER, + } + headers.update(self._extra_headers) + if not _has_header_case_insensitive(headers, "User-Agent"): + # Header names are case-insensitive; preserve caller overrides in any casing. + headers["User-Agent"] = _USER_AGENT + + # Each protocol owns its handshake subprotocol, not the caller's kwargs. ws_connect_kwargs = dict(self._kwargs) ws_connect_kwargs.pop("protocols", None) - connection = await session.ws_connect( - url, headers=headers, params=params, protocols=("realtime",), **ws_connect_kwargs - ) - except BaseException as exc: + try: + connection = await session.ws_connect( + url, + headers=headers, + params=params, + protocols=(self._subprotocol,) if self._subprotocol else (), + **ws_connect_kwargs, + ) + except Exception as exc: + if isinstance(exc, (ValueError, RuntimeError)): + raise + raise ConnectionError( + f"Failed to open the {self._protocol_name} WebSocket connection to voice agent " + f"'{self._agent_name}' at '{url}': {exc}" + ) from exc + self._connection = self._create_connection(cast("ClientWebSocketResponse", connection), session) + self._logger.debug("WebSocket CONNECTED target=%s", target_url) + except BaseException: await session.close() - if not isinstance(exc, Exception) or isinstance(exc, (ValueError, RuntimeError)): - raise - raise ConnectionError( - f"Failed to open the realtime WebSocket connection to voice agent " - f"'{self._agent_name}' at '{url}': {exc}" - ) from exc - _LOGGER.debug("WebSocket CONNECTED target=%s", target_url) - self._connection = AsyncBetaRealtimeConnection(cast("ClientWebSocketResponse", connection), session) + raise return self._connection + def _get_url(self) -> str: + return _to_ws_url(self._endpoint, self._agent_name) + + def _get_query(self) -> Dict[str, str]: + params: Dict[str, str] = {"api-version": self._api_version, "x-ms-client-sdk": _USER_AGENT} + if self._agent_session_id is not None: + params["agent_session_id"] = self._agent_session_id + if self._structured_inputs is not None: + # Realtime expects structured_input (singular), serialized as JSON. + params["structured_input"] = json.dumps(self._structured_inputs, cls=SdkJSONEncoder) + params.update(self._extra_query) + if self._transport is not None: + params["transport"] = self._transport + return params + + def _create_connection(self, connection: "ClientWebSocketResponse", session: "ClientSession") -> _ConnectionT: + raise NotImplementedError + async def __aexit__(self, *exc_details: Any) -> None: - if self._connection is not None: - await self._connection.close() - self._connection = None + connection = self._connection + self._connection = None + if connection is not None: + await connection.close() + + +class AsyncBetaRealtimeConnectionManager(_BaseWebSocketConnectionManager[AsyncBetaRealtimeConnection]): + """Async context manager that opens an :class:`AsyncBetaRealtimeConnection`. + + Returned by :meth:`AsyncBetaRealtime.connect`; you normally use it as + ``async with client.beta.voice_agents.realtime.connect(...) as conn:``. + + :keyword str endpoint: The Foundry project endpoint. + :keyword credential: The credential used for the WebSocket handshake. + :paramtype credential: ~azure.core.credentials_async.AsyncTokenCredential + :keyword credential_scopes: The token scopes for the handshake. + :paramtype credential_scopes: list[str] + :keyword str api_version: The API version for the handshake. + :keyword str agent_name: The name of the voice agent. + :keyword agent_session_id: An optional session correlation identifier. + :paramtype agent_session_id: str or None + :keyword transport: The optional connection transport. The SDK encodes its wire value internally. + :paramtype transport: str or ~azure.ai.projects.models.VoiceAgentTransport or None + :keyword structured_inputs: Optional structured prompt inputs. + :paramtype structured_inputs: Mapping[str, Any] or None + :keyword connection_url: An optional WebSocket URL overriding the computed route. + :paramtype connection_url: str or None + :keyword extra_query: Additional handshake query parameters. + :paramtype extra_query: Mapping[str, str] or None + :keyword extra_headers: Additional handshake headers. + :paramtype extra_headers: Mapping[str, str] or None + """ + + async def __aenter__(self) -> AsyncBetaRealtimeConnection: + return await self.enter() + + async def enter(self) -> AsyncBetaRealtimeConnection: + """Open the connection. + + :return: The live realtime connection. + :rtype: ~azure.ai.projects.aio.AsyncBetaRealtimeConnection + :raises RuntimeError: If ``aiohttp`` is not installed. + :raises ValueError: If the computed or supplied WebSocket URL does not use ``wss://``. + :raises ConnectionError: If the WebSocket upgrade handshake fails (for example, a + network error, DNS failure, or a non-101 response from the service). + """ + return await super().enter() + + def _create_connection( + self, connection: "ClientWebSocketResponse", session: "ClientSession" + ) -> AsyncBetaRealtimeConnection: + return AsyncBetaRealtimeConnection(connection, session) class AsyncBetaRealtime: # pylint: disable=too-few-public-methods diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch.py index 2ca0414ab9ea..92281b3eb90d 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/operations/_patch.py @@ -27,6 +27,13 @@ ConversationItem, ServerEvent, ) +from .._live import ( + AsyncBetaLive, + AsyncBetaLiveConnection, + AsyncBetaLiveConnectionManager, + LiveClientEvent, + LiveServerEvent, +) from ._operations import ( BetaEvaluationTaxonomiesOperations, BetaInsightsOperations, @@ -36,6 +43,7 @@ BetaSchedulesOperations, BetaSkillsOperations, BetaVoiceAgentsConversationsOperations, + BetaVoiceAgentsLiveConversationsOperations, BetaVoiceAgentsOperations as GeneratedBetaVoiceAgentsOperations, BetaVoiceAgentsTelephonyOperations, ) @@ -57,6 +65,10 @@ class BetaVoiceAgentsOperations(GeneratedBetaVoiceAgentsOperations): """:class:`~azure.ai.projects.aio.operations.BetaVoiceAgentsTelephonyOperations` operations""" realtime: AsyncBetaRealtime """:class:`~azure.ai.projects.aio.operations.AsyncBetaRealtime` operations""" + live: AsyncBetaLive + """:class:`~azure.ai.projects.aio.operations.AsyncBetaLive` operations""" + live_conversations: BetaVoiceAgentsLiveConversationsOperations + """:class:`~azure.ai.projects.aio.operations.BetaVoiceAgentsLiveConversationsOperations` operations""" def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) @@ -64,6 +76,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: # pipeline has no way to keep a WebSocket upgrade's resulting socket alive. Add our # hand-written client, which manages a real, long-lived connection, in its place. self.realtime = AsyncBetaRealtime(self) + self.live = AsyncBetaLive(self) class BetaOperations(GeneratedBetaOperations): @@ -127,6 +140,11 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: "AsyncBetaRealtime", "AsyncBetaRealtimeConnection", "AsyncBetaRealtimeConnectionManager", + "AsyncBetaLive", + "AsyncBetaLiveConnection", + "AsyncBetaLiveConnectionManager", + "LiveClientEvent", + "LiveServerEvent", "BetaAgentInsightMonitorsOperations", "BetaAgentsOperations", "BetaEvaluationTaxonomiesOperations", @@ -139,6 +157,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: "BetaSchedulesOperations", "BetaSkillsOperations", "BetaVoiceAgentsConversationsOperations", + "BetaVoiceAgentsLiveConversationsOperations", "BetaVoiceAgentsOperations", "BetaVoiceAgentsTelephonyOperations", "ClientEvent", diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch.py b/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch.py index a29e31ebd39b..e548c1d46863 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/operations/_patch.py @@ -20,6 +20,7 @@ ConversationItem, ServerEvent, ) +from .._live import BetaLive, BetaLiveConnection, BetaLiveConnectionManager, LiveClientEvent, LiveServerEvent from ._patch_agents import AgentsOperations, BetaAgentsOperations from ._patch_agent_insights import BetaAgentInsightMonitorsOperations from ._patch_datasets import DatasetsOperations @@ -38,6 +39,7 @@ BetaSchedulesOperations, BetaSkillsOperations, BetaVoiceAgentsConversationsOperations, + BetaVoiceAgentsLiveConversationsOperations, BetaVoiceAgentsOperations as GeneratedBetaVoiceAgentsOperations, BetaVoiceAgentsTelephonyOperations, ) @@ -137,6 +139,10 @@ class BetaVoiceAgentsOperations(GeneratedBetaVoiceAgentsOperations): """:class:`~azure.ai.projects.operations.BetaVoiceAgentsTelephonyOperations` operations""" realtime: BetaRealtime """:class:`~azure.ai.projects.operations.BetaRealtime` operations""" + live: BetaLive + """:class:`~azure.ai.projects.operations.BetaLive` operations""" + live_conversations: BetaVoiceAgentsLiveConversationsOperations + """:class:`~azure.ai.projects.operations.BetaVoiceAgentsLiveConversationsOperations` operations""" def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) @@ -144,6 +150,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: # pipeline has no way to keep a WebSocket upgrade's resulting socket alive. Add our # hand-written client, which manages a real, long-lived connection, in its place. self.realtime = BetaRealtime(self) + self.live = BetaLive(self) class BetaOperations(GeneratedBetaOperations): @@ -216,6 +223,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: "BetaSchedulesOperations", "BetaSkillsOperations", "BetaVoiceAgentsConversationsOperations", + "BetaVoiceAgentsLiveConversationsOperations", "BetaVoiceAgentsOperations", "BetaVoiceAgentsTelephonyOperations", "ClientEvent", @@ -227,6 +235,11 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: "BetaRealtime", "BetaRealtimeConnection", "BetaRealtimeConnectionManager", + "BetaLive", + "BetaLiveConnection", + "BetaLiveConnectionManager", + "LiveClientEvent", + "LiveServerEvent", "ServerEvent", "TelemetryOperations", ] # Add all objects you want publicly available to users at this package level diff --git a/sdk/ai/azure-ai-projects/docs/GeneratePublicMethodsDoc.ps1 b/sdk/ai/azure-ai-projects/docs/GeneratePublicMethodsDoc.ps1 index b1d3017632f5..1e5f70540895 100644 --- a/sdk/ai/azure-ai-projects/docs/GeneratePublicMethodsDoc.ps1 +++ b/sdk/ai/azure-ai-projects/docs/GeneratePublicMethodsDoc.ps1 @@ -57,7 +57,7 @@ def unwrap_operation(value: Any) -> Any: # Hand-written sub-client classes that don't follow the generated *Operations naming # convention but are still part of the public surface and should be discovered/counted the # same way, e.g. `beta.voice_agents.realtime` (`BetaRealtime`/`AsyncBetaRealtime`). -_HANDWRITTEN_SUBCLIENT_CLASS_NAMES = {"BetaRealtime", "AsyncBetaRealtime"} +_HANDWRITTEN_SUBCLIENT_CLASS_NAMES = {"BetaRealtime", "AsyncBetaRealtime", "BetaLive", "AsyncBetaLive"} def _is_operation_group(value: Any) -> bool: diff --git a/sdk/ai/azure-ai-projects/docs/public-methods.md b/sdk/ai/azure-ai-projects/docs/public-methods.md index 3e60266b2083..b8e92f8dab33 100644 --- a/sdk/ai/azure-ai-projects/docs/public-methods.md +++ b/sdk/ai/azure-ai-projects/docs/public-methods.md @@ -6,11 +6,11 @@ This document lists all public methods available on `AIProjectClient` and its su ## Summary -There are a total of 201 unique public methods: +There are a total of 202 unique public methods: - 5 stable methods on the client - 84 stable methods on top-level sub-clients -- 112 beta methods on nested beta sub-clients +- 113 beta methods on nested beta sub-clients ### Top-level sub-clients (stable operations) @@ -42,6 +42,7 @@ There are a total of 201 unique public methods: | `beta.schedules` | BetaSchedulesOperations | 6 | | `beta.skills` | BetaSkillsOperations | 11 | | `beta.voice_agents.conversations` | BetaVoiceAgentsConversationsOperations | 14 | +| `beta.voice_agents.live` | BetaLive | 1 | | `beta.voice_agents.live_conversations` | BetaVoiceAgentsLiveConversationsOperations | 9 | | `beta.voice_agents.realtime` | BetaRealtime | 1 | | `beta.voice_agents.telephony` | BetaVoiceAgentsTelephonyOperations | 14 | @@ -262,6 +263,8 @@ Alphabetically sorted. An asterisk at the end of the method name means it is a h .beta.voice_agents.conversations.list_response_items .beta.voice_agents.conversations.list_responses +.beta.voice_agents.live.connect* + .beta.voice_agents.live_conversations.delete .beta.voice_agents.live_conversations.download_audio .beta.voice_agents.live_conversations.download_audio_item diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py b/sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py new file mode 100644 index 000000000000..68865d9d6ca8 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py @@ -0,0 +1,221 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +"""Shared lifecycle and event helpers for the GPT Live samples in this folder.""" + +from __future__ import annotations + +import asyncio +from contextlib import suppress +import os +import sys +import time +from typing import Any, Mapping, Optional, Sequence, TYPE_CHECKING +from uuid import uuid4 + +from azure.ai.projects import models + +if TYPE_CHECKING: + from azure.ai.projects.operations import BetaLiveConnection, LiveServerEvent + from azure.ai.projects.aio.operations import AsyncBetaLiveConnection + + +RESPONSE_TIMEOUT = 60.0 +CLOSE_TIMEOUT = 10.0 + + +def sample_agent_name(scenario: str) -> str: + """Treat the optional name as a prefix so each run owns a distinct version.""" + prefix = os.environ.get("FOUNDRY_VOICE_AGENT_NAME") or f"sample-live-{scenario}" + return f"{prefix}-{uuid4().hex[:8]}" + + +def text_conversation_prompts(prompts: Optional[Sequence[str]]) -> list[str]: + """Keep the original single-turn example unless explicit scripted prompts are supplied.""" + if prompts is None: + return ["Why is Seattle known for rain? Answer in one short sentence."] + if isinstance(prompts, str): + raise ValueError("Pass prompts as a sequence of strings, not one string.") + result = list(prompts) + if not result or any(not isinstance(prompt, str) or not prompt.strip() for prompt in result): + raise ValueError("Provide at least one nonblank text prompt.") + return result + + +def audio_settings() -> models.LiveInitialSessionAudioParam: + """Use the same PCM16, mono, 24 kHz format in both directions.""" + return models.LiveInitialSessionAudioParam( + format=models.LiveSessionAudioFormatPCMParam(rate=24000), + output=models.LiveInitialSessionAudioOutputParam(voice="marin"), + ) + + +def safe_print(text: str) -> None: + """Display model text even on a console with a legacy encoding.""" + encoding = sys.stdout.encoding or "utf-8" + print(text.encode(encoding, errors="replace").decode(encoding)) + + +def remaining_time(deadline: float, operation: str) -> float: + """Keep a stream of nonterminal events from extending a timeout forever.""" + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(f"Timed out waiting for {operation}.") + return remaining + + +def check_event(event: LiveServerEvent) -> Mapping[str, Any]: + """Raise service/backend failures and unwrap a nested Responses event.""" + if event.get("type") == "error": + error = event.get("error") or {} + raise RuntimeError(f"Live session error: {error.get('code')}: {error.get('message')}") + + if isinstance(event, models.LiveServerEventLiveResponseEvent): + nested: Mapping[str, Any] = event.event + elif event.get("type") == "response.event": + nested = event["event"] + else: + return {} + + response = nested.get("response") or {} + status = response.get("status") + if nested.get("type") in ("error", "response.failed", "response.incomplete", "response.cancelled") or status in ( + "failed", + "incomplete", + "cancelled", + ): + detail = nested.get("error") or response.get("error") or response.get("incomplete_details") + raise RuntimeError(f"Responses backend {nested.get('type')}: {detail or status or 'unknown error'}") + return nested + + +def check_session_open(event: LiveServerEvent) -> None: + """Do not mistake a premature shutdown for a completed reply.""" + if event.get("type") == "session.closed": + raise RuntimeError(f"Live session closed before the workflow completed: {event.get('reason')}") + + +def wait_for_event( + connection: BetaLiveConnection, event_type: str, timeout: float = RESPONSE_TIMEOUT +) -> LiveServerEvent: + """Receive a lifecycle acknowledgement within one bounded deadline.""" + deadline = time.monotonic() + timeout + while True: + try: + event = connection.recv(timeout=remaining_time(deadline, event_type)) + except TimeoutError as exc: + raise TimeoutError(f"Timed out waiting for {event_type}.") from exc + check_event(event) + if event.get("type") == event_type: + return event + check_session_open(event) + + +async def wait_for_event_async( + connection: AsyncBetaLiveConnection, event_type: str, timeout: float = RESPONSE_TIMEOUT +) -> LiveServerEvent: + """Receive an async lifecycle acknowledgement without an unbounded wait.""" + deadline = time.monotonic() + timeout + while True: + try: + remaining = remaining_time(deadline, event_type) + event = await asyncio.wait_for(connection.recv(), timeout=remaining) + except (asyncio.TimeoutError, TimeoutError) as exc: + raise TimeoutError(f"Timed out waiting for {event_type}.") from exc + check_event(event) + if event.get("type") == event_type: + return event + check_session_open(event) + + +def close_session(connection: BetaLiveConnection) -> None: + """Call in finally; preserve the original failure while still requesting close.""" + if sys.exc_info()[0] is not None: + with suppress(Exception): + connection.session.close() + return + connection.session.close() + event = wait_for_event(connection, "session.closed", timeout=CLOSE_TIMEOUT) + if event.get("reason") != "close_requested": + raise RuntimeError(f"Live session did not close normally: {event.get('reason')}") + print("Session closed: close_requested") + + +async def close_session_async(connection: AsyncBetaLiveConnection) -> None: + """Request protocol shutdown before the connection context closes the transport.""" + if sys.exc_info()[0] is not None: + with suppress(Exception): + await asyncio.wait_for(connection.session.close(), timeout=CLOSE_TIMEOUT) + return + deadline = time.monotonic() + CLOSE_TIMEOUT + try: + remaining = remaining_time(deadline, "session.close") + await asyncio.wait_for(connection.session.close(), timeout=remaining) + event = await wait_for_event_async( + connection, "session.closed", timeout=remaining_time(deadline, "session.closed") + ) + except (asyncio.TimeoutError, TimeoutError) as exc: + raise TimeoutError("Timed out closing the GPT Live session.") from exc + if event.get("reason") != "close_requested": + raise RuntimeError(f"Live session did not close normally: {event.get('reason')}") + print("Session closed: close_requested") + + +def _collect_response_event( + event: LiveServerEvent, + text_parts: dict[tuple[str, int], str], + calls: dict[str, Mapping[str, Any]], +) -> bool: + nested = check_event(event) + check_session_open(event) + event_type = nested.get("type") + if event_type in ("response.output_text.delta", "response.output_text.done"): + key = (nested["item_id"], nested["content_index"]) + if event_type == "response.output_text.delta": + text_parts[key] = text_parts.get(key, "") + nested["delta"] + else: + text_parts[key] = nested["text"] + elif event_type == "response.output_item.done": + item = nested["item"] + if item.get("type") == "function_call": + # Argument events identify item_id; the complete item supplies the tool-result call_id. + call_id = item.get("call_id") + if not isinstance(call_id, str) or not call_id: + raise RuntimeError("The completed Responses function_call item omitted a usable call_id.") + calls[call_id] = item + return event_type == "response.completed" + + +def receive_response( + connection: BetaLiveConnection, timeout: float = RESPONSE_TIMEOUT +) -> tuple[str, list[Mapping[str, Any]]]: + """Collect one nested Responses cycle, with function calls from complete output items.""" + deadline = time.monotonic() + timeout + text_parts: dict[tuple[str, int], str] = {} + calls: dict[str, Mapping[str, Any]] = {} + while True: + try: + event = connection.recv(timeout=remaining_time(deadline, "the Responses backend")) + except TimeoutError as exc: + raise TimeoutError("Timed out waiting for the Responses backend to complete.") from exc + if _collect_response_event(event, text_parts, calls): + # Live strips output from lifecycle snapshots; the granular events carry it. + return "".join(text_parts.values()).strip(), list(calls.values()) + + +async def receive_response_async( + connection: AsyncBetaLiveConnection, timeout: float = RESPONSE_TIMEOUT +) -> tuple[str, list[Mapping[str, Any]]]: + """Collect the same backend text and tool events without blocking the event loop.""" + deadline = time.monotonic() + timeout + text_parts: dict[tuple[str, int], str] = {} + calls: dict[str, Mapping[str, Any]] = {} + while True: + try: + remaining = remaining_time(deadline, "the Responses backend") + event = await asyncio.wait_for(connection.recv(), timeout=remaining) + except (asyncio.TimeoutError, TimeoutError) as exc: + raise TimeoutError("Timed out waiting for the Responses backend to complete.") from exc + if _collect_response_event(event, text_parts, calls): + return "".join(text_parts.values()).strip(), list(calls.values()) diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py new file mode 100644 index 000000000000..7cb71767c3b2 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py @@ -0,0 +1,269 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Stream a real WAV recording to a preview GPT Live voice agent while receiving + its speech concurrently. The input must contain spoken audio encoded as + PCM16 little-endian, mono, 24 kHz. Only raw frames are sent, without the WAV + header, in paced 50 ms chunks followed by one second of silence. + + Aggregate audio byte counts and any emitted input/output transcripts are + displayed. Live has no speech-finished event: after sending finishes, this + bounded demonstration stops after three seconds without non-silent reply + audio or nonblank output transcript updates. This is an inactivity heuristic; + a long speech pause can still split a reply. Low-level PCM with peak + amplitude at or below 256 does not restart the timer or count as a reply. + Captured PCM, including silence, is still counted and optionally saved. + Client-owned delegations are answered with commentary tied to their ID. + +USAGE: + python sample_voice_agent_live_audio_conversation_async.py + + pip install "azure-ai-projects[voice]" azure-identity python-dotenv + + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. + 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live DEPLOYMENT name. + 3) FOUNDRY_VOICE_AUDIO_INPUT_PATH (required) - Path to an actual spoken-audio + WAV file: uncompressed PCM16, mono, 24000 Hz. No microphone is needed. + 4) FOUNDRY_VOICE_AUDIO_OUTPUT_PATH (optional) - Explicit WAV output path. + When set, saves actual returned audio, not telemetry. Without it no audio + file is written. The path must differ from the input path. + 5) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for a uniquely named sample + agent. Only the version created by this run is deleted. + + Authenticates with DefaultAzureCredential. No audio-device dependency. + Choose a short recording; Ctrl-C cancels local tasks and closes the session. + Conversation persistence is disabled; explicitly saving audio still writes + conversation content to your chosen local file. +""" + +from __future__ import annotations + +import asyncio +import base64 +from contextlib import ExitStack +from dataclasses import dataclass +import os +from pathlib import Path +import time +from typing import Optional, TYPE_CHECKING +import wave + +from dotenv import load_dotenv +from azure.identity.aio import DefaultAzureCredential +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects import models + +from live_sample_utils import ( + audio_settings, + check_event, + check_session_open, + close_session_async, + remaining_time, + safe_print, + sample_agent_name, + wait_for_event_async, +) + +if TYPE_CHECKING: + from azure.ai.projects.aio.operations import AsyncBetaLiveConnection + + +_SAMPLE_RATE = 24000 +_CHUNK_FRAMES = 1200 +_OUTPUT_IDLE_SECONDS = 3.0 +_SILENCE_PEAK_THRESHOLD = 256 + + +@dataclass +class _AudioOutput: + num_bytes: int = 0 + chunks: int = 0 + input_transcript: str = "" + output_transcript: str = "" + has_audio_activity: bool = False + + +def _has_audio_activity(pcm: bytes) -> bool: + return any( + abs(int.from_bytes(pcm[offset : offset + 2], "little", signed=True)) > _SILENCE_PEAK_THRESHOLD + for offset in range(0, len(pcm), 2) + ) + + +def _validate_recording(recording: wave.Wave_read) -> None: + if ( + recording.getnchannels() != 1 + or recording.getsampwidth() != 2 + or recording.getframerate() != _SAMPLE_RATE + or recording.getcomptype() != "NONE" + ): + raise ValueError("FOUNDRY_VOICE_AUDIO_INPUT_PATH must be an uncompressed PCM16, mono, 24000 Hz WAV file.") + if recording.getnframes() == 0: + raise ValueError("The input WAV contains no audio frames.") + + +async def _send_audio(connection: AsyncBetaLiveConnection, recording: wave.Wave_read, finished: asyncio.Event) -> int: + num_bytes = 0 + while chunk := recording.readframes(_CHUNK_FRAMES): + await connection.session.input_audio.append(audio=chunk) + num_bytes += len(chunk) + await asyncio.sleep(len(chunk) / (2 * _SAMPLE_RATE)) + if num_bytes == 0: + raise RuntimeError("No recorded audio frames were sent.") + # Trailing silence lets the model finish the recorded speech turn. + for _ in range(_SAMPLE_RATE // _CHUNK_FRAMES): + silence = b"\x00\x00" * _CHUNK_FRAMES + await connection.session.input_audio.append(audio=silence) + num_bytes += len(silence) + await asyncio.sleep(_CHUNK_FRAMES / _SAMPLE_RATE) + finished.set() + return num_bytes + + +async def _receive_audio( + connection: AsyncBetaLiveConnection, + finished: asyncio.Event, + timeout: float, + output_file: Optional[wave.Wave_write], +) -> _AudioOutput: + output = _AudioOutput() + deadline = time.monotonic() + timeout + last_content = time.monotonic() + while True: + if finished.is_set() and output.has_audio_activity and time.monotonic() - last_content >= _OUTPUT_IDLE_SECONDS: + return output + remaining = remaining_time(deadline, "GPT Live reply audio") + try: + event = await asyncio.wait_for(connection.recv(), timeout=min(remaining, 1.0)) + except asyncio.TimeoutError: + continue + check_event(event) + check_session_open(event) + if isinstance(event, models.LiveOutputAudioDelta): + pcm = base64.b64decode(event.delta, validate=True) + if len(pcm) % 2: + raise ValueError("The service returned an incomplete PCM16 sample.") + output.num_bytes += len(pcm) + output.chunks += 1 + if output_file is not None: + output_file.writeframesraw(pcm) + if _has_audio_activity(pcm): + output.has_audio_activity = True + last_content = time.monotonic() + elif isinstance(event, models.LiveServerEventLiveInputTranscriptDelta): + output.input_transcript += event.delta + elif isinstance(event, models.LiveServerEventLiveOutputTranscriptDelta): + output.output_transcript += event.delta + if event.delta.strip(): + last_content = time.monotonic() + elif isinstance(event, models.LiveServerEventLiveDelegationCreated): + if event.delegation.target == "client": + await connection.session.commentary.append( + content="This sample cannot look up external information. Briefly acknowledge the recorded request.", + delegation_id=event.delegation.id, + ) + print("Replied to a client-owned delegation with associated commentary.") + + +async def _stream_recording( + connection: AsyncBetaLiveConnection, recording: wave.Wave_read, output_file: Optional[wave.Wave_write] +) -> None: + finished = asyncio.Event() + timeout = recording.getnframes() / _SAMPLE_RATE + 60 + sender = asyncio.create_task(_send_audio(connection, recording, finished)) + receiver = asyncio.create_task(_receive_audio(connection, finished, timeout, output_file)) + try: + input_bytes, output = await asyncio.wait_for(asyncio.gather(sender, receiver), timeout=timeout) + except asyncio.TimeoutError as exc: + raise TimeoutError("Timed out streaming the recording or waiting for nonempty GPT Live reply audio.") from exc + finally: + # Live has no response.cancel. Cancel local work and then request session.close. + for task in (sender, receiver): + if not task.done(): + task.cancel() + await asyncio.gather(sender, receiver, return_exceptions=True) + if input_bytes <= 0 or output.num_bytes <= 0 or not output.has_audio_activity: + raise RuntimeError("A successful conversation requires nonzero input and non-silent reply audio.") + for label, transcript in (("Input", output.input_transcript), ("Output", output.output_transcript)): + if transcript and not transcript.strip(): + raise RuntimeError(f"{label} transcript events contained no usable text.") + safe_print(f"{label} transcript: {transcript.strip() or '(not emitted by the service)'}") + print(f"Sent {input_bytes} PCM16 bytes, including 48000 bytes of trailing silence.") + print(f"Received {output.num_bytes} PCM16 bytes in {output.chunks} chunks ({output.num_bytes / 48000:.2f}s).") + + +async def _run_audio_conversation( + project_client: AIProjectClient, + agent_name: str, + agent_version: str, + recording: wave.Wave_read, + output_file: Optional[wave.Wave_write], +) -> None: + async with project_client.beta.voice_agents.live.connect( + agent_name=agent_name, agent_version=agent_version, store=False + ) as connection: + try: + await wait_for_event_async(connection, "session.started") + print("Live session started; sending the recording and receiving audio concurrently.") + await _stream_recording(connection, recording, output_file) + finally: + await close_session_async(connection) + + +async def main() -> None: + load_dotenv() + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + model = os.environ["FOUNDRY_VOICE_AGENT_MODEL"] + input_path = Path(os.environ["FOUNDRY_VOICE_AUDIO_INPUT_PATH"]) + output_path = os.environ.get("FOUNDRY_VOICE_AUDIO_OUTPUT_PATH") + if output_path and Path(output_path).resolve() == input_path.resolve(): + raise ValueError("The audio output path must not overwrite the input recording.") + agent_name = sample_agent_name("audio") + + with ExitStack() as files: + recording = files.enter_context(wave.open(str(input_path), "rb")) + _validate_recording(recording) + output_file = files.enter_context(wave.open(output_path, "wb")) if output_path else None + if output_file is not None: + output_file.setnchannels(1) + output_file.setsampwidth(2) + output_file.setframerate(_SAMPLE_RATE) + async with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, + ): + created_version = None + try: + created_version = await project_client.agents.create_version( + agent_name=agent_name, + definition=models.VoiceAgentLiveDefinition( + model=model, + instructions=( + "You are a friendly voice assistant. Briefly answer the recorded speech. " + "Keep replies to one short sentence. Delegate external lookups to the client." + ), + audio=audio_settings(), + delegation=models.LiveClientDelegationParam(), + store=False, + ), + ) + await _run_audio_conversation( + project_client, agent_name, created_version.version, recording, output_file + ) + finally: + if created_version is not None: + await project_client.agents.delete_version( + agent_name=agent_name, agent_version=created_version.version + ) + print(f"Deleted sample agent version: {created_version.version}") + if output_path: + print("Saved actual returned PCM16 WAV audio to FOUNDRY_VOICE_AUDIO_OUTPUT_PATH (not telemetry).") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_connection.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_connection.py new file mode 100644 index 000000000000..43dda481541d --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_connection.py @@ -0,0 +1,84 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Open a preview GPT Live voice-agent session, update its client delegation, + append instructions, and confirm graceful session shutdown. Foundry starts + the session from the persisted agent definition; do not send session.start. + This is a protocol/lifecycle example, not a text or microphone conversation. + +USAGE: + python sample_voice_agent_live_connection.py + + pip install "azure-ai-projects[voice]" azure-identity python-dotenv + + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. + 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live DEPLOYMENT name, + not a model catalog identifier. + 3) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for a uniquely named sample + agent. Only the version created by this run is deleted. + + Authenticates with DefaultAzureCredential. No audio devices are needed. +""" + +import os + +from dotenv import load_dotenv +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient, models + +from live_sample_utils import audio_settings, close_session, sample_agent_name, wait_for_event + + +def main() -> None: + load_dotenv() + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + model = os.environ["FOUNDRY_VOICE_AGENT_MODEL"] + agent_name = sample_agent_name("connection") + + with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, + ): + created_version = None + try: + created_version = project_client.agents.create_version( + agent_name=agent_name, + definition=models.VoiceAgentLiveDefinition( + model=model, + instructions="You are a concise, friendly voice assistant.", + audio=audio_settings(), + delegation=models.LiveClientDelegationParam(), + store=False, + ), + ) + with project_client.beta.voice_agents.live.connect( + agent_name=agent_name, agent_version=created_version.version, store=False + ) as connection: + try: + started = wait_for_event(connection, "session.started") + print(f"Live session started: {started['session']['id']}") + connection.session.update( + session=models.LiveSessionUpdateParams(delegation=models.LiveClientDelegationParam()) + ) + wait_for_event(connection, "session.updated") + connection.session.instructions.append( + content="Keep responses brief. Do not invent external facts.", delegation_id=None + ) + wait_for_event(connection, "session.instructions.appended") + print("Confirmed session.updated and session.instructions.appended.") + finally: + # session.close is a command; leaving the context closes the transport. + close_session(connection) + finally: + if created_version is not None: + project_client.agents.delete_version(agent_name=agent_name, agent_version=created_version.version) + print(f"Deleted sample agent version: {created_version.version}") + + +if __name__ == "__main__": + main() diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py new file mode 100644 index 000000000000..6be341df6acb --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py @@ -0,0 +1,166 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Resolve a client-executed Responses function tool in a preview GPT Live + session. The deterministic get_weather tool returns sample data for Seattle + (72 F), not a real weather observation. After the tool response completes, + submit function_call_output and request the next backend response. + Read call_id from the complete response.output_item.done function_call item, + not from response.function_call_arguments.done (which identifies item_id). + +USAGE: + python sample_voice_agent_live_function_tool.py + + pip install "azure-ai-projects[voice]" azure-identity python-dotenv + + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. + 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live DEPLOYMENT name. + 3) FOUNDRY_MODEL_NAME (required) - A tool-capable Responses backend + DEPLOYMENT name available to the Live resource. + 4) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for a uniquely named sample + agent. Only the version created by this run is deleted. + + Authenticates with DefaultAzureCredential. No audio devices are needed. +""" + +from __future__ import annotations + +import json +import os +import time +from typing import Any, Mapping + +from dotenv import load_dotenv +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient, models + +from live_sample_utils import ( + audio_settings, + close_session, + receive_response, + remaining_time, + safe_print, + sample_agent_name, + wait_for_event, +) + + +def get_weather(city: str) -> str: + """Return deterministic sample weather, rather than calling an external service.""" + if city.strip().casefold() != "seattle": + raise ValueError("This sample only provides weather data for Seattle.") + return json.dumps({"city": "Seattle", "condition": "sunny", "temperature_f": 72}) + + +def _execute_tool(call: Mapping[str, Any]) -> str: + if call.get("name") != "get_weather": + raise RuntimeError(f"Unexpected backend function: {call.get('name')}") + arguments = json.loads(call["arguments"]) + if not isinstance(arguments, dict) or set(arguments) != {"city"} or not isinstance(arguments["city"], str): + raise ValueError("get_weather requires exactly one string argument named city.") + return get_weather(arguments["city"]) + + +def _run_tool_turn(project_client: AIProjectClient, agent_name: str, agent_version: str) -> str: + with project_client.beta.voice_agents.live.connect( + agent_name=agent_name, agent_version=agent_version, store=False + ) as connection: + try: + wait_for_event(connection, "session.started") + connection.response.item.create( + item={ + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "Use get_weather for Seattle, then report the sample result."} + ], + } + ) + connection.response.create() + deadline = time.monotonic() + 120 + handled_calls: set[str] = set() + for _ in range(6): + text, calls = receive_response(connection, timeout=remaining_time(deadline, "the tool-backed answer")) + if not calls: + if not handled_calls: + raise RuntimeError("The backend answered without calling get_weather.") + if "72" not in text or "seattle" not in text.casefold(): + raise RuntimeError("The backend answer did not include Seattle and the tool's 72 F result.") + return text + # Wait for this cycle's response.completed before continuing, avoiding concurrent responses. + for call in calls: + call_id = call["call_id"] + if not call_id or call_id in handled_calls: + raise RuntimeError("The backend returned a missing or repeated function call ID.") + result = _execute_tool(call) + connection.response.item.create( + item={"type": "function_call_output", "call_id": call_id, "output": result} + ) + handled_calls.add(call_id) + print("get_weather returned sample data: Seattle, sunny, 72 F.") + connection.response.create() + raise RuntimeError("The backend exceeded this sample's six-response tool-call limit.") + finally: + close_session(connection) + + +def main() -> None: + load_dotenv() + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + model = os.environ["FOUNDRY_VOICE_AGENT_MODEL"] + backend_model = os.environ["FOUNDRY_MODEL_NAME"] + agent_name = sample_agent_name("function-tool") + + with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, + ): + created_version = None + try: + created_version = project_client.agents.create_version( + agent_name=agent_name, + definition=models.VoiceAgentLiveDefinition( + model=model, + instructions="Delegate weather questions to the Responses backend.", + audio=audio_settings(), + delegation=models.LiveResponsesDelegationParam( + responses=models.LiveResponsesDelegationSettingsInputParam( + model=backend_model, + instructions=( + "Call get_weather for Seattle. After its result, answer once without calling it again. " + "Explicitly include Seattle and the numeric temperature in F. Label it sample weather." + ), + tools=[ + models.LiveFunctionToolInputParam( + name="get_weather", + description="Return deterministic sample weather for Seattle, not a live observation.", + parameters={ + "type": "object", + "properties": {"city": {"type": "string", "enum": ["Seattle"]}}, + "required": ["city"], + "additionalProperties": False, + }, + strict=True, + ) + ], + parallel_tool_calls=False, + ) + ), + store=False, + ), + ) + answer = _run_tool_turn(project_client, agent_name, created_version.version) + safe_print(f"Responses backend: {answer}") + finally: + if created_version is not None: + project_client.agents.delete_version(agent_name=agent_name, agent_version=created_version.version) + print(f"Deleted sample agent version: {created_version.version}") + + +if __name__ == "__main__": + main() diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py new file mode 100644 index 000000000000..905d77191541 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py @@ -0,0 +1,118 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Send one or more text messages through one preview GPT Live session configured + with a Responses delegation. Each turn waits for nested response.completed + before sending the next prompt, retaining the conversation in that session. + This is not the Realtime protocol's response.done. + +USAGE: + python sample_voice_agent_live_text_conversation.py + python sample_voice_agent_live_text_conversation.py "My name is Ada." "What is my name?" + + Omit prompts to run the original single-turn example. Quote each positional + prompt to run a scripted, headless multi-turn conversation. + + pip install "azure-ai-projects[voice]" azure-identity python-dotenv + + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. + 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live DEPLOYMENT name. + 3) FOUNDRY_MODEL_NAME (required) - A Responses backend DEPLOYMENT name + available to the Live resource, not a model catalog identifier. + 4) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for a uniquely named sample + agent. Only the version created by this run is deleted. + + Authenticates with DefaultAzureCredential. No microphone or speaker needed. + This sample prints the backend text; use the audio sample for Live speech. +""" + +from __future__ import annotations + +import argparse +import os +from typing import Optional, Sequence + +from dotenv import load_dotenv +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient, models + +from live_sample_utils import ( + audio_settings, + close_session, + receive_response, + safe_print, + sample_agent_name, + text_conversation_prompts, + wait_for_event, +) + + +def _run_text_conversation( + project_client: AIProjectClient, agent_name: str, agent_version: str, prompts: Sequence[str] +) -> None: + with project_client.beta.voice_agents.live.connect( + agent_name=agent_name, agent_version=agent_version, store=False + ) as connection: + try: + wait_for_event(connection, "session.started") + for prompt in prompts: + safe_print(f"You: {prompt}") + connection.response.item.create( + item={"type": "message", "role": "user", "content": [{"type": "input_text", "text": prompt}]} + ) + connection.response.create() + text, calls = receive_response(connection) + if calls: + raise RuntimeError("The text-only backend unexpectedly requested a function tool.") + if not text: + raise RuntimeError("The Responses backend completed without returning any output text.") + safe_print(f"Responses backend: {text}") + finally: + close_session(connection) + + +def main(prompts: Optional[Sequence[str]] = None) -> None: + prompts = text_conversation_prompts(prompts) + load_dotenv() + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + model = os.environ["FOUNDRY_VOICE_AGENT_MODEL"] + backend_model = os.environ["FOUNDRY_MODEL_NAME"] + agent_name = sample_agent_name("text") + + with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, + ): + created_version = None + try: + created_version = project_client.agents.create_version( + agent_name=agent_name, + definition=models.VoiceAgentLiveDefinition( + model=model, + instructions="You are a friendly voice assistant. Delegate text questions to the backend.", + audio=audio_settings(), + delegation=models.LiveResponsesDelegationParam( + responses=models.LiveResponsesDelegationSettingsInputParam( + model=backend_model, instructions="Answer in one short sentence." + ) + ), + store=False, + ), + ) + _run_text_conversation(project_client, agent_name, created_version.version, prompts) + finally: + if created_version is not None: + project_client.agents.delete_version(agent_name=agent_name, agent_version=created_version.version) + print(f"Deleted sample agent version: {created_version.version}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run a headless GPT Live text conversation.") + parser.add_argument("prompts", nargs="*", help="Quoted text prompts to send sequentially in one session.") + arguments = parser.parse_args() + main(arguments.prompts or None) diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py new file mode 100644 index 000000000000..cdf54c39c807 --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py @@ -0,0 +1,120 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Run a headless, asynchronous GPT Live text conversation using Responses + delegation. Send multiple prompts sequentially over the same session, + waiting for nested response.completed before beginning the next turn. + Backend text is printed; it does not signal completion of frontend speech. + +USAGE: + python sample_voice_agent_live_text_conversation_async.py + python sample_voice_agent_live_text_conversation_async.py "My name is Ada." "What is my name?" + + Omit prompts for the same single-turn example as the synchronous sample. + + pip install "azure-ai-projects[voice]" azure-identity python-dotenv + + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. + 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live DEPLOYMENT name. + 3) FOUNDRY_MODEL_NAME (required) - A Responses backend DEPLOYMENT name + available to the Live resource, not a model catalog identifier. + 4) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for a uniquely named sample + agent. Only the version created by this run is deleted. + + Authenticates with DefaultAzureCredential. No microphone, speaker, or + audio-device dependency is needed. Ctrl-C cancels local work and closes + the session; Live has no response.cancel command. +""" + +from __future__ import annotations + +import argparse +import asyncio +import os +from typing import Optional, Sequence + +from dotenv import load_dotenv +from azure.identity.aio import DefaultAzureCredential +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects import models + +from live_sample_utils import ( + audio_settings, + close_session_async, + receive_response_async, + safe_print, + sample_agent_name, + text_conversation_prompts, + wait_for_event_async, +) + + +async def _run_text_conversation( + project_client: AIProjectClient, agent_name: str, agent_version: str, prompts: Sequence[str] +) -> None: + async with project_client.beta.voice_agents.live.connect( + agent_name=agent_name, agent_version=agent_version, store=False + ) as connection: + try: + await wait_for_event_async(connection, "session.started") + for prompt in prompts: + safe_print(f"You: {prompt}") + await connection.response.item.create( + item={"type": "message", "role": "user", "content": [{"type": "input_text", "text": prompt}]} + ) + await connection.response.create() + text, calls = await receive_response_async(connection) + if calls: + raise RuntimeError("The text-only backend unexpectedly requested a function tool.") + if not text: + raise RuntimeError("The Responses backend completed without returning any output text.") + safe_print(f"Responses backend: {text}") + finally: + await close_session_async(connection) + + +async def main(prompts: Optional[Sequence[str]] = None) -> None: + prompts = text_conversation_prompts(prompts) + load_dotenv() + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + model = os.environ["FOUNDRY_VOICE_AGENT_MODEL"] + backend_model = os.environ["FOUNDRY_MODEL_NAME"] + agent_name = sample_agent_name("text") + + async with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, + ): + created_version = None + try: + created_version = await project_client.agents.create_version( + agent_name=agent_name, + definition=models.VoiceAgentLiveDefinition( + model=model, + instructions="You are a friendly voice assistant. Delegate text questions to the backend.", + audio=audio_settings(), + delegation=models.LiveResponsesDelegationParam( + responses=models.LiveResponsesDelegationSettingsInputParam( + model=backend_model, instructions="Answer in one short sentence." + ) + ), + store=False, + ), + ) + await _run_text_conversation(project_client, agent_name, created_version.version, prompts) + finally: + if created_version is not None: + await project_client.agents.delete_version(agent_name=agent_name, agent_version=created_version.version) + print(f"Deleted sample agent version: {created_version.version}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run a headless, asynchronous GPT Live text conversation.") + parser.add_argument("prompts", nargs="*", help="Quoted text prompts to send sequentially in one session.") + arguments = parser.parse_args() + asyncio.run(main(arguments.prompts or None)) diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_live_client.py b/sdk/ai/azure-ai-projects/tests/agents/test_live_client.py new file mode 100644 index 000000000000..ab96c3e03c07 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_live_client.py @@ -0,0 +1,1486 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +"""Deterministic public Live client tests, with only WebSocket transports replaced. + +Both clients use the same wire assertions. No service, recording, real credential, +or audio device is involved; audio payloads are small synthetic byte sequences. +""" + +import asyncio +import base64 +import inspect +import json +import logging +import threading +from collections.abc import AsyncIterator, Iterator +from concurrent.futures import ThreadPoolExecutor +from contextlib import asynccontextmanager +from types import MappingProxyType, SimpleNamespace +from typing import Optional, TypeVar, get_args, get_origin, get_type_hints +from unittest.mock import AsyncMock, MagicMock, Mock, patch +from urllib.parse import parse_qs, urlsplit + +import aiohttp +import pytest +import pytest_asyncio +from azure.core.credentials import AccessToken +from websockets.exceptions import ConnectionClosedError, ConnectionClosedOK +from websockets.frames import Close + +from azure.ai.projects import AIProjectClient, __version__, models, operations +from azure.ai.projects.aio import AIProjectClient as AsyncAIProjectClient +from azure.ai.projects.aio import operations as async_operations + + +_ENDPOINT = "https://fake-account.services.ai.azure.com/api/projects/fake-project" +_HOST = "fake-account.services.ai.azure.com" +_AGENT = "agent-test" +_TOKEN = "unit-test-token" +_SCOPES = ("https://ai.azure.com/.default",) +_SESSION = { + "id": "live-session-test", + "model": "gpt-live-1", + "expires_at": 9_999_999_999, + "status": "active", + "store": False, + "delegation": {"type": "client"}, +} +_STARTED = {"type": "session.started", "event_id": "evt-started", "session": _SESSION} +_CLOSED = { + "type": "session.closed", + "event_id": "evt-closed", + "client_event_id": "evt-close", + "reason": "close_requested", + "session": _SESSION, + "usage": {"seconds": 1.25}, +} + +pytestmark = pytest.mark.asyncio + + +class _FakeCredential: + def __init__(self): + self.calls = [] + self.failure = None + + def get_token(self, *scopes, **kwargs): + self.calls.append((scopes, kwargs)) + if self.failure is not None: + raise self.failure + return AccessToken(_TOKEN, 9_999_999_999) + + +class _AsyncFakeCredential(_FakeCredential): + async def get_token(self, *scopes, **kwargs): + return super().get_token(*scopes, **kwargs) + + +def _message(kind, data=None): + return SimpleNamespace(type=kind, data=data) + + +def _handshake_query(url, kwargs): + query = parse_qs(urlsplit(url).query, keep_blank_values=True) + for name, value in kwargs.get("params", {}).items(): + query.setdefault(name, []).append(value) + return query + + +def _contains_type_variable(annotation): + return isinstance(annotation, TypeVar) or any( + _contains_type_variable(argument) for argument in get_args(annotation) + ) + + +class _Transport: + def __init__(self, is_async): + self.is_async = is_async + self.socket = MagicMock(name="websocket") + self.socket.closed = False + self.socket.close_code = None + self.socket.recv.return_value = json.dumps(_STARTED) + self.socket.receive = AsyncMock(return_value=_message(aiohttp.WSMsgType.TEXT, json.dumps(_STARTED))) + self.socket.send_str = AsyncMock() + self.socket.exception.return_value = None + close = AsyncMock if is_async else Mock + self.socket.close = close(side_effect=self._close_socket) + self.session = MagicMock(name="websocket_session") + self.session.closed = False + self.session.ws_connect = AsyncMock(return_value=self.socket) + self.session.close = AsyncMock(side_effect=self._close_session) + + def _close_socket(self, **kwargs): + self.socket.closed = True + + def _close_session(self): + self.session.closed = True + + @property + def writes(self): + return self.socket.send_str if self.is_async else self.socket.send + + @property + def sent(self): + calls = self.writes.await_args_list if self.is_async else self.writes.call_args_list + return [json.loads(call.args[0]) for call in calls] + + def queue(self, *payloads, binary=False): + frames = [json.dumps(payload) for payload in payloads] + if binary: + frames = [frame.encode("utf-8") for frame in frames] + self.socket.recv.side_effect = frames + kind = aiohttp.WSMsgType.BINARY if binary else aiohttp.WSMsgType.TEXT + self.socket.receive.side_effect = [_message(kind, frame) for frame in frames] + + def raw(self, frame): + self.socket.recv.return_value = frame + kind = aiohttp.WSMsgType.BINARY if isinstance(frame, (bytes, bytearray)) else aiohttp.WSMsgType.TEXT + self.socket.receive.return_value = _message(kind, frame) + + def assert_closed_once(self): + if self.is_async: + self.socket.close.assert_awaited_once() + self.session.close.assert_awaited_once() + assert self.session.closed is True + else: + self.socket.close.assert_called_once() + assert self.socket.closed is True + + +class _LiveHarness: + def __init__(self, is_async): + self.is_async = is_async + self.credential = _AsyncFakeCredential() if is_async else _FakeCredential() + self.transport = _Transport(is_async) + self.connect = Mock(return_value=self.transport.socket) + self.session_factory = Mock(return_value=self.transport.session) + self.clients = [] + self.client = self.new_client(allow_preview=True) + + def new_client(self, **options): + options.setdefault("endpoint", _ENDPOINT) + options.setdefault("credential", self.credential) + client_type = AsyncAIProjectClient if self.is_async else AIProjectClient + client = client_type(**options) + self.clients.append(client) + return client + + @property + def live(self): + return self.client.beta.voice_agents.live + + @property + def handshake(self): + call = self.transport.session.ws_connect.await_args if self.is_async else self.connect.call_args + assert call is not None + return call.args[0], call.kwargs + + @property + def headers(self): + return self.handshake[1]["headers" if self.is_async else "additional_headers"] + + def manager(self, *, protocol="live", **options): + options.setdefault("agent_name", _AGENT) + return getattr(self.client.beta.voice_agents, protocol).connect(**options) + + async def call(self, method, *args, **kwargs): + result = method(*args, **kwargs) + return await result if self.is_async else result + + async def exit(self, manager): + method = manager.__aexit__ if self.is_async else manager.__exit__ + await self.call(method, None, None, None) + + async def collect(self, connection): + if self.is_async: + return [event async for event in connection] + return list(connection) + + @asynccontextmanager + async def open(self, *, protocol="live", **options): + manager = self.manager(protocol=protocol, **options) + if self.is_async: + async with manager as connection: + yield connection + else: + with manager as connection: + yield connection + + +@pytest.fixture(params=[False, True], ids=["sync", "async"]) +def is_async(request): + return request.param + + +@pytest_asyncio.fixture +async def live(is_async, monkeypatch): + harness = _LiveHarness(is_async) + monkeypatch.setattr("websockets.sync.client.connect", harness.connect) + monkeypatch.setattr("aiohttp.ClientSession", harness.session_factory) + try: + yield harness + finally: + for client in reversed(harness.clients): + await harness.call(client.close) + + +async def test_live_is_public_and_connect_is_lazy(live): + public_operations = async_operations if live.is_async else operations + prefix = "Async" if live.is_async else "" + assert isinstance(live.live, getattr(public_operations, f"{prefix}BetaLive")) + assert live.live is live.client.beta.voice_agents.live + assert live.live is not live.client.beta.voice_agents.realtime + assert tuple(inspect.signature(type(live.live).__init__).parameters) == ("self", "args", "kwargs") + + manager = live.manager() + assert isinstance(manager, getattr(public_operations, f"{prefix}BetaLiveConnectionManager")) + assert live.credential.calls == [] + live.connect.assert_not_called() + live.session_factory.assert_not_called() + + connection = await live.call(manager.enter) + try: + assert isinstance(connection, getattr(public_operations, f"{prefix}BetaLiveConnection")) + assert connection.closed is False + assert "open" in repr(connection) + live.transport.writes.assert_not_called() + finally: + await live.exit(manager) + live.transport.assert_closed_once() + assert "closed" in repr(connection) + + +async def test_connect_signature_has_only_live_options(live): + parameters = inspect.signature(live.live.connect).parameters + assert tuple(parameters) == ( + "agent_name", + "agent_version", + "structured_inputs", + "store", + "connection_url", + "api_version", + "credential_scopes", + "extra_query", + "extra_headers", + "kwargs", + ) + assert parameters["agent_name"].default is inspect.Parameter.empty + for name, parameter in parameters.items(): + if name == "kwargs": + assert parameter.kind is inspect.Parameter.VAR_KEYWORD + else: + assert parameter.kind is inspect.Parameter.KEYWORD_ONLY + if name != "agent_name": + assert parameter.default is None + assert "model" not in parameters + assert "transport" not in parameters + assert models.LiveServerEvent.__mapping__ + + +@pytest.mark.parametrize("protocol", ["live", "realtime"]) +@pytest.mark.parametrize("member", ["enter", "recv", "send", "iterator"]) +async def test_public_websocket_signatures_have_concrete_type_hints(is_async, protocol, member): + public_operations = async_operations if is_async else operations + prefix = f"{'Async' if is_async else ''}Beta{protocol.capitalize()}" + connection_type = getattr(public_operations, f"{prefix}Connection") + manager_type = getattr(public_operations, f"{prefix}ConnectionManager") + client_event = getattr(public_operations, "LiveClientEvent" if protocol == "live" else "ClientEvent") + server_event = getattr(public_operations, "LiveServerEvent" if protocol == "live" else "ServerEvent") + owner = manager_type if member == "enter" else connection_type + method_name = ("__aiter__" if is_async else "__iter__") if member == "iterator" else member + method = getattr(owner, method_name) + parameters = inspect.signature(method).parameters + hints = get_type_hints(method) + + assert hints, f"{owner.__name__}.{method_name} must retain public annotations" + assert not any( + _contains_type_variable(annotation) for annotation in hints.values() + ), f"{owner.__name__}.{method_name} exposes unresolved generic type variables" + assert "return" in hints + + if member == "enter": + assert tuple(parameters) == ("self",) + assert hints == {"return": connection_type} + elif member == "recv": + assert hints["return"] == server_event + if is_async: + assert tuple(parameters) == ("self",) + assert set(hints) == {"return"} + else: + assert tuple(parameters) == ("self", "timeout") + assert parameters["timeout"].kind is inspect.Parameter.KEYWORD_ONLY + assert parameters["timeout"].default is None + assert hints["timeout"] == Optional[float] + assert set(hints) == {"timeout", "return"} + elif member == "send": + assert tuple(parameters) == ("self", "event") + assert parameters["event"].kind is inspect.Parameter.POSITIONAL_OR_KEYWORD + assert parameters["event"].default is inspect.Parameter.empty + assert hints == {"event": client_event, "return": type(None)} + else: + assert tuple(parameters) == ("self",) + assert set(hints) == {"return"} + assert get_origin(hints["return"]) is (AsyncIterator if is_async else Iterator) + assert get_args(hints["return"]) == (server_event,) + + +@pytest.mark.parametrize("preview", [None, False], ids=["default", "false"]) +async def test_beta_namespace_implies_preview_opt_in_without_allow_preview(live, preview): + options = {} if preview is None else {"allow_preview": preview} + client = live.new_client(**options) + manager = client.beta.voice_agents.live.connect(agent_name=_AGENT) + assert live.credential.calls == [] + live.connect.assert_not_called() + live.session_factory.assert_not_called() + + connection = await live.call(manager.enter) + try: + assert connection.closed is False + assert live.headers["Foundry-Features"] == "VoiceAgents=V1Preview" + assert live.headers["Authorization"] == f"Bearer {_TOKEN}" + assert live.credential.calls == [(_SCOPES, {})] + live.transport.writes.assert_not_called() + if live.is_async: + live.transport.session.ws_connect.assert_awaited_once() + else: + live.connect.assert_called_once() + finally: + await live.exit(manager) + live.transport.assert_closed_once() + + +async def test_handshake_encodes_live_route_authentication_and_all_options(live): + structured_inputs = {"name": "Zoë / guest", "count": 0, "enabled": False, "optional": None} + extra_query = MappingProxyType({"custom": "space + /?&=#"}) + extra_headers = MappingProxyType({"X-Custom": "example"}) + options = {"heartbeat": 3, "max_msg_size": 1024} if live.is_async else {"open_timeout": 3, "max_size": 1024} + scopes = ["https://scope-one.example/.default", "https://scope-two.example/.default"] + async with live.open( + agent_name="name /?#% 雪", + agent_version="7", + structured_inputs=structured_inputs, + store=False, + api_version="live-test-version", + credential_scopes=scopes, + extra_query=extra_query, + extra_headers=extra_headers, + **options, + ): + url, kwargs = live.handshake + assert urlsplit(url).path == ( + "/api/projects/fake-project/agents/name%20%2F%3F%23%25%20%E9%9B%AA" + "/endpoint/protocols/voice/live/sessions" + ) + assert urlsplit(url).scheme == "wss" + assert urlsplit(url).netloc == _HOST + query = _handshake_query(url, kwargs) + user_agent = live.headers["User-Agent"] + assert f"azsdk-python-ai-projects/{__version__}" in user_agent + assert query == { + "api-version": ["live-test-version"], + "x-ms-client-sdk": [user_agent], + "x-agent-version-override": ["7"], + "structured_inputs": [query["structured_inputs"][0]], + "store": ["false"], + "custom": ["space + /?&=#"], + } + assert json.loads(query["structured_inputs"][0]) == structured_inputs + assert live.headers == { + "Authorization": f"Bearer {_TOKEN}", + "Foundry-Features": "VoiceAgents=V1Preview", + "User-Agent": user_agent, + "X-Custom": "example", + } + assert live.credential.calls == [(tuple(scopes), {})] + assert all(kwargs[name] == value for name, value in options.items()) + assert not kwargs.get("protocols" if live.is_async else "subprotocols") + assert "Sec-WebSocket-Protocol" not in live.headers + assert "model" not in query + assert "transport" not in query + assert "model" not in kwargs + assert "transport" not in kwargs + if not live.is_async: + assert kwargs["user_agent_header"] is None + live.transport.writes.assert_not_called() + assert dict(extra_query) == {"custom": "space + /?&=#"} + assert dict(extra_headers) == {"X-Custom": "example"} + assert structured_inputs == {"name": "Zoë / guest", "count": 0, "enabled": False, "optional": None} + live.transport.assert_closed_once() + + +async def test_defaults_do_not_add_model_transport_startup_or_optional_query(live): + async with live.open(): + url, kwargs = live.handshake + assert urlsplit(url).path.endswith(f"/agents/{_AGENT}/endpoint/protocols/voice/live/sessions") + assert set(_handshake_query(url, kwargs)) == {"api-version", "x-ms-client-sdk"} + assert live.credential.calls == [(_SCOPES, {})] + assert not kwargs.get("protocols" if live.is_async else "subprotocols") + live.transport.writes.assert_not_called() + + +@pytest.mark.parametrize("header_name", ["User-Agent", "user-agent", "uSeR-aGeNt"]) +async def test_caller_user_agent_is_respected_case_insensitively(live, header_name): + headers = {header_name: "custom-test-client"} + async with live.open(extra_headers=headers): + matching = [(name, value) for name, value in live.headers.items() if name.lower() == "user-agent"] + assert matching == [(header_name, "custom-test-client")] + query = _handshake_query(*live.handshake) + assert f"azsdk-python-ai-projects/{__version__}" in query["x-ms-client-sdk"][0] + assert headers == {header_name: "custom-test-client"} + + +@pytest.mark.parametrize("authority", [_HOST, f"{_HOST.upper()}:443"]) +async def test_same_origin_override_preserves_opaque_query_and_caller_mappings(live, authority): + override = ( + f"wss://{authority}/custom/live" "?sig=opaque%2fsignature%2Bvalue%3D&keep=one&keep=two&empty=&path=folder%2F" + ) + query = {"custom": "space + /?&=#"} + headers = {"X-Custom": "unchanged"} + async with live.open(connection_url=override, extra_query=query, extra_headers=headers): + url, kwargs = live.handshake + assert url == override if live.is_async else url.startswith(override + "&") + parsed = _handshake_query(url, kwargs) + assert parsed["sig"] == ["opaque/signature+value="] + assert parsed["keep"] == ["one", "two"] + assert parsed["empty"] == [""] + assert parsed["path"] == ["folder/"] + assert parsed["custom"] == ["space + /?&=#"] + assert url.count("?") == 1 + assert "sig=opaque%2fsignature%2Bvalue%3D" in url + assert "path=folder%2F" in url + assert query == {"custom": "space + /?&=#"} + assert headers == {"X-Custom": "unchanged"} + + +@pytest.mark.parametrize( + "override", + [ + "wss://untrusted.example/custom", + f"wss://{_HOST}:8443/custom", + f"ws://{_HOST}/custom", + f"ws://{_HOST}:443/custom", + f"https://{_HOST}/custom", + f"http://{_HOST}:443/custom", + f"wss://{_HOST}:invalid/custom", + "not-a-url", + "", + ], +) +async def test_invalid_override_fails_before_token_or_transport(live, override): + manager = live.manager(connection_url=override) + with pytest.raises(ValueError): + await live.call(manager.enter) + assert live.credential.calls == [] + live.connect.assert_not_called() + live.session_factory.assert_not_called() + + +@pytest.mark.parametrize("endpoint", [f"http://{_HOST}/project", f"ftp://{_HOST}/project"]) +async def test_insecure_computed_endpoint_fails_before_token_or_transport(live, endpoint): + client = live.new_client(endpoint=endpoint, allow_preview=True) + manager = client.beta.voice_agents.live.connect(agent_name=_AGENT) + with pytest.raises(ValueError, match="wss"): + await live.call(manager.enter) + assert live.credential.calls == [] + live.connect.assert_not_called() + live.session_factory.assert_not_called() + + +@pytest.mark.parametrize( + "structured_inputs,extra_query", + [ + ({"name": "guest"}, {"structured_input": "{}"}), + (None, {"structured_inputs": "{}", "structured_input": "{}"}), + ({}, {"structured_inputs": "{}", "structured_input": "{}"}), + ], +) +async def test_structured_input_spellings_are_mutually_exclusive_before_auth(live, structured_inputs, extra_query): + manager = live.manager(structured_inputs=structured_inputs, extra_query=extra_query) + with pytest.raises(ValueError, match="structured_input.*structured_inputs"): + await live.call(manager.enter) + assert live.credential.calls == [] + live.connect.assert_not_called() + live.session_factory.assert_not_called() + assert "structured_input" in extra_query + + +@pytest.mark.parametrize("name", ["structured_input", "structured_inputs"]) +async def test_one_explicit_structured_input_query_spelling_is_preserved(live, name): + query = {name: '{"name": "guest"}'} + async with live.open(extra_query=query): + values = _handshake_query(*live.handshake) + assert values[name] == ['{"name": "guest"}'] + assert ("structured_inputs" if name == "structured_input" else "structured_input") not in values + assert query == {name: '{"name": "guest"}'} + + +@pytest.mark.parametrize("store", [True, False]) +async def test_typed_query_options_override_extra_query_without_mutating_it(live, store): + extra_query = { + "x-agent-version-override": "old-version", + "structured_inputs": '{"old":true}', + "store": "old-store", + } + async with live.open(agent_version="8", structured_inputs={}, store=store, extra_query=extra_query): + query = _handshake_query(*live.handshake) + assert query["x-agent-version-override"] == ["8"] + assert query["structured_inputs"] == ["{}"] + assert query["store"] == [str(store).lower()] + assert extra_query == { + "x-agent-version-override": "old-version", + "structured_inputs": '{"old":true}', + "store": "old-store", + } + + +async def test_manager_snapshots_extra_query_and_headers(live): + extra_query = {"custom": "original"} + extra_headers = {"X-Custom": "original"} + manager = live.manager(extra_query=extra_query, extra_headers=extra_headers) + extra_query["custom"] = "changed" + extra_headers["X-Custom"] = "changed" + await live.call(manager.enter) + try: + assert _handshake_query(*live.handshake)["custom"] == ["original"] + assert live.headers["X-Custom"] == "original" + finally: + await live.exit(manager) + + +async def test_optional_dependency_is_only_required_on_enter(live): + modules = {"aiohttp": None} if live.is_async else {"websockets.sync.client": None, "websockets.typing": None} + dependency = "aiohttp" if live.is_async else "websockets" + with patch.dict("sys.modules", modules): + client = live.new_client(allow_preview=True) + manager = client.beta.voice_agents.live.connect(agent_name=_AGENT) + assert live.credential.calls == [] + with pytest.raises(RuntimeError, match=f"pip install {dependency}") as caught: + await live.call(manager.enter) + assert "live" in str(caught.value).lower() + assert "realtime" not in str(caught.value).lower() + live.connect.assert_not_called() + live.session_factory.assert_not_called() + assert live.credential.calls == [] + + +async def test_token_failure_propagates_without_opening_websocket_or_leaking_session(live): + failure = RuntimeError("fake credential rejected the scope") + live.credential.failure = failure + with pytest.raises(RuntimeError) as caught: + await live.call(live.manager().enter) + assert caught.value is failure + assert live.credential.calls == [(_SCOPES, {})] + live.connect.assert_not_called() + if live.is_async: + live.transport.session.ws_connect.assert_not_awaited() + assert live.session_factory.call_count <= 1 + assert live.transport.session.close.await_count == live.session_factory.call_count + else: + live.session_factory.assert_not_called() + + +@pytest.mark.parametrize( + "failure,expected", + [ + (OSError("upgrade refused"), ConnectionError), + (ValueError("invalid websocket option"), ValueError), + (RuntimeError("websocket already closing"), RuntimeError), + ], +) +async def test_handshake_errors_preserve_causes_and_release_owned_resources(live, failure, expected): + connector = live.transport.session.ws_connect if live.is_async else live.connect + connector.side_effect = failure + manager = live.manager() + with pytest.raises(expected) as caught: + await live.call(manager.enter) + if expected is ConnectionError: + assert caught.value.__cause__ is failure + assert _AGENT in str(caught.value) + assert "live" in str(caught.value).lower() + assert "realtime" not in str(caught.value).lower() + else: + assert caught.value is failure + if live.is_async: + live.transport.session.close.assert_awaited_once() + live.transport.socket.close.assert_not_awaited() + await live.exit(manager) + await live.exit(manager) + if live.is_async: + live.transport.session.close.assert_awaited_once() + + +async def test_live_does_not_negotiate_realtime_even_with_caller_protocol_override(live): + option = "protocols" if live.is_async else "subprotocols" + async with live.open(**{option: ["realtime"]}): + assert not live.handshake[1].get(option) + assert "Sec-WebSocket-Protocol" not in live.headers + + +@pytest.mark.parametrize("kind", ["model", "mapping", "json"]) +async def test_raw_send_supports_generated_model_mapping_and_exact_json(live, kind): + expected = {"type": "response.create", "event_id": "evt-response"} + raw = '{ "type": "response.create", "event_id": "evt-response" }' + event = { + "model": models.LiveClientEventLiveResponseCreateParam(event_id="evt-response"), + "mapping": expected, + "json": raw, + }[kind] + async with live.open() as connection: + await live.call(connection.send, event) + assert live.transport.sent == [expected] + if kind == "json": + live.transport.writes.assert_called_once_with(raw) + assert expected == {"type": "response.create", "event_id": "evt-response"} + + +async def test_generated_session_start_is_available_only_when_explicitly_sent(live): + event = models.LiveSessionStartEvent( + session=models.LiveSessionCreateParams(model="gpt-live-1", store=False), + event_id="evt-raw-start", + ) + async with live.open() as connection: + live.transport.writes.assert_not_called() + await live.call(connection.send, event) + assert live.transport.sent == [ + { + "type": "session.start", + "event_id": "evt-raw-start", + "session": {"model": "gpt-live-1", "store": False}, + } + ] + + +@pytest.mark.parametrize("raw", ["not-json", '{"type":', ""]) +async def test_invalid_outgoing_json_never_reaches_socket(live, raw): + async with live.open() as connection: + with pytest.raises(ValueError, match="not valid JSON"): + await live.call(connection.send, raw) + live.transport.writes.assert_not_called() + assert connection.closed is False + + +@pytest.mark.parametrize("raw", ["null", "[]", '"future-value"', "true", "42", '{"type":[]}']) +async def test_valid_json_values_match_realtime_pass_through_behavior(live, raw): + async with live.open() as connection: + await live.call(connection.send, raw) + live.transport.writes.assert_called_once_with(raw) + assert live.transport.sent == [json.loads(raw)] + assert connection.closed is False + + +async def test_unserializable_mapping_is_rejected_before_socket_write(live): + async with live.open() as connection: + with pytest.raises(TypeError): + await live.call(connection.send, {"type": "response.create", "future": object()}) + live.transport.writes.assert_not_called() + assert connection.closed is False + + +@pytest.mark.parametrize("audio", [b"\x00\x01\xfe\xff", "AAH+/w==", b"", ""]) +@pytest.mark.parametrize("event_id", [None, "evt-audio"]) +async def test_input_audio_append_encodes_bytes_once_and_preserves_base64(live, audio, event_id): + expected = { + "type": "session.input_audio.append", + "audio": base64.b64encode(audio).decode("ascii") if isinstance(audio, bytes) else audio, + } + if event_id is not None: + expected["event_id"] = event_id + async with live.open() as connection: + await live.call(connection.session.input_audio.append, audio=audio, event_id=event_id) + assert live.transport.sent == [expected] + + +async def test_input_audio_mute_and_unmute_emit_only_live_commands(live): + async with live.open() as connection: + await live.call(connection.session.input_audio.mute, event_id="evt-mute") + await live.call(connection.session.input_audio.unmute) + assert live.transport.sent == [ + {"type": "session.input_audio.mute", "event_id": "evt-mute"}, + {"type": "session.input_audio.unmute"}, + ] + + +@pytest.mark.parametrize("resource", ["instructions", "thinking", "commentary"]) +@pytest.mark.parametrize("delegation_id", [None, "delegation-test"]) +@pytest.mark.parametrize("event_id", [None, "evt-text"]) +async def test_text_append_preserves_required_nullable_delegation_on_wire(live, resource, delegation_id, event_id): + expected = { + "type": f"session.{resource}.append", + "content": "Context for the Live model.", + "delegation_id": delegation_id, + } + if event_id is not None: + expected["event_id"] = event_id + async with live.open() as connection: + await live.call( + getattr(connection.session, resource).append, + content="Context for the Live model.", + delegation_id=delegation_id, + event_id=event_id, + ) + assert live.transport.sent == [expected] + assert "delegation_id" in live.transport.sent[0] + + +@pytest.mark.parametrize("resource", ["instructions", "thinking", "commentary"]) +async def test_text_append_cannot_omit_delegation_id(live, resource): + async with live.open() as connection: + append = getattr(connection.session, resource).append + parameter = inspect.signature(append).parameters["delegation_id"] + assert parameter.kind is inspect.Parameter.KEYWORD_ONLY + assert parameter.default is inspect.Parameter.empty + with pytest.raises(TypeError, match="delegation_id"): + await live.call(append, content="Context without a delegation argument") + live.transport.writes.assert_not_called() + + +@pytest.mark.parametrize("typed", [False, True], ids=["mapping", "generated-model"]) +async def test_session_update_serializes_sparse_delegation_settings(live, typed): + settings = { + "delegation": { + "type": "responses", + "responses": { + "instructions": "Answer concisely.", + "parallel_tool_calls": False, + "max_output_tokens": 32, + }, + } + } + session = models.LiveSessionUpdateParams(settings) if typed else settings + async with live.open() as connection: + await live.call(connection.session.update, session=session, event_id="evt-update") + assert live.transport.sent == [{"type": "session.update", "event_id": "evt-update", "session": settings}] + assert "model" not in settings + assert "audio" not in settings + + +@pytest.mark.parametrize( + "item", + [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Check this itinerary."}], + }, + { + "type": "function_call_output", + "call_id": "call-test", + "output": '{"available":false,"count":0}', + }, + ], + ids=["message", "function-result"], +) +async def test_response_item_create_preserves_responses_item_mapping(live, item): + expected = json.loads(json.dumps(item)) + async with live.open() as connection: + await live.call(connection.response.item.create, item=MappingProxyType(item), event_id="evt-item") + assert live.transport.sent == [{"type": "response.item.create", "event_id": "evt-item", "item": expected}] + assert item == expected + + +@pytest.mark.parametrize("event_id", [None, "evt-create"]) +async def test_response_create_has_only_event_id_and_no_cancel_helper(live, event_id): + async with live.open() as connection: + assert tuple(inspect.signature(connection.response.create).parameters) == ("event_id",) + assert not hasattr(connection.response, "cancel") + await live.call(connection.response.create, event_id=event_id) + expected = {"type": "response.create"} + if event_id is not None: + expected["event_id"] = event_id + assert live.transport.sent == [expected] + + +async def test_session_close_requests_shutdown_without_closing_transport(live): + live.transport.queue(_STARTED, _CLOSED) + async with live.open() as connection: + started = await live.call(connection.recv) + assert isinstance(started, models.LiveServerEventLiveSessionStarted) + assert isinstance(started.session, models.LiveSessionResourceParam) + assert started.session.id == "live-session-test" + assert isinstance(started.session.delegation, models.LiveClientDelegationParam) + await live.call(connection.session.close, event_id="evt-close") + assert live.transport.sent == [{"type": "session.close", "event_id": "evt-close"}] + assert connection.closed is False + live.transport.socket.close.assert_not_called() + closed = await live.call(connection.recv) + assert isinstance(closed, models.LiveServerEventLiveSessionClosed) + assert closed.reason == "close_requested" + assert closed.client_event_id == "evt-close" + assert closed.session.status == "active" + assert isinstance(closed.usage, models.LiveSessionUsage) + assert closed.usage.seconds == 1.25 + assert closed.as_dict() == _CLOSED + assert connection.closed is False + live.transport.socket.close.assert_not_called() + live.transport.assert_closed_once() + + +@pytest.mark.parametrize( + "event_type,event_model", + list(models.LiveServerEvent.__mapping__.items()), + ids=list(models.LiveServerEvent.__mapping__), +) +async def test_every_generated_live_server_event_is_dispatched(live, event_type, event_model): + payload = {"type": event_type, "event_id": "evt-dispatch", "future_field": {"enabled": False}} + live.transport.queue(payload) + async with live.open() as connection: + event = await live.call(connection.recv) + assert type(event) is event_model + assert isinstance(event, models.LiveServerEvent) + assert event.type == event_type + assert event.as_dict() == payload + + +@pytest.mark.parametrize("frame_type", [bytes, bytearray], ids=["bytes", "bytearray"]) +async def test_binary_json_audio_event_uses_generated_base64_string_field(live, frame_type): + payload = {"type": "session.output_audio.delta", "delta": "AAH+/w==", "start_ms": 0, "end_ms": 20} + live.transport.raw(frame_type(json.dumps(payload).encode("utf-8"))) + async with live.open() as connection: + event = await live.call(connection.recv) + assert isinstance(event, models.LiveOutputAudioDelta) + assert isinstance(event.delta, str) + assert base64.b64decode(event.delta) == b"\x00\x01\xfe\xff" + assert (event.start_ms, event.end_ms) == (0, 20) + assert event.as_dict() == payload + + +async def test_reflected_audio_and_transcript_fields_are_not_realtime_aliases(live): + reflected = {"type": "session.input_audio.append", "audio": "AAH+/w=="} + transcript = { + "type": "session.output_transcript.delta", + "event_id": "evt-transcript", + "client_event_id": "evt-client", + "delta": "A synthetic transcript.", + "start_ms": 5, + "end_ms": 15, + } + live.transport.queue(reflected, transcript) + async with live.open() as connection: + audio = await live.call(connection.recv) + text = await live.call(connection.recv) + assert isinstance(audio, models.LiveInputAudioAppend) + assert audio.audio == "AAH+/w==" + assert isinstance(text, models.LiveServerEventLiveOutputTranscriptDelta) + assert text.delta == "A synthetic transcript." + assert (text.start_ms, text.end_ms) == (5, 15) + assert audio.as_dict() == reflected + assert text.as_dict() == transcript + + +async def test_delegation_and_wrapped_responses_events_retain_full_wire_shape(live): + delegation = { + "type": "session.delegation.created", + "event_id": "evt-delegation", + "offset_ms": 20, + "delegation": { + "id": "delegation-test", + "type": "delegation", + "target": "responses", + "response_id": "response-test", + }, + } + response = { + "type": "response.event", + "event_id": "evt-response-event", + "client_event_id": "evt-response-create", + "delegation_id": "delegation-test", + "event": { + "type": "response.function_call_arguments.done", + "call_id": "call-test", + "arguments": '{"city":"Seattle"}', + "future": {"preserved": True}, + }, + } + live.transport.queue(delegation, response) + async with live.open() as connection: + created = await live.call(connection.recv) + wrapped = await live.call(connection.recv) + assert isinstance(created, models.LiveServerEventLiveDelegationCreated) + assert isinstance(created.delegation, models.LiveDelegationItem) + assert created.delegation.id == "delegation-test" + assert created.delegation.response_id == "response-test" + assert isinstance(wrapped, models.LiveServerEventLiveResponseEvent) + assert wrapped.delegation_id == "delegation-test" + assert wrapped.event == response["event"] + assert created.as_dict() == delegation + assert wrapped.as_dict() == response + + +@pytest.mark.parametrize("event_type", ["future.live.event", None, 1, False, [], {}]) +async def test_unknown_or_non_string_event_type_returns_unchanged_mapping(live, event_type): + payload = {"type": event_type, "future": {"text": "preserved", "optional": None}} + live.transport.queue(payload, {"future": ["missing", "type"]}) + async with live.open() as connection: + assert await live.call(connection.recv) == payload + assert await live.call(connection.recv) == {"future": ["missing", "type"]} + assert connection.closed is False + + +async def test_server_error_is_returned_as_typed_error_not_success_or_swallowed(live): + payload = { + "type": "error", + "event_id": "evt-server-error", + "client_event_id": "evt-request", + "error": { + "type": "invalid_request_error", + "code": "invalid_delegation", + "message": "The delegation does not exist.", + "param": "delegation_id", + "client_event_id": "evt-request", + }, + } + live.transport.queue(payload, _STARTED) + async with live.open() as connection: + event = await live.call(connection.recv) + assert isinstance(event, models.LiveServerEventLiveErrorEvent) + assert isinstance(event.error, models.LiveLiveError) + assert event.error.code == "invalid_delegation" + assert event.error.message == "The delegation does not exist." + assert event.client_event_id == "evt-request" + assert event.as_dict() == payload + assert connection.closed is False + assert isinstance(await live.call(connection.recv), models.LiveServerEventLiveSessionStarted) + + +@pytest.mark.parametrize("raw,exception", [("{invalid", ValueError), (b"\xff", UnicodeDecodeError)]) +async def test_invalid_incoming_frame_is_not_replaced_with_success(live, raw, exception): + live.transport.raw(raw) + async with live.open() as connection: + with pytest.raises(exception): + await live.call(connection.recv) + assert connection.closed is False + live.transport.writes.assert_not_called() + live.transport.assert_closed_once() + + +@pytest.mark.parametrize("protocol", ["live", "realtime"]) +@pytest.mark.parametrize("payload", [None, [], [{"type": "session.started"}], "session.started", 0, True]) +@pytest.mark.parametrize("binary", [False, True], ids=["text", "binary"]) +async def test_incoming_non_object_json_raises_value_error_without_poisoning_connection( + live, protocol, payload, binary +): + next_payload = _STARTED if protocol == "live" else {"type": "session.created", "session": {}} + live.transport.queue(payload, next_payload, binary=binary) + async with live.open(protocol=protocol) as connection: + with pytest.raises(ValueError): + await live.call(connection.recv) + assert connection.closed is False + live.transport.socket.close.assert_not_called() + event = await live.call(connection.recv) + expected_type = ( + models.LiveServerEventLiveSessionStarted if protocol == "live" else models.RealtimeServerEventSessionCreated + ) + assert isinstance(event, expected_type) + assert event.as_dict() == next_payload + live.transport.writes.assert_not_called() + live.transport.assert_closed_once() + + +@pytest.mark.parametrize("fail_in_body", [False, True], ids=["normal", "exception"]) +async def test_context_manager_releases_transport_and_preserves_body_exception(live, fail_in_body): + failure = RuntimeError("application failed") + try: + async with live.open() as connection: + if fail_in_body: + raise failure + except RuntimeError as caught: + assert fail_in_body + assert caught is failure + else: + assert not fail_in_body + assert connection.closed is True + live.transport.assert_closed_once() + + +async def test_explicit_close_is_idempotent_and_forwards_code_and_reason(live): + manager = live.manager() + connection = await live.call(manager.enter) + try: + await live.call(connection.close, code=1001, reason="finished ✓") + assert connection.closed is True + await live.call(connection.close) + finally: + await live.exit(manager) + await live.exit(manager) + live.transport.assert_closed_once() + expected = {"code": 1001} + expected["message" if live.is_async else "reason"] = "finished ✓".encode("utf-8") if live.is_async else "finished ✓" + live.transport.socket.close.assert_called_once_with(**expected) + live.transport.writes.assert_not_called() + + +async def test_connection_itself_is_a_context_manager(live): + manager = live.manager() + connection = await live.call(manager.enter) + try: + if live.is_async: + async with connection as entered: + assert entered is connection + else: + with connection as entered: + assert entered is connection + assert connection.closed is True + finally: + await live.exit(manager) + live.transport.assert_closed_once() + + +@pytest.mark.parametrize("protocol", ["live", "realtime"]) +async def test_task_cancellation_inside_connection_context_releases_resources(live, protocol): + entered = asyncio.Event() + block = asyncio.Event() + + async def use_connection(): + async with live.open(protocol=protocol): + entered.set() + await block.wait() + + task = asyncio.create_task(use_connection()) + try: + await asyncio.wait_for(entered.wait(), timeout=2) + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + live.transport.assert_closed_once() + + +@pytest.mark.parametrize("is_async", [False], ids=["sync"]) +async def test_sync_close_error_propagates_and_does_not_repeat_transport_cleanup(live): + failure = OSError("fake socket close failed") + live.transport.socket.close.side_effect = failure + manager = live.manager() + connection = manager.enter() + with pytest.raises(OSError) as caught: + connection.close() + assert caught.value is failure + assert connection.closed is True + connection.close() + await live.exit(manager) + live.transport.socket.close.assert_called_once_with(code=1000, reason="") + + +@pytest.mark.parametrize("protocol", ["live", "realtime"]) +@pytest.mark.parametrize("code", [1000, 1001]) +async def test_graceful_websocket_closure_ends_iteration_for_both_protocols(live, protocol, code): + if live.is_async: + live.transport.socket.closed = True + live.transport.socket.close_code = code + live.transport.socket.receive.side_effect = [ + _message(aiohttp.WSMsgType.TEXT, json.dumps(_STARTED if protocol == "live" else {"type": "future.event"})), + _message(aiohttp.WSMsgType.CLOSE), + ] + else: + live.transport.socket.recv.side_effect = [ + json.dumps(_STARTED if protocol == "live" else {"type": "future.event"}), + ConnectionClosedOK(Close(code, "finished"), None), + ] + async with live.open(protocol=protocol) as connection: + events = await live.collect(connection) + assert len(events) == 1 + assert events[0].type == "session.started" if protocol == "live" else events[0] == {"type": "future.event"} + assert connection.closed is True + if live.is_async: + live.transport.session.close.assert_awaited_once() + + +@pytest.mark.parametrize("protocol", ["live", "realtime"]) +@pytest.mark.parametrize("code", [1006, 1011]) +async def test_abnormal_websocket_closure_is_not_silently_end_of_stream(live, protocol, code): + if live.is_async: + live.transport.socket.closed = True + live.transport.socket.close_code = code + live.transport.socket.receive.return_value = _message(aiohttp.WSMsgType.CLOSE) + else: + live.transport.socket.recv.side_effect = ConnectionClosedError(Close(code, "failed"), None) + async with live.open(protocol=protocol) as connection: + with pytest.raises(ConnectionResetError) as caught: + await live.collect(connection) + assert caught.value.__cause__ is not None + assert protocol in str(caught.value).lower() + if protocol == "live": + assert "realtime" not in str(caught.value).lower() + assert connection.closed is True + if live.is_async: + live.transport.session.close.assert_awaited_once() + + +@pytest.mark.parametrize("is_async", [False], ids=["sync"]) +async def test_sync_receive_timeout_is_forwarded_and_does_not_close_connection(live): + live.transport.socket.recv.side_effect = [TimeoutError("not ready"), json.dumps(_STARTED)] + async with live.open() as connection: + with pytest.raises(TimeoutError, match="not ready"): + connection.recv(timeout=0.25) + assert connection.closed is False + assert isinstance(connection.recv(timeout=1), models.LiveServerEventLiveSessionStarted) + assert [call.kwargs for call in live.transport.socket.recv.call_args_list] == [ + {"timeout": 0.25}, + {"timeout": 1}, + ] + + +async def test_write_error_propagates_and_context_still_releases_resources(live): + failure = OSError("fake socket write failed") + live.transport.writes.side_effect = failure + with pytest.raises(OSError) as caught: + async with live.open() as connection: + await live.call(connection.response.create) + assert caught.value is failure + live.transport.assert_closed_once() + + +async def test_live_and_realtime_routes_events_and_resources_stay_independent(live): + first = live.transport + second = _Transport(live.is_async) + live.connect.side_effect = [first.socket, second.socket] + live.session_factory.side_effect = [first.session, second.session] + first.queue(_STARTED) + second.queue({"type": "session.created", "session": {}}) + first_manager = live.manager() + second_manager = live.manager(protocol="realtime") + first_connection = await live.call(first_manager.enter) + try: + second_connection = await live.call(second_manager.enter) + try: + assert isinstance(await live.call(first_connection.recv), models.LiveServerEventLiveSessionStarted) + assert isinstance(await live.call(second_connection.recv), models.RealtimeServerEventSessionCreated) + await live.call(first_connection.session.input_audio.mute) + await live.call(second_connection.input_audio_buffer.commit) + await live.call(second_connection.response.cancel, response_id="response-test") + assert first.sent == [{"type": "session.input_audio.mute"}] + assert second.sent == [ + {"type": "input_audio_buffer.commit"}, + {"type": "response.cancel", "response_id": "response-test"}, + ] + await live.call(first_connection.close) + assert first_connection.closed is True + assert second_connection.closed is False + assert not hasattr(first_connection, "input_audio_buffer") + assert not hasattr(first_connection.response, "cancel") + finally: + await live.exit(second_manager) + finally: + await live.exit(first_manager) + first.assert_closed_once() + second.assert_closed_once() + calls = ( + [first.session.ws_connect.await_args, second.session.ws_connect.await_args] + if live.is_async + else live.connect.call_args_list + ) + assert urlsplit(calls[0].args[0]).path.endswith("/endpoint/protocols/voice/live/sessions") + assert urlsplit(calls[1].args[0]).path.endswith("/endpoint/protocols/voice") + option = "protocols" if live.is_async else "subprotocols" + assert not calls[0].kwargs.get(option) + assert list(calls[1].kwargs[option]) == ["realtime"] + + +async def test_sending_while_receiving_does_not_serialize_or_poison_the_other_direction(live): + if live.is_async: + receiving = asyncio.Event() + release = asyncio.Event() + + async def receive(): + receiving.set() + await release.wait() + return _message(aiohttp.WSMsgType.TEXT, json.dumps(_STARTED)) + + live.transport.socket.receive.side_effect = receive + async with live.open() as connection: + task = asyncio.create_task(connection.recv()) + try: + await asyncio.wait_for(receiving.wait(), timeout=2) + await asyncio.wait_for(connection.session.input_audio.append(audio=b"\x00\x01"), timeout=2) + assert not task.done() + assert live.transport.sent == [{"type": "session.input_audio.append", "audio": "AAE="}] + finally: + release.set() + result = await asyncio.wait_for(task, timeout=2) + assert isinstance(result, models.LiveServerEventLiveSessionStarted) + else: + receiving = threading.Event() + release = threading.Event() + + def receive(**kwargs): + receiving.set() + assert release.wait(timeout=5), "The test did not release the fake receiver" + return json.dumps(_STARTED) + + live.transport.socket.recv.side_effect = receive + async with live.open() as connection: + with ThreadPoolExecutor(max_workers=2) as executor: + task = executor.submit(connection.recv) + try: + assert receiving.wait(timeout=2) + sender = executor.submit(connection.session.input_audio.append, audio=b"\x00\x01") + sender.result(timeout=2) + assert not task.done() + assert live.transport.sent == [{"type": "session.input_audio.append", "audio": "AAE="}] + finally: + release.set() + assert isinstance(task.result(timeout=2), models.LiveServerEventLiveSessionStarted) + live.transport.assert_closed_once() + + +async def test_logging_contains_metadata_but_never_tokens_queries_audio_or_text(live, caplog): + synthetic_audio = base64.b64encode(b"synthetic-private-audio").decode("ascii") + live.transport.queue({"type": "session.output_audio.delta", "delta": synthetic_audio}) + caplog.set_level(logging.DEBUG, logger="azure.ai.projects.live") + async with live.open( + connection_url=f"wss://{_HOST}/custom?sig=private-query", + structured_inputs={"name": "private-structured-input"}, + ) as connection: + await live.call(connection.session.input_audio.append, audio=synthetic_audio) + await live.call(connection.session.thinking.append, content="private-outbound-context", delegation_id=None) + await live.call(connection.recv) + messages = "\n".join(record.getMessage() for record in caplog.records if record.name == "azure.ai.projects.live") + assert f"WebSocket CONNECT target=wss://{_HOST}/custom" in messages + assert f"WebSocket CONNECTED target=wss://{_HOST}/custom" in messages + assert "WebSocket SEND type=session.input_audio.append bytes=" in messages + assert "WebSocket RECEIVE type=session.output_audio.delta bytes=" in messages + assert "WebSocket CLOSE code=1000" in messages + for value in (_TOKEN, "private-query", "private-structured-input", "private-outbound-context", synthetic_audio): + assert value not in messages + + +@pytest.mark.parametrize("is_async", [True], ids=["async"]) +async def test_async_receive_ignores_ping_and_pong_frames(live): + live.transport.socket.receive.side_effect = [ + _message(aiohttp.WSMsgType.PING, b""), + _message(aiohttp.WSMsgType.PONG, b""), + _message(aiohttp.WSMsgType.TEXT, json.dumps(_STARTED)), + ] + async with live.open() as connection: + event = await connection.recv() + assert isinstance(event, models.LiveServerEventLiveSessionStarted) + assert live.transport.socket.receive.await_count == 3 + live.transport.assert_closed_once() + + +@pytest.mark.parametrize("is_async", [True], ids=["async"]) +@pytest.mark.parametrize("frame_type", [aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSING, aiohttp.WSMsgType.CLOSED]) +async def test_async_remote_close_releases_session_even_when_socket_already_closed(live, frame_type): + live.transport.socket.closed = True + live.transport.socket.close_code = 1000 + live.transport.socket.receive.return_value = _message(frame_type) + manager = live.manager() + connection = await manager.enter() + try: + with pytest.raises(ConnectionResetError) as caught: + await connection.recv() + assert caught.value.__cause__ is None + assert connection.closed is True + finally: + await live.exit(manager) + await connection.close() + await live.exit(manager) + live.transport.session.close.assert_awaited_once() + assert live.transport.session.closed is True + + +@pytest.mark.parametrize("is_async", [True], ids=["async"]) +@pytest.mark.parametrize("protocol", ["live", "realtime"]) +async def test_async_error_frame_preserves_transport_failure_and_releases_session(live, protocol): + failure = OSError("fake websocket failed") + live.transport.socket.exception.return_value = failure + live.transport.socket.receive.return_value = _message(aiohttp.WSMsgType.ERROR) + async with live.open(protocol=protocol) as connection: + with pytest.raises(ConnectionResetError) as caught: + await live.collect(connection) + assert caught.value.__cause__ is failure + assert protocol in str(caught.value).lower() + if protocol == "live": + assert "realtime" not in str(caught.value).lower() + live.transport.assert_closed_once() + + +@pytest.mark.parametrize("is_async", [True], ids=["async"]) +@pytest.mark.parametrize("failure", [OSError("close failed"), asyncio.CancelledError()], ids=["error", "cancellation"]) +async def test_async_socket_close_failure_still_releases_session_once(live, failure): + live.transport.socket.close.side_effect = failure + manager = live.manager() + connection = await manager.enter() + with pytest.raises(type(failure)) as caught: + await connection.close() + assert caught.value is failure + live.transport.session.close.assert_awaited_once() + await connection.close() + await live.exit(manager) + live.transport.session.close.assert_awaited_once() + + +@pytest.mark.parametrize("is_async", [True], ids=["async"]) +@pytest.mark.parametrize("protocol", ["live", "realtime"]) +async def test_async_concurrent_close_only_releases_each_owned_resource_once(live, protocol): + closing = asyncio.Event() + second_started = asyncio.Event() + release = asyncio.Event() + + async def close_socket(**kwargs): + closing.set() + await release.wait() + live.transport.socket.closed = True + + live.transport.socket.close.side_effect = close_socket + manager = live.manager(protocol=protocol) + connection = await manager.enter() + + async def close_again(): + second_started.set() + await connection.close() + + first = asyncio.create_task(connection.close()) + second = None + try: + await asyncio.wait_for(closing.wait(), timeout=2) + second = asyncio.create_task(close_again()) + await asyncio.wait_for(second_started.wait(), timeout=2) + finally: + release.set() + tasks = [first] if second is None else [first, second] + await asyncio.wait_for(asyncio.gather(*tasks), timeout=2) + await live.exit(manager) + live.transport.assert_closed_once() + assert connection.closed is True + + +@pytest.mark.parametrize("is_async", [True], ids=["async"]) +@pytest.mark.parametrize("protocol", ["live", "realtime"]) +async def test_async_cancellation_during_authentication_cannot_leak_session(live, protocol): + authenticating = asyncio.Event() + block = asyncio.Event() + + async def get_token(*scopes, **kwargs): + live.credential.calls.append((scopes, kwargs)) + authenticating.set() + await block.wait() + return AccessToken(_TOKEN, 9_999_999_999) + + live.credential.get_token = get_token + manager = live.manager(protocol=protocol) + task = asyncio.create_task(manager.enter()) + try: + await asyncio.wait_for(authenticating.wait(), timeout=2) + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert live.credential.calls == [(_SCOPES, {})] + live.transport.session.ws_connect.assert_not_awaited() + live.transport.socket.close.assert_not_awaited() + assert live.session_factory.call_count <= 1 + assert live.transport.session.close.await_count == live.session_factory.call_count + await live.exit(manager) + assert live.transport.session.close.await_count == live.session_factory.call_count + + +@pytest.mark.parametrize("is_async", [True], ids=["async"]) +@pytest.mark.parametrize("protocol", ["live", "realtime"]) +async def test_async_cancelled_handshake_releases_session_and_propagates_cancellation(live, protocol): + entering = asyncio.Event() + block = asyncio.Event() + + async def ws_connect(*args, **kwargs): + entering.set() + await block.wait() + return live.transport.socket + + live.transport.session.ws_connect.side_effect = ws_connect + manager = live.manager(protocol=protocol) + task = asyncio.create_task(manager.enter()) + try: + await asyncio.wait_for(entering.wait(), timeout=2) + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + live.transport.session.close.assert_awaited_once() + live.transport.socket.close.assert_not_awaited() + await live.exit(manager) + await live.exit(manager) + live.transport.session.close.assert_awaited_once() + + +@pytest.mark.parametrize("is_async", [True], ids=["async"]) +@pytest.mark.parametrize("protocol", ["live", "realtime"]) +async def test_async_competing_receive_failure_does_not_poison_active_receiver(live, protocol): + receiving = asyncio.Event() + release = asyncio.Event() + active = False + payload = _STARTED if protocol == "live" else {"type": "session.created", "session": {}} + + async def receive(): + nonlocal active + if active: + raise RuntimeError("Concurrent call to receive() is not allowed") + active = True + receiving.set() + try: + await release.wait() + finally: + active = False + return _message(aiohttp.WSMsgType.TEXT, json.dumps(payload)) + + live.transport.socket.receive.side_effect = receive + async with live.open(protocol=protocol) as connection: + task = asyncio.create_task(connection.recv()) + try: + await asyncio.wait_for(receiving.wait(), timeout=2) + with pytest.raises(RuntimeError, match="Concurrent call to receive"): + await connection.recv() + assert not task.done() + assert connection.closed is False + live.transport.socket.close.assert_not_awaited() + live.transport.session.close.assert_not_awaited() + finally: + release.set() + event = await asyncio.wait_for(task, timeout=2) + assert event.type == payload["type"] + live.transport.assert_closed_once() + + +@pytest.mark.parametrize("is_async", [True], ids=["async"]) +@pytest.mark.parametrize("protocol", ["live", "realtime"]) +async def test_async_recv_cancellation_does_not_close_socket_or_cancel_unrelated_work(live, protocol): + receiving = asyncio.Event() + block = asyncio.Event() + calls = 0 + payload = _STARTED if protocol == "live" else {"type": "session.created", "session": {}} + + async def receive(): + nonlocal calls + calls += 1 + if calls == 1: + receiving.set() + try: + await block.wait() + except asyncio.CancelledError: + # aiohttp records 1006 on a cancelled receive without closing the socket. + live.transport.socket.close_code = 1006 + raise + return _message(aiohttp.WSMsgType.TEXT, json.dumps(payload)) + + live.transport.socket.receive.side_effect = receive + async with live.open(protocol=protocol) as connection: + task = asyncio.create_task(connection.recv()) + try: + await asyncio.wait_for(receiving.wait(), timeout=2) + unrelated = asyncio.create_task(connection.send({"type": "future.client.command", "event_id": "evt-other"})) + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + await asyncio.wait_for(unrelated, timeout=2) + assert connection.closed is False + live.transport.socket.close.assert_not_awaited() + live.transport.session.close.assert_not_awaited() + assert live.transport.sent == [{"type": "future.client.command", "event_id": "evt-other"}] + event = await asyncio.wait_for(connection.recv(), timeout=2) + expected_type = ( + models.LiveServerEventLiveSessionStarted if protocol == "live" else models.RealtimeServerEventSessionCreated + ) + assert isinstance(event, expected_type) + assert connection.closed is False + live.transport.assert_closed_once() diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py index 40f3dca663db..ac9b9428e120 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py +++ b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client.py @@ -232,17 +232,13 @@ def test_enter_caller_user_agent_overrides_default_case_insensitive(self): assert headers["user-agent"] == "custom-user-agent" def test_enter_source_retains_client_identification_wiring(self): - # Regression guard for the SDK client-identification fix (ported from azure-ai-voicelive - # PR #48848) surviving a future TypeSpec regeneration. `_realtime.py` is a hand-written - # file that is NOT `_patch.py`-named, so it isn't covered by the code generator's own - # "never touch _patch.py" guarantee -- nothing in the TypeSpec emitter is aware this file - # exists. The tests above already fail on a *behavioral* regression (wrong header/query - # value), but they exercise the code through mocks and could, in principle, still pass - # against a rewritten implementation that happens to produce the same observable values by - # a different (less safe) path. This inspects the actual source of `enter()` so a partial - # revert -- one that drops the case-insensitive guard, say, while keeping the header value - # correct for the common case -- is caught directly, independent of the tests above. - source = inspect.getsource(BetaRealtimeConnectionManager.enter) + # SDK identification belongs to the authenticated transport shared by Live and Realtime. + source = "\n".join( + inspect.getsource(cls.__dict__[method]) + for cls in BetaRealtimeConnectionManager.__mro__ + for method in ("enter", "_get_query") + if method in cls.__dict__ + ) assert "_USER_AGENT" in source assert "_has_header_case_insensitive" in source assert "x-ms-client-sdk" in source diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py index dc0018fb0232..fb069ba5921d 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py +++ b/sdk/ai/azure-ai-projects/tests/agents/test_realtime_client_async.py @@ -170,17 +170,13 @@ async def test_enter_caller_user_agent_overrides_default_case_insensitive(self): assert headers["user-agent"] == "custom-user-agent" async def test_enter_source_retains_client_identification_wiring(self): - # Regression guard for the SDK client-identification fix (ported from azure-ai-voicelive - # PR #48848) surviving a future TypeSpec regeneration. `aio/_realtime.py` is a hand-written - # file that is NOT `_patch.py`-named, so it isn't covered by the code generator's own - # "never touch _patch.py" guarantee -- nothing in the TypeSpec emitter is aware this file - # exists. The tests above already fail on a *behavioral* regression (wrong header/query - # value), but they exercise the code through mocks and could, in principle, still pass - # against a rewritten implementation that happens to produce the same observable values by - # a different (less safe) path. This inspects the actual source of `enter()` so a partial - # revert -- one that drops the case-insensitive guard, say, while keeping the header value - # correct for the common case -- is caught directly, independent of the tests above. - source = inspect.getsource(AsyncBetaRealtimeConnectionManager.enter) + # SDK identification belongs to the authenticated transport shared by Live and Realtime. + source = "\n".join( + inspect.getsource(cls.__dict__[method]) + for cls in AsyncBetaRealtimeConnectionManager.__mro__ + for method in ("enter", "_get_query") + if method in cls.__dict__ + ) assert "_USER_AGENT" in source assert "_has_header_case_insensitive" in source assert "x-ms-client-sdk" in source diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py new file mode 100644 index 000000000000..6b29b8b24e45 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py @@ -0,0 +1,813 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +"""Offline guards for the GPT Live sample protocol, cleanup, and WAV handling.""" + +import asyncio +import base64 +import importlib.util +import io +import json +from pathlib import Path +import sys +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock +import wave + +import pytest + +from azure.ai.projects import models + + +_SAMPLE_DIRECTORY = Path(__file__).resolve().parents[2] / "samples" / "agents" / "voice" +_SYNC_SCENARIOS = ("connection", "text_conversation", "function_tool") + + +def _load_module(monkeypatch, filename): + name = Path(filename).stem + spec = importlib.util.spec_from_file_location(name, _SAMPLE_DIRECTORY / filename) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, name, module) + spec.loader.exec_module(module) + return module + + +def _started(): + return models.LiveServerEventLiveSessionStarted({"event_id": "start-test", "session": {"id": "session-test"}}) + + +def _closed(): + return models.LiveServerEventLiveSessionClosed( + { + "event_id": "closed-test", + "reason": "close_requested", + "session": {"id": "session-test"}, + "usage": {"seconds": 1}, + } + ) + + +def _response(event_type, **fields): + return models.LiveServerEventLiveResponseEvent(event_id="response-event-test", event={"type": event_type, **fields}) + + +def _text_response(text): + return [ + _response("response.output_text.delta", item_id="message-test", content_index=0, delta=text), + _response("response.output_text.done", item_id="message-test", content_index=0, text=text), + _response("response.completed", response={"status": "completed", "output": []}), + ] + + +def _recording(payload=b"\x01\x00" * 1200, *, channels=1, rate=24000, sample_width=2): + stream = io.BytesIO() + with wave.open(stream, "wb") as writer: + writer.setnchannels(channels) + writer.setsampwidth(sample_width) + writer.setframerate(rate) + writer.writeframes(payload) + stream.seek(0) + return wave.open(stream, "rb") + + +@pytest.fixture +def sample_environment(monkeypatch): + monkeypatch.setenv("FOUNDRY_PROJECT_ENDPOINT", "https://example.invalid/api/projects/offline") + monkeypatch.setenv("FOUNDRY_VOICE_AGENT_MODEL", "live-deployment-test") + monkeypatch.setenv("FOUNDRY_MODEL_NAME", "backend-deployment-test") + monkeypatch.setenv("FOUNDRY_VOICE_AGENT_NAME", "live-offline") + monkeypatch.setenv("FOUNDRY_VOICE_AUDIO_INPUT_PATH", "offline-input.wav") + monkeypatch.delenv("FOUNDRY_VOICE_AUDIO_OUTPUT_PATH", raising=False) + monkeypatch.setattr("dotenv.load_dotenv", lambda: False) + + client = MagicMock() + client.__enter__.return_value = client + client.agents.create_version.return_value = SimpleNamespace(version="owned-version") + connection = client.beta.voice_agents.live.connect.return_value.__enter__.return_value + monkeypatch.setattr("azure.ai.projects.AIProjectClient", MagicMock(return_value=client)) + monkeypatch.setattr("azure.identity.DefaultAzureCredential", MagicMock()) + + async_client = MagicMock() + async_client.__aenter__.return_value = async_client + async_client.agents.create_version = AsyncMock(return_value=SimpleNamespace(version="owned-version")) + async_client.agents.delete_version = AsyncMock() + async_connection = async_client.beta.voice_agents.live.connect.return_value.__aenter__.return_value + async_connection.recv = AsyncMock() + async_connection.session.close = AsyncMock() + async_connection.session.input_audio.append = AsyncMock() + async_connection.session.commentary.append = AsyncMock() + async_connection.response.item.create = AsyncMock() + async_connection.response.create = AsyncMock() + monkeypatch.setattr("azure.ai.projects.aio.AIProjectClient", MagicMock(return_value=async_client)) + monkeypatch.setattr("azure.identity.aio.DefaultAzureCredential", MagicMock()) + + helper = _load_module(monkeypatch, "live_sample_utils.py") + return SimpleNamespace( + client=client, + connection=connection, + async_client=async_client, + async_connection=async_connection, + helper=helper, + ) + + +@pytest.mark.parametrize("scenario", _SYNC_SCENARIOS) +def test_sync_samples_delete_only_the_owned_version_on_failure(monkeypatch, sample_environment, scenario): + sample = _load_module(monkeypatch, f"sample_voice_agent_live_{scenario}.py") + client = sample_environment.client + connection = sample_environment.connection + connection.recv.side_effect = RuntimeError("offline receive failure") + connection.session.close.side_effect = ConnectionError("offline cleanup failure") + + with pytest.raises(RuntimeError, match="offline receive failure"): + sample.main() + + agent_name = client.agents.create_version.call_args.kwargs["agent_name"] + assert agent_name.startswith("live-offline-") + client.beta.voice_agents.live.connect.assert_called_once_with( + agent_name=agent_name, agent_version="owned-version", store=False + ) + client.agents.delete_version.assert_called_once_with(agent_name=agent_name, agent_version="owned-version") + client.agents.delete.assert_not_called() + connection.session.close.assert_called_once() + connection.session.start.assert_not_called() + connection.response.cancel.assert_not_called() + client.beta.voice_agents.realtime.connect.assert_not_called() + client.beta.voice_agents.live.connect.return_value.__exit__.assert_called_once() + + +@pytest.mark.parametrize("scenario", _SYNC_SCENARIOS) +def test_failed_agent_creation_does_not_delete_an_existing_version(monkeypatch, sample_environment, scenario): + sample = _load_module(monkeypatch, f"sample_voice_agent_live_{scenario}.py") + sample_environment.client.agents.create_version.side_effect = RuntimeError("create failed") + + with pytest.raises(RuntimeError, match="create failed"): + sample.main() + + sample_environment.client.agents.delete_version.assert_not_called() + sample_environment.client.beta.voice_agents.live.connect.assert_not_called() + + +def test_connection_waits_for_start_and_confirms_graceful_close(monkeypatch, sample_environment, capsys): + sample = _load_module(monkeypatch, "sample_voice_agent_live_connection.py") + connection = sample_environment.connection + connection.recv.side_effect = [ + {"type": "future.event"}, + _started(), + {"type": "session.updated"}, + {"type": "session.instructions.appended"}, + _closed(), + ] + + sample.main() + + definition = sample_environment.client.agents.create_version.call_args.kwargs["definition"] + assert isinstance(definition, models.VoiceAgentLiveDefinition) + assert definition.model == "live-deployment-test" + assert definition.sub_protocol == "live" + assert definition.model_type == "self_deployed" + assert definition.audio.format.rate == 24000 + assert definition.audio.output.voice == "marin" + connection.session.instructions.append.assert_called_once_with( + content="Keep responses brief. Do not invent external facts.", delegation_id=None + ) + assert "Live session started: session-test" in capsys.readouterr().out + + +def test_text_uses_responses_delegation_and_granular_text(monkeypatch, sample_environment, capsys): + sample = _load_module(monkeypatch, "sample_voice_agent_live_text_conversation.py") + connection = sample_environment.connection + connection.recv.side_effect = [_started(), *_text_response("Seattle has a rainy climate."), _closed()] + + sample.main() + + definition = sample_environment.client.agents.create_version.call_args.kwargs["definition"] + assert isinstance(definition.delegation, models.LiveResponsesDelegationParam) + assert definition.delegation.responses.model == "backend-deployment-test" + item = connection.response.item.create.call_args.kwargs["item"] + assert item["type"] == "message" and item["role"] == "user" + assert item["content"][0]["type"] == "input_text" + connection.response.create.assert_called_once_with() + assert "Responses backend: Seattle has a rainy climate." in capsys.readouterr().out + + +def test_text_rejects_empty_backend_completion(monkeypatch, sample_environment): + sample = _load_module(monkeypatch, "sample_voice_agent_live_text_conversation.py") + sample_environment.connection.recv.side_effect = [ + _started(), + _response("response.completed", response={"status": "completed", "output": []}), + ] + + with pytest.raises(RuntimeError, match="without returning any output text"): + sample.main() + + sample_environment.client.agents.delete_version.assert_called_once() + + +@pytest.mark.parametrize("event_type", ["error", "response.failed", "response.incomplete", "response.cancelled"]) +def test_backend_failures_are_not_successful_completions(sample_environment, event_type): + with pytest.raises(RuntimeError, match="Responses backend"): + sample_environment.helper.check_event(_response(event_type, response={"error": {"message": "backend failed"}})) + + +def test_outer_live_errors_are_raised(sample_environment): + event = models.LiveServerEventLiveErrorEvent( + event_id="error-test", + error=models.LiveLiveError(type="invalid_request_error", code="invalid_command", message="command rejected"), + ) + with pytest.raises(RuntimeError, match="invalid_command: command rejected"): + sample_environment.helper.check_event(event) + + +def test_lifecycle_wait_has_one_deadline(monkeypatch, sample_environment): + helper = sample_environment.helper + monkeypatch.setattr(helper, "time", SimpleNamespace(monotonic=MagicMock(side_effect=[0, 1, 61]))) + sample_environment.connection.recv.return_value = {"type": "future.event"} + + with pytest.raises(TimeoutError, match="session.started"): + helper.wait_for_event(sample_environment.connection, "session.started") + + sample_environment.connection.recv.assert_called_once_with(timeout=59) + + +@pytest.mark.asyncio +async def test_async_lifecycle_timeout_names_the_missing_event(sample_environment): + sample_environment.async_connection.recv.side_effect = asyncio.TimeoutError + with pytest.raises(TimeoutError, match="session.started"): + await sample_environment.helper.wait_for_event_async(sample_environment.async_connection, "session.started") + + +@pytest.mark.asyncio +async def test_async_close_send_is_bounded(monkeypatch, sample_environment): + monkeypatch.setattr(sample_environment.helper, "CLOSE_TIMEOUT", 0.01) + + async def stalled_close(): + await asyncio.Future() + + sample_environment.async_connection.session.close.side_effect = stalled_close + with pytest.raises(TimeoutError, match="closing the GPT Live session"): + await sample_environment.helper.close_session_async(sample_environment.async_connection) + sample_environment.async_connection.session.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_close_timeout_preserves_the_original_error(monkeypatch, sample_environment): + monkeypatch.setattr(sample_environment.helper, "CLOSE_TIMEOUT", 0.01) + + async def stalled_close(): + await asyncio.Future() + + sample_environment.async_connection.session.close.side_effect = stalled_close + with pytest.raises(RuntimeError, match="original receive failure"): + try: + raise RuntimeError("original receive failure") + finally: + await sample_environment.helper.close_session_async(sample_environment.async_connection) + sample_environment.async_connection.session.close.assert_awaited_once() + + +def test_function_tool_waits_for_each_completed_cycle(monkeypatch, sample_environment, capsys): + sample = _load_module(monkeypatch, "sample_voice_agent_live_function_tool.py") + call = { + "id": "fc-item-test", + "type": "function_call", + "name": "get_weather", + "call_id": "call-test", + "arguments": '{"city":"Seattle"}', + "status": "completed", + } + connection = sample_environment.connection + connection.recv.side_effect = [ + _started(), + _response("response.output_item.added", item={**call, "arguments": "", "status": "in_progress"}), + _response("response.function_call_arguments.delta", item_id="fc-item-test", delta='{"city":'), + _response("response.function_call_arguments.delta", item_id="fc-item-test", delta='"Seattle"}'), + _response( + "response.function_call_arguments.done", + item_id="fc-item-test", + name="get_weather", + arguments='{"city":"Seattle"}', + ), + _response("response.output_item.done", item=call), + _response("response.output_item.done", item=call), + _response("response.completed", response={"status": "completed", "output": []}), + *_text_response("The sample weather in Seattle is sunny, 72 F."), + _closed(), + ] + + sample.main() + + submitted = connection.response.item.create.call_args_list + assert len(submitted) == 2 + result = submitted[1].kwargs["item"] + assert result["type"] == "function_call_output" + assert result["call_id"] == "call-test" + assert json.loads(result["output"]) == {"city": "Seattle", "condition": "sunny", "temperature_f": 72} + assert connection.response.create.call_count == 2 + assert "Responses backend: The sample weather in Seattle is sunny, 72 F." in capsys.readouterr().out + + +def test_argument_done_without_call_id_does_not_invent_a_tool_call(sample_environment): + sample_environment.connection.recv.side_effect = [ + _response( + "response.function_call_arguments.done", + item_id="fc-item-test", + name="get_weather", + arguments='{"city":"Seattle"}', + ), + _response("response.completed", response={"status": "completed", "output": []}), + ] + + text, calls = sample_environment.helper.receive_response(sample_environment.connection) + + assert text == "" + assert calls == [] + + +def test_completed_function_call_requires_its_canonical_call_id(sample_environment): + sample_environment.connection.recv.return_value = _response( + "response.output_item.done", + item={"id": "fc-item-test", "type": "function_call", "name": "get_weather", "arguments": '{"city":"Seattle"}'}, + ) + + with pytest.raises(RuntimeError, match="function_call item omitted a usable call_id"): + sample_environment.helper.receive_response(sample_environment.connection) + + +@pytest.mark.parametrize("arguments", ['{"city":1}', '{"city":"Seattle","extra":true}', '["Seattle"]']) +def test_function_tool_rejects_invalid_arguments(monkeypatch, sample_environment, arguments): + sample = _load_module(monkeypatch, "sample_voice_agent_live_function_tool.py") + with pytest.raises(ValueError, match="exactly one string argument"): + sample._execute_tool({"name": "get_weather", "arguments": arguments}) + + +@pytest.mark.parametrize("options", [{"channels": 2}, {"rate": 16000}, {"sample_width": 1}, {"payload": b""}]) +def test_audio_rejects_wrong_or_empty_wav_before_connecting(monkeypatch, sample_environment, options): + sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + with _recording(**options) as recording: + with pytest.raises(ValueError, match="WAV|no audio frames"): + sample._validate_recording(recording) + sample_environment.async_client.beta.voice_agents.live.connect.assert_not_called() + + +@pytest.mark.asyncio +async def test_audio_streams_raw_frames_concurrently_and_collects_transcripts(monkeypatch, sample_environment, capsys): + sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + connection = sample_environment.async_connection + original_sleep = asyncio.sleep + + async def yield_without_delay(_delay): + await original_sleep(0) + + monkeypatch.setattr(sample.asyncio, "sleep", yield_without_delay) + monkeypatch.setattr(sample, "_OUTPUT_IDLE_SECONDS", 0) + payload = b"\x01\x00" * 2400 + reply = b"\x00\x08" * 1200 + events = iter( + [ + models.LiveServerEventLiveInputTranscriptDelta(event_id="input", start_ms=0, end_ms=50, delta="Hello."), + models.LiveOutputAudioDelta(delta=base64.b64encode(reply).decode("ascii")), + models.LiveServerEventLiveOutputTranscriptDelta(event_id="output", start_ms=0, end_ms=50, delta="Hi!"), + models.LiveServerEventLiveDelegationCreated( + event_id="delegation", + offset_ms=50, + delegation=models.LiveDelegationItem(id="delegation-test", target="client"), + ), + ] + ) + received_during_send = [] + + async def receive(): + await original_sleep(0) + try: + event = next(events) + except StopIteration as exc: + raise asyncio.TimeoutError from exc + received_during_send.append(connection.session.input_audio.append.await_count < 22) + return event + + connection.recv.side_effect = receive + output_stream = io.BytesIO() + with _recording(payload) as recording, wave.open(output_stream, "wb") as output_file: + output_file.setparams((1, 2, 24000, 0, "NONE", "not compressed")) + await sample._stream_recording(connection, recording, output_file) + output_stream.seek(0) + with wave.open(output_stream, "rb") as recording: + assert recording.readframes(recording.getnframes()) == reply + sent = [call.kwargs["audio"] for call in connection.session.input_audio.append.await_args_list] + assert b"".join(sent[:2]) == payload + assert sum(map(len, sent)) == len(payload) + 48000 + assert all(not chunk.startswith(b"RIFF") for chunk in sent) + assert any(received_during_send) + connection.session.commentary.append.assert_awaited_once_with( + content="This sample cannot look up external information. Briefly acknowledge the recorded request.", + delegation_id="delegation-test", + ) + text = capsys.readouterr().out + assert "Input transcript: Hello." in text and "Output transcript: Hi!" in text + assert f"Received {len(reply)} PCM16 bytes" in text + + +@pytest.mark.asyncio +async def test_audio_send_failure_cancels_the_receiver(monkeypatch, sample_environment): + sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + connection = sample_environment.async_connection + connection.session.input_audio.append.side_effect = ConnectionError("audio send failed") + receiver_stopped = asyncio.Event() + + async def receive(): + try: + await asyncio.Future() + finally: + receiver_stopped.set() + + connection.recv.side_effect = receive + with _recording() as recording: + with pytest.raises(ConnectionError, match="audio send failed"): + await sample._stream_recording(connection, recording, None) + assert receiver_stopped.is_set() + connection.response.cancel.assert_not_called() + + +@pytest.mark.asyncio +async def test_audio_cannot_complete_without_reply_bytes(monkeypatch, sample_environment): + sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + finished = asyncio.Event() + finished.set() + with pytest.raises(TimeoutError, match="GPT Live reply audio"): + await sample._receive_audio(sample_environment.async_connection, finished, 0, None) + sample_environment.async_connection.recv.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_audio_idle_completion_does_not_require_a_receive_timeout(monkeypatch, sample_environment): + sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + monkeypatch.setattr(sample, "_OUTPUT_IDLE_SECONDS", 0) + finished = asyncio.Event() + finished.set() + reply = b"\x00\x08" * 1200 + sample_environment.async_connection.recv.return_value = models.LiveOutputAudioDelta( + delta=base64.b64encode(reply).decode("ascii") + ) + output = await sample._receive_audio(sample_environment.async_connection, finished, 1, None) + assert output.num_bytes == len(reply) + sample_environment.async_connection.recv.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("delta", ["not valid base64", base64.b64encode(b"\x01").decode("ascii")]) +async def test_audio_rejects_invalid_or_partial_pcm(monkeypatch, sample_environment, delta): + sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + sample_environment.async_connection.recv.return_value = models.LiveOutputAudioDelta(delta=delta) + with pytest.raises(ValueError): + await sample._receive_audio(sample_environment.async_connection, asyncio.Event(), 1, None) + + +@pytest.mark.asyncio +async def test_audio_refuses_to_overwrite_the_input_recording(monkeypatch, sample_environment): + sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + monkeypatch.setenv("FOUNDRY_VOICE_AUDIO_OUTPUT_PATH", "offline-input.wav") + with pytest.raises(ValueError, match="must not overwrite"): + await sample.main() + sample_environment.async_client.agents.create_version.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [RuntimeError("offline receive failed"), asyncio.CancelledError()]) +async def test_async_sample_deletes_its_version_on_error_or_cancellation(monkeypatch, sample_environment, failure): + sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + with _recording() as recording: + monkeypatch.setattr(sample, "wave", SimpleNamespace(open=MagicMock(return_value=recording))) + sample_environment.async_connection.recv.side_effect = failure + with pytest.raises(type(failure)): + await sample.main() + + client = sample_environment.async_client + agent_name = client.agents.create_version.call_args.kwargs["agent_name"] + client.beta.voice_agents.live.connect.assert_called_once_with( + agent_name=agent_name, agent_version="owned-version", store=False + ) + client.agents.delete_version.assert_awaited_once_with(agent_name=agent_name, agent_version="owned-version") + client.agents.delete.assert_not_called() + sample_environment.async_connection.session.close.assert_awaited_once() + client.beta.voice_agents.live.connect.return_value.__aexit__.assert_awaited_once() + + +def _audio_sample_with_clock(monkeypatch, sample_environment): + sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + clock = SimpleNamespace(now=0.0) + timer = SimpleNamespace(monotonic=lambda: clock.now) + monkeypatch.setattr(sample, "time", timer) + monkeypatch.setattr(sample_environment.helper, "time", timer) + return sample, clock + + +@pytest.mark.asyncio +@pytest.mark.parametrize("speech_level", [257, -257, 32767, -32768]) +@pytest.mark.parametrize("quiet_level", [0, 1, -1, 256, -256]) +async def test_audio_completion_ignores_continuous_quiet_pcm( + monkeypatch, sample_environment, speech_level, quiet_level +): + sample, clock = _audio_sample_with_clock(monkeypatch, sample_environment) + connection = sample_environment.async_connection + speech = speech_level.to_bytes(2, "little", signed=True) * 480 + quiet = quiet_level.to_bytes(2, "little", signed=True) * 480 + + async def receive(): + clock.now += 1 + pcm = speech if clock.now == 1 else quiet + return models.LiveOutputAudioDelta(delta=base64.b64encode(pcm).decode("ascii")) + + connection.recv.side_effect = receive + finished = asyncio.Event() + finished.set() + output_file = MagicMock() + + output = await sample._receive_audio(connection, finished, 10, output_file) + + assert clock.now == 4 + assert output.has_audio_activity + assert output.num_bytes == len(speech) + 3 * len(quiet) + assert output.chunks == 4 + assert b"".join(call.args[0] for call in output_file.writeframesraw.call_args_list) == speech + quiet * 3 + assert connection.recv.await_count == 4 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("quiet_level", [0, 1, -1, 256, -256]) +async def test_audio_silence_only_cannot_be_a_successful_reply(monkeypatch, sample_environment, quiet_level): + sample, clock = _audio_sample_with_clock(monkeypatch, sample_environment) + connection = sample_environment.async_connection + quiet = quiet_level.to_bytes(2, "little", signed=True) * 480 + + async def receive(): + clock.now += 1 + if clock.now == 1: + return models.LiveOutputAudioDelta(delta=base64.b64encode(quiet).decode("ascii")) + raise asyncio.TimeoutError + + connection.recv.side_effect = receive + finished = asyncio.Event() + finished.set() + + with pytest.raises(TimeoutError, match="GPT Live reply audio"): + await sample._receive_audio(connection, finished, 10, None) + + assert clock.now == 10 + assert connection.recv.await_count == 10 + + +@pytest.mark.asyncio +async def test_continuous_reply_activity_does_not_extend_the_overall_audio_deadline(monkeypatch, sample_environment): + sample, clock = _audio_sample_with_clock(monkeypatch, sample_environment) + connection = sample_environment.async_connection + pcm = b"\x00\x08" * 480 + + async def receive(): + clock.now += 1 + return models.LiveOutputAudioDelta(delta=base64.b64encode(pcm).decode("ascii")) + + connection.recv.side_effect = receive + finished = asyncio.Event() + finished.set() + + with pytest.raises(TimeoutError, match="GPT Live reply audio"): + await sample._receive_audio(connection, finished, 10, None) + + assert clock.now == 10 + assert connection.recv.await_count == 10 + + +def _text_sample(monkeypatch, sample_environment, asynchronous): + suffix = "_async" if asynchronous else "" + sample = _load_module(monkeypatch, f"sample_voice_agent_live_text_conversation{suffix}.py") + client = sample_environment.async_client if asynchronous else sample_environment.client + connection = sample_environment.async_connection if asynchronous else sample_environment.connection + return sample, client, connection + + +async def _run_text_sample(sample, asynchronous, prompts=None): + if asynchronous: + await sample.main(prompts) + else: + sample.main(prompts) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_text_samples_reuse_one_session_and_wait_for_each_turn( + monkeypatch, sample_environment, capsys, asynchronous +): + sample, client, connection = _text_sample(monkeypatch, sample_environment, asynchronous) + prompts = ["My name is Ada.", "What is my name?"] + events = iter([_started(), *_text_response("Hello, Ada."), *_text_response("Your name is Ada."), _closed()]) + completed = 0 + submitted = [] + + def receive(*args, **kwargs): + nonlocal completed + event = next(events) + if event.get("type") == "response.event" and event["event"]["type"] == "response.completed": + completed += 1 + return event + + def submit(*, item): + assert completed == len(submitted) + submitted.append(item["content"][0]["text"]) + + connection.recv.side_effect = receive + connection.response.item.create.side_effect = submit + + await _run_text_sample(sample, asynchronous, prompts) + + assert submitted == prompts + assert completed == 2 + assert connection.response.create.call_count == 2 + assert connection.session.close.call_count == 1 + client.beta.voice_agents.live.connect.assert_called_once() + client.agents.create_version.assert_called_once() + client.agents.delete_version.assert_called_once() + client.agents.delete.assert_not_called() + connection.session.start.assert_not_called() + connection.response.cancel.assert_not_called() + if asynchronous: + assert connection.response.item.create.await_count == 2 + assert connection.response.create.await_count == 2 + connection.session.close.assert_awaited_once() + client.agents.delete_version.assert_awaited_once() + client.beta.voice_agents.live.connect.return_value.__aexit__.assert_awaited_once() + else: + client.beta.voice_agents.live.connect.return_value.__exit__.assert_called_once() + output = capsys.readouterr().out + assert output.count("Responses backend: Hello, Ada.\n") == 1 + assert output.count("Responses backend: Your name is Ada.\n") == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_text_samples_keep_the_original_single_prompt_default(monkeypatch, sample_environment, asynchronous): + sample, client, connection = _text_sample(monkeypatch, sample_environment, asynchronous) + connection.recv.side_effect = [_started(), *_text_response("Seattle has a rainy climate."), _closed()] + + await _run_text_sample(sample, asynchronous) + + definition = client.agents.create_version.call_args.kwargs["definition"] + assert isinstance(definition.delegation, models.LiveResponsesDelegationParam) + assert definition.delegation.responses.model == "backend-deployment-test" + connection.response.item.create.assert_called_once_with( + item={ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Why is Seattle known for rain? Answer in one short sentence."}], + } + ) + connection.response.create.assert_called_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("prompts", [[], [""], ["first", " "], "not-a-prompt-list"]) +async def test_text_samples_reject_empty_or_invalid_scripted_prompts( + monkeypatch, sample_environment, asynchronous, prompts +): + sample, client, _ = _text_sample(monkeypatch, sample_environment, asynchronous) + + with pytest.raises(ValueError, match="prompt"): + await _run_text_sample(sample, asynchronous, prompts) + + client.agents.create_version.assert_not_called() + client.beta.voice_agents.live.connect.assert_not_called() + client.agents.delete_version.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_text_samples_stop_on_a_later_backend_failure_and_clean_up(monkeypatch, sample_environment, asynchronous): + sample, client, connection = _text_sample(monkeypatch, sample_environment, asynchronous) + connection.recv.side_effect = [ + _started(), + *_text_response("First answer."), + _response("response.failed", response={"error": {"message": "backend failed"}}), + ] + + with pytest.raises(RuntimeError, match="Responses backend response.failed"): + await _run_text_sample(sample, asynchronous, ["first", "second", "must not be sent"]) + + assert connection.response.item.create.call_count == 2 + assert connection.response.create.call_count == 2 + connection.session.close.assert_called_once() + agent_name = client.agents.create_version.call_args.kwargs["agent_name"] + client.agents.delete_version.assert_called_once_with(agent_name=agent_name, agent_version="owned-version") + if asynchronous: + client.agents.delete_version.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("empty", [True, False], ids=["empty-text", "unexpected-tool"]) +async def test_text_samples_reject_non_text_backend_completions(monkeypatch, sample_environment, asynchronous, empty): + sample, client, connection = _text_sample(monkeypatch, sample_environment, asynchronous) + events = [_started()] + if not empty: + events.append( + _response( + "response.output_item.done", + item={"type": "function_call", "call_id": "unexpected-call", "name": "unexpected", "arguments": "{}"}, + ) + ) + events.append(_response("response.completed", response={"status": "completed", "output": []})) + connection.recv.side_effect = events + + with pytest.raises(RuntimeError, match="without returning any output text|unexpectedly requested"): + await _run_text_sample(sample, asynchronous, ["one", "must not be sent"]) + + connection.response.item.create.assert_called_once() + client.agents.delete_version.assert_called_once() + + +@pytest.mark.asyncio +async def test_async_text_create_failure_does_not_delete_an_existing_version(monkeypatch, sample_environment): + sample, client, _ = _text_sample(monkeypatch, sample_environment, True) + client.agents.create_version.side_effect = RuntimeError("create failed") + + with pytest.raises(RuntimeError, match="create failed"): + await sample.main() + + client.agents.delete_version.assert_not_awaited() + client.beta.voice_agents.live.connect.assert_not_called() + + +@pytest.mark.asyncio +async def test_async_text_cancellation_propagates_and_deletes_the_owned_version(monkeypatch, sample_environment): + sample, client, connection = _text_sample(monkeypatch, sample_environment, True) + connection.recv.side_effect = asyncio.CancelledError + + with pytest.raises(asyncio.CancelledError): + await sample.main() + + connection.session.close.assert_awaited_once() + agent_name = client.agents.create_version.call_args.kwargs["agent_name"] + client.agents.delete_version.assert_awaited_once_with(agent_name=agent_name, agent_version="owned-version") + client.beta.voice_agents.live.connect.return_value.__aexit__.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_response_collection_keeps_canonical_tool_ids_and_granular_text(sample_environment): + connection = sample_environment.async_connection + call = { + "type": "function_call", + "call_id": "canonical-call", + "name": "get_weather", + "arguments": '{"city":"Seattle"}', + } + connection.recv.side_effect = [ + _response("response.output_text.delta", item_id="message-test", content_index=0, delta="Partial"), + _response("response.output_text.done", item_id="message-test", content_index=0, text="Complete text"), + _response("response.function_call_arguments.done", item_id="different-item-id", arguments='{"city":"Seattle"}'), + _response("response.output_item.done", item=call), + _response("response.completed", response={"status": "completed", "output": []}), + ] + + text, calls = await sample_environment.helper.receive_response_async(connection) + + assert text == "Complete text" + assert calls == [call] + + +@pytest.mark.asyncio +async def test_async_response_collection_does_not_extend_its_deadline(monkeypatch, sample_environment): + helper = sample_environment.helper + monkeypatch.setattr(helper, "time", SimpleNamespace(monotonic=MagicMock(side_effect=[0, 1, 61]))) + sample_environment.async_connection.recv.return_value = {"type": "future.event"} + + with pytest.raises(TimeoutError, match="Responses backend to complete"): + await helper.receive_response_async(sample_environment.async_connection) + + sample_environment.async_connection.recv.assert_awaited_once() + assert sample_environment.async_connection.recv.call_count == 1 + + +@pytest.mark.asyncio +async def test_async_lifecycle_expired_deadline_does_not_create_an_unawaited_receive(monkeypatch, sample_environment): + helper = sample_environment.helper + monkeypatch.setattr(helper, "time", SimpleNamespace(monotonic=MagicMock(side_effect=[0, 1, 61]))) + sample_environment.async_connection.recv.return_value = {"type": "future.event"} + + with pytest.raises(TimeoutError, match="session.started"): + await helper.wait_for_event_async(sample_environment.async_connection, "session.started") + + sample_environment.async_connection.recv.assert_awaited_once() + assert sample_environment.async_connection.recv.call_count == 1 + + +@pytest.mark.asyncio +async def test_async_close_expired_deadline_does_not_create_an_unawaited_send(monkeypatch, sample_environment): + helper = sample_environment.helper + monkeypatch.setattr(helper, "time", SimpleNamespace(monotonic=MagicMock(side_effect=[0, 11]))) + + with pytest.raises(TimeoutError, match="closing the GPT Live session"): + await helper.close_session_async(sample_environment.async_connection) + + sample_environment.async_connection.session.close.assert_not_called() From bd0cf2b2ed22b277e9decd6bd45802c0d3ddf084 Mon Sep 17 00:00:00 2001 From: Xiting Zhang Date: Wed, 21 Oct 2026 10:42:10 -0700 Subject: [PATCH 02/10] Fix GPT Live event registry type narrowing Validate generated event registry values through a Mapping of objects so Pyright correctly narrows model classes without casts. Add invalid-registry regression coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/ai/azure-ai-projects/azure/ai/projects/_live_utils.py | 6 +++--- sdk/ai/azure-ai-projects/tests/agents/test_live_client.py | 8 ++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/_live_utils.py b/sdk/ai/azure-ai-projects/azure/ai/projects/_live_utils.py index 6945be4c068e..c1028ccdb93e 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/_live_utils.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/_live_utils.py @@ -22,15 +22,15 @@ _LOGGER = logging.getLogger("azure.ai.projects.live") -def _server_event_types() -> dict[str, Type[Model]]: +def _server_event_types(registry: Mapping[str, object]) -> dict[str, Type[Model]]: result: dict[str, Type[Model]] = {} - for name, cls in models.LiveServerEvent.__mapping__.items(): + for name, cls in registry.items(): if isinstance(cls, type) and issubclass(cls, Model): result[name] = cls return result -_SERVER_EVENT_TYPES = _server_event_types() +_SERVER_EVENT_TYPES = _server_event_types(models.LiveServerEvent.__mapping__) def _live_url(endpoint: str, agent_name: str) -> str: diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_live_client.py b/sdk/ai/azure-ai-projects/tests/agents/test_live_client.py index ab96c3e03c07..afbc3e04c2bd 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/test_live_client.py +++ b/sdk/ai/azure-ai-projects/tests/agents/test_live_client.py @@ -30,6 +30,7 @@ from websockets.frames import Close from azure.ai.projects import AIProjectClient, __version__, models, operations +from azure.ai.projects._live_utils import _server_event_types from azure.ai.projects.aio import AIProjectClient as AsyncAIProjectClient from azure.ai.projects.aio import operations as async_operations @@ -817,6 +818,13 @@ async def test_session_close_requests_shutdown_without_closing_transport(live): live.transport.assert_closed_once() +@pytest.mark.parametrize("invalid", [None, "not-a-model", 42, object, models.LiveServerEvent({"type": "future.event"})]) +async def test_live_event_registry_accepts_only_model_classes(invalid): + registry = {"session.started": models.LiveServerEventLiveSessionStarted, "invalid": invalid} + + assert _server_event_types(registry) == {"session.started": models.LiveServerEventLiveSessionStarted} + + @pytest.mark.parametrize( "event_type,event_model", list(models.LiveServerEvent.__mapping__.items()), From 08c70e84c77902c72296cd57c757fb3066eda02f Mon Sep 17 00:00:00 2001 From: Xiting Zhang Date: Wed, 21 Oct 2026 11:20:28 -0700 Subject: [PATCH 03/10] [azure-ai-projects] Add GPT Live REST conversation samples Add sync and async REST-only samples for stored conversation, item, and recording APIs, with opt-in downloads and deletion. Add 42 enabled offline guards and related documentation. Verified all 18 sync/async REST targets against the real service and 555 focused unit cases. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/ai/azure-ai-projects/CHANGELOG.md | 1 + sdk/ai/azure-ai-projects/README.md | 16 + .../samples/agents/voice/live_sample_utils.py | 28 +- ...mple_voice_agent_live_read_conversation.py | 184 ++++++++++++ ...oice_agent_live_read_conversation_async.py | 193 ++++++++++++ .../tests/samples/test_live_samples.py | 284 ++++++++++++++++++ 6 files changed, 704 insertions(+), 2 deletions(-) create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py create mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation_async.py diff --git a/sdk/ai/azure-ai-projects/CHANGELOG.md b/sdk/ai/azure-ai-projects/CHANGELOG.md index 3761928e5037..6b77372dd536 100644 --- a/sdk/ai/azure-ai-projects/CHANGELOG.md +++ b/sdk/ai/azure-ai-projects/CHANGELOG.md @@ -17,6 +17,7 @@ * Added GPT Live connection, text conversation, asynchronous PCM audio streaming, and function-tool examples under `samples/agents/voice`. * Updated GPT Live text examples with synchronous/asynchronous headless multi-turn conversations that reuse one session and preserve the default single-turn example. +* Added synchronous/asynchronous GPT Live REST examples for reading an existing stored conversation, its items, and audio metadata, with explicit opt-in recording downloads and conversation deletion. * Fixed the GPT Live WAV sample to ignore silent/very-low-level PCM when detecting reply inactivity, reject silence-only replies, and preserve captured PCM in audio counts and optional WAV output. ## 2.9.0 (2026-10-16) diff --git a/sdk/ai/azure-ai-projects/README.md b/sdk/ai/azure-ai-projects/README.md index f00851117ed5..c418ac4fec73 100644 --- a/sdk/ai/azure-ai-projects/README.md +++ b/sdk/ai/azure-ai-projects/README.md @@ -260,6 +260,22 @@ The WAV audio example uses a bounded speech-inactivity heuristic because Live ha event. Continued silent PCM does not keep it waiting forever, and silence-only output is not a successful spoken reply. A long speech pause can still split a reply. +The [synchronous REST sample](samples/agents/voice/sample_voice_agent_live_read_conversation.py) +and [asynchronous REST sample](samples/agents/voice/sample_voice_agent_live_read_conversation_async.py) +read an existing stored Live conversation without opening a WebSocket. Set `FOUNDRY_PROJECT_ENDPOINT`, +`FOUNDRY_VOICE_AGENT_NAME`, and `FOUNDRY_VOICE_CONVERSATION_ID`. The last value must be +`session.started.conversation_id` from a session with effective `store=True`, not `session.id`. +The session must have finished successfully and contain recorded speech. + +Both samples demonstrate conversation/item listing and retrieval, plus merged and per-item audio +metadata. Pass `--output-dir ` to download Foundry-managed WAV audio without overwriting +existing files. Downloads save conversation content locally, not telemetry. BYOS audio requires your +own storage credentials. Nothing is deleted unless `--delete` is explicitly supplied; that option +deletes the selected conversation and its stored items/audio only after all requested reads succeed. +These samples use the dedicated `.beta.voice_agents.live_conversations` routes. A deployment that +does not support those routes returns HTTP 404, which is surfaced rather than silently falling back +to `.beta.voice_agents.conversations`. + ## Client-side tracing See [Add client-side tracing to Foundry agents (preview)](https://learn.microsoft.com/azure/foundry/observability/how-to/trace-agent-client-side?tabs=python). diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py b/sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py index 68865d9d6ca8..9dfc4c8005bc 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py @@ -7,11 +7,12 @@ from __future__ import annotations import asyncio -from contextlib import suppress +from contextlib import contextmanager, suppress import os +from pathlib import Path import sys import time -from typing import Any, Mapping, Optional, Sequence, TYPE_CHECKING +from typing import Any, BinaryIO, Iterator, Mapping, Optional, Sequence, TYPE_CHECKING from uuid import uuid4 from azure.ai.projects import models @@ -57,6 +58,29 @@ def safe_print(text: str) -> None: print(text.encode(encoding, errors="replace").decode(encoding)) +@contextmanager +def recording_output(path: Path) -> Iterator[BinaryIO]: + """Create an audio file exclusively and remove incomplete or empty downloads. + + :param path: Explicit destination that must not already exist. + :type path: pathlib.Path + :return: A writable binary stream for the downloaded audio. + :rtype: ~typing.Iterator[~typing.BinaryIO] + """ + output = path.open("xb") + completed = False + try: + with output: + yield output + if output.tell() == 0: + raise RuntimeError("The audio download returned no bytes.") + completed = True + finally: + if not completed: + path.unlink() + print(f"Saved recording: {path}") + + def remaining_time(deadline: float, operation: str) -> float: """Keep a stream of nonterminal events from extending a timeout forever.""" remaining = deadline - time.monotonic() diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py new file mode 100644 index 000000000000..07afdaf239fb --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py @@ -0,0 +1,184 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Read a stored GPT Live conversation using the preview + project_client.beta.voice_agents.live_conversations REST APIs. Retrieve the + conversation, recent conversations, ordered items, individual items, and + merged/per-item audio metadata. No agent or WebSocket session is created. + + Requires an ended speech session with effective store=True. Use + session.started.conversation_id, not session.id. These dedicated Live REST + routes must be available on the service deployment; HTTP 404 is reported + and raised, never replaced by calls to the older Realtime conversation API. + + By default, no audio files are written and nothing is deleted. Pass + --output-dir to download Foundry-managed WAV recordings without overwriting + existing files. BYOS recordings must be downloaded with your own storage + credentials. Pass --delete only for a conversation you intend to remove: + deletion also removes its stored items and audio. A read/download failure + prevents deletion. + +USAGE: + python sample_voice_agent_live_read_conversation.py + python sample_voice_agent_live_read_conversation.py --output-dir live-recordings + python sample_voice_agent_live_read_conversation.py --delete + + pip install azure-ai-projects azure-identity python-dotenv + + Environment variables (all required): + 1) FOUNDRY_PROJECT_ENDPOINT - The Foundry project endpoint. + 2) FOUNDRY_VOICE_AGENT_NAME - The existing GPT Live agent name. + 3) FOUNDRY_VOICE_CONVERSATION_ID - The stored Foundry conversation ID. + + Authenticates with DefaultAzureCredential. No model deployment name, + microphone, or WebSocket dependency is needed. +""" + +from __future__ import annotations + +import argparse +from itertools import islice +import os +from pathlib import Path +from typing import Optional + +from dotenv import load_dotenv +from live_sample_utils import recording_output, safe_print + +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient, models +from azure.ai.projects.operations import BetaVoiceAgentsLiveConversationsOperations + + +def _read_item( + conversations: BetaVoiceAgentsLiveConversationsOperations, + agent_name: str, + conversation_id: str, + item_id: str, + index: int, + output_directory: Optional[Path], +) -> None: + detail = conversations.get_item(agent_name, conversation_id, item_id) + print(f"Item {detail.id}: type={detail.type}") + if not isinstance( + detail, (models.VoiceAgentLiveInputConversationItem, models.VoiceAgentLiveOutputConversationItem) + ): + print(" Non-speech item; there is no speech audio to download.") + return + safe_print(f" Transcript: {detail.transcription or '(not captured)'}") + if detail.audio is None: + print(" No captured audio metadata for this speech item.") + return + audio = conversations.get_audio_item(agent_name, conversation_id, item_id) + print(f" Audio: role={audio.role}, sample_rate={audio.sample_rate}, duration={audio.duration_ms}") + if audio.blob_uri: + print(" BYOS item: download from your storage using your own credentials.") + if output_directory is not None: + raise RuntimeError("This sample downloads Foundry-managed audio only, not BYOS audio.") + elif output_directory is not None: + with recording_output(output_directory / f"item-{index:04d}.wav") as output: + for chunk in conversations.download_audio_item(agent_name, conversation_id, item_id): + output.write(chunk) + + +def _read_conversation( + conversations: BetaVoiceAgentsLiveConversationsOperations, + agent_name: str, + conversation_id: str, + *, + output_directory: Optional[Path], + delete: bool, +) -> None: + conversation = conversations.get(agent_name, conversation_id) + if conversation.status != "completed": + raise RuntimeError( + f"The stored session must finish successfully before reading its recording; " + f"status={conversation.status}, error={conversation.last_error}" + ) + print(f"Conversation {conversation.id}: status={conversation.status}, created_at={conversation.created_at}") + + print("Recent stored Live conversations (up to five):") + for recent in islice(conversations.list(agent_name, limit=5, order="desc"), 5): + print(f" {recent.id}: status={recent.status}") + + if output_directory is not None: + output_directory.mkdir(parents=True, exist_ok=True) + + seen_items: set[str] = set() + for index, item in enumerate(conversations.list_items(agent_name, conversation_id, limit=20, order="asc"), 1): + if item.id in seen_items: + raise RuntimeError("The conversation pager repeated an item instead of advancing.") + seen_items.add(item.id) + _read_item(conversations, agent_name, conversation_id, item.id, index, output_directory) + + recording = conversations.get_audio(agent_name, conversation_id) + print( + f"Merged recording: format={recording.format}, sample_rate={recording.sample_rate}, " + f"channels={recording.channels}, duration={recording.duration_ms}" + ) + if recording.blob_uri: + print("BYOS recording: download from your storage using your own credentials.") + if output_directory is not None: + raise RuntimeError("This sample downloads Foundry-managed audio only, not BYOS audio.") + elif output_directory is not None: + with recording_output(output_directory / "conversation.wav") as output: + for chunk in conversations.download_audio(agent_name, conversation_id): + output.write(chunk) + + if delete: + conversations.delete(agent_name, conversation_id) + try: + conversations.get(agent_name, conversation_id) + except ResourceNotFoundError: + print(f"Deleted conversation {conversation_id}; confirmed HTTP 404.") + else: + raise RuntimeError("The conversation remains readable after deletion.") + else: + print("Read-only run: the conversation, items, and audio were not deleted.") + + +def main(*, output_directory: Optional[Path] = None, delete: bool = False) -> None: + load_dotenv() + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + agent_name = os.environ["FOUNDRY_VOICE_AGENT_NAME"] + conversation_id = os.environ["FOUNDRY_VOICE_CONVERSATION_ID"] + if not agent_name.strip() or not conversation_id.strip(): + raise ValueError("Provide a nonblank existing agent name and stored Foundry conversation ID.") + + with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, + ): + try: + _read_conversation( + project_client.beta.voice_agents.live_conversations, + agent_name, + conversation_id, + output_directory=output_directory, + delete=delete, + ) + except HttpResponseError as exc: + print(f"GPT Live conversation REST request failed: HTTP {exc.status_code}.") + if exc.status_code == 404: + print( + "Confirm the stored conversation ID and store=True, and that this deployment supports " + "/protocols/voice/live/conversations. No older-route fallback is used." + ) + elif exc.status_code == 409: + print("Recording finalization may still be pending or unavailable; inspect the service error code.") + raise + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Read a stored GPT Live conversation through the dedicated REST API.") + parser.add_argument( + "--output-dir", type=Path, help="Explicitly save audio to this directory; files are not overwritten." + ) + parser.add_argument("--delete", action="store_true", help="Delete the selected conversation, its items, and audio.") + options = parser.parse_args() + main(output_directory=options.output_dir, delete=options.delete) diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation_async.py new file mode 100644 index 000000000000..f4bc9e1c14dc --- /dev/null +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation_async.py @@ -0,0 +1,193 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + Read a stored GPT Live conversation using the asynchronous preview + project_client.beta.voice_agents.live_conversations REST APIs. Retrieve the + conversation, recent conversations, ordered items, individual items, and + merged/per-item audio metadata. No agent or WebSocket session is created. + + Requires an ended speech session with effective store=True. Use + session.started.conversation_id, not session.id. These dedicated Live REST + routes must be available on the service deployment; HTTP 404 is reported + and raised, never replaced by calls to the older Realtime conversation API. + + By default, no audio files are written and nothing is deleted. Pass + --output-dir to download Foundry-managed WAV recordings without overwriting + existing files. BYOS recordings must be downloaded with your own storage + credentials. Pass --delete only for a conversation you intend to remove: + deletion also removes its stored items and audio. A read/download failure + prevents deletion. + +USAGE: + python sample_voice_agent_live_read_conversation_async.py + python sample_voice_agent_live_read_conversation_async.py --output-dir live-recordings + python sample_voice_agent_live_read_conversation_async.py --delete + + pip install azure-ai-projects azure-identity python-dotenv aiohttp + + Environment variables (all required): + 1) FOUNDRY_PROJECT_ENDPOINT - The Foundry project endpoint. + 2) FOUNDRY_VOICE_AGENT_NAME - The existing GPT Live agent name. + 3) FOUNDRY_VOICE_CONVERSATION_ID - The stored Foundry conversation ID. + + Authenticates with DefaultAzureCredential. No model deployment name, + microphone, or WebSocket dependency is needed. +""" + +from __future__ import annotations + +import argparse +import asyncio +import os +from pathlib import Path +from typing import Optional + +from dotenv import load_dotenv +from live_sample_utils import recording_output, safe_print + +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError +from azure.identity.aio import DefaultAzureCredential +from azure.ai.projects import models +from azure.ai.projects.aio import AIProjectClient +from azure.ai.projects.aio.operations import BetaVoiceAgentsLiveConversationsOperations + + +async def _read_item( + conversations: BetaVoiceAgentsLiveConversationsOperations, + agent_name: str, + conversation_id: str, + item_id: str, + index: int, + output_directory: Optional[Path], +) -> None: + detail = await conversations.get_item(agent_name, conversation_id, item_id) + print(f"Item {detail.id}: type={detail.type}") + if not isinstance( + detail, (models.VoiceAgentLiveInputConversationItem, models.VoiceAgentLiveOutputConversationItem) + ): + print(" Non-speech item; there is no speech audio to download.") + return + safe_print(f" Transcript: {detail.transcription or '(not captured)'}") + if detail.audio is None: + print(" No captured audio metadata for this speech item.") + return + audio = await conversations.get_audio_item(agent_name, conversation_id, item_id) + print(f" Audio: role={audio.role}, sample_rate={audio.sample_rate}, duration={audio.duration_ms}") + if audio.blob_uri: + print(" BYOS item: download from your storage using your own credentials.") + if output_directory is not None: + raise RuntimeError("This sample downloads Foundry-managed audio only, not BYOS audio.") + elif output_directory is not None: + with recording_output(output_directory / f"item-{index:04d}.wav") as output: + stream = await conversations.download_audio_item(agent_name, conversation_id, item_id) + async for chunk in stream: + output.write(chunk) + + +async def _read_conversation( + conversations: BetaVoiceAgentsLiveConversationsOperations, + agent_name: str, + conversation_id: str, + *, + output_directory: Optional[Path], + delete: bool, +) -> None: + conversation = await conversations.get(agent_name, conversation_id) + if conversation.status != "completed": + raise RuntimeError( + f"The stored session must finish successfully before reading its recording; " + f"status={conversation.status}, error={conversation.last_error}" + ) + print(f"Conversation {conversation.id}: status={conversation.status}, created_at={conversation.created_at}") + + print("Recent stored Live conversations (up to five):") + count = 0 + async for recent in conversations.list(agent_name, limit=5, order="desc"): + print(f" {recent.id}: status={recent.status}") + count += 1 + if count == 5: + break + + if output_directory is not None: + output_directory.mkdir(parents=True, exist_ok=True) + + seen_items: set[str] = set() + index = 0 + async for item in conversations.list_items(agent_name, conversation_id, limit=20, order="asc"): + index += 1 + if item.id in seen_items: + raise RuntimeError("The conversation pager repeated an item instead of advancing.") + seen_items.add(item.id) + await _read_item(conversations, agent_name, conversation_id, item.id, index, output_directory) + + recording = await conversations.get_audio(agent_name, conversation_id) + print( + f"Merged recording: format={recording.format}, sample_rate={recording.sample_rate}, " + f"channels={recording.channels}, duration={recording.duration_ms}" + ) + if recording.blob_uri: + print("BYOS recording: download from your storage using your own credentials.") + if output_directory is not None: + raise RuntimeError("This sample downloads Foundry-managed audio only, not BYOS audio.") + elif output_directory is not None: + with recording_output(output_directory / "conversation.wav") as output: + stream = await conversations.download_audio(agent_name, conversation_id) + async for chunk in stream: + output.write(chunk) + + if delete: + await conversations.delete(agent_name, conversation_id) + try: + await conversations.get(agent_name, conversation_id) + except ResourceNotFoundError: + print(f"Deleted conversation {conversation_id}; confirmed HTTP 404.") + else: + raise RuntimeError("The conversation remains readable after deletion.") + else: + print("Read-only run: the conversation, items, and audio were not deleted.") + + +async def main(*, output_directory: Optional[Path] = None, delete: bool = False) -> None: + load_dotenv() + endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] + agent_name = os.environ["FOUNDRY_VOICE_AGENT_NAME"] + conversation_id = os.environ["FOUNDRY_VOICE_CONVERSATION_ID"] + if not agent_name.strip() or not conversation_id.strip(): + raise ValueError("Provide a nonblank existing agent name and stored Foundry conversation ID.") + + async with ( + DefaultAzureCredential() as credential, + AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, + ): + try: + await _read_conversation( + project_client.beta.voice_agents.live_conversations, + agent_name, + conversation_id, + output_directory=output_directory, + delete=delete, + ) + except HttpResponseError as exc: + print(f"GPT Live conversation REST request failed: HTTP {exc.status_code}.") + if exc.status_code == 404: + print( + "Confirm the stored conversation ID and store=True, and that this deployment supports " + "/protocols/voice/live/conversations. No older-route fallback is used." + ) + elif exc.status_code == 409: + print("Recording finalization may still be pending or unavailable; inspect the service error code.") + raise + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Read a stored GPT Live conversation through the dedicated REST API.") + parser.add_argument( + "--output-dir", type=Path, help="Explicitly save audio to this directory; files are not overwritten." + ) + parser.add_argument("--delete", action="store_true", help="Delete the selected conversation, its items, and audio.") + options = parser.parse_args() + asyncio.run(main(output_directory=options.output_dir, delete=options.delete)) diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py index 6b29b8b24e45..42a078864bcd 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py @@ -16,6 +16,7 @@ import wave import pytest +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError from azure.ai.projects import models @@ -811,3 +812,286 @@ async def test_async_close_expired_deadline_does_not_create_an_unawaited_send(mo await helper.close_session_async(sample_environment.async_connection) sample_environment.async_connection.session.close.assert_not_called() + + +def _rest_values(values, asynchronous): + if asynchronous: + + async def iterate(): + for value in values: + yield value + + return iterate() + return iter(values) + + +def _wav_bytes(channels): + stream = io.BytesIO() + with wave.open(stream, "wb") as writer: + writer.setparams((channels, 2, 24000, 0, "NONE", "not compressed")) + writer.writeframes(b"\x01\x02" * 1200 * channels) + return stream.getvalue() + + +@pytest.fixture(params=[False, True], ids=["sync", "async"]) +def rest_sample(request, monkeypatch, sample_environment): + asynchronous = request.param + filename = "sample_voice_agent_live_read_conversation" + ("_async.py" if asynchronous else ".py") + sample = _load_module(monkeypatch, filename) + monkeypatch.setenv("FOUNDRY_VOICE_AGENT_NAME", "stored-agent-test") + monkeypatch.setenv("FOUNDRY_VOICE_CONVERSATION_ID", "conversation-test") + client = sample_environment.async_client if asynchronous else sample_environment.client + operations = client.beta.voice_agents.live_conversations + method_mock = AsyncMock if asynchronous else MagicMock + conversation = models.VoiceAgentLiveConversation( + {"id": "conversation-test", "status": "completed", "created_at": 1, "completed_at": 2} + ) + audio = models.VoiceAudioItem( + { + "conversation_id": "conversation-test", + "item_id": "speech-test", + "role": "agent", + "format": "wav", + "codec": "pcm16", + "sample_rate": 24000, + "channels": 1, + "duration_ms": 50, + } + ) + speech = models.VoiceAgentLiveOutputConversationItem( + {"id": "speech-test", "transcription": "A synthetic transcript.", "audio": audio.as_dict()} + ) + delegation = models.VoiceAgentLiveClientDelegationConversationItem( + {"id": "delegation-test", "created_at": 1, "commentary": "Synthetic application guidance."} + ) + items = [speech, delegation] + recording = models.VoiceRecording( + { + "conversation_id": "conversation-test", + "format": "wav", + "sample_rate": 24000, + "channels": 2, + "channel_layout": {"left": "user", "right": "agent"}, + "duration_ms": 50, + } + ) + item_bytes, merged_bytes = _wav_bytes(1), _wav_bytes(2) + operations.get = method_mock(return_value=conversation) + operations.list = MagicMock(side_effect=lambda *args, **kwargs: _rest_values([conversation], asynchronous)) + operations.list_items = MagicMock(side_effect=lambda *args, **kwargs: _rest_values(items, asynchronous)) + operations.get_item = method_mock( + side_effect=lambda agent, conversation, item: next(v for v in items if v.id == item) + ) + operations.get_audio_item = method_mock(return_value=audio) + operations.get_audio = method_mock(return_value=recording) + operations.download_audio_item = method_mock( + side_effect=lambda *args: _rest_values([item_bytes[:12], item_bytes[12:]], asynchronous) + ) + operations.download_audio = method_mock( + side_effect=lambda *args: _rest_values([merged_bytes[:12], merged_bytes[12:]], asynchronous) + ) + operations.delete = method_mock() + + async def run(**kwargs): + if asynchronous: + await sample.main(**kwargs) + else: + sample.main(**kwargs) + + return SimpleNamespace( + sample=sample, + client=client, + operations=operations, + conversation=conversation, + audio=audio, + recording=recording, + items=items, + run=run, + item_bytes=item_bytes, + merged_bytes=merged_bytes, + asynchronous=asynchronous, + ) + + +@pytest.mark.asyncio +async def test_live_rest_samples_are_read_only_by_default(rest_sample, tmp_path, capsys, monkeypatch): + for variable in ("FOUNDRY_VOICE_AGENT_MODEL", "FOUNDRY_MODEL_NAME", "FOUNDRY_VOICE_AUDIO_INPUT_PATH"): + monkeypatch.delenv(variable, raising=False) + await rest_sample.run() + + operations = rest_sample.operations + operations.get.assert_called_once_with("stored-agent-test", "conversation-test") + operations.list.assert_called_once_with("stored-agent-test", limit=5, order="desc") + operations.list_items.assert_called_once_with("stored-agent-test", "conversation-test", limit=20, order="asc") + assert operations.get_item.call_count == 2 + operations.get_audio_item.assert_called_once_with("stored-agent-test", "conversation-test", "speech-test") + operations.get_audio.assert_called_once_with("stored-agent-test", "conversation-test") + operations.download_audio.assert_not_called() + operations.download_audio_item.assert_not_called() + operations.delete.assert_not_called() + rest_sample.client.agents.create_version.assert_not_called() + rest_sample.client.agents.delete_version.assert_not_called() + rest_sample.client.beta.voice_agents.live.connect.assert_not_called() + rest_sample.client.beta.voice_agents.conversations.get.assert_not_called() + assert not list(tmp_path.iterdir()) + output = capsys.readouterr().out + assert "A synthetic transcript." in output + assert "Non-speech item" in output + assert "Read-only run" in output + + +@pytest.mark.asyncio +async def test_live_rest_samples_download_both_recordings_and_delete_only_when_requested(rest_sample, tmp_path, capsys): + rest_sample.operations.get.side_effect = [rest_sample.conversation, ResourceNotFoundError("deleted")] + + await rest_sample.run(output_directory=tmp_path, delete=True) + + assert (tmp_path / "item-0001.wav").read_bytes() == rest_sample.item_bytes + assert (tmp_path / "conversation.wav").read_bytes() == rest_sample.merged_bytes + rest_sample.operations.download_audio_item.assert_called_once_with( + "stored-agent-test", "conversation-test", "speech-test" + ) + rest_sample.operations.download_audio.assert_called_once_with("stored-agent-test", "conversation-test") + rest_sample.operations.delete.assert_called_once_with("stored-agent-test", "conversation-test") + assert rest_sample.operations.get.call_count == 2 + if rest_sample.asynchronous: + for name in ("get_item", "get_audio_item", "get_audio", "download_audio_item", "download_audio", "delete"): + method = getattr(rest_sample.operations, name) + assert method.await_count == method.call_count + assert "confirmed HTTP 404" in capsys.readouterr().out + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", [404, 409, 500]) +async def test_live_rest_samples_surface_service_failures_without_fallback_or_deletion(rest_sample, status, capsys): + failure = HttpResponseError("service failure") + failure.status_code = status + rest_sample.operations.get.side_effect = failure + + with pytest.raises(HttpResponseError) as caught: + await rest_sample.run(delete=True) + + assert caught.value is failure + rest_sample.operations.delete.assert_not_called() + assert not rest_sample.client.beta.voice_agents.conversations.mock_calls + assert f"HTTP {status}" in capsys.readouterr().out + if rest_sample.asynchronous: + rest_sample.client.__aexit__.assert_awaited_once() + else: + rest_sample.client.__exit__.assert_called_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", ["in_progress", "failed"]) +async def test_live_rest_samples_require_successful_session_finalization(rest_sample, status): + rest_sample.conversation.status = status + + with pytest.raises(RuntimeError, match="must finish successfully"): + await rest_sample.run(delete=True) + + rest_sample.operations.list_items.assert_not_called() + rest_sample.operations.download_audio.assert_not_called() + rest_sample.operations.delete.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method,filename", + [ + ("download_audio_item", "item-0001.wav"), + ("download_audio", "conversation.wav"), + ], +) +@pytest.mark.parametrize("empty", [True, False], ids=["empty-stream", "failed-stream"]) +async def test_live_rest_download_failures_remove_incomplete_files_and_do_not_delete( + rest_sample, tmp_path, method, filename, empty +): + async def async_chunks(): + if not empty: + yield b"partial" + raise ConnectionError("stream failed") + + def sync_chunks(): + if not empty: + yield b"partial" + raise ConnectionError("stream failed") + + getattr(rest_sample.operations, method).side_effect = lambda *args: ( + async_chunks() if rest_sample.asynchronous else sync_chunks() + ) + + with pytest.raises(RuntimeError if empty else ConnectionError, match="no bytes|stream failed"): + await rest_sample.run(output_directory=tmp_path, delete=True) + + assert not (tmp_path / filename).exists() + rest_sample.operations.delete.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "filename,method", + [ + ("item-0001.wav", "download_audio_item"), + ("conversation.wav", "download_audio"), + ], +) +async def test_live_rest_samples_never_overwrite_an_existing_recording(rest_sample, tmp_path, filename, method): + path = tmp_path / filename + path.write_bytes(b"keep existing audio") + + with pytest.raises(FileExistsError): + await rest_sample.run(output_directory=tmp_path, delete=True) + + assert path.read_bytes() == b"keep existing audio" + getattr(rest_sample.operations, method).assert_not_called() + rest_sample.operations.delete.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("level", ["item", "merged"]) +@pytest.mark.parametrize("download", [False, True], ids=["metadata-only", "download-requested"]) +async def test_live_rest_samples_explain_byos_without_proxying_downloads( + rest_sample, tmp_path, capsys, level, download +): + metadata = rest_sample.audio if level == "item" else rest_sample.recording + metadata.blob_uri = "https://example.invalid/private-recording" + if download: + with pytest.raises(RuntimeError, match="not BYOS audio"): + await rest_sample.run(output_directory=tmp_path, delete=True) + rest_sample.operations.delete.assert_not_called() + else: + await rest_sample.run() + method = "download_audio_item" if level == "item" else "download_audio" + getattr(rest_sample.operations, method).assert_not_called() + output = capsys.readouterr().out + assert "BYOS" in output and "https://example.invalid/private-recording" not in output + + +@pytest.mark.asyncio +async def test_live_rest_samples_reject_repeated_paging_items(rest_sample): + rest_sample.items.append(rest_sample.items[0]) + + with pytest.raises(RuntimeError, match="pager repeated an item"): + await rest_sample.run(delete=True) + + rest_sample.operations.delete.assert_not_called() + + +@pytest.mark.asyncio +async def test_live_rest_samples_report_a_delete_that_did_not_take_effect(rest_sample): + with pytest.raises(RuntimeError, match="remains readable after deletion"): + await rest_sample.run(delete=True) + + rest_sample.operations.delete.assert_called_once_with("stored-agent-test", "conversation-test") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("variable", ["FOUNDRY_VOICE_AGENT_NAME", "FOUNDRY_VOICE_CONVERSATION_ID"]) +async def test_live_rest_samples_reject_blank_resource_identifiers(rest_sample, monkeypatch, variable): + monkeypatch.setenv(variable, " ") + + with pytest.raises(ValueError, match="nonblank"): + await rest_sample.run(delete=True) + + rest_sample.operations.get.assert_not_called() + rest_sample.operations.delete.assert_not_called() From 9e7255447ad378af144576532a3bec3ea78dcc2b Mon Sep 17 00:00:00 2001 From: Xiting Zhang Date: Wed, 21 Oct 2026 12:03:13 -0700 Subject: [PATCH 04/10] [azure-ai-projects] Address GPT Live review feedback Protect input WAV recordings from hard-linked output aliases, normalize mapping inputs in Live serialization, and await cancellation-safe shared asynchronous transport cleanup. Add deterministic regressions for all three findings while preserving existing close-error behavior. Verified 580 focused unit cases and type/lint/build checks. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/ai/azure-ai-projects/CHANGELOG.md | 3 + .../azure/ai/projects/_live.py | 2 +- .../azure/ai/projects/_live_utils.py | 2 +- .../azure/ai/projects/aio/_live.py | 2 +- .../azure/ai/projects/aio/_realtime.py | 32 ++++- ...ice_agent_live_audio_conversation_async.py | 9 +- .../tests/agents/test_live_client.py | 118 +++++++++++++++++- .../tests/samples/test_live_samples.py | 25 ++++ 8 files changed, 179 insertions(+), 14 deletions(-) diff --git a/sdk/ai/azure-ai-projects/CHANGELOG.md b/sdk/ai/azure-ai-projects/CHANGELOG.md index 6b77372dd536..a4621c9f36f6 100644 --- a/sdk/ai/azure-ai-projects/CHANGELOG.md +++ b/sdk/ai/azure-ai-projects/CHANGELOG.md @@ -12,6 +12,8 @@ ### Bugs Fixed * Preserved concrete Live/Realtime voice-agent types on agent readback without inserting an omitted `sub_protocol`. +* Accepted non-dictionary mappings in GPT Live raw events and structured connection inputs. +* Fixed concurrent asynchronous Voice Agent close calls to await shared cleanup, including when a caller is cancelled. ### Sample updates @@ -19,6 +21,7 @@ * Updated GPT Live text examples with synchronous/asynchronous headless multi-turn conversations that reuse one session and preserve the default single-turn example. * Added synchronous/asynchronous GPT Live REST examples for reading an existing stored conversation, its items, and audio metadata, with explicit opt-in recording downloads and conversation deletion. * Fixed the GPT Live WAV sample to ignore silent/very-low-level PCM when detecting reply inactivity, reject silence-only replies, and preserve captured PCM in audio counts and optional WAV output. +* Protected the GPT Live WAV sample's input recording from being overwritten through hard-linked output paths. ## 2.9.0 (2026-10-16) diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/_live.py b/sdk/ai/azure-ai-projects/azure/ai/projects/_live.py index 51bda73220db..b2be76a26795 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/_live.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/_live.py @@ -176,7 +176,7 @@ def send(self, event: LiveClientEvent) -> None: :type event: ~azure.ai.projects.operations.LiveClientEvent :raises ValueError: If the supplied JSON string is invalid. """ - super().send(event) + super().send(dict(event) if isinstance(event, Mapping) else event) class BetaLiveConnectionManager(_BaseWebSocketConnectionManager[BetaLiveConnection]): diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/_live_utils.py b/sdk/ai/azure-ai-projects/azure/ai/projects/_live_utils.py index c1028ccdb93e..d802bc67f6ee 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/_live_utils.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/_live_utils.py @@ -51,7 +51,7 @@ def _live_query( if structured_inputs is not None: if "structured_input" in result: raise ValueError("structured_input and structured_inputs cannot be supplied together.") - result["structured_inputs"] = json.dumps(structured_inputs, cls=SdkJSONEncoder) + result["structured_inputs"] = json.dumps(dict(structured_inputs), cls=SdkJSONEncoder) if "structured_inputs" in result and "structured_input" in result: raise ValueError("structured_input and structured_inputs cannot be supplied together.") if store is not None: diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_live.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_live.py index aa1f9a515592..100a10351bd7 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_live.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_live.py @@ -204,7 +204,7 @@ async def send(self, event: LiveClientEvent) -> None: :type event: ~azure.ai.projects.aio.operations.LiveClientEvent :raises ValueError: If the supplied JSON string is invalid. """ - await super().send(event) + await super().send(dict(event) if isinstance(event, Mapping) else event) class AsyncBetaLiveConnectionManager(_BaseWebSocketConnectionManager[AsyncBetaLiveConnection]): diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py index d5c7d1a667f4..30750f9728f6 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py @@ -31,6 +31,7 @@ from __future__ import annotations +import asyncio # pylint: disable=do-not-import-asyncio import base64 import json import logging @@ -595,7 +596,7 @@ class _BaseWebSocketConnection(Generic[_ServerEventT, _ClientEventT]): def __init__(self, connection: "ClientWebSocketResponse", session: "ClientSession") -> None: self._connection = connection self._session = session - self._closed = False + self._close_task: Optional[asyncio.Task[None]] = None async def __aenter__(self) -> "_BaseWebSocketConnection[_ServerEventT, _ClientEventT]": return self @@ -697,12 +698,30 @@ async def send(self, event: _ClientEventT) -> None: async def close(self, *, code: int = 1000, reason: str = "") -> None: """Close the socket and its HTTP session once. + Concurrent callers await the same cleanup. Caller cancellation is + propagated after both resources have finished closing. + :keyword int code: WebSocket close code. :keyword str reason: WebSocket close reason. """ - if self._closed: + if self._close_task is not None and self._close_task.done(): return - self._closed = True + if self._close_task is None: + self._close_task = asyncio.create_task(self._close_resources(code, reason)) + cancelled = False + while True: + try: + await asyncio.shield(self._close_task) + break + except asyncio.CancelledError: + if self._close_task.cancelled(): + await self._close_task + raise + cancelled = True + if cancelled: + raise asyncio.CancelledError + + async def _close_resources(self, code: int, reason: str) -> None: self._logger.debug("WebSocket CLOSE code=%d", code) try: await self._connection.close(code=code, message=reason.encode("utf-8")) @@ -943,9 +962,12 @@ def _create_connection(self, connection: "ClientWebSocketResponse", session: "Cl async def __aexit__(self, *exc_details: Any) -> None: connection = self._connection - self._connection = None if connection is not None: - await connection.close() + try: + await connection.close() + finally: + if self._connection is connection: + self._connection = None class AsyncBetaRealtimeConnectionManager(_BaseWebSocketConnectionManager[AsyncBetaRealtimeConnection]): diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py index 7cb71767c3b2..6e7243b969f9 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py @@ -31,7 +31,8 @@ WAV file: uncompressed PCM16, mono, 24000 Hz. No microphone is needed. 4) FOUNDRY_VOICE_AUDIO_OUTPUT_PATH (optional) - Explicit WAV output path. When set, saves actual returned audio, not telemetry. Without it no audio - file is written. The path must differ from the input path. + file is written. The output must not refer to the input file, including + through a symbolic link or hard link. 5) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for a uniquely named sample agent. Only the version created by this run is deleted. @@ -221,8 +222,10 @@ async def main() -> None: model = os.environ["FOUNDRY_VOICE_AGENT_MODEL"] input_path = Path(os.environ["FOUNDRY_VOICE_AUDIO_INPUT_PATH"]) output_path = os.environ.get("FOUNDRY_VOICE_AUDIO_OUTPUT_PATH") - if output_path and Path(output_path).resolve() == input_path.resolve(): - raise ValueError("The audio output path must not overwrite the input recording.") + if output_path: + output = Path(output_path) + if output.resolve() == input_path.resolve() or (output.exists() and output.samefile(input_path)): + raise ValueError("The audio output path must not overwrite the input recording.") agent_name = sample_agent_name("audio") with ExitStack() as files: diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_live_client.py b/sdk/ai/azure-ai-projects/tests/agents/test_live_client.py index afbc3e04c2bd..6db2a22c2c5e 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/test_live_client.py +++ b/sdk/ai/azure-ai-projects/tests/agents/test_live_client.py @@ -14,6 +14,7 @@ import json import logging import threading +from collections import UserDict from collections.abc import AsyncIterator, Iterator from concurrent.futures import ThreadPoolExecutor from contextlib import asynccontextmanager @@ -352,8 +353,9 @@ async def test_beta_namespace_implies_preview_opt_in_without_allow_preview(live, live.transport.assert_closed_once() -async def test_handshake_encodes_live_route_authentication_and_all_options(live): - structured_inputs = {"name": "Zoë / guest", "count": 0, "enabled": False, "optional": None} +@pytest.mark.parametrize("mapping_type", [dict, MappingProxyType, UserDict]) +async def test_handshake_encodes_live_route_authentication_and_all_options(live, mapping_type): + structured_inputs = mapping_type({"name": "Zoë / guest", "count": 0, "enabled": False, "optional": None}) extra_query = MappingProxyType({"custom": "space + /?&=#"}) extra_headers = MappingProxyType({"X-Custom": "example"}) options = {"heartbeat": 3, "max_msg_size": 1024} if live.is_async else {"open_timeout": 3, "max_size": 1024} @@ -619,13 +621,15 @@ async def test_live_does_not_negotiate_realtime_even_with_caller_protocol_overri assert "Sec-WebSocket-Protocol" not in live.headers -@pytest.mark.parametrize("kind", ["model", "mapping", "json"]) +@pytest.mark.parametrize("kind", ["model", "mapping", "mapping_proxy", "user_dict", "json"]) async def test_raw_send_supports_generated_model_mapping_and_exact_json(live, kind): expected = {"type": "response.create", "event_id": "evt-response"} raw = '{ "type": "response.create", "event_id": "evt-response" }' event = { "model": models.LiveClientEventLiveResponseCreateParam(event_id="evt-response"), "mapping": expected, + "mapping_proxy": MappingProxyType(expected), + "user_dict": UserDict(expected), "json": raw, }[kind] async with live.open() as connection: @@ -1055,6 +1059,114 @@ async def use_connection(): live.transport.assert_closed_once() +@pytest.mark.parametrize("is_async", [True], ids=["async"]) +@pytest.mark.parametrize("protocol", ["live", "realtime"]) +@pytest.mark.parametrize("blocked_resource", ["websocket", "http_session"]) +@pytest.mark.parametrize("cancel_caller", [None, "first_close", "context_exit"]) +async def test_async_close_callers_wait_for_shared_cancellation_safe_cleanup( + live, protocol, blocked_resource, cancel_caller +): + blocked = asyncio.Event() + release = asyncio.Event() + + async def close_socket(**kwargs): + if blocked_resource == "websocket": + blocked.set() + await release.wait() + live.transport.socket.closed = True + + async def close_session(): + if blocked_resource == "http_session": + blocked.set() + await release.wait() + live.transport.session.closed = True + + live.transport.socket.close.side_effect = close_socket + live.transport.session.close.side_effect = close_session + manager = live.manager(protocol=protocol) + connection = await manager.enter() + callers = {"first_close": asyncio.create_task(connection.close(code=1001, reason="first caller"))} + try: + await asyncio.wait_for(blocked.wait(), timeout=2) + callers["second_close"] = asyncio.create_task(connection.close(code=1000, reason="second caller")) + callers["context_exit"] = asyncio.create_task(live.exit(manager)) + callers["second_context_exit"] = asyncio.create_task(live.exit(manager)) + await asyncio.sleep(0) + assert all(not caller.done() for caller in callers.values()) + assert live.transport.session.closed is False + if cancel_caller is not None: + for _ in range(2): + callers[cancel_caller].cancel() + await asyncio.sleep(0) + assert all(not caller.done() for caller in callers.values()) + release.set() + results = await asyncio.wait_for(asyncio.gather(*callers.values(), return_exceptions=True), timeout=2) + for name, result in zip(callers, results): + if name == cancel_caller: + assert isinstance(result, asyncio.CancelledError) + else: + assert result is None + finally: + release.set() + await asyncio.wait_for(asyncio.gather(*callers.values(), return_exceptions=True), timeout=2) + await live.exit(manager) + live.transport.assert_closed_once() + live.transport.socket.close.assert_awaited_once_with(code=1001, message=b"first caller") + assert manager._connection is None + + +@pytest.mark.parametrize("is_async", [True], ids=["async"]) +@pytest.mark.parametrize("protocol", ["live", "realtime"]) +@pytest.mark.parametrize("failure_source", ["websocket", "http_session"]) +async def test_async_close_failure_propagates_to_every_waiter_without_repeating_cleanup(live, protocol, failure_source): + blocked = asyncio.Event() + release = asyncio.Event() + failure = OSError("offline cleanup failure") + + async def close_socket(**kwargs): + blocked.set() + await release.wait() + if failure_source == "websocket": + raise failure + live.transport.socket.closed = True + + async def close_session(): + live.transport.session.closed = True + if failure_source == "http_session": + raise failure + + live.transport.socket.close.side_effect = close_socket + live.transport.session.close.side_effect = close_session + manager = live.manager(protocol=protocol) + connection = await manager.enter() + first = asyncio.create_task(connection.close()) + second = None + context_exit = None + try: + await asyncio.wait_for(blocked.wait(), timeout=2) + second = asyncio.create_task(connection.close()) + context_exit = asyncio.create_task(live.exit(manager)) + await asyncio.sleep(0) + assert not second.done() + assert not context_exit.done() + release.set() + results = await asyncio.wait_for(asyncio.gather(first, second, context_exit, return_exceptions=True), timeout=2) + assert all(result is failure for result in results) + await connection.close() + await live.exit(manager) + finally: + release.set() + await asyncio.wait_for( + asyncio.gather( + *(task for task in (first, second, context_exit) if task is not None), return_exceptions=True + ), + timeout=2, + ) + live.transport.socket.close.assert_awaited_once() + live.transport.session.close.assert_awaited_once() + assert live.transport.session.closed is True + + @pytest.mark.parametrize("is_async", [False], ids=["sync"]) async def test_sync_close_error_propagates_and_does_not_repeat_transport_cleanup(live): failure = OSError("fake socket close failed") diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py index 42a078864bcd..c934d2cc31c3 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py @@ -475,6 +475,31 @@ async def test_audio_refuses_to_overwrite_the_input_recording(monkeypatch, sampl sample_environment.async_client.agents.create_version.assert_not_awaited() +@pytest.mark.asyncio +async def test_audio_refuses_a_hard_link_to_the_input_without_modifying_either_path( + monkeypatch, sample_environment, tmp_path +): + sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + input_path = tmp_path / "input.wav" + output_path = tmp_path / "hard-linked-output.wav" + original = _wav_bytes(1) + input_path.write_bytes(original) + output_path.hardlink_to(input_path) + assert input_path.resolve() != output_path.resolve() + assert output_path.samefile(input_path) + monkeypatch.setenv("FOUNDRY_VOICE_AUDIO_INPUT_PATH", str(input_path)) + monkeypatch.setenv("FOUNDRY_VOICE_AUDIO_OUTPUT_PATH", str(output_path)) + monkeypatch.setattr(sample, "_run_audio_conversation", AsyncMock()) + + with pytest.raises(ValueError, match="must not overwrite"): + await sample.main() + + assert input_path.read_bytes() == original + assert output_path.read_bytes() == original + sample_environment.async_client.agents.create_version.assert_not_awaited() + sample_environment.async_client.beta.voice_agents.live.connect.assert_not_called() + + @pytest.mark.asyncio @pytest.mark.parametrize("failure", [RuntimeError("offline receive failed"), asyncio.CancelledError()]) async def test_async_sample_deletes_its_version_on_error_or_cancellation(monkeypatch, sample_environment, failure): From d7bf0e2bce4f8705d68274a8fdd4b9aaa02a6627 Mon Sep 17 00:00:00 2001 From: Xiting Zhang Date: Wed, 21 Oct 2026 16:52:45 -0700 Subject: [PATCH 05/10] [azure-ai-projects] Add microphone mode to GPT Live audio sample Extend the existing audio sample with optional PyAudio microphone capture and speaker playback, preserving WAV mode. Bound device buffering, surface callback/send errors, report actual audio bytes, and gracefully close the session on Ctrl+C. Add 24 microphone guards and related documentation. Verified 162 focused tests, both CLI modes against the live service, and user-confirmed physical microphone operation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/ai/azure-ai-projects/CHANGELOG.md | 1 + sdk/ai/azure-ai-projects/README.md | 18 + .../samples/agents/voice/live_sample_utils.py | 4 +- ...ice_agent_live_audio_conversation_async.py | 296 +++++++++++++-- .../tests/samples/test_live_samples.py | 350 ++++++++++++++++++ 5 files changed, 643 insertions(+), 26 deletions(-) diff --git a/sdk/ai/azure-ai-projects/CHANGELOG.md b/sdk/ai/azure-ai-projects/CHANGELOG.md index a4621c9f36f6..85bcc18b12ef 100644 --- a/sdk/ai/azure-ai-projects/CHANGELOG.md +++ b/sdk/ai/azure-ai-projects/CHANGELOG.md @@ -22,6 +22,7 @@ * Added synchronous/asynchronous GPT Live REST examples for reading an existing stored conversation, its items, and audio metadata, with explicit opt-in recording downloads and conversation deletion. * Fixed the GPT Live WAV sample to ignore silent/very-low-level PCM when detecting reply inactivity, reject silence-only replies, and preserve captured PCM in audio counts and optional WAV output. * Protected the GPT Live WAV sample's input recording from being overwritten through hard-linked output paths. +* Added `--microphone` to the existing GPT Live audio sample for nonblocking microphone capture and speaker playback with optional PyAudio, preserving the WAV workflow and reporting actual audio transfer sizes. ## 2.9.0 (2026-10-16) diff --git a/sdk/ai/azure-ai-projects/README.md b/sdk/ai/azure-ai-projects/README.md index c418ac4fec73..2ab6ddcab2d3 100644 --- a/sdk/ai/azure-ai-projects/README.md +++ b/sdk/ai/azure-ai-projects/README.md @@ -260,6 +260,24 @@ The WAV audio example uses a bounded speech-inactivity heuristic because Live ha event. Continued silent PCM does not keep it waiting forever, and silence-only output is not a successful spoken reply. A long speech pause can still split a reply. +The same [GPT Live audio sample](samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py) +also supports live microphone capture and speaker playback, following the Realtime sample's +nonblocking PyAudio device pattern: + +```powershell +pip install pyaudio +python .\samples\agents\voice\sample_voice_agent_live_audio_conversation_async.py --microphone +``` + +Microphone mode requires `FOUNDRY_PROJECT_ENDPOINT` and `FOUNDRY_VOICE_AGENT_MODEL`, but not +`FOUNDRY_VOICE_AUDIO_INPUT_PATH`. Speak normally after the sample prints "Speak now"; use +headphones to avoid feedback, and press Ctrl-C to close the session and delete its temporary +agent version. The sample reports actual input/output PCM bytes, durations, and any dropped +capture frames. No audio file is saved unless `FOUNDRY_VOICE_AUDIO_OUTPUT_PATH` is explicitly +set; that file contains the received reply, not the microphone input. Conversation persistence +remains disabled. Live owns turn handling, so this mode does not use Realtime `response.cancel`. +Omit `--microphone` to retain the original WAV workflow without requiring PyAudio. + The [synchronous REST sample](samples/agents/voice/sample_voice_agent_live_read_conversation.py) and [asynchronous REST sample](samples/agents/voice/sample_voice_agent_live_read_conversation_async.py) read an existing stored Live conversation without opening a WebSocket. Set `FOUNDRY_PROJECT_ENDPOINT`, diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py b/sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py index 9dfc4c8005bc..de8441ff63e4 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py @@ -52,10 +52,10 @@ def audio_settings() -> models.LiveInitialSessionAudioParam: ) -def safe_print(text: str) -> None: +def safe_print(text: str, *, end: str = "\n") -> None: """Display model text even on a console with a legacy encoding.""" encoding = sys.stdout.encoding or "utf-8" - print(text.encode(encoding, errors="replace").decode(encoding)) + print(text.encode(encoding, errors="replace").decode(encoding), end=end, flush=True) @contextmanager diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py index 6e7243b969f9..8ad4e97da032 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py @@ -5,14 +5,26 @@ """ DESCRIPTION: - Stream a real WAV recording to a preview GPT Live voice agent while receiving - its speech concurrently. The input must contain spoken audio encoded as - PCM16 little-endian, mono, 24 kHz. Only raw frames are sent, without the WAV - header, in paced 50 ms chunks followed by one second of silence. + Talk to a preview GPT Live voice agent using --microphone, or stream a real + WAV recording without that flag. Both modes use PCM16 little-endian, mono, + 24 kHz audio and send/receive concurrently. + + Microphone mode uses optional PyAudio capture and speaker callbacks, + following the Realtime audio sample's nonblocking device pattern. Speak + normally and hear the reply through your speakers; use headphones to avoid + feedback. Capture backpressure drops a frame instead of queuing unlimited + sends, with the dropped-frame count reported at shutdown. Playback buffering + is bounded and device/send errors are raised. Press Ctrl-C to stop capture, + close the Live session and delete only the sample's agent version. Live owns + turn handling; this sample does not send Realtime response.cancel commands. + + WAV mode needs spoken audio in uncompressed PCM16, mono, 24 kHz format. + Only raw frames are sent, without the WAV header, in paced 50 ms chunks + followed by one second of silence. No audio-device dependency is needed. Aggregate audio byte counts and any emitted input/output transcripts are - displayed. Live has no speech-finished event: after sending finishes, this - bounded demonstration stops after three seconds without non-silent reply + displayed. Live has no speech-finished event: after WAV sending finishes, this + bounded mode stops after three seconds without non-silent reply audio or nonblank output transcript updates. This is an inactivity heuristic; a long speech pause can still split a reply. Low-level PCM with peak amplitude at or below 256 does not restart the timer or count as a reply. @@ -21,14 +33,16 @@ USAGE: python sample_voice_agent_live_audio_conversation_async.py + python sample_voice_agent_live_audio_conversation_async.py --microphone pip install "azure-ai-projects[voice]" azure-identity python-dotenv + pip install pyaudio # Only required for --microphone. Environment variables: 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live DEPLOYMENT name. - 3) FOUNDRY_VOICE_AUDIO_INPUT_PATH (required) - Path to an actual spoken-audio - WAV file: uncompressed PCM16, mono, 24000 Hz. No microphone is needed. + 3) FOUNDRY_VOICE_AUDIO_INPUT_PATH (required for WAV mode only) - Path to an + actual spoken-audio WAV file: uncompressed PCM16, mono, 24000 Hz. 4) FOUNDRY_VOICE_AUDIO_OUTPUT_PATH (optional) - Explicit WAV output path. When set, saves actual returned audio, not telemetry. Without it no audio file is written. The output must not refer to the input file, including @@ -36,7 +50,7 @@ 5) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for a uniquely named sample agent. Only the version created by this run is deleted. - Authenticates with DefaultAzureCredential. No audio-device dependency. + Authenticates with DefaultAzureCredential. Choose a short recording; Ctrl-C cancels local tasks and closes the session. Conversation persistence is disabled; explicitly saving audio still writes conversation content to your chosen local file. @@ -44,14 +58,17 @@ from __future__ import annotations +import argparse import asyncio import base64 +import concurrent.futures from contextlib import ExitStack from dataclasses import dataclass import os from pathlib import Path +import queue import time -from typing import Optional, TYPE_CHECKING +from typing import Any, Optional, TYPE_CHECKING import wave from dotenv import load_dotenv @@ -89,6 +106,206 @@ class _AudioOutput: has_audio_activity: bool = False +def _microphone_backend() -> Any: + try: + import pyaudio + except ImportError as exc: + raise RuntimeError("Microphone mode requires PyAudio. Install it with: pip install pyaudio") from exc + return pyaudio + + +class _MicrophoneAudio: + """Bounded, nonblocking microphone capture and PCM speaker playback.""" + + def __init__(self, connection: AsyncBetaLiveConnection, backend: Any) -> None: + self._connection = connection + self._backend = backend + self._audio = backend.PyAudio() + self._loop: Optional[asyncio.AbstractEventLoop] = None + self._input_stream: Any = None + self._output_stream: Any = None + self._pending_send: Optional[concurrent.futures.Future[None]] = None + self._capture_error: Optional[BaseException] = None + self._capture_active = False + self._playback_queue: queue.Queue[bytes] = queue.Queue(maxsize=200) + self._playback_remaining = b"" + self.input_bytes = 0 + self.output_bytes = 0 + self.dropped_frames = 0 + + async def _append_input_audio(self, pcm: bytes) -> None: + await self._connection.session.input_audio.append(audio=pcm) + self.input_bytes += len(pcm) + + def _capture_callback(self, pcm: bytes, _frame_count: int, _time_info: Any, status: int) -> tuple[None, int]: + if not self._capture_active: + return None, self._backend.paAbort + if status: + self._capture_error = RuntimeError(f"Microphone callback reported audio status {status}.") + return None, self._backend.paAbort + if self._pending_send is not None: + if not self._pending_send.done(): + self.dropped_frames += 1 + return None, self._backend.paContinue + if self._pending_send.cancelled(): + self._capture_error = RuntimeError("The microphone audio send was cancelled.") + return None, self._backend.paAbort + failure = self._pending_send.exception() + if failure is not None: + self._capture_error = failure + return None, self._backend.paAbort + assert self._loop is not None + self._pending_send = asyncio.run_coroutine_threadsafe(self._append_input_audio(pcm), self._loop) + return None, self._backend.paContinue + + def _playback_callback(self, _pcm: Any, frame_count: int, _time_info: Any, status: int) -> tuple[bytes, int]: + wanted = frame_count * 2 + if status: + self._capture_error = RuntimeError(f"Speaker callback reported audio status {status}.") + return bytes(wanted), self._backend.paAbort + output = self._playback_remaining[:wanted] + self._playback_remaining = self._playback_remaining[wanted:] + while len(output) < wanted: + try: + chunk = self._playback_queue.get_nowait() + except queue.Empty: + output += bytes(wanted - len(output)) + break + take = wanted - len(output) + output += chunk[:take] + self._playback_remaining = chunk[take:] + return output, self._backend.paContinue + + def start(self) -> None: + audio = self._audio + if audio is None: + raise RuntimeError("Cannot restart closed microphone audio devices.") + self._loop = asyncio.get_running_loop() + self._output_stream = audio.open( + format=self._backend.paInt16, + channels=1, + rate=_SAMPLE_RATE, + output=True, + frames_per_buffer=_CHUNK_FRAMES, + stream_callback=self._playback_callback, + start=False, + ) + self._input_stream = audio.open( + format=self._backend.paInt16, + channels=1, + rate=_SAMPLE_RATE, + input=True, + frames_per_buffer=_CHUNK_FRAMES, + stream_callback=self._capture_callback, + start=False, + ) + self._capture_active = True + self._output_stream.start_stream() + self._input_stream.start_stream() + + def check_capture_error(self) -> None: + if self._capture_error is not None: + raise self._capture_error + if self._pending_send is not None and self._pending_send.done(): + self._pending_send.result() + + def enqueue_audio(self, pcm: bytes) -> None: + if len(pcm) % 2: + raise ValueError("The service returned an incomplete PCM16 sample.") + self.output_bytes += len(pcm) + try: + self._playback_queue.put_nowait(pcm) + except queue.Full as exc: + raise RuntimeError("Speaker playback fell behind; the audio buffer is full.") from exc + + async def stop(self) -> None: + self._capture_active = False + try: + try: + if self._input_stream is not None: + stream, self._input_stream = self._input_stream, None + try: + stream.stop_stream() + finally: + stream.close() + finally: + if self._pending_send is not None: + pending, self._pending_send = self._pending_send, None + if pending.cancelled(): + raise RuntimeError("The microphone audio send was cancelled.") + await asyncio.wrap_future(pending) + finally: + try: + if self._output_stream is not None: + stream, self._output_stream = self._output_stream, None + try: + stream.stop_stream() + finally: + stream.close() + finally: + if self._audio is not None: + audio, self._audio = self._audio, None + audio.terminate() + + +async def _stream_microphone( + connection: AsyncBetaLiveConnection, backend: Any, output_file: Optional[wave.Wave_write] +) -> bool: + audio = _MicrophoneAudio(connection, backend) + last_role = "" + try: + audio.start() + print("Speak now -- GPT Live replies through your speakers. Use headphones to avoid feedback.") + print("Press Ctrl-C to stop the microphone, close the session, and clean up.") + while True: + audio.check_capture_error() + try: + event = await asyncio.wait_for(connection.recv(), timeout=0.5) + except asyncio.TimeoutError: + continue + check_event(event) + if event.get("type") == "session.closed": + reason = event.get("reason") + if reason not in ("close_requested", "expired", "remote_hangup"): + raise RuntimeError(f"Live session ended unexpectedly: {reason}") + print(f"\nSession closed: {reason}") + return True + if isinstance(event, models.LiveOutputAudioDelta): + pcm = base64.b64decode(event.delta, validate=True) + audio.enqueue_audio(pcm) + if output_file is not None: + output_file.writeframesraw(pcm) + elif isinstance( + event, (models.LiveServerEventLiveInputTranscriptDelta, models.LiveServerEventLiveOutputTranscriptDelta) + ): + role = "You" if isinstance(event, models.LiveServerEventLiveInputTranscriptDelta) else "Agent" + if role != last_role: + safe_print(f"\n{role}: ", end="") + last_role = role + safe_print(event.delta, end="") + elif isinstance(event, models.LiveServerEventLiveDelegationCreated): + if event.delegation.target == "client": + await connection.session.commentary.append( + content="This sample cannot look up external information. Briefly acknowledge the spoken request.", + delegation_id=event.delegation.id, + ) + finally: + try: + await audio.stop() + finally: + print( + f"\nInput audio: {audio.input_bytes} bytes ({audio.input_bytes / 1048576:.2f} MiB), " + f"PCM16, 24000 Hz, mono, {audio.input_bytes / 48000:.2f}s." + ) + print( + f"Output audio: {audio.output_bytes} bytes ({audio.output_bytes / 1048576:.2f} MiB), " + f"PCM16, 24000 Hz, mono, {audio.output_bytes / 48000:.2f}s." + ) + print(f"Total audio transferred: {audio.input_bytes + audio.output_bytes} bytes.") + if audio.dropped_frames: + print(f"Dropped {audio.dropped_frames} microphone frame(s) while a previous send was in flight.") + + def _has_audio_activity(pcm: bytes) -> bool: return any( abs(int.from_bytes(pcm[offset : offset + 2], "little", signed=True)) > _SILENCE_PEAK_THRESHOLD @@ -202,35 +419,53 @@ async def _run_audio_conversation( project_client: AIProjectClient, agent_name: str, agent_version: str, - recording: wave.Wave_read, + recording: Optional[wave.Wave_read], output_file: Optional[wave.Wave_write], + *, + audio_backend: Any = None, ) -> None: + if recording is None and audio_backend is None: + raise ValueError("Microphone mode requires an audio backend.") + session_closed = False + cancelled = False async with project_client.beta.voice_agents.live.connect( agent_name=agent_name, agent_version=agent_version, store=False ) as connection: try: await wait_for_event_async(connection, "session.started") - print("Live session started; sending the recording and receiving audio concurrently.") - await _stream_recording(connection, recording, output_file) + if recording is None: + try: + session_closed = await _stream_microphone(connection, audio_backend, output_file) + except asyncio.CancelledError: + print("\n(ending microphone conversation...)") + cancelled = True + else: + print("Live session started; sending the recording and receiving audio concurrently.") + await _stream_recording(connection, recording, output_file) finally: - await close_session_async(connection) + if not session_closed: + await close_session_async(connection) + if cancelled: + raise asyncio.CancelledError -async def main() -> None: +async def main(*, microphone: bool = False) -> None: load_dotenv() endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] model = os.environ["FOUNDRY_VOICE_AGENT_MODEL"] - input_path = Path(os.environ["FOUNDRY_VOICE_AUDIO_INPUT_PATH"]) + backend = _microphone_backend() if microphone else None + input_path = None if microphone else Path(os.environ["FOUNDRY_VOICE_AUDIO_INPUT_PATH"]) output_path = os.environ.get("FOUNDRY_VOICE_AUDIO_OUTPUT_PATH") - if output_path: + if output_path and input_path is not None: output = Path(output_path) if output.resolve() == input_path.resolve() or (output.exists() and output.samefile(input_path)): raise ValueError("The audio output path must not overwrite the input recording.") agent_name = sample_agent_name("audio") with ExitStack() as files: - recording = files.enter_context(wave.open(str(input_path), "rb")) - _validate_recording(recording) + recording = files.enter_context(wave.open(str(input_path), "rb")) if input_path is not None else None + if recording is not None: + _validate_recording(recording) output_file = files.enter_context(wave.open(output_path, "wb")) if output_path else None if output_file is not None: output_file.setnchannels(1) @@ -247,7 +482,7 @@ async def main() -> None: definition=models.VoiceAgentLiveDefinition( model=model, instructions=( - "You are a friendly voice assistant. Briefly answer the recorded speech. " + "You are a friendly voice assistant. Briefly answer the user's speech. " "Keep replies to one short sentence. Delegate external lookups to the client." ), audio=audio_settings(), @@ -255,9 +490,14 @@ async def main() -> None: store=False, ), ) - await _run_audio_conversation( - project_client, agent_name, created_version.version, recording, output_file - ) + if microphone: + await _run_audio_conversation( + project_client, agent_name, created_version.version, None, output_file, audio_backend=backend + ) + else: + await _run_audio_conversation( + project_client, agent_name, created_version.version, recording, output_file + ) finally: if created_version is not None: await project_client.agents.delete_version( @@ -269,4 +509,12 @@ async def main() -> None: if __name__ == "__main__": - asyncio.run(main()) + parser = argparse.ArgumentParser(description="Talk to GPT Live using a WAV file or your microphone.") + parser.add_argument( + "--microphone", action="store_true", help="Stream the default microphone and play replies using PyAudio." + ) + arguments = parser.parse_args() + try: + asyncio.run(main(microphone=arguments.microphone)) + except KeyboardInterrupt: + print("\nStopped.") diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py index c934d2cc31c3..337fa5a0fc12 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py @@ -6,10 +6,12 @@ import asyncio import base64 +from concurrent.futures import Future import importlib.util import io import json from pathlib import Path +import queue import sys from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -1120,3 +1122,351 @@ async def test_live_rest_samples_reject_blank_resource_identifiers(rest_sample, rest_sample.operations.get.assert_not_called() rest_sample.operations.delete.assert_not_called() + + +@pytest.fixture +def microphone_sample(monkeypatch, sample_environment): + sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + speaker = MagicMock() + microphone = MagicMock() + engine = MagicMock() + engine.open.side_effect = [speaker, microphone] + backend = SimpleNamespace(PyAudio=MagicMock(return_value=engine), paInt16=8, paContinue=0, paAbort=2) + monkeypatch.setattr(sample, "_microphone_backend", lambda: backend) + return SimpleNamespace( + sample=sample, + connection=sample_environment.async_connection, + client=sample_environment.async_client, + backend=backend, + engine=engine, + speaker=speaker, + microphone=microphone, + make=lambda: sample._MicrophoneAudio(sample_environment.async_connection, backend), + ) + + +@pytest.mark.asyncio +async def test_microphone_mode_does_not_require_a_wav_and_preserves_live_lifecycle( + monkeypatch, microphone_sample, capsys +): + setup = microphone_sample + monkeypatch.delenv("FOUNDRY_VOICE_AUDIO_INPUT_PATH") + setup.connection.recv.side_effect = [_started(), _closed()] + + await setup.sample.main(microphone=True) + + assert setup.engine.open.call_count == 2 + output_options, input_options = [call.kwargs for call in setup.engine.open.call_args_list] + assert output_options["output"] is True and input_options["input"] is True + for options in (output_options, input_options): + assert options["format"] == setup.backend.paInt16 + assert options["rate"] == 24000 and options["channels"] == 1 + assert options["frames_per_buffer"] == 1200 and options["start"] is False + setup.speaker.start_stream.assert_called_once() + setup.microphone.start_stream.assert_called_once() + setup.speaker.close.assert_called_once() + setup.microphone.close.assert_called_once() + setup.engine.terminate.assert_called_once() + definition = setup.client.agents.create_version.call_args.kwargs["definition"] + assert isinstance(definition, models.VoiceAgentLiveDefinition) and definition.store is False + setup.client.agents.delete_version.assert_awaited_once() + setup.connection.session.start.assert_not_called() + setup.connection.response.cancel.assert_not_called() + setup.client.beta.voice_agents.realtime.connect.assert_not_called() + assert "Speak now" in capsys.readouterr().out + + +@pytest.mark.asyncio +async def test_microphone_dependency_error_precedes_agent_creation(monkeypatch, sample_environment): + sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + monkeypatch.setitem(sys.modules, "pyaudio", None) + monkeypatch.delenv("FOUNDRY_VOICE_AUDIO_INPUT_PATH") + + with pytest.raises(RuntimeError, match="pip install pyaudio"): + await sample.main(microphone=True) + + sample_environment.async_client.agents.create_version.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_microphone_counts_only_successfully_sent_pcm(microphone_sample): + audio = microphone_sample.make() + pcm = b"\x01\x02" * 1200 + await audio._append_input_audio(pcm) + assert audio.input_bytes == len(pcm) + microphone_sample.connection.session.input_audio.append.assert_awaited_once_with(audio=pcm) + microphone_sample.connection.session.input_audio.append.side_effect = ConnectionError("send failed") + with pytest.raises(ConnectionError, match="send failed"): + await audio._append_input_audio(pcm) + assert audio.input_bytes == len(pcm) + await audio.stop() + + +@pytest.mark.asyncio +async def test_microphone_callback_sends_raw_pcm_on_the_event_loop(microphone_sample): + audio = microphone_sample.make() + audio.start() + pcm = b"\x01\x02" * 1200 + + assert audio._capture_callback(pcm, 1200, None, 0) == (None, microphone_sample.backend.paContinue) + await asyncio.wrap_future(audio._pending_send) + + assert audio.input_bytes == len(pcm) + microphone_sample.connection.session.input_audio.append.assert_awaited_once_with(audio=pcm) + await audio.stop() + + +@pytest.mark.asyncio +async def test_microphone_capture_backpressure_drops_a_frame_without_creating_more_sends(microphone_sample): + audio = microphone_sample.make() + audio.start() + pending = Future() + audio._pending_send = pending + + assert audio._capture_callback(b"\x01\x02" * 1200, 1200, None, 0) == (None, microphone_sample.backend.paContinue) + assert audio._pending_send is pending and audio.dropped_frames == 1 + microphone_sample.connection.session.input_audio.append.assert_not_called() + pending.set_result(None) + await audio.stop() + + +@pytest.mark.asyncio +async def test_microphone_callback_send_failure_is_raised_by_the_receiver(microphone_sample): + audio = microphone_sample.make() + audio.start() + failure = ConnectionError("captured audio send failed") + pending = Future() + pending.set_exception(failure) + audio._pending_send = pending + + assert audio._capture_callback(b"\x00\x00" * 1200, 1200, None, 0) == (None, microphone_sample.backend.paAbort) + with pytest.raises(ConnectionError) as caught: + audio.check_capture_error() + assert caught.value is failure + with pytest.raises(ConnectionError): + await audio.stop() + microphone_sample.speaker.close.assert_called_once() + microphone_sample.microphone.close.assert_called_once() + microphone_sample.engine.terminate.assert_called_once() + + +@pytest.mark.asyncio +async def test_microphone_shutdown_waits_for_the_last_send_and_closes_once(microphone_sample): + audio = microphone_sample.make() + audio.start() + pending = Future() + audio._pending_send = pending + shutdown = asyncio.create_task(audio.stop()) + await asyncio.sleep(0) + assert not shutdown.done() + microphone_sample.microphone.close.assert_called_once() + pending.set_result(None) + await shutdown + await audio.stop() + assert audio._pending_send is None + microphone_sample.microphone.close.assert_called_once() + microphone_sample.speaker.close.assert_called_once() + microphone_sample.engine.terminate.assert_called_once() + + +@pytest.mark.asyncio +async def test_microphone_playback_preserves_remainder_and_pads_exact_callback_frames(microphone_sample): + audio = microphone_sample.make() + audio.start() + audio.enqueue_audio(b"\x01\x02" * 5) + + first, state = audio._playback_callback(None, 3, None, 0) + second, _ = audio._playback_callback(None, 4, None, 0) + + assert first == b"\x01\x02" * 3 + assert second == b"\x01\x02" * 2 + b"\x00" * 4 + assert state == microphone_sample.backend.paContinue + assert audio.output_bytes == 10 + await audio.stop() + + +@pytest.mark.asyncio +async def test_microphone_rejects_partial_pcm_and_reports_playback_backpressure(microphone_sample): + audio = microphone_sample.make() + audio._playback_queue = queue.Queue(maxsize=1) + with pytest.raises(ValueError, match="PCM16"): + audio.enqueue_audio(b"\x01") + assert audio.output_bytes == 0 + audio.enqueue_audio(b"\x01\x02") + with pytest.raises(RuntimeError, match="playback"): + audio.enqueue_audio(b"\x03\x04") + assert audio.output_bytes == 4 + await audio.stop() + + +@pytest.mark.asyncio +async def test_microphone_input_device_failure_releases_the_opened_speaker(microphone_sample): + microphone_sample.engine.open.side_effect = [microphone_sample.speaker, OSError("microphone unavailable")] + with pytest.raises(OSError, match="microphone unavailable"): + await microphone_sample.sample._stream_microphone(microphone_sample.connection, microphone_sample.backend, None) + microphone_sample.speaker.close.assert_called_once() + microphone_sample.engine.terminate.assert_called_once() + + +@pytest.mark.asyncio +async def test_microphone_receives_audio_and_transcripts_and_saves_only_explicit_reply_output( + microphone_sample, capsys +): + pcm = b"\x01\x02" * 1200 + microphone_sample.connection.recv.side_effect = [ + models.LiveServerEventLiveInputTranscriptDelta(delta="Hello"), + models.LiveServerEventLiveOutputTranscriptDelta(delta="Hi"), + models.LiveOutputAudioDelta(delta=base64.b64encode(pcm).decode("ascii")), + _closed(), + ] + output_file = MagicMock() + + closed = await microphone_sample.sample._stream_microphone( + microphone_sample.connection, microphone_sample.backend, output_file + ) + + assert closed is True + output_file.writeframesraw.assert_called_once_with(pcm) + text = capsys.readouterr().out + assert "You: Hello" in text and "Agent: Hi" in text + assert f"{len(pcm)} bytes" in text + microphone_sample.engine.terminate.assert_called_once() + + +@pytest.mark.asyncio +async def test_microphone_cancellation_closes_the_session_and_deletes_only_its_version(microphone_sample, monkeypatch): + monkeypatch.delenv("FOUNDRY_VOICE_AUDIO_INPUT_PATH") + microphone_sample.connection.recv.side_effect = [_started(), asyncio.CancelledError(), _closed()] + + with pytest.raises(asyncio.CancelledError): + await microphone_sample.sample.main(microphone=True) + + microphone_sample.connection.session.close.assert_awaited_once() + microphone_sample.client.agents.delete_version.assert_awaited_once() + microphone_sample.client.agents.delete.assert_not_called() + microphone_sample.microphone.close.assert_called_once() + microphone_sample.speaker.close.assert_called_once() + microphone_sample.engine.terminate.assert_called_once() + + +@pytest.mark.asyncio +async def test_microphone_service_errors_are_not_silently_successful(microphone_sample, monkeypatch): + monkeypatch.delenv("FOUNDRY_VOICE_AUDIO_INPUT_PATH") + microphone_sample.connection.recv.side_effect = [ + _started(), + {"type": "error", "error": {"code": "offline_error", "message": "service failed"}}, + ] + with pytest.raises(RuntimeError, match="service failed"): + await microphone_sample.sample.main(microphone=True) + microphone_sample.client.agents.delete_version.assert_awaited_once() + microphone_sample.microphone.close.assert_called_once() + microphone_sample.speaker.close.assert_called_once() + microphone_sample.engine.terminate.assert_called_once() + + +@pytest.mark.asyncio +async def test_microphone_stopped_capture_does_not_schedule_an_audio_send(microphone_sample): + audio = microphone_sample.make() + assert audio._capture_callback(b"\x00\x00" * 1200, 1200, None, 0) == (None, microphone_sample.backend.paAbort) + microphone_sample.connection.session.input_audio.append.assert_not_called() + await audio.stop() + + +@pytest.mark.asyncio +async def test_cancelled_microphone_send_is_an_error_not_a_successful_user_stop(microphone_sample): + audio = microphone_sample.make() + audio.start() + pending = Future() + pending.cancel() + audio._pending_send = pending + assert audio._capture_callback(b"\x00\x00" * 1200, 1200, None, 0) == (None, microphone_sample.backend.paAbort) + with pytest.raises(RuntimeError, match="send was cancelled"): + audio.check_capture_error() + with pytest.raises(RuntimeError, match="send was cancelled"): + await audio.stop() + microphone_sample.engine.terminate.assert_called_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("device,status", [("microphone", 2), ("speaker", 4)]) +async def test_microphone_device_status_errors_are_reported_to_the_receiver(microphone_sample, device, status): + audio = microphone_sample.make() + audio.start() + if device == "microphone": + result, state = audio._capture_callback(b"\x00\x00" * 1200, 1200, None, status) + assert result is None + else: + result, state = audio._playback_callback(None, 1200, None, status) + assert len(result) == 2400 + assert state == microphone_sample.backend.paAbort + with pytest.raises(RuntimeError, match="callback reported audio status"): + audio.check_capture_error() + await audio.stop() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reason", ["connection_lost", "content"]) +async def test_microphone_abnormal_session_termination_is_not_a_normal_completion(microphone_sample, reason): + closed = _closed() + closed.reason = reason + microphone_sample.connection.recv.return_value = closed + with pytest.raises(RuntimeError, match="ended unexpectedly"): + await microphone_sample.sample._stream_microphone(microphone_sample.connection, microphone_sample.backend, None) + microphone_sample.engine.terminate.assert_called_once() + + +@pytest.mark.asyncio +async def test_microphone_client_delegation_uses_associated_live_commentary(microphone_sample): + microphone_sample.connection.recv.side_effect = [ + models.LiveServerEventLiveDelegationCreated( + {"delegation": {"id": "delegation-test", "target": "client", "created_at": 1}} + ), + _closed(), + ] + await microphone_sample.sample._stream_microphone(microphone_sample.connection, microphone_sample.backend, None) + microphone_sample.connection.session.commentary.append.assert_awaited_once_with( + content="This sample cannot look up external information. Briefly acknowledge the spoken request.", + delegation_id="delegation-test", + ) + microphone_sample.connection.response.cancel.assert_not_called() + + +@pytest.mark.asyncio +async def test_wav_mode_does_not_import_or_initialize_optional_microphone_devices(monkeypatch, sample_environment): + sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + monkeypatch.setitem(sys.modules, "pyaudio", None) + microphone_backend = MagicMock(side_effect=AssertionError("WAV mode initialized a microphone backend")) + monkeypatch.setattr(sample, "_microphone_backend", microphone_backend) + monkeypatch.setattr(sample, "_stream_recording", AsyncMock()) + sample_environment.async_connection.recv.side_effect = [_started(), _closed()] + with _recording() as recording: + monkeypatch.setattr(sample, "wave", SimpleNamespace(open=MagicMock(return_value=recording))) + await sample.main() + microphone_backend.assert_not_called() + sample._stream_recording.assert_awaited_once() + sample_environment.async_client.agents.delete_version.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_microphone_cannot_restart_devices_after_shutdown(microphone_sample): + audio = microphone_sample.make() + await audio.stop() + with pytest.raises(RuntimeError, match="Cannot restart closed"): + audio.start() + microphone_sample.engine.open.assert_not_called() + + +@pytest.mark.asyncio +async def test_microphone_stop_failure_still_drains_the_last_send_and_releases_every_device(microphone_sample): + audio = microphone_sample.make() + audio.start() + pending = Future() + pending.set_result(None) + audio._pending_send = pending + microphone_sample.microphone.stop_stream.side_effect = OSError("device stop failed") + with pytest.raises(OSError, match="device stop failed"): + await audio.stop() + assert audio._pending_send is None + microphone_sample.microphone.close.assert_called_once() + microphone_sample.speaker.close.assert_called_once() + microphone_sample.engine.terminate.assert_called_once() From 184af8c9bdb6a79e325befbbe414184144fe414b Mon Sep 17 00:00:00 2001 From: Xiting Zhang Date: Wed, 21 Oct 2026 19:03:18 -0700 Subject: [PATCH 06/10] Add recorded GPT Live REST coverage Cover all nine dedicated Live REST methods in sync and async Test Proxy lifecycles, including pagination, audio content, HTTP statuses, deletion and safe fixture cleanup. Reference the published recording assets and add offline playback regressions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/ai/azure-ai-projects/assets.json | 2 +- .../agents/live_conversation_test_base.py | 312 +++++++++++++++++ ...est_live_conversation_recording_helpers.py | 313 ++++++++++++++++++ .../test_voice_agent_live_conversations.py | 118 +++++++ ...st_voice_agent_live_conversations_async.py | 113 +++++++ 5 files changed, 857 insertions(+), 1 deletion(-) create mode 100644 sdk/ai/azure-ai-projects/tests/agents/live_conversation_test_base.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_live_conversation_recording_helpers.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_live_conversations.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_live_conversations_async.py diff --git a/sdk/ai/azure-ai-projects/assets.json b/sdk/ai/azure-ai-projects/assets.json index 47291692e6fa..d0ae5b6975b9 100644 --- a/sdk/ai/azure-ai-projects/assets.json +++ b/sdk/ai/azure-ai-projects/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "python", "TagPrefix": "python/ai/azure-ai-projects", - "Tag": "python/ai/azure-ai-projects_6e955764df" + "Tag": "python/ai/azure-ai-projects_6289cbf17e" } diff --git a/sdk/ai/azure-ai-projects/tests/agents/live_conversation_test_base.py b/sdk/ai/azure-ai-projects/tests/agents/live_conversation_test_base.py new file mode 100644 index 000000000000..52e68de4365c --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/live_conversation_test_base.py @@ -0,0 +1,312 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +"""Recording-only Live fixtures and shared REST assertions. + +The preparer runs outside the recorded decorator. Recording creates two owned +stored greeting sessions and waits for durable audio before REST recording +starts. Playback uses sanitized IDs and never opens a WebSocket or creates an +agent. Recording uses an existing GPT Live deployment and needs no microphone +or prerecorded audio file. +""" + +import array +import asyncio +import base64 +from contextlib import AsyncExitStack +from dataclasses import dataclass +import datetime +import functools +import io +import re +import sys +import time +from typing import Any +from urllib.parse import urlsplit +import uuid +import wave + +from azure.ai.projects import models +from azure.ai.projects.aio import AIProjectClient +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError +from devtools_testutils import EnvironmentVariableLoader, add_general_regex_sanitizer, is_live +from devtools_testutils.preparers import AbstractPreparer + + +liveConversationServicePreparer = functools.partial( + EnvironmentVariableLoader, + "", + foundry_project_endpoint="https://sanitized-account-name.services.ai.azure.com/api/projects/sanitized-project-name", + foundry_voice_live_model_name="sanitized-live-deployment-name", +) + +_AGENT = "sanitized-live-rest-agent" +_CONVERSATIONS = ("sanitized-live-conversation-1", "sanitized-live-conversation-2") +_SILENCE = bytes(2400) +_SESSION_TIMEOUT = 45 +_FINALIZATION_TIMEOUT = 90 + + +@dataclass(frozen=True) +class LiveConversationFixture: + agent_name: str + conversation_ids: tuple[str, ...] + item_ids: tuple[tuple[str, ...], ...] = () + version: str = "" + + +def playback_fixture() -> LiveConversationFixture: + return LiveConversationFixture(_AGENT, _CONVERSATIONS) + + +def add_live_conversation_sanitizers(fixture: LiveConversationFixture) -> None: + pairs = [(fixture.agent_name, _AGENT)] + for index, conversation_id in enumerate(fixture.conversation_ids): + pairs.append((conversation_id, _CONVERSATIONS[index])) + for item_index, item_id in enumerate(fixture.item_ids[index]): + pairs.append((item_id, f"sanitized-live-item-{index + 1}-{item_index + 1}")) + for actual, replacement in pairs: + add_general_regex_sanitizer(regex=re.escape(actual), value=replacement, function_scoped=True) + + +def _check_live_error(event: Any) -> None: + if event.get("type") == "error": + error = event.get("error") or {} + raise RuntimeError(f"Live recording fixture failed: {error.get('code')}: {error.get('message')}") + + +async def _receive_event(connection, event_type: str): + deadline = time.monotonic() + _SESSION_TIMEOUT + while time.monotonic() < deadline: + event = await asyncio.wait_for(connection.recv(), timeout=deadline - time.monotonic()) + _check_live_error(event) + if event.get("type") == event_type: + return event + if event.get("type") == "session.closed": + raise RuntimeError(f"Live recording fixture closed early: {event.get('reason')}") + raise TimeoutError(f"Live recording fixture did not receive {event_type}") + + +async def _hold_greeting(client: AIProjectClient, agent_name: str, version: str, conversations: list[str]) -> None: + async with client.beta.voice_agents.live.connect( + agent_name=agent_name, agent_version=version, store=True + ) as connection: + started = await _receive_event(connection, "session.started") + assert started.conversation_id and started.conversation_id != started.session.id + conversations.append(started.conversation_id) + stop = asyncio.Event() + + async def clock(): + while not stop.is_set(): + await connection.session.input_audio.append(audio=_SILENCE) + await asyncio.sleep(0.05) + + sender = asyncio.create_task(clock()) + speech = False + transcript = "" + last_content = time.monotonic() + deadline = time.monotonic() + _SESSION_TIMEOUT + try: + while time.monotonic() < deadline: + if sender.done(): + sender.result() + if speech and transcript.strip() and time.monotonic() - last_content >= 2: + break + try: + event = await asyncio.wait_for(connection.recv(), timeout=min(0.5, deadline - time.monotonic())) + except asyncio.TimeoutError: + continue + _check_live_error(event) + if isinstance(event, models.LiveOutputAudioDelta): + pcm = base64.b64decode(event.delta, validate=True) + assert len(pcm) % 2 == 0 + if any( + abs(int.from_bytes(pcm[i : i + 2], "little", signed=True)) > 256 for i in range(0, len(pcm), 2) + ): + speech = True + last_content = time.monotonic() + elif isinstance(event, models.LiveServerEventLiveOutputTranscriptDelta): + transcript += event.delta + if event.delta.strip(): + last_content = time.monotonic() + elif event.get("type") == "session.closed": + raise RuntimeError("Live recording fixture ended before greeting audio was captured") + else: + raise TimeoutError("Live recording fixture did not finish greeting speech") + finally: + stop.set() + try: + await sender + finally: + await connection.session.close() + closed = await _receive_event(connection, "session.closed") + assert closed.reason == "close_requested" + + +async def _wait_for_recording(client: AIProjectClient, agent_name: str, conversation_id: str) -> tuple[str, ...]: + operations = client.beta.voice_agents.live_conversations + deadline = time.monotonic() + _FINALIZATION_TIMEOUT + while time.monotonic() < deadline: + try: + conversation = await operations.get(agent_name, conversation_id) + except ResourceNotFoundError: + await asyncio.sleep(1) + continue + if conversation.status == "failed": + raise AssertionError(f"Live recording persistence failed: {conversation.last_error}") + if conversation.status == "completed": + try: + recording = await operations.get_audio(agent_name, conversation_id) + except HttpResponseError as exc: + if exc.status_code != 409 or getattr(exc.error, "code", None) != "recording_not_ready": + raise + else: + assert recording.blob_uri is None, "Recorded audio tests require Foundry-managed storage, not BYOS" + items = [item async for item in operations.list_items(agent_name, conversation_id)] + assert items and any( + isinstance(item, models.VoiceAgentLiveOutputConversationItem) and item.audio is not None + for item in items + ) + return tuple(item.id for item in items) + await asyncio.sleep(1) + raise TimeoutError("Live recording fixture did not reach completed persistence/audio state") + + +async def _delete_fixture(client: AIProjectClient, fixture: LiveConversationFixture) -> None: + async def delete_conversation(conversation_id: str) -> None: + try: + await client.beta.voice_agents.live_conversations.delete(fixture.agent_name, conversation_id) + except ResourceNotFoundError: + pass + + async with AsyncExitStack() as cleanup: + cleanup.push_async_callback( + client.agents.delete_version, agent_name=fixture.agent_name, agent_version=fixture.version + ) + for conversation_id in reversed(fixture.conversation_ids): + cleanup.push_async_callback(delete_conversation, conversation_id) + + +class LiveConversationPreparer(AbstractPreparer): + def __init__(self) -> None: + super().__init__("live-rest-", 24, disable_recording=True) + self.resource_moniker = _AGENT + self.fixture: LiveConversationFixture | None = None + self.endpoint: str | None = None + + def create_resource(self, name, **kwargs): + self.fixture = None + self.endpoint = None + if not is_live(): + return {"live_conversation_fixture": playback_fixture()} + self.endpoint = kwargs["foundry_project_endpoint"] + model = kwargs["foundry_voice_live_model_name"] + self.fixture = asyncio.run(self._create_fixture(f"{name}-{uuid.uuid4().hex[:8]}", model)) + return {"live_conversation_fixture": self.fixture} + + async def _create_fixture(self, agent_name: str, model: str) -> LiveConversationFixture: + endpoint = self.endpoint + assert isinstance(endpoint, str) and endpoint.strip(), "A recording fixture needs a project endpoint" + credential = self.test_class_instance.get_credential(AIProjectClient, is_async=True) + async with credential, AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as client: + agent = None + conversation_ids: list[str] = [] + complete = False + try: + agent = await client.agents.create_version( + agent_name=agent_name, + definition=models.VoiceAgentLiveDefinition( + model=model, + instructions="Greet the caller briefly.", + greeting=models.VoiceAgentTemplateGreetingConfig(text="Hello, welcome."), + audio=models.LiveInitialSessionAudioParam( + format=models.LiveSessionAudioFormatPCMParam(rate=24000), + output=models.LiveInitialSessionAudioOutputParam(voice="marin"), + ), + delegation=models.LiveClientDelegationParam(), + store=True, + ), + ) + items = [] + for _ in _CONVERSATIONS: + await _hold_greeting(client, agent_name, agent.version, conversation_ids) + items.append(await _wait_for_recording(client, agent_name, conversation_ids[-1])) + fixture = LiveConversationFixture(agent_name, tuple(conversation_ids), tuple(items), agent.version) + complete = True + return fixture + finally: + if agent is not None and not complete: + await _delete_fixture( + client, LiveConversationFixture(agent_name, tuple(conversation_ids), version=agent.version) + ) + + def remove_resource(self, name, **kwargs): + if self.fixture is not None: + asyncio.run(self._cleanup_fixture()) + self.fixture = None + self.endpoint = None + + async def _cleanup_fixture(self) -> None: + fixture, endpoint = self.fixture, self.endpoint + assert fixture is not None and isinstance(endpoint, str), "No owned recording fixture was prepared" + credential = self.test_class_instance.get_credential(AIProjectClient, is_async=True) + async with credential, AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as client: + await _delete_fixture(client, fixture) + + +def assert_live_status(expected: int): + def verify(response): + assert response.http_response.status_code == expected + assert "/protocols/voice/live/conversations" in urlsplit(response.http_request.url).path + assert response.http_request.headers["Foundry-Features"] == "VoiceAgents=V1Preview" + + return verify + + +def assert_completed_conversation(conversation, conversation_id: str) -> None: + assert isinstance(conversation, models.VoiceAgentLiveConversation) + assert conversation.id == conversation_id + assert conversation.object == "voice.conversation" and conversation.status == "completed" + assert isinstance(conversation.created_at, datetime.datetime) + assert isinstance(conversation.completed_at, datetime.datetime) + assert conversation.last_error is None + + +def assert_speech_item(item, expected_id: str | None = None) -> models.VoiceAgentLiveOutputConversationItem: + assert isinstance(item, models.VoiceAgentLiveOutputConversationItem) + assert item.id and (expected_id is None or item.id == expected_id) + assert item.transcription and item.transcription.strip() + assert isinstance(item.audio, models.VoiceAudioItem) + assert item.audio.item_id == item.id + return item + + +def assert_audio_metadata(audio, conversation_id: str, *, item_id: str | None = None) -> None: + assert audio.conversation_id == conversation_id + assert audio.format == "wav" and audio.sample_rate == 24000 + assert isinstance(audio.duration_ms, datetime.timedelta) and audio.duration_ms.total_seconds() > 0 + assert audio.blob_uri is None, "This test must exercise the Foundry-managed content routes" + if item_id is None: + assert isinstance(audio, models.VoiceRecording) and audio.channels == 2 + assert audio.channel_layout.left == "user" and audio.channel_layout.right == "agent" + else: + assert isinstance(audio, models.VoiceAudioItem) and audio.item_id == item_id + assert audio.role == "agent" and audio.channels == 1 and audio.codec == "pcm16" + + +def assert_wav(content: bytes, metadata) -> None: + assert content[:4] == b"RIFF" and content[8:12] == b"WAVE" + with wave.open(io.BytesIO(content), "rb") as recording: + assert recording.getcomptype() == "NONE" and recording.getsampwidth() == 2 + assert recording.getframerate() == metadata.sample_rate + assert recording.getnchannels() == metadata.channels + frames = recording.getnframes() + pcm = recording.readframes(frames) + assert frames > 0 and len(pcm) == frames * metadata.channels * 2 + assert abs(frames / metadata.sample_rate - metadata.duration_ms.total_seconds()) <= 0.01 + samples = array.array("h", pcm) + if sys.byteorder != "little": + samples.byteswap() + channel = 1 if metadata.channels == 2 else 0 + assert any(abs(sample) > 256 for sample in samples[channel :: metadata.channels]), "Expected non-silent agent audio" diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_live_conversation_recording_helpers.py b/sdk/ai/azure-ai-projects/tests/agents/test_live_conversation_recording_helpers.py new file mode 100644 index 000000000000..e36b60caae1e --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_live_conversation_recording_helpers.py @@ -0,0 +1,313 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +"""Offline guards for the recorded REST tests; no service or cassette is used here.""" + +import datetime +import importlib +import io +import json +from pathlib import Path +import re +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock +import wave + +import pytest +from azure.ai.projects import models +from azure.core.exceptions import ResourceNotFoundError + +import live_conversation_test_base as helpers + + +def _audio(channels=1): + common = { + "conversation_id": "conversation-test", + "format": "wav", + "sample_rate": 24000, + "channels": channels, + "duration_ms": 50, + } + if channels == 1: + return models.VoiceAudioItem(dict(common, item_id="item-test", role="agent", codec="pcm16")) + return models.VoiceRecording(dict(common, channel_layout={"left": "user", "right": "agent"})) + + +def _wav(channels=1, silent=False): + stream = io.BytesIO() + sample = b"\x00\x00" if silent else b"\x00\x08" + frame = sample if channels == 1 else b"\x00\x00" + sample + with wave.open(stream, "wb") as recording: + recording.setparams((channels, 2, 24000, 0, "NONE", "not compressed")) + recording.writeframes(frame * 1200) + return stream.getvalue() + + +def test_playback_fixture_never_creates_an_agent_or_opens_a_websocket(monkeypatch): + monkeypatch.setattr(helpers, "is_live", lambda: False) + create_client = MagicMock(side_effect=AssertionError("Playback attempted live setup")) + monkeypatch.setattr(helpers, "AIProjectClient", create_client) + preparer = helpers.LiveConversationPreparer() + preparer.test_class_instance = SimpleNamespace(get_credential=MagicMock()) + preparer.fixture = helpers.LiveConversationFixture("previous-owned-agent", ("previous-conversation",), version="1") + preparer.endpoint = "https://example.invalid/previous-recording" + + result = preparer.create_resource("unused") + preparer.remove_resource("unused") + + assert result["live_conversation_fixture"] == helpers.playback_fixture() + create_client.assert_not_called() + preparer.test_class_instance.get_credential.assert_not_called() + + +def test_playback_preparer_does_not_require_the_legacy_test_resource_counter(monkeypatch): + monkeypatch.setattr(helpers, "is_live", lambda: False) + case = SimpleNamespace(is_live=False, in_recording=False) + received = [] + + def recorded_body(instance, live_conversation_fixture): + assert instance is case + received.append(live_conversation_fixture) + + helpers.LiveConversationPreparer()(recorded_body)(case) + assert received == [helpers.playback_fixture()] + + +@pytest.mark.parametrize("fail_test", [False, True]) +def test_websocket_setup_and_owned_cleanup_are_outside_recorded_transport_patching(monkeypatch, fail_test): + monkeypatch.setattr(helpers, "is_live", lambda: True) + active = {"recorded": False} + calls = [] + fixture = helpers.LiveConversationFixture( + "owned-agent", ("owned-conversation-1", "owned-conversation-2"), (("owned-item-1",), ("owned-item-2",)), "1" + ) + preparer = helpers.LiveConversationPreparer() + + async def create(*args): + assert not active["recorded"] + calls.append("setup") + return fixture + + async def cleanup(): + assert not active["recorded"] + calls.append("cleanup") + + monkeypatch.setattr(preparer, "_create_fixture", create) + monkeypatch.setattr(preparer, "_cleanup_fixture", cleanup) + + def recorded_body(instance, **kwargs): + active["recorded"] = True + try: + assert kwargs["live_conversation_fixture"] is fixture + calls.append("recorded-rest") + if fail_test: + raise ValueError("recorded test failed") + finally: + active["recorded"] = False + + instance = SimpleNamespace(in_recording=False, is_live=True, test_resources_count=0) + wrapped = preparer(recorded_body) + if fail_test: + with pytest.raises(ValueError, match="recorded test failed"): + wrapped( + instance, foundry_project_endpoint="https://example.invalid", foundry_voice_live_model_name="live-test" + ) + else: + wrapped(instance, foundry_project_endpoint="https://example.invalid", foundry_voice_live_model_name="live-test") + assert calls == ["setup", "recorded-rest", "cleanup"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_failed_recorded_assertions_do_not_append_unexpected_cleanup_requests(monkeypatch, asynchronous): + monkeypatch.syspath_prepend(str(Path(__file__).resolve().parents[1])) + module = importlib.import_module("test_voice_agent_live_conversations" + ("_async" if asynchronous else "")) + monkeypatch.setattr(module, "is_live", lambda: False) + case = getattr(module, "TestVoiceAgentLiveConversations" + ("Async" if asynchronous else "")) + body = getattr(case, "_test_live_conversation_rest_lifecycle" + ("_async" if asynchronous else "")) + client = MagicMock() + client.__enter__.return_value = client + client.__aenter__.return_value = client + client.beta.voice_agents.live_conversations.list.return_value.__iter__.return_value = iter(()) + client.beta.voice_agents.live_conversations.list.return_value.__aiter__.return_value = [] + instance = SimpleNamespace( + create_client=MagicMock(return_value=client), create_async_client=MagicMock(return_value=client) + ) + with pytest.raises(AssertionError): + if asynchronous: + await body(instance, live_conversation_fixture=helpers.playback_fixture()) + else: + body(instance, live_conversation_fixture=helpers.playback_fixture()) + client.beta.voice_agents.live_conversations.delete.assert_not_called() + + +def test_sanitizers_keep_agent_conversation_item_and_cursor_references_consistent(monkeypatch): + sanitize = MagicMock() + monkeypatch.setattr(helpers, "add_general_regex_sanitizer", sanitize) + fixture = helpers.LiveConversationFixture( + "owned-agent", + ("owned.conversation+1", "owned.conversation+2"), + (("owned.item+1", "owned.item+2"), ("owned.item+3",)), + "1", + ) + helpers.add_live_conversation_sanitizers(fixture) + assert sanitize.call_count == 6 + wire = json.dumps( + { + "agent_name": fixture.agent_name, + "conversation_ids": fixture.conversation_ids, + "item_ids": fixture.item_ids, + "after": fixture.conversation_ids[0], + "audio": {"conversation_id": fixture.conversation_ids[0], "item_id": fixture.item_ids[0][0]}, + } + ) + for call in sanitize.call_args_list: + assert call.kwargs["function_scoped"] is True + wire = re.sub(call.kwargs["regex"], call.kwargs["value"], wire) + parsed = json.loads(wire) + expected = helpers.playback_fixture() + assert parsed["agent_name"] == expected.agent_name + assert parsed["conversation_ids"] == list(expected.conversation_ids) + assert parsed["after"] == parsed["audio"]["conversation_id"] == expected.conversation_ids[0] + assert parsed["audio"]["item_id"] == parsed["item_ids"][0][0] == "sanitized-live-item-1-1" + assert "owned" not in wire + + +@pytest.mark.asyncio +async def test_fixture_cleanup_deletes_only_owned_versions_even_if_conversation_delete_fails(): + client = MagicMock() + client.beta.voice_agents.live_conversations.delete = AsyncMock(side_effect=[ValueError("delete failed"), None]) + client.agents.delete_version = AsyncMock() + fixture = helpers.LiveConversationFixture("owned-agent", ("first", "second"), version="7") + with pytest.raises(ValueError, match="delete failed"): + await helpers._delete_fixture(client, fixture) + assert client.beta.voice_agents.live_conversations.delete.await_count == 2 + assert [call.args for call in client.beta.voice_agents.live_conversations.delete.await_args_list] == [ + ("owned-agent", "first"), + ("owned-agent", "second"), + ] + client.agents.delete_version.assert_awaited_once_with(agent_name="owned-agent", agent_version="7") + client.agents.delete.assert_not_called() + + +@pytest.mark.asyncio +async def test_fixture_cleanup_tolerates_a_conversation_already_deleted_by_the_recorded_test(): + client = MagicMock() + client.beta.voice_agents.live_conversations.delete = AsyncMock(side_effect=[ResourceNotFoundError("deleted"), None]) + client.agents.delete_version = AsyncMock() + fixture = helpers.LiveConversationFixture("owned-agent", ("first", "second"), version="1") + await helpers._delete_fixture(client, fixture) + assert client.beta.voice_agents.live_conversations.delete.await_count == 2 + client.agents.delete_version.assert_awaited_once_with(agent_name="owned-agent", agent_version="1") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("setup_fails", [False, True]) +async def test_recording_setup_tracks_owned_ids_and_cleans_partial_websocket_failure(monkeypatch, setup_fails): + credential = MagicMock() + credential.__aenter__.return_value = credential + client = MagicMock() + client.__aenter__.return_value = client + client.agents.create_version = AsyncMock(return_value=SimpleNamespace(version="1")) + client.agents.delete_version = AsyncMock() + client.beta.voice_agents.live_conversations.delete = AsyncMock() + constructor = MagicMock(return_value=client) + monkeypatch.setattr(helpers, "AIProjectClient", constructor) + preparer = helpers.LiveConversationPreparer() + preparer.endpoint = "https://example.invalid/api/projects/test" + preparer.test_class_instance = SimpleNamespace(get_credential=MagicMock(return_value=credential)) + + async def hold_greeting(project_client, name, version, conversation_ids): + assert project_client is client and name == "owned-agent" and version == "1" + conversation_ids.append(f"owned-conversation-{len(conversation_ids) + 1}") + if setup_fails: + raise ConnectionError("WebSocket fixture failed") + + monkeypatch.setattr(helpers, "_hold_greeting", hold_greeting) + ready = AsyncMock(side_effect=[("owned-item-1",), ("owned-item-2",)]) + monkeypatch.setattr(helpers, "_wait_for_recording", ready) + if setup_fails: + with pytest.raises(ConnectionError, match="WebSocket fixture failed"): + await preparer._create_fixture("owned-agent", "live-model-test") + client.beta.voice_agents.live_conversations.delete.assert_awaited_once_with( + "owned-agent", "owned-conversation-1" + ) + ready.assert_not_awaited() + else: + fixture = await preparer._create_fixture("owned-agent", "live-model-test") + assert fixture.conversation_ids == ("owned-conversation-1", "owned-conversation-2") + assert fixture.item_ids == (("owned-item-1",), ("owned-item-2",)) + assert fixture.version == "1" + client.agents.delete_version.assert_not_awaited() + preparer.fixture = fixture + await preparer._cleanup_fixture() + assert client.beta.voice_agents.live_conversations.delete.await_count == 2 + client.agents.delete_version.assert_awaited_once_with(agent_name="owned-agent", agent_version="1") + client.agents.delete.assert_not_called() + definition = client.agents.create_version.call_args.kwargs["definition"] + assert definition.store is True and definition.sub_protocol == "live" + assert definition.greeting.text + + +@pytest.mark.parametrize("channels", [1, 2]) +def test_recorded_audio_assertions_parse_pcm_shape_duration_and_agent_channel(channels): + audio = _audio(channels) + helpers.assert_audio_metadata(audio, "conversation-test", item_id="item-test" if channels == 1 else None) + helpers.assert_wav(_wav(channels), audio) + + +@pytest.mark.parametrize("failure", ["silent", "truncated", "wrong-duration", "wrong-format"]) +def test_recorded_audio_assertions_reject_success_shaped_but_invalid_audio(failure): + audio = _audio() + content = _wav(silent=failure == "silent") + if failure == "truncated": + content = content[:-2] + elif failure == "wrong-duration": + audio.duration_ms = datetime.timedelta(seconds=5) + elif failure == "wrong-format": + content = b"not-a-wave-recording" + with pytest.raises(AssertionError): + helpers.assert_wav(content, audio) + + +def test_recorded_audio_requires_actual_managed_content_and_transcript(): + audio = _audio() + assert isinstance(audio, models.VoiceAudioItem) + audio.blob_uri = "https://example.invalid/customer-recording" + with pytest.raises(AssertionError, match="managed content"): + helpers.assert_audio_metadata(audio, "conversation-test", item_id="item-test") + audio.blob_uri = None + item = models.VoiceAgentLiveOutputConversationItem(id="item-test", transcription=" ", audio=audio) + with pytest.raises(AssertionError): + helpers.assert_speech_item(item) + item.transcription = "A synthetic recorded greeting." + helpers.assert_speech_item(item, "item-test") + + +@pytest.mark.parametrize("status", ["in_progress", "failed"]) +def test_recorded_conversation_must_be_durably_completed(status): + conversation = models.VoiceAgentLiveConversation( + {"id": "conversation-test", "status": status, "created_at": 1, "completed_at": 2} + ) + with pytest.raises(AssertionError): + helpers.assert_completed_conversation(conversation, "conversation-test") + + +@pytest.mark.parametrize("invalid", ["route", "status", "header"]) +def test_recorded_http_assertions_reject_old_routes_wrong_status_or_missing_preview(invalid): + response = SimpleNamespace( + http_response=SimpleNamespace(status_code=200), + http_request=SimpleNamespace( + url="https://example.invalid/agents/agent-test/endpoint/protocols/voice/live/conversations", + headers={"Foundry-Features": "VoiceAgents=V1Preview"}, + ), + ) + if invalid == "route": + response.http_request.url = response.http_request.url.replace("/live/", "/") + elif invalid == "status": + response.http_response.status_code = 204 + else: + response.http_request.headers["Foundry-Features"] = "wrong-feature" + with pytest.raises(AssertionError): + helpers.assert_live_status(200)(response) diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_live_conversations.py b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_live_conversations.py new file mode 100644 index 000000000000..04be0b260943 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_live_conversations.py @@ -0,0 +1,118 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +"""Test Proxy recorded coverage of all nine dedicated GPT Live REST operations. + +Recording needs FOUNDRY_PROJECT_ENDPOINT and FOUNDRY_VOICE_LIVE_MODEL_NAME. +The latter is the existing GPT Live deployment name, not a model catalog ID. +The outer fixture preparer creates owned stored sessions outside HTTP recording; +playback performs REST calls only. These cases remain enabled without recordings +so missing Test Proxy assets are visible rather than hidden by a skip marker. +""" + +import pytest + +from azure.ai.projects import models +from azure.core.exceptions import ResourceNotFoundError +from devtools_testutils import is_live, recorded_by_proxy +from test_base import TestBase +from live_conversation_test_base import ( + LiveConversationPreparer, + add_live_conversation_sanitizers, + assert_audio_metadata, + assert_completed_conversation, + assert_live_status, + assert_speech_item, + assert_wav, + liveConversationServicePreparer, +) + + +class TestVoiceAgentLiveConversations(TestBase): + @liveConversationServicePreparer() + @LiveConversationPreparer() + @recorded_by_proxy() + def test_live_conversation_rest_lifecycle(self, live_conversation_fixture, **kwargs): + self._test_live_conversation_rest_lifecycle(live_conversation_fixture, **kwargs) + + def _test_live_conversation_rest_lifecycle(self, live_conversation_fixture, **kwargs): + fixture = live_conversation_fixture + if is_live(): + add_live_conversation_sanitizers(fixture) + with self.create_client(allow_preview=True, **kwargs) as client: + conversations = client.beta.voice_agents.live_conversations + agent_name, conversation_id = fixture.agent_name, fixture.conversation_ids[0] + listed = list( + conversations.list(agent_name, limit=1, order="asc", raw_response_hook=assert_live_status(200)) + ) + assert [value.id for value in listed] == list(fixture.conversation_ids) + for value in listed: + assert_completed_conversation(value, value.id) + detail = conversations.get(agent_name, conversation_id, raw_response_hook=assert_live_status(200)) + assert_completed_conversation(detail, conversation_id) + + items = list( + conversations.list_items( + agent_name, conversation_id, limit=1, order="asc", raw_response_hook=assert_live_status(200) + ) + ) + assert items and len({item.id for item in items}) == len(items) + assert all(isinstance(item, models.VoiceAgentLiveConversationItem) for item in items) + speech = [item for item in items if isinstance(item, models.VoiceAgentLiveOutputConversationItem)] + assert speech, "Stored greeting speech must exercise item audio, not just metadata/list routes" + for item in speech: + assert_speech_item(item) + fetched = conversations.get_item( + agent_name, conversation_id, item.id, raw_response_hook=assert_live_status(200) + ) + fetched = assert_speech_item(fetched, item.id) + assert fetched.transcription == item.transcription + audio = conversations.get_audio_item( + agent_name, conversation_id, item.id, raw_response_hook=assert_live_status(200) + ) + assert_audio_metadata(audio, conversation_id, item_id=item.id) + assert_wav( + b"".join( + conversations.download_audio_item( + agent_name, conversation_id, item.id, raw_response_hook=assert_live_status(200) + ) + ), + audio, + ) + recording = conversations.get_audio(agent_name, conversation_id, raw_response_hook=assert_live_status(200)) + assert_audio_metadata(recording, conversation_id) + assert_wav( + b"".join( + conversations.download_audio(agent_name, conversation_id, raw_response_hook=assert_live_status(200)) + ), + recording, + ) + + conversations.delete(agent_name, conversation_id, raw_response_hook=assert_live_status(204)) + with pytest.raises(ResourceNotFoundError): + conversations.get(agent_name, conversation_id, raw_response_hook=assert_live_status(404)) + with pytest.raises(ResourceNotFoundError): + conversations.get_item( + agent_name, conversation_id, speech[0].id, raw_response_hook=assert_live_status(404) + ) + with pytest.raises(ResourceNotFoundError): + conversations.get_audio(agent_name, conversation_id, raw_response_hook=assert_live_status(404)) + with pytest.raises(ResourceNotFoundError): + conversations.get_audio_item( + agent_name, conversation_id, speech[0].id, raw_response_hook=assert_live_status(404) + ) + with pytest.raises(ResourceNotFoundError): + list( + conversations.download_audio(agent_name, conversation_id, raw_response_hook=assert_live_status(404)) + ) + with pytest.raises(ResourceNotFoundError): + list( + conversations.download_audio_item( + agent_name, conversation_id, speech[0].id, raw_response_hook=assert_live_status(404) + ) + ) + remaining = list( + conversations.list(agent_name, limit=1, order="asc", raw_response_hook=assert_live_status(200)) + ) + assert [value.id for value in remaining] == list(fixture.conversation_ids[1:]) diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_live_conversations_async.py b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_live_conversations_async.py new file mode 100644 index 000000000000..1c83a11a01a7 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_voice_agent_live_conversations_async.py @@ -0,0 +1,113 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +"""Async Test Proxy coverage of the same GPT Live REST lifecycle as the sync test.""" + +import pytest + +from azure.ai.projects import models +from azure.core.exceptions import ResourceNotFoundError +from devtools_testutils import is_live +from devtools_testutils.aio import recorded_by_proxy_async +from test_base import TestBase +from live_conversation_test_base import ( + LiveConversationPreparer, + add_live_conversation_sanitizers, + assert_audio_metadata, + assert_completed_conversation, + assert_live_status, + assert_speech_item, + assert_wav, + liveConversationServicePreparer, +) + + +class TestVoiceAgentLiveConversationsAsync(TestBase): + @liveConversationServicePreparer() + @LiveConversationPreparer() + @recorded_by_proxy_async() + async def test_live_conversation_rest_lifecycle_async(self, live_conversation_fixture, **kwargs): + await self._test_live_conversation_rest_lifecycle_async(live_conversation_fixture, **kwargs) + + async def _test_live_conversation_rest_lifecycle_async(self, live_conversation_fixture, **kwargs): + fixture = live_conversation_fixture + if is_live(): + add_live_conversation_sanitizers(fixture) + async with self.create_async_client(allow_preview=True, **kwargs) as client: + conversations = client.beta.voice_agents.live_conversations + agent_name, conversation_id = fixture.agent_name, fixture.conversation_ids[0] + listed = [ + value + async for value in conversations.list( + agent_name, limit=1, order="asc", raw_response_hook=assert_live_status(200) + ) + ] + assert [value.id for value in listed] == list(fixture.conversation_ids) + for value in listed: + assert_completed_conversation(value, value.id) + detail = await conversations.get(agent_name, conversation_id, raw_response_hook=assert_live_status(200)) + assert_completed_conversation(detail, conversation_id) + + items = [ + item + async for item in conversations.list_items( + agent_name, conversation_id, limit=1, order="asc", raw_response_hook=assert_live_status(200) + ) + ] + assert items and len({item.id for item in items}) == len(items) + assert all(isinstance(item, models.VoiceAgentLiveConversationItem) for item in items) + speech = [item for item in items if isinstance(item, models.VoiceAgentLiveOutputConversationItem)] + assert speech, "Stored greeting speech must exercise item audio, not just metadata/list routes" + for item in speech: + assert_speech_item(item) + fetched = await conversations.get_item( + agent_name, conversation_id, item.id, raw_response_hook=assert_live_status(200) + ) + fetched = assert_speech_item(fetched, item.id) + assert fetched.transcription == item.transcription + audio = await conversations.get_audio_item( + agent_name, conversation_id, item.id, raw_response_hook=assert_live_status(200) + ) + assert_audio_metadata(audio, conversation_id, item_id=item.id) + stream = await conversations.download_audio_item( + agent_name, conversation_id, item.id, raw_response_hook=assert_live_status(200) + ) + assert_wav(b"".join([chunk async for chunk in stream]), audio) + recording = await conversations.get_audio( + agent_name, conversation_id, raw_response_hook=assert_live_status(200) + ) + assert_audio_metadata(recording, conversation_id) + stream = await conversations.download_audio( + agent_name, conversation_id, raw_response_hook=assert_live_status(200) + ) + assert_wav(b"".join([chunk async for chunk in stream]), recording) + + await conversations.delete(agent_name, conversation_id, raw_response_hook=assert_live_status(204)) + with pytest.raises(ResourceNotFoundError): + await conversations.get(agent_name, conversation_id, raw_response_hook=assert_live_status(404)) + with pytest.raises(ResourceNotFoundError): + await conversations.get_item( + agent_name, conversation_id, speech[0].id, raw_response_hook=assert_live_status(404) + ) + with pytest.raises(ResourceNotFoundError): + await conversations.get_audio(agent_name, conversation_id, raw_response_hook=assert_live_status(404)) + with pytest.raises(ResourceNotFoundError): + await conversations.get_audio_item( + agent_name, conversation_id, speech[0].id, raw_response_hook=assert_live_status(404) + ) + with pytest.raises(ResourceNotFoundError): + await conversations.download_audio( + agent_name, conversation_id, raw_response_hook=assert_live_status(404) + ) + with pytest.raises(ResourceNotFoundError): + await conversations.download_audio_item( + agent_name, conversation_id, speech[0].id, raw_response_hook=assert_live_status(404) + ) + remaining = [ + value + async for value in conversations.list( + agent_name, limit=1, order="asc", raw_response_hook=assert_live_status(200) + ) + ] + assert [value.id for value in remaining] == list(fixture.conversation_ids[1:]) From 0fe197c313188302731057b882836485eea56d21 Mon Sep 17 00:00:00 2001 From: Xiting Zhang Date: Mon, 26 Oct 2026 16:40:28 -0700 Subject: [PATCH 07/10] Align GPT Live samples with review conventions Consolidate Live samples, remove CLI arguments, use user-temporary audio output, and cover resource ownership and sample conventions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/ai/azure-ai-projects/CHANGELOG.md | 7 +- sdk/ai/azure-ai-projects/README.md | 51 ++- ...ice_agent_live_audio_conversation_async.py | 60 +-- ...mple_voice_agent_live_read_conversation.py | 73 +-- ...oice_agent_live_read_conversation_async.py | 193 -------- ...mple_voice_agent_live_text_conversation.py | 19 +- ...oice_agent_live_text_conversation_async.py | 120 ----- .../tests/samples/test_live_samples.py | 433 ++++++++++-------- 8 files changed, 325 insertions(+), 631 deletions(-) delete mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation_async.py delete mode 100644 sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py diff --git a/sdk/ai/azure-ai-projects/CHANGELOG.md b/sdk/ai/azure-ai-projects/CHANGELOG.md index db9561a20cc1..6e66e2c6e95b 100644 --- a/sdk/ai/azure-ai-projects/CHANGELOG.md +++ b/sdk/ai/azure-ai-projects/CHANGELOG.md @@ -18,12 +18,7 @@ ### Sample updates -* Added GPT Live connection, text conversation, asynchronous PCM audio streaming, and function-tool examples under `samples/agents/voice`. -* Updated GPT Live text examples with synchronous/asynchronous headless multi-turn conversations that reuse one session and preserve the default single-turn example. -* Added synchronous/asynchronous GPT Live REST examples for reading an existing stored conversation, its items, and audio metadata, with explicit opt-in recording downloads and conversation deletion. -* Fixed the GPT Live WAV sample to ignore silent/very-low-level PCM when detecting reply inactivity, reject silence-only replies, and preserve captured PCM in audio counts and optional WAV output. -* Protected the GPT Live WAV sample's input recording from being overwritten through hard-linked output paths. -* Added `--microphone` to the existing GPT Live audio sample for nonblocking microphone capture and speaker playback with optional PyAudio, preserving the WAV workflow and reporting actual audio transfer sizes. +* Added GPT Live samples under `samples/agents/voice` demonstrating session lifecycle and configuration, single- and multi-turn text conversations, function-tool calling, WAV audio streaming, microphone capture with speaker playback, and stored-conversation/item/audio retrieval with Foundry-managed WAV downloads to a unique user-temporary folder. ## 2.9.0 (2026-10-16) diff --git a/sdk/ai/azure-ai-projects/README.md b/sdk/ai/azure-ai-projects/README.md index 30d475abe9e9..e92a3dd9287f 100644 --- a/sdk/ai/azure-ai-projects/README.md +++ b/sdk/ai/azure-ai-projects/README.md @@ -253,50 +253,59 @@ limitation does not affect WebSocket sessions. Connection, text, PCM audio-streaming, and function-tool examples are in `samples/agents/voice` in the [Samples][samples] folder. The optional `[voice]` dependencies are unchanged. -The synchronous and asynchronous text examples accept quoted prompts for a headless multi-turn -conversation in one session. Omit prompts to run the original single-turn example: +The [text sample](samples/agents/voice/sample_voice_agent_live_text_conversation.py) runs a +single-turn conversation by default. To demonstrate headless multi-turn conversation in one +session, set `TEXT_PROMPTS = ["My name is Ada.", "What is my name?"]` in the sample, then run: ```powershell -python .\samples\agents\voice\sample_voice_agent_live_text_conversation.py "My name is Ada." "What is my name?" -python .\samples\agents\voice\sample_voice_agent_live_text_conversation_async.py "My name is Ada." "What is my name?" +python .\samples\agents\voice\sample_voice_agent_live_text_conversation.py ``` -These text examples print Responses backend text, not completion of the voice frontend's speech. +The text sample prints Responses backend text, not completion of the voice frontend's speech. The WAV audio example uses a bounded speech-inactivity heuristic because Live has no speech-finished event. Continued silent PCM does not keep it waiting forever, and silence-only output is not a successful spoken reply. A long speech pause can still split a reply. The same [GPT Live audio sample](samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py) also supports live microphone capture and speaker playback, following the Realtime sample's -nonblocking PyAudio device pattern: +nonblocking PyAudio device pattern. This is the only asynchronous Live sample because sending +and receiving audio concurrently and handling audio devices benefit from async execution. +Set `USE_MICROPHONE = True` in the sample before running microphone mode: ```powershell pip install pyaudio -python .\samples\agents\voice\sample_voice_agent_live_audio_conversation_async.py --microphone +python .\samples\agents\voice\sample_voice_agent_live_audio_conversation_async.py ``` Microphone mode requires `FOUNDRY_PROJECT_ENDPOINT` and `FOUNDRY_VOICE_AGENT_MODEL`, but not `FOUNDRY_VOICE_AUDIO_INPUT_PATH`. Speak normally after the sample prints "Speak now"; use headphones to avoid feedback, and press Ctrl-C to close the session and delete its temporary agent version. The sample reports actual input/output PCM bytes, durations, and any dropped -capture frames. No audio file is saved unless `FOUNDRY_VOICE_AUDIO_OUTPUT_PATH` is explicitly -set; that file contains the received reply, not the microphone input. Conversation persistence -remains disabled. Live owns turn handling, so this mode does not use Realtime `response.cancel`. -Omit `--microphone` to retain the original WAV workflow without requiring PyAudio. - -The [synchronous REST sample](samples/agents/voice/sample_voice_agent_live_read_conversation.py) -and [asynchronous REST sample](samples/agents/voice/sample_voice_agent_live_read_conversation_async.py) -read an existing stored Live conversation without opening a WebSocket. Set `FOUNDRY_PROJECT_ENDPOINT`, +capture frames. No audio file is saved unless `SAVE_AUDIO = True` is set in the sample. Saved +reply audio goes into a unique user-temporary folder whose location is printed; it remains +available after exit, so remove the folder when it is no longer needed. The WAV contains the +received reply, not microphone input or telemetry. Conversation persistence remains disabled. +Live owns turn handling, so this mode does not use Realtime `response.cancel`. +Leave `USE_MICROPHONE = False` to use the WAV workflow without requiring PyAudio. + +The Live samples do not accept command-line arguments. Connection/session controls, text, +function calling, and stored-conversation REST each have one synchronous example; audio has +the single asynchronous example described above. Samples delete only the agent version they +create; the stored-conversation REST sample does not create or delete service resources. + +The [REST sample](samples/agents/voice/sample_voice_agent_live_read_conversation.py) +reads an existing stored Live conversation without opening a WebSocket. Set `FOUNDRY_PROJECT_ENDPOINT`, `FOUNDRY_VOICE_AGENT_NAME`, and `FOUNDRY_VOICE_CONVERSATION_ID`. The last value must be `session.started.conversation_id` from a session with effective `store=True`, not `session.id`. The session must have finished successfully and contain recorded speech. -Both samples demonstrate conversation/item listing and retrieval, plus merged and per-item audio -metadata. Pass `--output-dir ` to download Foundry-managed WAV audio without overwriting -existing files. Downloads save conversation content locally, not telemetry. BYOS audio requires your -own storage credentials. Nothing is deleted unless `--delete` is explicitly supplied; that option -deletes the selected conversation and its stored items/audio only after all requested reads succeed. -These samples use the dedicated `.beta.voice_agents.live_conversations` routes. A deployment that +The sample demonstrates conversation/item listing and retrieval, audio metadata, and merged/per-item +Foundry-managed WAV downloads. It creates a unique folder in your user temporary directory and prints +the location. Files remain available after the sample exits; remove the local folder when it is no +longer needed. These downloads contain conversation content, not telemetry. BYOS audio requires your +own storage credentials and is not downloaded by this sample. The sample never deletes the existing +agent or stored conversation, items, or audio, and it does not accept command-line options. +It uses the dedicated `.beta.voice_agents.live_conversations` routes. A deployment that does not support those routes returns HTTP 404, which is surfaced rather than silently falling back to `.beta.voice_agents.conversations`. diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py index 8ad4e97da032..04a98837c1d8 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py @@ -5,9 +5,11 @@ """ DESCRIPTION: - Talk to a preview GPT Live voice agent using --microphone, or stream a real - WAV recording without that flag. Both modes use PCM16 little-endian, mono, + Talk to a preview GPT Live voice agent using a microphone, or stream a real + WAV recording. Both modes use PCM16 little-endian, mono, 24 kHz audio and send/receive concurrently. + Async is used for full-duplex streaming and nonblocking audio devices, + rather than duplicating a synchronous sample. Microphone mode uses optional PyAudio capture and speaker callbacks, following the Realtime audio sample's nonblocking device pattern. Speak @@ -33,32 +35,32 @@ USAGE: python sample_voice_agent_live_audio_conversation_async.py - python sample_voice_agent_live_audio_conversation_async.py --microphone + + Set USE_MICROPHONE = True in this file for microphone capture and speaker + playback. Leave it False to stream a WAV file without audio devices. + Set SAVE_AUDIO = True to save returned audio in a unique user-temporary + folder. Its location is printed and the files remain after exit; remove + the folder when it is no longer needed. pip install "azure-ai-projects[voice]" azure-identity python-dotenv - pip install pyaudio # Only required for --microphone. + pip install pyaudio # Only required for microphone mode. Environment variables: 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live DEPLOYMENT name. 3) FOUNDRY_VOICE_AUDIO_INPUT_PATH (required for WAV mode only) - Path to an actual spoken-audio WAV file: uncompressed PCM16, mono, 24000 Hz. - 4) FOUNDRY_VOICE_AUDIO_OUTPUT_PATH (optional) - Explicit WAV output path. - When set, saves actual returned audio, not telemetry. Without it no audio - file is written. The output must not refer to the input file, including - through a symbolic link or hard link. - 5) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for a uniquely named sample + 4) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for a uniquely named sample agent. Only the version created by this run is deleted. Authenticates with DefaultAzureCredential. Choose a short recording; Ctrl-C cancels local tasks and closes the session. Conversation persistence is disabled; explicitly saving audio still writes - conversation content to your chosen local file. + conversation content locally. SAVE_AUDIO is False by default. """ from __future__ import annotations -import argparse import asyncio import base64 import concurrent.futures @@ -67,6 +69,7 @@ import os from pathlib import Path import queue +from tempfile import mkdtemp import time from typing import Any, Optional, TYPE_CHECKING import wave @@ -91,6 +94,9 @@ from azure.ai.projects.aio.operations import AsyncBetaLiveConnection +USE_MICROPHONE = False +SAVE_AUDIO = False + _SAMPLE_RATE = 24000 _CHUNK_FRAMES = 1200 _OUTPUT_IDLE_SECONDS = 3.0 @@ -449,25 +455,26 @@ async def _run_audio_conversation( raise asyncio.CancelledError -async def main(*, microphone: bool = False) -> None: +async def main() -> None: load_dotenv() endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] model = os.environ["FOUNDRY_VOICE_AGENT_MODEL"] - backend = _microphone_backend() if microphone else None - input_path = None if microphone else Path(os.environ["FOUNDRY_VOICE_AUDIO_INPUT_PATH"]) - output_path = os.environ.get("FOUNDRY_VOICE_AUDIO_OUTPUT_PATH") - if output_path and input_path is not None: - output = Path(output_path) - if output.resolve() == input_path.resolve() or (output.exists() and output.samefile(input_path)): - raise ValueError("The audio output path must not overwrite the input recording.") + backend = _microphone_backend() if USE_MICROPHONE else None + input_path = None if USE_MICROPHONE else Path(os.environ["FOUNDRY_VOICE_AUDIO_INPUT_PATH"]) + output_path: Optional[Path] = None agent_name = sample_agent_name("audio") with ExitStack() as files: recording = files.enter_context(wave.open(str(input_path), "rb")) if input_path is not None else None if recording is not None: _validate_recording(recording) - output_file = files.enter_context(wave.open(output_path, "wb")) if output_path else None - if output_file is not None: + output_file: Optional[wave.Wave_write] = None + if SAVE_AUDIO: + output_directory = Path(mkdtemp(prefix="voice-agent-live-audio-")) + output_path = output_directory / "reply.wav" + print(f"Audio output directory: {output_directory}") + output_stream = files.enter_context(output_path.open("xb")) + output_file = files.enter_context(wave.open(output_stream, "wb")) output_file.setnchannels(1) output_file.setsampwidth(2) output_file.setframerate(_SAMPLE_RATE) @@ -490,7 +497,7 @@ async def main(*, microphone: bool = False) -> None: store=False, ), ) - if microphone: + if USE_MICROPHONE: await _run_audio_conversation( project_client, agent_name, created_version.version, None, output_file, audio_backend=backend ) @@ -505,16 +512,11 @@ async def main(*, microphone: bool = False) -> None: ) print(f"Deleted sample agent version: {created_version.version}") if output_path: - print("Saved actual returned PCM16 WAV audio to FOUNDRY_VOICE_AUDIO_OUTPUT_PATH (not telemetry).") + print(f"Saved actual returned PCM16 WAV audio: {output_path} (not telemetry).") if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Talk to GPT Live using a WAV file or your microphone.") - parser.add_argument( - "--microphone", action="store_true", help="Stream the default microphone and play replies using PyAudio." - ) - arguments = parser.parse_args() try: - asyncio.run(main(microphone=arguments.microphone)) + asyncio.run(main()) except KeyboardInterrupt: print("\nStopped.") diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py index 07afdaf239fb..69a6c3952dcc 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py @@ -15,17 +15,15 @@ routes must be available on the service deployment; HTTP 404 is reported and raised, never replaced by calls to the older Realtime conversation API. - By default, no audio files are written and nothing is deleted. Pass - --output-dir to download Foundry-managed WAV recordings without overwriting - existing files. BYOS recordings must be downloaded with your own storage - credentials. Pass --delete only for a conversation you intend to remove: - deletion also removes its stored items and audio. A read/download failure - prevents deletion. + Download Foundry-managed WAV recordings into a unique folder in your user + temporary directory and print its location. The files remain available + after the sample exits and are not overwritten. Downloads contain stored + conversation content; remove the local folder when it is no longer needed. + BYOS recordings require your own storage credentials and are not downloaded. + The existing agent, conversation, items, and stored audio are never deleted. USAGE: python sample_voice_agent_live_read_conversation.py - python sample_voice_agent_live_read_conversation.py --output-dir live-recordings - python sample_voice_agent_live_read_conversation.py --delete pip install azure-ai-projects azure-identity python-dotenv @@ -40,16 +38,15 @@ from __future__ import annotations -import argparse from itertools import islice import os from pathlib import Path -from typing import Optional +from tempfile import mkdtemp from dotenv import load_dotenv from live_sample_utils import recording_output, safe_print -from azure.core.exceptions import HttpResponseError, ResourceNotFoundError +from azure.core.exceptions import HttpResponseError from azure.identity import DefaultAzureCredential from azure.ai.projects import AIProjectClient, models from azure.ai.projects.operations import BetaVoiceAgentsLiveConversationsOperations @@ -61,7 +58,7 @@ def _read_item( conversation_id: str, item_id: str, index: int, - output_directory: Optional[Path], + output_directory: Path, ) -> None: detail = conversations.get_item(agent_name, conversation_id, item_id) print(f"Item {detail.id}: type={detail.type}") @@ -78,21 +75,16 @@ def _read_item( print(f" Audio: role={audio.role}, sample_rate={audio.sample_rate}, duration={audio.duration_ms}") if audio.blob_uri: print(" BYOS item: download from your storage using your own credentials.") - if output_directory is not None: - raise RuntimeError("This sample downloads Foundry-managed audio only, not BYOS audio.") - elif output_directory is not None: - with recording_output(output_directory / f"item-{index:04d}.wav") as output: - for chunk in conversations.download_audio_item(agent_name, conversation_id, item_id): - output.write(chunk) + raise RuntimeError("This sample downloads Foundry-managed audio only, not BYOS audio.") + with recording_output(output_directory / f"item-{index:04d}.wav") as output: + for chunk in conversations.download_audio_item(agent_name, conversation_id, item_id): + output.write(chunk) def _read_conversation( conversations: BetaVoiceAgentsLiveConversationsOperations, agent_name: str, conversation_id: str, - *, - output_directory: Optional[Path], - delete: bool, ) -> None: conversation = conversations.get(agent_name, conversation_id) if conversation.status != "completed": @@ -106,8 +98,8 @@ def _read_conversation( for recent in islice(conversations.list(agent_name, limit=5, order="desc"), 5): print(f" {recent.id}: status={recent.status}") - if output_directory is not None: - output_directory.mkdir(parents=True, exist_ok=True) + output_directory = Path(mkdtemp(prefix="voice-agent-live-")) + print(f"Audio output directory: {output_directory}") seen_items: set[str] = set() for index, item in enumerate(conversations.list_items(agent_name, conversation_id, limit=20, order="asc"), 1): @@ -123,26 +115,15 @@ def _read_conversation( ) if recording.blob_uri: print("BYOS recording: download from your storage using your own credentials.") - if output_directory is not None: - raise RuntimeError("This sample downloads Foundry-managed audio only, not BYOS audio.") - elif output_directory is not None: - with recording_output(output_directory / "conversation.wav") as output: - for chunk in conversations.download_audio(agent_name, conversation_id): - output.write(chunk) - - if delete: - conversations.delete(agent_name, conversation_id) - try: - conversations.get(agent_name, conversation_id) - except ResourceNotFoundError: - print(f"Deleted conversation {conversation_id}; confirmed HTTP 404.") - else: - raise RuntimeError("The conversation remains readable after deletion.") - else: - print("Read-only run: the conversation, items, and audio were not deleted.") + raise RuntimeError("This sample downloads Foundry-managed audio only, not BYOS audio.") + with recording_output(output_directory / "conversation.wav") as output: + for chunk in conversations.download_audio(agent_name, conversation_id): + output.write(chunk) + + print("Read-only REST run: the existing agent, conversation, items, and stored audio were not modified.") -def main(*, output_directory: Optional[Path] = None, delete: bool = False) -> None: +def main() -> None: load_dotenv() endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] agent_name = os.environ["FOUNDRY_VOICE_AGENT_NAME"] @@ -159,8 +140,6 @@ def main(*, output_directory: Optional[Path] = None, delete: bool = False) -> No project_client.beta.voice_agents.live_conversations, agent_name, conversation_id, - output_directory=output_directory, - delete=delete, ) except HttpResponseError as exc: print(f"GPT Live conversation REST request failed: HTTP {exc.status_code}.") @@ -175,10 +154,4 @@ def main(*, output_directory: Optional[Path] = None, delete: bool = False) -> No if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Read a stored GPT Live conversation through the dedicated REST API.") - parser.add_argument( - "--output-dir", type=Path, help="Explicitly save audio to this directory; files are not overwritten." - ) - parser.add_argument("--delete", action="store_true", help="Delete the selected conversation, its items, and audio.") - options = parser.parse_args() - main(output_directory=options.output_dir, delete=options.delete) + main() diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation_async.py deleted file mode 100644 index f4bc9e1c14dc..000000000000 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation_async.py +++ /dev/null @@ -1,193 +0,0 @@ -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ - -""" -DESCRIPTION: - Read a stored GPT Live conversation using the asynchronous preview - project_client.beta.voice_agents.live_conversations REST APIs. Retrieve the - conversation, recent conversations, ordered items, individual items, and - merged/per-item audio metadata. No agent or WebSocket session is created. - - Requires an ended speech session with effective store=True. Use - session.started.conversation_id, not session.id. These dedicated Live REST - routes must be available on the service deployment; HTTP 404 is reported - and raised, never replaced by calls to the older Realtime conversation API. - - By default, no audio files are written and nothing is deleted. Pass - --output-dir to download Foundry-managed WAV recordings without overwriting - existing files. BYOS recordings must be downloaded with your own storage - credentials. Pass --delete only for a conversation you intend to remove: - deletion also removes its stored items and audio. A read/download failure - prevents deletion. - -USAGE: - python sample_voice_agent_live_read_conversation_async.py - python sample_voice_agent_live_read_conversation_async.py --output-dir live-recordings - python sample_voice_agent_live_read_conversation_async.py --delete - - pip install azure-ai-projects azure-identity python-dotenv aiohttp - - Environment variables (all required): - 1) FOUNDRY_PROJECT_ENDPOINT - The Foundry project endpoint. - 2) FOUNDRY_VOICE_AGENT_NAME - The existing GPT Live agent name. - 3) FOUNDRY_VOICE_CONVERSATION_ID - The stored Foundry conversation ID. - - Authenticates with DefaultAzureCredential. No model deployment name, - microphone, or WebSocket dependency is needed. -""" - -from __future__ import annotations - -import argparse -import asyncio -import os -from pathlib import Path -from typing import Optional - -from dotenv import load_dotenv -from live_sample_utils import recording_output, safe_print - -from azure.core.exceptions import HttpResponseError, ResourceNotFoundError -from azure.identity.aio import DefaultAzureCredential -from azure.ai.projects import models -from azure.ai.projects.aio import AIProjectClient -from azure.ai.projects.aio.operations import BetaVoiceAgentsLiveConversationsOperations - - -async def _read_item( - conversations: BetaVoiceAgentsLiveConversationsOperations, - agent_name: str, - conversation_id: str, - item_id: str, - index: int, - output_directory: Optional[Path], -) -> None: - detail = await conversations.get_item(agent_name, conversation_id, item_id) - print(f"Item {detail.id}: type={detail.type}") - if not isinstance( - detail, (models.VoiceAgentLiveInputConversationItem, models.VoiceAgentLiveOutputConversationItem) - ): - print(" Non-speech item; there is no speech audio to download.") - return - safe_print(f" Transcript: {detail.transcription or '(not captured)'}") - if detail.audio is None: - print(" No captured audio metadata for this speech item.") - return - audio = await conversations.get_audio_item(agent_name, conversation_id, item_id) - print(f" Audio: role={audio.role}, sample_rate={audio.sample_rate}, duration={audio.duration_ms}") - if audio.blob_uri: - print(" BYOS item: download from your storage using your own credentials.") - if output_directory is not None: - raise RuntimeError("This sample downloads Foundry-managed audio only, not BYOS audio.") - elif output_directory is not None: - with recording_output(output_directory / f"item-{index:04d}.wav") as output: - stream = await conversations.download_audio_item(agent_name, conversation_id, item_id) - async for chunk in stream: - output.write(chunk) - - -async def _read_conversation( - conversations: BetaVoiceAgentsLiveConversationsOperations, - agent_name: str, - conversation_id: str, - *, - output_directory: Optional[Path], - delete: bool, -) -> None: - conversation = await conversations.get(agent_name, conversation_id) - if conversation.status != "completed": - raise RuntimeError( - f"The stored session must finish successfully before reading its recording; " - f"status={conversation.status}, error={conversation.last_error}" - ) - print(f"Conversation {conversation.id}: status={conversation.status}, created_at={conversation.created_at}") - - print("Recent stored Live conversations (up to five):") - count = 0 - async for recent in conversations.list(agent_name, limit=5, order="desc"): - print(f" {recent.id}: status={recent.status}") - count += 1 - if count == 5: - break - - if output_directory is not None: - output_directory.mkdir(parents=True, exist_ok=True) - - seen_items: set[str] = set() - index = 0 - async for item in conversations.list_items(agent_name, conversation_id, limit=20, order="asc"): - index += 1 - if item.id in seen_items: - raise RuntimeError("The conversation pager repeated an item instead of advancing.") - seen_items.add(item.id) - await _read_item(conversations, agent_name, conversation_id, item.id, index, output_directory) - - recording = await conversations.get_audio(agent_name, conversation_id) - print( - f"Merged recording: format={recording.format}, sample_rate={recording.sample_rate}, " - f"channels={recording.channels}, duration={recording.duration_ms}" - ) - if recording.blob_uri: - print("BYOS recording: download from your storage using your own credentials.") - if output_directory is not None: - raise RuntimeError("This sample downloads Foundry-managed audio only, not BYOS audio.") - elif output_directory is not None: - with recording_output(output_directory / "conversation.wav") as output: - stream = await conversations.download_audio(agent_name, conversation_id) - async for chunk in stream: - output.write(chunk) - - if delete: - await conversations.delete(agent_name, conversation_id) - try: - await conversations.get(agent_name, conversation_id) - except ResourceNotFoundError: - print(f"Deleted conversation {conversation_id}; confirmed HTTP 404.") - else: - raise RuntimeError("The conversation remains readable after deletion.") - else: - print("Read-only run: the conversation, items, and audio were not deleted.") - - -async def main(*, output_directory: Optional[Path] = None, delete: bool = False) -> None: - load_dotenv() - endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] - agent_name = os.environ["FOUNDRY_VOICE_AGENT_NAME"] - conversation_id = os.environ["FOUNDRY_VOICE_CONVERSATION_ID"] - if not agent_name.strip() or not conversation_id.strip(): - raise ValueError("Provide a nonblank existing agent name and stored Foundry conversation ID.") - - async with ( - DefaultAzureCredential() as credential, - AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, - ): - try: - await _read_conversation( - project_client.beta.voice_agents.live_conversations, - agent_name, - conversation_id, - output_directory=output_directory, - delete=delete, - ) - except HttpResponseError as exc: - print(f"GPT Live conversation REST request failed: HTTP {exc.status_code}.") - if exc.status_code == 404: - print( - "Confirm the stored conversation ID and store=True, and that this deployment supports " - "/protocols/voice/live/conversations. No older-route fallback is used." - ) - elif exc.status_code == 409: - print("Recording finalization may still be pending or unavailable; inspect the service error code.") - raise - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Read a stored GPT Live conversation through the dedicated REST API.") - parser.add_argument( - "--output-dir", type=Path, help="Explicitly save audio to this directory; files are not overwritten." - ) - parser.add_argument("--delete", action="store_true", help="Delete the selected conversation, its items, and audio.") - options = parser.parse_args() - asyncio.run(main(output_directory=options.output_dir, delete=options.delete)) diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py index 905d77191541..09c6736661fc 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py @@ -12,10 +12,10 @@ USAGE: python sample_voice_agent_live_text_conversation.py - python sample_voice_agent_live_text_conversation.py "My name is Ada." "What is my name?" - Omit prompts to run the original single-turn example. Quote each positional - prompt to run a scripted, headless multi-turn conversation. + Leave TEXT_PROMPTS as None for the single-turn example. To demonstrate + a headless multi-turn conversation, set TEXT_PROMPTS in this file to + ["My name is Ada.", "What is my name?"] before running it. pip install "azure-ai-projects[voice]" azure-identity python-dotenv @@ -33,7 +33,6 @@ from __future__ import annotations -import argparse import os from typing import Optional, Sequence @@ -52,6 +51,9 @@ ) +TEXT_PROMPTS: Optional[Sequence[str]] = None + + def _run_text_conversation( project_client: AIProjectClient, agent_name: str, agent_version: str, prompts: Sequence[str] ) -> None: @@ -76,8 +78,8 @@ def _run_text_conversation( close_session(connection) -def main(prompts: Optional[Sequence[str]] = None) -> None: - prompts = text_conversation_prompts(prompts) +def main() -> None: + prompts = text_conversation_prompts(TEXT_PROMPTS) load_dotenv() endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] model = os.environ["FOUNDRY_VOICE_AGENT_MODEL"] @@ -112,7 +114,4 @@ def main(prompts: Optional[Sequence[str]] = None) -> None: if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Run a headless GPT Live text conversation.") - parser.add_argument("prompts", nargs="*", help="Quoted text prompts to send sequentially in one session.") - arguments = parser.parse_args() - main(arguments.prompts or None) + main() diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py deleted file mode 100644 index cdf54c39c807..000000000000 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation_async.py +++ /dev/null @@ -1,120 +0,0 @@ -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ - -""" -DESCRIPTION: - Run a headless, asynchronous GPT Live text conversation using Responses - delegation. Send multiple prompts sequentially over the same session, - waiting for nested response.completed before beginning the next turn. - Backend text is printed; it does not signal completion of frontend speech. - -USAGE: - python sample_voice_agent_live_text_conversation_async.py - python sample_voice_agent_live_text_conversation_async.py "My name is Ada." "What is my name?" - - Omit prompts for the same single-turn example as the synchronous sample. - - pip install "azure-ai-projects[voice]" azure-identity python-dotenv - - Environment variables: - 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. - 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live DEPLOYMENT name. - 3) FOUNDRY_MODEL_NAME (required) - A Responses backend DEPLOYMENT name - available to the Live resource, not a model catalog identifier. - 4) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for a uniquely named sample - agent. Only the version created by this run is deleted. - - Authenticates with DefaultAzureCredential. No microphone, speaker, or - audio-device dependency is needed. Ctrl-C cancels local work and closes - the session; Live has no response.cancel command. -""" - -from __future__ import annotations - -import argparse -import asyncio -import os -from typing import Optional, Sequence - -from dotenv import load_dotenv -from azure.identity.aio import DefaultAzureCredential -from azure.ai.projects.aio import AIProjectClient -from azure.ai.projects import models - -from live_sample_utils import ( - audio_settings, - close_session_async, - receive_response_async, - safe_print, - sample_agent_name, - text_conversation_prompts, - wait_for_event_async, -) - - -async def _run_text_conversation( - project_client: AIProjectClient, agent_name: str, agent_version: str, prompts: Sequence[str] -) -> None: - async with project_client.beta.voice_agents.live.connect( - agent_name=agent_name, agent_version=agent_version, store=False - ) as connection: - try: - await wait_for_event_async(connection, "session.started") - for prompt in prompts: - safe_print(f"You: {prompt}") - await connection.response.item.create( - item={"type": "message", "role": "user", "content": [{"type": "input_text", "text": prompt}]} - ) - await connection.response.create() - text, calls = await receive_response_async(connection) - if calls: - raise RuntimeError("The text-only backend unexpectedly requested a function tool.") - if not text: - raise RuntimeError("The Responses backend completed without returning any output text.") - safe_print(f"Responses backend: {text}") - finally: - await close_session_async(connection) - - -async def main(prompts: Optional[Sequence[str]] = None) -> None: - prompts = text_conversation_prompts(prompts) - load_dotenv() - endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] - model = os.environ["FOUNDRY_VOICE_AGENT_MODEL"] - backend_model = os.environ["FOUNDRY_MODEL_NAME"] - agent_name = sample_agent_name("text") - - async with ( - DefaultAzureCredential() as credential, - AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, - ): - created_version = None - try: - created_version = await project_client.agents.create_version( - agent_name=agent_name, - definition=models.VoiceAgentLiveDefinition( - model=model, - instructions="You are a friendly voice assistant. Delegate text questions to the backend.", - audio=audio_settings(), - delegation=models.LiveResponsesDelegationParam( - responses=models.LiveResponsesDelegationSettingsInputParam( - model=backend_model, instructions="Answer in one short sentence." - ) - ), - store=False, - ), - ) - await _run_text_conversation(project_client, agent_name, created_version.version, prompts) - finally: - if created_version is not None: - await project_client.agents.delete_version(agent_name=agent_name, agent_version=created_version.version) - print(f"Deleted sample agent version: {created_version.version}") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Run a headless, asynchronous GPT Live text conversation.") - parser.add_argument("prompts", nargs="*", help="Quoted text prompts to send sequentially in one session.") - arguments = parser.parse_args() - asyncio.run(main(arguments.prompts or None)) diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py index 4d1d24a00ef2..51af775fa7d9 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py @@ -8,6 +8,7 @@ import base64 from concurrent.futures import Future import importlib.util +import inspect import io import json from pathlib import Path @@ -18,7 +19,7 @@ import wave import pytest -from azure.core.exceptions import HttpResponseError, ResourceNotFoundError +from azure.core.exceptions import HttpResponseError from azure.ai.projects import models @@ -27,6 +28,23 @@ _SYNC_SCENARIOS = ("connection", "text_conversation", "function_tool") +@pytest.mark.parametrize( + "filename", sorted(path.name for path in _SAMPLE_DIRECTORY.glob("sample_voice_agent_live_*.py")) +) +def test_all_live_samples_have_no_cli_or_entrypoint_arguments(monkeypatch, sample_environment, filename): + sample = _load_module(monkeypatch, filename) + + assert not inspect.signature(sample.main).parameters + assert not hasattr(sample, "argparse") + assert "sys.argv" not in inspect.getsource(sample) + + +def test_only_full_duplex_audio_needs_a_separate_async_live_sample(): + assert {path.name for path in _SAMPLE_DIRECTORY.glob("sample_voice_agent_live_*_async.py")} == { + "sample_voice_agent_live_audio_conversation_async.py" + } + + def _load_module(monkeypatch, filename): name = Path(filename).stem spec = importlib.util.spec_from_file_location(name, _SAMPLE_DIRECTORY / filename) @@ -75,6 +93,20 @@ def _recording(payload=b"\x01\x00" * 1200, *, channels=1, rate=24000, sample_wid return wave.open(stream, "rb") +def _capture_sample_output_directories(monkeypatch, sample, tmp_path): + make_directory = sample.mkdtemp + directories = [] + + def create_directory(*, prefix): + directory = Path(make_directory(prefix=prefix, dir=tmp_path)) + directories.append(directory) + return str(directory) + + factory = MagicMock(side_effect=create_directory) + monkeypatch.setattr(sample, "mkdtemp", factory) + return factory, directories + + @pytest.fixture def sample_environment(monkeypatch): monkeypatch.setenv("FOUNDRY_PROJECT_ENDPOINT", "https://example.invalid/api/projects/offline") @@ -82,7 +114,6 @@ def sample_environment(monkeypatch): monkeypatch.setenv("FOUNDRY_MODEL_NAME", "backend-deployment-test") monkeypatch.setenv("FOUNDRY_VOICE_AGENT_NAME", "live-offline") monkeypatch.setenv("FOUNDRY_VOICE_AUDIO_INPUT_PATH", "offline-input.wav") - monkeypatch.delenv("FOUNDRY_VOICE_AUDIO_OUTPUT_PATH", raising=False) monkeypatch.setattr("dotenv.load_dotenv", lambda: False) client = MagicMock() @@ -469,37 +500,104 @@ async def test_audio_rejects_invalid_or_partial_pcm(monkeypatch, sample_environm @pytest.mark.asyncio -async def test_audio_refuses_to_overwrite_the_input_recording(monkeypatch, sample_environment): +@pytest.mark.parametrize("hard_link", [False, True], ids=["input-path", "hard-linked-input"]) +async def test_audio_temp_output_never_overwrites_input_audio(monkeypatch, sample_environment, tmp_path, hard_link): sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") - monkeypatch.setenv("FOUNDRY_VOICE_AUDIO_OUTPUT_PATH", "offline-input.wav") - with pytest.raises(ValueError, match="must not overwrite"): + input_path = tmp_path / ("input.wav" if hard_link else "reply.wav") + output_path = tmp_path / "reply.wav" + original = _wav_bytes(1) + input_path.write_bytes(original) + if hard_link: + output_path.hardlink_to(input_path) + assert output_path.samefile(input_path) + monkeypatch.setenv("FOUNDRY_VOICE_AUDIO_INPUT_PATH", str(input_path)) + monkeypatch.setattr(sample, "SAVE_AUDIO", True) + monkeypatch.setattr(sample, "mkdtemp", MagicMock(return_value=str(tmp_path))) + monkeypatch.setattr(sample, "_run_audio_conversation", AsyncMock()) + + with pytest.raises(FileExistsError): await sample.main() + + assert input_path.read_bytes() == original + assert output_path.read_bytes() == original sample_environment.async_client.agents.create_version.assert_not_awaited() + sample_environment.async_client.beta.voice_agents.live.connect.assert_not_called() @pytest.mark.asyncio -async def test_audio_refuses_a_hard_link_to_the_input_without_modifying_either_path( - monkeypatch, sample_environment, tmp_path +@pytest.mark.parametrize("microphone", [False, True], ids=["wav", "microphone"]) +@pytest.mark.parametrize("save_audio", [False, True], ids=["no-recording", "save-reply"]) +async def test_audio_recording_is_opt_in_and_uses_unique_persistent_user_temp_folders( + monkeypatch, sample_environment, tmp_path, capsys, microphone, save_audio ): sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") input_path = tmp_path / "input.wav" - output_path = tmp_path / "hard-linked-output.wav" original = _wav_bytes(1) input_path.write_bytes(original) - output_path.hardlink_to(input_path) - assert input_path.resolve() != output_path.resolve() - assert output_path.samefile(input_path) + if microphone: + monkeypatch.delenv("FOUNDRY_VOICE_AUDIO_INPUT_PATH") + else: + monkeypatch.setenv("FOUNDRY_VOICE_AUDIO_INPUT_PATH", str(input_path)) + monkeypatch.setattr(sample, "USE_MICROPHONE", microphone) + monkeypatch.setattr(sample, "SAVE_AUDIO", save_audio) + backend = object() + monkeypatch.setattr(sample, "_microphone_backend", lambda: backend) + output_factory, directories = _capture_sample_output_directories(monkeypatch, sample, tmp_path) + reply = b"\x00\x08" * 1200 + + async def stream(client, agent_name, agent_version, recording, output_file, *, audio_backend=None): + assert client is sample_environment.async_client + assert agent_version == "owned-version" + assert agent_name.startswith("live-offline-") + if microphone: + assert recording is None and audio_backend is backend + else: + assert recording.readframes(recording.getnframes()) == b"\x01\x02" * 1200 + if save_audio: + output_file.writeframesraw(reply) + else: + assert output_file is None + + run_audio = AsyncMock(side_effect=stream) + monkeypatch.setattr(sample, "_run_audio_conversation", run_audio) + await sample.main() + await sample.main() + + assert input_path.read_bytes() == original + assert run_audio.await_count == 2 + if save_audio: + assert len(directories) == 2 and directories[0] != directories[1] + assert output_factory.call_count == 2 + for directory in directories: + assert directory.parent == tmp_path + with wave.open(str(directory / "reply.wav"), "rb") as recording: + assert recording.getparams()[:3] == (1, 2, 24000) + assert recording.readframes(recording.getnframes()) == reply + output = capsys.readouterr().out + assert all(str(directory) in output for directory in directories) + else: + output_factory.assert_not_called() + assert directories == [] + assert sample_environment.async_client.agents.delete_version.await_count == 2 + sample_environment.async_client.agents.delete.assert_not_called() + + +@pytest.mark.asyncio +async def test_audio_temp_directory_failure_precedes_service_resource_creation( + monkeypatch, sample_environment, tmp_path +): + sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + input_path = tmp_path / "input.wav" + input_path.write_bytes(_wav_bytes(1)) monkeypatch.setenv("FOUNDRY_VOICE_AUDIO_INPUT_PATH", str(input_path)) - monkeypatch.setenv("FOUNDRY_VOICE_AUDIO_OUTPUT_PATH", str(output_path)) - monkeypatch.setattr(sample, "_run_audio_conversation", AsyncMock()) + monkeypatch.setattr(sample, "SAVE_AUDIO", True) + monkeypatch.setattr(sample, "mkdtemp", MagicMock(side_effect=PermissionError("temporary directory unavailable"))) - with pytest.raises(ValueError, match="must not overwrite"): + with pytest.raises(PermissionError, match="temporary directory unavailable"): await sample.main() - assert input_path.read_bytes() == original - assert output_path.read_bytes() == original sample_environment.async_client.agents.create_version.assert_not_awaited() - sample_environment.async_client.beta.voice_agents.live.connect.assert_not_called() + sample_environment.async_client.agents.delete_version.assert_not_awaited() @pytest.mark.asyncio @@ -608,28 +706,15 @@ async def receive(): assert connection.recv.await_count == 10 -def _text_sample(monkeypatch, sample_environment, asynchronous): - suffix = "_async" if asynchronous else "" - sample = _load_module(monkeypatch, f"sample_voice_agent_live_text_conversation{suffix}.py") - client = sample_environment.async_client if asynchronous else sample_environment.client - connection = sample_environment.async_connection if asynchronous else sample_environment.connection - return sample, client, connection - - -async def _run_text_sample(sample, asynchronous, prompts=None): - if asynchronous: - await sample.main(prompts) - else: - sample.main(prompts) +def _text_sample(monkeypatch, sample_environment): + sample = _load_module(monkeypatch, "sample_voice_agent_live_text_conversation.py") + return sample, sample_environment.client, sample_environment.connection -@pytest.mark.asyncio -@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) -async def test_text_samples_reuse_one_session_and_wait_for_each_turn( - monkeypatch, sample_environment, capsys, asynchronous -): - sample, client, connection = _text_sample(monkeypatch, sample_environment, asynchronous) +def test_text_sample_reuses_one_session_and_waits_for_each_turn(monkeypatch, sample_environment, capsys): + sample, client, connection = _text_sample(monkeypatch, sample_environment) prompts = ["My name is Ada.", "What is my name?"] + monkeypatch.setattr(sample, "TEXT_PROMPTS", prompts) events = iter([_started(), *_text_response("Hello, Ada."), *_text_response("Your name is Ada."), _closed()]) completed = 0 submitted = [] @@ -648,7 +733,7 @@ def submit(*, item): connection.recv.side_effect = receive connection.response.item.create.side_effect = submit - await _run_text_sample(sample, asynchronous, prompts) + sample.main() assert submitted == prompts assert completed == 2 @@ -660,26 +745,17 @@ def submit(*, item): client.agents.delete.assert_not_called() connection.session.start.assert_not_called() connection.response.cancel.assert_not_called() - if asynchronous: - assert connection.response.item.create.await_count == 2 - assert connection.response.create.await_count == 2 - connection.session.close.assert_awaited_once() - client.agents.delete_version.assert_awaited_once() - client.beta.voice_agents.live.connect.return_value.__aexit__.assert_awaited_once() - else: - client.beta.voice_agents.live.connect.return_value.__exit__.assert_called_once() + client.beta.voice_agents.live.connect.return_value.__exit__.assert_called_once() output = capsys.readouterr().out assert output.count("Responses backend: Hello, Ada.\n") == 1 assert output.count("Responses backend: Your name is Ada.\n") == 1 -@pytest.mark.asyncio -@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) -async def test_text_samples_keep_the_original_single_prompt_default(monkeypatch, sample_environment, asynchronous): - sample, client, connection = _text_sample(monkeypatch, sample_environment, asynchronous) +def test_text_sample_keeps_the_original_single_prompt_default(monkeypatch, sample_environment): + sample, client, connection = _text_sample(monkeypatch, sample_environment) connection.recv.side_effect = [_started(), *_text_response("Seattle has a rainy climate."), _closed()] - await _run_text_sample(sample, asynchronous) + sample.main() definition = client.agents.create_version.call_args.kwargs["definition"] assert isinstance(definition.delegation, models.LiveResponsesDelegationParam) @@ -694,26 +770,22 @@ async def test_text_samples_keep_the_original_single_prompt_default(monkeypatch, connection.response.create.assert_called_once() -@pytest.mark.asyncio -@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) @pytest.mark.parametrize("prompts", [[], [""], ["first", " "], "not-a-prompt-list"]) -async def test_text_samples_reject_empty_or_invalid_scripted_prompts( - monkeypatch, sample_environment, asynchronous, prompts -): - sample, client, _ = _text_sample(monkeypatch, sample_environment, asynchronous) +def test_text_sample_rejects_empty_or_invalid_scripted_prompts(monkeypatch, sample_environment, prompts): + sample, client, _ = _text_sample(monkeypatch, sample_environment) + monkeypatch.setattr(sample, "TEXT_PROMPTS", prompts) with pytest.raises(ValueError, match="prompt"): - await _run_text_sample(sample, asynchronous, prompts) + sample.main() client.agents.create_version.assert_not_called() client.beta.voice_agents.live.connect.assert_not_called() client.agents.delete_version.assert_not_called() -@pytest.mark.asyncio -@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) -async def test_text_samples_stop_on_a_later_backend_failure_and_clean_up(monkeypatch, sample_environment, asynchronous): - sample, client, connection = _text_sample(monkeypatch, sample_environment, asynchronous) +def test_text_sample_stops_on_a_later_backend_failure_and_cleans_up(monkeypatch, sample_environment): + sample, client, connection = _text_sample(monkeypatch, sample_environment) + monkeypatch.setattr(sample, "TEXT_PROMPTS", ["first", "second", "must not be sent"]) connection.recv.side_effect = [ _started(), *_text_response("First answer."), @@ -721,22 +793,19 @@ async def test_text_samples_stop_on_a_later_backend_failure_and_clean_up(monkeyp ] with pytest.raises(RuntimeError, match="Responses backend response.failed"): - await _run_text_sample(sample, asynchronous, ["first", "second", "must not be sent"]) + sample.main() assert connection.response.item.create.call_count == 2 assert connection.response.create.call_count == 2 connection.session.close.assert_called_once() agent_name = client.agents.create_version.call_args.kwargs["agent_name"] client.agents.delete_version.assert_called_once_with(agent_name=agent_name, agent_version="owned-version") - if asynchronous: - client.agents.delete_version.assert_awaited_once() -@pytest.mark.asyncio -@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) @pytest.mark.parametrize("empty", [True, False], ids=["empty-text", "unexpected-tool"]) -async def test_text_samples_reject_non_text_backend_completions(monkeypatch, sample_environment, asynchronous, empty): - sample, client, connection = _text_sample(monkeypatch, sample_environment, asynchronous) +def test_text_sample_rejects_non_text_backend_completions(monkeypatch, sample_environment, empty): + sample, client, connection = _text_sample(monkeypatch, sample_environment) + monkeypatch.setattr(sample, "TEXT_PROMPTS", ["one", "must not be sent"]) events = [_started()] if not empty: events.append( @@ -749,38 +818,27 @@ async def test_text_samples_reject_non_text_backend_completions(monkeypatch, sam connection.recv.side_effect = events with pytest.raises(RuntimeError, match="without returning any output text|unexpectedly requested"): - await _run_text_sample(sample, asynchronous, ["one", "must not be sent"]) + sample.main() connection.response.item.create.assert_called_once() client.agents.delete_version.assert_called_once() @pytest.mark.asyncio -async def test_async_text_create_failure_does_not_delete_an_existing_version(monkeypatch, sample_environment): - sample, client, _ = _text_sample(monkeypatch, sample_environment, True) +async def test_async_audio_create_failure_does_not_delete_an_existing_version(monkeypatch, sample_environment): + sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + client = sample_environment.async_client client.agents.create_version.side_effect = RuntimeError("create failed") - with pytest.raises(RuntimeError, match="create failed"): - await sample.main() + with _recording() as recording: + monkeypatch.setattr(sample, "wave", SimpleNamespace(open=MagicMock(return_value=recording))) + with pytest.raises(RuntimeError, match="create failed"): + await sample.main() client.agents.delete_version.assert_not_awaited() client.beta.voice_agents.live.connect.assert_not_called() -@pytest.mark.asyncio -async def test_async_text_cancellation_propagates_and_deletes_the_owned_version(monkeypatch, sample_environment): - sample, client, connection = _text_sample(monkeypatch, sample_environment, True) - connection.recv.side_effect = asyncio.CancelledError - - with pytest.raises(asyncio.CancelledError): - await sample.main() - - connection.session.close.assert_awaited_once() - agent_name = client.agents.create_version.call_args.kwargs["agent_name"] - client.agents.delete_version.assert_awaited_once_with(agent_name=agent_name, agent_version="owned-version") - client.beta.voice_agents.live.connect.return_value.__aexit__.assert_awaited_once() - - @pytest.mark.asyncio async def test_async_response_collection_keeps_canonical_tool_ids_and_granular_text(sample_environment): connection = sample_environment.async_connection @@ -841,15 +899,11 @@ async def test_async_close_expired_deadline_does_not_create_an_unawaited_send(mo sample_environment.async_connection.session.close.assert_not_called() -def _rest_values(values, asynchronous): - if asynchronous: +def test_live_rest_sample_entrypoint_has_no_cli_or_mutation_options(monkeypatch, sample_environment): + sample = _load_module(monkeypatch, "sample_voice_agent_live_read_conversation.py") - async def iterate(): - for value in values: - yield value - - return iterate() - return iter(values) + assert not inspect.signature(sample.main).parameters + assert not hasattr(sample, "argparse") def _wav_bytes(channels): @@ -860,16 +914,13 @@ def _wav_bytes(channels): return stream.getvalue() -@pytest.fixture(params=[False, True], ids=["sync", "async"]) -def rest_sample(request, monkeypatch, sample_environment): - asynchronous = request.param - filename = "sample_voice_agent_live_read_conversation" + ("_async.py" if asynchronous else ".py") - sample = _load_module(monkeypatch, filename) +@pytest.fixture +def rest_sample(monkeypatch, sample_environment, tmp_path): + sample = _load_module(monkeypatch, "sample_voice_agent_live_read_conversation.py") monkeypatch.setenv("FOUNDRY_VOICE_AGENT_NAME", "stored-agent-test") monkeypatch.setenv("FOUNDRY_VOICE_CONVERSATION_ID", "conversation-test") - client = sample_environment.async_client if asynchronous else sample_environment.client + client = sample_environment.client operations = client.beta.voice_agents.live_conversations - method_mock = AsyncMock if asynchronous else MagicMock conversation = models.VoiceAgentLiveConversation( {"id": "conversation-test", "status": "completed", "created_at": 1, "completed_at": 2} ) @@ -903,27 +954,18 @@ def rest_sample(request, monkeypatch, sample_environment): } ) item_bytes, merged_bytes = _wav_bytes(1), _wav_bytes(2) - operations.get = method_mock(return_value=conversation) - operations.list = MagicMock(side_effect=lambda *args, **kwargs: _rest_values([conversation], asynchronous)) - operations.list_items = MagicMock(side_effect=lambda *args, **kwargs: _rest_values(items, asynchronous)) - operations.get_item = method_mock( + operations.get = MagicMock(return_value=conversation) + operations.list = MagicMock(side_effect=lambda *args, **kwargs: iter([conversation])) + operations.list_items = MagicMock(side_effect=lambda *args, **kwargs: iter(items)) + operations.get_item = MagicMock( side_effect=lambda agent, conversation, item: next(v for v in items if v.id == item) ) - operations.get_audio_item = method_mock(return_value=audio) - operations.get_audio = method_mock(return_value=recording) - operations.download_audio_item = method_mock( - side_effect=lambda *args: _rest_values([item_bytes[:12], item_bytes[12:]], asynchronous) - ) - operations.download_audio = method_mock( - side_effect=lambda *args: _rest_values([merged_bytes[:12], merged_bytes[12:]], asynchronous) - ) - operations.delete = method_mock() - - async def run(**kwargs): - if asynchronous: - await sample.main(**kwargs) - else: - sample.main(**kwargs) + operations.get_audio_item = MagicMock(return_value=audio) + operations.get_audio = MagicMock(return_value=recording) + operations.download_audio_item = MagicMock(side_effect=lambda *args: iter([item_bytes[:12], item_bytes[12:]])) + operations.download_audio = MagicMock(side_effect=lambda *args: iter([merged_bytes[:12], merged_bytes[12:]])) + operations.delete = MagicMock() + output_factory, directories = _capture_sample_output_directories(monkeypatch, sample, tmp_path) return SimpleNamespace( sample=sample, @@ -933,18 +975,20 @@ async def run(**kwargs): audio=audio, recording=recording, items=items, - run=run, + run=sample.main, item_bytes=item_bytes, merged_bytes=merged_bytes, - asynchronous=asynchronous, + output_factory=output_factory, + directories=directories, ) -@pytest.mark.asyncio -async def test_live_rest_samples_are_read_only_by_default(rest_sample, tmp_path, capsys, monkeypatch): +def test_live_rest_sample_downloads_to_user_temp_without_modifying_resources( + rest_sample, tmp_path, capsys, monkeypatch +): for variable in ("FOUNDRY_VOICE_AGENT_MODEL", "FOUNDRY_MODEL_NAME", "FOUNDRY_VOICE_AUDIO_INPUT_PATH"): monkeypatch.delenv(variable, raising=False) - await rest_sample.run() + rest_sample.run() operations = rest_sample.operations operations.get.assert_called_once_with("stored-agent-test", "conversation-test") @@ -953,75 +997,69 @@ async def test_live_rest_samples_are_read_only_by_default(rest_sample, tmp_path, assert operations.get_item.call_count == 2 operations.get_audio_item.assert_called_once_with("stored-agent-test", "conversation-test", "speech-test") operations.get_audio.assert_called_once_with("stored-agent-test", "conversation-test") - operations.download_audio.assert_not_called() - operations.download_audio_item.assert_not_called() + operations.download_audio.assert_called_once_with("stored-agent-test", "conversation-test") + operations.download_audio_item.assert_called_once_with("stored-agent-test", "conversation-test", "speech-test") operations.delete.assert_not_called() - rest_sample.client.agents.create_version.assert_not_called() - rest_sample.client.agents.delete_version.assert_not_called() + assert not rest_sample.client.agents.mock_calls rest_sample.client.beta.voice_agents.live.connect.assert_not_called() rest_sample.client.beta.voice_agents.conversations.get.assert_not_called() - assert not list(tmp_path.iterdir()) + rest_sample.output_factory.assert_called_once_with(prefix="voice-agent-live-") + output_directory = rest_sample.directories[0] + assert output_directory.parent == tmp_path + assert (output_directory / "item-0001.wav").read_bytes() == rest_sample.item_bytes + assert (output_directory / "conversation.wav").read_bytes() == rest_sample.merged_bytes output = capsys.readouterr().out + assert str(output_directory) in output assert "A synthetic transcript." in output assert "Non-speech item" in output - assert "Read-only run" in output - + assert "Read-only REST run" in output + rest_sample.client.__exit__.assert_called_once() -@pytest.mark.asyncio -async def test_live_rest_samples_download_both_recordings_and_delete_only_when_requested(rest_sample, tmp_path, capsys): - rest_sample.operations.get.side_effect = [rest_sample.conversation, ResourceNotFoundError("deleted")] - await rest_sample.run(output_directory=tmp_path, delete=True) +def test_live_rest_sample_uses_unique_persistent_output_folders(rest_sample): + rest_sample.run() + first = rest_sample.directories[0] + rest_sample.run() + second = rest_sample.directories[1] - assert (tmp_path / "item-0001.wav").read_bytes() == rest_sample.item_bytes - assert (tmp_path / "conversation.wav").read_bytes() == rest_sample.merged_bytes - rest_sample.operations.download_audio_item.assert_called_once_with( - "stored-agent-test", "conversation-test", "speech-test" - ) - rest_sample.operations.download_audio.assert_called_once_with("stored-agent-test", "conversation-test") - rest_sample.operations.delete.assert_called_once_with("stored-agent-test", "conversation-test") - assert rest_sample.operations.get.call_count == 2 - if rest_sample.asynchronous: - for name in ("get_item", "get_audio_item", "get_audio", "download_audio_item", "download_audio", "delete"): - method = getattr(rest_sample.operations, name) - assert method.await_count == method.call_count - assert "confirmed HTTP 404" in capsys.readouterr().out + assert first != second + assert rest_sample.output_factory.call_count == 2 + for directory in (first, second): + assert (directory / "conversation.wav").read_bytes() == rest_sample.merged_bytes + assert (directory / "item-0001.wav").read_bytes() == rest_sample.item_bytes + rest_sample.operations.delete.assert_not_called() -@pytest.mark.asyncio @pytest.mark.parametrize("status", [404, 409, 500]) -async def test_live_rest_samples_surface_service_failures_without_fallback_or_deletion(rest_sample, status, capsys): +def test_live_rest_sample_surfaces_service_failures_without_fallback_or_deletion(rest_sample, status, capsys): failure = HttpResponseError("service failure") failure.status_code = status rest_sample.operations.get.side_effect = failure with pytest.raises(HttpResponseError) as caught: - await rest_sample.run(delete=True) + rest_sample.run() assert caught.value is failure rest_sample.operations.delete.assert_not_called() assert not rest_sample.client.beta.voice_agents.conversations.mock_calls assert f"HTTP {status}" in capsys.readouterr().out - if rest_sample.asynchronous: - rest_sample.client.__aexit__.assert_awaited_once() - else: - rest_sample.client.__exit__.assert_called_once() + rest_sample.output_factory.assert_not_called() + rest_sample.client.__exit__.assert_called_once() -@pytest.mark.asyncio @pytest.mark.parametrize("status", ["in_progress", "failed"]) -async def test_live_rest_samples_require_successful_session_finalization(rest_sample, status): +def test_live_rest_sample_requires_successful_session_finalization(rest_sample, status): rest_sample.conversation.status = status with pytest.raises(RuntimeError, match="must finish successfully"): - await rest_sample.run(delete=True) + rest_sample.run() rest_sample.operations.list_items.assert_not_called() rest_sample.operations.download_audio.assert_not_called() rest_sample.operations.delete.assert_not_called() + rest_sample.output_factory.assert_not_called() -@pytest.mark.asyncio @pytest.mark.parametrize( "method,filename", [ @@ -1030,31 +1068,23 @@ async def test_live_rest_samples_require_successful_session_finalization(rest_sa ], ) @pytest.mark.parametrize("empty", [True, False], ids=["empty-stream", "failed-stream"]) -async def test_live_rest_download_failures_remove_incomplete_files_and_do_not_delete( - rest_sample, tmp_path, method, filename, empty +def test_live_rest_download_failures_remove_incomplete_files_without_deleting_resources( + rest_sample, method, filename, empty ): - async def async_chunks(): - if not empty: - yield b"partial" - raise ConnectionError("stream failed") - def sync_chunks(): if not empty: yield b"partial" raise ConnectionError("stream failed") - getattr(rest_sample.operations, method).side_effect = lambda *args: ( - async_chunks() if rest_sample.asynchronous else sync_chunks() - ) + getattr(rest_sample.operations, method).side_effect = lambda *args: sync_chunks() with pytest.raises(RuntimeError if empty else ConnectionError, match="no bytes|stream failed"): - await rest_sample.run(output_directory=tmp_path, delete=True) + rest_sample.run() - assert not (tmp_path / filename).exists() + assert not (rest_sample.directories[0] / filename).exists() rest_sample.operations.delete.assert_not_called() -@pytest.mark.asyncio @pytest.mark.parametrize( "filename,method", [ @@ -1062,71 +1092,69 @@ def sync_chunks(): ("conversation.wav", "download_audio"), ], ) -async def test_live_rest_samples_never_overwrite_an_existing_recording(rest_sample, tmp_path, filename, method): +def test_live_rest_sample_never_overwrites_an_existing_recording(rest_sample, tmp_path, filename, method): path = tmp_path / filename path.write_bytes(b"keep existing audio") + rest_sample.output_factory.side_effect = lambda **kwargs: str(tmp_path) with pytest.raises(FileExistsError): - await rest_sample.run(output_directory=tmp_path, delete=True) + rest_sample.run() assert path.read_bytes() == b"keep existing audio" getattr(rest_sample.operations, method).assert_not_called() rest_sample.operations.delete.assert_not_called() -@pytest.mark.asyncio @pytest.mark.parametrize("level", ["item", "merged"]) -@pytest.mark.parametrize("download", [False, True], ids=["metadata-only", "download-requested"]) -async def test_live_rest_samples_explain_byos_without_proxying_downloads( - rest_sample, tmp_path, capsys, level, download -): +def test_live_rest_sample_explains_byos_without_proxying_downloads(rest_sample, capsys, level): metadata = rest_sample.audio if level == "item" else rest_sample.recording metadata.blob_uri = "https://example.invalid/private-recording" - if download: - with pytest.raises(RuntimeError, match="not BYOS audio"): - await rest_sample.run(output_directory=tmp_path, delete=True) - rest_sample.operations.delete.assert_not_called() - else: - await rest_sample.run() + with pytest.raises(RuntimeError, match="not BYOS audio"): + rest_sample.run() method = "download_audio_item" if level == "item" else "download_audio" getattr(rest_sample.operations, method).assert_not_called() + rest_sample.operations.delete.assert_not_called() output = capsys.readouterr().out assert "BYOS" in output and "https://example.invalid/private-recording" not in output -@pytest.mark.asyncio -async def test_live_rest_samples_reject_repeated_paging_items(rest_sample): +def test_live_rest_sample_rejects_repeated_paging_items(rest_sample): rest_sample.items.append(rest_sample.items[0]) with pytest.raises(RuntimeError, match="pager repeated an item"): - await rest_sample.run(delete=True) + rest_sample.run() rest_sample.operations.delete.assert_not_called() -@pytest.mark.asyncio -async def test_live_rest_samples_report_a_delete_that_did_not_take_effect(rest_sample): - with pytest.raises(RuntimeError, match="remains readable after deletion"): - await rest_sample.run(delete=True) +def test_live_rest_sample_temp_directory_failure_leaves_service_resources_untouched(rest_sample): + rest_sample.output_factory.side_effect = PermissionError("temporary directory unavailable") - rest_sample.operations.delete.assert_called_once_with("stored-agent-test", "conversation-test") + with pytest.raises(PermissionError, match="temporary directory unavailable"): + rest_sample.run() + + rest_sample.operations.list_items.assert_not_called() + rest_sample.operations.download_audio.assert_not_called() + rest_sample.operations.delete.assert_not_called() + rest_sample.client.__exit__.assert_called_once() -@pytest.mark.asyncio @pytest.mark.parametrize("variable", ["FOUNDRY_VOICE_AGENT_NAME", "FOUNDRY_VOICE_CONVERSATION_ID"]) -async def test_live_rest_samples_reject_blank_resource_identifiers(rest_sample, monkeypatch, variable): +def test_live_rest_sample_rejects_blank_resource_identifiers(rest_sample, monkeypatch, variable): monkeypatch.setenv(variable, " ") with pytest.raises(ValueError, match="nonblank"): - await rest_sample.run(delete=True) + rest_sample.run() rest_sample.operations.get.assert_not_called() rest_sample.operations.delete.assert_not_called() + rest_sample.output_factory.assert_not_called() @pytest.fixture def microphone_sample(monkeypatch, sample_environment): sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") + monkeypatch.setattr(sample, "USE_MICROPHONE", True) speaker = MagicMock() microphone = MagicMock() engine = MagicMock() @@ -1153,7 +1181,7 @@ async def test_microphone_mode_does_not_require_a_wav_and_preserves_live_lifecyc monkeypatch.delenv("FOUNDRY_VOICE_AUDIO_INPUT_PATH") setup.connection.recv.side_effect = [_started(), _closed()] - await setup.sample.main(microphone=True) + await setup.sample.main() assert setup.engine.open.call_count == 2 output_options, input_options = [call.kwargs for call in setup.engine.open.call_args_list] @@ -1181,9 +1209,10 @@ async def test_microphone_dependency_error_precedes_agent_creation(monkeypatch, sample = _load_module(monkeypatch, "sample_voice_agent_live_audio_conversation_async.py") monkeypatch.setitem(sys.modules, "pyaudio", None) monkeypatch.delenv("FOUNDRY_VOICE_AUDIO_INPUT_PATH") + monkeypatch.setattr(sample, "USE_MICROPHONE", True) with pytest.raises(RuntimeError, match="pip install pyaudio"): - await sample.main(microphone=True) + await sample.main() sample_environment.async_client.agents.create_version.assert_not_awaited() @@ -1339,7 +1368,7 @@ async def test_microphone_cancellation_closes_the_session_and_deletes_only_its_v microphone_sample.connection.recv.side_effect = [_started(), asyncio.CancelledError(), _closed()] with pytest.raises(asyncio.CancelledError): - await microphone_sample.sample.main(microphone=True) + await microphone_sample.sample.main() microphone_sample.connection.session.close.assert_awaited_once() microphone_sample.client.agents.delete_version.assert_awaited_once() @@ -1357,7 +1386,7 @@ async def test_microphone_service_errors_are_not_silently_successful(microphone_ {"type": "error", "error": {"code": "offline_error", "message": "service failed"}}, ] with pytest.raises(RuntimeError, match="service failed"): - await microphone_sample.sample.main(microphone=True) + await microphone_sample.sample.main() microphone_sample.client.agents.delete_version.assert_awaited_once() microphone_sample.microphone.close.assert_called_once() microphone_sample.speaker.close.assert_called_once() From 06a65902ed19fa106643b502dd136a021c9a430b Mon Sep 17 00:00:00 2001 From: Xiting Zhang Date: Tue, 27 Oct 2026 13:56:06 -0700 Subject: [PATCH 08/10] Make GPT Live conversation REST sample self-contained Create and clean up a stored greeting conversation when no existing ID is supplied. Preserve read-only behavior for supplied IDs and remove source-shape and filename assertions per review while retaining behavioral coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/ai/azure-ai-projects/CHANGELOG.md | 2 +- sdk/ai/azure-ai-projects/README.md | 22 +- ...mple_voice_agent_live_read_conversation.py | 198 +++++++++++++++--- .../tests/samples/test_live_samples.py | 169 ++++++++++++--- 4 files changed, 327 insertions(+), 64 deletions(-) diff --git a/sdk/ai/azure-ai-projects/CHANGELOG.md b/sdk/ai/azure-ai-projects/CHANGELOG.md index 6e66e2c6e95b..d139f03d9554 100644 --- a/sdk/ai/azure-ai-projects/CHANGELOG.md +++ b/sdk/ai/azure-ai-projects/CHANGELOG.md @@ -18,7 +18,7 @@ ### Sample updates -* Added GPT Live samples under `samples/agents/voice` demonstrating session lifecycle and configuration, single- and multi-turn text conversations, function-tool calling, WAV audio streaming, microphone capture with speaker playback, and stored-conversation/item/audio retrieval with Foundry-managed WAV downloads to a unique user-temporary folder. +* Added GPT Live samples under `samples/agents/voice` demonstrating session lifecycle and configuration, single- and multi-turn text conversations, function-tool calling, WAV audio streaming, microphone capture with speaker playback, and stored-conversation/item/audio retrieval with Foundry-managed WAV downloads to a unique user-temporary folder. The REST sample can create and clean up its own stored greeting conversation or read an existing conversation without modifying it. ## 2.9.0 (2026-10-16) diff --git a/sdk/ai/azure-ai-projects/README.md b/sdk/ai/azure-ai-projects/README.md index e92a3dd9287f..1aedd54c8d86 100644 --- a/sdk/ai/azure-ai-projects/README.md +++ b/sdk/ai/azure-ai-projects/README.md @@ -291,20 +291,28 @@ Leave `USE_MICROPHONE = False` to use the WAV workflow without requiring PyAudio The Live samples do not accept command-line arguments. Connection/session controls, text, function calling, and stored-conversation REST each have one synchronous example; audio has the single asynchronous example described above. Samples delete only the agent version they -create; the stored-conversation REST sample does not create or delete service resources. +create. The stored-conversation REST sample also deletes a conversation only when it created it. The [REST sample](samples/agents/voice/sample_voice_agent_live_read_conversation.py) -reads an existing stored Live conversation without opening a WebSocket. Set `FOUNDRY_PROJECT_ENDPOINT`, -`FOUNDRY_VOICE_AGENT_NAME`, and `FOUNDRY_VOICE_CONVERSATION_ID`. The last value must be -`session.started.conversation_id` from a session with effective `store=True`, not `session.id`. -The session must have finished successfully and contain recorded speech. +can run without a pre-created conversation. Set `FOUNDRY_PROJECT_ENDPOINT` and +`FOUNDRY_VOICE_AGENT_MODEL`. When `FOUNDRY_VOICE_CONVERSATION_ID` is unset or blank, the sample +creates a uniquely named temporary agent and a stored greeting session over a WebSocket, +waits for the conversation and its audio to finish persisting, and reads them through REST. +It deletes its own conversation and agent version afterward, including on failure. No microphone, +input WAV, or Responses backend is needed. An optional `FOUNDRY_VOICE_AGENT_NAME` is used only +as a name prefix in this mode, so an existing agent with that name is not modified. + +Alternatively, supply both `FOUNDRY_VOICE_AGENT_NAME` and `FOUNDRY_VOICE_CONVERSATION_ID` to read +an existing conversation without opening a WebSocket or requiring a model deployment name. +The ID must be `session.started.conversation_id` from a successfully ended speech session with +effective `store=True`, not `session.id`. Existing conversations and agents are never deleted. The sample demonstrates conversation/item listing and retrieval, audio metadata, and merged/per-item Foundry-managed WAV downloads. It creates a unique folder in your user temporary directory and prints the location. Files remain available after the sample exits; remove the local folder when it is no longer needed. These downloads contain conversation content, not telemetry. BYOS audio requires your -own storage credentials and is not downloaded by this sample. The sample never deletes the existing -agent or stored conversation, items, or audio, and it does not accept command-line options. +own storage credentials and is not downloaded by this sample. The sample deletes only service +resources it created, and it does not accept command-line options. It uses the dedicated `.beta.voice_agents.live_conversations` routes. A deployment that does not support those routes returns HTTP 404, which is surfaced rather than silently falling back to `.beta.voice_agents.conversations`. diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py index 69a6c3952dcc..2fbcb8760c1a 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py @@ -8,48 +8,183 @@ Read a stored GPT Live conversation using the preview project_client.beta.voice_agents.live_conversations REST APIs. Retrieve the conversation, recent conversations, ordered items, individual items, and - merged/per-item audio metadata. No agent or WebSocket session is created. + merged/per-item audio metadata. - Requires an ended speech session with effective store=True. Use - session.started.conversation_id, not session.id. These dedicated Live REST - routes must be available on the service deployment; HTTP 404 is reported - and raised, never replaced by calls to the older Realtime conversation API. + If FOUNDRY_VOICE_CONVERSATION_ID is omitted or blank, create a temporary + GPT Live agent and a stored greeting session, close it, wait for persistence, + read its recordings, then delete the conversation and agent version. + No microphone, input recording, or Responses backend is needed. + + If an existing conversation ID is supplied, read it without creating an + agent or opening a WebSocket and never delete its resources. It must come + from session.started.conversation_id, not session.id, after a successful + speech session with effective store=True. Dedicated Live REST routes must + be available; no fallback to older Realtime conversation routes is used. Download Foundry-managed WAV recordings into a unique folder in your user temporary directory and print its location. The files remain available after the sample exits and are not overwritten. Downloads contain stored conversation content; remove the local folder when it is no longer needed. BYOS recordings require your own storage credentials and are not downloaded. - The existing agent, conversation, items, and stored audio are never deleted. + Only the agent version and conversation created by this run are deleted. USAGE: python sample_voice_agent_live_read_conversation.py - pip install azure-ai-projects azure-identity python-dotenv + pip install "azure-ai-projects[voice]" azure-identity python-dotenv - Environment variables (all required): - 1) FOUNDRY_PROJECT_ENDPOINT - The Foundry project endpoint. - 2) FOUNDRY_VOICE_AGENT_NAME - The existing GPT Live agent name. - 3) FOUNDRY_VOICE_CONVERSATION_ID - The stored Foundry conversation ID. + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. + 2) FOUNDRY_VOICE_CONVERSATION_ID (optional) - An existing stored conversation. + When unset or blank, this sample creates and deletes its own conversation. + 3) FOUNDRY_VOICE_AGENT_NAME (required with an existing conversation ID) - + The existing GPT Live agent name. Otherwise, an optional name prefix + for the uniquely named temporary agent; an existing agent is not changed. + 4) FOUNDRY_VOICE_AGENT_MODEL (required without a conversation ID) - + Your GPT Live deployment name, used to create the temporary agent. - Authenticates with DefaultAzureCredential. No model deployment name, - microphone, or WebSocket dependency is needed. + Authenticates with DefaultAzureCredential. Existing-conversation mode + requires no model deployment name or WebSocket dependency. """ from __future__ import annotations +import base64 +from contextlib import contextmanager, ExitStack from itertools import islice import os from pathlib import Path +import struct from tempfile import mkdtemp +import time +from typing import Iterator from dotenv import load_dotenv -from live_sample_utils import recording_output, safe_print +from live_sample_utils import ( + audio_settings, + check_event, + check_session_open, + close_session, + recording_output, + remaining_time, + safe_print, + sample_agent_name, + wait_for_event, +) -from azure.core.exceptions import HttpResponseError +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError from azure.identity import DefaultAzureCredential from azure.ai.projects import AIProjectClient, models -from azure.ai.projects.operations import BetaVoiceAgentsLiveConversationsOperations +from azure.ai.projects.operations import BetaLiveConnection, BetaVoiceAgentsLiveConversationsOperations + + +_SESSION_TIMEOUT = 45.0 +_FINALIZATION_TIMEOUT = 90.0 + + +def _receive_greeting(connection: BetaLiveConnection) -> None: + deadline = time.monotonic() + _SESSION_TIMEOUT + next_input = time.monotonic() + last_content = next_input + speech_received = False + input_bytes = output_bytes = 0 + while True: + remaining = remaining_time(deadline, "the sample greeting audio") + now = time.monotonic() + if speech_received and now - last_content >= 2: + print(f"Greeting audio: sent {input_bytes} silence bytes; received {output_bytes} PCM16 bytes.") + return + if now >= next_input: + connection.session.input_audio.append(audio=bytes(2400)) + input_bytes += 2400 + next_input = now + 0.05 + try: + event = connection.recv(timeout=min(remaining, max(0.001, next_input - time.monotonic()))) + except TimeoutError: + continue + check_event(event) + check_session_open(event) + if isinstance(event, models.LiveOutputAudioDelta): + pcm = base64.b64decode(event.delta, validate=True) + if len(pcm) % 2: + raise ValueError("The service returned an incomplete PCM16 sample.") + output_bytes += len(pcm) + if any(abs(sample) > 256 for (sample,) in struct.iter_unpack(" None: + print("Waiting for the sample conversation and audio to finish persisting.") + deadline = time.monotonic() + _FINALIZATION_TIMEOUT + while True: + remaining_time(deadline, "stored conversation finalization") + try: + conversation = conversations.get(agent_name, conversation_id) + except ResourceNotFoundError: + pass + else: + if conversation.id != conversation_id: + raise RuntimeError("The service returned a different stored conversation ID.") + if conversation.status == "failed": + raise RuntimeError(f"Sample conversation persistence failed: {conversation.last_error}") + if conversation.status == "completed": + try: + conversations.get_audio(agent_name, conversation_id) + except HttpResponseError as exc: + if exc.status_code != 409 or getattr(exc.error, "code", None) != "recording_not_ready": + raise + else: + items = conversations.list_items(agent_name, conversation_id, limit=20, order="asc") + if any( + isinstance(item, models.VoiceAgentLiveOutputConversationItem) and item.audio is not None + for item in islice(items, 20) + ): + return + elif conversation.status != "in_progress": + raise RuntimeError(f"Unexpected sample conversation status: {conversation.status}") + time.sleep(min(1.0, remaining_time(deadline, "stored conversation finalization"))) + + +@contextmanager +def _temporary_conversation(project_client: AIProjectClient, model: str) -> Iterator[tuple[str, str]]: + agent_name = sample_agent_name("rest") + conversations = project_client.beta.voice_agents.live_conversations + with ExitStack() as cleanup: + agent = project_client.agents.create_version( + agent_name=agent_name, + definition=models.VoiceAgentLiveDefinition( + model=model, + instructions="Greet the caller briefly.", + greeting=models.VoiceAgentTemplateGreetingConfig(text="Hello, welcome."), + audio=audio_settings(), + delegation=models.LiveClientDelegationParam(), + store=True, + ), + ) + cleanup.callback(project_client.agents.delete_version, agent_name=agent_name, agent_version=agent.version) + with project_client.beta.voice_agents.live.connect( + agent_name=agent_name, agent_version=agent.version, store=True + ) as connection: + try: + started = wait_for_event(connection, "session.started") + stored_id = started.get("conversation_id") + if not isinstance(stored_id, str) or not stored_id.strip() or stored_id == started["session"]["id"]: + raise RuntimeError("The stored session did not return a valid stored conversation ID.") + conversation_id = stored_id + cleanup.callback(conversations.delete, agent_name, conversation_id) + print(f"Created conversation: {conversation_id}") + _receive_greeting(connection) + finally: + close_session(connection) + _wait_for_recording(conversations, agent_name, conversation_id) + yield agent_name, conversation_id + print(f"Deleted sample conversation: {conversation_id} and temporary agent version: {agent.version}") def _read_item( @@ -120,27 +255,36 @@ def _read_conversation( for chunk in conversations.download_audio(agent_name, conversation_id): output.write(chunk) - print("Read-only REST run: the existing agent, conversation, items, and stored audio were not modified.") - def main() -> None: load_dotenv() endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] - agent_name = os.environ["FOUNDRY_VOICE_AGENT_NAME"] - conversation_id = os.environ["FOUNDRY_VOICE_CONVERSATION_ID"] - if not agent_name.strip() or not conversation_id.strip(): - raise ValueError("Provide a nonblank existing agent name and stored Foundry conversation ID.") + agent_name = os.environ.get("FOUNDRY_VOICE_AGENT_NAME", "").strip() + conversation_id = os.environ.get("FOUNDRY_VOICE_CONVERSATION_ID", "").strip() + model = "" + if conversation_id: + if not agent_name: + raise ValueError("Provide a nonblank existing agent name with the stored Foundry conversation ID.") + else: + model = os.environ["FOUNDRY_VOICE_AGENT_MODEL"].strip() + if not model: + raise ValueError("Provide a nonblank GPT Live deployment name to create the sample conversation.") with ( DefaultAzureCredential() as credential, AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project_client, ): try: - _read_conversation( - project_client.beta.voice_agents.live_conversations, - agent_name, - conversation_id, - ) + if conversation_id: + _read_conversation(project_client.beta.voice_agents.live_conversations, agent_name, conversation_id) + print( + "Read-only REST run: the existing agent, conversation, items, and stored audio were not modified." + ) + else: + with _temporary_conversation(project_client, model) as (created_agent, created_conversation): + _read_conversation( + project_client.beta.voice_agents.live_conversations, created_agent, created_conversation + ) except HttpResponseError as exc: print(f"GPT Live conversation REST request failed: HTTP {exc.status_code}.") if exc.status_code == 404: diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py index 51af775fa7d9..7bd4284fa09d 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py @@ -8,7 +8,6 @@ import base64 from concurrent.futures import Future import importlib.util -import inspect import io import json from pathlib import Path @@ -19,7 +18,7 @@ import wave import pytest -from azure.core.exceptions import HttpResponseError +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError from azure.ai.projects import models @@ -28,23 +27,6 @@ _SYNC_SCENARIOS = ("connection", "text_conversation", "function_tool") -@pytest.mark.parametrize( - "filename", sorted(path.name for path in _SAMPLE_DIRECTORY.glob("sample_voice_agent_live_*.py")) -) -def test_all_live_samples_have_no_cli_or_entrypoint_arguments(monkeypatch, sample_environment, filename): - sample = _load_module(monkeypatch, filename) - - assert not inspect.signature(sample.main).parameters - assert not hasattr(sample, "argparse") - assert "sys.argv" not in inspect.getsource(sample) - - -def test_only_full_duplex_audio_needs_a_separate_async_live_sample(): - assert {path.name for path in _SAMPLE_DIRECTORY.glob("sample_voice_agent_live_*_async.py")} == { - "sample_voice_agent_live_audio_conversation_async.py" - } - - def _load_module(monkeypatch, filename): name = Path(filename).stem spec = importlib.util.spec_from_file_location(name, _SAMPLE_DIRECTORY / filename) @@ -899,13 +881,6 @@ async def test_async_close_expired_deadline_does_not_create_an_unawaited_send(mo sample_environment.async_connection.session.close.assert_not_called() -def test_live_rest_sample_entrypoint_has_no_cli_or_mutation_options(monkeypatch, sample_environment): - sample = _load_module(monkeypatch, "sample_voice_agent_live_read_conversation.py") - - assert not inspect.signature(sample.main).parameters - assert not hasattr(sample, "argparse") - - def _wav_bytes(channels): stream = io.BytesIO() with wave.open(stream, "wb") as writer: @@ -983,6 +958,139 @@ def rest_sample(monkeypatch, sample_environment, tmp_path): ) +@pytest.fixture +def created_rest_sample(monkeypatch, rest_sample, sample_environment): + monkeypatch.delenv("FOUNDRY_VOICE_CONVERSATION_ID") + clock = SimpleNamespace(now=0.0) + clock.monotonic = lambda: clock.now + + def sleep(duration): + clock.now += duration + + clock.sleep = sleep + monkeypatch.setattr(rest_sample.sample, "time", clock, raising=False) + monkeypatch.setattr(sample_environment.helper, "time", clock) + started = _started() + started.conversation_id = rest_sample.conversation.id + events = [ + started, + models.LiveOutputAudioDelta(delta=base64.b64encode(b"\x00\x08" * 1200).decode("ascii")), + models.LiveServerEventLiveOutputTranscriptDelta(delta="Hello, welcome."), + ] + connection = sample_environment.connection + + def receive(*, timeout): + if connection.session.close.called: + return _closed() + if events: + return events.pop(0) + clock.now += timeout + raise TimeoutError + + connection.recv.side_effect = receive + rest_sample.connection = connection + rest_sample.clock = clock + rest_sample.events = events + return rest_sample + + +@pytest.mark.parametrize("conversation_id", [None, "", " "], ids=["unset", "empty", "blank"]) +def test_live_rest_sample_creates_and_deletes_its_conversation_when_id_is_omitted( + created_rest_sample, capsys, monkeypatch, conversation_id +): + setup = created_rest_sample + if conversation_id is not None: + monkeypatch.setenv("FOUNDRY_VOICE_CONVERSATION_ID", conversation_id) + setup.run() + + create = setup.client.agents.create_version.call_args.kwargs + agent_name = create["agent_name"] + assert agent_name.startswith("stored-agent-test-") and agent_name != "stored-agent-test" + definition = create["definition"] + assert isinstance(definition, models.VoiceAgentLiveDefinition) + assert definition.model == "live-deployment-test" + assert definition.store is True and definition.greeting.text + assert definition.audio.format.rate == 24000 + setup.client.beta.voice_agents.live.connect.assert_called_once_with( + agent_name=agent_name, agent_version="owned-version", store=True + ) + setup.connection.session.start.assert_not_called() + setup.connection.session.close.assert_called_once() + setup.connection.session.input_audio.append.assert_called() + assert all( + call.kwargs["audio"] == bytes(2400) for call in setup.connection.session.input_audio.append.call_args_list + ) + setup.operations.get.assert_any_call(agent_name, "conversation-test") + setup.operations.download_audio.assert_called_once_with(agent_name, "conversation-test") + setup.operations.delete.assert_called_once_with(agent_name, "conversation-test") + setup.client.agents.delete_version.assert_called_once_with(agent_name=agent_name, agent_version="owned-version") + setup.client.agents.delete.assert_not_called() + assert (setup.directories[0] / "conversation.wav").read_bytes() == setup.merged_bytes + methods = [call[0] for call in setup.client.mock_calls] + assert ( + methods.index("beta.voice_agents.live_conversations.download_audio") + < methods.index("beta.voice_agents.live_conversations.delete") + < methods.index("agents.delete_version") + ) + output = capsys.readouterr().out + assert "Created conversation: conversation-test" in output + assert "Deleted sample conversation: conversation-test" in output + assert "Read-only REST run" not in output + + +@pytest.mark.parametrize("stage", ["create-agent", "connect", "greeting", "read", "delete"]) +def test_live_rest_failure_cleans_up_only_resources_created_by_the_sample(created_rest_sample, stage): + setup = created_rest_sample + failure = RuntimeError("sample workflow failed") + if stage == "create-agent": + setup.client.agents.create_version.side_effect = failure + elif stage == "connect": + setup.client.beta.voice_agents.live.connect.return_value.__enter__.side_effect = failure + elif stage == "greeting": + setup.events[1] = {"type": "error", "error": {"code": "test_error", "message": str(failure)}} + elif stage == "read": + setup.operations.download_audio.side_effect = failure + else: + setup.operations.delete.side_effect = failure + + with pytest.raises(RuntimeError, match="sample workflow failed"): + setup.run() + + if stage in ("create-agent", "connect"): + setup.operations.delete.assert_not_called() + else: + setup.operations.delete.assert_called_once() + if stage == "create-agent": + setup.client.agents.delete_version.assert_not_called() + else: + setup.client.agents.delete_version.assert_called_once() + + +def test_live_rest_finalization_waits_for_durable_speech_and_recording(created_rest_sample): + setup = created_rest_sample + in_progress = models.VoiceAgentLiveConversation( + {"id": "conversation-test", "status": "in_progress", "created_at": 1} + ) + not_ready = HttpResponseError("recording not ready") + not_ready.status_code = 409 + not_ready.error = SimpleNamespace(code="recording_not_ready") + setup.operations.get.side_effect = [ + ResourceNotFoundError("not persisted yet"), + in_progress, + setup.conversation, + setup.conversation, + ] + setup.operations.get_audio.side_effect = [not_ready, setup.recording] + + setup.sample._wait_for_recording(setup.operations, "owned-agent", "conversation-test") + + assert setup.clock.now == 3 + assert setup.operations.get.call_count == 4 + assert setup.operations.get_audio.call_count == 2 + setup.operations.list_items.assert_called_once_with("owned-agent", "conversation-test", limit=20, order="asc") + setup.operations.delete.assert_not_called() + + def test_live_rest_sample_downloads_to_user_temp_without_modifying_resources( rest_sample, tmp_path, capsys, monkeypatch ): @@ -1139,9 +1247,12 @@ def test_live_rest_sample_temp_directory_failure_leaves_service_resources_untouc rest_sample.client.__exit__.assert_called_once() -@pytest.mark.parametrize("variable", ["FOUNDRY_VOICE_AGENT_NAME", "FOUNDRY_VOICE_CONVERSATION_ID"]) -def test_live_rest_sample_rejects_blank_resource_identifiers(rest_sample, monkeypatch, variable): - monkeypatch.setenv(variable, " ") +@pytest.mark.parametrize("agent_name", [None, "", " "], ids=["unset", "empty", "blank"]) +def test_live_rest_sample_requires_an_agent_name_for_an_existing_conversation(rest_sample, monkeypatch, agent_name): + if agent_name is None: + monkeypatch.delenv("FOUNDRY_VOICE_AGENT_NAME") + else: + monkeypatch.setenv("FOUNDRY_VOICE_AGENT_NAME", agent_name) with pytest.raises(ValueError, match="nonblank"): rest_sample.run() From d8638f6ec9cd644e13f4794601ed851221e41f5f Mon Sep 17 00:00:00 2001 From: Xiting Zhang Date: Tue, 27 Oct 2026 15:05:14 -0700 Subject: [PATCH 09/10] Simplify GPT Live REST sample test coverage Reuse the existing REST fixture for success and failure cleanup checks. Remove the simulated greeting, clock, and persistence test setup without changing the sample. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../tests/samples/test_live_samples.py | 145 +++--------------- 1 file changed, 19 insertions(+), 126 deletions(-) diff --git a/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py b/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py index 7bd4284fa09d..3529030f86ed 100644 --- a/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py +++ b/sdk/ai/azure-ai-projects/tests/samples/test_live_samples.py @@ -18,7 +18,7 @@ import wave import pytest -from azure.core.exceptions import HttpResponseError, ResourceNotFoundError +from azure.core.exceptions import HttpResponseError from azure.ai.projects import models @@ -958,137 +958,30 @@ def rest_sample(monkeypatch, sample_environment, tmp_path): ) -@pytest.fixture -def created_rest_sample(monkeypatch, rest_sample, sample_environment): +@pytest.mark.parametrize("read_fails", [False, True], ids=["success", "read-failure"]) +def test_live_rest_sample_cleans_up_its_own_conversation(rest_sample, sample_environment, monkeypatch, read_fails): monkeypatch.delenv("FOUNDRY_VOICE_CONVERSATION_ID") - clock = SimpleNamespace(now=0.0) - clock.monotonic = lambda: clock.now - - def sleep(duration): - clock.now += duration - - clock.sleep = sleep - monkeypatch.setattr(rest_sample.sample, "time", clock, raising=False) - monkeypatch.setattr(sample_environment.helper, "time", clock) + monkeypatch.setattr(rest_sample.sample, "_receive_greeting", MagicMock()) + monkeypatch.setattr(rest_sample.sample, "_wait_for_recording", MagicMock()) started = _started() started.conversation_id = rest_sample.conversation.id - events = [ - started, - models.LiveOutputAudioDelta(delta=base64.b64encode(b"\x00\x08" * 1200).decode("ascii")), - models.LiveServerEventLiveOutputTranscriptDelta(delta="Hello, welcome."), - ] - connection = sample_environment.connection - - def receive(*, timeout): - if connection.session.close.called: - return _closed() - if events: - return events.pop(0) - clock.now += timeout - raise TimeoutError - - connection.recv.side_effect = receive - rest_sample.connection = connection - rest_sample.clock = clock - rest_sample.events = events - return rest_sample - - -@pytest.mark.parametrize("conversation_id", [None, "", " "], ids=["unset", "empty", "blank"]) -def test_live_rest_sample_creates_and_deletes_its_conversation_when_id_is_omitted( - created_rest_sample, capsys, monkeypatch, conversation_id -): - setup = created_rest_sample - if conversation_id is not None: - monkeypatch.setenv("FOUNDRY_VOICE_CONVERSATION_ID", conversation_id) - setup.run() - - create = setup.client.agents.create_version.call_args.kwargs - agent_name = create["agent_name"] - assert agent_name.startswith("stored-agent-test-") and agent_name != "stored-agent-test" - definition = create["definition"] - assert isinstance(definition, models.VoiceAgentLiveDefinition) - assert definition.model == "live-deployment-test" - assert definition.store is True and definition.greeting.text - assert definition.audio.format.rate == 24000 - setup.client.beta.voice_agents.live.connect.assert_called_once_with( - agent_name=agent_name, agent_version="owned-version", store=True - ) - setup.connection.session.start.assert_not_called() - setup.connection.session.close.assert_called_once() - setup.connection.session.input_audio.append.assert_called() - assert all( - call.kwargs["audio"] == bytes(2400) for call in setup.connection.session.input_audio.append.call_args_list - ) - setup.operations.get.assert_any_call(agent_name, "conversation-test") - setup.operations.download_audio.assert_called_once_with(agent_name, "conversation-test") - setup.operations.delete.assert_called_once_with(agent_name, "conversation-test") - setup.client.agents.delete_version.assert_called_once_with(agent_name=agent_name, agent_version="owned-version") - setup.client.agents.delete.assert_not_called() - assert (setup.directories[0] / "conversation.wav").read_bytes() == setup.merged_bytes - methods = [call[0] for call in setup.client.mock_calls] - assert ( - methods.index("beta.voice_agents.live_conversations.download_audio") - < methods.index("beta.voice_agents.live_conversations.delete") - < methods.index("agents.delete_version") - ) - output = capsys.readouterr().out - assert "Created conversation: conversation-test" in output - assert "Deleted sample conversation: conversation-test" in output - assert "Read-only REST run" not in output - - -@pytest.mark.parametrize("stage", ["create-agent", "connect", "greeting", "read", "delete"]) -def test_live_rest_failure_cleans_up_only_resources_created_by_the_sample(created_rest_sample, stage): - setup = created_rest_sample - failure = RuntimeError("sample workflow failed") - if stage == "create-agent": - setup.client.agents.create_version.side_effect = failure - elif stage == "connect": - setup.client.beta.voice_agents.live.connect.return_value.__enter__.side_effect = failure - elif stage == "greeting": - setup.events[1] = {"type": "error", "error": {"code": "test_error", "message": str(failure)}} - elif stage == "read": - setup.operations.download_audio.side_effect = failure - else: - setup.operations.delete.side_effect = failure - - with pytest.raises(RuntimeError, match="sample workflow failed"): - setup.run() - - if stage in ("create-agent", "connect"): - setup.operations.delete.assert_not_called() - else: - setup.operations.delete.assert_called_once() - if stage == "create-agent": - setup.client.agents.delete_version.assert_not_called() + sample_environment.connection.recv.side_effect = [started, _closed()] + if read_fails: + rest_sample.operations.download_audio.side_effect = RuntimeError("REST read failed") + with pytest.raises(RuntimeError, match="REST read failed"): + rest_sample.run() else: - setup.client.agents.delete_version.assert_called_once() - + rest_sample.run() -def test_live_rest_finalization_waits_for_durable_speech_and_recording(created_rest_sample): - setup = created_rest_sample - in_progress = models.VoiceAgentLiveConversation( - {"id": "conversation-test", "status": "in_progress", "created_at": 1} + rest_sample.client.agents.create_version.assert_called_once() + agent_name = rest_sample.client.agents.create_version.call_args.kwargs["agent_name"] + assert agent_name != "stored-agent-test" + rest_sample.operations.get.assert_called_once_with(agent_name, "conversation-test") + rest_sample.operations.delete.assert_called_once_with(agent_name, "conversation-test") + rest_sample.client.agents.delete_version.assert_called_once_with( + agent_name=agent_name, agent_version="owned-version" ) - not_ready = HttpResponseError("recording not ready") - not_ready.status_code = 409 - not_ready.error = SimpleNamespace(code="recording_not_ready") - setup.operations.get.side_effect = [ - ResourceNotFoundError("not persisted yet"), - in_progress, - setup.conversation, - setup.conversation, - ] - setup.operations.get_audio.side_effect = [not_ready, setup.recording] - - setup.sample._wait_for_recording(setup.operations, "owned-agent", "conversation-test") - - assert setup.clock.now == 3 - assert setup.operations.get.call_count == 4 - assert setup.operations.get_audio.call_count == 2 - setup.operations.list_items.assert_called_once_with("owned-agent", "conversation-test", limit=20, order="asc") - setup.operations.delete.assert_not_called() + rest_sample.client.agents.delete.assert_not_called() def test_live_rest_sample_downloads_to_user_temp_without_modifying_resources( From f7328667c7ac3888a10aacdc751f00248f84613b Mon Sep 17 00:00:00 2001 From: Xiting Zhang Date: Wed, 28 Oct 2026 17:21:43 -0700 Subject: [PATCH 10/10] Simplify handwritten VoiceAgents documentation Clarify Realtime and GPT Live setup, sample choices, and client descriptions without changing runtime behavior, public APIs, generated code, or tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- sdk/ai/azure-ai-projects/CHANGELOG.md | 10 +- sdk/ai/azure-ai-projects/README.md | 158 ++++++++---------- .../azure/ai/projects/_live.py | 53 +++--- .../azure/ai/projects/_realtime.py | 63 +++---- .../azure/ai/projects/aio/_live.py | 57 +++---- .../azure/ai/projects/aio/_realtime.py | 78 +++------ .../samples/agents/voice/live_sample_utils.py | 26 +-- .../agents/voice/sample_voice_agent_basic.py | 24 +-- .../voice/sample_voice_agent_basic_async.py | 19 +-- .../voice/sample_voice_agent_generate.py | 26 +-- ...ice_agent_live_audio_conversation_async.py | 64 +++---- .../sample_voice_agent_live_connection.py | 16 +- .../sample_voice_agent_live_function_tool.py | 25 ++- ...mple_voice_agent_live_read_conversation.py | 55 +++--- ...mple_voice_agent_live_text_conversation.py | 28 ++-- .../sample_voice_agent_read_conversation.py | 47 ++---- ...ple_voice_agent_read_conversation_audio.py | 58 +++---- ...agent_realtime_audio_conversation_async.py | 40 ++--- ...mple_voice_agent_realtime_function_tool.py | 33 ++-- ..._voice_agent_realtime_text_conversation.py | 36 ++-- ..._agent_realtime_text_conversation_async.py | 34 ++-- .../voice/sample_voice_agent_versions.py | 21 +-- .../voice/sample_voice_agent_with_tools.py | 38 ++--- .../samples/agents/voice/voice_sample_util.py | 19 +-- .../logs/sample_log_from_voice_agent.py | 18 +- 25 files changed, 402 insertions(+), 644 deletions(-) diff --git a/sdk/ai/azure-ai-projects/CHANGELOG.md b/sdk/ai/azure-ai-projects/CHANGELOG.md index d139f03d9554..26917000b65d 100644 --- a/sdk/ai/azure-ai-projects/CHANGELOG.md +++ b/sdk/ai/azure-ai-projects/CHANGELOG.md @@ -4,7 +4,7 @@ ### Features Added -* Added preview GPT Live support through `beta.voice_agents.live`, with synchronous and asynchronous WebSocket clients, typed Live events, session controls, audio streaming, and Responses-delegation tool input/output. +* Added preview GPT Live clients at `beta.voice_agents.live` for text and audio streaming, typed events, session controls, and Responses function-tool results. Both synchronous and asynchronous clients are available. * Added generated GPT Live voice-agent configuration, delegation, and client/server event models. * Added `VoiceAgentLiveSessionResource` and `VoiceAgentLiveDelegationItem` so Live lifecycle and delegation events preserve effective Foundry-managed ownership and resolved agent versions. * Added persisted Live conversation, item, and audio REST operations under `beta.voice_agents.live_conversations`. @@ -12,13 +12,13 @@ ### Bugs Fixed -* Preserved concrete Live/Realtime voice-agent types on agent readback without inserting an omitted `sub_protocol`. -* Accepted non-dictionary mappings in GPT Live raw events and structured connection inputs. -* Fixed concurrent asynchronous Voice Agent close calls to await shared cleanup, including when a caller is cancelled. +* Preserved Live and Realtime agent types when reading agents, without adding `sub_protocol` when it was omitted. +* Accepted mappings other than dictionaries for GPT Live raw events and structured inputs. +* Fixed overlapping async Voice Agent close calls to wait for the same cleanup, even if a caller is cancelled. ### Sample updates -* Added GPT Live samples under `samples/agents/voice` demonstrating session lifecycle and configuration, single- and multi-turn text conversations, function-tool calling, WAV audio streaming, microphone capture with speaker playback, and stored-conversation/item/audio retrieval with Foundry-managed WAV downloads to a unique user-temporary folder. The REST sample can create and clean up its own stored greeting conversation or read an existing conversation without modifying it. +* Added GPT Live samples for session setup, text conversations, function tools, WAV and microphone audio, and stored-conversation readback. The REST sample creates and cleans up a conversation when no ID is supplied, or reads an existing conversation without changing it. Audio downloads use a unique user-temporary folder. ## 2.9.0 (2026-10-16) diff --git a/sdk/ai/azure-ai-projects/README.md b/sdk/ai/azure-ai-projects/README.md index 1aedd54c8d86..676bf33161ce 100644 --- a/sdk/ai/azure-ai-projects/README.md +++ b/sdk/ai/azure-ai-projects/README.md @@ -197,9 +197,8 @@ The table below lists the operation groups supported by the client library, with ### Voice Agent realtime transport (preview) -Select the connection transport with the optional `transport` keyword. The SDK constructs the wire -query internally; callers do not need to put the transport in `extra_query`. Omitting the option -preserves the service's default WebSocket behavior. +Use `project_client.beta.voice_agents.realtime` for agents created with `VoiceAgentDefinition`. +Connections use WebSocket by default. You can also select the transport explicitly: ```python from azure.ai.projects.models import VoiceAgentTransport @@ -221,101 +220,82 @@ async with async_project_client.beta.voice_agents.realtime.connect( print("Voice Agent connection is ready.") ``` -`VoiceAgentTransport.WEBRTC` selects WebRTC signaling over the WebSocket. The application must manage -the peer connection, media, and SDP offer/answer exchange using `rtc.call.sdp.create` and -`rtc.call.sdp.created` events. Selecting this value does not automatically create a WebRTC media client. +Use `transport`, not `extra_query`, to select a transport. `VoiceAgentTransport.WEBRTC` enables +WebRTC signaling over the WebSocket, but your app must handle the peer connection, audio/video, +and SDP offer/answer exchange (`rtc.call.sdp.create` and `rtc.call.sdp.created`). -### Voice Agent GPT Live (preview) - -GPT Live is a separate protocol at `project_client.beta.voice_agents.live` (and the same namespace -on the asynchronous client). Create a `VoiceAgentLiveDefinition` using your GPT Live **deployment -name**; existing `VoiceAgentDefinition` and `.realtime` calls retain Realtime behavior. -Creating the preview definition through `project_client.agents` requires constructing the client -with `allow_preview=True`, as the samples do. The `.beta` namespace opts in automatically. - -Use `.live.connect(agent_name=..., agent_version=...)` as a context manager and receive -`session.started` before sending commands. Foundry starts the session from the saved definition; -do not send another `session.start`. Live uses `session.input_audio`, `session.instructions`, -`session.thinking`, and `session.commentary`, while `response.item.create` and `response.create` -require Responses delegation. Live has no Realtime `response.cancel`; cancel local receive tasks -normally or request `session.close` and receive `session.closed`, then close the connection. - -Lifecycle snapshots use `VoiceAgentLiveSessionResource`, and delegation-trigger metadata uses -`VoiceAgentLiveDelegationItem`. For Foundry-managed sessions, `session.delegation` reports the -target agent name and resolved version, while delegation-created events report -`target="foundry_managed_agent"`. This is ownership metadata, not the delegated task text; -Foundry handles GPT Live's native client delegation for these sessions. - -Live conversation REST operations use the emitted implementation without handwritten paging overrides. -The current emitter's REST pagers follow `last_id` without checking `has_more`; this known generation -limitation does not affect WebSocket sessions. +Start with the [text conversation sample](samples/agents/voice/sample_voice_agent_realtime_text_conversation.py) +or [microphone sample](samples/agents/voice/sample_voice_agent_realtime_audio_conversation_async.py). -Connection, text, PCM audio-streaming, and function-tool examples are in `samples/agents/voice` -in the [Samples][samples] folder. The optional `[voice]` dependencies are unchanged. +### Voice Agent GPT Live (preview) -The [text sample](samples/agents/voice/sample_voice_agent_live_text_conversation.py) runs a -single-turn conversation by default. To demonstrate headless multi-turn conversation in one -session, set `TEXT_PROMPTS = ["My name is Ada.", "What is my name?"]` in the sample, then run: +GPT Live uses a separate client and agent definition; it does not change Realtime behavior. +The synchronous and asynchronous clients use the same access paths: -```powershell -python .\samples\agents\voice\sample_voice_agent_live_text_conversation.py -``` - -The text sample prints Responses backend text, not completion of the voice frontend's speech. -The WAV audio example uses a bounded speech-inactivity heuristic because Live has no speech-finished -event. Continued silent PCM does not keep it waiting forever, and silence-only output is not a -successful spoken reply. A long speech pause can still split a reply. +| | Realtime | GPT Live | +|---|---|---| +| Agent definition | `VoiceAgentDefinition` | `VoiceAgentLiveDefinition` | +| WebSocket client | `.beta.voice_agents.realtime` | `.beta.voice_agents.live` | +| Stored conversations | `.beta.voice_agents.conversations` | `.beta.voice_agents.live_conversations` | -The same [GPT Live audio sample](samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py) -also supports live microphone capture and speaker playback, following the Realtime sample's -nonblocking PyAudio device pattern. This is the only asynchronous Live sample because sending -and receiving audio concurrently and handling audio devices benefit from async execution. -Set `USE_MICROPHONE = True` in the sample before running microphone mode: +Install the WebSocket dependencies: ```powershell -pip install pyaudio -python .\samples\agents\voice\sample_voice_agent_live_audio_conversation_async.py +pip install "azure-ai-projects[voice]" azure-identity python-dotenv ``` -Microphone mode requires `FOUNDRY_PROJECT_ENDPOINT` and `FOUNDRY_VOICE_AGENT_MODEL`, but not -`FOUNDRY_VOICE_AUDIO_INPUT_PATH`. Speak normally after the sample prints "Speak now"; use -headphones to avoid feedback, and press Ctrl-C to close the session and delete its temporary -agent version. The sample reports actual input/output PCM bytes, durations, and any dropped -capture frames. No audio file is saved unless `SAVE_AUDIO = True` is set in the sample. Saved -reply audio goes into a unique user-temporary folder whose location is printed; it remains -available after exit, so remove the folder when it is no longer needed. The WAV contains the -received reply, not microphone input or telemetry. Conversation persistence remains disabled. -Live owns turn handling, so this mode does not use Realtime `response.cancel`. -Leave `USE_MICROPHONE = False` to use the WAV workflow without requiring PyAudio. - -The Live samples do not accept command-line arguments. Connection/session controls, text, -function calling, and stored-conversation REST each have one synchronous example; audio has -the single asynchronous example described above. Samples delete only the agent version they -create. The stored-conversation REST sample also deletes a conversation only when it created it. - -The [REST sample](samples/agents/voice/sample_voice_agent_live_read_conversation.py) -can run without a pre-created conversation. Set `FOUNDRY_PROJECT_ENDPOINT` and -`FOUNDRY_VOICE_AGENT_MODEL`. When `FOUNDRY_VOICE_CONVERSATION_ID` is unset or blank, the sample -creates a uniquely named temporary agent and a stored greeting session over a WebSocket, -waits for the conversation and its audio to finish persisting, and reads them through REST. -It deletes its own conversation and agent version afterward, including on failure. No microphone, -input WAV, or Responses backend is needed. An optional `FOUNDRY_VOICE_AGENT_NAME` is used only -as a name prefix in this mode, so an existing agent with that name is not modified. - -Alternatively, supply both `FOUNDRY_VOICE_AGENT_NAME` and `FOUNDRY_VOICE_CONVERSATION_ID` to read -an existing conversation without opening a WebSocket or requiring a model deployment name. -The ID must be `session.started.conversation_id` from a successfully ended speech session with -effective `store=True`, not `session.id`. Existing conversations and agents are never deleted. - -The sample demonstrates conversation/item listing and retrieval, audio metadata, and merged/per-item -Foundry-managed WAV downloads. It creates a unique folder in your user temporary directory and prints -the location. Files remain available after the sample exits; remove the local folder when it is no -longer needed. These downloads contain conversation content, not telemetry. BYOS audio requires your -own storage credentials and is not downloaded by this sample. The sample deletes only service -resources it created, and it does not accept command-line options. -It uses the dedicated `.beta.voice_agents.live_conversations` routes. A deployment that -does not support those routes returns HTTP 404, which is surfaced rather than silently falling back -to `.beta.voice_agents.conversations`. +Set `FOUNDRY_PROJECT_ENDPOINT` and `FOUNDRY_VOICE_AGENT_MODEL` to your project endpoint and +GPT Live **deployment name**, not a model catalog ID. Authenticate with `DefaultAzureCredential`. +Use `allow_preview=True` when creating an agent through `project_client.agents`; the `.beta` +client opts in automatically. + +Choose a sample: + +| Task | Sample | +|---|---| +| Open, update, and close a session | [Connection](samples/agents/voice/sample_voice_agent_live_connection.py) | +| Send one or more text messages | [Text conversation](samples/agents/voice/sample_voice_agent_live_text_conversation.py) | +| Handle a function call and return its result | [Function tool](samples/agents/voice/sample_voice_agent_live_function_tool.py) | +| Send a WAV file or use a microphone and speakers | [Audio conversation](samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py) | +| Read a stored conversation and download its audio | [Conversation REST](samples/agents/voice/sample_voice_agent_live_read_conversation.py) | + +Run samples without command-line arguments; edit the settings in each file as needed: + +- **Text and tools:** also set `FOUNDRY_MODEL_NAME` to a Responses model deployment available + to your Live resource. This model handles delegated requests. Set `TEXT_PROMPTS` in the text + sample to a list of messages for a multi-turn conversation. Printed backend text does not + mean the voice agent has finished speaking. +- **Audio:** by default, set `FOUNDRY_VOICE_AUDIO_INPUT_PATH` to a PCM16, mono, 24-kHz WAV file. + For microphone mode, install `pyaudio` and set `USE_MICROPHONE = True` instead. Use headphones + and press Ctrl-C to stop. Async lets the sample send and receive audio at the same time. + The WAV mode ends after three seconds without active reply audio or transcript updates; + long pauses can cut a reply short, and silence alone is not a successful reply. +- **Stored conversations:** omit `FOUNDRY_VOICE_CONVERSATION_ID` to create a short greeting + conversation and delete it afterward. No microphone, input WAV, or Responses model is needed. + To read an existing conversation, set its ID and `FOUNDRY_VOICE_AGENT_NAME`; no Live model or + WebSocket is then needed. Use `session.started.conversation_id`, not `session.id`, from a + completed session with `store=True`. The existing agent and conversation are never changed. + +GPT Live samples delete only the service resources they create. Streaming samples use `store=False`; +set `SAVE_AUDIO = True` in the audio sample to save reply audio locally. Saved replies and REST +downloads contain conversation content, not telemetry. They remain in a unique user-temporary +folder whose path is printed; remove the folder when finished. Bring-your-own-storage (BYOS) +recordings need your own storage credentials and are not downloaded by the REST sample. + +**Session basics:** open `.live.connect(...)` in a context manager and wait for `session.started`. +Foundry starts the session; do not send `session.start`. Use `session.input_audio` for audio, +`session.instructions` for guidance, `session.thinking` for unspoken context, and +`session.commentary` for spoken commentary. The `response` methods require Responses delegation. +To end a session, send `session.close`, wait for `session.closed`, and leave the context. +Live has no `response.cancel`; cancel local receive tasks when needed. + +Session events use `VoiceAgentLiveSessionResource`; delegation events use `VoiceAgentLiveDelegationItem`. +For Foundry-managed sessions, these report the target agent and resolved version, not the task text. +Foundry handles delegations with `target="foundry_managed_agent"`. + +**Known REST limitation:** the pager advances using `last_id` without checking `has_more`. +This does not affect WebSocket sessions. Unsupported Live REST routes return HTTP 404; +the SDK does not fall back to the Realtime conversation API. ## Client-side tracing diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/_live.py b/sdk/ai/azure-ai-projects/azure/ai/projects/_live.py index b2be76a26795..d0b694e50bcc 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/_live.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/_live.py @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------- -"""Synchronous GPT Live WebSocket client, alongside the Realtime client.""" +"""Synchronous GPT Live sessions at ``client.beta.voice_agents.live``.""" from __future__ import annotations @@ -25,17 +25,17 @@ def __init__(self, connection: BetaLiveConnection) -> None: class TextResource(_Resource): - """Append instructions, silent thinking, or speakable commentary.""" + """Send instructions, unspoken context, or spoken commentary.""" def __init__(self, connection: BetaLiveConnection, event_model: type[models.LiveClientEvent]) -> None: super().__init__(connection) self._event_model = event_model def append(self, *, content: str, delegation_id: Optional[str], event_id: Optional[str] = None) -> None: - """Append text, optionally associated with a client delegation. + """Send text, optionally tied to a request delegated to your app. :keyword str content: Text to append. - :keyword delegation_id: Delegation ID, or None for independent context. + :keyword delegation_id: The delegated request's ID, or None for standalone text. :paramtype delegation_id: str or None :keyword event_id: Optional client event identifier. :paramtype event_id: str or None @@ -44,7 +44,7 @@ def append(self, *, content: str, delegation_id: Optional[str], event_id: Option class InputAudioResource(_Resource): - """Stream primary-transport audio in the session's configured format.""" + """Send audio using the session's configured format.""" def append(self, *, audio: Union[str, bytes], event_id: Optional[str] = None) -> None: """Send raw audio bytes or base64 without a container header. @@ -75,7 +75,7 @@ def unmute(self, *, event_id: Optional[str] = None) -> None: class SessionResource(_Resource): - """Live session controls; startup is owned by Foundry, not the client.""" + """Control a Live session that Foundry starts when the connection opens.""" def __init__(self, connection: BetaLiveConnection) -> None: super().__init__(connection) @@ -87,9 +87,9 @@ def __init__(self, connection: BetaLiveConnection) -> None: def update( self, *, session: Union[models.LiveSessionUpdateParams, Mapping[str, Any]], event_id: Optional[str] = None ) -> None: - """Update mutable delegation settings, not startup model/audio. + """Change delegation settings. The model and audio format cannot change mid-session. - :keyword session: Sparse session delegation settings. + :keyword session: Only the delegation settings to change. :paramtype session: ~azure.ai.projects.models.LiveSessionUpdateParams or Mapping[str, Any] :keyword event_id: Optional client event identifier. :paramtype event_id: str or None @@ -106,10 +106,10 @@ def close(self, *, event_id: Optional[str] = None) -> None: class ResponseItemResource(_Resource): - """Append user messages or function results to a Responses delegation.""" + """Send user messages or tool results to the Responses backend model.""" def create(self, *, item: Mapping[str, Any], event_id: Optional[str] = None) -> None: - """Submit an item; requires Responses delegation. + """Send a message or tool result. Requires a Responses backend. :keyword item: Responses input item, including function_call_output. :paramtype item: Mapping[str, Any] @@ -120,7 +120,7 @@ def create(self, *, item: Mapping[str, Any], event_id: Optional[str] = None) -> class ResponseResource(_Resource): - """Controls for the Responses backend, not Realtime response controls.""" + """Request replies from the Responses backend model.""" def __init__(self, connection: BetaLiveConnection) -> None: super().__init__(connection) @@ -136,10 +136,11 @@ def create(self, *, event_id: Optional[str] = None) -> None: class BetaLiveConnection(_BaseWebSocketConnection[LiveServerEvent, LiveClientEvent]): - """An open Live connection. Server errors are returned as typed events. + """Send and receive events on a GPT Live connection. - Local cancellation closes the transport through the context manager. Live - has no response.cancel command; session.close requests protocol shutdown. + Service errors arrive as error events. Use the context manager to close + the connection. Send ``session.close`` to end the session; Live has no + ``response.cancel`` command. """ _logger = _LOGGER @@ -158,7 +159,7 @@ def __iter__(self) -> Iterator[LiveServerEvent]: return self._iter() def recv(self, *, timeout: Optional[float] = None) -> LiveServerEvent: - """Receive a generated Live event, or a mapping for an unknown event. + """Receive an SDK event model, or a dictionary for an unknown event type. :keyword timeout: Maximum receive wait in seconds; None waits indefinitely. :paramtype timeout: float or None @@ -170,7 +171,7 @@ def recv(self, *, timeout: Optional[float] = None) -> LiveServerEvent: return super().recv(timeout=timeout) def send(self, event: LiveClientEvent) -> None: - """Send a generated Live event, a mapping, or a JSON string. + """Send an SDK event model, mapping, or JSON string. :param event: The client event. :type event: ~azure.ai.projects.operations.LiveClientEvent @@ -201,7 +202,7 @@ def __enter__(self) -> BetaLiveConnection: return self.enter() def enter(self) -> BetaLiveConnection: - """Open the Live connection without sending a second startup event. + """Open the connection. Foundry starts the Live session automatically. :return: The Live connection. :rtype: ~azure.ai.projects.operations.BetaLiveConnection @@ -221,10 +222,10 @@ def _create_connection(self, connection: ClientConnection) -> BetaLiveConnection class BetaLive: - """Live protocol entry point at client.beta.voice_agents.live. + """Connect to GPT Live through ``client.beta.voice_agents.live``. - Foundry starts the session using the persisted Live agent definition. Receive - session.started before sending session commands; do not send session.start. + Foundry starts the session from the saved agent settings. Wait for + ``session.started`` before sending commands; do not send ``session.start``. """ def __init__(self, *args: Any, **kwargs: Any) -> None: @@ -246,16 +247,16 @@ def connect( extra_headers: Optional[Mapping[str, str]] = None, **kwargs: Any, ) -> BetaLiveConnectionManager: - """Open a Live WebSocket session; network I/O starts on context entry. + """Connect to a GPT Live agent when entering the context manager. - :keyword str agent_name: Persisted Live agent name. - :keyword agent_version: Optional pinned agent version. + :keyword str agent_name: The name of an existing GPT Live agent. + :keyword agent_version: A specific agent version, if needed. :paramtype agent_version: str or None - :keyword structured_inputs: Session template values. + :keyword structured_inputs: Values for placeholders in the agent's instructions and greeting. :paramtype structured_inputs: Mapping[str, Any] or None - :keyword store: Optional conversation persistence override. + :keyword store: Save conversation content and audio. None uses the agent's setting. :paramtype store: bool or None - :keyword connection_url: Same-origin wss URL override. + :keyword connection_url: A wss URL on the project endpoint's origin, instead of the default route. :paramtype connection_url: str or None :keyword api_version: API version override. :paramtype api_version: str or None diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py b/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py index b11f64b00c31..bd5735a34144 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/_realtime.py @@ -4,15 +4,11 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------- -"""Hand-written sync realtime (WebSocket) streaming client for voice agents. +"""Synchronous Realtime voice-agent client. -This is the synchronous counterpart of :mod:`azure.ai.projects.aio._realtime`. See that -module's docstring for the full design rationale; the two modules are kept structurally -identical (sync method names drop the ``async``/``await`` keywords) so fixes/features land in -both at once. - -``websockets`` is required for this feature and is *not* a hard dependency of the package; it -is imported lazily so importing the SDK never fails when it is absent. +Use ``client.beta.voice_agents.realtime.connect(...)`` as a context manager. +Send text or audio through the connection and receive model events. +Install ``azure-ai-projects[voice]`` for the optional WebSocket dependencies. """ from __future__ import annotations @@ -49,15 +45,10 @@ _LOGGER = logging.getLogger("azure.ai.projects.realtime") -# The realtime WebSocket route is voice-agent-specific (see `_to_ws_url`'s -# `/endpoint/protocols/voice` path), so this is always the correct opt-in value -- callers -# cannot and do not need to override it. +# Voice-agent WebSocket requests need the same preview header as REST requests. _VOICE_AGENT_FEATURE_HEADER: str = _AgentDefinitionOptInKeys.VOICE_AGENTS_V1_PREVIEW.value -# Identifies the SDK to the service on the WebSocket handshake, which otherwise falls back to -# the underlying `websockets` library's generic default (the generated HTTP surface gets this -# for free from the pipeline's own UserAgentPolicy; this hand-written client builds its own -# request instead, so it needs to opt in explicitly the same way). +# The WebSocket handshake does not use the HTTP pipeline, so identify the SDK explicitly. _USER_AGENT: str = UserAgentPolicy(sdk_moniker=f"ai-projects/{VERSION}").user_agent if TYPE_CHECKING: @@ -67,14 +58,7 @@ class _ConfigProvider(Protocol): - """Anything exposing the shared client configuration (endpoint, credential, etc.). - - :class:`~azure.ai.projects.operations.BetaVoiceAgentsOperations` (accessed as - ``client.beta.voice_agents``) satisfies this: it is constructed with the same shared - configuration instance as the top-level client, so ``client.beta.voice_agents.realtime`` can - reuse the endpoint/credential wiring without needing a back-reference to the top-level client - itself. - """ + """Provide the project endpoint, credential, and shared client settings.""" _config: "AIProjectClientConfiguration" @@ -679,8 +663,8 @@ class BetaRealtimeConnection( ): # pylint: disable=too-many-instance-attributes """An open realtime WebSocket connection to a voice agent. - Iterate over the connection to receive strongly-typed server events, and use the - sub-namespaces to send strongly-typed client events:: + Iterate to receive events. Use ``conversation``, ``response``, and the audio + buffers to send messages and control replies:: with client.beta.voice_agents.realtime.connect(agent_name="my-agent") as conn: for event in conn: @@ -730,9 +714,8 @@ def _iter(self) -> Iterator[ServerEvent]: def recv(self, *, timeout: Optional[float] = None) -> ServerEvent: """Receive and parse the next server event. - Known event types are returned as their strongly-typed - ``VoiceAgentServerEventXxx`` model. Event types not (yet) represented by a - generated model are returned as a plain ``dict`` for forward compatibility. + Known events use SDK model classes. Unknown event types are returned + as dictionaries. :keyword timeout: Maximum time in seconds to wait for the next event. If ``None`` (the default), block until an event is received. If no event arrives within @@ -750,7 +733,7 @@ def recv(self, *, timeout: Optional[float] = None) -> ServerEvent: def send(self, event: ClientEvent) -> None: """Send a client event over the connection. - :param event: A strongly-typed client event, a ready-made mapping, or a raw JSON string. + :param event: A client event model, mapping, or JSON string. :type event: ~azure.ai.projects.ClientEvent or str :raises ValueError: If ``event`` is a ``str`` that is not valid JSON. """ @@ -940,10 +923,9 @@ def _create_connection(self, connection: "ClientConnection") -> BetaRealtimeConn class BetaRealtime: # pylint: disable=too-few-public-methods - """BetaRealtime streaming entry point, exposed as ``client.beta.voice_agents.realtime``. + """Connect to a voice agent through ``client.beta.voice_agents.realtime``. - Follows the OpenAI Python realtime surface: obtain it from the HTTP client and open a - connection with :meth:`connect`:: + Open a connection with :meth:`connect`:: from azure.ai.projects import AIProjectClient from azure.identity import DefaultAzureCredential @@ -957,9 +939,7 @@ class BetaRealtime: # pylint: disable=too-few-public-methods if event.type == RealtimeServerEventType.RESPONSE_DONE: break - :param client: The object whose endpoint and credential are reused for the realtime - handshake -- the ``.beta.voice_agents`` sub-client, which shares the same underlying - configuration as the top-level client. + :param client: The voice-agent client that supplies the project endpoint and credential. :type client: ~azure.ai.projects.operations.BetaVoiceAgentsOperations """ @@ -988,16 +968,11 @@ def connect( # pylint: disable=too-many-arguments :keyword agent_session_id: An optional identifier used to correlate the voice session. Default value is None. :paramtype agent_session_id: str or None - :keyword transport: The transport used for this connection. Known values are - ``VoiceAgentTransport.WEBSOCKET`` and ``VoiceAgentTransport.WEBRTC``. When omitted, the - service defaults to WebSocket. The SDK constructs the wire query internally; an explicit - selection overrides an existing transport query value. For WebRTC, the caller must - manage the peer connection, media, and SDP offer/answer exchange. + :keyword transport: ``VoiceAgentTransport.WEBSOCKET`` (service default) or + ``VoiceAgentTransport.WEBRTC``. An explicit value overrides ``extra_query``. + WebRTC requires your app to handle the peer connection, media, and SDP exchange. :paramtype transport: str or ~azure.ai.projects.models.VoiceAgentTransport or None - :keyword structured_inputs: A mapping of structured-input names to their values for this - session (see :attr:`~azure.ai.projects.models.CreateTelephonyCallJobRequest.structured_inputs` - for the analogous shape used elsewhere). Serialized to JSON on the wire. Default value is - None. + :keyword structured_inputs: Values for the agent's structured-input placeholders. :paramtype structured_inputs: Mapping[str, Any] or None :keyword connection_url: Full ``wss://`` URL that overrides the route computed from the client endpoint. Query parameters are still appended. Default value is None. diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_live.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_live.py index 100a10351bd7..974ff44bcae7 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_live.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_live.py @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------- -"""Asynchronous GPT Live WebSocket client, alongside the Realtime client.""" +"""Asynchronous GPT Live sessions at ``client.beta.voice_agents.live``.""" from __future__ import annotations @@ -34,7 +34,7 @@ def __init__(self, connection: AsyncBetaLiveConnection) -> None: class TextResource(_Resource): - """Append instructions, silent thinking, or speakable commentary. + """Send instructions, unspoken context, or spoken commentary. :param connection: The parent Live connection. :type connection: ~azure.ai.projects.aio.operations.AsyncBetaLiveConnection @@ -47,10 +47,10 @@ def __init__(self, connection: AsyncBetaLiveConnection, event_model: type[models self._event_model = event_model async def append(self, *, content: str, delegation_id: Optional[str], event_id: Optional[str] = None) -> None: - """Append text, optionally associated with a client delegation. + """Send text, optionally tied to a request delegated to your app. :keyword str content: Text to append. - :keyword delegation_id: Delegation ID, or None for independent context. + :keyword delegation_id: The delegated request's ID, or None for standalone text. :paramtype delegation_id: str or None :keyword event_id: Optional client event identifier. :paramtype event_id: str or None @@ -59,7 +59,7 @@ async def append(self, *, content: str, delegation_id: Optional[str], event_id: class InputAudioResource(_Resource): - """Stream primary-transport audio in the session's configured format.""" + """Send audio using the session's configured format.""" async def append(self, *, audio: Union[str, bytes], event_id: Optional[str] = None) -> None: """Send raw audio bytes or base64 without a container header. @@ -90,7 +90,7 @@ async def unmute(self, *, event_id: Optional[str] = None) -> None: class SessionResource(_Resource): - """Live session controls; startup is owned by Foundry, not the client. + """Control a Live session that Foundry starts when the connection opens. :param connection: The parent Live connection. :type connection: ~azure.ai.projects.aio.operations.AsyncBetaLiveConnection @@ -106,9 +106,9 @@ def __init__(self, connection: AsyncBetaLiveConnection) -> None: async def update( self, *, session: Union[models.LiveSessionUpdateParams, Mapping[str, Any]], event_id: Optional[str] = None ) -> None: - """Update mutable delegation settings, not startup model/audio. + """Change delegation settings. The model and audio format cannot change mid-session. - :keyword session: Sparse session delegation settings. + :keyword session: Only the delegation settings to change. :paramtype session: ~azure.ai.projects.models.LiveSessionUpdateParams or Mapping[str, Any] :keyword event_id: Optional client event identifier. :paramtype event_id: str or None @@ -125,10 +125,10 @@ async def close(self, *, event_id: Optional[str] = None) -> None: class ResponseItemResource(_Resource): - """Append user messages or function results to a Responses delegation.""" + """Send user messages or tool results to the Responses backend model.""" async def create(self, *, item: Mapping[str, Any], event_id: Optional[str] = None) -> None: - """Submit an item; requires Responses delegation. + """Send a message or tool result. Requires a Responses backend. :keyword item: Responses input item, including function_call_output. :paramtype item: Mapping[str, Any] @@ -141,7 +141,7 @@ async def create(self, *, item: Mapping[str, Any], event_id: Optional[str] = Non class ResponseResource(_Resource): - """Controls for the Responses backend, not Realtime response controls. + """Request replies from the Responses backend model. :param connection: The parent Live connection. :type connection: ~azure.ai.projects.aio.operations.AsyncBetaLiveConnection @@ -161,10 +161,11 @@ async def create(self, *, event_id: Optional[str] = None) -> None: class AsyncBetaLiveConnection(_BaseWebSocketConnection[LiveServerEvent, LiveClientEvent]): - """An open Live connection. Server errors are returned as typed events. + """Send and receive events on a GPT Live connection. - Local cancellation closes the transport through the context manager. Live - has no response.cancel command; session.close requests protocol shutdown. + Service errors arrive as error events. Use the context manager to close + the connection. Send ``session.close`` to end the session; Live has no + ``response.cancel`` command. :param connection: The open WebSocket. :type connection: ~aiohttp.ClientWebSocketResponse @@ -188,7 +189,7 @@ def __aiter__(self) -> AsyncIterator[LiveServerEvent]: return super().__aiter__() async def recv(self) -> LiveServerEvent: - """Receive a generated Live event, or a mapping for an unknown event. + """Receive an SDK event model, or a dictionary for an unknown event type. :return: The server event. :rtype: ~azure.ai.projects.aio.operations.LiveServerEvent @@ -198,7 +199,7 @@ async def recv(self) -> LiveServerEvent: return await super().recv() async def send(self, event: LiveClientEvent) -> None: - """Send a generated Live event, a mapping, or a JSON string. + """Send an SDK event model, mapping, or JSON string. :param event: The client event. :type event: ~azure.ai.projects.aio.operations.LiveClientEvent @@ -210,9 +211,9 @@ async def send(self, event: LiveClientEvent) -> None: class AsyncBetaLiveConnectionManager(_BaseWebSocketConnectionManager[AsyncBetaLiveConnection]): """Context manager for one Live session using the project's credential. - :keyword agent_version: Optional pinned agent version. + :keyword agent_version: A specific agent version, if needed. :paramtype agent_version: str or None - :keyword store: Optional conversation persistence override. + :keyword store: Save conversation content and audio. None uses the agent's setting. :paramtype store: bool or None """ @@ -235,7 +236,7 @@ async def __aenter__(self) -> AsyncBetaLiveConnection: return await self.enter() async def enter(self) -> AsyncBetaLiveConnection: - """Open the Live connection without sending a second startup event. + """Open the connection. Foundry starts the Live session automatically. :return: The Live connection. :rtype: ~azure.ai.projects.aio.operations.AsyncBetaLiveConnection @@ -260,10 +261,10 @@ def _create_connection( class AsyncBetaLive: - """Live protocol entry point at client.beta.voice_agents.live. + """Connect to GPT Live through ``client.beta.voice_agents.live``. - Foundry starts the session using the persisted Live agent definition. Receive - session.started before sending session commands; do not send session.start. + Foundry starts the session from the saved agent settings. Wait for + ``session.started`` before sending commands; do not send ``session.start``. :param args: The voice-agent sub-client whose shared configuration is reused. :type args: ~azure.ai.projects.aio.operations.BetaVoiceAgentsOperations @@ -288,16 +289,16 @@ def connect( extra_headers: Optional[Mapping[str, str]] = None, **kwargs: Any, ) -> AsyncBetaLiveConnectionManager: - """Open a Live WebSocket session; network I/O starts on context entry. + """Connect to a GPT Live agent when entering the context manager. - :keyword str agent_name: Persisted Live agent name. - :keyword agent_version: Optional pinned agent version. + :keyword str agent_name: The name of an existing GPT Live agent. + :keyword agent_version: A specific agent version, if needed. :paramtype agent_version: str or None - :keyword structured_inputs: Session template values. + :keyword structured_inputs: Values for placeholders in the agent's instructions and greeting. :paramtype structured_inputs: Mapping[str, Any] or None - :keyword store: Optional conversation persistence override. + :keyword store: Save conversation content and audio. None uses the agent's setting. :paramtype store: bool or None - :keyword connection_url: Same-origin wss URL override. + :keyword connection_url: A wss URL on the project endpoint's origin, instead of the default route. :paramtype connection_url: str or None :keyword api_version: API version override. :paramtype api_version: str or None diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py index 30750f9728f6..ee62741f0bed 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/aio/_realtime.py @@ -4,29 +4,12 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------- -"""Hand-written async realtime (WebSocket) streaming client for voice agents. +"""Asynchronous Realtime voice-agent client. -BetaRealtime uses a fundamentally different transport (a persistent WebSocket) than the -request/response HTTP surface generated from the service's TypeSpec definition, so it is -hand-written and exposed as the ``AIProjectClient.beta.voice_agents.realtime`` namespace. - -The connection ergonomics follow the OpenAI Python realtime client so that developers moving -between the libraries get a familiar surface: - -* :meth:`AsyncBetaRealtime.connect` returns an async context manager. -* Entering the context yields an :class:`AsyncBetaRealtimeConnection`. -* The connection is async-iterable over inbound, strongly-typed server events and exposes - sub-namespaces (``session``, ``input_audio_buffer``, ``output_audio_buffer``, - ``conversation``, ``response``) for sending strongly-typed outbound client events. - -Outbound and inbound events use the generated ``VoiceAgentClientEventXxx``/ -``VoiceAgentServerEventXxx`` models directly where one exists. ``send`` and ``recv`` still -accept/return plain ``dict`` objects as a forward-compatible fallback for any event ``type`` -the generated models don't yet know about (for example ``conversation.created``, which is a -valid event but does not (yet) have a dedicated generated model in this package). - -``aiohttp`` is required for this feature and is *not* a hard dependency of the package; it is -imported lazily so importing the SDK never fails when it is absent. +Use ``async with client.beta.voice_agents.realtime.connect(...)`` to open a session. +Send text or audio through the connection and use ``async for`` to receive events. +Known events use SDK models; unknown event types are returned as dictionaries. +Install ``azure-ai-projects[voice]`` for the optional WebSocket dependencies. """ from __future__ import annotations @@ -64,15 +47,10 @@ _LOGGER = logging.getLogger("azure.ai.projects.realtime") -# The realtime WebSocket route is voice-agent-specific (see `_to_ws_url`'s -# `/endpoint/protocols/voice` path), so this is always the correct opt-in value -- callers -# cannot and do not need to override it. +# Voice-agent WebSocket requests need the same preview header as REST requests. _VOICE_AGENT_FEATURE_HEADER: str = _AgentDefinitionOptInKeys.VOICE_AGENTS_V1_PREVIEW.value -# Identifies the SDK to the service on the WebSocket handshake, which otherwise falls back to -# the underlying `aiohttp` library's generic default (the generated HTTP surface gets this for -# free from the pipeline's own UserAgentPolicy; this hand-written client builds its own request -# instead, so it needs to opt in explicitly the same way). +# The WebSocket handshake does not use the HTTP pipeline, so identify the SDK explicitly. _USER_AGENT: str = UserAgentPolicy(sdk_moniker=f"ai-projects/{VERSION}").user_agent if TYPE_CHECKING: @@ -82,14 +60,7 @@ class _ConfigProvider(Protocol): - """Anything exposing the shared client configuration (endpoint, credential, etc.). - - :class:`~azure.ai.projects.aio.operations.BetaVoiceAgentsOperations` (accessed as - ``async_client.beta.voice_agents``) satisfies this: it is constructed with the same shared - configuration instance as the top-level client, so ``async_client.beta.voice_agents.realtime`` - can reuse the endpoint/credential wiring without needing a back-reference to the top-level - client itself. - """ + """Provide the project endpoint, credential, and shared client settings.""" _config: "AIProjectClientConfiguration" @@ -734,8 +705,8 @@ class AsyncBetaRealtimeConnection( # pylint: disable=too-many-instance-attribut ): """An open realtime WebSocket connection to a voice agent. - Iterate over the connection to receive strongly-typed server events, and use the - sub-namespaces to send strongly-typed client events:: + Iterate to receive events. Use ``conversation``, ``response``, and the audio + buffers to send messages and control replies:: async with client.beta.voice_agents.realtime.connect(agent_name="my-agent") as conn: await conn.input_audio_buffer.append(audio=chunk) @@ -774,9 +745,8 @@ def __aiter__(self) -> AsyncIterator[ServerEvent]: async def recv(self) -> ServerEvent: """Receive and parse the next server event. - Known event types are returned as their strongly-typed - ``VoiceAgentServerEventXxx`` model. Event types not (yet) represented by a - generated model are returned as a plain ``dict`` for forward compatibility. + Known events use SDK model classes. Unknown event types are returned + as dictionaries. :return: The parsed server event. :rtype: ~azure.ai.projects.aio.ServerEvent @@ -791,7 +761,7 @@ async def recv(self) -> ServerEvent: async def send(self, event: ClientEvent) -> None: """Send a client event over the connection. - :param event: A strongly-typed client event, a ready-made mapping, or a raw JSON string. + :param event: A client event model, mapping, or JSON string. :type event: ~azure.ai.projects.aio.ClientEvent or str :raises ValueError: If ``event`` is a ``str`` that is not valid JSON. """ @@ -1019,10 +989,9 @@ def _create_connection( class AsyncBetaRealtime: # pylint: disable=too-few-public-methods - """BetaRealtime streaming entry point, exposed as ``client.beta.voice_agents.realtime``. + """Connect to a voice agent through ``client.beta.voice_agents.realtime``. - Follows the OpenAI Python realtime surface: obtain it from the HTTP client and open a - connection with :meth:`connect`:: + Open a connection with :meth:`connect`:: from azure.ai.projects.aio import AIProjectClient from azure.identity.aio import DefaultAzureCredential @@ -1036,9 +1005,7 @@ class AsyncBetaRealtime: # pylint: disable=too-few-public-methods if event.type == RealtimeServerEventType.RESPONSE_DONE: break - :param client: The object whose endpoint and credential are reused for the realtime - handshake -- the ``.beta.voice_agents`` sub-client, which shares the same underlying - configuration as the top-level client. + :param client: The voice-agent client that supplies the project endpoint and credential. :type client: ~azure.ai.projects.aio.operations.BetaVoiceAgentsOperations """ @@ -1067,16 +1034,11 @@ def connect( # pylint: disable=too-many-arguments :keyword agent_session_id: An optional identifier used to correlate the voice session. Default value is None. :paramtype agent_session_id: str or None - :keyword transport: The transport used for this connection. Known values are - ``VoiceAgentTransport.WEBSOCKET`` and ``VoiceAgentTransport.WEBRTC``. When omitted, the - service defaults to WebSocket. The SDK constructs the wire query internally; an explicit - selection overrides an existing transport query value. For WebRTC, the caller must - manage the peer connection, media, and SDP offer/answer exchange. + :keyword transport: ``VoiceAgentTransport.WEBSOCKET`` (service default) or + ``VoiceAgentTransport.WEBRTC``. An explicit value overrides ``extra_query``. + WebRTC requires your app to handle the peer connection, media, and SDP exchange. :paramtype transport: str or ~azure.ai.projects.models.VoiceAgentTransport or None - :keyword structured_inputs: A mapping of structured-input names to their values for this - session (see :attr:`~azure.ai.projects.models.CreateTelephonyCallJobRequest.structured_inputs` - for the analogous shape used elsewhere). Serialized to JSON on the wire. Default value is - None. + :keyword structured_inputs: Values for the agent's structured-input placeholders. :paramtype structured_inputs: Mapping[str, Any] or None :keyword connection_url: Full ``wss://`` URL that overrides the route computed from the client endpoint. Query parameters are still appended. Default value is None. diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py b/sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py index de8441ff63e4..e54e6a48660e 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/live_sample_utils.py @@ -27,13 +27,13 @@ def sample_agent_name(scenario: str) -> str: - """Treat the optional name as a prefix so each run owns a distinct version.""" + """Choose a unique agent name so cleanup affects only this run.""" prefix = os.environ.get("FOUNDRY_VOICE_AGENT_NAME") or f"sample-live-{scenario}" return f"{prefix}-{uuid4().hex[:8]}" def text_conversation_prompts(prompts: Optional[Sequence[str]]) -> list[str]: - """Keep the original single-turn example unless explicit scripted prompts are supplied.""" + """Use one default message, or validate the supplied list of messages.""" if prompts is None: return ["Why is Seattle known for rain? Answer in one short sentence."] if isinstance(prompts, str): @@ -60,7 +60,7 @@ def safe_print(text: str, *, end: str = "\n") -> None: @contextmanager def recording_output(path: Path) -> Iterator[BinaryIO]: - """Create an audio file exclusively and remove incomplete or empty downloads. + """Save a recording without overwriting a file; remove failed or empty downloads. :param path: Explicit destination that must not already exist. :type path: pathlib.Path @@ -82,7 +82,7 @@ def recording_output(path: Path) -> Iterator[BinaryIO]: def remaining_time(deadline: float, operation: str) -> float: - """Keep a stream of nonterminal events from extending a timeout forever.""" + """Return the time left, or raise when the deadline expires.""" remaining = deadline - time.monotonic() if remaining <= 0: raise TimeoutError(f"Timed out waiting for {operation}.") @@ -90,7 +90,7 @@ def remaining_time(deadline: float, operation: str) -> float: def check_event(event: LiveServerEvent) -> Mapping[str, Any]: - """Raise service/backend failures and unwrap a nested Responses event.""" + """Raise service errors and read events from the Responses backend.""" if event.get("type") == "error": error = event.get("error") or {} raise RuntimeError(f"Live session error: {error.get('code')}: {error.get('message')}") @@ -115,7 +115,7 @@ def check_event(event: LiveServerEvent) -> Mapping[str, Any]: def check_session_open(event: LiveServerEvent) -> None: - """Do not mistake a premature shutdown for a completed reply.""" + """Raise if the session closes before the requested work finishes.""" if event.get("type") == "session.closed": raise RuntimeError(f"Live session closed before the workflow completed: {event.get('reason')}") @@ -123,7 +123,7 @@ def check_session_open(event: LiveServerEvent) -> None: def wait_for_event( connection: BetaLiveConnection, event_type: str, timeout: float = RESPONSE_TIMEOUT ) -> LiveServerEvent: - """Receive a lifecycle acknowledgement within one bounded deadline.""" + """Wait for a specific event, with a timeout.""" deadline = time.monotonic() + timeout while True: try: @@ -139,7 +139,7 @@ def wait_for_event( async def wait_for_event_async( connection: AsyncBetaLiveConnection, event_type: str, timeout: float = RESPONSE_TIMEOUT ) -> LiveServerEvent: - """Receive an async lifecycle acknowledgement without an unbounded wait.""" + """Wait asynchronously for a specific event, with a timeout.""" deadline = time.monotonic() + timeout while True: try: @@ -154,7 +154,7 @@ async def wait_for_event_async( def close_session(connection: BetaLiveConnection) -> None: - """Call in finally; preserve the original failure while still requesting close.""" + """Close the session without hiding an error already being handled.""" if sys.exc_info()[0] is not None: with suppress(Exception): connection.session.close() @@ -167,7 +167,7 @@ def close_session(connection: BetaLiveConnection) -> None: async def close_session_async(connection: AsyncBetaLiveConnection) -> None: - """Request protocol shutdown before the connection context closes the transport.""" + """End the session before closing its WebSocket.""" if sys.exc_info()[0] is not None: with suppress(Exception): await asyncio.wait_for(connection.session.close(), timeout=CLOSE_TIMEOUT) @@ -214,7 +214,7 @@ def _collect_response_event( def receive_response( connection: BetaLiveConnection, timeout: float = RESPONSE_TIMEOUT ) -> tuple[str, list[Mapping[str, Any]]]: - """Collect one nested Responses cycle, with function calls from complete output items.""" + """Wait for a backend reply and return its text and completed tool calls.""" deadline = time.monotonic() + timeout text_parts: dict[tuple[str, int], str] = {} calls: dict[str, Mapping[str, Any]] = {} @@ -224,14 +224,14 @@ def receive_response( except TimeoutError as exc: raise TimeoutError("Timed out waiting for the Responses backend to complete.") from exc if _collect_response_event(event, text_parts, calls): - # Live strips output from lifecycle snapshots; the granular events carry it. + # Text and tool results arrive before response.completed. return "".join(text_parts.values()).strip(), list(calls.values()) async def receive_response_async( connection: AsyncBetaLiveConnection, timeout: float = RESPONSE_TIMEOUT ) -> tuple[str, list[Mapping[str, Any]]]: - """Collect the same backend text and tool events without blocking the event loop.""" + """Wait asynchronously for a backend reply and its completed tool calls.""" deadline = time.monotonic() + timeout text_parts: dict[tuple[str, int], str] = {} calls: dict[str, Mapping[str, Any]] = {} diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic.py index 590a313c1b90..bccc87c9260a 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic.py @@ -6,28 +6,20 @@ """ DESCRIPTION: - This sample demonstrates the voice-agent management lifecycle using the - synchronous AIProjectClient: creating a voice agent, retrieving it, - listing the voice agents in the project, and deleting it. - - Voice agents are exposed through `project_client.agents` with - `kind="voice"`, the same surface used for prompt, workflow, hosted, and - external agents. + Create a voice agent, read its settings, add a version, and list voice + agents in the project. Delete the versions created by this sample. + Voice agents use project_client.agents with kind="voice". USAGE: python sample_voice_agent_basic.py - Before running the sample: - pip install "azure-ai-projects>=2.7.0" python-dotenv - Set these environment variables with your own values: - 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the Overview - page of your Microsoft Foundry portal. - 2) FOUNDRY_VOICE_AGENT_MODEL - Optional. The managed (service-hosted) realtime model - identifier. Defaults to "gpt-realtime". - 3) FOUNDRY_VOICE_AGENT_NAME - Optional. The name of the voice agent. If not - set, defaults to "MyVoiceAgent". + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. + 2) FOUNDRY_VOICE_AGENT_MODEL (optional) - Managed Realtime model; + defaults to "gpt-realtime". + 3) FOUNDRY_VOICE_AGENT_NAME (optional) - Agent name; defaults to "MyVoiceAgent". """ import os diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic_async.py index ef63f207e1c0..daff1a0c9690 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_basic_async.py @@ -6,24 +6,19 @@ """ DESCRIPTION: - This sample demonstrates the voice-agent management lifecycle using the - asynchronous AIProjectClient: creating a voice agent, retrieving it, - listing the voice agents in the project, and deleting it. + Create a voice agent, read its settings, and list voice agents using the + async client. Delete the agent version created by this sample. USAGE: python sample_voice_agent_basic_async.py - Before running the sample: - pip install "azure-ai-projects>=2.7.0" aiohttp python-dotenv - Set these environment variables with your own values: - 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the Overview - page of your Microsoft Foundry portal. - 2) FOUNDRY_VOICE_AGENT_MODEL - Optional. The managed (service-hosted) realtime model - identifier. Defaults to "gpt-realtime". - 3) FOUNDRY_VOICE_AGENT_NAME - Optional. The name of the voice agent. If not - set, defaults to "MyVoiceAgentAsync". + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. + 2) FOUNDRY_VOICE_AGENT_MODEL (optional) - Managed Realtime model; + defaults to "gpt-realtime". + 3) FOUNDRY_VOICE_AGENT_NAME (optional) - Agent name; defaults to "MyVoiceAgentAsync". """ import asyncio diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_generate.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_generate.py index 7b42328d5490..f34c39b7f1db 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_generate.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_generate.py @@ -6,22 +6,19 @@ """ DESCRIPTION: - This sample demonstrates guided authoring: generating and creating a voice - agent through `create_from_prompt` with `kind="voice"`. - The service creates a voice agent with a service-selected starter definition, - which is fully editable afterward through the standard create_version/update flow. + Describe the voice agent you want and create it with create_from_prompt. + The service chooses initial settings. You can later customize them by + creating another agent version. USAGE: python sample_voice_agent_generate.py - Before running the sample: - pip install "azure-ai-projects>=2.7.0" python-dotenv - Set these environment variables with your own values: - 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. - 2) FOUNDRY_VOICE_AGENT_NAME - Optional. The name of the voice agent. If not - set, defaults to "MyGeneratedVoiceAgent". + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. + 2) FOUNDRY_VOICE_AGENT_NAME (optional) - Agent name; + defaults to "MyGeneratedVoiceAgent". """ import os @@ -35,14 +32,7 @@ def _safe_print(text: str) -> None: - """Print text that may contain characters the current console can't display. - - The instructions below are model-generated and can contain characters (curly - quotes, em-dashes, etc.) outside some legacy, non-Unicode console encodings - (for example when stdout is piped/redirected on Windows). Rather than crashing - with UnicodeEncodeError, fall back to replacing just the unsupported characters; - a real interactive UTF-8 console prints unaffected. - """ + """Print model text, replacing characters the console cannot display.""" try: print(text) except UnicodeEncodeError: diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py index 04a98837c1d8..8993f37817db 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_audio_conversation_async.py @@ -5,58 +5,42 @@ """ DESCRIPTION: - Talk to a preview GPT Live voice agent using a microphone, or stream a real - WAV recording. Both modes use PCM16 little-endian, mono, - 24 kHz audio and send/receive concurrently. - Async is used for full-duplex streaming and nonblocking audio devices, - rather than duplicating a synchronous sample. - - Microphone mode uses optional PyAudio capture and speaker callbacks, - following the Realtime audio sample's nonblocking device pattern. Speak - normally and hear the reply through your speakers; use headphones to avoid - feedback. Capture backpressure drops a frame instead of queuing unlimited - sends, with the dropped-frame count reported at shutdown. Playback buffering - is bounded and device/send errors are raised. Press Ctrl-C to stop capture, - close the Live session and delete only the sample's agent version. Live owns - turn handling; this sample does not send Realtime response.cancel commands. - - WAV mode needs spoken audio in uncompressed PCM16, mono, 24 kHz format. - Only raw frames are sent, without the WAV header, in paced 50 ms chunks - followed by one second of silence. No audio-device dependency is needed. - - Aggregate audio byte counts and any emitted input/output transcripts are - displayed. Live has no speech-finished event: after WAV sending finishes, this - bounded mode stops after three seconds without non-silent reply - audio or nonblank output transcript updates. This is an inactivity heuristic; - a long speech pause can still split a reply. Low-level PCM with peak - amplitude at or below 256 does not restart the timer or count as a reply. - Captured PCM, including silence, is still counted and optionally saved. - Client-owned delegations are answered with commentary tied to their ID. + Send a WAV recording to GPT Live, or talk through a microphone and hear + replies through your speakers. Async lets audio send and receive together. + Both modes use PCM16 little-endian, mono, 24-kHz audio. + + The sample prints transcripts and audio byte counts, then deletes its + temporary agent version. Microphone mode also reports dropped capture + frames if sending cannot keep up. Device and playback errors are raised. USAGE: python sample_voice_agent_live_audio_conversation_async.py - Set USE_MICROPHONE = True in this file for microphone capture and speaker - playback. Leave it False to stream a WAV file without audio devices. - Set SAVE_AUDIO = True to save returned audio in a unique user-temporary - folder. Its location is printed and the files remain after exit; remove - the folder when it is no longer needed. + Settings in this file: + - USE_MICROPHONE = False (default): send a WAV file; no audio devices needed. + - USE_MICROPHONE = True: use a microphone and speakers. Use headphones + to avoid feedback; press Ctrl-C to stop. + - SAVE_AUDIO = True: save reply audio in a unique user-temporary folder. + The path is printed. Remove the folder when finished; it contains + conversation content, not telemetry. Saving is off by default. pip install "azure-ai-projects[voice]" azure-identity python-dotenv pip install pyaudio # Only required for microphone mode. Environment variables: 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. - 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live DEPLOYMENT name. + 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live deployment name. 3) FOUNDRY_VOICE_AUDIO_INPUT_PATH (required for WAV mode only) - Path to an actual spoken-audio WAV file: uncompressed PCM16, mono, 24000 Hz. - 4) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for a uniquely named sample - agent. Only the version created by this run is deleted. - - Authenticates with DefaultAzureCredential. - Choose a short recording; Ctrl-C cancels local tasks and closes the session. - Conversation persistence is disabled; explicitly saving audio still writes - conversation content locally. SAVE_AUDIO is False by default. + 4) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for the temporary agent name. + + Uses DefaultAzureCredential; conversation persistence is disabled. + Choose a short spoken recording. WAV mode sends raw audio in 50-ms chunks + and adds one second of silence. It finishes after three seconds without + active reply audio or transcript updates because Live has no speech-finished + event. Long pauses can cut a reply short; silence alone is not a reply. + Low-level PCM (peak <= 256) does not extend the wait, but is still counted + and saved when requested. Live handles turn-taking without response.cancel. """ from __future__ import annotations diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_connection.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_connection.py index 43dda481541d..049783db71f6 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_connection.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_connection.py @@ -5,10 +5,9 @@ """ DESCRIPTION: - Open a preview GPT Live voice-agent session, update its client delegation, - append instructions, and confirm graceful session shutdown. Foundry starts - the session from the persisted agent definition; do not send session.start. - This is a protocol/lifecycle example, not a text or microphone conversation. + Create a temporary GPT Live agent, open a session, update its settings, + add instructions, and close it. The sample deletes the agent version it + creates. No user input or audio devices are needed. USAGE: python sample_voice_agent_live_connection.py @@ -17,12 +16,11 @@ Environment variables: 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. - 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live DEPLOYMENT name, - not a model catalog identifier. - 3) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for a uniquely named sample - agent. Only the version created by this run is deleted. + 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live deployment name. + 3) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for the temporary agent name. - Authenticates with DefaultAzureCredential. No audio devices are needed. + Uses DefaultAzureCredential. Foundry starts the session automatically; + wait for session.started rather than sending session.start. """ import os diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py index 6be341df6acb..cfcddad684f2 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_function_tool.py @@ -5,12 +5,10 @@ """ DESCRIPTION: - Resolve a client-executed Responses function tool in a preview GPT Live - session. The deterministic get_weather tool returns sample data for Seattle - (72 F), not a real weather observation. After the tool response completes, - submit function_call_output and request the next backend response. - Read call_id from the complete response.output_item.done function_call item, - not from response.function_call_arguments.done (which identifies item_id). + Let a GPT Live session call a local get_weather function through a Responses + backend model. Return the function result and print the model's final reply. + The weather is fixed sample data for Seattle (72 F), not a live forecast. + The sample deletes its temporary agent version when finished. USAGE: python sample_voice_agent_live_function_tool.py @@ -19,13 +17,14 @@ Environment variables: 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. - 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live DEPLOYMENT name. - 3) FOUNDRY_MODEL_NAME (required) - A tool-capable Responses backend - DEPLOYMENT name available to the Live resource. - 4) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for a uniquely named sample - agent. Only the version created by this run is deleted. - - Authenticates with DefaultAzureCredential. No audio devices are needed. + 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live deployment name. + 3) FOUNDRY_MODEL_NAME (required) - A tool-capable Responses model deployment + available to your Live resource. + 4) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for the temporary agent name. + + Uses DefaultAzureCredential. No audio devices are needed. + Tool results use call_id from response.output_item.done, not the item_id + in response.function_call_arguments.done. """ from __future__ import annotations diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py index 2fbcb8760c1a..115dbd256693 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_read_conversation.py @@ -5,28 +5,18 @@ """ DESCRIPTION: - Read a stored GPT Live conversation using the preview - project_client.beta.voice_agents.live_conversations REST APIs. Retrieve the - conversation, recent conversations, ordered items, individual items, and - merged/per-item audio metadata. - - If FOUNDRY_VOICE_CONVERSATION_ID is omitted or blank, create a temporary - GPT Live agent and a stored greeting session, close it, wait for persistence, - read its recordings, then delete the conversation and agent version. - No microphone, input recording, or Responses backend is needed. - - If an existing conversation ID is supplied, read it without creating an - agent or opening a WebSocket and never delete its resources. It must come - from session.started.conversation_id, not session.id, after a successful - speech session with effective store=True. Dedicated Live REST routes must - be available; no fallback to older Realtime conversation routes is used. - - Download Foundry-managed WAV recordings into a unique folder in your user - temporary directory and print its location. The files remain available - after the sample exits and are not overwritten. Downloads contain stored - conversation content; remove the local folder when it is no longer needed. - BYOS recordings require your own storage credentials and are not downloaded. - Only the agent version and conversation created by this run are deleted. + Read a GPT Live conversation, list its items, and download its audio using + project_client.beta.voice_agents.live_conversations. + + With no conversation ID, the sample creates a short greeting session and + deletes its conversation and temporary agent version afterward. No microphone, + input WAV, or Responses model is needed. Supplying an existing ID skips + creation and leaves the existing agent and conversation untouched. + + WAV downloads remain in a unique user-temporary folder. The sample prints + its path; remove the folder when finished. Files contain conversation + content, not telemetry. BYOS audio requires your own storage credentials + and is not downloaded. USAGE: python sample_voice_agent_live_read_conversation.py @@ -35,16 +25,17 @@ Environment variables: 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. - 2) FOUNDRY_VOICE_CONVERSATION_ID (optional) - An existing stored conversation. - When unset or blank, this sample creates and deletes its own conversation. - 3) FOUNDRY_VOICE_AGENT_NAME (required with an existing conversation ID) - - The existing GPT Live agent name. Otherwise, an optional name prefix - for the uniquely named temporary agent; an existing agent is not changed. - 4) FOUNDRY_VOICE_AGENT_MODEL (required without a conversation ID) - - Your GPT Live deployment name, used to create the temporary agent. - - Authenticates with DefaultAzureCredential. Existing-conversation mode - requires no model deployment name or WebSocket dependency. + 2) FOUNDRY_VOICE_AGENT_MODEL (required when creating a conversation) - + Your GPT Live deployment name. + 3) FOUNDRY_VOICE_CONVERSATION_ID (optional) - An existing conversation ID. + Unset or blank means create a conversation for this run. + 4) FOUNDRY_VOICE_AGENT_NAME - Required with an existing ID; otherwise an + optional prefix for the temporary agent name. + + Uses DefaultAzureCredential. An existing conversation needs no model + deployment or WebSocket. Use session.started.conversation_id, not session.id, + from a completed speech session with store=True. The service must support + Live REST routes; HTTP errors are raised without a Realtime fallback. """ from __future__ import annotations diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py index 09c6736661fc..e332ec25db34 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_live_text_conversation.py @@ -5,30 +5,28 @@ """ DESCRIPTION: - Send one or more text messages through one preview GPT Live session configured - with a Responses delegation. Each turn waits for nested response.completed - before sending the next prompt, retaining the conversation in that session. - This is not the Realtime protocol's response.done. + Send text messages through GPT Live and print the replies from a Responses + backend model. Multiple messages share one session. The sample waits for + response.completed before sending the next message and deletes its temporary + agent version when finished. USAGE: python sample_voice_agent_live_text_conversation.py - Leave TEXT_PROMPTS as None for the single-turn example. To demonstrate - a headless multi-turn conversation, set TEXT_PROMPTS in this file to - ["My name is Ada.", "What is my name?"] before running it. + Leave TEXT_PROMPTS as None for one message, or set it to a list such as + ["My name is Ada.", "What is my name?"] for a multi-turn conversation. pip install "azure-ai-projects[voice]" azure-identity python-dotenv Environment variables: 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. - 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live DEPLOYMENT name. - 3) FOUNDRY_MODEL_NAME (required) - A Responses backend DEPLOYMENT name - available to the Live resource, not a model catalog identifier. - 4) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for a uniquely named sample - agent. Only the version created by this run is deleted. - - Authenticates with DefaultAzureCredential. No microphone or speaker needed. - This sample prints the backend text; use the audio sample for Live speech. + 2) FOUNDRY_VOICE_AGENT_MODEL (required) - Your GPT Live deployment name. + 3) FOUNDRY_MODEL_NAME (required) - A Responses model deployment available + to your Live resource. It handles the delegated text requests. + 4) FOUNDRY_VOICE_AGENT_NAME (optional) - Prefix for the temporary agent name. + + Uses DefaultAzureCredential. No microphone or speaker is needed. + Printed text does not signal that voice playback has finished. """ from __future__ import annotations diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation.py index acbd0aca3c96..478c72963766 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation.py @@ -6,44 +6,27 @@ """ DESCRIPTION: - This sample demonstrates reading a persisted voice conversation back over - the read-only conversation API exposed by `project_client.beta.voice_agents.conversations`: - the conversation envelope, its responses (model inference turns), and its - ordered items (the transcript). Conversations are created and written by - the voice orchestrator during a live session; this client can only read - them, and only when the agent was configured with `store=True` (see - sample_voice_agent_basic.py). - - Runs with no setup beyond the endpoint: if FOUNDRY_VOICE_CONVERSATION_ID is - not set, this sample creates a temporary voice agent, holds one short - realtime text turn to produce a real conversation, reads it back, then - deletes the agent version it created. Set FOUNDRY_VOICE_AGENT_NAME to hold - that conversation against your own existing agent instead -- it is never - created, modified, or deleted by this sample. Additionally set - FOUNDRY_VOICE_CONVERSATION_ID to skip holding a new conversation entirely - and just read back one your agent already produced. + Read a stored Realtime conversation, its model replies, and its transcript + through project_client.beta.voice_agents.conversations. + + With no conversation ID, send one text message to create a conversation. + With no agent name, also create a temporary agent and delete its version + afterward. An existing agent is not changed or deleted. Conversations + created on an existing agent remain stored. USAGE: python sample_voice_agent_read_conversation.py - Before running the sample: - pip install "azure-ai-projects[voice]>=2.7.0" python-dotenv - Set these environment variables with your own values: - 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. - 2) FOUNDRY_VOICE_AGENT_NAME - Optional. The name of an existing voice agent - (configured with `store=True`) to hold or read a conversation on. - Defaults to a temporary agent, created and deleted by this sample, when - unset. - 3) FOUNDRY_VOICE_CONVERSATION_ID - Optional. The id of a persisted - conversation owned by FOUNDRY_VOICE_AGENT_NAME. If unset, this sample - holds one short realtime text turn to produce one; see - voice_sample_util.py in this folder and - sample_voice_agent_realtime_text_conversation.py for a full interactive - version. - 4) FOUNDRY_VOICE_AGENT_MODEL - Optional. The managed (service-hosted) realtime model - identifier, used only when creating the temporary agent. Defaults to "gpt-realtime". + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. + 2) FOUNDRY_VOICE_AGENT_NAME (optional) - Existing agent with store=True; + omit to create a temporary agent. + 3) FOUNDRY_VOICE_CONVERSATION_ID (optional) - Completed conversation from + that agent. Omit to create a conversation with a text message. + 4) FOUNDRY_VOICE_AGENT_MODEL (optional) - Model for a temporary agent; + defaults to "gpt-realtime". """ import os diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation_audio.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation_audio.py index 4cdbf5bcd897..40171cba2fec 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation_audio.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_read_conversation_audio.py @@ -6,50 +6,32 @@ """ DESCRIPTION: - This sample demonstrates reading the persisted audio of a voice - conversation via `project_client.beta.voice_agents.conversations`, both the - merged whole-call recording and a single turn's audio segment. For each it - reads the metadata first, then streams the WAV bytes to a local file. The - merged recording is stereo: the caller on the left channel and the agent - on the right. - - Audio is available only after the session has ended and only when the - agent was configured with `store=True`. For bring-your-own-storage (BYOS) - accounts the metadata carries a `blob_uri` instead, and the bytes are read - from your own storage rather than streamed here. - - Runs with no setup beyond the endpoint: if FOUNDRY_VOICE_CONVERSATION_ID is - not set, this sample creates a temporary voice agent, holds one short - realtime text turn to produce a real conversation, then reads its audio - back -- the agent's reply is real synthesized speech either way, so both - the merged recording and the reply's own audio segment are available even - though the turn itself was typed. Set FOUNDRY_VOICE_AGENT_NAME to hold that - conversation against your own existing agent instead -- it is never - created, modified, or deleted by this sample. Additionally set - FOUNDRY_VOICE_CONVERSATION_ID to skip holding a new conversation entirely - and just read back the audio of one your agent already produced. + Download a stored Realtime conversation's WAV recording and one reply's + audio. The whole-call recording has caller audio on the left channel and + agent audio on the right. The session must have ended with store=True. + + With no conversation ID, send a text message to produce a spoken reply. + With no agent name, also create a temporary agent and delete its version + afterward. An existing agent is not changed or deleted. Conversations + created on an existing agent remain stored. + + Files are saved in the current directory and contain conversation content. + For bring-your-own-storage (BYOS) recordings, the sample prints the blob + location; use your own storage credentials to download the audio. USAGE: python sample_voice_agent_read_conversation_audio.py - Before running the sample: - pip install "azure-ai-projects[voice]>=2.7.0" python-dotenv - Set these environment variables with your own values: - 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. - 2) FOUNDRY_VOICE_AGENT_NAME - Optional. The name of an existing voice - agent (configured with `store=True`) to hold or read a conversation on. - Defaults to a temporary agent, created and deleted by this sample, when - unset. - 3) FOUNDRY_VOICE_CONVERSATION_ID - Optional. The id of a persisted - conversation owned by FOUNDRY_VOICE_AGENT_NAME. If unset, this sample - holds one short realtime text turn to produce one; see - voice_sample_util.py in this folder and - sample_voice_agent_realtime_text_conversation.py for a full interactive - version. - 4) FOUNDRY_VOICE_AGENT_MODEL - Optional. The managed (service-hosted) realtime model - identifier, used only when creating the temporary agent. Defaults to "gpt-realtime". + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. + 2) FOUNDRY_VOICE_AGENT_NAME (optional) - Existing agent with store=True; + omit to create a temporary agent. + 3) FOUNDRY_VOICE_CONVERSATION_ID (optional) - Completed conversation from + that agent. Omit to create a conversation with a text message. + 4) FOUNDRY_VOICE_AGENT_MODEL (optional) - Model for a temporary agent; + defaults to "gpt-realtime". """ import os diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_audio_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_audio_conversation_async.py index e974e8dbb70f..61ffb9165c61 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_audio_conversation_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_audio_conversation_async.py @@ -6,38 +6,24 @@ """ DESCRIPTION: - End-to-end hands-free, bidirectional voice conversation using the - ``client.beta.voice_agents.realtime`` namespace added on top of the generated - azure-ai-projects client (see ``azure.ai.projects.aio.operations.AsyncBetaRealtime``). - This mirrors the ergonomics of the OpenAI Python realtime client. - - 1. Create a voice agent with conversation persistence enabled - (`store=True`) so the conversation can be read back afterward. - 2. Stream live mic audio and let the agent's server-side VAD detect your - turns: your speech is transcribed, the agent replies through the - speakers, and talking over it barges in. - 3. Fetch the persisted conversation back by id. - 4. Delete the agent created for this sample. - - Capture and playback use non-blocking pyaudio callbacks; reply audio is - sequence-numbered so a barge-in can skip whatever is still queued. The - agent owns turn detection and noise suppression server-side. Use a headset - to avoid echo. - - Mic audio is sent as base64 PCM16; the reply arrives as typed - ``response.output_audio.*`` events, decoded to PCM16, mono, 24 kHz. - Requires ``aiohttp`` and ``pyaudio``. - - pip install "azure-ai-projects[voice]>=2.7.0" azure-identity pyaudio + Talk to a Realtime voice agent through your microphone and speakers. + The service detects speech turns and lets you interrupt a reply by speaking. + The sample prints transcripts. Use headphones to avoid echo. + + The sample sends and receives PCM16, mono, 24-kHz audio at the same time. + It creates an agent with store=True, reads back the conversation when you + stop, and deletes its agent version. USAGE: python sample_voice_agent_realtime_audio_conversation_async.py + pip install "azure-ai-projects[voice]>=2.7.0" azure-identity python-dotenv pyaudio + Environment variables: 1) FOUNDRY_PROJECT_ENDPOINT (required) - Foundry project endpoint: https://.services.ai.azure.com/api/projects/ - 2) FOUNDRY_VOICE_AGENT_MODEL - Optional. The managed (service-hosted) realtime model - identifier. Defaults to "gpt-realtime". + 2) FOUNDRY_VOICE_AGENT_MODEL (optional) - Managed Realtime model; + defaults to "gpt-realtime". 3) FOUNDRY_VOICE_AGENT_NAME - Optional. Name for the agent created by this sample. Defaults to "sample-realtime-audio-conversation-agent-async". @@ -56,9 +42,7 @@ from azure.core.exceptions import HttpResponseError from azure.identity.aio import DefaultAzureCredential -# AsyncBetaRealtimeConnection is re-exported dynamically via aio/operations/_patch.py's `__all__`; -# pylint's static import resolution cannot trace that, but the symbol is valid (verified by -# Pyright/mypy). +# This public re-export is not recognized by Pylint's import resolver. from azure.ai.projects.aio.operations import AsyncBetaRealtimeConnection # pylint: disable=no-name-in-module from azure.ai.projects.aio import AIProjectClient from azure.ai.projects.models import ( diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_function_tool.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_function_tool.py index ec3f85c02435..051d3ace3137 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_function_tool.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_function_tool.py @@ -6,26 +6,20 @@ """ DESCRIPTION: - This sample demonstrates handling a client-executed `function` tool during - a live voice-agent session: - - 1) Create a voice agent configured with a `get_weather` function tool. - 2) Open a realtime session and send a text turn that should trigger the tool. - 3) Listen for `response.function_call_arguments.done`, execute the function - locally, and send the result back with `conversation.item.create` + - `response.create` so the agent can finish its reply using the tool output. + Create a Realtime voice agent with a local get_weather tool. Send a text + request, run the requested function, return its result, and print the final + reply. Weather values are sample data, not a live forecast. + The sample deletes the agent version it creates. USAGE: python sample_voice_agent_realtime_function_tool.py - Before running the sample: - pip install "azure-ai-projects[voice]>=2.7.0" azure-identity python-dotenv - Set these environment variables with your own values: - 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. - 2) FOUNDRY_VOICE_AGENT_MODEL - Optional. The managed (service-hosted) realtime model - identifier. Defaults to "gpt-realtime". + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. + 2) FOUNDRY_VOICE_AGENT_MODEL (optional) - Managed Realtime model; + defaults to "gpt-realtime". 3) FOUNDRY_VOICE_AGENT_NAME - Optional. Name for the sample voice agent created and deleted by this script. Defaults to "sample-voice-agent-function-tool". @@ -61,7 +55,7 @@ def get_weather(city: str) -> str: - """A trivial local "tool" implementation the agent can call. + """Return sample weather without calling a weather service. :param city: The city to look up. :type city: str @@ -72,14 +66,7 @@ def get_weather(city: str) -> str: def _safe_print(text: str) -> None: - """Print text that may contain characters the current console can't display. - - The agent's reply below is model-generated and can contain characters (curly - quotes, em-dashes, etc.) outside some legacy, non-Unicode console encodings - (for example when stdout is piped/redirected on Windows). Rather than crashing - with UnicodeEncodeError, fall back to replacing just the unsupported characters; - a real interactive UTF-8 console prints unaffected. - """ + """Print model text, replacing characters the console cannot display.""" try: print(text) except UnicodeEncodeError: diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_text_conversation.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_text_conversation.py index 409c530cb79a..a7210b882281 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_text_conversation.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_text_conversation.py @@ -6,36 +6,26 @@ """ DESCRIPTION: - End-to-end typed conversation using the ``client.beta.voice_agents.realtime`` namespace added - on top of the generated azure-ai-projects client (see - ``azure.ai.projects.operations.BetaRealtime``). - - 1. Create a voice agent with conversation persistence enabled - (`store=True`) so the conversation can be read back afterward. - 2. Hold a typed, multi-turn conversation: each prompt is sent as a - ``RealtimeConversationItemMessageUser`` and the reply streams back as - typed audio and transcript events. Blank line (or ``exit`` / ``quit``) - ends it. - 3. Fetch the persisted conversation back by id. - 4. Delete the agent created for this sample. - - Reply audio is PCM16, mono, 24 kHz and plays through the speakers when - ``pyaudio`` is installed; runs headless otherwise. For a hands-free mic - conversation with barge-in, see sample_voice_agent_realtime_audio_conversation_async.py - (that sample needs concurrent send/receive so it stays async-only; see - sample_voice_agent_realtime_text_conversation_async.py for the async version of - this one). - - pip install "azure-ai-projects[voice]>=2.7.0" azure-identity pyaudio + Type messages to a Realtime voice agent and print its replies. If PyAudio + is installed, also play reply audio through your speakers. Enter a blank + line, "exit", or "quit" to finish. + + The sample creates an agent with store=True, reads back the conversation, + and deletes its agent version. Audio is PCM16, mono, 24 kHz. + For microphone input, use sample_voice_agent_realtime_audio_conversation_async.py. + For the async text version, use sample_voice_agent_realtime_text_conversation_async.py. USAGE: python sample_voice_agent_realtime_text_conversation.py + pip install "azure-ai-projects[voice]>=2.7.0" azure-identity python-dotenv + pip install pyaudio # Optional speaker playback. + Environment variables: 1) FOUNDRY_PROJECT_ENDPOINT (required) - Foundry project endpoint: https://.services.ai.azure.com/api/projects/ - 2) FOUNDRY_VOICE_AGENT_MODEL - Optional. The managed (service-hosted) realtime model - identifier. Defaults to "gpt-realtime". + 2) FOUNDRY_VOICE_AGENT_MODEL (optional) - Managed Realtime model; + defaults to "gpt-realtime". 3) FOUNDRY_VOICE_AGENT_NAME - Optional. Name for the agent created by this sample. Defaults to "sample-realtime-text-conversation-agent". diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_text_conversation_async.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_text_conversation_async.py index c638124194c0..e860753a9303 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_text_conversation_async.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_realtime_text_conversation_async.py @@ -6,33 +6,25 @@ """ DESCRIPTION: - End-to-end typed conversation using the ``client.beta.voice_agents.realtime`` namespace added - on top of the generated azure-ai-projects client (see - ``azure.ai.projects.aio.operations.AsyncBetaRealtime``). + Type messages to a Realtime voice agent using the async client and print + its replies. If PyAudio is installed, also play reply audio through your + speakers. Enter a blank line, "exit", or "quit" to finish. - 1. Create a voice agent with conversation persistence enabled - (`store=True`) so the conversation can be read back afterward. - 2. Hold a typed, multi-turn conversation: each prompt is sent as a - ``RealtimeConversationItemMessageUser`` and the reply streams back as - typed audio and transcript events. Blank line (or ``exit`` / ``quit``) - ends it. - 3. Fetch the persisted conversation back by id. - 4. Delete the agent created for this sample. - - Reply audio is PCM16, mono, 24 kHz and plays through the speakers when - ``pyaudio`` is installed; runs headless otherwise. For a hands-free mic - conversation with barge-in, see sample_voice_agent_realtime_audio_conversation_async.py. - - pip install "azure-ai-projects[voice]>=2.7.0" azure-identity pyaudio + The sample creates an agent with store=True, reads back the conversation, + and deletes its agent version. Audio is PCM16, mono, 24 kHz. + For microphone input, use sample_voice_agent_realtime_audio_conversation_async.py. USAGE: python sample_voice_agent_realtime_text_conversation_async.py + pip install "azure-ai-projects[voice]>=2.7.0" azure-identity python-dotenv + pip install pyaudio # Optional speaker playback. + Environment variables: 1) FOUNDRY_PROJECT_ENDPOINT (required) - Foundry project endpoint: https://.services.ai.azure.com/api/projects/ - 2) FOUNDRY_VOICE_AGENT_MODEL - Optional. The managed (service-hosted) realtime model - identifier. Defaults to "gpt-realtime". + 2) FOUNDRY_VOICE_AGENT_MODEL (optional) - Managed Realtime model; + defaults to "gpt-realtime". 3) FOUNDRY_VOICE_AGENT_NAME - Optional. Name for the agent created by this sample. Defaults to "sample-realtime-text-conversation-agent-async". @@ -48,9 +40,7 @@ from azure.core.exceptions import HttpResponseError from azure.identity.aio import DefaultAzureCredential -# AsyncBetaRealtimeConnection is re-exported dynamically via aio/operations/_patch.py's `__all__`; -# pylint's static import resolution cannot trace that, but the symbol is valid (verified by -# Pyright/mypy). +# This public re-export is not recognized by Pylint's import resolver. from azure.ai.projects.aio.operations import AsyncBetaRealtimeConnection # pylint: disable=no-name-in-module from azure.ai.projects.aio import AIProjectClient from azure.ai.projects.models import ( diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_versions.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_versions.py index ebbff2f47dbc..0e0a83bccfa4 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_versions.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_versions.py @@ -6,24 +6,21 @@ """ DESCRIPTION: - This sample demonstrates working with voice-agent versions. Agents are - immutable: every `create_version` call produces a new version. This sample - creates an agent, adds a new version to it, adds a draft version, lists the - versions, and reads a single version back. + Create published and draft versions of a voice agent, list them, and read + their settings. Each create_version call makes a new version rather than + changing an existing one. USAGE: python sample_voice_agent_versions.py - Before running the sample: - pip install "azure-ai-projects>=2.7.0" python-dotenv - Set these environment variables with your own values: - 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. - 2) FOUNDRY_VOICE_AGENT_MODEL - Optional. The managed (service-hosted) realtime model - identifier. Defaults to "gpt-realtime". - 3) FOUNDRY_VOICE_AGENT_NAME - Optional. The name of the voice agent. If not - set, defaults to "sample-versioned-voice-agent". + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. + 2) FOUNDRY_VOICE_AGENT_MODEL (optional) - Managed Realtime model; + defaults to "gpt-realtime". + 3) FOUNDRY_VOICE_AGENT_NAME (optional) - Agent name; + defaults to "sample-versioned-voice-agent". """ import os diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_with_tools.py b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_with_tools.py index 8b12a235abbb..779605fa8fe0 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_with_tools.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/sample_voice_agent_with_tools.py @@ -6,34 +6,26 @@ """ DESCRIPTION: - This sample demonstrates the richer parts of a voice agent definition: + Configure a voice agent with audio settings, speech detection, transcription, + a local function tool, and a service-managed end-conversation tool. + MCP and toolbox settings are shown as examples, not executed. - * Input (microphone) audio configuration: audio format, server-side turn - detection (VAD), input-audio transcription. - * Tools the agent may use during a live session: a client-executed - `function` tool and a service-managed `system` control tool (`mcp` and - `toolbox` tools are shown as constructed objects for illustration). - * Bring-your-own-model (BYOM): set `model_type="self_deployed"` to point - the agent at your own Foundry model deployment instead of a - service-managed model. + Use a managed model, or select "self_deployed" to use your own Foundry + model deployment. USAGE: python sample_voice_agent_with_tools.py - Before running the sample: - pip install "azure-ai-projects>=2.7.0" python-dotenv - Set these environment variables with your own values: - 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. - 2) FOUNDRY_VOICE_AGENT_MODEL - Optional. The realtime model (managed) or the - Foundry deployment name (BYOM). Defaults to "gpt-realtime". (FOUNDRY_VOICE_MODEL - is a deprecated alias for this variable, still read as a fallback for compatibility - with earlier samples.) - 3) FOUNDRY_VOICE_MODEL_TYPE - Optional. "managed" (default) for a - service-hosted model, or "self_deployed" to bring your own deployment. - 4) FOUNDRY_VOICE_AGENT_NAME - Optional. The name of the voice agent. If not - set, defaults to "sample-voice-agent-with-tools". + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. + 2) FOUNDRY_VOICE_AGENT_MODEL (optional) - Managed model or your deployment + name; defaults to "gpt-realtime". The deprecated FOUNDRY_VOICE_MODEL + variable is still accepted as a fallback. + 3) FOUNDRY_VOICE_MODEL_TYPE (optional) - "managed" (default) or "self_deployed". + 4) FOUNDRY_VOICE_AGENT_NAME (optional) - Agent name; + defaults to "sample-voice-agent-with-tools". """ import os @@ -65,9 +57,7 @@ endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] # Keep existing BYOM configurations targeting the caller's deployment. model = os.environ.get("FOUNDRY_VOICE_AGENT_MODEL") or os.environ.get("FOUNDRY_VOICE_MODEL") or "gpt-realtime" -# "managed" runs a service-hosted model; "self_deployed" (BYOM) uses your own -# Foundry deployment named by `model`. The service derives whether the model is -# realtime or cascaded; you don't set that here. +# Choose a managed model or your own deployment; the service selects its protocol. model_type = os.environ.get("FOUNDRY_VOICE_MODEL_TYPE") or VoiceModelType.MANAGED agent_name = os.environ.get("FOUNDRY_VOICE_AGENT_NAME") or "sample-voice-agent-with-tools" diff --git a/sdk/ai/azure-ai-projects/samples/agents/voice/voice_sample_util.py b/sdk/ai/azure-ai-projects/samples/agents/voice/voice_sample_util.py index a4855a8b6f8a..f3a28cb71b12 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/voice/voice_sample_util.py +++ b/sdk/ai/azure-ai-projects/samples/agents/voice/voice_sample_util.py @@ -30,15 +30,10 @@ def hold_sample_conversation( project_client: AIProjectClient, agent_name: str, prompt: str = "Say a short, friendly hello." ) -> str: - """Send one short realtime text turn to an existing voice agent and return the resulting - persisted conversation id, once it has finished finalizing. + """Send one text message and return its stored Realtime conversation ID. - Samples that read a conversation back (transcript, responses, audio) do not always have a - live call handy to read. This helper produces a minimal real conversation on demand so those - samples can run without requiring a conversation id up front. The agent named ``agent_name`` - must already exist (see sample_voice_agent_basic.py) and be configured with ``store=True`` so - its conversations are persisted. For a full interactive conversation and a fuller explanation - of the realtime event flow used here, see sample_voice_agent_realtime_text_conversation.py. + The agent must already exist with ``store=True``. Wait for storage to finish + before returning so callers can read the replies and transcript. :param project_client: The Foundry project client. :param agent_name: The name of an existing voice agent, configured with ``store=True``. @@ -63,8 +58,7 @@ def hold_sample_conversation( while True: event = conn.recv(timeout=_RESPONSE_TIMEOUT) if isinstance(event, RealtimeServerEventSessionCreated): - # The persisted conversation id (only present when conversation persistence is - # enabled) is set here, not on response.done. + # The stored conversation ID arrives at session creation, not response.done. conversation_id = event.conversation_id or conversation_id if isinstance(event, RealtimeServerEventResponseDone): break @@ -77,10 +71,7 @@ def hold_sample_conversation( "was configured with `store=True`." ) - # `response.done` only means the model finished replying, not that the service has finished - # persisting the conversation -- immediately after the session closes, the conversation can - # still briefly report `in_progress` while that finalization completes in the background. - # Poll until it settles so callers can read a complete transcript right away. + # A reply can finish before its transcript is stored; wait for storage to finish too. conversations = project_client.beta.voice_agents.conversations for _ in range(_FINALIZE_POLL_ATTEMPTS): conversation = conversations.get(agent_name, conversation_id) diff --git a/sdk/ai/azure-ai-projects/samples/logs/sample_log_from_voice_agent.py b/sdk/ai/azure-ai-projects/samples/logs/sample_log_from_voice_agent.py index 698d689f9574..e2b387a59c89 100644 --- a/sdk/ai/azure-ai-projects/samples/logs/sample_log_from_voice_agent.py +++ b/sdk/ai/azure-ai-projects/samples/logs/sample_log_from_voice_agent.py @@ -5,23 +5,21 @@ """ DESCRIPTION: - Capture Voice Agent WebSocket diagnostics from azure.ai.projects.realtime - into a timestamped temporary log file. The SDK logs connection lifecycle, - event types and serialized event sizes, not event payloads or credentials. - No OpenTelemetry configuration or GenAI tracing opt-in is required. + Save Realtime voice-agent diagnostics to a temporary log file using + azure.ai.projects.realtime. Logs contain connection events, event types, + and byte counts, not message content or credentials. OpenTelemetry is + not required. - Create a preview Voice Agent, send one text prompt, wait for its response, - and delete the created version. No microphone or speaker is required. + The sample sends one text prompt and deletes the agent version it creates. + No microphone or speaker is needed. USAGE: python samples/logs/sample_log_from_voice_agent.py - Before running the sample: - pip install "azure-ai-projects[voice]" azure-identity python-dotenv - Set these environment variables with your own values: - 1) FOUNDRY_PROJECT_ENDPOINT - The Azure AI Project endpoint. + Environment variables: + 1) FOUNDRY_PROJECT_ENDPOINT (required) - The Foundry project endpoint. 2) FOUNDRY_VOICE_AGENT_MODEL - Optional managed model; defaults to "gpt-realtime". 3) FOUNDRY_VOICE_AGENT_NAME - Optional agent name; defaults to a unique sample name.