From 72803daea53971b7e449c2ab5a80dff19cba8ff0 Mon Sep 17 00:00:00 2001 From: Julian Toledo Date: Fri, 23 Oct 2026 21:49:03 +0200 Subject: [PATCH 1/6] feat(agent): add Gemini Live (gemini-3.8-live) voice session support for AppFunctions --- agent/app/src/main/AndroidManifest.xml | 2 + .../agent/data/GeminiLiveAudioHelper.kt | 233 +++++++ .../agent/data/GeminiToolConverter.kt | 35 + .../appfunctions/agent/di/DataModule.kt | 2 + .../agent/domain/AgentOrchestrator.kt | 238 +++++++ .../agent/domain/GeminiLiveSessionManager.kt | 608 ++++++++++++++++++ .../agent/domain/LiveSessionState.kt | 27 + .../ui/screens/agentdemo/AgentDemoScreen.kt | 169 ++++- .../screens/agentdemo/AgentDemoViewModel.kt | 31 + .../ui/screens/agentdemo/AgentUiState.kt | 6 + agent/app/src/main/res/values/strings.xml | 4 + .../agent/data/GeminiToolConverterTest.kt | 64 ++ .../domain/GeminiLiveSessionManagerTest.kt | 106 +++ .../agentdemo/AgentDemoViewModelTest.kt | 49 +- 14 files changed, 1563 insertions(+), 11 deletions(-) create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelper.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManager.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveSessionState.kt create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManagerTest.kt diff --git a/agent/app/src/main/AndroidManifest.xml b/agent/app/src/main/AndroidManifest.xml index 7e327172..6d535d75 100644 --- a/agent/app/src/main/AndroidManifest.xml +++ b/agent/app/src/main/AndroidManifest.xml @@ -22,6 +22,8 @@ + + diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelper.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelper.kt new file mode 100644 index 00000000..1a125f0c --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelper.kt @@ -0,0 +1,233 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import android.annotation.SuppressLint +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioRecord +import android.media.AudioTrack +import android.media.MediaRecorder +import android.media.audiofx.AcousticEchoCanceler +import android.util.Log +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.atomic.AtomicBoolean + +/** + * Manages low-latency PCM audio recording (16 kHz mono 16-bit) and playback (24 kHz mono 16-bit) + * for a Gemini Live bidirectional audio session. + */ +class GeminiLiveAudioHelper { + private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + private val isReleased = AtomicBoolean(false) + private val isRecordingPaused = AtomicBoolean(false) + + private var audioRecord: AudioRecord? = null + private var echoCanceler: AcousticEchoCanceler? = null + private var audioTrack: AudioTrack? = null + private var playbackJob: Job? = null + + private val audioQueue = ConcurrentLinkedQueue() + + @SuppressLint("MissingPermission") + fun start() { + val minRecordBuf = + AudioRecord.getMinBufferSize( + INPUT_SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + ).coerceAtLeast(DEFAULT_READ_BUFFER_SIZE) + + val recorder = + AudioRecord( + MediaRecorder.AudioSource.VOICE_COMMUNICATION, + INPUT_SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + minRecordBuf, + ) + if (recorder.state != AudioRecord.STATE_INITIALIZED) { + recorder.release() + throw IllegalStateException("Failed to initialize AudioRecord") + } + + if (AcousticEchoCanceler.isAvailable()) { + runCatching { + echoCanceler = + AcousticEchoCanceler.create(recorder.audioSessionId)?.apply { + enabled = true + } + }.onFailure { e -> + Log.w(TAG, "Failed to enable AcousticEchoCanceler", e) + } + } + + val minTrackBuf = + AudioTrack.getMinBufferSize( + OUTPUT_SAMPLE_RATE, + AudioFormat.CHANNEL_OUT_MONO, + AudioFormat.ENCODING_PCM_16BIT, + ).coerceAtLeast(DEFAULT_READ_BUFFER_SIZE) + + val track = + AudioTrack.Builder() + .setAudioAttributes( + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_MEDIA) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build(), + ) + .setAudioFormat( + AudioFormat.Builder() + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setSampleRate(OUTPUT_SAMPLE_RATE) + .setChannelMask(AudioFormat.CHANNEL_OUT_MONO) + .build(), + ) + .setBufferSizeInBytes(minTrackBuf) + .setTransferMode(AudioTrack.MODE_STREAM) + .build() + + track.play() + recorder.startRecording() + + audioRecord = recorder + audioTrack = track + + playbackJob = + scope.launch { + while (isActive && !isReleased.get()) { + val chunk = audioQueue.poll() + if (chunk != null) { + val currentTrack = audioTrack + if (currentTrack != null && !isReleased.get()) { + currentTrack.write(chunk, 0, chunk.size) + } + } else { + delay(PLAYBACK_POLL_DELAY_MS) + } + } + } + } + + /** Emits PCM 16-bit 16 kHz mono byte chunks recorded from the microphone. */ + fun listenToRecording(): Flow = + flow { + val buffer = ByteArray(DEFAULT_READ_BUFFER_SIZE) + while (currentCoroutineContext().isActive && !isReleased.get()) { + if (isRecordingPaused.get()) { + delay(RECORD_PAUSE_DELAY_MS) + continue + } + val recorder = audioRecord ?: break + val bytesRead = recorder.read(buffer, 0, buffer.size) + if (bytesRead > 0 && !isRecordingPaused.get()) { + emit(buffer.copyOf(bytesRead)) + } else if (bytesRead < 0) { + Log.w(TAG, "AudioRecord.read returned error code: $bytesRead") + break + } + } + }.flowOn(Dispatchers.IO) + + fun queueAudio(data: ByteArray) { + if (!isReleased.get()) { + audioQueue.add(data) + } + } + + /** Clears any queued model audio when the user interrupts the model. */ + fun stopAudio() { + audioQueue.clear() + } + + fun pauseRecording() { + isRecordingPaused.set(true) + runCatching { + val recorder = audioRecord + if (recorder?.recordingState == AudioRecord.RECORDSTATE_RECORDING) { + recorder.stop() + } + } + } + + fun resumeRecording() { + if (!isReleased.get()) { + runCatching { + val recorder = audioRecord + if (recorder?.state == AudioRecord.STATE_INITIALIZED && + recorder.recordingState != AudioRecord.RECORDSTATE_RECORDING + ) { + recorder.startRecording() + } + } + isRecordingPaused.set(false) + } + } + + fun release() { + if (!isReleased.compareAndSet(false, true)) return + audioQueue.clear() + playbackJob?.cancel() + scope.cancel() + + runCatching { echoCanceler?.release() } + echoCanceler = null + + runCatching { + audioRecord?.let { recorder -> + if (recorder.recordingState == AudioRecord.RECORDSTATE_RECORDING) { + recorder.stop() + } + recorder.release() + } + } + audioRecord = null + + runCatching { + audioTrack?.let { track -> + if (track.playState == AudioTrack.PLAYSTATE_PLAYING) { + track.pause() + track.flush() + track.stop() + } + track.release() + } + } + audioTrack = null + } + + companion object { + private const val TAG = "GeminiLiveAudioHelper" + private const val INPUT_SAMPLE_RATE = 16000 + private const val OUTPUT_SAMPLE_RATE = 24000 + private const val DEFAULT_READ_BUFFER_SIZE = 2048 + private const val PLAYBACK_POLL_DELAY_MS = 10L + private const val RECORD_PAUSE_DELAY_MS = 25L + } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt index cc109813..e195b146 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt @@ -45,6 +45,40 @@ class GeminiToolConverter return convertToolToGeminiSchema(tool) } + /** + * Converts an [AppFunctionMetadata] into a Gemini Live (`BidiGenerateContent`) + * `FunctionDeclaration` JSON object, normalizing OpenAPI schema fields. + */ + fun convertForLiveApi(tool: AppFunctionMetadata): JsonObject { + val baseSchema = convertToolToGeminiSchema(tool) + return normalizeForLiveApi(baseSchema) + } + + private fun normalizeForLiveApi(obj: JsonObject): JsonObject { + val typeValue = (obj[KEY_TYPE] as? JsonPrimitive)?.content + return buildJsonObject { + for ((key, value) in obj) { + when { + key == KEY_FORMAT && + (value as? JsonPrimitive)?.content == VALUE_FILE_REFERENCE -> { + // Omit Interactions-API-specific file_reference format in BidiGenerateContent + } + key == KEY_ENUMS -> { + if (typeValue == VALUE_STRING) { + put(KEY_ENUM_LIVE, value) + } + } + value is JsonObject -> { + put(key, normalizeForLiveApi(value)) + } + else -> { + put(key, value) + } + } + } + } + } + override fun getToolName(tool: AppFunctionMetadata): String { return getGeminiFunctionName(tool) } @@ -295,6 +329,7 @@ class GeminiToolConverter private const val VALUE_ARRAY = "array" private const val KEY_ITEMS = "items" private const val KEY_ENUMS = "enums" + private const val KEY_ENUM_LIVE = "enum" private val INVALID_NAME_CHARS = Regex("[^a-zA-Z0-9_]") private const val MAX_NAME_LENGTH = 64 } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt index 3757f9df..347257d8 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt @@ -38,6 +38,7 @@ import io.ktor.client.HttpClient import io.ktor.client.engine.okhttp.OkHttp import io.ktor.client.plugins.HttpTimeout import io.ktor.client.plugins.contentnegotiation.ContentNegotiation +import io.ktor.client.plugins.websocket.WebSockets import io.ktor.serialization.kotlinx.json.json import kotlinx.serialization.json.Json import javax.inject.Singleton @@ -109,6 +110,7 @@ abstract class DataModule { ) } install(HttpTimeout) { socketTimeoutMillis = 30000 } + install(WebSockets) } } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index d2af9ee1..7c2c3e5b 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -94,6 +94,15 @@ class AgentOrchestrator /** The current status of the agent. */ val status: StateFlow = _status.asStateFlow() + /** Returns all currently enabled AppFunction and internal tools. */ + suspend fun getEnabledTools(targetPackageName: String? = null): List { + val disconnectedApps = settingsRepository.disconnectedApps.first() + val allTools = + getAppFunctionsUseCase().first().values.flatten() + + agentInternalTools.getInternalToolsMetadata().map { it.first } + return filterTools(allTools, disconnectedApps, targetPackageName) + } + /** * Observes messages for a specific thread and processes any pending agent responses. This * method suspends and collects messages until cancelled. @@ -524,6 +533,235 @@ class AgentOrchestrator return ExecuteToolCallsResult.Success(results) } + /** + * Executes tool calls initiated by a Gemini Live voice session for [threadId], updating + * [status], saving any generated attachments or [PendingIntent] confirmation actions to the + * chat thread, and returning a [ToolOutput] for every call. + */ + suspend fun executeLiveToolCalls( + threadId: String, + toolCalls: List, + tools: List, + ): List { + val outputs = mutableListOf() + val capturedAttachments = mutableListOf() + try { + for (toolCall in toolCalls) { + _status.value = + AgentStatus.InvokingTool(toolCall.functionId, toolCall.packageName) + + if (toolCall.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) { + val executionResult = executeInternalTool(toolCall) + executionResult + .onSuccess { jsonString -> + runCatching { + val json = JSONObject(jsonString) + val uri = json.optString("imageUri") + if (uri.isNotBlank()) { + val mimeType = + json.optString("mimeType").takeIf { it.isNotBlank() } + ?: "image/png" + capturedAttachments.add( + MessageAttachment(uri = uri, mimeType = mimeType), + ) + } + } + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = jsonString, + ), + ) + } + .onFailure { exception -> + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = + JSONObject().apply { + put( + "error", + "Internal tool execution failed: ${exception.message}", + ) + }.toString(), + ), + ) + } + continue + } + + val matchingTool = + tools.find { + it.packageName == toolCall.packageName && it.id == toolCall.functionId + } + + if (matchingTool == null) { + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = + JSONObject().apply { + put("error", "Tool not found: ${toolCall.functionId}") + }.toString(), + ), + ) + continue + } + + @Suppress("UNCHECKED_CAST") + val rawConvertedInputs = + toolCall.arguments.filterValues { it != null } as Map + val convertedInputs = + try { + withContext(Dispatchers.IO) { + resolveRemoteFileReferencesRecursively( + context = context, + parametersMetadata = matchingTool.parameters, + inputs = rawConvertedInputs, + ) + } + } catch (e: Exception) { + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = + JSONObject().apply { + put( + "error", + "Failed to download remote file reference: ${e.message}", + ) + }.toString(), + ), + ) + continue + } + + grantContentUriPermissionsRecursively( + context, + toolCall.packageName, + convertedInputs, + ) + + val appFunctionDataResult = + withContext(Dispatchers.Default) { + convertInputToAppFunctionDataUseCase( + parameters = matchingTool.parameters, + components = matchingTool.components, + inputs = convertedInputs, + ) + } + if (appFunctionDataResult.isFailure) { + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = + JSONObject().apply { + put( + "error", + "Failed to convert arguments for ${toolCall.functionId}: ${appFunctionDataResult.exceptionOrNull()?.message}", + ) + }.toString(), + ), + ) + continue + } + + val executionResult = + executeAppFunctionUseCase( + function = matchingTool, + parameters = appFunctionDataResult.getOrThrow(), + threadId = threadId, + ) + + when (executionResult) { + is ExecuteAppFunctionResult.Data -> { + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = executionResult.formattedJson, + ), + ) + } + + is ExecuteAppFunctionResult.PendingIntentAction -> { + val pendingIntentId = UUID.randomUUID().toString() + savePendingIntentUseCase(pendingIntentId, executionResult.pendingIntent) + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = "", + processingStatus = MessageProcessingStatus.PROCESSED, + pendingIntentId = pendingIntentId, + ) + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = + JSONObject().apply { + put("status", "confirmation_required") + put( + "message", + "A confirmation button has been displayed in the chat. Ask the user to tap it to confirm.", + ) + }.toString(), + ), + ) + } + + is ExecuteAppFunctionResult.Error -> { + val exception = executionResult.exception + if (exception is CancellationException) { + throw exception + } + val appFunctionException = + AppFunctionExceptionFormatter.getAppFunctionException(exception) + val errorJson = + if (appFunctionException != null) { + AppFunctionExceptionFormatter.format( + appFunctionException, + toolCall.functionId, + ) + } else { + JSONObject().apply { + put( + "error", + "Tool execution failed for ${toolCall.functionId}: ${exception.message}", + ) + }.toString() + } + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = errorJson, + ), + ) + } + } + } + + if (capturedAttachments.isNotEmpty()) { + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = "", + processingStatus = MessageProcessingStatus.PROCESSED, + attachments = capturedAttachments, + ) + } + } finally { + _status.value = AgentStatus.Idle + } + return outputs + } + private suspend fun completeMessageWithError( messageId: String, threadId: String, diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManager.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManager.kt new file mode 100644 index 00000000..997dc03f --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManager.kt @@ -0,0 +1,608 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain + +import android.util.Base64 +import android.util.Log +import androidx.appfunctions.metadata.AppFunctionMetadata +import com.example.appfunctions.agent.data.GeminiLiveAudioHelper +import com.example.appfunctions.agent.data.GeminiToolConverter +import com.example.appfunctions.agent.data.SettingsRepository +import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus +import com.example.appfunctions.agent.data.db.entities.MessageRole +import com.example.appfunctions.agent.domain.chat.SendMessageUseCase +import io.ktor.client.HttpClient +import io.ktor.client.plugins.websocket.DefaultClientWebSocketSession +import io.ktor.client.plugins.websocket.webSocket +import io.ktor.websocket.CloseReason +import io.ktor.websocket.Frame +import io.ktor.websocket.close +import io.ktor.websocket.readBytes +import io.ktor.websocket.readText +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.booleanOrNull +import kotlinx.serialization.json.buildJsonArray +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.put +import java.time.LocalDate +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Manages a bidirectional voice session with the Gemini Live API (`models/gemini-3.8-live`), + * bridging realtime audio I/O, transcriptions, and Android AppFunction tool execution. + */ +@Singleton +class GeminiLiveSessionManager + @Inject + constructor( + private val httpClient: HttpClient, + private val toolConverter: GeminiToolConverter, + private val settingsRepository: SettingsRepository, + private val agentOrchestrator: AgentOrchestrator, + private val sendMessageUseCase: SendMessageUseCase, + ) { + private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + + private val _state = MutableStateFlow(LiveSessionState.Idle) + + /** Current state of the Gemini Live session. */ + val state: StateFlow = _state.asStateFlow() + + private var sessionJob: Job? = null + private var audioHelper: GeminiLiveAudioHelper? = null + + internal var audioHelperFactory: () -> GeminiLiveAudioHelper = { GeminiLiveAudioHelper() } + + /** Toggles the Gemini Live voice session for [threadId]. */ + fun toggleSession( + threadId: String, + targetPackageName: String? = null, + ) { + when (_state.value) { + is LiveSessionState.Connecting, + is LiveSessionState.Listening, + -> stopSession() + is LiveSessionState.Idle, + is LiveSessionState.Error, + -> startSession(threadId, targetPackageName) + } + } + + /** Starts a new Gemini Live voice session for [threadId]. */ + fun startSession( + threadId: String, + targetPackageName: String? = null, + ) { + stopSessionInternal(resetState = false) + _state.value = LiveSessionState.Connecting + + sessionJob = + scope.launch { + val userTranscriptBuilder = StringBuilder() + val modelTranscriptBuilder = StringBuilder() + try { + val apiKey = + settingsRepository.geminiApiKey.first()?.takeIf { it.isNotBlank() } + if (apiKey == null) { + val errorMsg = "API key is missing for GEMINI" + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = errorMsg, + processingStatus = MessageProcessingStatus.FAILED, + ) + _state.value = LiveSessionState.Error(errorMsg) + return@launch + } + + val tools = agentOrchestrator.getEnabledTools(targetPackageName) + val wsUrl = "$GEMINI_LIVE_WS_URL?key=$apiKey" + + httpClient.webSocket(urlString = wsUrl) { + val setupMessage = buildSetupMessage(tools) + Log.d(TAG, "Sending Gemini Live setup for model $LIVE_MODEL_NAME") + send(Frame.Text(setupMessage.toString())) + + val firstFrameText = receiveNextJsonText() + if (firstFrameText == null) { + throw IllegalStateException( + "WebSocket closed before setupComplete was received", + ) + } + val firstJson = Json.parseToJsonElement(firstFrameText).jsonObject + if (!firstJson.containsKey("setupComplete")) { + throw IllegalStateException( + "Unexpected setup response: $firstFrameText", + ) + } + + val helper = + audioHelperFactory().also { + it.start() + audioHelper = it + } + _state.value = LiveSessionState.Listening + + val sendAudioJob = + launch { + helper.listenToRecording().collect { pcmBytes -> + val base64Audio = + Base64.encodeToString(pcmBytes, Base64.NO_WRAP) + val realtimeInput = + buildJsonObject { + put( + "realtimeInput", + buildJsonObject { + put( + "audio", + buildJsonObject { + put( + "mimeType", + "audio/pcm;rate=16000", + ) + put("data", base64Audio) + }, + ) + }, + ) + } + send(Frame.Text(realtimeInput.toString())) + } + } + + try { + while (isActive) { + val messageText = receiveNextJsonText() ?: break + val serverJson = + runCatching { + Json.parseToJsonElement(messageText).jsonObject + }.getOrNull() ?: continue + + handleServerMessage( + serverJson = serverJson, + threadId = threadId, + tools = tools, + helper = helper, + userTranscriptBuilder = userTranscriptBuilder, + modelTranscriptBuilder = modelTranscriptBuilder, + ) + } + } finally { + sendAudioJob.cancel() + runCatching { + close( + CloseReason( + CloseReason.Codes.NORMAL, + "Session ended by client", + ), + ) + } + } + } + + flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) + _state.value = LiveSessionState.Idle + } catch (e: CancellationException) { + scope.launch { + flushTranscripts( + threadId, + userTranscriptBuilder, + modelTranscriptBuilder, + ) + } + throw e + } catch (e: Exception) { + Log.e(TAG, "Gemini Live session error", e) + flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) + val errorText = + "Gemini Live error: ${e.message ?: "Connection failed"}" + runCatching { + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = errorText, + processingStatus = MessageProcessingStatus.FAILED, + ) + } + _state.value = LiveSessionState.Error(errorText) + } finally { + audioHelper?.release() + audioHelper = null + } + } + } + + /** Stops any active Gemini Live voice session. */ + fun stopSession() { + stopSessionInternal(resetState = true) + } + + private fun stopSessionInternal(resetState: Boolean) { + sessionJob?.cancel() + sessionJob = null + audioHelper?.release() + audioHelper = null + if (resetState) { + _state.value = LiveSessionState.Idle + } + } + + private suspend fun DefaultClientWebSocketSession.receiveNextJsonText(): String? { + while (isActive) { + val frame = runCatching { incoming.receive() }.getOrNull() ?: return null + when (frame) { + is Frame.Text -> return frame.readText() + is Frame.Binary -> return frame.readBytes().decodeToString() + is Frame.Close -> return null + else -> continue + } + } + return null + } + + private suspend fun DefaultClientWebSocketSession.handleServerMessage( + serverJson: JsonObject, + threadId: String, + tools: List, + helper: GeminiLiveAudioHelper, + userTranscriptBuilder: StringBuilder, + modelTranscriptBuilder: StringBuilder, + ) { + val serverContent = serverJson["serverContent"]?.jsonObject + if (serverContent != null) { + val interrupted = + serverContent["interrupted"]?.jsonPrimitive?.booleanOrNull ?: false + if (interrupted) { + helper.stopAudio() + flushTranscript(threadId, MessageRole.ASSISTANT, modelTranscriptBuilder) + } + + val inputTranscription = + serverContent["inputTranscription"] + ?.jsonObject + ?.get("text") + ?.jsonPrimitive + ?.contentOrNull + if (!inputTranscription.isNullOrEmpty()) { + userTranscriptBuilder.append(inputTranscription) + } + + val outputTranscription = + serverContent["outputTranscription"] + ?.jsonObject + ?.get("text") + ?.jsonPrimitive + ?.contentOrNull + if (!outputTranscription.isNullOrEmpty()) { + // Flush user transcript before model response transcript starts + flushTranscript(threadId, MessageRole.USER, userTranscriptBuilder) + modelTranscriptBuilder.append(outputTranscription) + } + + val parts = + serverContent["modelTurn"]?.jsonObject?.get("parts")?.jsonArray + if (parts != null) { + for (part in parts) { + val inlineData = part.jsonObject["inlineData"]?.jsonObject ?: continue + val mimeType = + inlineData["mimeType"]?.jsonPrimitive?.contentOrNull ?: "" + val base64Data = + inlineData["data"]?.jsonPrimitive?.contentOrNull ?: continue + if (mimeType.startsWith("audio/pcm")) { + val pcmBytes = Base64.decode(base64Data, Base64.DEFAULT) + helper.queueAudio(pcmBytes) + } + } + } + + val turnComplete = + serverContent["turnComplete"]?.jsonPrimitive?.booleanOrNull ?: false + if (turnComplete) { + flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) + } + } + + val toolCallObj = serverJson["toolCall"]?.jsonObject + if (toolCallObj != null) { + flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) + val functionCalls = toolCallObj["functionCalls"]?.jsonArray ?: JsonArray(emptyList()) + if (functionCalls.isNotEmpty()) { + helper.pauseRecording() + try { + val parsedCalls = mutableListOf() + val unknownCalls = mutableListOf>() + + for (callElement in functionCalls) { + val callObj = callElement.jsonObject + val callId = callObj["id"]?.jsonPrimitive?.contentOrNull ?: "" + val name = callObj["name"]?.jsonPrimitive?.contentOrNull ?: continue + val matchingTool = + tools.find { tool -> toolConverter.getToolName(tool) == name } + + if (matchingTool == null) { + unknownCalls.add(callId to name) + continue + } + + val argsObj = callObj["args"]?.jsonObject + val argumentsMap = mutableMapOf() + if (argsObj != null) { + for ((key, value) in argsObj) { + argumentsMap[key] = value.toPrimitive() + } + } + parsedCalls.add( + LlmResponsePart.ToolCall( + packageName = matchingTool.packageName, + functionId = matchingTool.id, + arguments = argumentsMap, + callId = callId, + ), + ) + } + + val toolOutputs = + if (parsedCalls.isNotEmpty()) { + agentOrchestrator.executeLiveToolCalls( + threadId = threadId, + toolCalls = parsedCalls, + tools = tools, + ) + } else { + emptyList() + } + + val toolResponseMessage = + buildToolResponseMessage( + toolOutputs = toolOutputs, + unknownCalls = unknownCalls, + tools = tools, + ) + Log.d(TAG, "Sending Gemini Live toolResponse: $toolResponseMessage") + send(Frame.Text(toolResponseMessage.toString())) + } finally { + helper.resumeRecording() + } + } + } + } + + private suspend fun flushTranscripts( + threadId: String, + userTranscriptBuilder: StringBuilder, + modelTranscriptBuilder: StringBuilder, + ) { + flushTranscript(threadId, MessageRole.USER, userTranscriptBuilder) + flushTranscript(threadId, MessageRole.ASSISTANT, modelTranscriptBuilder) + } + + private suspend fun flushTranscript( + threadId: String, + role: MessageRole, + builder: StringBuilder, + ) { + val text = builder.toString().trim() + builder.clear() + if (text.isNotEmpty()) { + sendMessageUseCase( + threadId = threadId, + role = role, + textContent = text, + processingStatus = MessageProcessingStatus.PROCESSED, + ) + } + } + + internal fun buildSetupMessage(tools: List): JsonObject { + val functionDeclarations = + tools + .sortedByDescending { it.id.startsWith(it.packageName) } + .distinctBy { toolConverter.getToolName(it) } + .mapNotNull { tool -> + try { + toolConverter.convertForLiveApi(tool) + } catch (e: IllegalArgumentException) { + Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e) + null + } + } + + return buildJsonObject { + put( + "setup", + buildJsonObject { + put("model", LIVE_MODEL_NAME) + put( + "generationConfig", + buildJsonObject { + put( + "responseModalities", + buildJsonArray { add(JsonPrimitive("AUDIO")) }, + ) + put( + "speechConfig", + buildJsonObject { + put( + "voiceConfig", + buildJsonObject { + put( + "prebuiltVoiceConfig", + buildJsonObject { + put("voiceName", DEFAULT_VOICE_NAME) + }, + ) + }, + ) + }, + ) + }, + ) + put( + "systemInstruction", + buildJsonObject { + put( + "parts", + buildJsonArray { + add( + buildJsonObject { + put("text", getSystemInstruction()) + }, + ) + }, + ) + }, + ) + if (functionDeclarations.isNotEmpty()) { + put( + "tools", + buildJsonArray { + add( + buildJsonObject { + put( + "functionDeclarations", + JsonArray(functionDeclarations), + ) + }, + ) + }, + ) + } + put("inputAudioTranscription", buildJsonObject {}) + put("outputAudioTranscription", buildJsonObject {}) + }, + ) + } + } + + private fun buildToolResponseMessage( + toolOutputs: List, + unknownCalls: List>, + tools: List, + ): JsonObject { + return buildJsonObject { + put( + "toolResponse", + buildJsonObject { + put( + "functionResponses", + buildJsonArray { + for (output in toolOutputs) { + val matchingTool = tools.find { it.id == output.functionId } + val mappedName = + if (matchingTool != null) { + toolConverter.getToolName(matchingTool) + } else { + output.functionId + } + val parsedResult = + runCatching { + Json.parseToJsonElement(output.result) + }.getOrElse { + JsonPrimitive(output.result) + } + add( + buildJsonObject { + if (output.callId.isNotEmpty()) { + put("id", output.callId) + } + put("name", mappedName) + put( + "response", + buildJsonObject { + put("result", parsedResult) + }, + ) + }, + ) + } + for ((callId, unknownName) in unknownCalls) { + add( + buildJsonObject { + if (callId.isNotEmpty()) { + put("id", callId) + } + put("name", unknownName) + put( + "response", + buildJsonObject { + put( + "error", + "Unknown function: $unknownName", + ) + }, + ) + }, + ) + } + }, + ) + }, + ) + } + } + + private fun JsonElement.toPrimitive(): Any? { + return when (this) { + is JsonPrimitive -> { + if (this.isString) return this.content + return this.content.toBooleanStrictOrNull() + ?: this.content.toLongOrNull() + ?: this.content.toDoubleOrNull() + } + is JsonObject -> this.mapValues { it.value.toPrimitive() } + is JsonArray -> this.map { it.toPrimitive() } + } + } + + private fun getSystemInstruction(): String { + val currentDate = LocalDate.now().toString() + return """ + You are an AI voice assistant running on Android using Gemini Live. Today's date is $currentDate. + Always reply to the user using concise, friendly, natural spoken language. + You have access to Android AppFunctions and internal device tools. Use them whenever the user asks to perform an action or look up information on their device. + When a user asks you to generate an image, call the generateImage tool and then confirm to the user in a natural sentence. + When a tool returns status "confirmation_required", let the user know that a confirmation button is on their screen and ask them to tap it to proceed. + """.trimIndent() + } + + companion object { + private const val TAG = "GeminiLiveSessionMgr" + const val LIVE_MODEL_NAME = "models/gemini-3.8-live" + private const val DEFAULT_VOICE_NAME = "CHARON" + private const val GEMINI_LIVE_WS_URL = + "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent" + } + } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveSessionState.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveSessionState.kt new file mode 100644 index 00000000..f9fc51c0 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveSessionState.kt @@ -0,0 +1,27 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain + +/** Represents the state of a Gemini Live voice session. */ +sealed class LiveSessionState { + object Idle : LiveSessionState() + + object Connecting : LiveSessionState() + + object Listening : LiveSessionState() + + data class Error(val message: String) : LiveSessionState() +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt index b1c9f3b3..3f7665af 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt @@ -15,7 +15,10 @@ */ package com.example.appfunctions.agent.ui.screens.agentdemo +import android.Manifest import android.content.pm.PackageManager +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.animation.AnimatedVisibility import androidx.compose.animation.expandHorizontally import androidx.compose.animation.shrinkHorizontally @@ -49,6 +52,8 @@ import androidx.compose.material.icons.automirrored.filled.Send import androidx.compose.material.icons.filled.Add import androidx.compose.material.icons.filled.ArrowDropDown import androidx.compose.material.icons.filled.Menu +import androidx.compose.material.icons.filled.Mic +import androidx.compose.material.icons.filled.Stop import androidx.compose.material.icons.filled.Warning import androidx.compose.material3.ButtonDefaults import androidx.compose.material3.Card @@ -72,6 +77,7 @@ import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.material3.rememberDrawerState import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf @@ -116,6 +122,7 @@ import androidx.compose.ui.unit.dp import androidx.compose.ui.window.Popup import androidx.compose.ui.window.PopupPositionProvider import androidx.compose.ui.window.PopupProperties +import androidx.core.content.ContextCompat import androidx.core.graphics.drawable.toBitmap import androidx.hilt.navigation.compose.hiltViewModel import androidx.lifecycle.compose.collectAsStateWithLifecycle @@ -127,6 +134,7 @@ import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentStatus +import com.example.appfunctions.agent.domain.LiveSessionState import com.example.appfunctions.agent.domain.appfunction.AppInfo import com.example.appfunctions.agent.ui.screens.debugging.LazyExposedDropdownMenu import com.mikepenz.markdown.m3.Markdown @@ -137,6 +145,12 @@ import kotlinx.coroutines.launch fun AgentDemoScreen(viewModel: AgentDemoViewModel = hiltViewModel()) { val uiState by viewModel.uiState.collectAsStateWithLifecycle() + DisposableEffect(Unit) { + onDispose { + viewModel.onEvent(AgentUiEvent.OnStopLiveSession) + } + } + AgentDemoContent(uiState = uiState, onEvent = viewModel::onEvent) } @@ -230,6 +244,44 @@ fun AgentDemoLoadedScreen( var isSidePanelVisible by remember { mutableStateOf(initialSidePanelVisible) } var selectedAppPackageName by remember { mutableStateOf(null) } + val context = LocalContext.current + val audioPermissionLauncher = + rememberLauncherForActivityResult( + contract = ActivityResultContracts.RequestPermission(), + ) { isGranted -> + if (isGranted) { + onEvent(AgentUiEvent.OnToggleLiveSession(selectedAppPackageName)) + } + } + + val toggleLiveSession = { + val isActive = + uiState.liveSessionState is LiveSessionState.Connecting || + uiState.liveSessionState is LiveSessionState.Listening + if (isActive) { + onEvent(AgentUiEvent.OnStopLiveSession) + } else { + val hasRecordAudioPermission = + try { + ContextCompat.checkSelfPermission( + context, + Manifest.permission.RECORD_AUDIO, + ) == PackageManager.PERMISSION_GRANTED + } catch (_: Exception) { + true + } + if (hasRecordAudioPermission) { + onEvent(AgentUiEvent.OnToggleLiveSession(selectedAppPackageName)) + } else { + try { + audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + } catch (_: IllegalStateException) { + onEvent(AgentUiEvent.OnToggleLiveSession(selectedAppPackageName)) + } + } + } + } + val chipBgColor = MaterialTheme.colorScheme.primaryContainer val chipTextColor = MaterialTheme.colorScheme.onPrimaryContainer val visualTransformation = @@ -324,6 +376,18 @@ fun AgentDemoLoadedScreen( } } + if ( + uiState.liveSessionState is LiveSessionState.Connecting || + uiState.liveSessionState is LiveSessionState.Listening + ) { + item { + LiveSessionIndicator( + liveSessionState = uiState.liveSessionState, + onStopClick = { onEvent(AgentUiEvent.OnStopLiveSession) }, + ) + } + } + items( items = uiState.messages.reversed(), key = { message -> message.messageId }, @@ -440,17 +504,49 @@ fun AgentDemoLoadedScreen( focusedBorderColor = Color.Transparent, ), trailingIcon = { - IconButton( - onClick = sendMessage, - enabled = - messageText.text.isNotBlank() && - uiState.status == AgentStatus.Idle, + val isLiveActive = + uiState.liveSessionState is LiveSessionState.Connecting || + uiState.liveSessionState is LiveSessionState.Listening + Row( + verticalAlignment = Alignment.CenterVertically, + modifier = Modifier.padding(end = 4.dp), ) { - Icon( - imageVector = Icons.AutoMirrored.Filled.Send, - contentDescription = - stringResource(R.string.agent_demo_send), - ) + IconButton( + onClick = toggleLiveSession, + ) { + Icon( + imageVector = + if (isLiveActive) { + Icons.Default.Stop + } else { + Icons.Default.Mic + }, + contentDescription = + if (isLiveActive) { + stringResource(R.string.agent_demo_live_stop) + } else { + stringResource(R.string.agent_demo_live_start) + }, + tint = + if (isLiveActive) { + MaterialTheme.colorScheme.error + } else { + MaterialTheme.colorScheme.primary + }, + ) + } + IconButton( + onClick = sendMessage, + enabled = + messageText.text.isNotBlank() && + uiState.status == AgentStatus.Idle, + ) { + Icon( + imageVector = Icons.AutoMirrored.Filled.Send, + contentDescription = + stringResource(R.string.agent_demo_send), + ) + } } }, ) @@ -907,6 +1003,59 @@ fun StatusIndicator( } } +@Composable +fun LiveSessionIndicator( + liveSessionState: LiveSessionState, + onStopClick: () -> Unit, +) { + val label = + when (liveSessionState) { + LiveSessionState.Connecting -> stringResource(R.string.agent_demo_live_connecting) + LiveSessionState.Listening -> stringResource(R.string.agent_demo_live_listening) + else -> return + } + + Surface( + modifier = + Modifier + .fillMaxWidth() + .padding(vertical = 8.dp), + shape = MaterialTheme.shapes.large, + color = MaterialTheme.colorScheme.secondaryContainer, + shadowElevation = 2.dp, + ) { + Row( + modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.SpaceBetween, + ) { + Row( + verticalAlignment = Alignment.CenterVertically, + modifier = Modifier.weight(1f), + ) { + CircularProgressIndicator( + modifier = Modifier.size(20.dp), + strokeWidth = 2.dp, + color = MaterialTheme.colorScheme.onSecondaryContainer, + ) + Spacer(modifier = Modifier.width(12.dp)) + Text( + text = label, + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSecondaryContainer, + ) + } + IconButton(onClick = onStopClick) { + Icon( + imageVector = Icons.Default.Stop, + contentDescription = stringResource(R.string.agent_demo_live_stop), + tint = MaterialTheme.colorScheme.error, + ) + } + } + } +} + @Composable fun ChatHistorySidePanel( threads: List, diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt index 2f92660d..2212ddb7 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt @@ -27,6 +27,7 @@ import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentOrchestrator import com.example.appfunctions.agent.domain.AgentStatus +import com.example.appfunctions.agent.domain.GeminiLiveSessionManager import com.example.appfunctions.agent.domain.appfunction.AppInfo import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase @@ -64,6 +65,7 @@ class AgentDemoViewModel private val consumePendingIntentUseCase: ConsumePendingIntentUseCase, private val getInstalledAppsUseCase: GetInstalledAppsUseCase, private val getAppFunctionsUseCase: GetAppFunctionsUseCase, + private val geminiLiveSessionManager: GeminiLiveSessionManager, ) : ViewModel() { private val _uiState = MutableStateFlow(AgentUiState.Loading) val uiState: StateFlow = _uiState.asStateFlow() @@ -95,6 +97,15 @@ class AgentDemoViewModel } } + viewModelScope.launch { + geminiLiveSessionManager.state.collect { liveState -> + val currentState = _uiState.value + if (currentState is AgentUiState.Loaded) { + _uiState.value = currentState.copy(liveSessionState = liveState) + } + } + } + viewModelScope.launch { val threads = manageThreadsUseCase.getThreads().first() if (threads.isEmpty()) { @@ -140,10 +151,14 @@ class AgentDemoViewModel activePendingActionIds = currentLoadedState?.activePendingActionIds ?: emptySet(), installedApps = apps, + liveSessionState = geminiLiveSessionManager.state.value, ) // Start observing messages for the current thread if not already doing so if (observationJob == null || previousThreadId != currentThread.threadId) { + if (previousThreadId != null && previousThreadId != currentThread.threadId) { + geminiLiveSessionManager.stopSession() + } observationJob?.cancel() observationJob = viewModelScope.launch { @@ -207,9 +222,25 @@ class AgentDemoViewModel launchPendingIntentUseCase(pendingIntent) } } + is AgentUiEvent.OnToggleLiveSession -> { + if (currentState is AgentUiState.Loaded) { + geminiLiveSessionManager.toggleSession( + threadId = currentState.currentThread.threadId, + targetPackageName = event.targetPackageName, + ) + } + } + AgentUiEvent.OnStopLiveSession -> { + geminiLiveSessionManager.stopSession() + } } } + override fun onCleared() { + super.onCleared() + geminiLiveSessionManager.stopSession() + } + private suspend fun createAndSelectThread(llmModel: LlmModel) { val threadId = manageThreadsUseCase.createThread(llmModel) savedStateHandle[MainActivity.ARG_THREAD_ID] = threadId diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt index 4798f798..04c2ecdc 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt @@ -19,6 +19,7 @@ import com.example.appfunctions.agent.data.LlmModel import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentStatus +import com.example.appfunctions.agent.domain.LiveSessionState import com.example.appfunctions.agent.domain.appfunction.AppInfo /** Represents the UI state for the Agent Demo screen. */ @@ -32,6 +33,7 @@ sealed class AgentUiState { val threads: List = emptyList(), val activePendingActionIds: Set = emptySet(), val installedApps: List = emptyList(), + val liveSessionState: LiveSessionState = LiveSessionState.Idle, ) : AgentUiState() } @@ -49,4 +51,8 @@ sealed class AgentUiEvent { data class OnThreadSelected(val threadId: String) : AgentUiEvent() data class OnConfirmAction(val pendingIntentId: String) : AgentUiEvent() + + data class OnToggleLiveSession(val targetPackageName: String? = null) : AgentUiEvent() + + object OnStopLiveSession : AgentUiEvent() } diff --git a/agent/app/src/main/res/values/strings.xml b/agent/app/src/main/res/values/strings.xml index 8a74b210..243f2eeb 100644 --- a/agent/app/src/main/res/values/strings.xml +++ b/agent/app/src/main/res/values/strings.xml @@ -110,6 +110,10 @@ Yes, go ahead Done I can perform this action for you. Should I proceed? + Start Gemini Live + Stop Gemini Live + Connecting to Gemini Live (gemini-3.8-live)… + Gemini Live (gemini-3.8-live) is listening… Copied value of %1$s Copy value diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt index 62cb16ab..8388855d 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt @@ -520,4 +520,68 @@ class GeminiToolConverterTest { assertEquals(expectedJson, schema) } + + @Test + fun convertForLiveApi_normalizesEnumsAndStripsFileReferenceFormat() { + val enumParam = + AppFunctionParameterMetadata( + name = "mode", + isRequired = true, + dataType = + AppFunctionStringTypeMetadata(isNullable = false, enumValues = setOf("A", "B")), + description = "Mode parameter", + ) + val uriParam = + AppFunctionParameterMetadata( + name = "wallpaperUri", + isRequired = false, + dataType = AppFunctionStringTypeMetadata(isNullable = true), + description = "Wallpaper URI", + ) + val tool = + AppFunctionMetadata( + id = "com.example.my_function", + packageName = "com.example", + isEnabled = true, + schema = null, + parameters = listOf(enumParam, uriParam), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Test function", + deprecation = null, + ) + + val liveSchema = converter.convertForLiveApi(tool) + + val expectedJson = + Json.parseToJsonElement( + """ + { + "name": "com_example_my_function", + "description": "Test function", + "parameters": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["A", "B"], + "description": "Mode parameter" + }, + "wallpaperUri": { + "type": "string", + "description": "Wallpaper URI" + } + }, + "required": ["mode"] + } + } + """, + ) + + assertEquals(expectedJson, liveSchema) + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManagerTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManagerTest.kt new file mode 100644 index 00000000..9d1381f9 --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManagerTest.kt @@ -0,0 +1,106 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain + +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionResponseMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import com.example.appfunctions.agent.data.GeminiToolConverter +import com.example.appfunctions.agent.data.SettingsRepository +import com.example.appfunctions.agent.domain.chat.SendMessageUseCase +import io.ktor.client.HttpClient +import io.mockk.mockk +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test + +class GeminiLiveSessionManagerTest { + private lateinit var sessionManager: GeminiLiveSessionManager + + @Before + fun setUp() { + sessionManager = + GeminiLiveSessionManager( + httpClient = mockk(relaxed = true), + toolConverter = GeminiToolConverter(), + settingsRepository = mockk(relaxed = true), + agentOrchestrator = mockk(relaxed = true), + sendMessageUseCase = mockk(relaxed = true), + ) + } + + @Test + fun `buildSetupMessage configures gemini-3_8-live model, audio modality, transcriptions, and tools`() { + val tool = + AppFunctionMetadata( + id = "com.example.notes.createNote", + packageName = "com.example.notes", + isEnabled = true, + schema = null, + parameters = + listOf( + AppFunctionParameterMetadata( + name = "title", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(isNullable = false), + description = "Note title", + ), + ), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Creates a note", + deprecation = null, + ) + + val setupEnvelope = sessionManager.buildSetupMessage(listOf(tool)) + val setup = setupEnvelope["setup"]!!.jsonObject + + assertEquals("models/gemini-3.8-live", setup["model"]!!.jsonPrimitive.content) + + val modalities = + setup["generationConfig"]!! + .jsonObject["responseModalities"]!! + .jsonArray + .map { it.jsonPrimitive.content } + assertEquals(listOf("AUDIO"), modalities) + + assertTrue(setup.containsKey("inputAudioTranscription")) + assertTrue(setup.containsKey("outputAudioTranscription")) + assertTrue(setup.containsKey("systemInstruction")) + + val declarations = + setup["tools"]!! + .jsonArray + .first() + .jsonObject["functionDeclarations"]!! + .jsonArray + assertEquals(1, declarations.size) + assertEquals( + "com_example_notes_createNote", + declarations.first().jsonObject["name"]!!.jsonPrimitive.content, + ) + } +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt index 27a48055..a2aba834 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt @@ -25,6 +25,8 @@ import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentOrchestrator import com.example.appfunctions.agent.domain.AgentStatus +import com.example.appfunctions.agent.domain.GeminiLiveSessionManager +import com.example.appfunctions.agent.domain.LiveSessionState import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase import com.example.appfunctions.agent.domain.chat.GetChatHistoryUseCase @@ -37,6 +39,7 @@ import io.mockk.coEvery import io.mockk.coVerify import io.mockk.every import io.mockk.mockk +import io.mockk.verify import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.flow.MutableStateFlow @@ -65,6 +68,7 @@ class AgentDemoViewModelTest { private lateinit var consumePendingIntentUseCase: ConsumePendingIntentUseCase private lateinit var getInstalledAppsUseCase: GetInstalledAppsUseCase private lateinit var getAppFunctionsUseCase: GetAppFunctionsUseCase + private lateinit var geminiLiveSessionManager: GeminiLiveSessionManager private lateinit var viewModel: AgentDemoViewModel @@ -73,6 +77,7 @@ class AgentDemoViewModelTest { private val agentStatusFlow = MutableStateFlow(AgentStatus.Idle) private val messagesFlow = MutableStateFlow>(emptyList()) private val activePendingActionIdsFlow = MutableStateFlow>(emptySet()) + private val liveSessionStateFlow = MutableStateFlow(LiveSessionState.Idle) @Before fun setUp() { @@ -89,6 +94,7 @@ class AgentDemoViewModelTest { consumePendingIntentUseCase = mockk(relaxed = true) getInstalledAppsUseCase = mockk() getAppFunctionsUseCase = mockk() + geminiLiveSessionManager = mockk(relaxed = true) every { getInstalledAppsUseCase() } returns emptyList() every { getAppFunctionsUseCase() } returns flowOf(emptyMap()) @@ -98,7 +104,7 @@ class AgentDemoViewModelTest { every { agentOrchestrator.status } returns agentStatusFlow every { getChatHistoryUseCase(any()) } returns messagesFlow every { observeActivePendingIntentsUseCase() } returns activePendingActionIdsFlow - every { observeActivePendingIntentsUseCase() } returns activePendingActionIdsFlow + every { geminiLiveSessionManager.state } returns liveSessionStateFlow coEvery { agentOrchestrator.observeAndProcessMessages(any()) } returns Unit } @@ -136,6 +142,7 @@ class AgentDemoViewModelTest { consumePendingIntentUseCase, getInstalledAppsUseCase, getAppFunctionsUseCase, + geminiLiveSessionManager, ) viewModel.onEvent(AgentUiEvent.OnModelSelected(LlmModel.GEMINI_3_1_PRO_PREVIEW)) @@ -164,6 +171,7 @@ class AgentDemoViewModelTest { consumePendingIntentUseCase, getInstalledAppsUseCase, getAppFunctionsUseCase, + geminiLiveSessionManager, ) coVerify { manageThreadsUseCase.createThread(LlmModel.DEFAULT) } @@ -195,6 +203,7 @@ class AgentDemoViewModelTest { consumePendingIntentUseCase, getInstalledAppsUseCase, getAppFunctionsUseCase, + geminiLiveSessionManager, ) coVerify(exactly = 0) { manageThreadsUseCase.createThread(any()) } @@ -230,6 +239,7 @@ class AgentDemoViewModelTest { consumePendingIntentUseCase, getInstalledAppsUseCase, getAppFunctionsUseCase, + geminiLiveSessionManager, ) coEvery { sendMessageUseCase(any(), any(), any(), any()) } returns Unit @@ -272,6 +282,7 @@ class AgentDemoViewModelTest { consumePendingIntentUseCase, getInstalledAppsUseCase, getAppFunctionsUseCase, + geminiLiveSessionManager, ) val message = @@ -321,8 +332,44 @@ class AgentDemoViewModelTest { consumePendingIntentUseCase, getInstalledAppsUseCase, getAppFunctionsUseCase, + geminiLiveSessionManager, ) assertEquals(newThread, (viewModel.uiState.value as AgentUiState.Loaded).currentThread) } + + @Test + fun `onEvent OnToggleLiveSession delegates to GeminiLiveSessionManager`() = + runTest { + val existingThread = + ThreadEntity( + "thread-1", + System.currentTimeMillis(), + LlmModel.GEMINI_3_1_PRO_PREVIEW, + null, + ) + threadsFlow.value = listOf(existingThread) + + viewModel = + AgentDemoViewModel( + savedStateHandle, + getChatHistoryUseCase, + manageThreadsUseCase, + sendMessageUseCase, + agentOrchestrator, + settingsRepository, + observeActivePendingIntentsUseCase, + launchPendingIntentUseCase, + consumePendingIntentUseCase, + getInstalledAppsUseCase, + getAppFunctionsUseCase, + geminiLiveSessionManager, + ) + + viewModel.onEvent(AgentUiEvent.OnToggleLiveSession("com.example.target")) + + verify { + geminiLiveSessionManager.toggleSession("thread-1", "com.example.target") + } + } } From 0561b6b8f6cf50c560f9142d7d4e73c62d602cae Mon Sep 17 00:00:00 2001 From: Julian Toledo Date: Fri, 23 Oct 2026 22:11:41 +0200 Subject: [PATCH 2/6] feat(ui): display gemini-3.8-live in top model bar when Live session is active --- .../agent/domain/AgentOrchestrator.kt | 3 +- .../ui/screens/agentdemo/AgentDemoScreen.kt | 42 ++++++++++++++----- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index 7c2c3e5b..0b3b7a8d 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -655,6 +655,7 @@ class AgentOrchestrator ) } if (appFunctionDataResult.isFailure) { + val errMsg = appFunctionDataResult.exceptionOrNull()?.message outputs.add( ToolOutput( functionId = toolCall.functionId, @@ -663,7 +664,7 @@ class AgentOrchestrator JSONObject().apply { put( "error", - "Failed to convert arguments for ${toolCall.functionId}: ${appFunctionDataResult.exceptionOrNull()?.message}", + "Failed to convert arguments for ${toolCall.functionId}: $errMsg", ) }.toString(), ), diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt index 3f7665af..4ab33cd3 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt @@ -303,6 +303,7 @@ fun AgentDemoLoadedScreen( .weight(1f) .padding(horizontal = 8.dp), currentThread = uiState.currentThread, + liveSessionState = uiState.liveSessionState, onModelSelected = { onEvent(AgentUiEvent.OnModelSelected(it)) }, onMenuClick = { if (isWideScreen) { @@ -620,14 +621,29 @@ fun AgentDemoLoadedScreen( fun ModelDropdown( modifier: Modifier = Modifier, currentThread: ThreadEntity?, + liveSessionState: LiveSessionState = LiveSessionState.Idle, onModelSelected: (LlmModel) -> Unit, onMenuClick: () -> Unit, ) { var expanded by remember { mutableStateOf(false) } + val isLiveActive = + liveSessionState is LiveSessionState.Connecting || + liveSessionState is LiveSessionState.Listening + + LaunchedEffect(isLiveActive) { + if (isLiveActive) { + expanded = false + } + } + ExposedDropdownMenuBox( modifier = modifier, - expanded = expanded, - onExpandedChange = { expanded = !expanded }, + expanded = expanded && !isLiveActive, + onExpandedChange = { + if (!isLiveActive) { + expanded = !expanded + } + }, ) { Surface( modifier = Modifier.padding(bottom = 8.dp), @@ -636,13 +652,17 @@ fun ModelDropdown( color = MaterialTheme.colorScheme.surfaceBright, ) { val text = - currentThread?.llmModel?.modelName - ?: stringResource(R.string.agent_demo_select_model_to_create_thread) - val textColor = - if (currentThread != null) { - MaterialTheme.colorScheme.onSurface + if (isLiveActive) { + "gemini-3.8-live" } else { - MaterialTheme.colorScheme.error + currentThread?.llmModel?.modelName + ?: stringResource(R.string.agent_demo_select_model_to_create_thread) + } + val textColor = + when { + isLiveActive -> MaterialTheme.colorScheme.primary + currentThread != null -> MaterialTheme.colorScheme.onSurface + else -> MaterialTheme.colorScheme.error } Row( @@ -663,7 +683,7 @@ fun ModelDropdown( .fillMaxHeight() .menuAnchor( ExposedDropdownMenuAnchorType.PrimaryEditable, - enabled = true, + enabled = !isLiveActive, ), verticalAlignment = Alignment.CenterVertically, ) { @@ -681,7 +701,9 @@ fun ModelDropdown( overflow = TextOverflow.Ellipsis, ) } - Icon(imageVector = Icons.Default.ArrowDropDown, contentDescription = null) + if (!isLiveActive) { + Icon(imageVector = Icons.Default.ArrowDropDown, contentDescription = null) + } } } } From 69eef684d3a7bd40e647dec2fef09f581573302a Mon Sep 17 00:00:00 2001 From: Julian Toledo Date: Mon, 26 Oct 2026 22:30:07 +0100 Subject: [PATCH 3/6] refactor(agent): decouple LiveAgentOrchestrator and host Live session in VoiceInteractionService - Extract GetEnabledToolsUseCase and ExecuteToolCallUseCase in domain/appfunction/ so AgentOrchestrator and LiveAgentOrchestrator share a single tool discovery and execution pipeline - Replace GeminiLiveSessionManager with provider-agnostic LiveAgentOrchestrator in domain/ and move Gemini BidiGenerateContent WebSocket/JSON protocol details into GeminiLiveProviderImpl in data/ - Move Gemini Live UI state and session lifecycle out of AgentDemoScreen and AgentDemoViewModel into AgentVoiceInteractionService, AgentVoiceInteractionSession, and LiveVoiceSessionOverlay (with Digital Assistant selection via ACTION_VOICE_INPUT_SETTINGS) - Stabilize GeminiLiveAudioHelper full-duplex audio using VoIP playback/capture with AEC and NoiseSuppressor in MODE_IN_COMMUNICATION, software mic gating during tool calls, and serialized off-main-thread teardown --- agent/app/src/main/AndroidManifest.xml | 36 + .../agent/data/GeminiJsonUtils.kt | 43 ++ .../agent/data/GeminiLiveAudioHelper.kt | 476 +++++++++--- .../agent/data/GeminiLiveProviderImpl.kt | 396 ++++++++++ .../agent/data/GeminiProviderImpl.kt | 42 +- .../agent/data/GeminiToolConverter.kt | 24 + .../agent/data/LlmProviderFactoryImpl.kt | 10 +- .../appfunctions/agent/di/DataModule.kt | 19 +- .../agent/domain/AgentOrchestrator.kt | 692 +----------------- .../agent/domain/GeminiLiveSessionManager.kt | 608 --------------- .../agent/domain/LiveAgentOrchestrator.kt | 514 +++++++++++++ .../agent/domain/LiveAudioSession.kt | 47 ++ .../agent/domain/LiveLlmProvider.kt | 65 ++ .../agent/domain/LiveSessionState.kt | 2 +- .../agent/domain/LlmProviderFactory.kt | 7 +- .../appfunction/ExecuteToolCallUseCase.kt | 456 ++++++++++++ .../appfunction/GetEnabledToolsUseCase.kt | 64 ++ .../ui/screens/agentdemo/AgentDemoScreen.kt | 203 +---- .../screens/agentdemo/AgentDemoViewModel.kt | 31 - .../ui/screens/agentdemo/AgentUiState.kt | 6 - .../agent/voice/AgentRecognitionService.kt | 38 + .../voice/AgentVoiceInteractionService.kt | 167 +++++ .../voice/AgentVoiceInteractionSession.kt | 300 ++++++++ .../AgentVoiceInteractionSessionService.kt | 43 ++ .../agent/voice/LiveVoiceSessionOverlay.kt | 384 ++++++++++ agent/app/src/main/res/values/strings.xml | 12 +- .../res/xml/voice_interaction_service.xml | 23 + .../agent/data/GeminiLiveAudioHelperTest.kt | 330 +++++++++ .../agent/data/GeminiLiveProviderImplTest.kt | 324 ++++++++ .../agent/domain/AgentOrchestratorTest.kt | 25 +- .../domain/GeminiLiveSessionManagerTest.kt | 106 --- .../agent/domain/LiveAgentOrchestratorTest.kt | 435 +++++++++++ .../appfunction/ExecuteToolCallUseCaseTest.kt | 231 ++++++ .../appfunction/GetEnabledToolsUseCaseTest.kt | 119 +++ .../agentdemo/AgentDemoViewModelTest.kt | 151 +--- .../voice/AgentVoiceInteractionServiceTest.kt | 131 ++++ .../voice/AgentVoiceInteractionSessionTest.kt | 256 +++++++ 37 files changed, 4945 insertions(+), 1871 deletions(-) create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiJsonUtils.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt delete mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManager.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAgentOrchestrator.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAudioSession.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ExecuteToolCallUseCase.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetEnabledToolsUseCase.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentRecognitionService.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionService.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSession.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionService.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/voice/LiveVoiceSessionOverlay.kt create mode 100644 agent/app/src/main/res/xml/voice_interaction_service.xml create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelperTest.kt create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt delete mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManagerTest.kt create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/domain/LiveAgentOrchestratorTest.kt create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteToolCallUseCaseTest.kt create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetEnabledToolsUseCaseTest.kt create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionServiceTest.kt create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionTest.kt diff --git a/agent/app/src/main/AndroidManifest.xml b/agent/app/src/main/AndroidManifest.xml index 6d535d75..b79e0dd7 100644 --- a/agent/app/src/main/AndroidManifest.xml +++ b/agent/app/src/main/AndroidManifest.xml @@ -52,6 +52,14 @@ + + + + + + + + + + + + + + + + + + + + + + + null + is JsonPrimitive -> { + if (this.isString) return this.content + return this.content.toBooleanStrictOrNull() + ?: this.content.toLongOrNull() + ?: this.content.toDoubleOrNull() + } + is JsonObject -> this.mapValues { it.value.toPrimitive() } + is JsonArray -> this.map { it.toPrimitive() } + } +} + +/** Converts a nullable JSON arguments object into a Kotlin arguments map. */ +internal fun JsonObject?.toArgumentsMap(): Map { + if (this == null) return emptyMap() + return mapValues { (_, value) -> value.toPrimitive() } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelper.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelper.kt index 1a125f0c..ac828a74 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelper.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelper.kt @@ -17,12 +17,17 @@ package com.example.appfunctions.agent.data import android.annotation.SuppressLint import android.media.AudioAttributes +import android.media.AudioDeviceInfo +import android.media.AudioFocusRequest import android.media.AudioFormat +import android.media.AudioManager import android.media.AudioRecord import android.media.AudioTrack import android.media.MediaRecorder import android.media.audiofx.AcousticEchoCanceler +import android.media.audiofx.NoiseSuppressor import android.util.Log +import com.example.appfunctions.agent.domain.LiveAudioSession import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job @@ -42,66 +47,44 @@ import java.util.concurrent.atomic.AtomicBoolean * Manages low-latency PCM audio recording (16 kHz mono 16-bit) and playback (24 kHz mono 16-bit) * for a Gemini Live bidirectional audio session. */ -class GeminiLiveAudioHelper { +class GeminiLiveAudioHelper( + private val audioManager: AudioManager? = null, +) : LiveAudioSession { private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + private val stateLock = Any() private val isReleased = AtomicBoolean(false) private val isRecordingPaused = AtomicBoolean(false) + private val isFocusLost = AtomicBoolean(false) + private val flushRequested = AtomicBoolean(false) - private var audioRecord: AudioRecord? = null + @Volatile private var audioRecord: AudioRecord? = null private var echoCanceler: AcousticEchoCanceler? = null - private var audioTrack: AudioTrack? = null + private var noiseSuppressor: NoiseSuppressor? = null + + @Volatile private var audioTrack: AudioTrack? = null + private var audioFocusRequest: AudioFocusRequest? = null + private var previousAudioMode: Int? = null + private var didSetCommunicationDevice: Boolean = false private var playbackJob: Job? = null private val audioQueue = ConcurrentLinkedQueue() @SuppressLint("MissingPermission") - fun start() { - val minRecordBuf = - AudioRecord.getMinBufferSize( - INPUT_SAMPLE_RATE, - AudioFormat.CHANNEL_IN_MONO, - AudioFormat.ENCODING_PCM_16BIT, - ).coerceAtLeast(DEFAULT_READ_BUFFER_SIZE) - - val recorder = - AudioRecord( - MediaRecorder.AudioSource.VOICE_COMMUNICATION, - INPUT_SAMPLE_RATE, - AudioFormat.CHANNEL_IN_MONO, - AudioFormat.ENCODING_PCM_16BIT, - minRecordBuf, - ) - if (recorder.state != AudioRecord.STATE_INITIALIZED) { - recorder.release() - throw IllegalStateException("Failed to initialize AudioRecord") - } - - if (AcousticEchoCanceler.isAvailable()) { - runCatching { - echoCanceler = - AcousticEchoCanceler.create(recorder.audioSessionId)?.apply { - enabled = true - } - }.onFailure { e -> - Log.w(TAG, "Failed to enable AcousticEchoCanceler", e) - } - } - - val minTrackBuf = - AudioTrack.getMinBufferSize( - OUTPUT_SAMPLE_RATE, - AudioFormat.CHANNEL_OUT_MONO, - AudioFormat.ENCODING_PCM_16BIT, - ).coerceAtLeast(DEFAULT_READ_BUFFER_SIZE) + internal var audioRecordFactory: (minBufferSize: Int) -> AudioRecord = { minRecordBuf -> + AudioRecord( + MediaRecorder.AudioSource.VOICE_COMMUNICATION, + INPUT_SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + minRecordBuf, + ) + } - val track = + internal var audioTrackFactory: + (attributes: AudioAttributes, minBufferSize: Int) -> AudioTrack = + { playbackAttributes, minTrackBuf -> AudioTrack.Builder() - .setAudioAttributes( - AudioAttributes.Builder() - .setUsage(AudioAttributes.USAGE_MEDIA) - .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) - .build(), - ) + .setAudioAttributes(playbackAttributes) .setAudioFormat( AudioFormat.Builder() .setEncoding(AudioFormat.ENCODING_PCM_16BIT) @@ -112,114 +95,368 @@ class GeminiLiveAudioHelper { .setBufferSizeInBytes(minTrackBuf) .setTransferMode(AudioTrack.MODE_STREAM) .build() + } - track.play() - recorder.startRecording() + private val focusChangeListener = + AudioManager.OnAudioFocusChangeListener { focusChange -> + when (focusChange) { + AudioManager.AUDIOFOCUS_LOSS, + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT, + -> { + isFocusLost.set(true) + stopAudio() + } + AudioManager.AUDIOFOCUS_GAIN -> { + isFocusLost.set(false) + } + } + } - audioRecord = recorder - audioTrack = track + @SuppressLint("MissingPermission") + override fun start() { + if (isReleased.get()) return - playbackJob = - scope.launch { - while (isActive && !isReleased.get()) { - val chunk = audioQueue.poll() - if (chunk != null) { - val currentTrack = audioTrack - if (currentTrack != null && !isReleased.get()) { - currentTrack.write(chunk, 0, chunk.size) - } - } else { - delay(PLAYBACK_POLL_DELAY_MS) + synchronized(stateLock) { + if (isReleased.get() || audioRecord != null) return + + var createdFocusRequest: AudioFocusRequest? = null + var savedPreviousMode: Int? = null + var commDeviceSelected = false + var createdRecorder: AudioRecord? = null + var createdEchoCanceler: AcousticEchoCanceler? = null + var createdNoiseSuppressor: NoiseSuppressor? = null + var createdTrack: AudioTrack? = null + + try { + // Pair USAGE_VOICE_COMMUNICATION playback with VOICE_COMMUNICATION recording under + // MODE_IN_COMMUNICATION so the hardware VoIP DSP AcousticEchoCanceler receives the + // far-end speaker reference (VOIP_PLAYBACK <-> VOIP_RECORD) instead of feeding + // uncancelled media/assistant speaker audio back into the microphone. + val playbackAttributes = + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + + val manager = audioManager + if (manager != null) { + val focusReq = + AudioFocusRequest.Builder( + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT, + ) + .setAudioAttributes(playbackAttributes) + .setOnAudioFocusChangeListener(focusChangeListener) + .build() + val focusResult = manager.requestAudioFocus(focusReq) + createdFocusRequest = focusReq + if (focusResult != AudioManager.AUDIOFOCUS_REQUEST_GRANTED) { + Log.w(TAG, "Audio focus request not granted (result=$focusResult)") + } + + runCatching { + savedPreviousMode = manager.mode + manager.mode = AudioManager.MODE_IN_COMMUNICATION + }.onFailure { e -> + Log.w(TAG, "Failed to enter MODE_IN_COMMUNICATION", e) + } + commDeviceSelected = selectSpeakerphoneIfNoExternalDevice(manager) + } + + val minRecordBuf = + AudioRecord.getMinBufferSize( + INPUT_SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + ).coerceAtLeast(DEFAULT_READ_BUFFER_SIZE) + + val recorder = audioRecordFactory(minRecordBuf) + createdRecorder = recorder + if (recorder.state != AudioRecord.STATE_INITIALIZED) { + throw IllegalStateException("Failed to initialize AudioRecord") + } + + if (AcousticEchoCanceler.isAvailable()) { + runCatching { + createdEchoCanceler = + AcousticEchoCanceler.create(recorder.audioSessionId)?.apply { + enabled = true + } + }.onFailure { e -> + Log.w(TAG, "Failed to enable AcousticEchoCanceler", e) + } + } + + if (NoiseSuppressor.isAvailable()) { + runCatching { + createdNoiseSuppressor = + NoiseSuppressor.create(recorder.audioSessionId)?.apply { + enabled = true + } + }.onFailure { e -> + Log.w(TAG, "Failed to enable NoiseSuppressor", e) } } + + val minTrackBuf = + ( + AudioTrack.getMinBufferSize( + OUTPUT_SAMPLE_RATE, + AudioFormat.CHANNEL_OUT_MONO, + AudioFormat.ENCODING_PCM_16BIT, + ) * 2 + ).coerceAtLeast(DEFAULT_READ_BUFFER_SIZE) + + val track = audioTrackFactory(playbackAttributes, minTrackBuf) + createdTrack = track + + if (isReleased.get()) { + cleanupResources( + focusRequest = createdFocusRequest, + previousMode = savedPreviousMode, + clearCommDevice = commDeviceSelected, + echoCanceler = createdEchoCanceler, + noiseSuppressor = createdNoiseSuppressor, + recorder = createdRecorder, + track = createdTrack, + ) + return + } + + track.play() + recorder.startRecording() + + audioFocusRequest = createdFocusRequest + previousAudioMode = savedPreviousMode + didSetCommunicationDevice = commDeviceSelected + echoCanceler = createdEchoCanceler + noiseSuppressor = createdNoiseSuppressor + audioRecord = recorder + audioTrack = track + + playbackJob = + scope.launch { + while (isActive && !isReleased.get()) { + if (flushRequested.getAndSet(false)) { + val currentTrack = audioTrack + if (currentTrack != null && !isReleased.get()) { + runCatching { + if (currentTrack.playState == AudioTrack.PLAYSTATE_PLAYING) { + currentTrack.pause() + currentTrack.flush() + currentTrack.play() + } + } + } + } + val chunk = audioQueue.poll() + if (chunk != null) { + if (flushRequested.get()) { + continue + } + val currentTrack = audioTrack + if (currentTrack != null && !isReleased.get()) { + runCatching { + currentTrack.write(chunk, 0, chunk.size) + } + } + } else { + delay(PLAYBACK_POLL_DELAY_MS) + } + } + } + } catch (t: Throwable) { + cleanupResources( + focusRequest = createdFocusRequest, + previousMode = savedPreviousMode, + clearCommDevice = commDeviceSelected, + echoCanceler = createdEchoCanceler, + noiseSuppressor = createdNoiseSuppressor, + recorder = createdRecorder, + track = createdTrack, + ) + throw t } + } } - /** Emits PCM 16-bit 16 kHz mono byte chunks recorded from the microphone. */ - fun listenToRecording(): Flow = + /** + * Emits PCM 16-bit 16 kHz mono byte chunks recorded from the microphone. + * + * Keeps [AudioRecord] continuously recording across tool calls so `audioserver` does not + * broadcast system-wide port updates on pause/resume, while draining the HAL ring buffer and + * gating emission in software when paused or when audio focus is temporarily lost. + */ + override fun listenToRecording(): Flow = flow { val buffer = ByteArray(DEFAULT_READ_BUFFER_SIZE) while (currentCoroutineContext().isActive && !isReleased.get()) { - if (isRecordingPaused.get()) { - delay(RECORD_PAUSE_DELAY_MS) - continue - } val recorder = audioRecord ?: break - val bytesRead = recorder.read(buffer, 0, buffer.size) - if (bytesRead > 0 && !isRecordingPaused.get()) { - emit(buffer.copyOf(bytesRead)) + val bytesRead = + runCatching { + recorder.read(buffer, 0, buffer.size) + }.getOrElse { e -> + if (!isReleased.get()) { + Log.w(TAG, "AudioRecord.read failed", e) + } + -1 + } + if (bytesRead > 0) { + if (!isRecordingPaused.get() && !isFocusLost.get() && !isReleased.get()) { + emit(buffer.copyOf(bytesRead)) + } } else if (bytesRead < 0) { - Log.w(TAG, "AudioRecord.read returned error code: $bytesRead") + if (!isReleased.get()) { + Log.w(TAG, "AudioRecord.read returned error code: $bytesRead") + } break + } else { + delay(RECORD_IDLE_DELAY_MS) } } }.flowOn(Dispatchers.IO) - fun queueAudio(data: ByteArray) { + override fun queueAudio(pcmBytes: ByteArray) { if (!isReleased.get()) { - audioQueue.add(data) + audioQueue.add(pcmBytes) } } - /** Clears any queued model audio when the user interrupts the model. */ - fun stopAudio() { + /** Clears any queued model audio and flushes the playback track when interrupted. */ + override fun stopAudio() { audioQueue.clear() + flushRequested.set(true) } - fun pauseRecording() { + /** + * Gates outgoing microphone emissions in software without stopping the underlying + * [AudioRecord] hardware stream. + */ + override fun pauseRecording() { isRecordingPaused.set(true) - runCatching { - val recorder = audioRecord - if (recorder?.recordingState == AudioRecord.RECORDSTATE_RECORDING) { - recorder.stop() - } - } } - fun resumeRecording() { + /** Resumes outgoing microphone emissions after [pauseRecording]. */ + override fun resumeRecording() { if (!isReleased.get()) { - runCatching { - val recorder = audioRecord - if (recorder?.state == AudioRecord.STATE_INITIALIZED && - recorder.recordingState != AudioRecord.RECORDSTATE_RECORDING - ) { - recorder.startRecording() - } - } isRecordingPaused.set(false) } } - fun release() { + internal fun isPausedForTesting(): Boolean = isRecordingPaused.get() || isFocusLost.get() + + internal fun notifyAudioFocusChangeForTesting(focusChange: Int) { + focusChangeListener.onAudioFocusChange(focusChange) + } + + override fun release() { if (!isReleased.compareAndSet(false, true)) return audioQueue.clear() - playbackJob?.cancel() - scope.cancel() + synchronized(stateLock) { + playbackJob?.cancel() + playbackJob = null + scope.cancel() + + val focusReq = audioFocusRequest + audioFocusRequest = null + val prevMode = previousAudioMode + previousAudioMode = null + val clearComm = didSetCommunicationDevice + didSetCommunicationDevice = false + val aec = echoCanceler + echoCanceler = null + val ns = noiseSuppressor + noiseSuppressor = null + val recorder = audioRecord + audioRecord = null + val track = audioTrack + audioTrack = null + + cleanupResources( + focusRequest = focusReq, + previousMode = prevMode, + clearCommDevice = clearComm, + echoCanceler = aec, + noiseSuppressor = ns, + recorder = recorder, + track = track, + ) + } + } + + private fun selectSpeakerphoneIfNoExternalDevice(manager: AudioManager): Boolean = + runCatching { + val activeDevice = manager.communicationDevice + if (activeDevice != null && activeDevice.type in EXTERNAL_COMMUNICATION_DEVICE_TYPES) { + return@runCatching false + } + val availableDevices = manager.availableCommunicationDevices + if (availableDevices.any { it.type in EXTERNAL_COMMUNICATION_DEVICE_TYPES }) { + return@runCatching false + } + val speaker = + availableDevices.firstOrNull { + it.type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER + } ?: return@runCatching false + manager.setCommunicationDevice(speaker) + }.getOrElse { e -> + Log.w(TAG, "Failed to route communication audio to speakerphone", e) + false + } + + private fun cleanupResources( + focusRequest: AudioFocusRequest?, + previousMode: Int?, + clearCommDevice: Boolean, + echoCanceler: AcousticEchoCanceler?, + noiseSuppressor: NoiseSuppressor?, + recorder: AudioRecord?, + track: AudioTrack?, + ) { runCatching { echoCanceler?.release() } - echoCanceler = null + runCatching { noiseSuppressor?.release() } runCatching { - audioRecord?.let { recorder -> - if (recorder.recordingState == AudioRecord.RECORDSTATE_RECORDING) { - recorder.stop() + recorder?.let { rec -> + if (rec.state == AudioRecord.STATE_INITIALIZED && + rec.recordingState == AudioRecord.RECORDSTATE_RECORDING + ) { + rec.stop() } - recorder.release() + rec.release() } } - audioRecord = null runCatching { - audioTrack?.let { track -> - if (track.playState == AudioTrack.PLAYSTATE_PLAYING) { - track.pause() - track.flush() - track.stop() + track?.let { trk -> + if (trk.state == AudioTrack.STATE_INITIALIZED && + trk.playState == AudioTrack.PLAYSTATE_PLAYING + ) { + trk.pause() + trk.flush() + trk.stop() } - track.release() + trk.release() + } + } + + runCatching { + if (clearCommDevice) { + audioManager?.clearCommunicationDevice() + } + } + + runCatching { + if (previousMode != null) { + audioManager?.mode = previousMode + } + } + + runCatching { + if (focusRequest != null) { + audioManager?.abandonAudioFocusRequest(focusRequest) } } - audioTrack = null } companion object { @@ -228,6 +465,25 @@ class GeminiLiveAudioHelper { private const val OUTPUT_SAMPLE_RATE = 24000 private const val DEFAULT_READ_BUFFER_SIZE = 2048 private const val PLAYBACK_POLL_DELAY_MS = 10L - private const val RECORD_PAUSE_DELAY_MS = 25L + private const val RECORD_IDLE_DELAY_MS = 10L + + private val EXTERNAL_COMMUNICATION_DEVICE_TYPES = + setOf( + AudioDeviceInfo.TYPE_WIRED_HEADSET, + AudioDeviceInfo.TYPE_WIRED_HEADPHONES, + AudioDeviceInfo.TYPE_BLUETOOTH_SCO, + AudioDeviceInfo.TYPE_BLUETOOTH_A2DP, + AudioDeviceInfo.TYPE_BLE_HEADSET, + AudioDeviceInfo.TYPE_BLE_SPEAKER, + AudioDeviceInfo.TYPE_BLE_BROADCAST, + AudioDeviceInfo.TYPE_USB_HEADSET, + AudioDeviceInfo.TYPE_USB_DEVICE, + AudioDeviceInfo.TYPE_USB_ACCESSORY, + AudioDeviceInfo.TYPE_HEARING_AID, + AudioDeviceInfo.TYPE_HDMI, + AudioDeviceInfo.TYPE_AUX_LINE, + AudioDeviceInfo.TYPE_LINE_ANALOG, + AudioDeviceInfo.TYPE_LINE_DIGITAL, + ) } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt new file mode 100644 index 00000000..b7b6af4f --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt @@ -0,0 +1,396 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import android.util.Log +import androidx.appfunctions.metadata.AppFunctionMetadata +import com.example.appfunctions.agent.domain.LiveLlmEvent +import com.example.appfunctions.agent.domain.LiveLlmProvider +import com.example.appfunctions.agent.domain.LiveLlmSession +import com.example.appfunctions.agent.domain.LlmResponsePart +import com.example.appfunctions.agent.domain.ToolOutput +import io.ktor.client.HttpClient +import io.ktor.client.plugins.websocket.webSocket +import io.ktor.websocket.CloseReason +import io.ktor.websocket.Frame +import io.ktor.websocket.WebSocketSession +import io.ktor.websocket.close +import io.ktor.websocket.readBytes +import io.ktor.websocket.readText +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.isActive +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.booleanOrNull +import kotlinx.serialization.json.buildJsonArray +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.put +import java.time.LocalDate +import java.util.Base64 +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Gemini implementation of [LiveLlmProvider] over the `BidiGenerateContent` WebSocket API + * (`models/gemini-3.8-live`). + */ +@Singleton +class GeminiLiveProviderImpl + @Inject + constructor( + private val httpClient: HttpClient, + private val toolConverter: GeminiToolConverter, + ) : LiveLlmProvider { + internal var webSocketConnector: + suspend (wsUrl: String, block: suspend WebSocketSession.() -> Unit) -> Unit = + { wsUrl, block -> + httpClient.webSocket(urlString = wsUrl, block = block) + } + + override suspend fun connect( + apiKey: String, + tools: List, + block: suspend (LiveLlmSession) -> Unit, + ) { + val wsUrl = "$GEMINI_LIVE_WS_URL?key=$apiKey" + webSocketConnector(wsUrl) { + val setupMessage = buildSetupMessage(tools) + Log.d(TAG, "Sending Gemini Live setup for model $LIVE_MODEL_NAME") + send(Frame.Text(setupMessage.toString())) + + val firstFrameText = + receiveNextJsonText() + ?: throw IllegalStateException( + "WebSocket closed before setupComplete was received", + ) + val firstJson = Json.parseToJsonElement(firstFrameText).jsonObject + if (!firstJson.containsKey("setupComplete")) { + throw IllegalStateException("Unexpected setup response: $firstFrameText") + } + + val session = createSession(webSocketSession = this, tools = tools) + try { + block(session) + } finally { + runCatching { + close( + CloseReason( + CloseReason.Codes.NORMAL, + "Session ended by client", + ), + ) + } + } + } + } + + private fun createSession( + webSocketSession: WebSocketSession, + tools: List, + ): LiveLlmSession = + object : LiveLlmSession { + override val events: Flow = + flow { + while (currentCoroutineContext().isActive && webSocketSession.isActive) { + val messageText = webSocketSession.receiveNextJsonText() ?: break + val serverJson = + runCatching { + Json.parseToJsonElement(messageText).jsonObject + }.getOrNull() ?: continue + + for (event in parseServerMessage(serverJson, tools)) { + emit(event) + } + } + } + + override suspend fun sendAudio(pcmBytes: ByteArray) { + val realtimeInput = buildRealtimeAudioMessage(pcmBytes) + webSocketSession.send(Frame.Text(realtimeInput.toString())) + } + + override suspend fun sendToolResponses(outputs: List) { + val toolResponseMessage = buildToolResponseMessage(outputs, tools) + Log.d(TAG, "Sending Gemini Live toolResponse: $toolResponseMessage") + webSocketSession.send(Frame.Text(toolResponseMessage.toString())) + } + } + + private suspend fun WebSocketSession.receiveNextJsonText(): String? { + while (isActive) { + val frame = runCatching { incoming.receive() }.getOrNull() ?: return null + when (frame) { + is Frame.Text -> return frame.readText() + is Frame.Binary -> return frame.readBytes().decodeToString() + is Frame.Close -> return null + else -> continue + } + } + return null + } + + internal fun parseServerMessage( + serverJson: JsonObject, + tools: List, + ): List { + val events = mutableListOf() + + val serverContent = serverJson["serverContent"]?.jsonObject + if (serverContent != null) { + val interrupted = + serverContent["interrupted"]?.jsonPrimitive?.booleanOrNull ?: false + if (interrupted) { + events.add(LiveLlmEvent.Interrupted) + } + + val inputTranscription = + serverContent["inputTranscription"] + ?.jsonObject + ?.get("text") + ?.jsonPrimitive + ?.contentOrNull + if (!inputTranscription.isNullOrEmpty()) { + events.add(LiveLlmEvent.InputTranscription(inputTranscription)) + } + + val outputTranscription = + serverContent["outputTranscription"] + ?.jsonObject + ?.get("text") + ?.jsonPrimitive + ?.contentOrNull + if (!outputTranscription.isNullOrEmpty()) { + events.add(LiveLlmEvent.OutputTranscription(outputTranscription)) + } + + val parts = serverContent["modelTurn"]?.jsonObject?.get("parts")?.jsonArray + if (parts != null) { + for (part in parts) { + val inlineData = part.jsonObject["inlineData"]?.jsonObject ?: continue + val mimeType = + inlineData["mimeType"]?.jsonPrimitive?.contentOrNull ?: "" + val base64Data = + inlineData["data"]?.jsonPrimitive?.contentOrNull ?: continue + if (mimeType.startsWith("audio/pcm")) { + val pcmBytes = Base64.getMimeDecoder().decode(base64Data) + events.add(LiveLlmEvent.AudioChunk(pcmBytes)) + } + } + } + + val turnComplete = + serverContent["turnComplete"]?.jsonPrimitive?.booleanOrNull ?: false + if (turnComplete) { + events.add(LiveLlmEvent.TurnComplete) + } + } + + val toolCallObj = serverJson["toolCall"]?.jsonObject + if (toolCallObj != null) { + val functionCalls = + toolCallObj["functionCalls"]?.jsonArray ?: JsonArray(emptyList()) + val parsedCalls = + functionCalls.mapNotNull { callElement -> + val callObj = callElement.jsonObject + val callId = callObj["id"]?.jsonPrimitive?.contentOrNull ?: "" + val name = + callObj["name"]?.jsonPrimitive?.contentOrNull ?: return@mapNotNull null + val matchingTool = toolConverter.findToolByName(name, tools) + val argumentsMap = callObj["args"]?.jsonObject.toArgumentsMap() + + LlmResponsePart.ToolCall( + packageName = matchingTool?.packageName ?: "", + functionId = matchingTool?.id ?: name, + arguments = argumentsMap, + callId = callId, + ) + } + if (parsedCalls.isNotEmpty()) { + events.add(LiveLlmEvent.ToolCallRequest(parsedCalls)) + } + } + + return events + } + + internal fun buildSetupMessage(tools: List): JsonObject { + val functionDeclarations = + toolConverter + .selectDistinctTools(tools) + .mapNotNull { tool -> + try { + toolConverter.convertForLiveApi(tool) + } catch (e: IllegalArgumentException) { + Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e) + null + } + } + + return buildJsonObject { + put( + "setup", + buildJsonObject { + put("model", LIVE_MODEL_NAME) + put( + "generationConfig", + buildJsonObject { + put( + "responseModalities", + buildJsonArray { add(JsonPrimitive("AUDIO")) }, + ) + put( + "speechConfig", + buildJsonObject { + put( + "voiceConfig", + buildJsonObject { + put( + "prebuiltVoiceConfig", + buildJsonObject { + put("voiceName", DEFAULT_VOICE_NAME) + }, + ) + }, + ) + }, + ) + }, + ) + put( + "systemInstruction", + buildJsonObject { + put( + "parts", + buildJsonArray { + add( + buildJsonObject { + put("text", getSystemInstruction()) + }, + ) + }, + ) + }, + ) + if (functionDeclarations.isNotEmpty()) { + put( + "tools", + buildJsonArray { + add( + buildJsonObject { + put( + "functionDeclarations", + JsonArray(functionDeclarations), + ) + }, + ) + }, + ) + } + put("inputAudioTranscription", buildJsonObject {}) + put("outputAudioTranscription", buildJsonObject {}) + }, + ) + } + } + + internal fun buildRealtimeAudioMessage(pcmBytes: ByteArray): JsonObject { + val base64Audio = Base64.getEncoder().encodeToString(pcmBytes) + return buildJsonObject { + put( + "realtimeInput", + buildJsonObject { + put( + "audio", + buildJsonObject { + put("mimeType", INPUT_AUDIO_MIME_TYPE) + put("data", base64Audio) + }, + ) + }, + ) + } + } + + internal fun buildToolResponseMessage( + toolOutputs: List, + tools: List, + ): JsonObject { + return buildJsonObject { + put( + "toolResponse", + buildJsonObject { + put( + "functionResponses", + buildJsonArray { + for (output in toolOutputs) { + val mappedName = + toolConverter.resolveToolName(output.functionId, tools) + val parsedResult = + runCatching { + Json.parseToJsonElement(output.result) + }.getOrElse { + JsonPrimitive(output.result) + } + add( + buildJsonObject { + if (output.callId.isNotEmpty()) { + put("id", output.callId) + } + put("name", mappedName) + put( + "response", + buildJsonObject { + put("result", parsedResult) + }, + ) + }, + ) + } + }, + ) + }, + ) + } + } + + private fun getSystemInstruction(): String { + val currentDate = LocalDate.now().toString() + return """ + You are an AI voice assistant running on Android using Gemini Live. Today's date is $currentDate. + Always reply to the user using concise, friendly, natural spoken language. + You have access to Android AppFunctions and internal device tools. Use them whenever the user asks to perform an action or look up information on their device. + When a user asks you to generate an image, call the generateImage tool and then confirm to the user in a natural sentence. + When a tool returns status "confirmation_required", let the user know that a confirmation button is on their screen and ask them to tap it to proceed. + """.trimIndent() + } + + companion object { + private const val TAG = "GeminiLiveProvider" + const val LIVE_MODEL_NAME = "models/gemini-3.8-live" + private const val DEFAULT_VOICE_NAME = "CHARON" + private const val INPUT_AUDIO_MIME_TYPE = "audio/pcm;rate=16000" + private const val GEMINI_LIVE_WS_URL = + "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent" + } + } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt index 30bb74fe..097b30ac 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt @@ -31,9 +31,8 @@ import io.ktor.http.ContentType import io.ktor.http.contentType import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArray -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.buildJsonArray import kotlinx.serialization.json.buildJsonObject import kotlinx.serialization.json.contentOrNull import kotlinx.serialization.json.jsonArray @@ -60,9 +59,8 @@ class GeminiProviderImpl serviceTier: ServiceTier, ): LlmResponse { val convertedTools = - tools - .sortedByDescending { it.id.startsWith(it.packageName) } - .distinctBy { toolConverter.getToolName(it) } + toolConverter + .selectDistinctTools(tools) .mapNotNull { tool -> try { buildJsonObject { @@ -89,15 +87,10 @@ class GeminiProviderImpl when (input) { is LlmInput.ToolResponse -> { val inputElement = - kotlinx.serialization.json.buildJsonArray { + buildJsonArray { input.outputs.forEach { output -> - val matchingTool = tools.find { it.id == output.functionId } val mappedName = - if (matchingTool != null) { - toolConverter.getToolName(matchingTool) - } else { - output.functionId - } + toolConverter.resolveToolName(output.functionId, tools) add( buildJsonObject { @@ -119,7 +112,7 @@ class GeminiProviderImpl } if (convertedTools.isNotEmpty()) { - put(KEY_TOOLS, kotlinx.serialization.json.JsonArray(convertedTools)) + put(KEY_TOOLS, JsonArray(convertedTools)) } if (previousInteractionId != null) { @@ -200,19 +193,13 @@ class GeminiProviderImpl stepObj[KEY_NAME]?.jsonPrimitive?.content ?: return LlmResponse.Error("Called function without a name") val matchingTool = - tools.find { tool -> toolConverter.getToolName(tool) == name } + toolConverter.findToolByName(name, tools) ?: return LlmResponse.Error("Called unknown function: $name") val packageName = matchingTool.packageName val functionId = matchingTool.id - val args = stepObj[KEY_ARGUMENTS]?.jsonObject - val argumentsMap = mutableMapOf() - if (args != null) { - for ((key, value) in args) { - argumentsMap[key] = value.toPrimitive() - } - } + val argumentsMap = stepObj[KEY_ARGUMENTS]?.jsonObject.toArgumentsMap() val callId = stepObj["id"]?.jsonPrimitive?.content ?: return LlmResponse.Error( @@ -238,19 +225,6 @@ class GeminiProviderImpl ) } - private fun JsonElement.toPrimitive(): Any? { - return when (this) { - is JsonPrimitive -> { - if (this.isString) return this.content - return this.content.toBooleanStrictOrNull() - ?: this.content.toLongOrNull() - ?: this.content.toDoubleOrNull() - } - is JsonObject -> this.mapValues { it.value.toPrimitive() } - is JsonArray -> this.map { it.toPrimitive() } - } - } - private fun getSystemInstruction(): String { val currentDate = LocalDate.now().toString() return """ diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt index e195b146..3abbf9ee 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt @@ -83,6 +83,30 @@ class GeminiToolConverter return getGeminiFunctionName(tool) } + /** + * Deduplicates [tools] by their converted Gemini function name, prioritizing functions + * whose ID starts with their package name. + */ + fun selectDistinctTools(tools: List): List = + tools + .sortedByDescending { it.id.startsWith(it.packageName) } + .distinctBy { getToolName(it) } + + /** Finds the [AppFunctionMetadata] in [tools] matching a converted Gemini function [name]. */ + fun findToolByName( + name: String, + tools: List, + ): AppFunctionMetadata? = tools.find { getToolName(it) == name } + + /** + * Resolves the converted Gemini function name for [functionId] from [tools], falling back + * to [functionId] if no matching tool metadata is present. + */ + fun resolveToolName( + functionId: String, + tools: List, + ): String = tools.find { it.id == functionId }?.let(::getToolName) ?: functionId + private fun convertToolToGeminiSchema(tool: AppFunctionMetadata): JsonObject { val combinedName = getGeminiFunctionName(tool) return buildJsonObject { diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt index 0475b8c0..8d543c37 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt @@ -15,21 +15,29 @@ */ package com.example.appfunctions.agent.data +import com.example.appfunctions.agent.domain.LiveLlmProvider import com.example.appfunctions.agent.domain.LlmProvider import com.example.appfunctions.agent.domain.LlmProviderFactory import javax.inject.Inject import javax.inject.Singleton -/** Implementation of LlmProviderFactory. */ +/** Implementation of [LlmProviderFactory]. */ @Singleton class LlmProviderFactoryImpl @Inject constructor( private val geminiProvider: GeminiProviderImpl, + private val geminiLiveProvider: GeminiLiveProviderImpl, ) : LlmProviderFactory { override fun getProvider(name: LlmProviderName): LlmProvider { return when (name) { LlmProviderName.GEMINI -> geminiProvider } } + + override fun getLiveProvider(name: LlmProviderName): LiveLlmProvider { + return when (name) { + LlmProviderName.GEMINI -> geminiLiveProvider + } + } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt index 347257d8..31c0c6ad 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt @@ -16,18 +16,23 @@ package com.example.appfunctions.agent.di import android.content.Context +import android.media.AudioManager import androidx.datastore.core.DataStore import androidx.datastore.preferences.core.Preferences import androidx.datastore.preferences.preferencesDataStore import androidx.room.Room import com.example.appfunctions.agent.data.ChatRepository import com.example.appfunctions.agent.data.DataStoreSettingsRepository +import com.example.appfunctions.agent.data.GeminiLiveAudioHelper import com.example.appfunctions.agent.data.InMemoryPendingIntentRepository +import com.example.appfunctions.agent.data.LlmProviderFactoryImpl import com.example.appfunctions.agent.data.PendingIntentRepository import com.example.appfunctions.agent.data.RoomChatRepository import com.example.appfunctions.agent.data.SettingsRepository import com.example.appfunctions.agent.data.db.AppDatabase import com.example.appfunctions.agent.data.db.dao.ChatDao +import com.example.appfunctions.agent.domain.LiveAudioSessionFactory +import com.example.appfunctions.agent.domain.LlmProviderFactory import dagger.Binds import dagger.Module import dagger.Provides @@ -63,9 +68,7 @@ abstract class DataModule { @Binds @Singleton - abstract fun bindLlmProviderFactory( - impl: com.example.appfunctions.agent.data.LlmProviderFactoryImpl, - ): com.example.appfunctions.agent.domain.LlmProviderFactory + abstract fun bindLlmProviderFactory(impl: LlmProviderFactoryImpl): LlmProviderFactory companion object { @Provides @@ -113,5 +116,15 @@ abstract class DataModule { install(WebSockets) } } + + @Provides + @Singleton + fun provideLiveAudioSessionFactory( + @ApplicationContext context: Context, + ): LiveAudioSessionFactory { + return LiveAudioSessionFactory { + GeminiLiveAudioHelper(context.getSystemService(AudioManager::class.java)) + } + } } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index 0b3b7a8d..e14d6b72 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -16,18 +16,8 @@ package com.example.appfunctions.agent.domain import android.app.PendingIntent -import android.content.Context -import android.content.Intent -import android.net.Uri import android.util.Log -import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata -import androidx.appfunctions.metadata.AppFunctionDataTypeMetadata import androidx.appfunctions.metadata.AppFunctionMetadata -import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata -import androidx.appfunctions.metadata.AppFunctionParameterMetadata -import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata -import androidx.core.content.FileProvider -import com.example.appfunctions.agent.data.AgentInternalTools import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.SettingsRepository import com.example.appfunctions.agent.data.db.entities.MessageAttachment @@ -35,12 +25,9 @@ import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity -import com.example.appfunctions.agent.domain.appfunction.AppFunctionExceptionFormatter -import com.example.appfunctions.agent.domain.appfunction.ConvertInputToAppFunctionDataUseCase -import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult -import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionUseCase -import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionStatesUseCase -import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase +import com.example.appfunctions.agent.domain.appfunction.ExecuteToolCallUseCase +import com.example.appfunctions.agent.domain.appfunction.GetEnabledToolsUseCase +import com.example.appfunctions.agent.domain.appfunction.ToolCallExecutionResult import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase import com.example.appfunctions.agent.domain.chat.ObservePendingMessagesUseCase import com.example.appfunctions.agent.domain.chat.SendMessageUseCase @@ -49,9 +36,7 @@ import com.example.appfunctions.agent.domain.chat.UpdateMessageUseCase import com.example.appfunctions.agent.domain.chat.UpdateThreadParams import com.example.appfunctions.agent.domain.chat.UpdateThreadUseCase import com.example.appfunctions.agent.domain.pendingintent.SavePendingIntentUseCase -import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharingStarted @@ -60,12 +45,6 @@ import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.stateIn -import kotlinx.coroutines.withContext -import org.json.JSONObject -import java.io.File -import java.net.HttpURLConnection -import java.net.URL -import java.util.UUID import javax.inject.Inject import javax.inject.Singleton @@ -74,7 +53,6 @@ import javax.inject.Singleton class AgentOrchestrator @Inject constructor( - @ApplicationContext private val context: Context, private val manageThreadsUseCase: ManageThreadsUseCase, private val observePendingMessagesUseCase: ObservePendingMessagesUseCase, private val sendMessageUseCase: SendMessageUseCase, @@ -82,27 +60,15 @@ class AgentOrchestrator private val updateThreadUseCase: UpdateThreadUseCase, private val llmProviderFactory: LlmProviderFactory, private val settingsRepository: SettingsRepository, - private val getAppFunctionsUseCase: GetAppFunctionsUseCase, - private val getAppFunctionStatesUseCase: GetAppFunctionStatesUseCase, - private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase, - private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase, + private val getEnabledToolsUseCase: GetEnabledToolsUseCase, + private val executeToolCallUseCase: ExecuteToolCallUseCase, private val savePendingIntentUseCase: SavePendingIntentUseCase, - private val agentInternalTools: AgentInternalTools, ) { private val _status = MutableStateFlow(AgentStatus.Idle) /** The current status of the agent. */ val status: StateFlow = _status.asStateFlow() - /** Returns all currently enabled AppFunction and internal tools. */ - suspend fun getEnabledTools(targetPackageName: String? = null): List { - val disconnectedApps = settingsRepository.disconnectedApps.first() - val allTools = - getAppFunctionsUseCase().first().values.flatten() + - agentInternalTools.getInternalToolsMetadata().map { it.first } - return filterTools(allTools, disconnectedApps, targetPackageName) - } - /** * Observes messages for a specific thread and processes any pending agent responses. This * method suspends and collects messages until cancelled. @@ -141,25 +107,20 @@ class AgentOrchestrator return } - val disconnectedApps = settingsRepository.disconnectedApps.first() - val allTools = - getAppFunctionsUseCase().first().values.flatten() + - agentInternalTools.getInternalToolsMetadata().map { it.first } - - val targetPackageName = message.targetPackageName - val queryText = message.textContent - - val tools = filterTools(allTools, disconnectedApps, targetPackageName) + val tools = getEnabledToolsUseCase(message.targetPackageName) runInteractionLoop( message = message, thread = thread, apiKey = apiKey, tools = tools, - initialInput = queryText, + initialInput = message.textContent, ) _status.value = AgentStatus.Idle + } catch (e: CancellationException) { + _status.value = AgentStatus.Idle + throw e } catch (e: Exception) { Log.e("AgentOrchestrator", "Error processing message", e) completeMessageWithError( @@ -171,31 +132,6 @@ class AgentOrchestrator } } - private suspend fun filterTools( - allTools: List, - disconnectedApps: Set, - targetPackageName: String?, - ): List { - val functionNames = allTools.map { it.name } - val states = getAppFunctionStatesUseCase(functionNames) + agentInternalTools.getInternalToolsMetadata().map { it.second } - val stateMap = states.associateBy { it.functionName } - - return allTools - .filter { metadata -> - val isEnabled = - if (metadata.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) { - true - } else { - stateMap[metadata.name]?.isEnabled ?: false - } - isEnabled && - metadata.packageName !in disconnectedApps && - (targetPackageName == null || metadata.packageName == targetPackageName) - } - .sortedByDescending { metadata -> metadata.id.startsWith(metadata.packageName) } - .distinctBy { metadata -> metadata.id } - } - private suspend fun runInteractionLoop( message: MessageEntity, thread: ThreadEntity, @@ -211,7 +147,7 @@ class AgentOrchestrator var previousInteractionId = thread.latestInteractionId var currentToolOutputs = emptyList() var continueLoop = true - var currentInput = initialInput + val currentInput = initialInput val capturedAttachments = mutableListOf() while (continueLoop) { @@ -302,23 +238,11 @@ class AgentOrchestrator val textContent = textParts.joinToString("\n") { it.text } if (toolCalls.isNotEmpty()) { - when (val toolResult = executeToolCalls(toolCalls, tools, message)) { + when ( + val toolResult = + executeToolCalls(toolCalls, tools, message, capturedAttachments) + ) { is ExecuteToolCallsResult.Success -> { - for (output in toolResult.toolOutputs) { - runCatching { - val json = JSONObject(output.result) - val uri = json.optString("imageUri") - if (uri.isNotBlank()) { - val mimeType = - json.optString("mimeType") - .takeIf { it.isNotBlank() } - ?: "image/png" - capturedAttachments.add( - MessageAttachment(uri = uri, mimeType = mimeType), - ) - } - } - } if (textContent.isNotEmpty()) { sendMessageUseCase( threadId = message.threadId, @@ -383,386 +307,48 @@ class AgentOrchestrator toolCalls: List, tools: List, message: MessageEntity, + capturedAttachments: MutableList, ): ExecuteToolCallsResult { val results = mutableListOf() for (toolCall in toolCalls) { _status.value = AgentStatus.InvokingTool(toolCall.functionId, toolCall.packageName) - if (toolCall.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) { - val executionResult = executeInternalTool(toolCall) - executionResult - .onSuccess { jsonString -> - results.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = jsonString, - ), - ) - } - .onFailure { exception -> - completeMessageWithError( - message.messageId, - message.threadId, - "Internal tool execution failed: ${exception.message}", - ) - return ExecuteToolCallsResult.Error - } - continue - } - - val matchingTool = - tools.find { - it.packageName == toolCall.packageName && it.id == toolCall.functionId + when ( + val result = + executeToolCallUseCase( + toolCall = toolCall, + tools = tools, + threadId = message.threadId, + ) + ) { + is ToolCallExecutionResult.Success -> { + results.add(result.toolOutput) + result.attachment?.let { capturedAttachments.add(it) } } - if (matchingTool == null) { - completeMessageWithError( - message.messageId, - message.threadId, - "Tool not found: ${toolCall.functionId}", - ) - return ExecuteToolCallsResult.Error - } + is ToolCallExecutionResult.PendingIntentRequired -> { + return ExecuteToolCallsResult.PendingIntentAction( + result.pendingIntentId, + result.pendingIntent, + ) + } - val rawConvertedInputs = toolCall.arguments.filterValues { it != null } as Map - val convertedInputs = - try { - withContext(Dispatchers.IO) { - resolveRemoteFileReferencesRecursively( - context = context, - parametersMetadata = matchingTool.parameters, - inputs = rawConvertedInputs, - ) + is ToolCallExecutionResult.Failure -> { + result.exception?.let { exception -> + Log.e("AgentOrchestrator", result.errorMessage, exception) } - } catch (e: Exception) { completeMessageWithError( message.messageId, message.threadId, - "Failed to download remote file reference: ${e.message}", + result.errorMessage, ) return ExecuteToolCallsResult.Error } - - grantContentUriPermissionsRecursively(context, toolCall.packageName, convertedInputs) - - for (value in convertedInputs.values) { - if (value is String && value.startsWith("content://")) { - runCatching { - val uri = Uri.parse(value) - context.grantUriPermission( - toolCall.packageName, - uri, - Intent.FLAG_GRANT_READ_URI_PERMISSION, - ) - } - } - } - - val appFunctionDataResult = - withContext(Dispatchers.Default) { - convertInputToAppFunctionDataUseCase( - parameters = matchingTool.parameters, - components = matchingTool.components, - inputs = convertedInputs, - ) - } - if (appFunctionDataResult.isFailure) { - completeMessageWithError( - message.messageId, - message.threadId, - "Failed to convert arguments for ${toolCall.functionId}\n ${appFunctionDataResult.exceptionOrNull()}", - ) - return ExecuteToolCallsResult.Error - } - - val executionResult = - executeAppFunctionUseCase( - function = matchingTool, - parameters = appFunctionDataResult.getOrThrow(), - threadId = message.threadId, - ) - - when (executionResult) { - is ExecuteAppFunctionResult.Data -> { - results.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = executionResult.formattedJson, - ), - ) - } - - is ExecuteAppFunctionResult.PendingIntentAction -> { - val pendingIntentId = java.util.UUID.randomUUID().toString() - return ExecuteToolCallsResult.PendingIntentAction( - pendingIntentId, - executionResult.pendingIntent, - ) - } - - is ExecuteAppFunctionResult.Error -> { - val exception = executionResult.exception - if (exception is CancellationException) { - throw exception - } - val appFunctionException = - AppFunctionExceptionFormatter.getAppFunctionException(exception) - if (appFunctionException != null) { - results.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = - AppFunctionExceptionFormatter.format( - appFunctionException, - toolCall.functionId, - ), - ), - ) - } else { - throw IllegalStateException( - "Tool execution failed for ${toolCall.functionId}: ${exception.message}", - exception, - ) - } - } } } return ExecuteToolCallsResult.Success(results) } - /** - * Executes tool calls initiated by a Gemini Live voice session for [threadId], updating - * [status], saving any generated attachments or [PendingIntent] confirmation actions to the - * chat thread, and returning a [ToolOutput] for every call. - */ - suspend fun executeLiveToolCalls( - threadId: String, - toolCalls: List, - tools: List, - ): List { - val outputs = mutableListOf() - val capturedAttachments = mutableListOf() - try { - for (toolCall in toolCalls) { - _status.value = - AgentStatus.InvokingTool(toolCall.functionId, toolCall.packageName) - - if (toolCall.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) { - val executionResult = executeInternalTool(toolCall) - executionResult - .onSuccess { jsonString -> - runCatching { - val json = JSONObject(jsonString) - val uri = json.optString("imageUri") - if (uri.isNotBlank()) { - val mimeType = - json.optString("mimeType").takeIf { it.isNotBlank() } - ?: "image/png" - capturedAttachments.add( - MessageAttachment(uri = uri, mimeType = mimeType), - ) - } - } - outputs.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = jsonString, - ), - ) - } - .onFailure { exception -> - outputs.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = - JSONObject().apply { - put( - "error", - "Internal tool execution failed: ${exception.message}", - ) - }.toString(), - ), - ) - } - continue - } - - val matchingTool = - tools.find { - it.packageName == toolCall.packageName && it.id == toolCall.functionId - } - - if (matchingTool == null) { - outputs.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = - JSONObject().apply { - put("error", "Tool not found: ${toolCall.functionId}") - }.toString(), - ), - ) - continue - } - - @Suppress("UNCHECKED_CAST") - val rawConvertedInputs = - toolCall.arguments.filterValues { it != null } as Map - val convertedInputs = - try { - withContext(Dispatchers.IO) { - resolveRemoteFileReferencesRecursively( - context = context, - parametersMetadata = matchingTool.parameters, - inputs = rawConvertedInputs, - ) - } - } catch (e: Exception) { - outputs.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = - JSONObject().apply { - put( - "error", - "Failed to download remote file reference: ${e.message}", - ) - }.toString(), - ), - ) - continue - } - - grantContentUriPermissionsRecursively( - context, - toolCall.packageName, - convertedInputs, - ) - - val appFunctionDataResult = - withContext(Dispatchers.Default) { - convertInputToAppFunctionDataUseCase( - parameters = matchingTool.parameters, - components = matchingTool.components, - inputs = convertedInputs, - ) - } - if (appFunctionDataResult.isFailure) { - val errMsg = appFunctionDataResult.exceptionOrNull()?.message - outputs.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = - JSONObject().apply { - put( - "error", - "Failed to convert arguments for ${toolCall.functionId}: $errMsg", - ) - }.toString(), - ), - ) - continue - } - - val executionResult = - executeAppFunctionUseCase( - function = matchingTool, - parameters = appFunctionDataResult.getOrThrow(), - threadId = threadId, - ) - - when (executionResult) { - is ExecuteAppFunctionResult.Data -> { - outputs.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = executionResult.formattedJson, - ), - ) - } - - is ExecuteAppFunctionResult.PendingIntentAction -> { - val pendingIntentId = UUID.randomUUID().toString() - savePendingIntentUseCase(pendingIntentId, executionResult.pendingIntent) - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = "", - processingStatus = MessageProcessingStatus.PROCESSED, - pendingIntentId = pendingIntentId, - ) - outputs.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = - JSONObject().apply { - put("status", "confirmation_required") - put( - "message", - "A confirmation button has been displayed in the chat. Ask the user to tap it to confirm.", - ) - }.toString(), - ), - ) - } - - is ExecuteAppFunctionResult.Error -> { - val exception = executionResult.exception - if (exception is CancellationException) { - throw exception - } - val appFunctionException = - AppFunctionExceptionFormatter.getAppFunctionException(exception) - val errorJson = - if (appFunctionException != null) { - AppFunctionExceptionFormatter.format( - appFunctionException, - toolCall.functionId, - ) - } else { - JSONObject().apply { - put( - "error", - "Tool execution failed for ${toolCall.functionId}: ${exception.message}", - ) - }.toString() - } - outputs.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = errorJson, - ), - ) - } - } - } - - if (capturedAttachments.isNotEmpty()) { - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = "", - processingStatus = MessageProcessingStatus.PROCESSED, - attachments = capturedAttachments, - ) - } - } finally { - _status.value = AgentStatus.Idle - } - return outputs - } - private suspend fun completeMessageWithError( messageId: String, threadId: String, @@ -779,208 +365,4 @@ class AgentOrchestrator processingStatus = MessageProcessingStatus.FAILED, ) } - - private fun resolveRemoteFileReferencesRecursively( - context: Context, - parametersMetadata: List, - inputs: Map, - ): Map { - val paramMap = parametersMetadata.associateBy { it.name } - return inputs.mapValues { (key, value) -> - val paramMeta = paramMap[key] - resolveValueRecursively(context, paramMeta?.dataType, value, key) - } - } - - private fun resolveValueRecursively( - context: Context, - dataType: AppFunctionDataTypeMetadata?, - value: Any, - paramName: String?, - ): Any { - return when (value) { - is String -> { - val shouldResolve = - isFileReferenceParameter(paramName) || isUriMetadata(dataType) - if (shouldResolve && ( - value.startsWith( - "http://", - ) || value.startsWith("https://") - ) - ) { - downloadRemoteFileToContentUri(context, value) - } else { - value - } - } - is Map<*, *> -> { - value.entries.associate { entry -> - val k = entry.key as String - val propType = - (dataType as? AppFunctionObjectTypeMetadata)?.properties?.get(k) - k to ( - entry.value?.let { - resolveValueRecursively( - context, - propType, - it, - k, - ) - } ?: "" - ) - } - } - is List<*> -> { - val itemType = (dataType as? AppFunctionArrayTypeMetadata)?.itemType - value.mapNotNull { item -> - if (item != null) resolveValueRecursively(context, itemType, item, paramName) else null - } - } - else -> value - } - } - - private fun downloadRemoteFileToContentUri( - context: Context, - urlString: String, - ): String { - val url = URL(urlString) - val connection = - (url.openConnection() as HttpURLConnection).apply { - connectTimeout = 10000 - readTimeout = 15000 - } - try { - connection.connect() - val contentType = connection.contentType ?: "" - val ext = - when { - contentType.contains("png", ignoreCase = true) -> "png" - contentType.contains("jpeg", ignoreCase = true) || - contentType.contains("jpg", ignoreCase = true) -> "jpg" - contentType.contains("gif", ignoreCase = true) -> "gif" - contentType.contains("webp", ignoreCase = true) -> "webp" - urlString.substringAfterLast("/", "").contains(".") -> - urlString.substringAfterLast("/").substringAfterLast(".") - else -> "jpg" - } - val cacheDir = File(context.cacheDir, "file_references").apply { mkdirs() } - val file = File(cacheDir, "generated_${UUID.randomUUID()}.$ext") - connection.inputStream.use { input -> - file.outputStream().use { output -> - input.copyTo(output) - } - } - val contentUri = - FileProvider.getUriForFile( - context, - "${context.packageName}.fileprovider", - file, - ) - return contentUri.toString() - } finally { - connection.disconnect() - } - } - - private fun grantContentUriPermissionsRecursively( - context: Context, - targetPackageName: String, - value: Any?, - ) { - when (value) { - is String -> { - if (value.startsWith("content://")) { - runCatching { - val uri = Uri.parse(value) - context.grantUriPermission( - targetPackageName, - uri, - Intent.FLAG_GRANT_READ_URI_PERMISSION, - ) - } - } - } - is Map<*, *> -> - value.values.forEach { - grantContentUriPermissionsRecursively(context, targetPackageName, it) - } - is List<*> -> - value.forEach { - grantContentUriPermissionsRecursively(context, targetPackageName, it) - } - } - } - - private fun isFileReferenceParameter(parameterName: String?): Boolean { - if (parameterName == null) return false - if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true - return parameterName.endsWith("Uri", ignoreCase = true) || - parameterName.endsWith("Uris", ignoreCase = true) - } - - private fun isUriMetadata(dataType: AppFunctionDataTypeMetadata?): Boolean { - if (dataType == null) return false - if (dataType is AppFunctionObjectTypeMetadata && dataType.qualifiedName == "android.net.Uri") return true - if (dataType is AppFunctionReferenceTypeMetadata && dataType.referenceDataType == "android.net.Uri") return true - return false - } - - private suspend fun executeInternalTool(toolCall: LlmResponsePart.ToolCall): Result { - return runCatching { - when (toolCall.functionId) { - "getCurrentLocation" -> { - val location = agentInternalTools.getCurrentLocation() - if (location != null) { - JSONObject().apply { - put("latitude", location.latitude) - put("longitude", location.longitude) - }.toString() - } else { - "null" - } - } - "geocodeAddress" -> { - val address = - toolCall.arguments["address"] as? String - ?: throw IllegalArgumentException("Missing required parameter 'address'") - val location = agentInternalTools.geocodeAddress(address) - if (location != null) { - JSONObject().apply { - put("latitude", location.latitude) - put("longitude", location.longitude) - }.toString() - } else { - "null" - } - } - "generateImage" -> { - val prompt = - toolCall.arguments["prompt"] as? String - ?: throw IllegalArgumentException("Missing required parameter 'prompt'") - val aspectRatio = toolCall.arguments["aspectRatio"] as? String - val result = agentInternalTools.generateImage(prompt, aspectRatio) - JSONObject().apply { - put("imageUri", result.imageUri) - put("mimeType", result.mimeType) - put("prompt", result.prompt) - }.toString() - } - else -> throw IllegalArgumentException("Unknown internal tool: ${toolCall.functionId}") - } - } - } - - companion object { - private val KNOWN_FILE_REFERENCE_PARAM_NAMES = - setOf( - "wallpaperUri", - "imageUri", - "attachmentUri", - "ringtoneUri", - "profilePictureUri", - "audioUri", - "voiceNoteUri", - ) - } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManager.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManager.kt deleted file mode 100644 index 997dc03f..00000000 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManager.kt +++ /dev/null @@ -1,608 +0,0 @@ -/* - * Copyright 2026 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.example.appfunctions.agent.domain - -import android.util.Base64 -import android.util.Log -import androidx.appfunctions.metadata.AppFunctionMetadata -import com.example.appfunctions.agent.data.GeminiLiveAudioHelper -import com.example.appfunctions.agent.data.GeminiToolConverter -import com.example.appfunctions.agent.data.SettingsRepository -import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus -import com.example.appfunctions.agent.data.db.entities.MessageRole -import com.example.appfunctions.agent.domain.chat.SendMessageUseCase -import io.ktor.client.HttpClient -import io.ktor.client.plugins.websocket.DefaultClientWebSocketSession -import io.ktor.client.plugins.websocket.webSocket -import io.ktor.websocket.CloseReason -import io.ktor.websocket.Frame -import io.ktor.websocket.close -import io.ktor.websocket.readBytes -import io.ktor.websocket.readText -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.flow.MutableStateFlow -import kotlinx.coroutines.flow.StateFlow -import kotlinx.coroutines.flow.asStateFlow -import kotlinx.coroutines.flow.first -import kotlinx.coroutines.isActive -import kotlinx.coroutines.launch -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonArray -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.JsonPrimitive -import kotlinx.serialization.json.booleanOrNull -import kotlinx.serialization.json.buildJsonArray -import kotlinx.serialization.json.buildJsonObject -import kotlinx.serialization.json.contentOrNull -import kotlinx.serialization.json.jsonArray -import kotlinx.serialization.json.jsonObject -import kotlinx.serialization.json.jsonPrimitive -import kotlinx.serialization.json.put -import java.time.LocalDate -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Manages a bidirectional voice session with the Gemini Live API (`models/gemini-3.8-live`), - * bridging realtime audio I/O, transcriptions, and Android AppFunction tool execution. - */ -@Singleton -class GeminiLiveSessionManager - @Inject - constructor( - private val httpClient: HttpClient, - private val toolConverter: GeminiToolConverter, - private val settingsRepository: SettingsRepository, - private val agentOrchestrator: AgentOrchestrator, - private val sendMessageUseCase: SendMessageUseCase, - ) { - private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) - - private val _state = MutableStateFlow(LiveSessionState.Idle) - - /** Current state of the Gemini Live session. */ - val state: StateFlow = _state.asStateFlow() - - private var sessionJob: Job? = null - private var audioHelper: GeminiLiveAudioHelper? = null - - internal var audioHelperFactory: () -> GeminiLiveAudioHelper = { GeminiLiveAudioHelper() } - - /** Toggles the Gemini Live voice session for [threadId]. */ - fun toggleSession( - threadId: String, - targetPackageName: String? = null, - ) { - when (_state.value) { - is LiveSessionState.Connecting, - is LiveSessionState.Listening, - -> stopSession() - is LiveSessionState.Idle, - is LiveSessionState.Error, - -> startSession(threadId, targetPackageName) - } - } - - /** Starts a new Gemini Live voice session for [threadId]. */ - fun startSession( - threadId: String, - targetPackageName: String? = null, - ) { - stopSessionInternal(resetState = false) - _state.value = LiveSessionState.Connecting - - sessionJob = - scope.launch { - val userTranscriptBuilder = StringBuilder() - val modelTranscriptBuilder = StringBuilder() - try { - val apiKey = - settingsRepository.geminiApiKey.first()?.takeIf { it.isNotBlank() } - if (apiKey == null) { - val errorMsg = "API key is missing for GEMINI" - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = errorMsg, - processingStatus = MessageProcessingStatus.FAILED, - ) - _state.value = LiveSessionState.Error(errorMsg) - return@launch - } - - val tools = agentOrchestrator.getEnabledTools(targetPackageName) - val wsUrl = "$GEMINI_LIVE_WS_URL?key=$apiKey" - - httpClient.webSocket(urlString = wsUrl) { - val setupMessage = buildSetupMessage(tools) - Log.d(TAG, "Sending Gemini Live setup for model $LIVE_MODEL_NAME") - send(Frame.Text(setupMessage.toString())) - - val firstFrameText = receiveNextJsonText() - if (firstFrameText == null) { - throw IllegalStateException( - "WebSocket closed before setupComplete was received", - ) - } - val firstJson = Json.parseToJsonElement(firstFrameText).jsonObject - if (!firstJson.containsKey("setupComplete")) { - throw IllegalStateException( - "Unexpected setup response: $firstFrameText", - ) - } - - val helper = - audioHelperFactory().also { - it.start() - audioHelper = it - } - _state.value = LiveSessionState.Listening - - val sendAudioJob = - launch { - helper.listenToRecording().collect { pcmBytes -> - val base64Audio = - Base64.encodeToString(pcmBytes, Base64.NO_WRAP) - val realtimeInput = - buildJsonObject { - put( - "realtimeInput", - buildJsonObject { - put( - "audio", - buildJsonObject { - put( - "mimeType", - "audio/pcm;rate=16000", - ) - put("data", base64Audio) - }, - ) - }, - ) - } - send(Frame.Text(realtimeInput.toString())) - } - } - - try { - while (isActive) { - val messageText = receiveNextJsonText() ?: break - val serverJson = - runCatching { - Json.parseToJsonElement(messageText).jsonObject - }.getOrNull() ?: continue - - handleServerMessage( - serverJson = serverJson, - threadId = threadId, - tools = tools, - helper = helper, - userTranscriptBuilder = userTranscriptBuilder, - modelTranscriptBuilder = modelTranscriptBuilder, - ) - } - } finally { - sendAudioJob.cancel() - runCatching { - close( - CloseReason( - CloseReason.Codes.NORMAL, - "Session ended by client", - ), - ) - } - } - } - - flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) - _state.value = LiveSessionState.Idle - } catch (e: CancellationException) { - scope.launch { - flushTranscripts( - threadId, - userTranscriptBuilder, - modelTranscriptBuilder, - ) - } - throw e - } catch (e: Exception) { - Log.e(TAG, "Gemini Live session error", e) - flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) - val errorText = - "Gemini Live error: ${e.message ?: "Connection failed"}" - runCatching { - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = errorText, - processingStatus = MessageProcessingStatus.FAILED, - ) - } - _state.value = LiveSessionState.Error(errorText) - } finally { - audioHelper?.release() - audioHelper = null - } - } - } - - /** Stops any active Gemini Live voice session. */ - fun stopSession() { - stopSessionInternal(resetState = true) - } - - private fun stopSessionInternal(resetState: Boolean) { - sessionJob?.cancel() - sessionJob = null - audioHelper?.release() - audioHelper = null - if (resetState) { - _state.value = LiveSessionState.Idle - } - } - - private suspend fun DefaultClientWebSocketSession.receiveNextJsonText(): String? { - while (isActive) { - val frame = runCatching { incoming.receive() }.getOrNull() ?: return null - when (frame) { - is Frame.Text -> return frame.readText() - is Frame.Binary -> return frame.readBytes().decodeToString() - is Frame.Close -> return null - else -> continue - } - } - return null - } - - private suspend fun DefaultClientWebSocketSession.handleServerMessage( - serverJson: JsonObject, - threadId: String, - tools: List, - helper: GeminiLiveAudioHelper, - userTranscriptBuilder: StringBuilder, - modelTranscriptBuilder: StringBuilder, - ) { - val serverContent = serverJson["serverContent"]?.jsonObject - if (serverContent != null) { - val interrupted = - serverContent["interrupted"]?.jsonPrimitive?.booleanOrNull ?: false - if (interrupted) { - helper.stopAudio() - flushTranscript(threadId, MessageRole.ASSISTANT, modelTranscriptBuilder) - } - - val inputTranscription = - serverContent["inputTranscription"] - ?.jsonObject - ?.get("text") - ?.jsonPrimitive - ?.contentOrNull - if (!inputTranscription.isNullOrEmpty()) { - userTranscriptBuilder.append(inputTranscription) - } - - val outputTranscription = - serverContent["outputTranscription"] - ?.jsonObject - ?.get("text") - ?.jsonPrimitive - ?.contentOrNull - if (!outputTranscription.isNullOrEmpty()) { - // Flush user transcript before model response transcript starts - flushTranscript(threadId, MessageRole.USER, userTranscriptBuilder) - modelTranscriptBuilder.append(outputTranscription) - } - - val parts = - serverContent["modelTurn"]?.jsonObject?.get("parts")?.jsonArray - if (parts != null) { - for (part in parts) { - val inlineData = part.jsonObject["inlineData"]?.jsonObject ?: continue - val mimeType = - inlineData["mimeType"]?.jsonPrimitive?.contentOrNull ?: "" - val base64Data = - inlineData["data"]?.jsonPrimitive?.contentOrNull ?: continue - if (mimeType.startsWith("audio/pcm")) { - val pcmBytes = Base64.decode(base64Data, Base64.DEFAULT) - helper.queueAudio(pcmBytes) - } - } - } - - val turnComplete = - serverContent["turnComplete"]?.jsonPrimitive?.booleanOrNull ?: false - if (turnComplete) { - flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) - } - } - - val toolCallObj = serverJson["toolCall"]?.jsonObject - if (toolCallObj != null) { - flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) - val functionCalls = toolCallObj["functionCalls"]?.jsonArray ?: JsonArray(emptyList()) - if (functionCalls.isNotEmpty()) { - helper.pauseRecording() - try { - val parsedCalls = mutableListOf() - val unknownCalls = mutableListOf>() - - for (callElement in functionCalls) { - val callObj = callElement.jsonObject - val callId = callObj["id"]?.jsonPrimitive?.contentOrNull ?: "" - val name = callObj["name"]?.jsonPrimitive?.contentOrNull ?: continue - val matchingTool = - tools.find { tool -> toolConverter.getToolName(tool) == name } - - if (matchingTool == null) { - unknownCalls.add(callId to name) - continue - } - - val argsObj = callObj["args"]?.jsonObject - val argumentsMap = mutableMapOf() - if (argsObj != null) { - for ((key, value) in argsObj) { - argumentsMap[key] = value.toPrimitive() - } - } - parsedCalls.add( - LlmResponsePart.ToolCall( - packageName = matchingTool.packageName, - functionId = matchingTool.id, - arguments = argumentsMap, - callId = callId, - ), - ) - } - - val toolOutputs = - if (parsedCalls.isNotEmpty()) { - agentOrchestrator.executeLiveToolCalls( - threadId = threadId, - toolCalls = parsedCalls, - tools = tools, - ) - } else { - emptyList() - } - - val toolResponseMessage = - buildToolResponseMessage( - toolOutputs = toolOutputs, - unknownCalls = unknownCalls, - tools = tools, - ) - Log.d(TAG, "Sending Gemini Live toolResponse: $toolResponseMessage") - send(Frame.Text(toolResponseMessage.toString())) - } finally { - helper.resumeRecording() - } - } - } - } - - private suspend fun flushTranscripts( - threadId: String, - userTranscriptBuilder: StringBuilder, - modelTranscriptBuilder: StringBuilder, - ) { - flushTranscript(threadId, MessageRole.USER, userTranscriptBuilder) - flushTranscript(threadId, MessageRole.ASSISTANT, modelTranscriptBuilder) - } - - private suspend fun flushTranscript( - threadId: String, - role: MessageRole, - builder: StringBuilder, - ) { - val text = builder.toString().trim() - builder.clear() - if (text.isNotEmpty()) { - sendMessageUseCase( - threadId = threadId, - role = role, - textContent = text, - processingStatus = MessageProcessingStatus.PROCESSED, - ) - } - } - - internal fun buildSetupMessage(tools: List): JsonObject { - val functionDeclarations = - tools - .sortedByDescending { it.id.startsWith(it.packageName) } - .distinctBy { toolConverter.getToolName(it) } - .mapNotNull { tool -> - try { - toolConverter.convertForLiveApi(tool) - } catch (e: IllegalArgumentException) { - Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e) - null - } - } - - return buildJsonObject { - put( - "setup", - buildJsonObject { - put("model", LIVE_MODEL_NAME) - put( - "generationConfig", - buildJsonObject { - put( - "responseModalities", - buildJsonArray { add(JsonPrimitive("AUDIO")) }, - ) - put( - "speechConfig", - buildJsonObject { - put( - "voiceConfig", - buildJsonObject { - put( - "prebuiltVoiceConfig", - buildJsonObject { - put("voiceName", DEFAULT_VOICE_NAME) - }, - ) - }, - ) - }, - ) - }, - ) - put( - "systemInstruction", - buildJsonObject { - put( - "parts", - buildJsonArray { - add( - buildJsonObject { - put("text", getSystemInstruction()) - }, - ) - }, - ) - }, - ) - if (functionDeclarations.isNotEmpty()) { - put( - "tools", - buildJsonArray { - add( - buildJsonObject { - put( - "functionDeclarations", - JsonArray(functionDeclarations), - ) - }, - ) - }, - ) - } - put("inputAudioTranscription", buildJsonObject {}) - put("outputAudioTranscription", buildJsonObject {}) - }, - ) - } - } - - private fun buildToolResponseMessage( - toolOutputs: List, - unknownCalls: List>, - tools: List, - ): JsonObject { - return buildJsonObject { - put( - "toolResponse", - buildJsonObject { - put( - "functionResponses", - buildJsonArray { - for (output in toolOutputs) { - val matchingTool = tools.find { it.id == output.functionId } - val mappedName = - if (matchingTool != null) { - toolConverter.getToolName(matchingTool) - } else { - output.functionId - } - val parsedResult = - runCatching { - Json.parseToJsonElement(output.result) - }.getOrElse { - JsonPrimitive(output.result) - } - add( - buildJsonObject { - if (output.callId.isNotEmpty()) { - put("id", output.callId) - } - put("name", mappedName) - put( - "response", - buildJsonObject { - put("result", parsedResult) - }, - ) - }, - ) - } - for ((callId, unknownName) in unknownCalls) { - add( - buildJsonObject { - if (callId.isNotEmpty()) { - put("id", callId) - } - put("name", unknownName) - put( - "response", - buildJsonObject { - put( - "error", - "Unknown function: $unknownName", - ) - }, - ) - }, - ) - } - }, - ) - }, - ) - } - } - - private fun JsonElement.toPrimitive(): Any? { - return when (this) { - is JsonPrimitive -> { - if (this.isString) return this.content - return this.content.toBooleanStrictOrNull() - ?: this.content.toLongOrNull() - ?: this.content.toDoubleOrNull() - } - is JsonObject -> this.mapValues { it.value.toPrimitive() } - is JsonArray -> this.map { it.toPrimitive() } - } - } - - private fun getSystemInstruction(): String { - val currentDate = LocalDate.now().toString() - return """ - You are an AI voice assistant running on Android using Gemini Live. Today's date is $currentDate. - Always reply to the user using concise, friendly, natural spoken language. - You have access to Android AppFunctions and internal device tools. Use them whenever the user asks to perform an action or look up information on their device. - When a user asks you to generate an image, call the generateImage tool and then confirm to the user in a natural sentence. - When a tool returns status "confirmation_required", let the user know that a confirmation button is on their screen and ask them to tap it to proceed. - """.trimIndent() - } - - companion object { - private const val TAG = "GeminiLiveSessionMgr" - const val LIVE_MODEL_NAME = "models/gemini-3.8-live" - private const val DEFAULT_VOICE_NAME = "CHARON" - private const val GEMINI_LIVE_WS_URL = - "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent" - } - } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAgentOrchestrator.kt new file mode 100644 index 00000000..d4f2a663 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAgentOrchestrator.kt @@ -0,0 +1,514 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain + +import android.util.Log +import androidx.appfunctions.metadata.AppFunctionMetadata +import com.example.appfunctions.agent.data.LlmProviderName +import com.example.appfunctions.agent.data.SettingsRepository +import com.example.appfunctions.agent.data.db.entities.MessageAttachment +import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus +import com.example.appfunctions.agent.data.db.entities.MessageRole +import com.example.appfunctions.agent.domain.appfunction.ExecuteToolCallUseCase +import com.example.appfunctions.agent.domain.appfunction.GetEnabledToolsUseCase +import com.example.appfunctions.agent.domain.appfunction.ToolCallExecutionResult +import com.example.appfunctions.agent.domain.chat.SendMessageUseCase +import com.example.appfunctions.agent.domain.pendingintent.SavePendingIntentUseCase +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.takeWhile +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Provider-agnostic orchestrator for bidirectional live voice sessions, bridging realtime audio + * capture/playback, transcript persistence, and Android AppFunction tool execution via + * [LiveLlmProvider]. + */ +@Singleton +class LiveAgentOrchestrator + @Inject + constructor( + private val llmProviderFactory: LlmProviderFactory, + private val settingsRepository: SettingsRepository, + private val getEnabledToolsUseCase: GetEnabledToolsUseCase, + private val executeToolCallUseCase: ExecuteToolCallUseCase, + private val savePendingIntentUseCase: SavePendingIntentUseCase, + private val sendMessageUseCase: SendMessageUseCase, + private val audioSessionFactory: LiveAudioSessionFactory, + ) { + private val scope = CoroutineScope(SupervisorJob()) + private val stateLock = Any() + private val sessionMutex = Mutex() + + private val _state = MutableStateFlow(LiveSessionState.Idle) + + /** Current connection/listening state of the live voice session. */ + val state: StateFlow = _state.asStateFlow() + + private val _status = MutableStateFlow(AgentStatus.Idle) + + /** Current tool invocation status emitted during active live voice sessions. */ + val status: StateFlow = _status.asStateFlow() + + private var sessionJob: Job? = null + private var activeSessionId: Long = 0L + + internal var ioDispatcher: CoroutineDispatcher = Dispatchers.IO + + /** Toggles the live voice session for [threadId]. */ + fun toggleSession( + threadId: String, + targetPackageName: String? = null, + ) { + val shouldStop = + synchronized(stateLock) { + _state.value is LiveSessionState.Connecting || + _state.value is LiveSessionState.Listening + } + if (shouldStop) { + stopSession() + } else { + startSession(threadId, targetPackageName) + } + } + + /** Starts a new live voice session for [threadId]. */ + fun startSession( + threadId: String, + targetPackageName: String? = null, + ) { + val (mySessionId, previousJob) = + synchronized(stateLock) { + val nextId = ++activeSessionId + val prev = sessionJob + prev?.cancel() + _state.value = LiveSessionState.Connecting + _status.value = AgentStatus.Idle + nextId to prev + } + + lateinit var launchedJob: Job + launchedJob = + scope.launch(ioDispatcher) { + try { + // Serialize full audio hardware teardown of any prior session before + // opening a new session so two AudioRecord instances never overlap. + previousJob?.cancelAndJoin() + sessionMutex.withLock { + if (!isActive || !isCurrentSession(mySessionId)) { + return@withLock + } + runSession( + sessionId = mySessionId, + threadId = threadId, + targetPackageName = targetPackageName, + ) + } + } finally { + synchronized(stateLock) { + if (sessionJob === launchedJob) { + sessionJob = null + } + } + } + } + + synchronized(stateLock) { + if (activeSessionId == mySessionId) { + sessionJob = launchedJob + } else { + launchedJob.cancel() + } + } + } + + /** + * Stops any active live voice session without blocking the calling (UI) thread. + * + * Audio hardware teardown runs asynchronously on [ioDispatcher] inside the session + * coroutine's `finally` block under [sessionMutex]. + */ + fun stopSession() { + val jobToCancel = + synchronized(stateLock) { + activeSessionId++ + _state.value = LiveSessionState.Idle + _status.value = AgentStatus.Idle + sessionJob + } + jobToCancel?.cancel() + } + + private fun isCurrentSession(sessionId: Long): Boolean = synchronized(stateLock) { activeSessionId == sessionId } + + private fun updateStateIfCurrent( + sessionId: Long, + newState: LiveSessionState, + ) { + synchronized(stateLock) { + if (activeSessionId == sessionId) { + _state.value = newState + } + } + } + + private fun updateStatusIfCurrent( + sessionId: Long, + newStatus: AgentStatus, + ) { + synchronized(stateLock) { + if (activeSessionId == sessionId) { + _status.value = newStatus + } + } + } + + private suspend fun getApiKey(provider: LlmProviderName): String? { + return when (provider) { + LlmProviderName.GEMINI -> settingsRepository.geminiApiKey.first() + } + } + + private suspend fun runSession( + sessionId: Long, + threadId: String, + targetPackageName: String?, + ) { + val userTranscriptBuilder = StringBuilder() + val modelTranscriptBuilder = StringBuilder() + var localAudioSession: LiveAudioSession? = null + + try { + val providerName = settingsRepository.selectedProvider.first() + val apiKey = getApiKey(providerName)?.takeIf { it.isNotBlank() } + if (!currentCoroutineContext().isActive || !isCurrentSession(sessionId)) { + return + } + if (apiKey == null) { + val errorMsg = "API key is missing for ${providerName.name}" + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = errorMsg, + processingStatus = MessageProcessingStatus.FAILED, + ) + updateStateIfCurrent(sessionId, LiveSessionState.Error(errorMsg)) + return + } + + val tools = getEnabledToolsUseCase(targetPackageName) + currentCoroutineContext().ensureActive() + if (!isCurrentSession(sessionId)) return + + val liveProvider = llmProviderFactory.getLiveProvider(providerName) + + liveProvider.connect(apiKey = apiKey, tools = tools) { session -> + currentCoroutineContext().ensureActive() + if (!isCurrentSession(sessionId)) return@connect + + val audioSession = audioSessionFactory.create() + localAudioSession = audioSession + audioSession.start() + + if (!currentCoroutineContext().isActive || !isCurrentSession(sessionId)) { + return@connect + } + updateStateIfCurrent(sessionId, LiveSessionState.Listening) + + coroutineScope { + val sendAudioJob = + launch { + audioSession.listenToRecording().collect { pcmBytes -> + session.sendAudio(pcmBytes) + } + } + + try { + session.events + .takeWhile { + currentCoroutineContext().isActive && + isCurrentSession(sessionId) + } + .collect { event -> + handleLiveEvent( + event = event, + sessionId = sessionId, + session = session, + threadId = threadId, + tools = tools, + audioSession = audioSession, + userTranscriptBuilder = userTranscriptBuilder, + modelTranscriptBuilder = modelTranscriptBuilder, + ) + } + } finally { + sendAudioJob.cancel() + } + } + } + + if (currentCoroutineContext().isActive && isCurrentSession(sessionId)) { + flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) + updateStateIfCurrent(sessionId, LiveSessionState.Idle) + } else { + flushRemainingTranscriptsAsync( + threadId = threadId, + userTranscriptBuilder = userTranscriptBuilder, + modelTranscriptBuilder = modelTranscriptBuilder, + ) + } + } catch (e: CancellationException) { + flushRemainingTranscriptsAsync( + threadId = threadId, + userTranscriptBuilder = userTranscriptBuilder, + modelTranscriptBuilder = modelTranscriptBuilder, + ) + throw e + } catch (e: Exception) { + if (!currentCoroutineContext().isActive || !isCurrentSession(sessionId)) { + flushRemainingTranscriptsAsync( + threadId = threadId, + userTranscriptBuilder = userTranscriptBuilder, + modelTranscriptBuilder = modelTranscriptBuilder, + ) + return + } + Log.e(TAG, "Live session error", e) + flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) + val errorText = "Live session error: ${e.message ?: "Connection failed"}" + runCatching { + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = errorText, + processingStatus = MessageProcessingStatus.FAILED, + ) + } + updateStateIfCurrent(sessionId, LiveSessionState.Error(errorText)) + } finally { + updateStatusIfCurrent(sessionId, AgentStatus.Idle) + localAudioSession?.release() + localAudioSession = null + } + } + + private suspend fun handleLiveEvent( + event: LiveLlmEvent, + sessionId: Long, + session: LiveLlmSession, + threadId: String, + tools: List, + audioSession: LiveAudioSession, + userTranscriptBuilder: StringBuilder, + modelTranscriptBuilder: StringBuilder, + ) { + when (event) { + LiveLlmEvent.Interrupted -> { + audioSession.stopAudio() + flushTranscript(threadId, MessageRole.ASSISTANT, modelTranscriptBuilder) + } + + is LiveLlmEvent.InputTranscription -> { + userTranscriptBuilder.append(event.text) + } + + is LiveLlmEvent.OutputTranscription -> { + // Flush user transcript before model response transcript starts. + flushTranscript(threadId, MessageRole.USER, userTranscriptBuilder) + modelTranscriptBuilder.append(event.text) + } + + is LiveLlmEvent.AudioChunk -> { + audioSession.queueAudio(event.pcmBytes) + } + + LiveLlmEvent.TurnComplete -> { + flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) + } + + is LiveLlmEvent.ToolCallRequest -> { + flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) + if (event.calls.isNotEmpty()) { + audioSession.pauseRecording() + try { + val toolOutputs = + executeLiveToolCalls( + sessionId = sessionId, + threadId = threadId, + toolCalls = event.calls, + tools = tools, + ) + session.sendToolResponses(toolOutputs) + } finally { + audioSession.resumeRecording() + } + } + } + } + } + + private suspend fun executeLiveToolCalls( + sessionId: Long, + threadId: String, + toolCalls: List, + tools: List, + ): List { + val outputs = mutableListOf() + val capturedAttachments = mutableListOf() + try { + for (toolCall in toolCalls) { + updateStatusIfCurrent( + sessionId, + AgentStatus.InvokingTool(toolCall.functionId, toolCall.packageName), + ) + when ( + val result = + executeToolCallUseCase( + toolCall = toolCall, + tools = tools, + threadId = threadId, + ) + ) { + is ToolCallExecutionResult.Success -> { + outputs.add(result.toolOutput) + result.attachment?.let { capturedAttachments.add(it) } + } + + is ToolCallExecutionResult.PendingIntentRequired -> { + savePendingIntentUseCase(result.pendingIntentId, result.pendingIntent) + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = "", + processingStatus = MessageProcessingStatus.PROCESSED, + pendingIntentId = result.pendingIntentId, + ) + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = CONFIRMATION_REQUIRED_RESULT, + ), + ) + } + + is ToolCallExecutionResult.Failure -> { + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = result.errorMessage, + ), + ) + } + } + } + + if (capturedAttachments.isNotEmpty()) { + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = "", + processingStatus = MessageProcessingStatus.PROCESSED, + attachments = capturedAttachments, + ) + } + } finally { + updateStatusIfCurrent(sessionId, AgentStatus.Idle) + } + return outputs + } + + private fun flushRemainingTranscriptsAsync( + threadId: String, + userTranscriptBuilder: StringBuilder, + modelTranscriptBuilder: StringBuilder, + ) { + val remainingUserText = userTranscriptBuilder.toString().trim() + val remainingModelText = modelTranscriptBuilder.toString().trim() + userTranscriptBuilder.clear() + modelTranscriptBuilder.clear() + if (remainingUserText.isEmpty() && remainingModelText.isEmpty()) return + + scope.launch(ioDispatcher) { + if (remainingUserText.isNotEmpty()) { + sendMessageUseCase( + threadId = threadId, + role = MessageRole.USER, + textContent = remainingUserText, + processingStatus = MessageProcessingStatus.PROCESSED, + ) + } + if (remainingModelText.isNotEmpty()) { + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = remainingModelText, + processingStatus = MessageProcessingStatus.PROCESSED, + ) + } + } + } + + private suspend fun flushTranscripts( + threadId: String, + userTranscriptBuilder: StringBuilder, + modelTranscriptBuilder: StringBuilder, + ) { + flushTranscript(threadId, MessageRole.USER, userTranscriptBuilder) + flushTranscript(threadId, MessageRole.ASSISTANT, modelTranscriptBuilder) + } + + private suspend fun flushTranscript( + threadId: String, + role: MessageRole, + builder: StringBuilder, + ) { + val text = builder.toString().trim() + builder.clear() + if (text.isNotEmpty()) { + sendMessageUseCase( + threadId = threadId, + role = role, + textContent = text, + processingStatus = MessageProcessingStatus.PROCESSED, + ) + } + } + + companion object { + private const val TAG = "LiveAgentOrchestrator" + private const val CONFIRMATION_REQUIRED_RESULT = + """{"status":"confirmation_required","message":"A confirmation button has been displayed in the chat. Ask the user to tap it to confirm."}""" + } + } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAudioSession.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAudioSession.kt new file mode 100644 index 00000000..a6209e12 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAudioSession.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain + +import kotlinx.coroutines.flow.Flow + +/** Abstraction over full-duplex audio recording and playback for a live voice session. */ +interface LiveAudioSession { + /** Acquires audio focus/routing and starts microphone capture and audio playback resources. */ + fun start() + + /** Streams captured PCM microphone audio chunks while recording is active and unpaused. */ + fun listenToRecording(): Flow + + /** Temporarily suppresses microphone emissions (e.g. while executing tools). */ + fun pauseRecording() + + /** Resumes microphone emissions after [pauseRecording]. */ + fun resumeRecording() + + /** Enqueues a PCM audio chunk from the assistant for speaker/headset playback. */ + fun queueAudio(pcmBytes: ByteArray) + + /** Immediately clears queued assistant audio and flushes active playback on interruption. */ + fun stopAudio() + + /** Releases all underlying audio recording, playback, and audio-focus resources. */ + fun release() +} + +/** Factory for creating a new [LiveAudioSession] per voice session. */ +fun interface LiveAudioSessionFactory { + fun create(): LiveAudioSession +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt new file mode 100644 index 00000000..2b1c2025 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt @@ -0,0 +1,65 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain + +import androidx.appfunctions.metadata.AppFunctionMetadata +import kotlinx.coroutines.flow.Flow + +/** Interface for LLM providers supporting bidirectional realtime audio and tool sessions. */ +interface LiveLlmProvider { + /** + * Connects to the live LLM backend, completes session initialization with [tools], and invokes + * [block] with an active [LiveLlmSession] until [block] returns or is cancelled. + */ + suspend fun connect( + apiKey: String, + tools: List, + block: suspend (LiveLlmSession) -> Unit, + ) +} + +/** Represents an active bidirectional voice and tool session with an LLM provider. */ +interface LiveLlmSession { + /** Sequential stream of server-originated events for this live session. */ + val events: Flow + + /** Sends a chunk of raw 16-bit 16kHz mono PCM user microphone audio to the model. */ + suspend fun sendAudio(pcmBytes: ByteArray) + + /** Sends tool execution [outputs] back to the model in response to a [LiveLlmEvent.ToolCallRequest]. */ + suspend fun sendToolResponses(outputs: List) +} + +/** Provider-agnostic events emitted by a [LiveLlmSession]. */ +sealed class LiveLlmEvent { + /** Indicates the user interrupted the model's current audio response. */ + object Interrupted : LiveLlmEvent() + + /** Incremental speech-to-text transcription of the user's audio input. */ + data class InputTranscription(val text: String) : LiveLlmEvent() + + /** Incremental transcription of the model's spoken response. */ + data class OutputTranscription(val text: String) : LiveLlmEvent() + + /** Raw PCM audio chunk generated by the model for playback. */ + data class AudioChunk(val pcmBytes: ByteArray) : LiveLlmEvent() + + /** Indicates the model finished its current conversational turn. */ + object TurnComplete : LiveLlmEvent() + + /** Requests execution of one or more [calls] before continuing the live conversation. */ + data class ToolCallRequest(val calls: List) : LiveLlmEvent() +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveSessionState.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveSessionState.kt index f9fc51c0..a8893355 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveSessionState.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveSessionState.kt @@ -15,7 +15,7 @@ */ package com.example.appfunctions.agent.domain -/** Represents the state of a Gemini Live voice session. */ +/** Represents the state of a live voice session. */ sealed class LiveSessionState { object Idle : LiveSessionState() diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt index 054c287a..c9ae94f8 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt @@ -17,8 +17,11 @@ package com.example.appfunctions.agent.domain import com.example.appfunctions.agent.data.LlmProviderName -/** Factory interface to get the correct LlmProvider. */ +/** Factory interface to get the correct [LlmProvider] or [LiveLlmProvider]. */ interface LlmProviderFactory { - /** Returns the LlmProvider instance for the given name. */ + /** Returns the [LlmProvider] instance for the given [name]. */ fun getProvider(name: LlmProviderName): LlmProvider + + /** Returns the [LiveLlmProvider] instance for the given [name]. */ + fun getLiveProvider(name: LlmProviderName): LiveLlmProvider } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ExecuteToolCallUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ExecuteToolCallUseCase.kt new file mode 100644 index 00000000..2e14a187 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ExecuteToolCallUseCase.kt @@ -0,0 +1,456 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain.appfunction + +import android.app.PendingIntent +import android.content.Context +import android.content.Intent +import android.net.Uri +import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata +import androidx.appfunctions.metadata.AppFunctionDataTypeMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata +import androidx.core.content.FileProvider +import com.example.appfunctions.agent.data.AgentInternalTools +import com.example.appfunctions.agent.data.db.entities.MessageAttachment +import com.example.appfunctions.agent.domain.LlmResponsePart +import com.example.appfunctions.agent.domain.ToolOutput +import dagger.hilt.android.qualifiers.ApplicationContext +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.json.JSONObject +import java.io.File +import java.net.HttpURLConnection +import java.net.URL +import java.util.UUID +import javax.inject.Inject + +/** Result of executing a single [LlmResponsePart.ToolCall]. */ +sealed class ToolCallExecutionResult { + data class Success( + val toolOutput: ToolOutput, + val attachment: MessageAttachment? = null, + ) : ToolCallExecutionResult() + + data class PendingIntentRequired( + val pendingIntentId: String, + val pendingIntent: PendingIntent, + ) : ToolCallExecutionResult() + + data class Failure( + val errorMessage: String, + val exception: Throwable? = null, + ) : ToolCallExecutionResult() +} + +/** + * Shared pipeline that executes a single [LlmResponsePart.ToolCall] against internal agent tools or + * discovered Android AppFunctions, handling remote file reference downloads, recursive content URI + * permission grants, argument conversion, and [AppFunctionExceptionFormatter] formatting. + */ +class ExecuteToolCallUseCase + @Inject + constructor( + @ApplicationContext private val context: Context, + private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase, + private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase, + private val agentInternalTools: AgentInternalTools, + ) { + suspend operator fun invoke( + toolCall: LlmResponsePart.ToolCall, + tools: List, + threadId: String, + ): ToolCallExecutionResult { + if (toolCall.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) { + return executeInternalTool(toolCall) + .fold( + onSuccess = { jsonString -> + ToolCallExecutionResult.Success( + toolOutput = + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = jsonString, + ), + attachment = extractAttachment(jsonString), + ) + }, + onFailure = { exception -> + if (exception is CancellationException) { + throw exception + } + ToolCallExecutionResult.Failure( + errorMessage = + "Internal tool execution failed: ${exception.message}", + exception = exception, + ) + }, + ) + } + + val matchingTool = + tools.find { + it.packageName == toolCall.packageName && it.id == toolCall.functionId + } + ?: return ToolCallExecutionResult.Failure( + errorMessage = "Tool not found: ${toolCall.functionId}", + ) + + val rawConvertedInputs = + buildMap { + for ((key, value) in toolCall.arguments) { + if (value != null) { + put(key, value) + } + } + } + + val convertedInputs = + try { + withContext(Dispatchers.IO) { + resolveRemoteFileReferencesRecursively( + context = context, + parametersMetadata = matchingTool.parameters, + inputs = rawConvertedInputs, + ) + } + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + return ToolCallExecutionResult.Failure( + errorMessage = "Failed to download remote file reference: ${e.message}", + exception = e, + ) + } + + grantContentUriPermissionsRecursively( + context = context, + targetPackageName = toolCall.packageName, + value = convertedInputs, + ) + + val appFunctionDataResult = + withContext(Dispatchers.Default) { + convertInputToAppFunctionDataUseCase( + parameters = matchingTool.parameters, + components = matchingTool.components, + inputs = convertedInputs, + ) + } + if (appFunctionDataResult.isFailure) { + val exception = appFunctionDataResult.exceptionOrNull() + if (exception is CancellationException) { + throw exception + } + return ToolCallExecutionResult.Failure( + errorMessage = + "Failed to convert arguments for ${toolCall.functionId}\n $exception", + exception = exception, + ) + } + + return when ( + val executionResult = + executeAppFunctionUseCase( + function = matchingTool, + parameters = appFunctionDataResult.getOrThrow(), + threadId = threadId, + ) + ) { + is ExecuteAppFunctionResult.Data -> { + ToolCallExecutionResult.Success( + toolOutput = + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = executionResult.formattedJson, + ), + attachment = extractAttachment(executionResult.formattedJson), + ) + } + + is ExecuteAppFunctionResult.PendingIntentAction -> { + ToolCallExecutionResult.PendingIntentRequired( + pendingIntentId = UUID.randomUUID().toString(), + pendingIntent = executionResult.pendingIntent, + ) + } + + is ExecuteAppFunctionResult.Error -> { + val exception = executionResult.exception + if (exception is CancellationException) { + throw exception + } + val appFunctionException = + AppFunctionExceptionFormatter.getAppFunctionException(exception) + if (appFunctionException != null) { + ToolCallExecutionResult.Success( + toolOutput = + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = + AppFunctionExceptionFormatter.format( + appFunctionException, + toolCall.functionId, + ), + ), + ) + } else { + ToolCallExecutionResult.Failure( + errorMessage = + "Tool execution failed for ${toolCall.functionId}: ${exception.message}", + exception = exception, + ) + } + } + } + } + + private fun extractAttachment(jsonString: String): MessageAttachment? { + return runCatching { + val json = JSONObject(jsonString) + val uri = json.optString("imageUri") + if (uri.isNotBlank()) { + val mimeType = + json.optString("mimeType").takeIf { it.isNotBlank() } ?: "image/png" + MessageAttachment(uri = uri, mimeType = mimeType) + } else { + null + } + }.getOrNull() + } + + private fun resolveRemoteFileReferencesRecursively( + context: Context, + parametersMetadata: List, + inputs: Map, + ): Map { + val paramMap = parametersMetadata.associateBy { it.name } + return inputs.mapValues { (key, value) -> + val paramMeta = paramMap[key] + resolveValueRecursively(context, paramMeta?.dataType, value, key) + } + } + + private fun resolveValueRecursively( + context: Context, + dataType: AppFunctionDataTypeMetadata?, + value: Any, + paramName: String?, + ): Any { + return when (value) { + is String -> { + val shouldResolve = + isFileReferenceParameter(paramName) || isUriMetadata(dataType) + if ( + shouldResolve && + (value.startsWith("http://") || value.startsWith("https://")) + ) { + downloadRemoteFileToContentUri(context, value) + } else { + value + } + } + is Map<*, *> -> { + value.entries.associate { entry -> + val k = entry.key as String + val propType = + (dataType as? AppFunctionObjectTypeMetadata)?.properties?.get(k) + k to ( + entry.value?.let { + resolveValueRecursively( + context, + propType, + it, + k, + ) + } ?: "" + ) + } + } + is List<*> -> { + val itemType = (dataType as? AppFunctionArrayTypeMetadata)?.itemType + value.mapNotNull { item -> + if (item != null) { + resolveValueRecursively(context, itemType, item, paramName) + } else { + null + } + } + } + else -> value + } + } + + private fun downloadRemoteFileToContentUri( + context: Context, + urlString: String, + ): String { + val url = URL(urlString) + val connection = + (url.openConnection() as HttpURLConnection).apply { + connectTimeout = 10000 + readTimeout = 15000 + } + try { + connection.connect() + val contentType = connection.contentType ?: "" + val ext = + when { + contentType.contains("png", ignoreCase = true) -> "png" + contentType.contains("jpeg", ignoreCase = true) || + contentType.contains("jpg", ignoreCase = true) -> "jpg" + contentType.contains("gif", ignoreCase = true) -> "gif" + contentType.contains("webp", ignoreCase = true) -> "webp" + urlString.substringAfterLast("/", "").contains(".") -> + urlString.substringAfterLast("/").substringAfterLast(".") + else -> "jpg" + } + val cacheDir = File(context.cacheDir, "file_references").apply { mkdirs() } + val file = File(cacheDir, "generated_${UUID.randomUUID()}.$ext") + connection.inputStream.use { input -> + file.outputStream().use { output -> + input.copyTo(output) + } + } + val contentUri = + FileProvider.getUriForFile( + context, + "${context.packageName}.fileprovider", + file, + ) + return contentUri.toString() + } finally { + connection.disconnect() + } + } + + private fun grantContentUriPermissionsRecursively( + context: Context, + targetPackageName: String, + value: Any?, + ) { + when (value) { + is String -> { + if (value.startsWith("content://")) { + runCatching { + val uri = Uri.parse(value) + context.grantUriPermission( + targetPackageName, + uri, + Intent.FLAG_GRANT_READ_URI_PERMISSION, + ) + } + } + } + is Map<*, *> -> + value.values.forEach { + grantContentUriPermissionsRecursively(context, targetPackageName, it) + } + is List<*> -> + value.forEach { + grantContentUriPermissionsRecursively(context, targetPackageName, it) + } + } + } + + private fun isFileReferenceParameter(parameterName: String?): Boolean { + if (parameterName == null) return false + if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true + return parameterName.endsWith("Uri", ignoreCase = true) || + parameterName.endsWith("Uris", ignoreCase = true) + } + + private fun isUriMetadata(dataType: AppFunctionDataTypeMetadata?): Boolean { + if (dataType == null) return false + if (dataType is AppFunctionObjectTypeMetadata && dataType.qualifiedName == "android.net.Uri") { + return true + } + if (dataType is AppFunctionReferenceTypeMetadata && dataType.referenceDataType == "android.net.Uri") { + return true + } + return false + } + + private suspend fun executeInternalTool(toolCall: LlmResponsePart.ToolCall): Result { + return runCatching { + when (toolCall.functionId) { + "getCurrentLocation" -> { + val location = agentInternalTools.getCurrentLocation() + if (location != null) { + JSONObject().apply { + put("latitude", location.latitude) + put("longitude", location.longitude) + }.toString() + } else { + "null" + } + } + "geocodeAddress" -> { + val address = + toolCall.arguments["address"] as? String + ?: throw IllegalArgumentException( + "Missing required parameter 'address'", + ) + val location = agentInternalTools.geocodeAddress(address) + if (location != null) { + JSONObject().apply { + put("latitude", location.latitude) + put("longitude", location.longitude) + }.toString() + } else { + "null" + } + } + "generateImage" -> { + val prompt = + toolCall.arguments["prompt"] as? String + ?: throw IllegalArgumentException( + "Missing required parameter 'prompt'", + ) + val aspectRatio = toolCall.arguments["aspectRatio"] as? String + val result = agentInternalTools.generateImage(prompt, aspectRatio) + JSONObject().apply { + put("imageUri", result.imageUri) + put("mimeType", result.mimeType) + put("prompt", result.prompt) + }.toString() + } + else -> + throw IllegalArgumentException( + "Unknown internal tool: ${toolCall.functionId}", + ) + } + } + } + + companion object { + private val KNOWN_FILE_REFERENCE_PARAM_NAMES = + setOf( + "wallpaperUri", + "imageUri", + "attachmentUri", + "ringtoneUri", + "profilePictureUri", + "audioUri", + "voiceNoteUri", + ) + } + } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetEnabledToolsUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetEnabledToolsUseCase.kt new file mode 100644 index 00000000..164624c8 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetEnabledToolsUseCase.kt @@ -0,0 +1,64 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain.appfunction + +import androidx.appfunctions.metadata.AppFunctionMetadata +import com.example.appfunctions.agent.data.AgentInternalTools +import com.example.appfunctions.agent.data.SettingsRepository +import kotlinx.coroutines.flow.first +import javax.inject.Inject + +/** + * Use case to discover all currently enabled AppFunction and internal agent tools, filtered by + * connected apps and optional [targetPackageName] scoping. + */ +class GetEnabledToolsUseCase + @Inject + constructor( + private val settingsRepository: SettingsRepository, + private val getAppFunctionsUseCase: GetAppFunctionsUseCase, + private val getAppFunctionStatesUseCase: GetAppFunctionStatesUseCase, + private val agentInternalTools: AgentInternalTools, + ) { + suspend operator fun invoke(targetPackageName: String? = null): List { + val disconnectedApps = settingsRepository.disconnectedApps.first() + val internalTools = agentInternalTools.getInternalToolsMetadata() + val allTools = + getAppFunctionsUseCase().first().values.flatten() + + internalTools.map { it.first } + + val functionNames = allTools.map { it.name } + val states = + getAppFunctionStatesUseCase(functionNames) + + internalTools.map { it.second } + val stateMap = states.associateBy { it.functionName } + + return allTools + .filter { metadata -> + val isEnabled = + if (metadata.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) { + true + } else { + stateMap[metadata.name]?.isEnabled ?: false + } + isEnabled && + metadata.packageName !in disconnectedApps && + (targetPackageName == null || metadata.packageName == targetPackageName) + } + .sortedByDescending { metadata -> metadata.id.startsWith(metadata.packageName) } + .distinctBy { metadata -> metadata.id } + } + } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt index 4ab33cd3..0966db4e 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt @@ -15,10 +15,7 @@ */ package com.example.appfunctions.agent.ui.screens.agentdemo -import android.Manifest import android.content.pm.PackageManager -import androidx.activity.compose.rememberLauncherForActivityResult -import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.animation.AnimatedVisibility import androidx.compose.animation.expandHorizontally import androidx.compose.animation.shrinkHorizontally @@ -51,9 +48,8 @@ import androidx.compose.material.icons.Icons import androidx.compose.material.icons.automirrored.filled.Send import androidx.compose.material.icons.filled.Add import androidx.compose.material.icons.filled.ArrowDropDown +import androidx.compose.material.icons.filled.GraphicEq import androidx.compose.material.icons.filled.Menu -import androidx.compose.material.icons.filled.Mic -import androidx.compose.material.icons.filled.Stop import androidx.compose.material.icons.filled.Warning import androidx.compose.material3.ButtonDefaults import androidx.compose.material3.Card @@ -77,7 +73,6 @@ import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.material3.rememberDrawerState import androidx.compose.runtime.Composable -import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf @@ -122,7 +117,6 @@ import androidx.compose.ui.unit.dp import androidx.compose.ui.window.Popup import androidx.compose.ui.window.PopupPositionProvider import androidx.compose.ui.window.PopupProperties -import androidx.core.content.ContextCompat import androidx.core.graphics.drawable.toBitmap import androidx.hilt.navigation.compose.hiltViewModel import androidx.lifecycle.compose.collectAsStateWithLifecycle @@ -134,9 +128,9 @@ import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentStatus -import com.example.appfunctions.agent.domain.LiveSessionState import com.example.appfunctions.agent.domain.appfunction.AppInfo import com.example.appfunctions.agent.ui.screens.debugging.LazyExposedDropdownMenu +import com.example.appfunctions.agent.voice.rememberLiveVoiceSessionLauncher import com.mikepenz.markdown.m3.Markdown import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.launch @@ -144,14 +138,13 @@ import kotlinx.coroutines.launch @Composable fun AgentDemoScreen(viewModel: AgentDemoViewModel = hiltViewModel()) { val uiState by viewModel.uiState.collectAsStateWithLifecycle() + val launchLiveSession = rememberLiveVoiceSessionLauncher() - DisposableEffect(Unit) { - onDispose { - viewModel.onEvent(AgentUiEvent.OnStopLiveSession) - } - } - - AgentDemoContent(uiState = uiState, onEvent = viewModel::onEvent) + AgentDemoContent( + uiState = uiState, + onEvent = viewModel::onEvent, + onLaunchLiveSession = launchLiveSession, + ) } @OptIn(ExperimentalMaterial3Api::class) @@ -160,6 +153,7 @@ fun AgentDemoContent( uiState: AgentUiState, onEvent: (AgentUiEvent) -> Unit, initialSidePanelVisible: Boolean = false, + onLaunchLiveSession: (threadId: String, targetPackageName: String?) -> Unit = { _, _ -> }, ) { val context = LocalContext.current val packageManager = context.packageManager @@ -189,6 +183,7 @@ fun AgentDemoContent( scope = scope, packageManager = packageManager, initialSidePanelVisible = initialSidePanelVisible, + onLaunchLiveSession = onLaunchLiveSession, ) } } @@ -239,49 +234,12 @@ fun AgentDemoLoadedScreen( scope: CoroutineScope, packageManager: PackageManager, initialSidePanelVisible: Boolean = false, + onLaunchLiveSession: (threadId: String, targetPackageName: String?) -> Unit = { _, _ -> }, ) { var messageText by remember { mutableStateOf(TextFieldValue("")) } var isSidePanelVisible by remember { mutableStateOf(initialSidePanelVisible) } var selectedAppPackageName by remember { mutableStateOf(null) } - val context = LocalContext.current - val audioPermissionLauncher = - rememberLauncherForActivityResult( - contract = ActivityResultContracts.RequestPermission(), - ) { isGranted -> - if (isGranted) { - onEvent(AgentUiEvent.OnToggleLiveSession(selectedAppPackageName)) - } - } - - val toggleLiveSession = { - val isActive = - uiState.liveSessionState is LiveSessionState.Connecting || - uiState.liveSessionState is LiveSessionState.Listening - if (isActive) { - onEvent(AgentUiEvent.OnStopLiveSession) - } else { - val hasRecordAudioPermission = - try { - ContextCompat.checkSelfPermission( - context, - Manifest.permission.RECORD_AUDIO, - ) == PackageManager.PERMISSION_GRANTED - } catch (_: Exception) { - true - } - if (hasRecordAudioPermission) { - onEvent(AgentUiEvent.OnToggleLiveSession(selectedAppPackageName)) - } else { - try { - audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) - } catch (_: IllegalStateException) { - onEvent(AgentUiEvent.OnToggleLiveSession(selectedAppPackageName)) - } - } - } - } - val chipBgColor = MaterialTheme.colorScheme.primaryContainer val chipTextColor = MaterialTheme.colorScheme.onPrimaryContainer val visualTransformation = @@ -303,7 +261,6 @@ fun AgentDemoLoadedScreen( .weight(1f) .padding(horizontal = 8.dp), currentThread = uiState.currentThread, - liveSessionState = uiState.liveSessionState, onModelSelected = { onEvent(AgentUiEvent.OnModelSelected(it)) }, onMenuClick = { if (isWideScreen) { @@ -377,18 +334,6 @@ fun AgentDemoLoadedScreen( } } - if ( - uiState.liveSessionState is LiveSessionState.Connecting || - uiState.liveSessionState is LiveSessionState.Listening - ) { - item { - LiveSessionIndicator( - liveSessionState = uiState.liveSessionState, - onStopClick = { onEvent(AgentUiEvent.OnStopLiveSession) }, - ) - } - } - items( items = uiState.messages.reversed(), key = { message -> message.messageId }, @@ -505,35 +450,23 @@ fun AgentDemoLoadedScreen( focusedBorderColor = Color.Transparent, ), trailingIcon = { - val isLiveActive = - uiState.liveSessionState is LiveSessionState.Connecting || - uiState.liveSessionState is LiveSessionState.Listening Row( verticalAlignment = Alignment.CenterVertically, modifier = Modifier.padding(end = 4.dp), ) { IconButton( - onClick = toggleLiveSession, + onClick = { + onLaunchLiveSession( + uiState.currentThread.threadId, + selectedAppPackageName, + ) + }, ) { Icon( - imageVector = - if (isLiveActive) { - Icons.Default.Stop - } else { - Icons.Default.Mic - }, + imageVector = Icons.Default.GraphicEq, contentDescription = - if (isLiveActive) { - stringResource(R.string.agent_demo_live_stop) - } else { - stringResource(R.string.agent_demo_live_start) - }, - tint = - if (isLiveActive) { - MaterialTheme.colorScheme.error - } else { - MaterialTheme.colorScheme.primary - }, + stringResource(R.string.agent_demo_live_start), + tint = MaterialTheme.colorScheme.primary, ) } IconButton( @@ -621,29 +554,15 @@ fun AgentDemoLoadedScreen( fun ModelDropdown( modifier: Modifier = Modifier, currentThread: ThreadEntity?, - liveSessionState: LiveSessionState = LiveSessionState.Idle, onModelSelected: (LlmModel) -> Unit, onMenuClick: () -> Unit, ) { var expanded by remember { mutableStateOf(false) } - val isLiveActive = - liveSessionState is LiveSessionState.Connecting || - liveSessionState is LiveSessionState.Listening - - LaunchedEffect(isLiveActive) { - if (isLiveActive) { - expanded = false - } - } ExposedDropdownMenuBox( modifier = modifier, - expanded = expanded && !isLiveActive, - onExpandedChange = { - if (!isLiveActive) { - expanded = !expanded - } - }, + expanded = expanded, + onExpandedChange = { expanded = !expanded }, ) { Surface( modifier = Modifier.padding(bottom = 8.dp), @@ -652,17 +571,13 @@ fun ModelDropdown( color = MaterialTheme.colorScheme.surfaceBright, ) { val text = - if (isLiveActive) { - "gemini-3.8-live" - } else { - currentThread?.llmModel?.modelName - ?: stringResource(R.string.agent_demo_select_model_to_create_thread) - } + currentThread?.llmModel?.modelName + ?: stringResource(R.string.agent_demo_select_model_to_create_thread) val textColor = - when { - isLiveActive -> MaterialTheme.colorScheme.primary - currentThread != null -> MaterialTheme.colorScheme.onSurface - else -> MaterialTheme.colorScheme.error + if (currentThread != null) { + MaterialTheme.colorScheme.onSurface + } else { + MaterialTheme.colorScheme.error } Row( @@ -681,10 +596,7 @@ fun ModelDropdown( Modifier .weight(1f) .fillMaxHeight() - .menuAnchor( - ExposedDropdownMenuAnchorType.PrimaryEditable, - enabled = !isLiveActive, - ), + .menuAnchor(ExposedDropdownMenuAnchorType.PrimaryEditable), verticalAlignment = Alignment.CenterVertically, ) { Column(modifier = Modifier.weight(1f)) { @@ -701,9 +613,7 @@ fun ModelDropdown( overflow = TextOverflow.Ellipsis, ) } - if (!isLiveActive) { - Icon(imageVector = Icons.Default.ArrowDropDown, contentDescription = null) - } + Icon(imageVector = Icons.Default.ArrowDropDown, contentDescription = null) } } } @@ -1025,59 +935,6 @@ fun StatusIndicator( } } -@Composable -fun LiveSessionIndicator( - liveSessionState: LiveSessionState, - onStopClick: () -> Unit, -) { - val label = - when (liveSessionState) { - LiveSessionState.Connecting -> stringResource(R.string.agent_demo_live_connecting) - LiveSessionState.Listening -> stringResource(R.string.agent_demo_live_listening) - else -> return - } - - Surface( - modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 8.dp), - shape = MaterialTheme.shapes.large, - color = MaterialTheme.colorScheme.secondaryContainer, - shadowElevation = 2.dp, - ) { - Row( - modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.SpaceBetween, - ) { - Row( - verticalAlignment = Alignment.CenterVertically, - modifier = Modifier.weight(1f), - ) { - CircularProgressIndicator( - modifier = Modifier.size(20.dp), - strokeWidth = 2.dp, - color = MaterialTheme.colorScheme.onSecondaryContainer, - ) - Spacer(modifier = Modifier.width(12.dp)) - Text( - text = label, - style = MaterialTheme.typography.bodyMedium, - color = MaterialTheme.colorScheme.onSecondaryContainer, - ) - } - IconButton(onClick = onStopClick) { - Icon( - imageVector = Icons.Default.Stop, - contentDescription = stringResource(R.string.agent_demo_live_stop), - tint = MaterialTheme.colorScheme.error, - ) - } - } - } -} - @Composable fun ChatHistorySidePanel( threads: List, diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt index 2212ddb7..2f92660d 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt @@ -27,7 +27,6 @@ import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentOrchestrator import com.example.appfunctions.agent.domain.AgentStatus -import com.example.appfunctions.agent.domain.GeminiLiveSessionManager import com.example.appfunctions.agent.domain.appfunction.AppInfo import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase @@ -65,7 +64,6 @@ class AgentDemoViewModel private val consumePendingIntentUseCase: ConsumePendingIntentUseCase, private val getInstalledAppsUseCase: GetInstalledAppsUseCase, private val getAppFunctionsUseCase: GetAppFunctionsUseCase, - private val geminiLiveSessionManager: GeminiLiveSessionManager, ) : ViewModel() { private val _uiState = MutableStateFlow(AgentUiState.Loading) val uiState: StateFlow = _uiState.asStateFlow() @@ -97,15 +95,6 @@ class AgentDemoViewModel } } - viewModelScope.launch { - geminiLiveSessionManager.state.collect { liveState -> - val currentState = _uiState.value - if (currentState is AgentUiState.Loaded) { - _uiState.value = currentState.copy(liveSessionState = liveState) - } - } - } - viewModelScope.launch { val threads = manageThreadsUseCase.getThreads().first() if (threads.isEmpty()) { @@ -151,14 +140,10 @@ class AgentDemoViewModel activePendingActionIds = currentLoadedState?.activePendingActionIds ?: emptySet(), installedApps = apps, - liveSessionState = geminiLiveSessionManager.state.value, ) // Start observing messages for the current thread if not already doing so if (observationJob == null || previousThreadId != currentThread.threadId) { - if (previousThreadId != null && previousThreadId != currentThread.threadId) { - geminiLiveSessionManager.stopSession() - } observationJob?.cancel() observationJob = viewModelScope.launch { @@ -222,25 +207,9 @@ class AgentDemoViewModel launchPendingIntentUseCase(pendingIntent) } } - is AgentUiEvent.OnToggleLiveSession -> { - if (currentState is AgentUiState.Loaded) { - geminiLiveSessionManager.toggleSession( - threadId = currentState.currentThread.threadId, - targetPackageName = event.targetPackageName, - ) - } - } - AgentUiEvent.OnStopLiveSession -> { - geminiLiveSessionManager.stopSession() - } } } - override fun onCleared() { - super.onCleared() - geminiLiveSessionManager.stopSession() - } - private suspend fun createAndSelectThread(llmModel: LlmModel) { val threadId = manageThreadsUseCase.createThread(llmModel) savedStateHandle[MainActivity.ARG_THREAD_ID] = threadId diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt index 04c2ecdc..4798f798 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt @@ -19,7 +19,6 @@ import com.example.appfunctions.agent.data.LlmModel import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentStatus -import com.example.appfunctions.agent.domain.LiveSessionState import com.example.appfunctions.agent.domain.appfunction.AppInfo /** Represents the UI state for the Agent Demo screen. */ @@ -33,7 +32,6 @@ sealed class AgentUiState { val threads: List = emptyList(), val activePendingActionIds: Set = emptySet(), val installedApps: List = emptyList(), - val liveSessionState: LiveSessionState = LiveSessionState.Idle, ) : AgentUiState() } @@ -51,8 +49,4 @@ sealed class AgentUiEvent { data class OnThreadSelected(val threadId: String) : AgentUiEvent() data class OnConfirmAction(val pendingIntentId: String) : AgentUiEvent() - - data class OnToggleLiveSession(val targetPackageName: String? = null) : AgentUiEvent() - - object OnStopLiveSession : AgentUiEvent() } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentRecognitionService.kt b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentRecognitionService.kt new file mode 100644 index 00000000..e1a7e355 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentRecognitionService.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.voice + +import android.content.Intent +import android.speech.RecognitionService +import android.speech.SpeechRecognizer + +/** + * Companion [RecognitionService] required by Android's `VoiceInteractionServiceInfo` manifest + * metadata validation so [AgentVoiceInteractionService] is selectable as the device's Default + * Digital Assistant (`ROLE_ASSISTANT`). + */ +class AgentRecognitionService : RecognitionService() { + override fun onStartListening( + recognizerIntent: Intent?, + listener: Callback?, + ) { + runCatching { listener?.error(SpeechRecognizer.ERROR_CLIENT) } + } + + override fun onCancel(listener: Callback?) = Unit + + override fun onStopListening(listener: Callback?) = Unit +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionService.kt b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionService.kt new file mode 100644 index 00000000..4a23a1d5 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionService.kt @@ -0,0 +1,167 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.voice + +import android.app.Activity +import android.content.ComponentName +import android.content.Context +import android.content.ContextWrapper +import android.content.Intent +import android.os.Bundle +import android.provider.Settings +import android.service.voice.VoiceInteractionService +import android.util.Log +import java.util.concurrent.atomic.AtomicReference + +/** + * Android [VoiceInteractionService] entry point for Gemini Live voice sessions. + * + * Running Live voice sessions inside a [VoiceInteractionService] / [AgentVoiceInteractionSession] + * keeps microphone recording active and renders a system-wide floating overlay even when + * `MainActivity` moves to the background (for example, when an AppFunction launches a confirmation + * `PendingIntent` or the user switches into a target app like `ChatApp`). + */ +class AgentVoiceInteractionService : VoiceInteractionService() { + @Volatile private var isServiceReady: Boolean = false + + override fun onReady() { + super.onReady() + isServiceReady = true + activeInstance = this + Log.d(TAG, "AgentVoiceInteractionService onReady") + } + + override fun onShutdown() { + isServiceReady = false + if (activeInstance === this) { + activeInstance = null + } + super.onShutdown() + } + + override fun onDestroy() { + isServiceReady = false + if (activeInstance === this) { + activeInstance = null + } + super.onDestroy() + } + + companion object { + private const val TAG = "AgentVoiceService" + + const val EXTRA_THREAD_ID = "extra_thread_id" + const val EXTRA_TARGET_PACKAGE_NAME = "extra_target_package_name" + + @Volatile private var activeInstance: AgentVoiceInteractionService? = null + private val pendingSessionArgs = AtomicReference(null) + + internal var isActiveServiceChecker: (Context) -> Boolean = { context -> + isActiveService( + context, + ComponentName(context, AgentVoiceInteractionService::class.java), + ) + } + + /** Returns true if [AgentVoiceInteractionService] is currently the active assistant service. */ + fun isVoiceServiceActive(context: Context): Boolean = isActiveServiceChecker(context) + + /** Builds the session arguments [Bundle] for [threadId] and optional [targetPackageName]. */ + fun createSessionArgs( + threadId: String?, + targetPackageName: String? = null, + ): Bundle = + Bundle().apply { + if (threadId != null) { + putString(EXTRA_THREAD_ID, threadId) + } + if (targetPackageName != null) { + putString(EXTRA_TARGET_PACKAGE_NAME, targetPackageName) + } + } + + /** Clears any queued pending session request. */ + internal fun clearPendingSession() { + pendingSessionArgs.set(null) + } + + internal fun consumePendingSession(): Bundle? = pendingSessionArgs.getAndSet(null) + + internal fun resetForTesting() { + activeInstance = null + pendingSessionArgs.set(null) + isActiveServiceChecker = { context -> + isActiveService( + context, + ComponentName(context, AgentVoiceInteractionService::class.java), + ) + } + } + + /** + * Triggers [showSession] if [AgentVoiceInteractionService] is currently the active and + * bound [VoiceInteractionService]. Returns `false` if the user must select this app in + * Digital Assistant settings. + */ + fun launchLiveSession( + context: Context, + threadId: String, + targetPackageName: String? = null, + ): Boolean { + if (!isVoiceServiceActive(context)) { + Log.d(TAG, "AgentVoiceInteractionService is not the active digital assistant") + return false + } + val args = createSessionArgs(threadId, targetPackageName) + val instance = activeInstance + if (instance != null && instance.isServiceReady) { + clearPendingSession() + Log.d(TAG, "Showing live voice session via bound service") + instance.showSession(args, 0) + return true + } + + pendingSessionArgs.set(args) + val showedViaActivity = + runCatching { context.findActivity()?.showAssist(args) == true } + .getOrDefault(false) + if (!showedViaActivity) { + clearPendingSession() + Log.w(TAG, "Voice service is selected in settings but not currently bound") + return false + } + return true + } + + /** + * Creates an [Intent] opening Android's Digital Assistant picker settings + * ([Settings.ACTION_VOICE_INPUT_SETTINGS]). + * + * Note: `RoleManager.ROLE_ASSISTANT` is marked `requestable="false"` in AOSP + * `PermissionController`, so `RoleManager.createRequestRoleIntent` immediately finishes + * without showing UI; Digital Assistant selection must go through + * [Settings.ACTION_VOICE_INPUT_SETTINGS]. + */ + fun createAssistantSettingsIntent(): Intent = Intent(Settings.ACTION_VOICE_INPUT_SETTINGS) + + private tailrec fun Context.findActivity(): Activity? = + when (this) { + is Activity -> this + is ContextWrapper -> baseContext.findActivity() + else -> null + } + } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSession.kt b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSession.kt new file mode 100644 index 00000000..88a84039 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSession.kt @@ -0,0 +1,300 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.voice + +import android.content.Context +import android.graphics.Rect +import android.os.Bundle +import android.service.voice.VoiceInteractionSession +import android.view.View +import android.view.ViewGroup +import android.view.WindowManager +import androidx.compose.runtime.getValue +import androidx.compose.ui.platform.ComposeView +import androidx.compose.ui.platform.ViewCompositionStrategy +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleOwner +import androidx.lifecycle.LifecycleRegistry +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import androidx.lifecycle.setViewTreeLifecycleOwner +import androidx.savedstate.SavedStateRegistry +import androidx.savedstate.SavedStateRegistryController +import androidx.savedstate.SavedStateRegistryOwner +import androidx.savedstate.setViewTreeSavedStateRegistryOwner +import com.example.appfunctions.agent.data.LlmModel +import com.example.appfunctions.agent.domain.LiveAgentOrchestrator +import com.example.appfunctions.agent.domain.LiveSessionState +import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase +import com.example.appfunctions.agent.ui.theme.AppFunctionsAgentTheme +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch + +/** + * System-wide [VoiceInteractionSession] hosting the floating Live voice session UI and driving + * [LiveAgentOrchestrator]. + * + * Uses [Insets.TOUCHABLE_INSETS_REGION] and non-modal window flags so touches outside the floating + * Live pill pass directly through to underlying apps (such as `MainActivity` confirmation buttons + * or target apps like `ChatApp`). + */ +class AgentVoiceInteractionSession( + context: Context, + private val liveAgentOrchestrator: LiveAgentOrchestrator, + private val manageThreadsUseCase: ManageThreadsUseCase, + mainDispatcher: CoroutineDispatcher = Dispatchers.Main.immediate, +) : VoiceInteractionSession(context), LifecycleOwner, SavedStateRegistryOwner { + private val lifecycleRegistry = LifecycleRegistry(this) + private val savedStateRegistryController = SavedStateRegistryController.create(this) + private val sessionScope = CoroutineScope(SupervisorJob() + mainDispatcher) + + private val overlayBounds = Rect() + + private var currentThreadId: String? = null + private var currentTargetPackageName: String? = null + private var startupJob: Job? = null + private var stateMonitorJob: Job? = null + + override val lifecycle: Lifecycle + get() = lifecycleRegistry + + override val savedStateRegistry: SavedStateRegistry + get() = savedStateRegistryController.savedStateRegistry + + override fun onCreate() { + super.onCreate() + savedStateRegistryController.performRestore(null) + lifecycleRegistry.handleLifecycleEvent(Lifecycle.Event.ON_CREATE) + configurePassThroughWindow() + } + + override fun onCreateContentView(): View { + configurePassThroughWindow() + window?.window?.decorView?.let { decorView -> + decorView.setViewTreeLifecycleOwner(this) + decorView.setViewTreeSavedStateRegistryOwner(this) + } + + return ComposeView(context).apply { + setViewTreeLifecycleOwner(this@AgentVoiceInteractionSession) + setViewTreeSavedStateRegistryOwner(this@AgentVoiceInteractionSession) + setViewCompositionStrategy(ViewCompositionStrategy.DisposeOnViewTreeLifecycleDestroyed) + setContent { + AppFunctionsAgentTheme { + val liveState by + liveAgentOrchestrator.state.collectAsStateWithLifecycle() + val agentStatus by + liveAgentOrchestrator.status.collectAsStateWithLifecycle() + + LiveVoiceSessionOverlay( + liveSessionState = liveState, + agentStatus = agentStatus, + onStopClick = ::stopAndHide, + onRetryClick = ::retrySession, + onOverlayBoundsChanged = ::updateOverlayBounds, + ) + } + } + } + } + + override fun onComputeInsets(outInsets: Insets) { + super.onComputeInsets(outInsets) + outInsets.contentInsets.setEmpty() + outInsets.touchableInsets = Insets.TOUCHABLE_INSETS_REGION + outInsets.touchableRegion.set(overlayBounds) + } + + override fun onShow( + args: Bundle?, + showFlags: Int, + ) { + super.onShow(args, showFlags) + val effectiveArgs = + if (args?.containsKey(AgentVoiceInteractionService.EXTRA_THREAD_ID) == true) { + AgentVoiceInteractionService.clearPendingSession() + args + } else { + AgentVoiceInteractionService.consumePendingSession() ?: args + } + configurePassThroughWindow() + window?.window?.decorView?.let { decorView -> + decorView.setViewTreeLifecycleOwner(this) + decorView.setViewTreeSavedStateRegistryOwner(this) + } + + if (lifecycleRegistry.currentState != Lifecycle.State.DESTROYED) { + lifecycleRegistry.handleLifecycleEvent(Lifecycle.Event.ON_START) + lifecycleRegistry.handleLifecycleEvent(Lifecycle.Event.ON_RESUME) + } + + val requestedThreadId = + effectiveArgs?.getString(AgentVoiceInteractionService.EXTRA_THREAD_ID) + val requestedTargetPkg = + effectiveArgs?.getString(AgentVoiceInteractionService.EXTRA_TARGET_PACKAGE_NAME) + + startOrAttachSession( + requestedThreadId = requestedThreadId, + requestedTargetPackageName = requestedTargetPkg, + ) + } + + override fun onCloseSystemDialogs() { + // Keep the floating Live pill visible when the user presses Home to switch to another app + // (e.g. ChatApp) while a voice conversation is active. + val state = liveAgentOrchestrator.state.value + val isActive = + state is LiveSessionState.Connecting || state is LiveSessionState.Listening + if (!isActive) { + super.onCloseSystemDialogs() + } + } + + override fun onHide() { + stateMonitorJob?.cancel() + stateMonitorJob = null + startupJob?.cancel() + startupJob = null + liveAgentOrchestrator.stopSession() + + if (lifecycleRegistry.currentState.isAtLeast(Lifecycle.State.STARTED)) { + if (lifecycleRegistry.currentState.isAtLeast(Lifecycle.State.RESUMED)) { + lifecycleRegistry.handleLifecycleEvent(Lifecycle.Event.ON_PAUSE) + } + lifecycleRegistry.handleLifecycleEvent(Lifecycle.Event.ON_STOP) + } + super.onHide() + } + + override fun onDestroy() { + stateMonitorJob?.cancel() + stateMonitorJob = null + startupJob?.cancel() + startupJob = null + liveAgentOrchestrator.stopSession() + sessionScope.cancel() + + if (lifecycleRegistry.currentState.isAtLeast(Lifecycle.State.CREATED)) { + lifecycleRegistry.handleLifecycleEvent(Lifecycle.Event.ON_DESTROY) + } + super.onDestroy() + } + + internal fun stopAndHide() { + stateMonitorJob?.cancel() + stateMonitorJob = null + startupJob?.cancel() + startupJob = null + liveAgentOrchestrator.stopSession() + hide() + } + + internal fun retrySession() { + startOrAttachSession( + requestedThreadId = currentThreadId, + requestedTargetPackageName = currentTargetPackageName, + ) + } + + internal fun updateOverlayBounds(bounds: Rect) { + if (overlayBounds != bounds) { + overlayBounds.set(bounds) + window?.window?.decorView?.let { decorView -> + decorView.requestLayout() + decorView.invalidate() + } + } + } + + private fun startOrAttachSession( + requestedThreadId: String?, + requestedTargetPackageName: String?, + ) { + stateMonitorJob?.cancel() + startupJob?.cancel() + + val currentState = liveAgentOrchestrator.state.value + val isCurrentlyRunning = + currentState is LiveSessionState.Connecting || + currentState is LiveSessionState.Listening + val matchesRunningSession = + requestedThreadId == null || + ( + requestedThreadId == currentThreadId && + requestedTargetPackageName == currentTargetPackageName + ) + + if (isCurrentlyRunning && matchesRunningSession) { + monitorSessionState(initiallyActive = true) + return + } + + startupJob = + sessionScope.launch { + val resolvedThreadId = requestedThreadId ?: resolveOrCreateDefaultThreadId() + currentThreadId = resolvedThreadId + currentTargetPackageName = requestedTargetPackageName + liveAgentOrchestrator.startSession( + threadId = resolvedThreadId, + targetPackageName = requestedTargetPackageName, + ) + monitorSessionState(initiallyActive = true) + } + } + + private fun monitorSessionState(initiallyActive: Boolean) { + stateMonitorJob?.cancel() + stateMonitorJob = + sessionScope.launch { + var sawActiveSession = initiallyActive + liveAgentOrchestrator.state.collect { state -> + if ( + state is LiveSessionState.Connecting || + state is LiveSessionState.Listening + ) { + sawActiveSession = true + } else if (sawActiveSession && state is LiveSessionState.Idle) { + hide() + } + } + } + } + + private suspend fun resolveOrCreateDefaultThreadId(): String { + val existingThread = manageThreadsUseCase.getThreads().first().firstOrNull() + return existingThread?.threadId ?: manageThreadsUseCase.createThread(LlmModel.DEFAULT) + } + + private fun configurePassThroughWindow() { + window?.window?.let { win -> + win.clearFlags(WindowManager.LayoutParams.FLAG_DIM_BEHIND) + win.addFlags( + WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL or + WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE, + ) + win.setBackgroundDrawableResource(android.R.color.transparent) + win.setLayout( + ViewGroup.LayoutParams.MATCH_PARENT, + ViewGroup.LayoutParams.MATCH_PARENT, + ) + } + } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionService.kt b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionService.kt new file mode 100644 index 00000000..8500f502 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionService.kt @@ -0,0 +1,43 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.voice + +import android.os.Bundle +import android.service.voice.VoiceInteractionSession +import android.service.voice.VoiceInteractionSessionService +import com.example.appfunctions.agent.domain.LiveAgentOrchestrator +import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase +import dagger.hilt.android.AndroidEntryPoint +import javax.inject.Inject + +/** + * Hilt-injected [VoiceInteractionSessionService] factory that creates [AgentVoiceInteractionSession] + * instances backed by [LiveAgentOrchestrator]. + */ +@AndroidEntryPoint +class AgentVoiceInteractionSessionService : VoiceInteractionSessionService() { + @Inject lateinit var liveAgentOrchestrator: LiveAgentOrchestrator + + @Inject lateinit var manageThreadsUseCase: ManageThreadsUseCase + + override fun onNewSession(args: Bundle?): VoiceInteractionSession { + return AgentVoiceInteractionSession( + context = this, + liveAgentOrchestrator = liveAgentOrchestrator, + manageThreadsUseCase = manageThreadsUseCase, + ) + } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/voice/LiveVoiceSessionOverlay.kt b/agent/app/src/main/java/com/example/appfunctions/agent/voice/LiveVoiceSessionOverlay.kt new file mode 100644 index 00000000..57b3ffa6 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/voice/LiveVoiceSessionOverlay.kt @@ -0,0 +1,384 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.voice + +import android.Manifest +import android.content.Intent +import android.content.pm.PackageManager +import android.graphics.Rect +import android.provider.Settings +import android.widget.Toast +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.animation.core.RepeatMode +import androidx.compose.animation.core.animateFloat +import androidx.compose.animation.core.infiniteRepeatable +import androidx.compose.animation.core.rememberInfiniteTransition +import androidx.compose.animation.core.tween +import androidx.compose.foundation.Image +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.navigationBarsPadding +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Close +import androidx.compose.material.icons.filled.GraphicEq +import androidx.compose.material.icons.filled.Warning +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.FilledIconButton +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButtonDefaults +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.scale +import androidx.compose.ui.graphics.asImageBitmap +import androidx.compose.ui.layout.boundsInWindow +import androidx.compose.ui.layout.onGloballyPositioned +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import androidx.core.content.ContextCompat +import androidx.core.graphics.drawable.toBitmap +import com.example.appfunctions.agent.R +import com.example.appfunctions.agent.domain.AgentStatus +import com.example.appfunctions.agent.domain.LiveSessionState +import kotlin.math.roundToInt + +/** + * Floating system-overlay pill displayed inside [AgentVoiceInteractionSession] while a Live voice + * session is connecting, listening, invoking an AppFunction tool, or reporting an error. + */ +@Composable +fun LiveVoiceSessionOverlay( + liveSessionState: LiveSessionState, + agentStatus: AgentStatus, + onStopClick: () -> Unit, + onRetryClick: () -> Unit, + onOverlayBoundsChanged: (Rect) -> Unit = {}, +) { + val context = LocalContext.current + val packageManager = context.packageManager + + Box( + modifier = Modifier.fillMaxSize(), + contentAlignment = Alignment.BottomCenter, + ) { + Surface( + modifier = + Modifier + .navigationBarsPadding() + .padding(horizontal = 16.dp, vertical = 16.dp) + .fillMaxWidth() + .onGloballyPositioned { coordinates -> + val bounds = coordinates.boundsInWindow() + onOverlayBoundsChanged( + Rect( + bounds.left.roundToInt(), + bounds.top.roundToInt(), + bounds.right.roundToInt(), + bounds.bottom.roundToInt(), + ), + ) + }, + shape = RoundedCornerShape(28.dp), + color = MaterialTheme.colorScheme.surfaceContainerHigh, + tonalElevation = 6.dp, + shadowElevation = 8.dp, + ) { + Row( + modifier = Modifier.padding(horizontal = 16.dp, vertical = 14.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.SpaceBetween, + ) { + Row( + modifier = Modifier.weight(1f), + verticalAlignment = Alignment.CenterVertically, + ) { + LiveStatusLeadingIcon( + liveSessionState = liveSessionState, + agentStatus = agentStatus, + packageManager = packageManager, + ) + Spacer(modifier = Modifier.width(12.dp)) + Column(modifier = Modifier.weight(1f)) { + Text( + text = stringResource(R.string.live_session_title), + style = MaterialTheme.typography.labelMedium, + color = MaterialTheme.colorScheme.primary, + maxLines = 1, + ) + val subtitle = + when { + agentStatus is AgentStatus.InvokingTool -> { + val appLabel = + resolveAppLabel(packageManager, agentStatus.packageName) + stringResource(R.string.live_session_invoking_tool, appLabel) + } + liveSessionState is LiveSessionState.Connecting -> + stringResource(R.string.agent_demo_live_connecting) + liveSessionState is LiveSessionState.Listening -> + stringResource(R.string.agent_demo_live_listening) + liveSessionState is LiveSessionState.Error -> + liveSessionState.message + else -> stringResource(R.string.agent_demo_live_connecting) + } + val subtitleColor = + if (liveSessionState is LiveSessionState.Error) { + MaterialTheme.colorScheme.error + } else { + MaterialTheme.colorScheme.onSurface + } + Text( + text = subtitle, + style = MaterialTheme.typography.bodyMedium, + color = subtitleColor, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } + } + + Row(verticalAlignment = Alignment.CenterVertically) { + if (liveSessionState is LiveSessionState.Error) { + TextButton(onClick = onRetryClick) { + Text(stringResource(R.string.live_session_retry)) + } + } + FilledIconButton( + onClick = onStopClick, + shape = CircleShape, + colors = + IconButtonDefaults.filledIconButtonColors( + containerColor = MaterialTheme.colorScheme.errorContainer, + contentColor = MaterialTheme.colorScheme.onErrorContainer, + ), + ) { + Icon( + imageVector = Icons.Default.Close, + contentDescription = stringResource(R.string.agent_demo_live_stop), + ) + } + } + } + } + } +} + +@Composable +private fun LiveStatusLeadingIcon( + liveSessionState: LiveSessionState, + agentStatus: AgentStatus, + packageManager: PackageManager, +) { + when { + agentStatus is AgentStatus.InvokingTool -> { + val appIcon = + remember(agentStatus.packageName) { + runCatching { + packageManager.getApplicationIcon(agentStatus.packageName) + }.getOrNull() + } + if (appIcon != null) { + Image( + bitmap = appIcon.toBitmap().asImageBitmap(), + contentDescription = null, + modifier = Modifier.size(32.dp), + ) + } else { + CircularProgressIndicator( + modifier = Modifier.size(24.dp), + strokeWidth = 2.5.dp, + ) + } + } + + liveSessionState is LiveSessionState.Listening -> { + val infiniteTransition = rememberInfiniteTransition(label = "live_pulse") + val scale by + infiniteTransition.animateFloat( + initialValue = 0.88f, + targetValue = 1.15f, + animationSpec = + infiniteRepeatable( + animation = tween(durationMillis = 700), + repeatMode = RepeatMode.Reverse, + ), + label = "live_pulse_scale", + ) + Surface( + modifier = Modifier.size(36.dp).scale(scale), + shape = CircleShape, + color = MaterialTheme.colorScheme.primaryContainer, + ) { + Box(contentAlignment = Alignment.Center) { + Icon( + imageVector = Icons.Default.GraphicEq, + contentDescription = null, + tint = MaterialTheme.colorScheme.onPrimaryContainer, + modifier = Modifier.size(20.dp), + ) + } + } + } + + liveSessionState is LiveSessionState.Error -> { + Icon( + imageVector = Icons.Default.Warning, + contentDescription = null, + tint = MaterialTheme.colorScheme.error, + modifier = Modifier.size(28.dp), + ) + } + + else -> { + CircularProgressIndicator( + modifier = Modifier.size(24.dp), + strokeWidth = 2.5.dp, + ) + } + } +} + +private fun resolveAppLabel( + packageManager: PackageManager, + packageName: String, +): String { + if (packageName.isBlank()) return "AppFunction" + return runCatching { + val appInfo = packageManager.getApplicationInfo(packageName, 0) + packageManager.getApplicationLabel(appInfo).toString() + }.getOrDefault(packageName) +} + +/** + * Remembers a launcher callback that verifies `RECORD_AUDIO` permission and starts + * [AgentVoiceInteractionService] (opening Digital Assistant settings via + * [Settings.ACTION_VOICE_INPUT_SETTINGS] if this app is not yet the active + * [android.service.voice.VoiceInteractionService]). + */ +@Composable +fun rememberLiveVoiceSessionLauncher(): (threadId: String, targetPackageName: String?) -> Unit { + val context = LocalContext.current + val selectAssistantPrompt = stringResource(R.string.live_session_select_assistant_prompt) + var pendingThreadId by rememberSaveable { mutableStateOf(null) } + var pendingTargetPackageName by rememberSaveable { mutableStateOf(null) } + + val assistantSettingsLauncher = + rememberLauncherForActivityResult( + contract = ActivityResultContracts.StartActivityForResult(), + ) { + val threadId = pendingThreadId + val targetPackageName = pendingTargetPackageName + pendingThreadId = null + pendingTargetPackageName = null + if (threadId != null && AgentVoiceInteractionService.isVoiceServiceActive(context)) { + AgentVoiceInteractionService.launchLiveSession( + context = context, + threadId = threadId, + targetPackageName = targetPackageName, + ) + } + } + + val startVoiceServiceForRequest: (String, String?) -> Unit = { threadId, targetPackageName -> + val launched = + AgentVoiceInteractionService.launchLiveSession( + context = context, + threadId = threadId, + targetPackageName = targetPackageName, + ) + if (!launched) { + pendingThreadId = threadId + pendingTargetPackageName = targetPackageName + Toast.makeText( + context, + selectAssistantPrompt, + Toast.LENGTH_LONG, + ).show() + runCatching { + assistantSettingsLauncher.launch( + AgentVoiceInteractionService.createAssistantSettingsIntent(), + ) + }.onFailure { + runCatching { + assistantSettingsLauncher.launch( + Intent(Settings.ACTION_MANAGE_DEFAULT_APPS_SETTINGS), + ) + }.onFailure { + pendingThreadId = null + pendingTargetPackageName = null + } + } + } + } + + val audioPermissionLauncher = + rememberLauncherForActivityResult( + contract = ActivityResultContracts.RequestPermission(), + ) { isGranted -> + val threadId = pendingThreadId + val targetPackageName = pendingTargetPackageName + pendingThreadId = null + pendingTargetPackageName = null + if (isGranted && threadId != null) { + startVoiceServiceForRequest(threadId, targetPackageName) + } + } + + return { threadId, targetPackageName -> + val hasRecordAudioPermission = + runCatching { + ContextCompat.checkSelfPermission( + context, + Manifest.permission.RECORD_AUDIO, + ) == PackageManager.PERMISSION_GRANTED + }.getOrDefault(true) + + if (hasRecordAudioPermission) { + startVoiceServiceForRequest(threadId, targetPackageName) + } else { + pendingThreadId = threadId + pendingTargetPackageName = targetPackageName + runCatching { + audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + }.onFailure { + pendingThreadId = null + pendingTargetPackageName = null + startVoiceServiceForRequest(threadId, targetPackageName) + } + } + } +} diff --git a/agent/app/src/main/res/values/strings.xml b/agent/app/src/main/res/values/strings.xml index 243f2eeb..5193e2ed 100644 --- a/agent/app/src/main/res/values/strings.xml +++ b/agent/app/src/main/res/values/strings.xml @@ -110,10 +110,14 @@ Yes, go ahead Done I can perform this action for you. Should I proceed? - Start Gemini Live - Stop Gemini Live - Connecting to Gemini Live (gemini-3.8-live)… - Gemini Live (gemini-3.8-live) is listening… + Start Live Voice Session + End Live Voice Session + Connecting… + Listening… + Gemini Live + Using %1$s… + Retry + Select AppFunction Testing Agent as your Digital Assistant to start Gemini Live Copied value of %1$s Copy value diff --git a/agent/app/src/main/res/xml/voice_interaction_service.xml b/agent/app/src/main/res/xml/voice_interaction_service.xml new file mode 100644 index 00000000..e78173de --- /dev/null +++ b/agent/app/src/main/res/xml/voice_interaction_service.xml @@ -0,0 +1,23 @@ + + + + diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelperTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelperTest.kt new file mode 100644 index 00000000..fb8b97ee --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelperTest.kt @@ -0,0 +1,330 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import android.content.Context +import android.media.AudioAttributes +import android.media.AudioDeviceInfo +import android.media.AudioManager +import android.media.AudioRecord +import android.media.AudioTrack +import androidx.test.core.app.ApplicationProvider +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertSame +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows +import org.robolectric.annotation.Config +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +class GeminiLiveAudioHelperTest { + @Test + fun `start enters communication mode with voice communication focus and release restores state`() { + val context = ApplicationProvider.getApplicationContext() + val audioManager = context.getSystemService(AudioManager::class.java) + audioManager.mode = AudioManager.MODE_NORMAL + val shadowAudioManager = Shadows.shadowOf(audioManager) + + val mockRecorder = createInitializedMockRecorder() + val mockTrack = createInitializedMockTrack() + var capturedAttributes: AudioAttributes? = null + + val helper = + GeminiLiveAudioHelper(audioManager).apply { + audioRecordFactory = { mockRecorder } + audioTrackFactory = { attrs, _ -> + capturedAttributes = attrs + mockTrack + } + } + + helper.start() + + val focusReq = shadowAudioManager.lastAudioFocusRequest?.audioFocusRequest + assertNotNull(focusReq) + assertEquals( + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT, + focusReq?.focusGain, + ) + assertEquals(AudioAttributes.USAGE_VOICE_COMMUNICATION, capturedAttributes?.usage) + assertEquals(AudioAttributes.CONTENT_TYPE_SPEECH, capturedAttributes?.contentType) + assertEquals(AudioManager.MODE_IN_COMMUNICATION, audioManager.mode) + assertNull(shadowAudioManager.lastAbandonedAudioFocusRequest) + + helper.release() + + assertEquals(AudioManager.MODE_NORMAL, audioManager.mode) + assertSame(focusReq, shadowAudioManager.lastAbandonedAudioFocusRequest) + verify(exactly = 1) { mockRecorder.stop() } + verify(exactly = 1) { mockRecorder.release() } + verify(exactly = 1) { mockTrack.release() } + } + + @Test + fun `start routes communication audio to speakerphone when no external headset is connected`() { + val earpiece = + mockk { + every { type } returns AudioDeviceInfo.TYPE_BUILTIN_EARPIECE + } + val speaker = + mockk { + every { type } returns AudioDeviceInfo.TYPE_BUILTIN_SPEAKER + } + val telephony = + mockk { + every { type } returns AudioDeviceInfo.TYPE_TELEPHONY + } + val audioManager = + mockk(relaxed = true) { + every { mode } returns AudioManager.MODE_NORMAL + every { requestAudioFocus(any()) } returns AudioManager.AUDIOFOCUS_REQUEST_GRANTED + every { communicationDevice } returns earpiece + every { availableCommunicationDevices } returns listOf(earpiece, speaker, telephony) + every { setCommunicationDevice(speaker) } returns true + } + + val helper = + GeminiLiveAudioHelper(audioManager).apply { + audioRecordFactory = { createInitializedMockRecorder() } + audioTrackFactory = { _, _ -> createInitializedMockTrack() } + } + + helper.start() + verify(exactly = 1) { audioManager.mode = AudioManager.MODE_IN_COMMUNICATION } + verify(exactly = 1) { audioManager.setCommunicationDevice(speaker) } + + helper.release() + verify(exactly = 1) { audioManager.clearCommunicationDevice() } + verify(exactly = 1) { audioManager.mode = AudioManager.MODE_NORMAL } + } + + @Test + fun `start preserves external headset routing when BLE or wired headset is available`() { + val speaker = + mockk { + every { type } returns AudioDeviceInfo.TYPE_BUILTIN_SPEAKER + } + val bleHeadset = + mockk { + every { type } returns AudioDeviceInfo.TYPE_BLE_HEADSET + } + val audioManager = + mockk(relaxed = true) { + every { mode } returns AudioManager.MODE_NORMAL + every { requestAudioFocus(any()) } returns AudioManager.AUDIOFOCUS_REQUEST_GRANTED + every { communicationDevice } returns null + every { availableCommunicationDevices } returns listOf(speaker, bleHeadset) + } + + val helper = + GeminiLiveAudioHelper(audioManager).apply { + audioRecordFactory = { createInitializedMockRecorder() } + audioTrackFactory = { _, _ -> createInitializedMockTrack() } + } + + helper.start() + verify(exactly = 0) { audioManager.setCommunicationDevice(any()) } + + helper.release() + verify(exactly = 0) { audioManager.clearCommunicationDevice() } + } + + @Test + fun `stopAudio flushes active AudioTrack and discards queued chunks`() { + val flushedLatch = CountDownLatch(1) + val mockTrack = + createInitializedMockTrack().apply { + every { flush() } answers { + flushedLatch.countDown() + } + } + val helper = + GeminiLiveAudioHelper().apply { + audioRecordFactory = { createInitializedMockRecorder() } + audioTrackFactory = { _, _ -> mockTrack } + } + + helper.start() + helper.stopAudio() + + assertTrue(flushedLatch.await(2, TimeUnit.SECONDS)) + verify(atLeast = 1) { mockTrack.pause() } + verify(atLeast = 1) { mockTrack.flush() } + verify(atLeast = 2) { mockTrack.play() } + + helper.release() + } + + @Test + fun `pauseRecording and resumeRecording drain AudioRecord in software without stopping hardware`() = + runBlocking { + val mockRecorder = createInitializedMockRecorder() + val mockTrack = createInitializedMockTrack() + val helper = + GeminiLiveAudioHelper().apply { + audioRecordFactory = { mockRecorder } + audioTrackFactory = { _, _ -> mockTrack } + } + + val readStep = AtomicInteger(0) + every { mockRecorder.read(any(), any(), any()) } answers { + val buf = firstArg() + when (readStep.incrementAndGet()) { + 1 -> { + buf[0] = 10 + buf[1] = 11 + 2 + } + 2 -> { + // Simulate pauseRecording() active during tool call: read() still runs to + // drain the HAL ring buffer, but chunk 2 must be discarded. + helper.pauseRecording() + buf[0] = 20 + buf[1] = 21 + 2 + } + 3 -> { + // Resume before delivering chunk 3 + helper.resumeRecording() + buf[0] = 30 + buf[1] = 31 + 2 + } + else -> { + helper.release() + -1 + } + } + } + + helper.start() + val emittedChunks = helper.listenToRecording().toList() + + assertEquals(2, emittedChunks.size) + assertArrayEquals(byteArrayOf(10, 11), emittedChunks[0]) + assertArrayEquals(byteArrayOf(30, 31), emittedChunks[1]) + + // Hardware AudioRecord was started only once in start(), and stop() was only called once + // during final release() — never during pauseRecording()/resumeRecording(). + verify(exactly = 1) { mockRecorder.startRecording() } + verify(exactly = 1) { mockRecorder.stop() } + } + + @Test + fun `audio focus loss pauses emission independently of tool-call pause state`() { + val helper = GeminiLiveAudioHelper() + + assertFalse(helper.isPausedForTesting()) + + helper.pauseRecording() + assertTrue(helper.isPausedForTesting()) + + // Focus loss followed by focus gain must not clear an active tool-call pause. + helper.notifyAudioFocusChangeForTesting(AudioManager.AUDIOFOCUS_LOSS_TRANSIENT) + assertTrue(helper.isPausedForTesting()) + helper.notifyAudioFocusChangeForTesting(AudioManager.AUDIOFOCUS_GAIN) + assertTrue(helper.isPausedForTesting()) + + helper.resumeRecording() + assertFalse(helper.isPausedForTesting()) + + // Focus loss while unpaused gates emission until AUDIOFOCUS_GAIN arrives. + helper.notifyAudioFocusChangeForTesting(AudioManager.AUDIOFOCUS_LOSS_TRANSIENT) + assertTrue(helper.isPausedForTesting()) + helper.notifyAudioFocusChangeForTesting(AudioManager.AUDIOFOCUS_GAIN) + assertFalse(helper.isPausedForTesting()) + + helper.release() + } + + @Test + fun `concurrent release during start cleans up resources without leaking active AudioRecord`() { + val context = ApplicationProvider.getApplicationContext() + val audioManager = context.getSystemService(AudioManager::class.java) + val shadowAudioManager = Shadows.shadowOf(audioManager) + + val mockRecorder = createInitializedMockRecorder() + val mockTrack = createInitializedMockTrack() + val insideStartLatch = CountDownLatch(1) + val releaseIssuedLatch = CountDownLatch(1) + + val helper = + GeminiLiveAudioHelper(audioManager).apply { + audioRecordFactory = { + insideStartLatch.countDown() + assertTrue(releaseIssuedLatch.await(2, TimeUnit.SECONDS)) + mockRecorder + } + audioTrackFactory = { _, _ -> mockTrack } + } + + val releaseThread = + Thread { + assertTrue(insideStartLatch.await(2, TimeUnit.SECONDS)) + // Trigger release() from a second thread while start() is mid-initialization. + // release() flips isReleased=true immediately and then waits on stateLock. + val releaseCaller = + Thread { + helper.release() + } + releaseCaller.start() + // Spin briefly until releaseCaller flips isReleased (before blocking on stateLock) + Thread.sleep(30) + releaseIssuedLatch.countDown() + releaseCaller.join(2000) + } + releaseThread.start() + + helper.start() + releaseThread.join(2000) + + // Because release() was triggered mid-start(), AudioRecord must never be left recording. + verify(exactly = 0) { mockRecorder.startRecording() } + verify(exactly = 1) { mockRecorder.release() } + verify(exactly = 1) { mockTrack.release() } + val focusReq = shadowAudioManager.lastAudioFocusRequest?.audioFocusRequest + assertNotNull(focusReq) + assertSame(focusReq, shadowAudioManager.lastAbandonedAudioFocusRequest) + } + + private fun createInitializedMockRecorder(): AudioRecord = + mockk(relaxed = true).apply { + every { state } returns AudioRecord.STATE_INITIALIZED + every { recordingState } returns AudioRecord.RECORDSTATE_RECORDING + every { audioSessionId } returns 42 + } + + private fun createInitializedMockTrack(): AudioTrack = + mockk(relaxed = true).apply { + every { state } returns AudioTrack.STATE_INITIALIZED + every { playState } returns AudioTrack.PLAYSTATE_PLAYING + } +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt new file mode 100644 index 00000000..e3ce6077 --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt @@ -0,0 +1,324 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import android.util.Log +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionResponseMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import com.example.appfunctions.agent.domain.LiveLlmEvent +import com.example.appfunctions.agent.domain.ToolOutput +import io.ktor.client.HttpClient +import io.ktor.websocket.Frame +import io.ktor.websocket.WebSocketSession +import io.ktor.websocket.readText +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import io.mockk.mockkStatic +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.test.runTest +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Assert.fail +import org.junit.Before +import org.junit.Test +import java.util.Base64 + +class GeminiLiveProviderImplTest { + private lateinit var provider: GeminiLiveProviderImpl + + @Before + fun setUp() { + mockkStatic(Log::class) + every { Log.d(any(), any()) } returns 0 + every { Log.w(any(), any()) } returns 0 + every { Log.e(any(), any(), any()) } returns 0 + + provider = + GeminiLiveProviderImpl( + httpClient = mockk(relaxed = true), + toolConverter = GeminiToolConverter(), + ) + } + + @Test + fun `buildSetupMessage configures gemini-3_8-live model, audio modality, transcriptions, and tools`() { + val tool = createSampleNoteTool() + + val setupEnvelope = provider.buildSetupMessage(listOf(tool)) + val setup = setupEnvelope["setup"]!!.jsonObject + + assertEquals("models/gemini-3.8-live", setup["model"]!!.jsonPrimitive.content) + + val modalities = + setup["generationConfig"]!! + .jsonObject["responseModalities"]!! + .jsonArray + .map { it.jsonPrimitive.content } + assertEquals(listOf("AUDIO"), modalities) + + assertTrue(setup.containsKey("inputAudioTranscription")) + assertTrue(setup.containsKey("outputAudioTranscription")) + assertTrue(setup.containsKey("systemInstruction")) + + val declarations = + setup["tools"]!! + .jsonArray + .first() + .jsonObject["functionDeclarations"]!! + .jsonArray + assertEquals(1, declarations.size) + assertEquals( + "com_example_notes_createNote", + declarations.first().jsonObject["name"]!!.jsonPrimitive.content, + ) + } + + @Test + fun `connect throws IllegalStateException when WebSocket closes before setupComplete`() = + runTest { + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.close() + + provider.webSocketConnector = { _, block -> + coroutineScope { + createMockWebSocketSession(coroutineContext, incomingFrames).block() + } + } + + try { + provider.connect(apiKey = "key-123", tools = emptyList()) {} + fail("Expected IllegalStateException") + } catch (e: IllegalStateException) { + assertTrue(e.message!!.contains("setupComplete")) + } + } + + @Test + fun `connect exchanges realtimeInput audio, parses server events, and sends mapped toolResponse`() = + runTest { + val tool = createSampleNoteTool() + val expectedPcm = byteArrayOf(10, 20, 30, 40) + val encodedPcm = Base64.getEncoder().encodeToString(expectedPcm) + + val incomingFrames = Channel(Channel.UNLIMITED) + val sentFrames = mutableListOf() + var capturedWsUrl: String? = null + + incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFrames.trySend( + Frame.Text( + """ + { + "serverContent": { + "interrupted": true, + "inputTranscription": { "text": "Create a note" }, + "outputTranscription": { "text": "Creating note" }, + "modelTurn": { + "parts": [ + { + "inlineData": { + "mimeType": "audio/pcm;rate=24000", + "data": "$encodedPcm" + } + } + ] + }, + "turnComplete": true + } + } + """.trimIndent(), + ), + ) + incomingFrames.trySend( + Frame.Text( + """ + { + "toolCall": { + "functionCalls": [ + { + "id": "call-1", + "name": "com_example_notes_createNote", + "args": { + "title": "Shopping", + "meta": { "pinned": true, "priority": 2 } + } + }, + { + "id": "call-unknown", + "name": "hallucinated_tool", + "args": {} + } + ] + } + } + """.trimIndent(), + ), + ) + incomingFrames.close() + + provider.webSocketConnector = { wsUrl, block -> + capturedWsUrl = wsUrl + coroutineScope { + val ws = + createMockWebSocketSession( + context = coroutineContext, + incomingFrames = incomingFrames, + onSendText = { sentFrames.add(it) }, + ) + ws.block() + } + } + + val receivedEvents = mutableListOf() + provider.connect(apiKey = "secret-key", tools = listOf(tool)) { session -> + session.sendAudio(byteArrayOf(1, 2, 3)) + receivedEvents.addAll(session.events.toList()) + session.sendToolResponses( + listOf( + ToolOutput( + functionId = tool.id, + callId = "call-1", + result = """{"noteId":"n-1"}""", + ), + ToolOutput( + functionId = "hallucinated_tool", + callId = "call-unknown", + result = "Tool not found: hallucinated_tool", + ), + ), + ) + } + + assertTrue(capturedWsUrl!!.endsWith("?key=secret-key")) + assertEquals(3, sentFrames.size) + + // Frame 0: setup + assertTrue(Json.parseToJsonElement(sentFrames[0]).jsonObject.containsKey("setup")) + + // Frame 1: realtimeInput audio + val realtimeAudio = + Json.parseToJsonElement(sentFrames[1]) + .jsonObject["realtimeInput"]!! + .jsonObject["audio"]!! + .jsonObject + assertEquals("audio/pcm;rate=16000", realtimeAudio["mimeType"]!!.jsonPrimitive.content) + assertArrayEquals( + byteArrayOf(1, 2, 3), + Base64.getDecoder().decode(realtimeAudio["data"]!!.jsonPrimitive.content), + ) + + // Received events + assertEquals(6, receivedEvents.size) + assertEquals(LiveLlmEvent.Interrupted, receivedEvents[0]) + assertEquals(LiveLlmEvent.InputTranscription("Create a note"), receivedEvents[1]) + assertEquals(LiveLlmEvent.OutputTranscription("Creating note"), receivedEvents[2]) + assertArrayEquals(expectedPcm, (receivedEvents[3] as LiveLlmEvent.AudioChunk).pcmBytes) + assertEquals(LiveLlmEvent.TurnComplete, receivedEvents[4]) + + val toolCallReq = receivedEvents[5] as LiveLlmEvent.ToolCallRequest + assertEquals(2, toolCallReq.calls.size) + assertEquals("com.example.notes", toolCallReq.calls[0].packageName) + assertEquals("com.example.notes.createNote", toolCallReq.calls[0].functionId) + assertEquals("Shopping", toolCallReq.calls[0].arguments["title"]) + @Suppress("UNCHECKED_CAST") + val meta = toolCallReq.calls[0].arguments["meta"] as Map + assertEquals(true, meta["pinned"]) + assertEquals(2L, meta["priority"]) + + assertEquals("", toolCallReq.calls[1].packageName) + assertEquals("hallucinated_tool", toolCallReq.calls[1].functionId) + + // Frame 2: toolResponse + val responses = + Json.parseToJsonElement(sentFrames[2]) + .jsonObject["toolResponse"]!! + .jsonObject["functionResponses"]!! + .jsonArray + assertEquals(2, responses.size) + assertEquals( + "com_example_notes_createNote", + responses[0].jsonObject["name"]!!.jsonPrimitive.content, + ) + assertEquals( + "n-1", + responses[0] + .jsonObject["response"]!! + .jsonObject["result"]!! + .jsonObject["noteId"]!! + .jsonPrimitive + .content, + ) + assertEquals( + "hallucinated_tool", + responses[1].jsonObject["name"]!!.jsonPrimitive.content, + ) + } + + private fun createMockWebSocketSession( + context: kotlin.coroutines.CoroutineContext, + incomingFrames: Channel, + onSendText: (String) -> Unit = {}, + ): WebSocketSession { + val outgoingFrames = Channel(Channel.UNLIMITED) + return mockk(relaxed = true).apply { + every { coroutineContext } returns context + every { incoming } returns incomingFrames + every { outgoing } returns outgoingFrames + coEvery { send(any()) } coAnswers { + val frame = firstArg() + if (frame is Frame.Text) { + onSendText(frame.readText()) + } + } + } + } + + private fun createSampleNoteTool(): AppFunctionMetadata = + AppFunctionMetadata( + id = "com.example.notes.createNote", + packageName = "com.example.notes", + isEnabled = true, + schema = null, + parameters = + listOf( + AppFunctionParameterMetadata( + name = "title", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(isNullable = false), + description = "Note title", + ), + ), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Creates a note", + deprecation = null, + ) +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt index 8c26dbd8..5973bf5d 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt @@ -34,8 +34,10 @@ import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.appfunction.ConvertInputToAppFunctionDataUseCase import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionUseCase +import com.example.appfunctions.agent.domain.appfunction.ExecuteToolCallUseCase import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionStatesUseCase import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase +import com.example.appfunctions.agent.domain.appfunction.GetEnabledToolsUseCase import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase import com.example.appfunctions.agent.domain.chat.ObservePendingMessagesUseCase import com.example.appfunctions.agent.domain.chat.SendMessageUseCase @@ -81,9 +83,23 @@ class AgentOrchestratorTest { @Before fun setUp() { + val getEnabledToolsUseCase = + GetEnabledToolsUseCase( + settingsRepository = settingsRepository, + getAppFunctionsUseCase = getAppFunctionsUseCase, + getAppFunctionStatesUseCase = getAppFunctionStatesUseCase, + agentInternalTools = agentInternalTools, + ) + val executeToolCallUseCase = + ExecuteToolCallUseCase( + context = context, + convertInputToAppFunctionDataUseCase = convertInputToAppFunctionDataUseCase, + executeAppFunctionUseCase = executeAppFunctionUseCase, + agentInternalTools = agentInternalTools, + ) + agentOrchestrator = AgentOrchestrator( - context = context, manageThreadsUseCase = manageThreadsUseCase, observePendingMessagesUseCase = observePendingMessagesUseCase, sendMessageUseCase = sendMessageUseCase, @@ -91,12 +107,9 @@ class AgentOrchestratorTest { updateThreadUseCase = updateThreadUseCase, llmProviderFactory = llmProviderFactory, settingsRepository = settingsRepository, - getAppFunctionsUseCase = getAppFunctionsUseCase, - getAppFunctionStatesUseCase = getAppFunctionStatesUseCase, - convertInputToAppFunctionDataUseCase = convertInputToAppFunctionDataUseCase, - executeAppFunctionUseCase = executeAppFunctionUseCase, + getEnabledToolsUseCase = getEnabledToolsUseCase, + executeToolCallUseCase = executeToolCallUseCase, savePendingIntentUseCase = savePendingIntentUseCase, - agentInternalTools = agentInternalTools, ) coEvery { getAppFunctionStatesUseCase(any()) } answers { diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManagerTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManagerTest.kt deleted file mode 100644 index 9d1381f9..00000000 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManagerTest.kt +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright 2026 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.example.appfunctions.agent.domain - -import androidx.appfunctions.metadata.AppFunctionComponentsMetadata -import androidx.appfunctions.metadata.AppFunctionMetadata -import androidx.appfunctions.metadata.AppFunctionParameterMetadata -import androidx.appfunctions.metadata.AppFunctionResponseMetadata -import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata -import com.example.appfunctions.agent.data.GeminiToolConverter -import com.example.appfunctions.agent.data.SettingsRepository -import com.example.appfunctions.agent.domain.chat.SendMessageUseCase -import io.ktor.client.HttpClient -import io.mockk.mockk -import kotlinx.serialization.json.jsonArray -import kotlinx.serialization.json.jsonObject -import kotlinx.serialization.json.jsonPrimitive -import org.junit.Assert.assertEquals -import org.junit.Assert.assertTrue -import org.junit.Before -import org.junit.Test - -class GeminiLiveSessionManagerTest { - private lateinit var sessionManager: GeminiLiveSessionManager - - @Before - fun setUp() { - sessionManager = - GeminiLiveSessionManager( - httpClient = mockk(relaxed = true), - toolConverter = GeminiToolConverter(), - settingsRepository = mockk(relaxed = true), - agentOrchestrator = mockk(relaxed = true), - sendMessageUseCase = mockk(relaxed = true), - ) - } - - @Test - fun `buildSetupMessage configures gemini-3_8-live model, audio modality, transcriptions, and tools`() { - val tool = - AppFunctionMetadata( - id = "com.example.notes.createNote", - packageName = "com.example.notes", - isEnabled = true, - schema = null, - parameters = - listOf( - AppFunctionParameterMetadata( - name = "title", - isRequired = true, - dataType = AppFunctionStringTypeMetadata(isNullable = false), - description = "Note title", - ), - ), - response = - AppFunctionResponseMetadata( - valueType = AppFunctionStringTypeMetadata(isNullable = false), - description = "", - ), - components = AppFunctionComponentsMetadata(emptyMap()), - description = "Creates a note", - deprecation = null, - ) - - val setupEnvelope = sessionManager.buildSetupMessage(listOf(tool)) - val setup = setupEnvelope["setup"]!!.jsonObject - - assertEquals("models/gemini-3.8-live", setup["model"]!!.jsonPrimitive.content) - - val modalities = - setup["generationConfig"]!! - .jsonObject["responseModalities"]!! - .jsonArray - .map { it.jsonPrimitive.content } - assertEquals(listOf("AUDIO"), modalities) - - assertTrue(setup.containsKey("inputAudioTranscription")) - assertTrue(setup.containsKey("outputAudioTranscription")) - assertTrue(setup.containsKey("systemInstruction")) - - val declarations = - setup["tools"]!! - .jsonArray - .first() - .jsonObject["functionDeclarations"]!! - .jsonArray - assertEquals(1, declarations.size) - assertEquals( - "com_example_notes_createNote", - declarations.first().jsonObject["name"]!!.jsonPrimitive.content, - ) - } -} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/LiveAgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/LiveAgentOrchestratorTest.kt new file mode 100644 index 00000000..f17f1f0a --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/LiveAgentOrchestratorTest.kt @@ -0,0 +1,435 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain + +import android.app.PendingIntent +import android.util.Log +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionResponseMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import com.example.appfunctions.agent.data.LlmProviderName +import com.example.appfunctions.agent.data.SettingsRepository +import com.example.appfunctions.agent.data.db.entities.MessageAttachment +import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus +import com.example.appfunctions.agent.data.db.entities.MessageRole +import com.example.appfunctions.agent.domain.appfunction.ExecuteToolCallUseCase +import com.example.appfunctions.agent.domain.appfunction.GetEnabledToolsUseCase +import com.example.appfunctions.agent.domain.appfunction.ToolCallExecutionResult +import com.example.appfunctions.agent.domain.chat.SendMessageUseCase +import com.example.appfunctions.agent.domain.pendingintent.SavePendingIntentUseCase +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.coVerifyOrder +import io.mockk.every +import io.mockk.mockk +import io.mockk.mockkStatic +import io.mockk.verify +import io.mockk.verifyOrder +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.emptyFlow +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.flow.receiveAsFlow +import kotlinx.coroutines.test.StandardTestDispatcher +import kotlinx.coroutines.test.advanceUntilIdle +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import java.util.Collections +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger + +@OptIn(ExperimentalCoroutinesApi::class) +class LiveAgentOrchestratorTest { + private lateinit var llmProviderFactory: LlmProviderFactory + private lateinit var liveLlmProvider: LiveLlmProvider + private lateinit var settingsRepository: SettingsRepository + private lateinit var getEnabledToolsUseCase: GetEnabledToolsUseCase + private lateinit var executeToolCallUseCase: ExecuteToolCallUseCase + private lateinit var savePendingIntentUseCase: SavePendingIntentUseCase + private lateinit var sendMessageUseCase: SendMessageUseCase + private var currentAudioFactory: () -> LiveAudioSession = { mockk(relaxed = true) } + + private lateinit var orchestrator: LiveAgentOrchestrator + + @Before + fun setUp() { + mockkStatic(Log::class) + every { Log.d(any(), any()) } returns 0 + every { Log.w(any(), any()) } returns 0 + every { Log.e(any(), any(), any()) } returns 0 + + llmProviderFactory = mockk() + liveLlmProvider = mockk() + settingsRepository = mockk(relaxed = true) + getEnabledToolsUseCase = mockk() + executeToolCallUseCase = mockk() + savePendingIntentUseCase = mockk(relaxed = true) + sendMessageUseCase = mockk(relaxed = true) + + every { settingsRepository.selectedProvider } returns flowOf(LlmProviderName.GEMINI) + every { settingsRepository.geminiApiKey } returns flowOf("test-api-key") + every { llmProviderFactory.getLiveProvider(LlmProviderName.GEMINI) } returns liveLlmProvider + coEvery { getEnabledToolsUseCase(any()) } returns emptyList() + + orchestrator = + LiveAgentOrchestrator( + llmProviderFactory = llmProviderFactory, + settingsRepository = settingsRepository, + getEnabledToolsUseCase = getEnabledToolsUseCase, + executeToolCallUseCase = executeToolCallUseCase, + savePendingIntentUseCase = savePendingIntentUseCase, + sendMessageUseCase = sendMessageUseCase, + audioSessionFactory = LiveAudioSessionFactory { currentAudioFactory() }, + ) + } + + @Test + fun `startSession reports Error and persists FAILED message when API key is missing`() = + runTest { + val ioDispatcher = StandardTestDispatcher(testScheduler) + orchestrator.ioDispatcher = ioDispatcher + every { settingsRepository.geminiApiKey } returns flowOf(null) + + orchestrator.startSession("thread-1") + advanceUntilIdle() + + assertEquals( + LiveSessionState.Error("API key is missing for GEMINI"), + orchestrator.state.value, + ) + coVerify { + sendMessageUseCase( + threadId = "thread-1", + role = MessageRole.ASSISTANT, + textContent = "API key is missing for GEMINI", + processingStatus = MessageProcessingStatus.FAILED, + ) + } + } + + @Test + fun `stopSession does not call audioSession release synchronously on caller thread`() = + runTest { + val ioDispatcher = StandardTestDispatcher(testScheduler) + orchestrator.ioDispatcher = ioDispatcher + + val mockAudio = + mockk(relaxed = true).apply { + every { listenToRecording() } returns emptyFlow() + } + currentAudioFactory = { mockAudio } + + val eventChannel = Channel(Channel.UNLIMITED) + stubLiveConnection(eventChannel.receiveAsFlow()) + + orchestrator.startSession("thread-1") + runCurrent() + assertEquals(LiveSessionState.Listening, orchestrator.state.value) + + orchestrator.stopSession() + assertEquals(LiveSessionState.Idle, orchestrator.state.value) + verify(exactly = 0) { mockAudio.release() } + + advanceUntilIdle() + verify(exactly = 1) { mockAudio.release() } + } + + @Test + fun `startSession waits for prior session audio teardown before starting new LiveAudioSession`() { + val events = Collections.synchronizedList(mutableListOf()) + val session1Listening = CountDownLatch(1) + val release1Entered = CountDownLatch(1) + val allowRelease1Finish = CountDownLatch(1) + val session2Started = CountDownLatch(1) + + val audio1 = + mockk(relaxed = true).apply { + every { listenToRecording() } returns emptyFlow() + every { start() } answers { + events.add("start1") + session1Listening.countDown() + } + every { release() } answers { + events.add("release1_begin") + release1Entered.countDown() + assertTrue(allowRelease1Finish.await(2, TimeUnit.SECONDS)) + events.add("release1_end") + } + } + val audio2 = + mockk(relaxed = true).apply { + every { listenToRecording() } returns emptyFlow() + every { start() } answers { + events.add("start2") + session2Started.countDown() + } + every { release() } answers { + events.add("release2") + } + } + + val counter = AtomicInteger(0) + currentAudioFactory = { if (counter.incrementAndGet() == 1) audio1 else audio2 } + + val ch1 = Channel(Channel.UNLIMITED) + val ch2 = Channel(Channel.UNLIMITED) + val connCounter = AtomicInteger(0) + coEvery { liveLlmProvider.connect(any(), any(), any()) } coAnswers { + val block = thirdArg Unit>() + val flow = if (connCounter.incrementAndGet() == 1) ch1.receiveAsFlow() else ch2.receiveAsFlow() + block(FakeLiveLlmSession(flow)) + } + + orchestrator.startSession("thread-1") + assertTrue(session1Listening.await(2, TimeUnit.SECONDS)) + + orchestrator.stopSession() + assertTrue(release1Entered.await(2, TimeUnit.SECONDS)) + + orchestrator.startSession("thread-1") + Thread.sleep(50) + assertEquals(listOf("start1", "release1_begin"), events.toList()) + + allowRelease1Finish.countDown() + assertTrue(session2Started.await(2, TimeUnit.SECONDS)) + assertEquals( + listOf("start1", "release1_begin", "release1_end", "start2"), + events.toList(), + ) + + orchestrator.stopSession() + } + + @Test + fun `streams audio chunks, flushes transcriptions in order, and stops playback on interruption`() = + runTest { + val ioDispatcher = StandardTestDispatcher(testScheduler) + orchestrator.ioDispatcher = ioDispatcher + + val micPcm = byteArrayOf(1, 2, 3, 4) + val modelPcm = byteArrayOf(9, 8, 7, 6) + val queuedChunks = mutableListOf() + val mockAudio = + mockk(relaxed = true).apply { + every { listenToRecording() } returns flowOf(micPcm) + every { queueAudio(any()) } answers { queuedChunks.add(firstArg()) } + } + currentAudioFactory = { mockAudio } + + val eventChannel = Channel(Channel.UNLIMITED) + val fakeSession = stubLiveConnection(eventChannel.receiveAsFlow()) + + orchestrator.startSession("thread-1") + runCurrent() + + eventChannel.trySend(LiveLlmEvent.InputTranscription("What is ")) + eventChannel.trySend(LiveLlmEvent.InputTranscription("the weather?")) + eventChannel.trySend(LiveLlmEvent.OutputTranscription("It is sunny")) + eventChannel.trySend(LiveLlmEvent.AudioChunk(modelPcm)) + eventChannel.trySend(LiveLlmEvent.Interrupted) + eventChannel.close() + advanceUntilIdle() + + assertEquals(1, fakeSession.sentAudio.size) + assertArrayEquals(micPcm, fakeSession.sentAudio.first()) + assertEquals(1, queuedChunks.size) + assertArrayEquals(modelPcm, queuedChunks.first()) + verify(exactly = 1) { mockAudio.stopAudio() } + coVerifyOrder { + sendMessageUseCase( + threadId = "thread-1", + role = MessageRole.USER, + textContent = "What is the weather?", + processingStatus = MessageProcessingStatus.PROCESSED, + ) + sendMessageUseCase( + threadId = "thread-1", + role = MessageRole.ASSISTANT, + textContent = "It is sunny", + processingStatus = MessageProcessingStatus.PROCESSED, + ) + } + } + + @Test + fun `ToolCallRequest gates recording, updates status, saves PendingIntent and attachments, and sends ToolOutputs`() = + runTest { + val ioDispatcher = StandardTestDispatcher(testScheduler) + orchestrator.ioDispatcher = ioDispatcher + + val noteTool = createSampleNoteTool() + coEvery { getEnabledToolsUseCase("com.example.notes") } returns listOf(noteTool) + + val noteCall = + LlmResponsePart.ToolCall( + packageName = "com.example.notes", + functionId = noteTool.id, + arguments = mapOf("title" to "Shopping"), + callId = "call-1", + ) + val confirmCall = + LlmResponsePart.ToolCall( + packageName = "com.example.notes", + functionId = "deleteAll", + arguments = emptyMap(), + callId = "call-2", + ) + val pendingIntent = mockk() + val observedStatuses = mutableListOf() + + coEvery { + executeToolCallUseCase(noteCall, listOf(noteTool), "thread-1") + } answers { + observedStatuses.add(orchestrator.status.value) + ToolCallExecutionResult.Success( + toolOutput = ToolOutput(noteTool.id, "call-1", """{"status":"ok"}"""), + attachment = MessageAttachment("content://img/1.png", "image/png"), + ) + } + coEvery { + executeToolCallUseCase(confirmCall, listOf(noteTool), "thread-1") + } answers { + observedStatuses.add(orchestrator.status.value) + ToolCallExecutionResult.PendingIntentRequired("pi-123", pendingIntent) + } + + val mockAudio = + mockk(relaxed = true).apply { + every { listenToRecording() } returns emptyFlow() + } + currentAudioFactory = { mockAudio } + + val eventChannel = Channel(Channel.UNLIMITED) + eventChannel.trySend(LiveLlmEvent.ToolCallRequest(listOf(noteCall, confirmCall))) + val toolResponseSent = CompletableDeferred() + val fakeSession = + stubLiveConnection( + events = eventChannel.receiveAsFlow(), + onToolResponses = { + toolResponseSent.complete(Unit) + eventChannel.close() + }, + ) + + orchestrator.startSession("thread-1", targetPackageName = "com.example.notes") + advanceUntilIdle() + + assertTrue(toolResponseSent.isCompleted) + assertEquals(AgentStatus.Idle, orchestrator.status.value) + assertEquals( + listOf( + AgentStatus.InvokingTool(noteTool.id, "com.example.notes"), + AgentStatus.InvokingTool("deleteAll", "com.example.notes"), + ), + observedStatuses, + ) + verifyOrder { + mockAudio.start() + mockAudio.pauseRecording() + mockAudio.resumeRecording() + mockAudio.release() + } + verify { savePendingIntentUseCase("pi-123", pendingIntent) } + coVerify { + sendMessageUseCase( + threadId = "thread-1", + role = MessageRole.ASSISTANT, + textContent = "", + processingStatus = MessageProcessingStatus.PROCESSED, + pendingIntentId = "pi-123", + ) + sendMessageUseCase( + threadId = "thread-1", + role = MessageRole.ASSISTANT, + textContent = "", + processingStatus = MessageProcessingStatus.PROCESSED, + attachments = listOf(MessageAttachment("content://img/1.png", "image/png")), + ) + } + assertEquals(2, fakeSession.sentToolOutputs.single().size) + assertEquals("""{"status":"ok"}""", fakeSession.sentToolOutputs.single()[0].result) + assertTrue( + fakeSession.sentToolOutputs + .single()[1] + .result + .contains("confirmation_required"), + ) + } + + private fun stubLiveConnection( + events: Flow, + onToolResponses: (List) -> Unit = {}, + ): FakeLiveLlmSession { + val fakeSession = FakeLiveLlmSession(events, onToolResponses) + coEvery { liveLlmProvider.connect(any(), any(), any()) } coAnswers { + val block = thirdArg Unit>() + block(fakeSession) + } + return fakeSession + } + + private class FakeLiveLlmSession( + override val events: Flow, + private val onToolResponses: (List) -> Unit = {}, + ) : LiveLlmSession { + val sentAudio = mutableListOf() + val sentToolOutputs = mutableListOf>() + + override suspend fun sendAudio(pcmBytes: ByteArray) { + sentAudio.add(pcmBytes) + } + + override suspend fun sendToolResponses(outputs: List) { + sentToolOutputs.add(outputs) + onToolResponses(outputs) + } + } + + private fun createSampleNoteTool(): AppFunctionMetadata = + AppFunctionMetadata( + id = "com.example.notes.createNote", + packageName = "com.example.notes", + isEnabled = true, + schema = null, + parameters = + listOf( + AppFunctionParameterMetadata( + name = "title", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(isNullable = false), + description = "Note title", + ), + ), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Creates a note", + deprecation = null, + ) +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteToolCallUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteToolCallUseCaseTest.kt new file mode 100644 index 00000000..e94dc542 --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteToolCallUseCaseTest.kt @@ -0,0 +1,231 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain.appfunction + +import android.app.PendingIntent +import android.content.Context +import android.content.Intent +import androidx.appfunctions.AppFunctionData +import androidx.appfunctions.AppFunctionPermissionRequiredException +import androidx.appfunctions.metadata.AppFunctionMetadata +import com.example.appfunctions.agent.data.AgentInternalTools +import com.example.appfunctions.agent.data.db.entities.MessageAttachment +import com.example.appfunctions.agent.domain.LlmResponsePart +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertSame +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +class ExecuteToolCallUseCaseTest { + private val context: Context = mockk(relaxed = true) + private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase = mockk() + private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase = mockk() + private val agentInternalTools: AgentInternalTools = mockk(relaxed = true) + + private lateinit var useCase: ExecuteToolCallUseCase + + @Before + fun setUp() { + useCase = + ExecuteToolCallUseCase( + context = context, + convertInputToAppFunctionDataUseCase = convertInputToAppFunctionDataUseCase, + executeAppFunctionUseCase = executeAppFunctionUseCase, + agentInternalTools = agentInternalTools, + ) + } + + @Test + fun `internal generateImage tool returns Success with extracted MessageAttachment`() = + runTest { + val imageResult = + AgentInternalTools.GeneratedImageResult( + imageUri = "content://com.example.appfunctions.agent.fileprovider/cache/cat.png", + mimeType = "image/png", + prompt = "a cat", + ) + coEvery { agentInternalTools.generateImage("a cat", "1:1") } returns imageResult + + val toolCall = + LlmResponsePart.ToolCall( + packageName = AgentInternalTools.INTERNAL_TOOL_PACKAGE, + functionId = "generateImage", + arguments = mapOf("prompt" to "a cat", "aspectRatio" to "1:1"), + callId = "call-1", + ) + + val result = useCase(toolCall = toolCall, tools = emptyList(), threadId = "thread-1") + + assertTrue(result is ToolCallExecutionResult.Success) + val success = result as ToolCallExecutionResult.Success + assertEquals("call-1", success.toolOutput.callId) + assertEquals("generateImage", success.toolOutput.functionId) + assertEquals( + MessageAttachment(uri = imageResult.imageUri, mimeType = "image/png"), + success.attachment, + ) + } + + @Test + fun `missing tool metadata returns Failure`() = + runTest { + val toolCall = + LlmResponsePart.ToolCall( + packageName = "com.example.notes", + functionId = "nonexistent", + arguments = emptyMap(), + callId = "call-2", + ) + + val result = useCase(toolCall = toolCall, tools = emptyList(), threadId = "thread-1") + + assertTrue(result is ToolCallExecutionResult.Failure) + assertEquals( + "Tool not found: nonexistent", + (result as ToolCallExecutionResult.Failure).errorMessage, + ) + } + + @Test + fun `AppFunction execution grants recursive URI permissions and returns Data output`() = + runTest { + val tool = createMockTool("com.example.notes", "attachImages") + coEvery { + convertInputToAppFunctionDataUseCase(any(), any(), any()) + } returns Result.success(AppFunctionData.EMPTY) + coEvery { + executeAppFunctionUseCase(tool, any(), "thread-1") + } returns ExecuteAppFunctionResult.Data(AppFunctionData.EMPTY, """{"ok":true}""") + + val nestedUri = "content://com.example.appfunctions.agent.fileprovider/cache/nested.jpg" + val toolCall = + LlmResponsePart.ToolCall( + packageName = "com.example.notes", + functionId = "attachImages", + arguments = + mapOf( + "payload" to mapOf("uris" to listOf(nestedUri)), + ), + callId = "call-3", + ) + + val result = useCase(toolCall = toolCall, tools = listOf(tool), threadId = "thread-1") + + assertTrue(result is ToolCallExecutionResult.Success) + assertEquals( + """{"ok":true}""", + (result as ToolCallExecutionResult.Success).toolOutput.result, + ) + verify(exactly = 1) { + context.grantUriPermission( + eq("com.example.notes"), + match { it.toString() == nestedUri }, + eq(Intent.FLAG_GRANT_READ_URI_PERMISSION), + ) + } + } + + @Test + fun `AppFunction PendingIntentAction returns PendingIntentRequired`() = + runTest { + val tool = createMockTool("com.example.payments", "sendPayment") + val pendingIntent = mockk() + coEvery { + convertInputToAppFunctionDataUseCase(any(), any(), any()) + } returns Result.success(AppFunctionData.EMPTY) + coEvery { + executeAppFunctionUseCase(tool, any(), "thread-1") + } returns ExecuteAppFunctionResult.PendingIntentAction(pendingIntent) + + val toolCall = + LlmResponsePart.ToolCall( + packageName = "com.example.payments", + functionId = "sendPayment", + arguments = emptyMap(), + callId = "call-4", + ) + + val result = useCase(toolCall = toolCall, tools = listOf(tool), threadId = "thread-1") + + assertTrue(result is ToolCallExecutionResult.PendingIntentRequired) + assertSame( + pendingIntent, + (result as ToolCallExecutionResult.PendingIntentRequired).pendingIntent, + ) + } + + @Test + fun `AppFunctionException returns formatted recoverable Success output while generic Exception returns Failure`() = + runTest { + val tool = createMockTool("com.example.calendar", "createEvent") + coEvery { + convertInputToAppFunctionDataUseCase(any(), any(), any()) + } returns Result.success(AppFunctionData.EMPTY) + + val appException = AppFunctionPermissionRequiredException("Calendar denied") + coEvery { + executeAppFunctionUseCase(tool, any(), "thread-1") + } returns ExecuteAppFunctionResult.Error(appException) + + val toolCall = + LlmResponsePart.ToolCall( + packageName = "com.example.calendar", + functionId = "createEvent", + arguments = emptyMap(), + callId = "call-5", + ) + + val recoverable = + useCase(toolCall = toolCall, tools = listOf(tool), threadId = "thread-1") + assertTrue(recoverable is ToolCallExecutionResult.Success) + assertEquals( + "Tool execution failed for createEvent: Error: AppFunctionPermissionRequiredException - Calendar denied", + (recoverable as ToolCallExecutionResult.Success).toolOutput.result, + ) + + coEvery { + executeAppFunctionUseCase(tool, any(), "thread-1") + } returns ExecuteAppFunctionResult.Error(IllegalStateException("Binder died")) + + val fatal = useCase(toolCall = toolCall, tools = listOf(tool), threadId = "thread-1") + assertTrue(fatal is ToolCallExecutionResult.Failure) + assertEquals( + "Tool execution failed for createEvent: Binder died", + (fatal as ToolCallExecutionResult.Failure).errorMessage, + ) + } + + private fun createMockTool( + packageName: String, + id: String, + ): AppFunctionMetadata = + mockk(relaxed = true).apply { + every { this@apply.packageName } returns packageName + every { this@apply.id } returns id + every { parameters } returns emptyList() + } +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetEnabledToolsUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetEnabledToolsUseCaseTest.kt new file mode 100644 index 00000000..cc87d875 --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetEnabledToolsUseCaseTest.kt @@ -0,0 +1,119 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain.appfunction + +import androidx.appfunctions.AppFunctionName +import androidx.appfunctions.AppFunctionState +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionPackageMetadata +import com.example.appfunctions.agent.data.AgentInternalTools +import com.example.appfunctions.agent.data.SettingsRepository +import io.mockk.coEvery +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test + +class GetEnabledToolsUseCaseTest { + private val settingsRepository: SettingsRepository = mockk() + private val getAppFunctionsUseCase: GetAppFunctionsUseCase = mockk() + private val getAppFunctionStatesUseCase: GetAppFunctionStatesUseCase = mockk() + private val agentInternalTools: AgentInternalTools = mockk() + + private lateinit var useCase: GetEnabledToolsUseCase + + @Before + fun setUp() { + every { settingsRepository.disconnectedApps } returns flowOf(emptySet()) + every { agentInternalTools.getInternalToolsMetadata() } returns emptyList() + coEvery { getAppFunctionStatesUseCase(any()) } answers { + firstArg>().map { name -> + createState(name, isEnabled = true) + } + } + + useCase = + GetEnabledToolsUseCase( + settingsRepository = settingsRepository, + getAppFunctionsUseCase = getAppFunctionsUseCase, + getAppFunctionStatesUseCase = getAppFunctionStatesUseCase, + agentInternalTools = agentInternalTools, + ) + } + + @Test + fun `invoke excludes disabled tools and disconnected apps while keeping internal tools`() = + runTest { + val enabledTool = createTool("com.example.notes", "com.example.notes.createNote") + val disabledTool = createTool("com.example.notes", "com.example.notes.deleteNote") + val disconnectedTool = createTool("com.example.chat", "com.example.chat.sendMessage") + val internalTool = + createTool(AgentInternalTools.INTERNAL_TOOL_PACKAGE, "generateImage") + val internalState = createState(internalTool.name, isEnabled = true) + + val pkgMeta = mockk(relaxed = true) + every { getAppFunctionsUseCase() } returns + flowOf(mapOf(pkgMeta to listOf(enabledTool, disabledTool, disconnectedTool))) + every { settingsRepository.disconnectedApps } returns flowOf(setOf("com.example.chat")) + every { agentInternalTools.getInternalToolsMetadata() } returns + listOf(internalTool to internalState) + coEvery { getAppFunctionStatesUseCase(any()) } answers { + firstArg>().map { name -> + createState(name, isEnabled = name != disabledTool.name) + } + } + + val result = useCase() + + assertEquals(listOf(enabledTool, internalTool), result) + } + + @Test + fun `invoke scopes returned tools to targetPackageName when specified`() = + runTest { + val notesTool = createTool("com.example.notes", "createNote") + val calendarTool = createTool("com.example.calendar", "createEvent") + val pkgMeta = mockk(relaxed = true) + every { getAppFunctionsUseCase() } returns + flowOf(mapOf(pkgMeta to listOf(notesTool, calendarTool))) + + val result = useCase(targetPackageName = "com.example.calendar") + + assertEquals(listOf(calendarTool), result) + } + + private fun createTool( + packageName: String, + id: String, + ): AppFunctionMetadata = + mockk(relaxed = true).apply { + every { this@apply.packageName } returns packageName + every { this@apply.id } returns id + every { this@apply.name } returns AppFunctionName(packageName, id) + } + + private fun createState( + name: AppFunctionName, + isEnabled: Boolean, + ): AppFunctionState = + mockk().apply { + every { functionName } returns name + every { this@apply.isEnabled } returns isEnabled + } +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt index a2aba834..2aa588c0 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt @@ -25,8 +25,6 @@ import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentOrchestrator import com.example.appfunctions.agent.domain.AgentStatus -import com.example.appfunctions.agent.domain.GeminiLiveSessionManager -import com.example.appfunctions.agent.domain.LiveSessionState import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase import com.example.appfunctions.agent.domain.chat.GetChatHistoryUseCase @@ -39,7 +37,6 @@ import io.mockk.coEvery import io.mockk.coVerify import io.mockk.every import io.mockk.mockk -import io.mockk.verify import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.flow.MutableStateFlow @@ -68,7 +65,6 @@ class AgentDemoViewModelTest { private lateinit var consumePendingIntentUseCase: ConsumePendingIntentUseCase private lateinit var getInstalledAppsUseCase: GetInstalledAppsUseCase private lateinit var getAppFunctionsUseCase: GetAppFunctionsUseCase - private lateinit var geminiLiveSessionManager: GeminiLiveSessionManager private lateinit var viewModel: AgentDemoViewModel @@ -77,7 +73,6 @@ class AgentDemoViewModelTest { private val agentStatusFlow = MutableStateFlow(AgentStatus.Idle) private val messagesFlow = MutableStateFlow>(emptyList()) private val activePendingActionIdsFlow = MutableStateFlow>(emptySet()) - private val liveSessionStateFlow = MutableStateFlow(LiveSessionState.Idle) @Before fun setUp() { @@ -94,7 +89,6 @@ class AgentDemoViewModelTest { consumePendingIntentUseCase = mockk(relaxed = true) getInstalledAppsUseCase = mockk() getAppFunctionsUseCase = mockk() - geminiLiveSessionManager = mockk(relaxed = true) every { getInstalledAppsUseCase() } returns emptyList() every { getAppFunctionsUseCase() } returns flowOf(emptyMap()) @@ -104,7 +98,6 @@ class AgentDemoViewModelTest { every { agentOrchestrator.status } returns agentStatusFlow every { getChatHistoryUseCase(any()) } returns messagesFlow every { observeActivePendingIntentsUseCase() } returns activePendingActionIdsFlow - every { geminiLiveSessionManager.state } returns liveSessionStateFlow coEvery { agentOrchestrator.observeAndProcessMessages(any()) } returns Unit } @@ -129,21 +122,7 @@ class AgentDemoViewModelTest { manageThreadsUseCase.updateThreadModel("thread-1", LlmModel.GEMINI_3_1_PRO_PREVIEW) } returns Unit - viewModel = - AgentDemoViewModel( - savedStateHandle, - getChatHistoryUseCase, - manageThreadsUseCase, - sendMessageUseCase, - agentOrchestrator, - settingsRepository, - observeActivePendingIntentsUseCase, - launchPendingIntentUseCase, - consumePendingIntentUseCase, - getInstalledAppsUseCase, - getAppFunctionsUseCase, - geminiLiveSessionManager, - ) + viewModel = createViewModel() viewModel.onEvent(AgentUiEvent.OnModelSelected(LlmModel.GEMINI_3_1_PRO_PREVIEW)) @@ -158,21 +137,7 @@ class AgentDemoViewModelTest { threadsFlow.value = emptyList() coEvery { manageThreadsUseCase.createThread(LlmModel.DEFAULT) } returns "thread-new" - viewModel = - AgentDemoViewModel( - savedStateHandle, - getChatHistoryUseCase, - manageThreadsUseCase, - sendMessageUseCase, - agentOrchestrator, - settingsRepository, - observeActivePendingIntentsUseCase, - launchPendingIntentUseCase, - consumePendingIntentUseCase, - getInstalledAppsUseCase, - getAppFunctionsUseCase, - geminiLiveSessionManager, - ) + viewModel = createViewModel() coVerify { manageThreadsUseCase.createThread(LlmModel.DEFAULT) } } @@ -190,21 +155,7 @@ class AgentDemoViewModelTest { threadsFlow.value = listOf(existingThread) selectedProviderFlow.value = LlmProviderName.GEMINI - viewModel = - AgentDemoViewModel( - savedStateHandle, - getChatHistoryUseCase, - manageThreadsUseCase, - sendMessageUseCase, - agentOrchestrator, - settingsRepository, - observeActivePendingIntentsUseCase, - launchPendingIntentUseCase, - consumePendingIntentUseCase, - getInstalledAppsUseCase, - getAppFunctionsUseCase, - geminiLiveSessionManager, - ) + viewModel = createViewModel() coVerify(exactly = 0) { manageThreadsUseCase.createThread(any()) } assertEquals( @@ -226,21 +177,7 @@ class AgentDemoViewModelTest { threadsFlow.value = listOf(existingThread) selectedProviderFlow.value = LlmProviderName.GEMINI - viewModel = - AgentDemoViewModel( - savedStateHandle, - getChatHistoryUseCase, - manageThreadsUseCase, - sendMessageUseCase, - agentOrchestrator, - settingsRepository, - observeActivePendingIntentsUseCase, - launchPendingIntentUseCase, - consumePendingIntentUseCase, - getInstalledAppsUseCase, - getAppFunctionsUseCase, - geminiLiveSessionManager, - ) + viewModel = createViewModel() coEvery { sendMessageUseCase(any(), any(), any(), any()) } returns Unit @@ -269,21 +206,7 @@ class AgentDemoViewModelTest { threadsFlow.value = listOf(existingThread) selectedProviderFlow.value = LlmProviderName.GEMINI - viewModel = - AgentDemoViewModel( - savedStateHandle, - getChatHistoryUseCase, - manageThreadsUseCase, - sendMessageUseCase, - agentOrchestrator, - settingsRepository, - observeActivePendingIntentsUseCase, - launchPendingIntentUseCase, - consumePendingIntentUseCase, - getInstalledAppsUseCase, - getAppFunctionsUseCase, - geminiLiveSessionManager, - ) + viewModel = createViewModel() val message = MessageEntity( @@ -319,57 +242,23 @@ class AgentDemoViewModelTest { threadsFlow.value = listOf(newThread, oldThread) selectedProviderFlow.value = LlmProviderName.GEMINI - viewModel = - AgentDemoViewModel( - savedStateHandle, - getChatHistoryUseCase, - manageThreadsUseCase, - sendMessageUseCase, - agentOrchestrator, - settingsRepository, - observeActivePendingIntentsUseCase, - launchPendingIntentUseCase, - consumePendingIntentUseCase, - getInstalledAppsUseCase, - getAppFunctionsUseCase, - geminiLiveSessionManager, - ) + viewModel = createViewModel() assertEquals(newThread, (viewModel.uiState.value as AgentUiState.Loaded).currentThread) } - @Test - fun `onEvent OnToggleLiveSession delegates to GeminiLiveSessionManager`() = - runTest { - val existingThread = - ThreadEntity( - "thread-1", - System.currentTimeMillis(), - LlmModel.GEMINI_3_1_PRO_PREVIEW, - null, - ) - threadsFlow.value = listOf(existingThread) - - viewModel = - AgentDemoViewModel( - savedStateHandle, - getChatHistoryUseCase, - manageThreadsUseCase, - sendMessageUseCase, - agentOrchestrator, - settingsRepository, - observeActivePendingIntentsUseCase, - launchPendingIntentUseCase, - consumePendingIntentUseCase, - getInstalledAppsUseCase, - getAppFunctionsUseCase, - geminiLiveSessionManager, - ) - - viewModel.onEvent(AgentUiEvent.OnToggleLiveSession("com.example.target")) - - verify { - geminiLiveSessionManager.toggleSession("thread-1", "com.example.target") - } - } + private fun createViewModel(): AgentDemoViewModel = + AgentDemoViewModel( + savedStateHandle, + getChatHistoryUseCase, + manageThreadsUseCase, + sendMessageUseCase, + agentOrchestrator, + settingsRepository, + observeActivePendingIntentsUseCase, + launchPendingIntentUseCase, + consumePendingIntentUseCase, + getInstalledAppsUseCase, + getAppFunctionsUseCase, + ) } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionServiceTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionServiceTest.kt new file mode 100644 index 00000000..8ab142e2 --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionServiceTest.kt @@ -0,0 +1,131 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.voice + +import android.content.Context +import android.os.Bundle +import android.provider.Settings +import androidx.test.core.app.ApplicationProvider +import io.mockk.Runs +import io.mockk.every +import io.mockk.just +import io.mockk.slot +import io.mockk.spyk +import io.mockk.verify +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.Robolectric +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +class AgentVoiceInteractionServiceTest { + private lateinit var context: Context + + @Before + fun setUp() { + context = ApplicationProvider.getApplicationContext() + AgentVoiceInteractionService.resetForTesting() + } + + @After + fun tearDown() { + AgentVoiceInteractionService.resetForTesting() + } + + @Test + fun `launchLiveSession returns false when service is not active digital assistant`() { + AgentVoiceInteractionService.isActiveServiceChecker = { false } + + val launched = + AgentVoiceInteractionService.launchLiveSession( + context = context, + threadId = "thread-1", + targetPackageName = "com.example.chat", + ) + + assertFalse(launched) + assertNull(AgentVoiceInteractionService.consumePendingSession()) + } + + @Test + fun `launchLiveSession calls showSession immediately when service is active and ready`() { + AgentVoiceInteractionService.isActiveServiceChecker = { true } + val service = + spyk(Robolectric.buildService(AgentVoiceInteractionService::class.java).create().get()) + val argsSlot = slot() + every { service.showSession(capture(argsSlot), any()) } just Runs + service.onReady() + + val launched = + AgentVoiceInteractionService.launchLiveSession( + context = context, + threadId = "thread-42", + targetPackageName = "com.example.notes", + ) + + assertTrue(launched) + verify(exactly = 1) { service.showSession(any(), 0) } + assertEquals( + "thread-42", + argsSlot.captured.getString(AgentVoiceInteractionService.EXTRA_THREAD_ID), + ) + assertEquals( + "com.example.notes", + argsSlot.captured.getString(AgentVoiceInteractionService.EXTRA_TARGET_PACKAGE_NAME), + ) + } + + @Test + fun `launchLiveSession returns false when active in settings but unbound and showAssist fails`() { + AgentVoiceInteractionService.isActiveServiceChecker = { true } + + val launched = + AgentVoiceInteractionService.launchLiveSession( + context = context, + threadId = "thread-1", + targetPackageName = null, + ) + + assertFalse(launched) + assertNull(AgentVoiceInteractionService.consumePendingSession()) + } + + @Test + fun `createAssistantSettingsIntent targets ACTION_VOICE_INPUT_SETTINGS`() { + val intent = AgentVoiceInteractionService.createAssistantSettingsIntent() + + assertEquals(Settings.ACTION_VOICE_INPUT_SETTINGS, intent.action) + } + + @Test + fun `onReady does not self-trigger showSession while settings UI is in foreground`() { + val service = + spyk(Robolectric.buildService(AgentVoiceInteractionService::class.java).create().get()) + every { service.showSession(any(), any()) } just Runs + + service.onReady() + + verify(exactly = 0) { service.showSession(any(), any()) } + } +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionTest.kt new file mode 100644 index 00000000..a1f0efa3 --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionTest.kt @@ -0,0 +1,256 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.voice + +import android.content.Context +import android.graphics.Rect +import android.os.Looper +import android.service.voice.VoiceInteractionSession +import androidx.test.core.app.ApplicationProvider +import com.example.appfunctions.agent.data.LlmModel +import com.example.appfunctions.agent.data.db.entities.ThreadEntity +import com.example.appfunctions.agent.domain.AgentStatus +import com.example.appfunctions.agent.domain.LiveAgentOrchestrator +import com.example.appfunctions.agent.domain.LiveSessionState +import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase +import io.mockk.Runs +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.just +import io.mockk.mockk +import io.mockk.spyk +import io.mockk.verify +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.test.StandardTestDispatcher +import kotlinx.coroutines.test.advanceUntilIdle +import kotlinx.coroutines.test.runTest +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows +import org.robolectric.annotation.Config +import org.robolectric.annotation.GraphicsMode + +@OptIn(ExperimentalCoroutinesApi::class) +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +@GraphicsMode(GraphicsMode.Mode.NATIVE) +class AgentVoiceInteractionSessionTest { + private lateinit var context: Context + private lateinit var liveAgentOrchestrator: LiveAgentOrchestrator + private lateinit var manageThreadsUseCase: ManageThreadsUseCase + + private val liveStateFlow = MutableStateFlow(LiveSessionState.Idle) + private val agentStatusFlow = MutableStateFlow(AgentStatus.Idle) + + @Before + fun setUp() { + context = ApplicationProvider.getApplicationContext() + AgentVoiceInteractionService.resetForTesting() + + liveAgentOrchestrator = mockk(relaxed = true) + manageThreadsUseCase = mockk(relaxed = true) + + every { liveAgentOrchestrator.state } returns liveStateFlow + every { liveAgentOrchestrator.status } returns agentStatusFlow + } + + @After + fun tearDown() { + AgentVoiceInteractionService.resetForTesting() + } + + @Test + fun `onShow starts LiveAgentOrchestrator with requested threadId and targetPackageName`() = + runTest { + val testDispatcher = StandardTestDispatcher(testScheduler) + val session = + spyk( + AgentVoiceInteractionSession( + context = context, + liveAgentOrchestrator = liveAgentOrchestrator, + manageThreadsUseCase = manageThreadsUseCase, + mainDispatcher = testDispatcher, + ), + ) + every { session.hide() } just Runs + + val args = + AgentVoiceInteractionService.createSessionArgs( + threadId = "thread-123", + targetPackageName = "com.example.chat", + ) + session.onShow(args, 0) + advanceUntilIdle() + + verify(exactly = 1) { + liveAgentOrchestrator.startSession( + threadId = "thread-123", + targetPackageName = "com.example.chat", + ) + } + session.onDestroy() + } + + @Test + fun `onShow without threadId resolves default thread via ManageThreadsUseCase`() = + runTest { + val testDispatcher = StandardTestDispatcher(testScheduler) + every { manageThreadsUseCase.getThreads() } returns flowOf(emptyList()) + coEvery { manageThreadsUseCase.createThread(LlmModel.DEFAULT) } returns "thread-created" + + val session = + spyk( + AgentVoiceInteractionSession( + context = context, + liveAgentOrchestrator = liveAgentOrchestrator, + manageThreadsUseCase = manageThreadsUseCase, + mainDispatcher = testDispatcher, + ), + ) + every { session.hide() } just Runs + + session.onShow(null, 0) + advanceUntilIdle() + + coVerify(exactly = 1) { manageThreadsUseCase.createThread(LlmModel.DEFAULT) } + verify(exactly = 1) { + liveAgentOrchestrator.startSession( + threadId = "thread-created", + targetPackageName = null, + ) + } + session.onDestroy() + } + + @Test + fun `onShow without threadId reuses newest existing thread when present`() = + runTest { + val testDispatcher = StandardTestDispatcher(testScheduler) + val existingThread = + ThreadEntity( + "thread-existing", + 1000L, + LlmModel.DEFAULT, + null, + ) + every { manageThreadsUseCase.getThreads() } returns flowOf(listOf(existingThread)) + + val session = + spyk( + AgentVoiceInteractionSession( + context = context, + liveAgentOrchestrator = liveAgentOrchestrator, + manageThreadsUseCase = manageThreadsUseCase, + mainDispatcher = testDispatcher, + ), + ) + every { session.hide() } just Runs + + session.onShow(null, 0) + advanceUntilIdle() + + coVerify(exactly = 0) { manageThreadsUseCase.createThread(any()) } + verify(exactly = 1) { + liveAgentOrchestrator.startSession( + threadId = "thread-existing", + targetPackageName = null, + ) + } + session.onDestroy() + } + + @Test + fun `onComputeInsets restricts touchable region to floating overlay bounds`() = + runTest { + val testDispatcher = StandardTestDispatcher(testScheduler) + val session = + AgentVoiceInteractionSession( + context = context, + liveAgentOrchestrator = liveAgentOrchestrator, + manageThreadsUseCase = manageThreadsUseCase, + mainDispatcher = testDispatcher, + ) + + val pillBounds = Rect(32, 1800, 1048, 1980) + session.updateOverlayBounds(pillBounds) + Shadows.shadowOf(Looper.getMainLooper()).idle() + + val insets = VoiceInteractionSession.Insets() + session.onComputeInsets(insets) + + assertEquals(VoiceInteractionSession.Insets.TOUCHABLE_INSETS_REGION, insets.touchableInsets) + assertEquals(pillBounds, insets.touchableRegion.bounds) + session.onDestroy() + } + + @Test + fun `onCloseSystemDialogs keeps session visible while Listening and hides when Idle`() = + runTest { + val testDispatcher = StandardTestDispatcher(testScheduler) + val session = + spyk( + AgentVoiceInteractionSession( + context = context, + liveAgentOrchestrator = liveAgentOrchestrator, + manageThreadsUseCase = manageThreadsUseCase, + mainDispatcher = testDispatcher, + ), + ) + every { session.hide() } just Runs + + liveStateFlow.value = LiveSessionState.Listening + session.onCloseSystemDialogs() + verify(exactly = 0) { session.hide() } + + liveStateFlow.value = LiveSessionState.Idle + session.onCloseSystemDialogs() + verify(exactly = 1) { session.hide() } + + session.onDestroy() + } + + @Test + fun `stopAndHide and onHide stop LiveAgentOrchestrator`() = + runTest { + val testDispatcher = StandardTestDispatcher(testScheduler) + val session = + spyk( + AgentVoiceInteractionSession( + context = context, + liveAgentOrchestrator = liveAgentOrchestrator, + manageThreadsUseCase = manageThreadsUseCase, + mainDispatcher = testDispatcher, + ), + ) + every { session.hide() } just Runs + + session.stopAndHide() + verify(exactly = 1) { liveAgentOrchestrator.stopSession() } + verify(exactly = 1) { session.hide() } + + session.onHide() + verify(exactly = 2) { liveAgentOrchestrator.stopSession() } + + session.onDestroy() + } +} From 81e3b6242b94403454cdc3be76f92607d8283d3d Mon Sep 17 00:00:00 2001 From: Julian Toledo Date: Tue, 27 Oct 2026 16:52:13 +0100 Subject: [PATCH 4/6] feat(agent): default Gemini Live to English while allowing language switching Update GeminiLiveProviderImpl.getSystemInstruction() so Gemini Live defaults to English for understanding and replying without locking out language switching when the user clearly speaks in or asks to switch to another language. --- .../agent/data/GeminiLiveProviderImpl.kt | 1 + .../agent/data/GeminiLiveProviderImplTest.kt | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt index b7b6af4f..a0767238 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt @@ -378,6 +378,7 @@ class GeminiLiveProviderImpl val currentDate = LocalDate.now().toString() return """ You are an AI voice assistant running on Android using Gemini Live. Today's date is $currentDate. + Default to English for understanding and replying unless the user clearly speaks in or asks to switch to another language. Always reply to the user using concise, friendly, natural spoken language. You have access to Android AppFunctions and internal device tools. Use them whenever the user asks to perform an action or look up information on their device. When a user asks you to generate an image, call the generateImage tool and then confirm to the user in a natural sentence. diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt index e3ce6077..2fe48986 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt @@ -82,7 +82,20 @@ class GeminiLiveProviderImplTest { assertTrue(setup.containsKey("inputAudioTranscription")) assertTrue(setup.containsKey("outputAudioTranscription")) - assertTrue(setup.containsKey("systemInstruction")) + val systemInstructionText = + setup["systemInstruction"]!! + .jsonObject["parts"]!! + .jsonArray + .first() + .jsonObject["text"]!! + .jsonPrimitive + .content + assertTrue( + systemInstructionText.contains( + "Default to English for understanding and replying unless the user clearly " + + "speaks in or asks to switch to another language.", + ), + ) val declarations = setup["tools"]!! From 267f77fb857b37a9ced21f172628bd358874e765 Mon Sep 17 00:00:00 2001 From: Julian Toledo Date: Tue, 27 Oct 2026 17:19:40 +0100 Subject: [PATCH 5/6] feat(agent): default Gemini Live voice to Zephyr and add voice selector in Settings Add GeminiLiveVoice with all 30 prebuilt Gemini Live studio voices (defaulting to Zephyr), persist the selection in DataStoreSettingsRepository, expose a Gemini Live voice dropdown in SettingsScreen, and read the selected voice when initializing GeminiLiveProviderImpl sessions. --- .../agent/data/DataStoreSettingsRepository.kt | 12 +++ .../agent/data/GeminiLiveProviderImpl.kt | 18 ++-- .../agent/data/GeminiLiveVoice.kt | 72 ++++++++++++++++ .../agent/data/SettingsRepository.kt | 6 ++ .../ui/contracts/SettingsScreenLayout.kt | 3 + .../ui/screens/agentdemo/SettingsScreen.kt | 86 +++++++++++++++++++ .../ui/screens/agentdemo/SettingsUiState.kt | 2 + .../ui/screens/agentdemo/SettingsViewModel.kt | 14 ++- agent/app/src/main/res/values/strings.xml | 3 + .../agentdemo/SettingsScreenScreenshotTest.kt | 5 ++ .../agent/data/GeminiLiveProviderImplTest.kt | 51 +++++++++++ .../agent/data/SettingsRepositoryTest.kt | 15 ++++ .../agentdemo/FakeSettingsRepository.kt | 8 ++ .../agentdemo/SettingsViewModelTest.kt | 18 ++++ 14 files changed, 306 insertions(+), 7 deletions(-) create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveVoice.kt diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt index aac24d4d..329fe35a 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt @@ -37,6 +37,7 @@ class DataStoreSettingsRepository val GEMINI_API_KEY = stringPreferencesKey("gemini_api_key") val SELECTED_PROVIDER = stringPreferencesKey("selected_provider") val SERVICE_TIER = stringPreferencesKey("service_tier") + val GEMINI_LIVE_VOICE = stringPreferencesKey("gemini_live_voice") val PINNED_APPS = stringSetPreferencesKey("pinned_apps") val DISCONNECTED_APPS = stringSetPreferencesKey("disconnected_apps") } @@ -65,6 +66,11 @@ class DataStoreSettingsRepository } ?: ServiceTier.PRIORITY } + override val geminiLiveVoice: Flow = + dataStore.data.map { preferences -> + GeminiLiveVoice.fromStoredValue(preferences[PreferencesKeys.GEMINI_LIVE_VOICE]) + } + override val pinnedApps: Flow> = dataStore.data.map { preferences -> preferences[PreferencesKeys.PINNED_APPS] ?: emptySet() } @@ -87,6 +93,12 @@ class DataStoreSettingsRepository dataStore.edit { preferences -> preferences[PreferencesKeys.SERVICE_TIER] = tier.name } } + override suspend fun setGeminiLiveVoice(voice: GeminiLiveVoice) { + dataStore.edit { preferences -> + preferences[PreferencesKeys.GEMINI_LIVE_VOICE] = voice.name + } + } + override suspend fun setAppPinned( packageName: String, pinned: Boolean, diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt index a0767238..328c0f70 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt @@ -32,6 +32,7 @@ import io.ktor.websocket.readBytes import io.ktor.websocket.readText import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.flow import kotlinx.coroutines.isActive import kotlinx.serialization.json.Json @@ -61,6 +62,7 @@ class GeminiLiveProviderImpl constructor( private val httpClient: HttpClient, private val toolConverter: GeminiToolConverter, + private val settingsRepository: SettingsRepository, ) : LiveLlmProvider { internal var webSocketConnector: suspend (wsUrl: String, block: suspend WebSocketSession.() -> Unit) -> Unit = @@ -73,10 +75,14 @@ class GeminiLiveProviderImpl tools: List, block: suspend (LiveLlmSession) -> Unit, ) { + val selectedVoice = settingsRepository.geminiLiveVoice.first() val wsUrl = "$GEMINI_LIVE_WS_URL?key=$apiKey" webSocketConnector(wsUrl) { - val setupMessage = buildSetupMessage(tools) - Log.d(TAG, "Sending Gemini Live setup for model $LIVE_MODEL_NAME") + val setupMessage = buildSetupMessage(tools = tools, voice = selectedVoice) + Log.d( + TAG, + "Sending Gemini Live setup for model $LIVE_MODEL_NAME (voice=${selectedVoice.apiName})", + ) send(Frame.Text(setupMessage.toString())) val firstFrameText = @@ -234,7 +240,10 @@ class GeminiLiveProviderImpl return events } - internal fun buildSetupMessage(tools: List): JsonObject { + internal fun buildSetupMessage( + tools: List, + voice: GeminiLiveVoice = GeminiLiveVoice.DEFAULT, + ): JsonObject { val functionDeclarations = toolConverter .selectDistinctTools(tools) @@ -268,7 +277,7 @@ class GeminiLiveProviderImpl put( "prebuiltVoiceConfig", buildJsonObject { - put("voiceName", DEFAULT_VOICE_NAME) + put("voiceName", voice.apiName) }, ) }, @@ -389,7 +398,6 @@ class GeminiLiveProviderImpl companion object { private const val TAG = "GeminiLiveProvider" const val LIVE_MODEL_NAME = "models/gemini-3.8-live" - private const val DEFAULT_VOICE_NAME = "CHARON" private const val INPUT_AUDIO_MIME_TYPE = "audio/pcm;rate=16000" private const val GEMINI_LIVE_WS_URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent" diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveVoice.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveVoice.kt new file mode 100644 index 00000000..e597c28b --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveVoice.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +/** + * Prebuilt studio voices supported by Gemini Live (`models/gemini-3.8-live`) via + * `speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName`. + */ +enum class GeminiLiveVoice( + /** Voice identifier sent in `prebuiltVoiceConfig.voiceName` and shown in Settings. */ + val apiName: String, + /** Short vocal style descriptor for the voice. */ + val style: String, +) { + ZEPHYR("Zephyr", "Bright"), + PUCK("Puck", "Upbeat"), + CHARON("Charon", "Informative"), + KORE("Kore", "Firm"), + FENRIR("Fenrir", "Excitable"), + LEDA("Leda", "Youthful"), + ORUS("Orus", "Firm"), + AOEDE("Aoede", "Breezy"), + CALLIRRHOE("Callirrhoe", "Easy-going"), + AUTONOE("Autonoe", "Bright"), + ENCELADUS("Enceladus", "Breathy"), + IAPETUS("Iapetus", "Clear"), + UMBRIEL("Umbriel", "Easy-going"), + ALGIEBA("Algieba", "Smooth"), + DESPINA("Despina", "Smooth"), + ERINOME("Erinome", "Clear"), + ALGENIB("Algenib", "Gravelly"), + RASALGETHI("Rasalgethi", "Informative"), + LAOMEDEIA("Laomedeia", "Upbeat"), + ACHERNAR("Achernar", "Soft"), + ALNILAM("Alnilam", "Firm"), + SCHEDAR("Schedar", "Even"), + GACRUX("Gacrux", "Mature"), + PULCHERRIMA("Pulcherrima", "Forward"), + ACHIRD("Achird", "Friendly"), + ZUBENELGENUBI("Zubenelgenubi", "Casual"), + VINDEMIATRIX("Vindemiatrix", "Gentle"), + SADACHBIA("Sadachbia", "Lively"), + SADALTAGER("Sadaltager", "Knowledgeable"), + SULAFAT("Sulafat", "Warm"), + ; + + companion object { + val DEFAULT: GeminiLiveVoice = ZEPHYR + + /** Resolves a persisted preference string back to a [GeminiLiveVoice], falling back to [DEFAULT]. */ + fun fromStoredValue(value: String?): GeminiLiveVoice = + value?.let { stored -> + entries.find { + it.name.equals(stored, ignoreCase = true) || + it.apiName.equals(stored, ignoreCase = true) + } + } ?: DEFAULT + } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt index 0feefb9d..92a2ca13 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt @@ -37,6 +37,12 @@ interface SettingsRepository { /** Sets the service tier requested for LLM API calls. */ suspend fun setServiceTier(tier: ServiceTier) + /** Flow of the prebuilt voice selected for Gemini Live sessions. */ + val geminiLiveVoice: Flow + + /** Sets the prebuilt voice selected for Gemini Live sessions. */ + suspend fun setGeminiLiveVoice(voice: GeminiLiveVoice) + /** Flow of pinned app package names. */ val pinnedApps: Flow> diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/contracts/SettingsScreenLayout.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/contracts/SettingsScreenLayout.kt index 103c7720..5d0047e1 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/contracts/SettingsScreenLayout.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/contracts/SettingsScreenLayout.kt @@ -18,6 +18,7 @@ package com.example.appfunctions.agent.ui.contracts import androidx.compose.foundation.text.input.TextFieldState import androidx.compose.runtime.Composable import androidx.compose.ui.Modifier +import com.example.appfunctions.agent.data.GeminiLiveVoice import com.example.appfunctions.agent.data.ServiceTier /** @@ -33,5 +34,7 @@ interface SettingsScreenLayout { onOpenLicenses: () -> Unit, onNavigateToConnectedApps: () -> Unit, modifier: Modifier, + geminiLiveVoice: GeminiLiveVoice = GeminiLiveVoice.DEFAULT, + onGeminiLiveVoiceSelected: (GeminiLiveVoice) -> Unit = {}, ) } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreen.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreen.kt index b10ad052..63dbf6ab 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreen.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreen.kt @@ -23,6 +23,7 @@ import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.consumeWindowInsets import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.heightIn import androidx.compose.foundation.layout.imePadding import androidx.compose.foundation.layout.padding import androidx.compose.foundation.rememberScrollState @@ -70,6 +71,7 @@ import androidx.hilt.navigation.compose.hiltViewModel import androidx.lifecycle.compose.collectAsStateWithLifecycle import com.example.appfunctions.agent.BuildConfig import com.example.appfunctions.agent.R +import com.example.appfunctions.agent.data.GeminiLiveVoice import com.example.appfunctions.agent.data.ServiceTier import com.google.android.gms.oss.licenses.v2.OssLicensesMenuActivity import kotlin.OptIn @@ -93,6 +95,8 @@ fun SettingsScreen( geminiApiKeyState = viewModel.geminiApiKeyState, serviceTier = uiState.serviceTier, onServiceTierSelected = viewModel::setServiceTier, + geminiLiveVoice = uiState.geminiLiveVoice, + onGeminiLiveVoiceSelected = viewModel::setGeminiLiveVoice, onOpenLicenses = onOpenLicensesClick, onNavigateToConnectedApps = onNavigateToConnectedApps, ) @@ -107,6 +111,8 @@ fun SettingsScreenContent( onServiceTierSelected: (ServiceTier) -> Unit, onOpenLicenses: () -> Unit, onNavigateToConnectedApps: () -> Unit, + geminiLiveVoice: GeminiLiveVoice = GeminiLiveVoice.DEFAULT, + onGeminiLiveVoiceSelected: (GeminiLiveVoice) -> Unit = {}, ) { Scaffold( modifier = Modifier.fillMaxSize(), @@ -205,6 +211,24 @@ fun SettingsScreenContent( ) } + Column(modifier = Modifier.padding(horizontal = 16.dp, vertical = 8.dp)) { + Text( + text = stringResource(id = R.string.settings_live_voice), + style = MaterialTheme.typography.bodyMedium, + modifier = Modifier.padding(bottom = 8.dp), + ) + GeminiLiveVoiceDropdown( + selectedVoice = geminiLiveVoice, + onVoiceSelected = onGeminiLiveVoiceSelected, + ) + Text( + text = stringResource(id = R.string.settings_live_voice_description), + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(top = 4.dp), + ) + } + ListItem( headlineContent = { Text( @@ -325,3 +349,65 @@ private fun ServiceTier.labelRes(): Int = ServiceTier.STANDARD -> R.string.settings_service_tier_standard ServiceTier.PRIORITY -> R.string.settings_service_tier_priority } + +/** Dropdown allowing the user to pick the [GeminiLiveVoice] for Gemini Live voice sessions. */ +@OptIn(ExperimentalMaterial3Api::class) +@Composable +private fun GeminiLiveVoiceDropdown( + selectedVoice: GeminiLiveVoice, + onVoiceSelected: (GeminiLiveVoice) -> Unit, +) { + var expanded by remember { mutableStateOf(false) } + ExposedDropdownMenuBox( + expanded = expanded, + onExpandedChange = { expanded = it }, + ) { + OutlinedTextField( + value = + stringResource( + id = R.string.settings_live_voice_option, + selectedVoice.apiName, + selectedVoice.style, + ), + onValueChange = {}, + readOnly = true, + singleLine = true, + trailingIcon = { + ExposedDropdownMenuDefaults.TrailingIcon(expanded = expanded) + }, + shape = CircleShape, + colors = + ExposedDropdownMenuDefaults.outlinedTextFieldColors( + unfocusedContainerColor = MaterialTheme.colorScheme.surfaceBright, + focusedContainerColor = MaterialTheme.colorScheme.surfaceBright, + ), + modifier = + Modifier.fillMaxWidth() + .menuAnchor(ExposedDropdownMenuAnchorType.PrimaryNotEditable, enabled = true), + ) + ExposedDropdownMenu( + expanded = expanded, + onDismissRequest = { expanded = false }, + modifier = Modifier.heightIn(max = 320.dp), + ) { + GeminiLiveVoice.entries.forEach { voice -> + DropdownMenuItem( + text = { + Text( + text = + stringResource( + id = R.string.settings_live_voice_option, + voice.apiName, + voice.style, + ), + ) + }, + onClick = { + onVoiceSelected(voice) + expanded = false + }, + ) + } + } + } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsUiState.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsUiState.kt index 3c6bada4..60dfaba6 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsUiState.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsUiState.kt @@ -15,6 +15,7 @@ */ package com.example.appfunctions.agent.ui.screens.agentdemo +import com.example.appfunctions.agent.data.GeminiLiveVoice import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.ServiceTier @@ -22,4 +23,5 @@ import com.example.appfunctions.agent.data.ServiceTier data class SettingsUiState( val selectedProvider: LlmProviderName = LlmProviderName.GEMINI, val serviceTier: ServiceTier = ServiceTier.STANDARD, + val geminiLiveVoice: GeminiLiveVoice = GeminiLiveVoice.DEFAULT, ) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModel.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModel.kt index b9092e69..9c372f5e 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModel.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModel.kt @@ -19,6 +19,7 @@ import androidx.compose.foundation.text.input.TextFieldState import androidx.compose.foundation.text.input.setTextAndPlaceCursorAtEnd import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope +import com.example.appfunctions.agent.data.GeminiLiveVoice import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.ServiceTier import com.example.appfunctions.agent.data.SettingsRepository @@ -46,8 +47,13 @@ class SettingsViewModel combine( settingsRepository.selectedProvider, settingsRepository.serviceTier, - ) { provider, serviceTier -> - SettingsUiState(selectedProvider = provider, serviceTier = serviceTier) + settingsRepository.geminiLiveVoice, + ) { provider, serviceTier, geminiLiveVoice -> + SettingsUiState( + selectedProvider = provider, + serviceTier = serviceTier, + geminiLiveVoice = geminiLiveVoice, + ) } .stateIn( scope = viewModelScope, @@ -79,4 +85,8 @@ class SettingsViewModel fun setServiceTier(tier: ServiceTier) { viewModelScope.launch { settingsRepository.setServiceTier(tier) } } + + fun setGeminiLiveVoice(voice: GeminiLiveVoice) { + viewModelScope.launch { settingsRepository.setGeminiLiveVoice(voice) } + } } diff --git a/agent/app/src/main/res/values/strings.xml b/agent/app/src/main/res/values/strings.xml index 5193e2ed..bd33f2bc 100644 --- a/agent/app/src/main/res/values/strings.xml +++ b/agent/app/src/main/res/values/strings.xml @@ -88,6 +88,9 @@ Priority lowers response latency but requires Tier 2+ billing and costs more. Standard Priority + Gemini Live voice + Spoken voice used for Gemini Live sessions. Changes apply to new sessions. + %1$s (%2$s) Preferred Provider Gemini Open source licenses diff --git a/agent/app/src/screenshotTest/kotlin/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreenScreenshotTest.kt b/agent/app/src/screenshotTest/kotlin/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreenScreenshotTest.kt index f06d860d..fa38d946 100644 --- a/agent/app/src/screenshotTest/kotlin/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreenScreenshotTest.kt +++ b/agent/app/src/screenshotTest/kotlin/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreenScreenshotTest.kt @@ -19,6 +19,7 @@ import androidx.compose.foundation.text.input.rememberTextFieldState import androidx.compose.runtime.Composable import androidx.compose.ui.tooling.preview.Preview import com.android.tools.screenshot.PreviewTest +import com.example.appfunctions.agent.data.GeminiLiveVoice import com.example.appfunctions.agent.data.ServiceTier /** Screenshot test for [SettingsScreen]. */ @@ -32,6 +33,10 @@ fun SettingsScreenScreenshotPreview() { onServiceTierSelected = { // no-op }, + geminiLiveVoice = GeminiLiveVoice.DEFAULT, + onGeminiLiveVoiceSelected = { + // no-op + }, onOpenLicenses = { // no-op }, diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt index 2fe48986..8e28c500 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt @@ -33,6 +33,7 @@ import io.mockk.mockk import io.mockk.mockkStatic import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.flow.toList import kotlinx.coroutines.test.runTest import kotlinx.serialization.json.Json @@ -48,6 +49,7 @@ import org.junit.Test import java.util.Base64 class GeminiLiveProviderImplTest { + private lateinit var settingsRepository: SettingsRepository private lateinit var provider: GeminiLiveProviderImpl @Before @@ -57,10 +59,14 @@ class GeminiLiveProviderImplTest { every { Log.w(any(), any()) } returns 0 every { Log.e(any(), any(), any()) } returns 0 + settingsRepository = mockk(relaxed = true) + every { settingsRepository.geminiLiveVoice } returns flowOf(GeminiLiveVoice.DEFAULT) + provider = GeminiLiveProviderImpl( httpClient = mockk(relaxed = true), toolConverter = GeminiToolConverter(), + settingsRepository = settingsRepository, ) } @@ -80,6 +86,16 @@ class GeminiLiveProviderImplTest { .map { it.jsonPrimitive.content } assertEquals(listOf("AUDIO"), modalities) + val voiceName = + setup["generationConfig"]!! + .jsonObject["speechConfig"]!! + .jsonObject["voiceConfig"]!! + .jsonObject["prebuiltVoiceConfig"]!! + .jsonObject["voiceName"]!! + .jsonPrimitive + .content + assertEquals("Zephyr", voiceName) + assertTrue(setup.containsKey("inputAudioTranscription")) assertTrue(setup.containsKey("outputAudioTranscription")) val systemInstructionText = @@ -130,6 +146,41 @@ class GeminiLiveProviderImplTest { } } + @Test + fun `connect sends selected GeminiLiveVoice from SettingsRepository in setup message`() = + runTest { + every { settingsRepository.geminiLiveVoice } returns flowOf(GeminiLiveVoice.AOEDE) + + val incomingFrames = Channel(Channel.UNLIMITED) + incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}""")) + incomingFrames.close() + + val sentFrames = mutableListOf() + provider.webSocketConnector = { _, block -> + coroutineScope { + createMockWebSocketSession( + context = coroutineContext, + incomingFrames = incomingFrames, + onSendText = { sentFrames.add(it) }, + ).block() + } + } + + provider.connect(apiKey = "key-123", tools = emptyList()) {} + + val setupVoiceName = + Json.parseToJsonElement(sentFrames.first()) + .jsonObject["setup"]!! + .jsonObject["generationConfig"]!! + .jsonObject["speechConfig"]!! + .jsonObject["voiceConfig"]!! + .jsonObject["prebuiltVoiceConfig"]!! + .jsonObject["voiceName"]!! + .jsonPrimitive + .content + assertEquals("Aoede", setupVoiceName) + } + @Test fun `connect exchanges realtimeInput audio, parses server events, and sends mapped toolResponse`() = runTest { diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt index ea05a1fb..b18b913c 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt @@ -77,4 +77,19 @@ class SettingsRepositoryTest { val value = repository.serviceTier.first() assertEquals(ServiceTier.PRIORITY, value) } + + @Test + fun geminiLiveVoice_defaultsToZephyr() = + runTest { + val value = repository.geminiLiveVoice.first() + assertEquals(GeminiLiveVoice.ZEPHYR, value) + } + + @Test + fun setGeminiLiveVoice_persistsValue() = + runTest { + repository.setGeminiLiveVoice(GeminiLiveVoice.AOEDE) + val value = repository.geminiLiveVoice.first() + assertEquals(GeminiLiveVoice.AOEDE, value) + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt index 7ea3b0be..e36b55b0 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt @@ -15,6 +15,7 @@ */ package com.example.appfunctions.agent.ui.screens.agentdemo +import com.example.appfunctions.agent.data.GeminiLiveVoice import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.ServiceTier import com.example.appfunctions.agent.data.SettingsRepository @@ -31,6 +32,9 @@ class FakeSettingsRepository : SettingsRepository { private val _serviceTier = MutableStateFlow(ServiceTier.STANDARD) override val serviceTier: Flow = _serviceTier + private val _geminiLiveVoice = MutableStateFlow(GeminiLiveVoice.DEFAULT) + override val geminiLiveVoice: Flow = _geminiLiveVoice + private val _pinnedApps = MutableStateFlow>(emptySet()) override val pinnedApps: Flow> = _pinnedApps @@ -46,6 +50,10 @@ class FakeSettingsRepository : SettingsRepository { _serviceTier.value = tier } + override suspend fun setGeminiLiveVoice(voice: GeminiLiveVoice) { + _geminiLiveVoice.value = voice + } + override suspend fun setAppPinned( packageName: String, pinned: Boolean, diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModelTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModelTest.kt index d0b176a5..d9a33ea4 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModelTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModelTest.kt @@ -16,6 +16,7 @@ package com.example.appfunctions.agent.ui.screens.agentdemo import androidx.compose.runtime.snapshots.Snapshot +import com.example.appfunctions.agent.data.GeminiLiveVoice import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.ServiceTier import kotlinx.coroutines.Dispatchers @@ -120,6 +121,23 @@ class SettingsViewModelTest { val uiState = viewModel.uiState.value assertEquals(ServiceTier.PRIORITY, uiState.serviceTier) + job.cancel() + } + + @Test + fun setGeminiLiveVoice_updatesRepository() = + runTest { + val job = backgroundScope.launch { viewModel.uiState.collect {} } + + assertEquals(GeminiLiveVoice.ZEPHYR, viewModel.uiState.value.geminiLiveVoice) + + viewModel.setGeminiLiveVoice(GeminiLiveVoice.AOEDE) + + testScheduler.advanceUntilIdle() + + val uiState = viewModel.uiState.value + assertEquals(GeminiLiveVoice.AOEDE, uiState.geminiLiveVoice) + job.cancel() } } From c6c9c2e633774c00cd05b1c8733281de3c22cba5 Mon Sep 17 00:00:00 2001 From: Julian Toledo Date: Tue, 27 Oct 2026 17:53:25 +0100 Subject: [PATCH 6/6] feat(agent): auto-launch PendingIntents in Gemini Live and open chat on overlay tap - Launch PendingIntent AppFunction results immediately via LaunchPendingIntentUseCase in LiveAgentOrchestrator instead of requiring a chat confirmation button - Update GeminiLiveProviderImpl system instruction to execute actions directly without claiming an on-screen confirmation card or button is displayed - Make LiveVoiceSessionOverlay tappable to open the active chat thread in MainActivity (singleTop) while keeping the Live voice session running --- agent/app/src/main/AndroidManifest.xml | 1 + .../appfunctions/agent/MainActivity.kt | 65 +++++++++++++++ .../agent/data/GeminiLiveProviderImpl.kt | 2 +- .../agent/domain/LiveAgentOrchestrator.kt | 45 +++++----- .../voice/AgentVoiceInteractionSession.kt | 17 ++++ .../agent/voice/LiveVoiceSessionOverlay.kt | 2 + .../agent/data/GeminiLiveProviderImplTest.kt | 8 ++ .../agent/domain/LiveAgentOrchestratorTest.kt | 83 +++++++++++++++---- .../voice/AgentVoiceInteractionSessionTest.kt | 52 ++++++++++++ 9 files changed, 238 insertions(+), 37 deletions(-) diff --git a/agent/app/src/main/AndroidManifest.xml b/agent/app/src/main/AndroidManifest.xml index b79e0dd7..3caa8f9f 100644 --- a/agent/app/src/main/AndroidManifest.xml +++ b/agent/app/src/main/AndroidManifest.xml @@ -41,6 +41,7 @@ diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/MainActivity.kt b/agent/app/src/main/java/com/example/appfunctions/agent/MainActivity.kt index 6b4a6a83..ac707c0f 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/MainActivity.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/MainActivity.kt @@ -16,8 +16,11 @@ package com.example.appfunctions.agent import android.Manifest +import android.content.Intent +import android.net.Uri import android.os.Bundle import androidx.activity.ComponentActivity +import androidx.activity.compose.LocalActivity import androidx.activity.compose.setContent import androidx.activity.enableEdgeToEdge import androidx.activity.result.contract.ActivityResultContracts @@ -36,13 +39,17 @@ import androidx.compose.material3.Scaffold import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.ui.Modifier import androidx.compose.ui.res.stringResource +import androidx.core.util.Consumer import androidx.hilt.navigation.compose.hiltViewModel import androidx.lifecycle.compose.collectAsStateWithLifecycle import androidx.navigation.NavDestination.Companion.hierarchy import androidx.navigation.NavGraph.Companion.findStartDestination +import androidx.navigation.NavHostController import androidx.navigation.NavType import androidx.navigation.compose.NavHost import androidx.navigation.compose.composable @@ -99,6 +106,8 @@ class MainActivity : ComponentActivity() { const val ROUTE_AGENT_DEMO = "agent_demo?threadId={threadId}" const val ARG_THREAD_ID = "threadId" const val DEEPLINK_URI_PATTERN = "appfunctions-agent://chat?threadId={threadId}" + const val ACTION_OPEN_CHAT = "com.example.appfunctions.agent.action.OPEN_CHAT" + const val EXTRA_OPEN_THREAD_ID = "EXTRA_OPEN_THREAD_ID" const val EXTRA_INSTRUMENTATION_ERROR = "EXTRA_INSTRUMENTATION_ERROR" } } @@ -115,6 +124,21 @@ fun MainScreen( ) { val navController = rememberNavController() val uiState by viewModel.uiState.collectAsStateWithLifecycle() + val activity = LocalActivity.current as? ComponentActivity + + LaunchedEffect(activity, navController) { + navigateToChatIfRequested(navController, activity?.intent) + } + + DisposableEffect(activity, navController) { + val listener = + Consumer { newIntent -> + activity?.intent = newIntent + navigateToChatIfRequested(navController, newIntent) + } + activity?.addOnNewIntentListener(listener) + onDispose { activity?.removeOnNewIntentListener(listener) } + } val items = listOf("debugging", "agent_demo", "settings") val icons = listOf(Icons.Default.Terminal, Icons.Default.PlayCircle, Icons.Default.Settings) @@ -215,3 +239,44 @@ fun MainScreen( } } } + +private fun navigateToChatIfRequested( + navController: NavHostController, + intent: Intent?, +) { + if (intent == null) return + val isOpenChatIntent = + intent.action == MainActivity.ACTION_OPEN_CHAT || + intent.hasExtra(MainActivity.EXTRA_OPEN_THREAD_ID) + val deepLinkUri = + intent.data?.takeIf { it.scheme == "appfunctions-agent" && it.host == "chat" } + if (!isOpenChatIntent && deepLinkUri == null) return + + val threadId = + intent.getStringExtra(MainActivity.EXTRA_OPEN_THREAD_ID) + ?: deepLinkUri?.getQueryParameter(MainActivity.ARG_THREAD_ID) + + if (intent.action == MainActivity.ACTION_OPEN_CHAT) { + intent.action = null + } + intent.removeExtra(MainActivity.EXTRA_OPEN_THREAD_ID) + + val route = + if (!threadId.isNullOrEmpty()) { + "agent_demo?threadId=${Uri.encode(threadId)}" + } else { + "agent_demo" + } + + navController.navigate(route) { + popUpTo(navController.graph.findStartDestination().id) { + saveState = true + } + launchSingleTop = true + } + if (!threadId.isNullOrEmpty()) { + navController.currentBackStackEntry + ?.savedStateHandle + ?.set(MainActivity.ARG_THREAD_ID, threadId) + } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt index 328c0f70..bee91e43 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt @@ -390,8 +390,8 @@ class GeminiLiveProviderImpl Default to English for understanding and replying unless the user clearly speaks in or asks to switch to another language. Always reply to the user using concise, friendly, natural spoken language. You have access to Android AppFunctions and internal device tools. Use them whenever the user asks to perform an action or look up information on their device. + Execute user-requested actions directly without claiming an on-screen confirmation card or button is shown; if you ever need confirmation before a sensitive action, ask the user verbally. When a user asks you to generate an image, call the generateImage tool and then confirm to the user in a natural sentence. - When a tool returns status "confirmation_required", let the user know that a confirmation button is on their screen and ask them to tap it to proceed. """.trimIndent() } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAgentOrchestrator.kt index d4f2a663..10db47a7 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAgentOrchestrator.kt @@ -26,7 +26,7 @@ import com.example.appfunctions.agent.domain.appfunction.ExecuteToolCallUseCase import com.example.appfunctions.agent.domain.appfunction.GetEnabledToolsUseCase import com.example.appfunctions.agent.domain.appfunction.ToolCallExecutionResult import com.example.appfunctions.agent.domain.chat.SendMessageUseCase -import com.example.appfunctions.agent.domain.pendingintent.SavePendingIntentUseCase +import com.example.appfunctions.agent.domain.pendingintent.LaunchPendingIntentUseCase import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope @@ -62,7 +62,7 @@ class LiveAgentOrchestrator private val settingsRepository: SettingsRepository, private val getEnabledToolsUseCase: GetEnabledToolsUseCase, private val executeToolCallUseCase: ExecuteToolCallUseCase, - private val savePendingIntentUseCase: SavePendingIntentUseCase, + private val launchPendingIntentUseCase: LaunchPendingIntentUseCase, private val sendMessageUseCase: SendMessageUseCase, private val audioSessionFactory: LiveAudioSessionFactory, ) { @@ -405,21 +405,28 @@ class LiveAgentOrchestrator } is ToolCallExecutionResult.PendingIntentRequired -> { - savePendingIntentUseCase(result.pendingIntentId, result.pendingIntent) - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = "", - processingStatus = MessageProcessingStatus.PROCESSED, - pendingIntentId = result.pendingIntentId, - ) - outputs.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = CONFIRMATION_REQUIRED_RESULT, - ), - ) + val launchResult = launchPendingIntentUseCase(result.pendingIntent) + if (launchResult.isSuccess) { + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = PENDING_INTENT_LAUNCHED_RESULT, + ), + ) + } else { + val reason = + launchResult.exceptionOrNull()?.message + ?: "PendingIntent canceled" + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = + "Tool execution failed for ${toolCall.functionId}: $reason", + ), + ) + } } is ToolCallExecutionResult.Failure -> { @@ -508,7 +515,7 @@ class LiveAgentOrchestrator companion object { private const val TAG = "LiveAgentOrchestrator" - private const val CONFIRMATION_REQUIRED_RESULT = - """{"status":"confirmation_required","message":"A confirmation button has been displayed in the chat. Ask the user to tap it to confirm."}""" + private const val PENDING_INTENT_LAUNCHED_RESULT = + """{"status":"launched","message":"The action was executed and the target app UI was opened."}""" } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSession.kt b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSession.kt index 88a84039..81ac5ce7 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSession.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSession.kt @@ -16,6 +16,7 @@ package com.example.appfunctions.agent.voice import android.content.Context +import android.content.Intent import android.graphics.Rect import android.os.Bundle import android.service.voice.VoiceInteractionSession @@ -34,6 +35,7 @@ import androidx.savedstate.SavedStateRegistry import androidx.savedstate.SavedStateRegistryController import androidx.savedstate.SavedStateRegistryOwner import androidx.savedstate.setViewTreeSavedStateRegistryOwner +import com.example.appfunctions.agent.MainActivity import com.example.appfunctions.agent.data.LlmModel import com.example.appfunctions.agent.domain.LiveAgentOrchestrator import com.example.appfunctions.agent.domain.LiveSessionState @@ -109,6 +111,7 @@ class AgentVoiceInteractionSession( agentStatus = agentStatus, onStopClick = ::stopAndHide, onRetryClick = ::retrySession, + onOpenChatClick = ::openChat, onOverlayBoundsChanged = ::updateOverlayBounds, ) } @@ -214,6 +217,20 @@ class AgentVoiceInteractionSession( ) } + internal fun openChat() { + val intent = + Intent(context, MainActivity::class.java).apply { + action = MainActivity.ACTION_OPEN_CHAT + addFlags( + Intent.FLAG_ACTIVITY_NEW_TASK or + Intent.FLAG_ACTIVITY_SINGLE_TOP or + Intent.FLAG_ACTIVITY_CLEAR_TOP, + ) + currentThreadId?.let { putExtra(MainActivity.EXTRA_OPEN_THREAD_ID, it) } + } + context.startActivity(intent) + } + internal fun updateOverlayBounds(bounds: Rect) { if (overlayBounds != bounds) { overlayBounds.set(bounds) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/voice/LiveVoiceSessionOverlay.kt b/agent/app/src/main/java/com/example/appfunctions/agent/voice/LiveVoiceSessionOverlay.kt index 57b3ffa6..2d8b0b7d 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/voice/LiveVoiceSessionOverlay.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/voice/LiveVoiceSessionOverlay.kt @@ -87,6 +87,7 @@ fun LiveVoiceSessionOverlay( agentStatus: AgentStatus, onStopClick: () -> Unit, onRetryClick: () -> Unit, + onOpenChatClick: () -> Unit = {}, onOverlayBoundsChanged: (Rect) -> Unit = {}, ) { val context = LocalContext.current @@ -97,6 +98,7 @@ fun LiveVoiceSessionOverlay( contentAlignment = Alignment.BottomCenter, ) { Surface( + onClick = onOpenChatClick, modifier = Modifier .navigationBarsPadding() diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt index 8e28c500..eaebf3ad 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt @@ -42,6 +42,7 @@ import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive import org.junit.Assert.assertArrayEquals import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Assert.fail import org.junit.Before @@ -112,6 +113,13 @@ class GeminiLiveProviderImplTest { "speaks in or asks to switch to another language.", ), ) + assertTrue( + systemInstructionText.contains( + "Execute user-requested actions directly without claiming an on-screen " + + "confirmation card or button is shown", + ), + ) + assertFalse(systemInstructionText.contains("confirmation_required")) val declarations = setup["tools"]!! diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/LiveAgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/LiveAgentOrchestratorTest.kt index f17f1f0a..b4ab1e4f 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/LiveAgentOrchestratorTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/LiveAgentOrchestratorTest.kt @@ -31,7 +31,7 @@ import com.example.appfunctions.agent.domain.appfunction.ExecuteToolCallUseCase import com.example.appfunctions.agent.domain.appfunction.GetEnabledToolsUseCase import com.example.appfunctions.agent.domain.appfunction.ToolCallExecutionResult import com.example.appfunctions.agent.domain.chat.SendMessageUseCase -import com.example.appfunctions.agent.domain.pendingintent.SavePendingIntentUseCase +import com.example.appfunctions.agent.domain.pendingintent.LaunchPendingIntentUseCase import io.mockk.coEvery import io.mockk.coVerify import io.mockk.coVerifyOrder @@ -68,7 +68,7 @@ class LiveAgentOrchestratorTest { private lateinit var settingsRepository: SettingsRepository private lateinit var getEnabledToolsUseCase: GetEnabledToolsUseCase private lateinit var executeToolCallUseCase: ExecuteToolCallUseCase - private lateinit var savePendingIntentUseCase: SavePendingIntentUseCase + private lateinit var launchPendingIntentUseCase: LaunchPendingIntentUseCase private lateinit var sendMessageUseCase: SendMessageUseCase private var currentAudioFactory: () -> LiveAudioSession = { mockk(relaxed = true) } @@ -86,9 +86,10 @@ class LiveAgentOrchestratorTest { settingsRepository = mockk(relaxed = true) getEnabledToolsUseCase = mockk() executeToolCallUseCase = mockk() - savePendingIntentUseCase = mockk(relaxed = true) + launchPendingIntentUseCase = mockk() sendMessageUseCase = mockk(relaxed = true) + every { launchPendingIntentUseCase(any()) } returns Result.success(Unit) every { settingsRepository.selectedProvider } returns flowOf(LlmProviderName.GEMINI) every { settingsRepository.geminiApiKey } returns flowOf("test-api-key") every { llmProviderFactory.getLiveProvider(LlmProviderName.GEMINI) } returns liveLlmProvider @@ -100,7 +101,7 @@ class LiveAgentOrchestratorTest { settingsRepository = settingsRepository, getEnabledToolsUseCase = getEnabledToolsUseCase, executeToolCallUseCase = executeToolCallUseCase, - savePendingIntentUseCase = savePendingIntentUseCase, + launchPendingIntentUseCase = launchPendingIntentUseCase, sendMessageUseCase = sendMessageUseCase, audioSessionFactory = LiveAudioSessionFactory { currentAudioFactory() }, ) @@ -275,7 +276,7 @@ class LiveAgentOrchestratorTest { } @Test - fun `ToolCallRequest gates recording, updates status, saves PendingIntent and attachments, and sends ToolOutputs`() = + fun `ToolCallRequest gates recording, updates status, launches PendingIntent directly, saves attachments, and sends ToolOutputs`() = runTest { val ioDispatcher = StandardTestDispatcher(testScheduler) orchestrator.ioDispatcher = ioDispatcher @@ -290,10 +291,10 @@ class LiveAgentOrchestratorTest { arguments = mapOf("title" to "Shopping"), callId = "call-1", ) - val confirmCall = + val callAction = LlmResponsePart.ToolCall( packageName = "com.example.notes", - functionId = "deleteAll", + functionId = "makeCall", arguments = emptyMap(), callId = "call-2", ) @@ -310,7 +311,7 @@ class LiveAgentOrchestratorTest { ) } coEvery { - executeToolCallUseCase(confirmCall, listOf(noteTool), "thread-1") + executeToolCallUseCase(callAction, listOf(noteTool), "thread-1") } answers { observedStatuses.add(orchestrator.status.value) ToolCallExecutionResult.PendingIntentRequired("pi-123", pendingIntent) @@ -323,7 +324,7 @@ class LiveAgentOrchestratorTest { currentAudioFactory = { mockAudio } val eventChannel = Channel(Channel.UNLIMITED) - eventChannel.trySend(LiveLlmEvent.ToolCallRequest(listOf(noteCall, confirmCall))) + eventChannel.trySend(LiveLlmEvent.ToolCallRequest(listOf(noteCall, callAction))) val toolResponseSent = CompletableDeferred() val fakeSession = stubLiveConnection( @@ -342,7 +343,7 @@ class LiveAgentOrchestratorTest { assertEquals( listOf( AgentStatus.InvokingTool(noteTool.id, "com.example.notes"), - AgentStatus.InvokingTool("deleteAll", "com.example.notes"), + AgentStatus.InvokingTool("makeCall", "com.example.notes"), ), observedStatuses, ) @@ -352,15 +353,17 @@ class LiveAgentOrchestratorTest { mockAudio.resumeRecording() mockAudio.release() } - verify { savePendingIntentUseCase("pi-123", pendingIntent) } - coVerify { + verify(exactly = 1) { launchPendingIntentUseCase(pendingIntent) } + coVerify(exactly = 0) { sendMessageUseCase( - threadId = "thread-1", - role = MessageRole.ASSISTANT, - textContent = "", - processingStatus = MessageProcessingStatus.PROCESSED, + threadId = any(), + role = any(), + textContent = any(), + processingStatus = any(), pendingIntentId = "pi-123", ) + } + coVerify { sendMessageUseCase( threadId = "thread-1", role = MessageRole.ASSISTANT, @@ -375,7 +378,53 @@ class LiveAgentOrchestratorTest { fakeSession.sentToolOutputs .single()[1] .result - .contains("confirmation_required"), + .contains("\"status\":\"launched\""), + ) + } + + @Test + fun `ToolCallRequest reports failure ToolOutput when PendingIntent launch fails`() = + runTest { + val ioDispatcher = StandardTestDispatcher(testScheduler) + orchestrator.ioDispatcher = ioDispatcher + + val pendingIntent = mockk() + every { launchPendingIntentUseCase(pendingIntent) } returns + Result.failure(PendingIntent.CanceledException()) + + val callAction = + LlmResponsePart.ToolCall( + packageName = "com.example.chat", + functionId = "makeCall", + arguments = emptyMap(), + callId = "call-err", + ) + coEvery { executeToolCallUseCase(callAction, emptyList(), "thread-1") } returns + ToolCallExecutionResult.PendingIntentRequired("pi-err", pendingIntent) + + val mockAudio = + mockk(relaxed = true).apply { + every { listenToRecording() } returns emptyFlow() + } + currentAudioFactory = { mockAudio } + + val eventChannel = Channel(Channel.UNLIMITED) + eventChannel.trySend(LiveLlmEvent.ToolCallRequest(listOf(callAction))) + val fakeSession = + stubLiveConnection( + events = eventChannel.receiveAsFlow(), + onToolResponses = { eventChannel.close() }, + ) + + orchestrator.startSession("thread-1") + advanceUntilIdle() + + assertEquals(1, fakeSession.sentToolOutputs.single().size) + assertTrue( + fakeSession.sentToolOutputs + .single()[0] + .result + .contains("Tool execution failed for makeCall: PendingIntent canceled"), ) } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionTest.kt index a1f0efa3..d290dfa1 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionTest.kt @@ -15,11 +15,14 @@ */ package com.example.appfunctions.agent.voice +import android.app.Application import android.content.Context +import android.content.Intent import android.graphics.Rect import android.os.Looper import android.service.voice.VoiceInteractionSession import androidx.test.core.app.ApplicationProvider +import com.example.appfunctions.agent.MainActivity import com.example.appfunctions.agent.data.LlmModel import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentStatus @@ -42,6 +45,8 @@ import kotlinx.coroutines.test.advanceUntilIdle import kotlinx.coroutines.test.runTest import org.junit.After import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Test import org.junit.runner.RunWith @@ -93,6 +98,9 @@ class AgentVoiceInteractionSessionTest { ), ) every { session.hide() } just Runs + every { liveAgentOrchestrator.startSession(any(), any()) } answers { + liveStateFlow.value = LiveSessionState.Listening + } val args = AgentVoiceInteractionService.createSessionArgs( @@ -251,6 +259,50 @@ class AgentVoiceInteractionSessionTest { session.onHide() verify(exactly = 2) { liveAgentOrchestrator.stopSession() } + session.onDestroy() + } + + @Test + fun `openChat launches MainActivity with active threadId while keeping Live session running`() = + runTest { + val testDispatcher = StandardTestDispatcher(testScheduler) + val session = + spyk( + AgentVoiceInteractionSession( + context = context, + liveAgentOrchestrator = liveAgentOrchestrator, + manageThreadsUseCase = manageThreadsUseCase, + mainDispatcher = testDispatcher, + ), + ) + every { session.hide() } just Runs + every { liveAgentOrchestrator.startSession(any(), any()) } answers { + liveStateFlow.value = LiveSessionState.Listening + } + + val args = + AgentVoiceInteractionService.createSessionArgs( + threadId = "thread-live-42", + targetPackageName = null, + ) + session.onShow(args, 0) + advanceUntilIdle() + + session.openChat() + + val startedIntent = Shadows.shadowOf(context as Application).nextStartedActivity + assertNotNull(startedIntent) + assertEquals(MainActivity::class.java.name, startedIntent.component?.className) + assertEquals(MainActivity.ACTION_OPEN_CHAT, startedIntent.action) + assertEquals( + "thread-live-42", + startedIntent.getStringExtra(MainActivity.EXTRA_OPEN_THREAD_ID), + ) + assertTrue(startedIntent.flags and Intent.FLAG_ACTIVITY_NEW_TASK != 0) + assertTrue(startedIntent.flags and Intent.FLAG_ACTIVITY_SINGLE_TOP != 0) + verify(exactly = 0) { liveAgentOrchestrator.stopSession() } + verify(exactly = 0) { session.hide() } + session.onDestroy() } }