diff --git a/agent/app/src/main/AndroidManifest.xml b/agent/app/src/main/AndroidManifest.xml
index 7e327172..3caa8f9f 100644
--- a/agent/app/src/main/AndroidManifest.xml
+++ b/agent/app/src/main/AndroidManifest.xml
@@ -22,6 +22,8 @@
+
+
@@ -39,6 +41,7 @@
@@ -50,6 +53,14 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
{ newIntent ->
+ activity?.intent = newIntent
+ navigateToChatIfRequested(navController, newIntent)
+ }
+ activity?.addOnNewIntentListener(listener)
+ onDispose { activity?.removeOnNewIntentListener(listener) }
+ }
val items = listOf("debugging", "agent_demo", "settings")
val icons = listOf(Icons.Default.Terminal, Icons.Default.PlayCircle, Icons.Default.Settings)
@@ -215,3 +239,44 @@ fun MainScreen(
}
}
}
+
+private fun navigateToChatIfRequested(
+ navController: NavHostController,
+ intent: Intent?,
+) {
+ if (intent == null) return
+ val isOpenChatIntent =
+ intent.action == MainActivity.ACTION_OPEN_CHAT ||
+ intent.hasExtra(MainActivity.EXTRA_OPEN_THREAD_ID)
+ val deepLinkUri =
+ intent.data?.takeIf { it.scheme == "appfunctions-agent" && it.host == "chat" }
+ if (!isOpenChatIntent && deepLinkUri == null) return
+
+ val threadId =
+ intent.getStringExtra(MainActivity.EXTRA_OPEN_THREAD_ID)
+ ?: deepLinkUri?.getQueryParameter(MainActivity.ARG_THREAD_ID)
+
+ if (intent.action == MainActivity.ACTION_OPEN_CHAT) {
+ intent.action = null
+ }
+ intent.removeExtra(MainActivity.EXTRA_OPEN_THREAD_ID)
+
+ val route =
+ if (!threadId.isNullOrEmpty()) {
+ "agent_demo?threadId=${Uri.encode(threadId)}"
+ } else {
+ "agent_demo"
+ }
+
+ navController.navigate(route) {
+ popUpTo(navController.graph.findStartDestination().id) {
+ saveState = true
+ }
+ launchSingleTop = true
+ }
+ if (!threadId.isNullOrEmpty()) {
+ navController.currentBackStackEntry
+ ?.savedStateHandle
+ ?.set(MainActivity.ARG_THREAD_ID, threadId)
+ }
+}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt
index aac24d4d..329fe35a 100644
--- a/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/DataStoreSettingsRepository.kt
@@ -37,6 +37,7 @@ class DataStoreSettingsRepository
val GEMINI_API_KEY = stringPreferencesKey("gemini_api_key")
val SELECTED_PROVIDER = stringPreferencesKey("selected_provider")
val SERVICE_TIER = stringPreferencesKey("service_tier")
+ val GEMINI_LIVE_VOICE = stringPreferencesKey("gemini_live_voice")
val PINNED_APPS = stringSetPreferencesKey("pinned_apps")
val DISCONNECTED_APPS = stringSetPreferencesKey("disconnected_apps")
}
@@ -65,6 +66,11 @@ class DataStoreSettingsRepository
} ?: ServiceTier.PRIORITY
}
+ override val geminiLiveVoice: Flow =
+ dataStore.data.map { preferences ->
+ GeminiLiveVoice.fromStoredValue(preferences[PreferencesKeys.GEMINI_LIVE_VOICE])
+ }
+
override val pinnedApps: Flow> =
dataStore.data.map { preferences -> preferences[PreferencesKeys.PINNED_APPS] ?: emptySet() }
@@ -87,6 +93,12 @@ class DataStoreSettingsRepository
dataStore.edit { preferences -> preferences[PreferencesKeys.SERVICE_TIER] = tier.name }
}
+ override suspend fun setGeminiLiveVoice(voice: GeminiLiveVoice) {
+ dataStore.edit { preferences ->
+ preferences[PreferencesKeys.GEMINI_LIVE_VOICE] = voice.name
+ }
+ }
+
override suspend fun setAppPinned(
packageName: String,
pinned: Boolean,
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiJsonUtils.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiJsonUtils.kt
new file mode 100644
index 00000000..22ad48b9
--- /dev/null
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiJsonUtils.kt
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.data
+
+import kotlinx.serialization.json.JsonArray
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.JsonNull
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.JsonPrimitive
+
+/** Converts a [JsonElement] recursively into standard Kotlin primitive, [Map], or [List] values. */
+internal fun JsonElement.toPrimitive(): Any? {
+ return when (this) {
+ JsonNull -> null
+ is JsonPrimitive -> {
+ if (this.isString) return this.content
+ return this.content.toBooleanStrictOrNull()
+ ?: this.content.toLongOrNull()
+ ?: this.content.toDoubleOrNull()
+ }
+ is JsonObject -> this.mapValues { it.value.toPrimitive() }
+ is JsonArray -> this.map { it.toPrimitive() }
+ }
+}
+
+/** Converts a nullable JSON arguments object into a Kotlin arguments map. */
+internal fun JsonObject?.toArgumentsMap(): Map {
+ if (this == null) return emptyMap()
+ return mapValues { (_, value) -> value.toPrimitive() }
+}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelper.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelper.kt
new file mode 100644
index 00000000..ac828a74
--- /dev/null
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelper.kt
@@ -0,0 +1,489 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.data
+
+import android.annotation.SuppressLint
+import android.media.AudioAttributes
+import android.media.AudioDeviceInfo
+import android.media.AudioFocusRequest
+import android.media.AudioFormat
+import android.media.AudioManager
+import android.media.AudioRecord
+import android.media.AudioTrack
+import android.media.MediaRecorder
+import android.media.audiofx.AcousticEchoCanceler
+import android.media.audiofx.NoiseSuppressor
+import android.util.Log
+import com.example.appfunctions.agent.domain.LiveAudioSession
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.cancel
+import kotlinx.coroutines.currentCoroutineContext
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.flow
+import kotlinx.coroutines.flow.flowOn
+import kotlinx.coroutines.isActive
+import kotlinx.coroutines.launch
+import java.util.concurrent.ConcurrentLinkedQueue
+import java.util.concurrent.atomic.AtomicBoolean
+
+/**
+ * Manages low-latency PCM audio recording (16 kHz mono 16-bit) and playback (24 kHz mono 16-bit)
+ * for a Gemini Live bidirectional audio session.
+ */
+class GeminiLiveAudioHelper(
+ private val audioManager: AudioManager? = null,
+) : LiveAudioSession {
+ private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob())
+ private val stateLock = Any()
+ private val isReleased = AtomicBoolean(false)
+ private val isRecordingPaused = AtomicBoolean(false)
+ private val isFocusLost = AtomicBoolean(false)
+ private val flushRequested = AtomicBoolean(false)
+
+ @Volatile private var audioRecord: AudioRecord? = null
+ private var echoCanceler: AcousticEchoCanceler? = null
+ private var noiseSuppressor: NoiseSuppressor? = null
+
+ @Volatile private var audioTrack: AudioTrack? = null
+ private var audioFocusRequest: AudioFocusRequest? = null
+ private var previousAudioMode: Int? = null
+ private var didSetCommunicationDevice: Boolean = false
+ private var playbackJob: Job? = null
+
+ private val audioQueue = ConcurrentLinkedQueue()
+
+ @SuppressLint("MissingPermission")
+ internal var audioRecordFactory: (minBufferSize: Int) -> AudioRecord = { minRecordBuf ->
+ AudioRecord(
+ MediaRecorder.AudioSource.VOICE_COMMUNICATION,
+ INPUT_SAMPLE_RATE,
+ AudioFormat.CHANNEL_IN_MONO,
+ AudioFormat.ENCODING_PCM_16BIT,
+ minRecordBuf,
+ )
+ }
+
+ internal var audioTrackFactory:
+ (attributes: AudioAttributes, minBufferSize: Int) -> AudioTrack =
+ { playbackAttributes, minTrackBuf ->
+ AudioTrack.Builder()
+ .setAudioAttributes(playbackAttributes)
+ .setAudioFormat(
+ AudioFormat.Builder()
+ .setEncoding(AudioFormat.ENCODING_PCM_16BIT)
+ .setSampleRate(OUTPUT_SAMPLE_RATE)
+ .setChannelMask(AudioFormat.CHANNEL_OUT_MONO)
+ .build(),
+ )
+ .setBufferSizeInBytes(minTrackBuf)
+ .setTransferMode(AudioTrack.MODE_STREAM)
+ .build()
+ }
+
+ private val focusChangeListener =
+ AudioManager.OnAudioFocusChangeListener { focusChange ->
+ when (focusChange) {
+ AudioManager.AUDIOFOCUS_LOSS,
+ AudioManager.AUDIOFOCUS_LOSS_TRANSIENT,
+ -> {
+ isFocusLost.set(true)
+ stopAudio()
+ }
+ AudioManager.AUDIOFOCUS_GAIN -> {
+ isFocusLost.set(false)
+ }
+ }
+ }
+
+ @SuppressLint("MissingPermission")
+ override fun start() {
+ if (isReleased.get()) return
+
+ synchronized(stateLock) {
+ if (isReleased.get() || audioRecord != null) return
+
+ var createdFocusRequest: AudioFocusRequest? = null
+ var savedPreviousMode: Int? = null
+ var commDeviceSelected = false
+ var createdRecorder: AudioRecord? = null
+ var createdEchoCanceler: AcousticEchoCanceler? = null
+ var createdNoiseSuppressor: NoiseSuppressor? = null
+ var createdTrack: AudioTrack? = null
+
+ try {
+ // Pair USAGE_VOICE_COMMUNICATION playback with VOICE_COMMUNICATION recording under
+ // MODE_IN_COMMUNICATION so the hardware VoIP DSP AcousticEchoCanceler receives the
+ // far-end speaker reference (VOIP_PLAYBACK <-> VOIP_RECORD) instead of feeding
+ // uncancelled media/assistant speaker audio back into the microphone.
+ val playbackAttributes =
+ AudioAttributes.Builder()
+ .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION)
+ .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
+ .build()
+
+ val manager = audioManager
+ if (manager != null) {
+ val focusReq =
+ AudioFocusRequest.Builder(
+ AudioManager.AUDIOFOCUS_GAIN_TRANSIENT,
+ )
+ .setAudioAttributes(playbackAttributes)
+ .setOnAudioFocusChangeListener(focusChangeListener)
+ .build()
+ val focusResult = manager.requestAudioFocus(focusReq)
+ createdFocusRequest = focusReq
+ if (focusResult != AudioManager.AUDIOFOCUS_REQUEST_GRANTED) {
+ Log.w(TAG, "Audio focus request not granted (result=$focusResult)")
+ }
+
+ runCatching {
+ savedPreviousMode = manager.mode
+ manager.mode = AudioManager.MODE_IN_COMMUNICATION
+ }.onFailure { e ->
+ Log.w(TAG, "Failed to enter MODE_IN_COMMUNICATION", e)
+ }
+ commDeviceSelected = selectSpeakerphoneIfNoExternalDevice(manager)
+ }
+
+ val minRecordBuf =
+ AudioRecord.getMinBufferSize(
+ INPUT_SAMPLE_RATE,
+ AudioFormat.CHANNEL_IN_MONO,
+ AudioFormat.ENCODING_PCM_16BIT,
+ ).coerceAtLeast(DEFAULT_READ_BUFFER_SIZE)
+
+ val recorder = audioRecordFactory(minRecordBuf)
+ createdRecorder = recorder
+ if (recorder.state != AudioRecord.STATE_INITIALIZED) {
+ throw IllegalStateException("Failed to initialize AudioRecord")
+ }
+
+ if (AcousticEchoCanceler.isAvailable()) {
+ runCatching {
+ createdEchoCanceler =
+ AcousticEchoCanceler.create(recorder.audioSessionId)?.apply {
+ enabled = true
+ }
+ }.onFailure { e ->
+ Log.w(TAG, "Failed to enable AcousticEchoCanceler", e)
+ }
+ }
+
+ if (NoiseSuppressor.isAvailable()) {
+ runCatching {
+ createdNoiseSuppressor =
+ NoiseSuppressor.create(recorder.audioSessionId)?.apply {
+ enabled = true
+ }
+ }.onFailure { e ->
+ Log.w(TAG, "Failed to enable NoiseSuppressor", e)
+ }
+ }
+
+ val minTrackBuf =
+ (
+ AudioTrack.getMinBufferSize(
+ OUTPUT_SAMPLE_RATE,
+ AudioFormat.CHANNEL_OUT_MONO,
+ AudioFormat.ENCODING_PCM_16BIT,
+ ) * 2
+ ).coerceAtLeast(DEFAULT_READ_BUFFER_SIZE)
+
+ val track = audioTrackFactory(playbackAttributes, minTrackBuf)
+ createdTrack = track
+
+ if (isReleased.get()) {
+ cleanupResources(
+ focusRequest = createdFocusRequest,
+ previousMode = savedPreviousMode,
+ clearCommDevice = commDeviceSelected,
+ echoCanceler = createdEchoCanceler,
+ noiseSuppressor = createdNoiseSuppressor,
+ recorder = createdRecorder,
+ track = createdTrack,
+ )
+ return
+ }
+
+ track.play()
+ recorder.startRecording()
+
+ audioFocusRequest = createdFocusRequest
+ previousAudioMode = savedPreviousMode
+ didSetCommunicationDevice = commDeviceSelected
+ echoCanceler = createdEchoCanceler
+ noiseSuppressor = createdNoiseSuppressor
+ audioRecord = recorder
+ audioTrack = track
+
+ playbackJob =
+ scope.launch {
+ while (isActive && !isReleased.get()) {
+ if (flushRequested.getAndSet(false)) {
+ val currentTrack = audioTrack
+ if (currentTrack != null && !isReleased.get()) {
+ runCatching {
+ if (currentTrack.playState == AudioTrack.PLAYSTATE_PLAYING) {
+ currentTrack.pause()
+ currentTrack.flush()
+ currentTrack.play()
+ }
+ }
+ }
+ }
+ val chunk = audioQueue.poll()
+ if (chunk != null) {
+ if (flushRequested.get()) {
+ continue
+ }
+ val currentTrack = audioTrack
+ if (currentTrack != null && !isReleased.get()) {
+ runCatching {
+ currentTrack.write(chunk, 0, chunk.size)
+ }
+ }
+ } else {
+ delay(PLAYBACK_POLL_DELAY_MS)
+ }
+ }
+ }
+ } catch (t: Throwable) {
+ cleanupResources(
+ focusRequest = createdFocusRequest,
+ previousMode = savedPreviousMode,
+ clearCommDevice = commDeviceSelected,
+ echoCanceler = createdEchoCanceler,
+ noiseSuppressor = createdNoiseSuppressor,
+ recorder = createdRecorder,
+ track = createdTrack,
+ )
+ throw t
+ }
+ }
+ }
+
+ /**
+ * Emits PCM 16-bit 16 kHz mono byte chunks recorded from the microphone.
+ *
+ * Keeps [AudioRecord] continuously recording across tool calls so `audioserver` does not
+ * broadcast system-wide port updates on pause/resume, while draining the HAL ring buffer and
+ * gating emission in software when paused or when audio focus is temporarily lost.
+ */
+ override fun listenToRecording(): Flow =
+ flow {
+ val buffer = ByteArray(DEFAULT_READ_BUFFER_SIZE)
+ while (currentCoroutineContext().isActive && !isReleased.get()) {
+ val recorder = audioRecord ?: break
+ val bytesRead =
+ runCatching {
+ recorder.read(buffer, 0, buffer.size)
+ }.getOrElse { e ->
+ if (!isReleased.get()) {
+ Log.w(TAG, "AudioRecord.read failed", e)
+ }
+ -1
+ }
+ if (bytesRead > 0) {
+ if (!isRecordingPaused.get() && !isFocusLost.get() && !isReleased.get()) {
+ emit(buffer.copyOf(bytesRead))
+ }
+ } else if (bytesRead < 0) {
+ if (!isReleased.get()) {
+ Log.w(TAG, "AudioRecord.read returned error code: $bytesRead")
+ }
+ break
+ } else {
+ delay(RECORD_IDLE_DELAY_MS)
+ }
+ }
+ }.flowOn(Dispatchers.IO)
+
+ override fun queueAudio(pcmBytes: ByteArray) {
+ if (!isReleased.get()) {
+ audioQueue.add(pcmBytes)
+ }
+ }
+
+ /** Clears any queued model audio and flushes the playback track when interrupted. */
+ override fun stopAudio() {
+ audioQueue.clear()
+ flushRequested.set(true)
+ }
+
+ /**
+ * Gates outgoing microphone emissions in software without stopping the underlying
+ * [AudioRecord] hardware stream.
+ */
+ override fun pauseRecording() {
+ isRecordingPaused.set(true)
+ }
+
+ /** Resumes outgoing microphone emissions after [pauseRecording]. */
+ override fun resumeRecording() {
+ if (!isReleased.get()) {
+ isRecordingPaused.set(false)
+ }
+ }
+
+ internal fun isPausedForTesting(): Boolean = isRecordingPaused.get() || isFocusLost.get()
+
+ internal fun notifyAudioFocusChangeForTesting(focusChange: Int) {
+ focusChangeListener.onAudioFocusChange(focusChange)
+ }
+
+ override fun release() {
+ if (!isReleased.compareAndSet(false, true)) return
+ audioQueue.clear()
+
+ synchronized(stateLock) {
+ playbackJob?.cancel()
+ playbackJob = null
+ scope.cancel()
+
+ val focusReq = audioFocusRequest
+ audioFocusRequest = null
+ val prevMode = previousAudioMode
+ previousAudioMode = null
+ val clearComm = didSetCommunicationDevice
+ didSetCommunicationDevice = false
+ val aec = echoCanceler
+ echoCanceler = null
+ val ns = noiseSuppressor
+ noiseSuppressor = null
+ val recorder = audioRecord
+ audioRecord = null
+ val track = audioTrack
+ audioTrack = null
+
+ cleanupResources(
+ focusRequest = focusReq,
+ previousMode = prevMode,
+ clearCommDevice = clearComm,
+ echoCanceler = aec,
+ noiseSuppressor = ns,
+ recorder = recorder,
+ track = track,
+ )
+ }
+ }
+
+ private fun selectSpeakerphoneIfNoExternalDevice(manager: AudioManager): Boolean =
+ runCatching {
+ val activeDevice = manager.communicationDevice
+ if (activeDevice != null && activeDevice.type in EXTERNAL_COMMUNICATION_DEVICE_TYPES) {
+ return@runCatching false
+ }
+ val availableDevices = manager.availableCommunicationDevices
+ if (availableDevices.any { it.type in EXTERNAL_COMMUNICATION_DEVICE_TYPES }) {
+ return@runCatching false
+ }
+ val speaker =
+ availableDevices.firstOrNull {
+ it.type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER
+ } ?: return@runCatching false
+ manager.setCommunicationDevice(speaker)
+ }.getOrElse { e ->
+ Log.w(TAG, "Failed to route communication audio to speakerphone", e)
+ false
+ }
+
+ private fun cleanupResources(
+ focusRequest: AudioFocusRequest?,
+ previousMode: Int?,
+ clearCommDevice: Boolean,
+ echoCanceler: AcousticEchoCanceler?,
+ noiseSuppressor: NoiseSuppressor?,
+ recorder: AudioRecord?,
+ track: AudioTrack?,
+ ) {
+ runCatching { echoCanceler?.release() }
+ runCatching { noiseSuppressor?.release() }
+
+ runCatching {
+ recorder?.let { rec ->
+ if (rec.state == AudioRecord.STATE_INITIALIZED &&
+ rec.recordingState == AudioRecord.RECORDSTATE_RECORDING
+ ) {
+ rec.stop()
+ }
+ rec.release()
+ }
+ }
+
+ runCatching {
+ track?.let { trk ->
+ if (trk.state == AudioTrack.STATE_INITIALIZED &&
+ trk.playState == AudioTrack.PLAYSTATE_PLAYING
+ ) {
+ trk.pause()
+ trk.flush()
+ trk.stop()
+ }
+ trk.release()
+ }
+ }
+
+ runCatching {
+ if (clearCommDevice) {
+ audioManager?.clearCommunicationDevice()
+ }
+ }
+
+ runCatching {
+ if (previousMode != null) {
+ audioManager?.mode = previousMode
+ }
+ }
+
+ runCatching {
+ if (focusRequest != null) {
+ audioManager?.abandonAudioFocusRequest(focusRequest)
+ }
+ }
+ }
+
+ companion object {
+ private const val TAG = "GeminiLiveAudioHelper"
+ private const val INPUT_SAMPLE_RATE = 16000
+ private const val OUTPUT_SAMPLE_RATE = 24000
+ private const val DEFAULT_READ_BUFFER_SIZE = 2048
+ private const val PLAYBACK_POLL_DELAY_MS = 10L
+ private const val RECORD_IDLE_DELAY_MS = 10L
+
+ private val EXTERNAL_COMMUNICATION_DEVICE_TYPES =
+ setOf(
+ AudioDeviceInfo.TYPE_WIRED_HEADSET,
+ AudioDeviceInfo.TYPE_WIRED_HEADPHONES,
+ AudioDeviceInfo.TYPE_BLUETOOTH_SCO,
+ AudioDeviceInfo.TYPE_BLUETOOTH_A2DP,
+ AudioDeviceInfo.TYPE_BLE_HEADSET,
+ AudioDeviceInfo.TYPE_BLE_SPEAKER,
+ AudioDeviceInfo.TYPE_BLE_BROADCAST,
+ AudioDeviceInfo.TYPE_USB_HEADSET,
+ AudioDeviceInfo.TYPE_USB_DEVICE,
+ AudioDeviceInfo.TYPE_USB_ACCESSORY,
+ AudioDeviceInfo.TYPE_HEARING_AID,
+ AudioDeviceInfo.TYPE_HDMI,
+ AudioDeviceInfo.TYPE_AUX_LINE,
+ AudioDeviceInfo.TYPE_LINE_ANALOG,
+ AudioDeviceInfo.TYPE_LINE_DIGITAL,
+ )
+ }
+}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt
new file mode 100644
index 00000000..bee91e43
--- /dev/null
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveProviderImpl.kt
@@ -0,0 +1,405 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.data
+
+import android.util.Log
+import androidx.appfunctions.metadata.AppFunctionMetadata
+import com.example.appfunctions.agent.domain.LiveLlmEvent
+import com.example.appfunctions.agent.domain.LiveLlmProvider
+import com.example.appfunctions.agent.domain.LiveLlmSession
+import com.example.appfunctions.agent.domain.LlmResponsePart
+import com.example.appfunctions.agent.domain.ToolOutput
+import io.ktor.client.HttpClient
+import io.ktor.client.plugins.websocket.webSocket
+import io.ktor.websocket.CloseReason
+import io.ktor.websocket.Frame
+import io.ktor.websocket.WebSocketSession
+import io.ktor.websocket.close
+import io.ktor.websocket.readBytes
+import io.ktor.websocket.readText
+import kotlinx.coroutines.currentCoroutineContext
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.first
+import kotlinx.coroutines.flow.flow
+import kotlinx.coroutines.isActive
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonArray
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.booleanOrNull
+import kotlinx.serialization.json.buildJsonArray
+import kotlinx.serialization.json.buildJsonObject
+import kotlinx.serialization.json.contentOrNull
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive
+import kotlinx.serialization.json.put
+import java.time.LocalDate
+import java.util.Base64
+import javax.inject.Inject
+import javax.inject.Singleton
+
+/**
+ * Gemini implementation of [LiveLlmProvider] over the `BidiGenerateContent` WebSocket API
+ * (`models/gemini-3.8-live`).
+ */
+@Singleton
+class GeminiLiveProviderImpl
+ @Inject
+ constructor(
+ private val httpClient: HttpClient,
+ private val toolConverter: GeminiToolConverter,
+ private val settingsRepository: SettingsRepository,
+ ) : LiveLlmProvider {
+ internal var webSocketConnector:
+ suspend (wsUrl: String, block: suspend WebSocketSession.() -> Unit) -> Unit =
+ { wsUrl, block ->
+ httpClient.webSocket(urlString = wsUrl, block = block)
+ }
+
+ override suspend fun connect(
+ apiKey: String,
+ tools: List,
+ block: suspend (LiveLlmSession) -> Unit,
+ ) {
+ val selectedVoice = settingsRepository.geminiLiveVoice.first()
+ val wsUrl = "$GEMINI_LIVE_WS_URL?key=$apiKey"
+ webSocketConnector(wsUrl) {
+ val setupMessage = buildSetupMessage(tools = tools, voice = selectedVoice)
+ Log.d(
+ TAG,
+ "Sending Gemini Live setup for model $LIVE_MODEL_NAME (voice=${selectedVoice.apiName})",
+ )
+ send(Frame.Text(setupMessage.toString()))
+
+ val firstFrameText =
+ receiveNextJsonText()
+ ?: throw IllegalStateException(
+ "WebSocket closed before setupComplete was received",
+ )
+ val firstJson = Json.parseToJsonElement(firstFrameText).jsonObject
+ if (!firstJson.containsKey("setupComplete")) {
+ throw IllegalStateException("Unexpected setup response: $firstFrameText")
+ }
+
+ val session = createSession(webSocketSession = this, tools = tools)
+ try {
+ block(session)
+ } finally {
+ runCatching {
+ close(
+ CloseReason(
+ CloseReason.Codes.NORMAL,
+ "Session ended by client",
+ ),
+ )
+ }
+ }
+ }
+ }
+
+ private fun createSession(
+ webSocketSession: WebSocketSession,
+ tools: List,
+ ): LiveLlmSession =
+ object : LiveLlmSession {
+ override val events: Flow =
+ flow {
+ while (currentCoroutineContext().isActive && webSocketSession.isActive) {
+ val messageText = webSocketSession.receiveNextJsonText() ?: break
+ val serverJson =
+ runCatching {
+ Json.parseToJsonElement(messageText).jsonObject
+ }.getOrNull() ?: continue
+
+ for (event in parseServerMessage(serverJson, tools)) {
+ emit(event)
+ }
+ }
+ }
+
+ override suspend fun sendAudio(pcmBytes: ByteArray) {
+ val realtimeInput = buildRealtimeAudioMessage(pcmBytes)
+ webSocketSession.send(Frame.Text(realtimeInput.toString()))
+ }
+
+ override suspend fun sendToolResponses(outputs: List) {
+ val toolResponseMessage = buildToolResponseMessage(outputs, tools)
+ Log.d(TAG, "Sending Gemini Live toolResponse: $toolResponseMessage")
+ webSocketSession.send(Frame.Text(toolResponseMessage.toString()))
+ }
+ }
+
+ private suspend fun WebSocketSession.receiveNextJsonText(): String? {
+ while (isActive) {
+ val frame = runCatching { incoming.receive() }.getOrNull() ?: return null
+ when (frame) {
+ is Frame.Text -> return frame.readText()
+ is Frame.Binary -> return frame.readBytes().decodeToString()
+ is Frame.Close -> return null
+ else -> continue
+ }
+ }
+ return null
+ }
+
+ internal fun parseServerMessage(
+ serverJson: JsonObject,
+ tools: List,
+ ): List {
+ val events = mutableListOf()
+
+ val serverContent = serverJson["serverContent"]?.jsonObject
+ if (serverContent != null) {
+ val interrupted =
+ serverContent["interrupted"]?.jsonPrimitive?.booleanOrNull ?: false
+ if (interrupted) {
+ events.add(LiveLlmEvent.Interrupted)
+ }
+
+ val inputTranscription =
+ serverContent["inputTranscription"]
+ ?.jsonObject
+ ?.get("text")
+ ?.jsonPrimitive
+ ?.contentOrNull
+ if (!inputTranscription.isNullOrEmpty()) {
+ events.add(LiveLlmEvent.InputTranscription(inputTranscription))
+ }
+
+ val outputTranscription =
+ serverContent["outputTranscription"]
+ ?.jsonObject
+ ?.get("text")
+ ?.jsonPrimitive
+ ?.contentOrNull
+ if (!outputTranscription.isNullOrEmpty()) {
+ events.add(LiveLlmEvent.OutputTranscription(outputTranscription))
+ }
+
+ val parts = serverContent["modelTurn"]?.jsonObject?.get("parts")?.jsonArray
+ if (parts != null) {
+ for (part in parts) {
+ val inlineData = part.jsonObject["inlineData"]?.jsonObject ?: continue
+ val mimeType =
+ inlineData["mimeType"]?.jsonPrimitive?.contentOrNull ?: ""
+ val base64Data =
+ inlineData["data"]?.jsonPrimitive?.contentOrNull ?: continue
+ if (mimeType.startsWith("audio/pcm")) {
+ val pcmBytes = Base64.getMimeDecoder().decode(base64Data)
+ events.add(LiveLlmEvent.AudioChunk(pcmBytes))
+ }
+ }
+ }
+
+ val turnComplete =
+ serverContent["turnComplete"]?.jsonPrimitive?.booleanOrNull ?: false
+ if (turnComplete) {
+ events.add(LiveLlmEvent.TurnComplete)
+ }
+ }
+
+ val toolCallObj = serverJson["toolCall"]?.jsonObject
+ if (toolCallObj != null) {
+ val functionCalls =
+ toolCallObj["functionCalls"]?.jsonArray ?: JsonArray(emptyList())
+ val parsedCalls =
+ functionCalls.mapNotNull { callElement ->
+ val callObj = callElement.jsonObject
+ val callId = callObj["id"]?.jsonPrimitive?.contentOrNull ?: ""
+ val name =
+ callObj["name"]?.jsonPrimitive?.contentOrNull ?: return@mapNotNull null
+ val matchingTool = toolConverter.findToolByName(name, tools)
+ val argumentsMap = callObj["args"]?.jsonObject.toArgumentsMap()
+
+ LlmResponsePart.ToolCall(
+ packageName = matchingTool?.packageName ?: "",
+ functionId = matchingTool?.id ?: name,
+ arguments = argumentsMap,
+ callId = callId,
+ )
+ }
+ if (parsedCalls.isNotEmpty()) {
+ events.add(LiveLlmEvent.ToolCallRequest(parsedCalls))
+ }
+ }
+
+ return events
+ }
+
+ internal fun buildSetupMessage(
+ tools: List,
+ voice: GeminiLiveVoice = GeminiLiveVoice.DEFAULT,
+ ): JsonObject {
+ val functionDeclarations =
+ toolConverter
+ .selectDistinctTools(tools)
+ .mapNotNull { tool ->
+ try {
+ toolConverter.convertForLiveApi(tool)
+ } catch (e: IllegalArgumentException) {
+ Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e)
+ null
+ }
+ }
+
+ return buildJsonObject {
+ put(
+ "setup",
+ buildJsonObject {
+ put("model", LIVE_MODEL_NAME)
+ put(
+ "generationConfig",
+ buildJsonObject {
+ put(
+ "responseModalities",
+ buildJsonArray { add(JsonPrimitive("AUDIO")) },
+ )
+ put(
+ "speechConfig",
+ buildJsonObject {
+ put(
+ "voiceConfig",
+ buildJsonObject {
+ put(
+ "prebuiltVoiceConfig",
+ buildJsonObject {
+ put("voiceName", voice.apiName)
+ },
+ )
+ },
+ )
+ },
+ )
+ },
+ )
+ put(
+ "systemInstruction",
+ buildJsonObject {
+ put(
+ "parts",
+ buildJsonArray {
+ add(
+ buildJsonObject {
+ put("text", getSystemInstruction())
+ },
+ )
+ },
+ )
+ },
+ )
+ if (functionDeclarations.isNotEmpty()) {
+ put(
+ "tools",
+ buildJsonArray {
+ add(
+ buildJsonObject {
+ put(
+ "functionDeclarations",
+ JsonArray(functionDeclarations),
+ )
+ },
+ )
+ },
+ )
+ }
+ put("inputAudioTranscription", buildJsonObject {})
+ put("outputAudioTranscription", buildJsonObject {})
+ },
+ )
+ }
+ }
+
+ internal fun buildRealtimeAudioMessage(pcmBytes: ByteArray): JsonObject {
+ val base64Audio = Base64.getEncoder().encodeToString(pcmBytes)
+ return buildJsonObject {
+ put(
+ "realtimeInput",
+ buildJsonObject {
+ put(
+ "audio",
+ buildJsonObject {
+ put("mimeType", INPUT_AUDIO_MIME_TYPE)
+ put("data", base64Audio)
+ },
+ )
+ },
+ )
+ }
+ }
+
+ internal fun buildToolResponseMessage(
+ toolOutputs: List,
+ tools: List,
+ ): JsonObject {
+ return buildJsonObject {
+ put(
+ "toolResponse",
+ buildJsonObject {
+ put(
+ "functionResponses",
+ buildJsonArray {
+ for (output in toolOutputs) {
+ val mappedName =
+ toolConverter.resolveToolName(output.functionId, tools)
+ val parsedResult =
+ runCatching {
+ Json.parseToJsonElement(output.result)
+ }.getOrElse {
+ JsonPrimitive(output.result)
+ }
+ add(
+ buildJsonObject {
+ if (output.callId.isNotEmpty()) {
+ put("id", output.callId)
+ }
+ put("name", mappedName)
+ put(
+ "response",
+ buildJsonObject {
+ put("result", parsedResult)
+ },
+ )
+ },
+ )
+ }
+ },
+ )
+ },
+ )
+ }
+ }
+
+ private fun getSystemInstruction(): String {
+ val currentDate = LocalDate.now().toString()
+ return """
+ You are an AI voice assistant running on Android using Gemini Live. Today's date is $currentDate.
+ Default to English for understanding and replying unless the user clearly speaks in or asks to switch to another language.
+ Always reply to the user using concise, friendly, natural spoken language.
+ You have access to Android AppFunctions and internal device tools. Use them whenever the user asks to perform an action or look up information on their device.
+ Execute user-requested actions directly without claiming an on-screen confirmation card or button is shown; if you ever need confirmation before a sensitive action, ask the user verbally.
+ When a user asks you to generate an image, call the generateImage tool and then confirm to the user in a natural sentence.
+ """.trimIndent()
+ }
+
+ companion object {
+ private const val TAG = "GeminiLiveProvider"
+ const val LIVE_MODEL_NAME = "models/gemini-3.8-live"
+ private const val INPUT_AUDIO_MIME_TYPE = "audio/pcm;rate=16000"
+ private const val GEMINI_LIVE_WS_URL =
+ "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent"
+ }
+ }
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveVoice.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveVoice.kt
new file mode 100644
index 00000000..e597c28b
--- /dev/null
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveVoice.kt
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.data
+
+/**
+ * Prebuilt studio voices supported by Gemini Live (`models/gemini-3.8-live`) via
+ * `speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName`.
+ */
+enum class GeminiLiveVoice(
+ /** Voice identifier sent in `prebuiltVoiceConfig.voiceName` and shown in Settings. */
+ val apiName: String,
+ /** Short vocal style descriptor for the voice. */
+ val style: String,
+) {
+ ZEPHYR("Zephyr", "Bright"),
+ PUCK("Puck", "Upbeat"),
+ CHARON("Charon", "Informative"),
+ KORE("Kore", "Firm"),
+ FENRIR("Fenrir", "Excitable"),
+ LEDA("Leda", "Youthful"),
+ ORUS("Orus", "Firm"),
+ AOEDE("Aoede", "Breezy"),
+ CALLIRRHOE("Callirrhoe", "Easy-going"),
+ AUTONOE("Autonoe", "Bright"),
+ ENCELADUS("Enceladus", "Breathy"),
+ IAPETUS("Iapetus", "Clear"),
+ UMBRIEL("Umbriel", "Easy-going"),
+ ALGIEBA("Algieba", "Smooth"),
+ DESPINA("Despina", "Smooth"),
+ ERINOME("Erinome", "Clear"),
+ ALGENIB("Algenib", "Gravelly"),
+ RASALGETHI("Rasalgethi", "Informative"),
+ LAOMEDEIA("Laomedeia", "Upbeat"),
+ ACHERNAR("Achernar", "Soft"),
+ ALNILAM("Alnilam", "Firm"),
+ SCHEDAR("Schedar", "Even"),
+ GACRUX("Gacrux", "Mature"),
+ PULCHERRIMA("Pulcherrima", "Forward"),
+ ACHIRD("Achird", "Friendly"),
+ ZUBENELGENUBI("Zubenelgenubi", "Casual"),
+ VINDEMIATRIX("Vindemiatrix", "Gentle"),
+ SADACHBIA("Sadachbia", "Lively"),
+ SADALTAGER("Sadaltager", "Knowledgeable"),
+ SULAFAT("Sulafat", "Warm"),
+ ;
+
+ companion object {
+ val DEFAULT: GeminiLiveVoice = ZEPHYR
+
+ /** Resolves a persisted preference string back to a [GeminiLiveVoice], falling back to [DEFAULT]. */
+ fun fromStoredValue(value: String?): GeminiLiveVoice =
+ value?.let { stored ->
+ entries.find {
+ it.name.equals(stored, ignoreCase = true) ||
+ it.apiName.equals(stored, ignoreCase = true)
+ }
+ } ?: DEFAULT
+ }
+}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt
index 30bb74fe..097b30ac 100644
--- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt
@@ -31,9 +31,8 @@ import io.ktor.http.ContentType
import io.ktor.http.contentType
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
-import kotlinx.serialization.json.JsonElement
-import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.buildJsonArray
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.jsonArray
@@ -60,9 +59,8 @@ class GeminiProviderImpl
serviceTier: ServiceTier,
): LlmResponse {
val convertedTools =
- tools
- .sortedByDescending { it.id.startsWith(it.packageName) }
- .distinctBy { toolConverter.getToolName(it) }
+ toolConverter
+ .selectDistinctTools(tools)
.mapNotNull { tool ->
try {
buildJsonObject {
@@ -89,15 +87,10 @@ class GeminiProviderImpl
when (input) {
is LlmInput.ToolResponse -> {
val inputElement =
- kotlinx.serialization.json.buildJsonArray {
+ buildJsonArray {
input.outputs.forEach { output ->
- val matchingTool = tools.find { it.id == output.functionId }
val mappedName =
- if (matchingTool != null) {
- toolConverter.getToolName(matchingTool)
- } else {
- output.functionId
- }
+ toolConverter.resolveToolName(output.functionId, tools)
add(
buildJsonObject {
@@ -119,7 +112,7 @@ class GeminiProviderImpl
}
if (convertedTools.isNotEmpty()) {
- put(KEY_TOOLS, kotlinx.serialization.json.JsonArray(convertedTools))
+ put(KEY_TOOLS, JsonArray(convertedTools))
}
if (previousInteractionId != null) {
@@ -200,19 +193,13 @@ class GeminiProviderImpl
stepObj[KEY_NAME]?.jsonPrimitive?.content
?: return LlmResponse.Error("Called function without a name")
val matchingTool =
- tools.find { tool -> toolConverter.getToolName(tool) == name }
+ toolConverter.findToolByName(name, tools)
?: return LlmResponse.Error("Called unknown function: $name")
val packageName = matchingTool.packageName
val functionId = matchingTool.id
- val args = stepObj[KEY_ARGUMENTS]?.jsonObject
- val argumentsMap = mutableMapOf()
- if (args != null) {
- for ((key, value) in args) {
- argumentsMap[key] = value.toPrimitive()
- }
- }
+ val argumentsMap = stepObj[KEY_ARGUMENTS]?.jsonObject.toArgumentsMap()
val callId =
stepObj["id"]?.jsonPrimitive?.content
?: return LlmResponse.Error(
@@ -238,19 +225,6 @@ class GeminiProviderImpl
)
}
- private fun JsonElement.toPrimitive(): Any? {
- return when (this) {
- is JsonPrimitive -> {
- if (this.isString) return this.content
- return this.content.toBooleanStrictOrNull()
- ?: this.content.toLongOrNull()
- ?: this.content.toDoubleOrNull()
- }
- is JsonObject -> this.mapValues { it.value.toPrimitive() }
- is JsonArray -> this.map { it.toPrimitive() }
- }
- }
-
private fun getSystemInstruction(): String {
val currentDate = LocalDate.now().toString()
return """
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt
index cc109813..3abbf9ee 100644
--- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt
@@ -45,10 +45,68 @@ class GeminiToolConverter
return convertToolToGeminiSchema(tool)
}
+ /**
+ * Converts an [AppFunctionMetadata] into a Gemini Live (`BidiGenerateContent`)
+ * `FunctionDeclaration` JSON object, normalizing OpenAPI schema fields.
+ */
+ fun convertForLiveApi(tool: AppFunctionMetadata): JsonObject {
+ val baseSchema = convertToolToGeminiSchema(tool)
+ return normalizeForLiveApi(baseSchema)
+ }
+
+ private fun normalizeForLiveApi(obj: JsonObject): JsonObject {
+ val typeValue = (obj[KEY_TYPE] as? JsonPrimitive)?.content
+ return buildJsonObject {
+ for ((key, value) in obj) {
+ when {
+ key == KEY_FORMAT &&
+ (value as? JsonPrimitive)?.content == VALUE_FILE_REFERENCE -> {
+ // Omit Interactions-API-specific file_reference format in BidiGenerateContent
+ }
+ key == KEY_ENUMS -> {
+ if (typeValue == VALUE_STRING) {
+ put(KEY_ENUM_LIVE, value)
+ }
+ }
+ value is JsonObject -> {
+ put(key, normalizeForLiveApi(value))
+ }
+ else -> {
+ put(key, value)
+ }
+ }
+ }
+ }
+ }
+
override fun getToolName(tool: AppFunctionMetadata): String {
return getGeminiFunctionName(tool)
}
+ /**
+ * Deduplicates [tools] by their converted Gemini function name, prioritizing functions
+ * whose ID starts with their package name.
+ */
+ fun selectDistinctTools(tools: List): List =
+ tools
+ .sortedByDescending { it.id.startsWith(it.packageName) }
+ .distinctBy { getToolName(it) }
+
+ /** Finds the [AppFunctionMetadata] in [tools] matching a converted Gemini function [name]. */
+ fun findToolByName(
+ name: String,
+ tools: List,
+ ): AppFunctionMetadata? = tools.find { getToolName(it) == name }
+
+ /**
+ * Resolves the converted Gemini function name for [functionId] from [tools], falling back
+ * to [functionId] if no matching tool metadata is present.
+ */
+ fun resolveToolName(
+ functionId: String,
+ tools: List,
+ ): String = tools.find { it.id == functionId }?.let(::getToolName) ?: functionId
+
private fun convertToolToGeminiSchema(tool: AppFunctionMetadata): JsonObject {
val combinedName = getGeminiFunctionName(tool)
return buildJsonObject {
@@ -295,6 +353,7 @@ class GeminiToolConverter
private const val VALUE_ARRAY = "array"
private const val KEY_ITEMS = "items"
private const val KEY_ENUMS = "enums"
+ private const val KEY_ENUM_LIVE = "enum"
private val INVALID_NAME_CHARS = Regex("[^a-zA-Z0-9_]")
private const val MAX_NAME_LENGTH = 64
}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt
index 0475b8c0..8d543c37 100644
--- a/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/LlmProviderFactoryImpl.kt
@@ -15,21 +15,29 @@
*/
package com.example.appfunctions.agent.data
+import com.example.appfunctions.agent.domain.LiveLlmProvider
import com.example.appfunctions.agent.domain.LlmProvider
import com.example.appfunctions.agent.domain.LlmProviderFactory
import javax.inject.Inject
import javax.inject.Singleton
-/** Implementation of LlmProviderFactory. */
+/** Implementation of [LlmProviderFactory]. */
@Singleton
class LlmProviderFactoryImpl
@Inject
constructor(
private val geminiProvider: GeminiProviderImpl,
+ private val geminiLiveProvider: GeminiLiveProviderImpl,
) : LlmProviderFactory {
override fun getProvider(name: LlmProviderName): LlmProvider {
return when (name) {
LlmProviderName.GEMINI -> geminiProvider
}
}
+
+ override fun getLiveProvider(name: LlmProviderName): LiveLlmProvider {
+ return when (name) {
+ LlmProviderName.GEMINI -> geminiLiveProvider
+ }
+ }
}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt
index 0feefb9d..92a2ca13 100644
--- a/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/SettingsRepository.kt
@@ -37,6 +37,12 @@ interface SettingsRepository {
/** Sets the service tier requested for LLM API calls. */
suspend fun setServiceTier(tier: ServiceTier)
+ /** Flow of the prebuilt voice selected for Gemini Live sessions. */
+ val geminiLiveVoice: Flow
+
+ /** Sets the prebuilt voice selected for Gemini Live sessions. */
+ suspend fun setGeminiLiveVoice(voice: GeminiLiveVoice)
+
/** Flow of pinned app package names. */
val pinnedApps: Flow>
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt
index 3757f9df..31c0c6ad 100644
--- a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt
@@ -16,18 +16,23 @@
package com.example.appfunctions.agent.di
import android.content.Context
+import android.media.AudioManager
import androidx.datastore.core.DataStore
import androidx.datastore.preferences.core.Preferences
import androidx.datastore.preferences.preferencesDataStore
import androidx.room.Room
import com.example.appfunctions.agent.data.ChatRepository
import com.example.appfunctions.agent.data.DataStoreSettingsRepository
+import com.example.appfunctions.agent.data.GeminiLiveAudioHelper
import com.example.appfunctions.agent.data.InMemoryPendingIntentRepository
+import com.example.appfunctions.agent.data.LlmProviderFactoryImpl
import com.example.appfunctions.agent.data.PendingIntentRepository
import com.example.appfunctions.agent.data.RoomChatRepository
import com.example.appfunctions.agent.data.SettingsRepository
import com.example.appfunctions.agent.data.db.AppDatabase
import com.example.appfunctions.agent.data.db.dao.ChatDao
+import com.example.appfunctions.agent.domain.LiveAudioSessionFactory
+import com.example.appfunctions.agent.domain.LlmProviderFactory
import dagger.Binds
import dagger.Module
import dagger.Provides
@@ -38,6 +43,7 @@ import io.ktor.client.HttpClient
import io.ktor.client.engine.okhttp.OkHttp
import io.ktor.client.plugins.HttpTimeout
import io.ktor.client.plugins.contentnegotiation.ContentNegotiation
+import io.ktor.client.plugins.websocket.WebSockets
import io.ktor.serialization.kotlinx.json.json
import kotlinx.serialization.json.Json
import javax.inject.Singleton
@@ -62,9 +68,7 @@ abstract class DataModule {
@Binds
@Singleton
- abstract fun bindLlmProviderFactory(
- impl: com.example.appfunctions.agent.data.LlmProviderFactoryImpl,
- ): com.example.appfunctions.agent.domain.LlmProviderFactory
+ abstract fun bindLlmProviderFactory(impl: LlmProviderFactoryImpl): LlmProviderFactory
companion object {
@Provides
@@ -109,6 +113,17 @@ abstract class DataModule {
)
}
install(HttpTimeout) { socketTimeoutMillis = 30000 }
+ install(WebSockets)
+ }
+ }
+
+ @Provides
+ @Singleton
+ fun provideLiveAudioSessionFactory(
+ @ApplicationContext context: Context,
+ ): LiveAudioSessionFactory {
+ return LiveAudioSessionFactory {
+ GeminiLiveAudioHelper(context.getSystemService(AudioManager::class.java))
}
}
}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt
index d2af9ee1..e14d6b72 100644
--- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt
@@ -16,18 +16,8 @@
package com.example.appfunctions.agent.domain
import android.app.PendingIntent
-import android.content.Context
-import android.content.Intent
-import android.net.Uri
import android.util.Log
-import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata
-import androidx.appfunctions.metadata.AppFunctionDataTypeMetadata
import androidx.appfunctions.metadata.AppFunctionMetadata
-import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata
-import androidx.appfunctions.metadata.AppFunctionParameterMetadata
-import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata
-import androidx.core.content.FileProvider
-import com.example.appfunctions.agent.data.AgentInternalTools
import com.example.appfunctions.agent.data.LlmProviderName
import com.example.appfunctions.agent.data.SettingsRepository
import com.example.appfunctions.agent.data.db.entities.MessageAttachment
@@ -35,12 +25,9 @@ import com.example.appfunctions.agent.data.db.entities.MessageEntity
import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus
import com.example.appfunctions.agent.data.db.entities.MessageRole
import com.example.appfunctions.agent.data.db.entities.ThreadEntity
-import com.example.appfunctions.agent.domain.appfunction.AppFunctionExceptionFormatter
-import com.example.appfunctions.agent.domain.appfunction.ConvertInputToAppFunctionDataUseCase
-import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult
-import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionUseCase
-import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionStatesUseCase
-import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase
+import com.example.appfunctions.agent.domain.appfunction.ExecuteToolCallUseCase
+import com.example.appfunctions.agent.domain.appfunction.GetEnabledToolsUseCase
+import com.example.appfunctions.agent.domain.appfunction.ToolCallExecutionResult
import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase
import com.example.appfunctions.agent.domain.chat.ObservePendingMessagesUseCase
import com.example.appfunctions.agent.domain.chat.SendMessageUseCase
@@ -49,9 +36,7 @@ import com.example.appfunctions.agent.domain.chat.UpdateMessageUseCase
import com.example.appfunctions.agent.domain.chat.UpdateThreadParams
import com.example.appfunctions.agent.domain.chat.UpdateThreadUseCase
import com.example.appfunctions.agent.domain.pendingintent.SavePendingIntentUseCase
-import dagger.hilt.android.qualifiers.ApplicationContext
import kotlinx.coroutines.CancellationException
-import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.SharingStarted
@@ -60,12 +45,6 @@ import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.filterNotNull
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.flow.stateIn
-import kotlinx.coroutines.withContext
-import org.json.JSONObject
-import java.io.File
-import java.net.HttpURLConnection
-import java.net.URL
-import java.util.UUID
import javax.inject.Inject
import javax.inject.Singleton
@@ -74,7 +53,6 @@ import javax.inject.Singleton
class AgentOrchestrator
@Inject
constructor(
- @ApplicationContext private val context: Context,
private val manageThreadsUseCase: ManageThreadsUseCase,
private val observePendingMessagesUseCase: ObservePendingMessagesUseCase,
private val sendMessageUseCase: SendMessageUseCase,
@@ -82,12 +60,9 @@ class AgentOrchestrator
private val updateThreadUseCase: UpdateThreadUseCase,
private val llmProviderFactory: LlmProviderFactory,
private val settingsRepository: SettingsRepository,
- private val getAppFunctionsUseCase: GetAppFunctionsUseCase,
- private val getAppFunctionStatesUseCase: GetAppFunctionStatesUseCase,
- private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase,
- private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase,
+ private val getEnabledToolsUseCase: GetEnabledToolsUseCase,
+ private val executeToolCallUseCase: ExecuteToolCallUseCase,
private val savePendingIntentUseCase: SavePendingIntentUseCase,
- private val agentInternalTools: AgentInternalTools,
) {
private val _status = MutableStateFlow(AgentStatus.Idle)
@@ -132,25 +107,20 @@ class AgentOrchestrator
return
}
- val disconnectedApps = settingsRepository.disconnectedApps.first()
- val allTools =
- getAppFunctionsUseCase().first().values.flatten() +
- agentInternalTools.getInternalToolsMetadata().map { it.first }
-
- val targetPackageName = message.targetPackageName
- val queryText = message.textContent
-
- val tools = filterTools(allTools, disconnectedApps, targetPackageName)
+ val tools = getEnabledToolsUseCase(message.targetPackageName)
runInteractionLoop(
message = message,
thread = thread,
apiKey = apiKey,
tools = tools,
- initialInput = queryText,
+ initialInput = message.textContent,
)
_status.value = AgentStatus.Idle
+ } catch (e: CancellationException) {
+ _status.value = AgentStatus.Idle
+ throw e
} catch (e: Exception) {
Log.e("AgentOrchestrator", "Error processing message", e)
completeMessageWithError(
@@ -162,31 +132,6 @@ class AgentOrchestrator
}
}
- private suspend fun filterTools(
- allTools: List,
- disconnectedApps: Set,
- targetPackageName: String?,
- ): List {
- val functionNames = allTools.map { it.name }
- val states = getAppFunctionStatesUseCase(functionNames) + agentInternalTools.getInternalToolsMetadata().map { it.second }
- val stateMap = states.associateBy { it.functionName }
-
- return allTools
- .filter { metadata ->
- val isEnabled =
- if (metadata.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) {
- true
- } else {
- stateMap[metadata.name]?.isEnabled ?: false
- }
- isEnabled &&
- metadata.packageName !in disconnectedApps &&
- (targetPackageName == null || metadata.packageName == targetPackageName)
- }
- .sortedByDescending { metadata -> metadata.id.startsWith(metadata.packageName) }
- .distinctBy { metadata -> metadata.id }
- }
-
private suspend fun runInteractionLoop(
message: MessageEntity,
thread: ThreadEntity,
@@ -202,7 +147,7 @@ class AgentOrchestrator
var previousInteractionId = thread.latestInteractionId
var currentToolOutputs = emptyList()
var continueLoop = true
- var currentInput = initialInput
+ val currentInput = initialInput
val capturedAttachments = mutableListOf()
while (continueLoop) {
@@ -293,23 +238,11 @@ class AgentOrchestrator
val textContent = textParts.joinToString("\n") { it.text }
if (toolCalls.isNotEmpty()) {
- when (val toolResult = executeToolCalls(toolCalls, tools, message)) {
+ when (
+ val toolResult =
+ executeToolCalls(toolCalls, tools, message, capturedAttachments)
+ ) {
is ExecuteToolCallsResult.Success -> {
- for (output in toolResult.toolOutputs) {
- runCatching {
- val json = JSONObject(output.result)
- val uri = json.optString("imageUri")
- if (uri.isNotBlank()) {
- val mimeType =
- json.optString("mimeType")
- .takeIf { it.isNotBlank() }
- ?: "image/png"
- capturedAttachments.add(
- MessageAttachment(uri = uri, mimeType = mimeType),
- )
- }
- }
- }
if (textContent.isNotEmpty()) {
sendMessageUseCase(
threadId = message.threadId,
@@ -374,151 +307,43 @@ class AgentOrchestrator
toolCalls: List,
tools: List,
message: MessageEntity,
+ capturedAttachments: MutableList,
): ExecuteToolCallsResult {
val results = mutableListOf()
for (toolCall in toolCalls) {
_status.value = AgentStatus.InvokingTool(toolCall.functionId, toolCall.packageName)
- if (toolCall.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) {
- val executionResult = executeInternalTool(toolCall)
- executionResult
- .onSuccess { jsonString ->
- results.add(
- ToolOutput(
- functionId = toolCall.functionId,
- callId = toolCall.callId,
- result = jsonString,
- ),
- )
- }
- .onFailure { exception ->
- completeMessageWithError(
- message.messageId,
- message.threadId,
- "Internal tool execution failed: ${exception.message}",
- )
- return ExecuteToolCallsResult.Error
- }
- continue
- }
-
- val matchingTool =
- tools.find {
- it.packageName == toolCall.packageName && it.id == toolCall.functionId
+ when (
+ val result =
+ executeToolCallUseCase(
+ toolCall = toolCall,
+ tools = tools,
+ threadId = message.threadId,
+ )
+ ) {
+ is ToolCallExecutionResult.Success -> {
+ results.add(result.toolOutput)
+ result.attachment?.let { capturedAttachments.add(it) }
}
- if (matchingTool == null) {
- completeMessageWithError(
- message.messageId,
- message.threadId,
- "Tool not found: ${toolCall.functionId}",
- )
- return ExecuteToolCallsResult.Error
- }
+ is ToolCallExecutionResult.PendingIntentRequired -> {
+ return ExecuteToolCallsResult.PendingIntentAction(
+ result.pendingIntentId,
+ result.pendingIntent,
+ )
+ }
- val rawConvertedInputs = toolCall.arguments.filterValues { it != null } as Map
- val convertedInputs =
- try {
- withContext(Dispatchers.IO) {
- resolveRemoteFileReferencesRecursively(
- context = context,
- parametersMetadata = matchingTool.parameters,
- inputs = rawConvertedInputs,
- )
+ is ToolCallExecutionResult.Failure -> {
+ result.exception?.let { exception ->
+ Log.e("AgentOrchestrator", result.errorMessage, exception)
}
- } catch (e: Exception) {
completeMessageWithError(
message.messageId,
message.threadId,
- "Failed to download remote file reference: ${e.message}",
+ result.errorMessage,
)
return ExecuteToolCallsResult.Error
}
-
- grantContentUriPermissionsRecursively(context, toolCall.packageName, convertedInputs)
-
- for (value in convertedInputs.values) {
- if (value is String && value.startsWith("content://")) {
- runCatching {
- val uri = Uri.parse(value)
- context.grantUriPermission(
- toolCall.packageName,
- uri,
- Intent.FLAG_GRANT_READ_URI_PERMISSION,
- )
- }
- }
- }
-
- val appFunctionDataResult =
- withContext(Dispatchers.Default) {
- convertInputToAppFunctionDataUseCase(
- parameters = matchingTool.parameters,
- components = matchingTool.components,
- inputs = convertedInputs,
- )
- }
- if (appFunctionDataResult.isFailure) {
- completeMessageWithError(
- message.messageId,
- message.threadId,
- "Failed to convert arguments for ${toolCall.functionId}\n ${appFunctionDataResult.exceptionOrNull()}",
- )
- return ExecuteToolCallsResult.Error
- }
-
- val executionResult =
- executeAppFunctionUseCase(
- function = matchingTool,
- parameters = appFunctionDataResult.getOrThrow(),
- threadId = message.threadId,
- )
-
- when (executionResult) {
- is ExecuteAppFunctionResult.Data -> {
- results.add(
- ToolOutput(
- functionId = toolCall.functionId,
- callId = toolCall.callId,
- result = executionResult.formattedJson,
- ),
- )
- }
-
- is ExecuteAppFunctionResult.PendingIntentAction -> {
- val pendingIntentId = java.util.UUID.randomUUID().toString()
- return ExecuteToolCallsResult.PendingIntentAction(
- pendingIntentId,
- executionResult.pendingIntent,
- )
- }
-
- is ExecuteAppFunctionResult.Error -> {
- val exception = executionResult.exception
- if (exception is CancellationException) {
- throw exception
- }
- val appFunctionException =
- AppFunctionExceptionFormatter.getAppFunctionException(exception)
- if (appFunctionException != null) {
- results.add(
- ToolOutput(
- functionId = toolCall.functionId,
- callId = toolCall.callId,
- result =
- AppFunctionExceptionFormatter.format(
- appFunctionException,
- toolCall.functionId,
- ),
- ),
- )
- } else {
- throw IllegalStateException(
- "Tool execution failed for ${toolCall.functionId}: ${exception.message}",
- exception,
- )
- }
- }
}
}
return ExecuteToolCallsResult.Success(results)
@@ -540,208 +365,4 @@ class AgentOrchestrator
processingStatus = MessageProcessingStatus.FAILED,
)
}
-
- private fun resolveRemoteFileReferencesRecursively(
- context: Context,
- parametersMetadata: List,
- inputs: Map,
- ): Map {
- val paramMap = parametersMetadata.associateBy { it.name }
- return inputs.mapValues { (key, value) ->
- val paramMeta = paramMap[key]
- resolveValueRecursively(context, paramMeta?.dataType, value, key)
- }
- }
-
- private fun resolveValueRecursively(
- context: Context,
- dataType: AppFunctionDataTypeMetadata?,
- value: Any,
- paramName: String?,
- ): Any {
- return when (value) {
- is String -> {
- val shouldResolve =
- isFileReferenceParameter(paramName) || isUriMetadata(dataType)
- if (shouldResolve && (
- value.startsWith(
- "http://",
- ) || value.startsWith("https://")
- )
- ) {
- downloadRemoteFileToContentUri(context, value)
- } else {
- value
- }
- }
- is Map<*, *> -> {
- value.entries.associate { entry ->
- val k = entry.key as String
- val propType =
- (dataType as? AppFunctionObjectTypeMetadata)?.properties?.get(k)
- k to (
- entry.value?.let {
- resolveValueRecursively(
- context,
- propType,
- it,
- k,
- )
- } ?: ""
- )
- }
- }
- is List<*> -> {
- val itemType = (dataType as? AppFunctionArrayTypeMetadata)?.itemType
- value.mapNotNull { item ->
- if (item != null) resolveValueRecursively(context, itemType, item, paramName) else null
- }
- }
- else -> value
- }
- }
-
- private fun downloadRemoteFileToContentUri(
- context: Context,
- urlString: String,
- ): String {
- val url = URL(urlString)
- val connection =
- (url.openConnection() as HttpURLConnection).apply {
- connectTimeout = 10000
- readTimeout = 15000
- }
- try {
- connection.connect()
- val contentType = connection.contentType ?: ""
- val ext =
- when {
- contentType.contains("png", ignoreCase = true) -> "png"
- contentType.contains("jpeg", ignoreCase = true) ||
- contentType.contains("jpg", ignoreCase = true) -> "jpg"
- contentType.contains("gif", ignoreCase = true) -> "gif"
- contentType.contains("webp", ignoreCase = true) -> "webp"
- urlString.substringAfterLast("/", "").contains(".") ->
- urlString.substringAfterLast("/").substringAfterLast(".")
- else -> "jpg"
- }
- val cacheDir = File(context.cacheDir, "file_references").apply { mkdirs() }
- val file = File(cacheDir, "generated_${UUID.randomUUID()}.$ext")
- connection.inputStream.use { input ->
- file.outputStream().use { output ->
- input.copyTo(output)
- }
- }
- val contentUri =
- FileProvider.getUriForFile(
- context,
- "${context.packageName}.fileprovider",
- file,
- )
- return contentUri.toString()
- } finally {
- connection.disconnect()
- }
- }
-
- private fun grantContentUriPermissionsRecursively(
- context: Context,
- targetPackageName: String,
- value: Any?,
- ) {
- when (value) {
- is String -> {
- if (value.startsWith("content://")) {
- runCatching {
- val uri = Uri.parse(value)
- context.grantUriPermission(
- targetPackageName,
- uri,
- Intent.FLAG_GRANT_READ_URI_PERMISSION,
- )
- }
- }
- }
- is Map<*, *> ->
- value.values.forEach {
- grantContentUriPermissionsRecursively(context, targetPackageName, it)
- }
- is List<*> ->
- value.forEach {
- grantContentUriPermissionsRecursively(context, targetPackageName, it)
- }
- }
- }
-
- private fun isFileReferenceParameter(parameterName: String?): Boolean {
- if (parameterName == null) return false
- if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true
- return parameterName.endsWith("Uri", ignoreCase = true) ||
- parameterName.endsWith("Uris", ignoreCase = true)
- }
-
- private fun isUriMetadata(dataType: AppFunctionDataTypeMetadata?): Boolean {
- if (dataType == null) return false
- if (dataType is AppFunctionObjectTypeMetadata && dataType.qualifiedName == "android.net.Uri") return true
- if (dataType is AppFunctionReferenceTypeMetadata && dataType.referenceDataType == "android.net.Uri") return true
- return false
- }
-
- private suspend fun executeInternalTool(toolCall: LlmResponsePart.ToolCall): Result {
- return runCatching {
- when (toolCall.functionId) {
- "getCurrentLocation" -> {
- val location = agentInternalTools.getCurrentLocation()
- if (location != null) {
- JSONObject().apply {
- put("latitude", location.latitude)
- put("longitude", location.longitude)
- }.toString()
- } else {
- "null"
- }
- }
- "geocodeAddress" -> {
- val address =
- toolCall.arguments["address"] as? String
- ?: throw IllegalArgumentException("Missing required parameter 'address'")
- val location = agentInternalTools.geocodeAddress(address)
- if (location != null) {
- JSONObject().apply {
- put("latitude", location.latitude)
- put("longitude", location.longitude)
- }.toString()
- } else {
- "null"
- }
- }
- "generateImage" -> {
- val prompt =
- toolCall.arguments["prompt"] as? String
- ?: throw IllegalArgumentException("Missing required parameter 'prompt'")
- val aspectRatio = toolCall.arguments["aspectRatio"] as? String
- val result = agentInternalTools.generateImage(prompt, aspectRatio)
- JSONObject().apply {
- put("imageUri", result.imageUri)
- put("mimeType", result.mimeType)
- put("prompt", result.prompt)
- }.toString()
- }
- else -> throw IllegalArgumentException("Unknown internal tool: ${toolCall.functionId}")
- }
- }
- }
-
- companion object {
- private val KNOWN_FILE_REFERENCE_PARAM_NAMES =
- setOf(
- "wallpaperUri",
- "imageUri",
- "attachmentUri",
- "ringtoneUri",
- "profilePictureUri",
- "audioUri",
- "voiceNoteUri",
- )
- }
}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAgentOrchestrator.kt
new file mode 100644
index 00000000..10db47a7
--- /dev/null
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAgentOrchestrator.kt
@@ -0,0 +1,521 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.domain
+
+import android.util.Log
+import androidx.appfunctions.metadata.AppFunctionMetadata
+import com.example.appfunctions.agent.data.LlmProviderName
+import com.example.appfunctions.agent.data.SettingsRepository
+import com.example.appfunctions.agent.data.db.entities.MessageAttachment
+import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus
+import com.example.appfunctions.agent.data.db.entities.MessageRole
+import com.example.appfunctions.agent.domain.appfunction.ExecuteToolCallUseCase
+import com.example.appfunctions.agent.domain.appfunction.GetEnabledToolsUseCase
+import com.example.appfunctions.agent.domain.appfunction.ToolCallExecutionResult
+import com.example.appfunctions.agent.domain.chat.SendMessageUseCase
+import com.example.appfunctions.agent.domain.pendingintent.LaunchPendingIntentUseCase
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.CoroutineDispatcher
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.cancelAndJoin
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.currentCoroutineContext
+import kotlinx.coroutines.ensureActive
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.StateFlow
+import kotlinx.coroutines.flow.asStateFlow
+import kotlinx.coroutines.flow.first
+import kotlinx.coroutines.flow.takeWhile
+import kotlinx.coroutines.isActive
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import javax.inject.Inject
+import javax.inject.Singleton
+
+/**
+ * Provider-agnostic orchestrator for bidirectional live voice sessions, bridging realtime audio
+ * capture/playback, transcript persistence, and Android AppFunction tool execution via
+ * [LiveLlmProvider].
+ */
+@Singleton
+class LiveAgentOrchestrator
+ @Inject
+ constructor(
+ private val llmProviderFactory: LlmProviderFactory,
+ private val settingsRepository: SettingsRepository,
+ private val getEnabledToolsUseCase: GetEnabledToolsUseCase,
+ private val executeToolCallUseCase: ExecuteToolCallUseCase,
+ private val launchPendingIntentUseCase: LaunchPendingIntentUseCase,
+ private val sendMessageUseCase: SendMessageUseCase,
+ private val audioSessionFactory: LiveAudioSessionFactory,
+ ) {
+ private val scope = CoroutineScope(SupervisorJob())
+ private val stateLock = Any()
+ private val sessionMutex = Mutex()
+
+ private val _state = MutableStateFlow(LiveSessionState.Idle)
+
+ /** Current connection/listening state of the live voice session. */
+ val state: StateFlow = _state.asStateFlow()
+
+ private val _status = MutableStateFlow(AgentStatus.Idle)
+
+ /** Current tool invocation status emitted during active live voice sessions. */
+ val status: StateFlow = _status.asStateFlow()
+
+ private var sessionJob: Job? = null
+ private var activeSessionId: Long = 0L
+
+ internal var ioDispatcher: CoroutineDispatcher = Dispatchers.IO
+
+ /** Toggles the live voice session for [threadId]. */
+ fun toggleSession(
+ threadId: String,
+ targetPackageName: String? = null,
+ ) {
+ val shouldStop =
+ synchronized(stateLock) {
+ _state.value is LiveSessionState.Connecting ||
+ _state.value is LiveSessionState.Listening
+ }
+ if (shouldStop) {
+ stopSession()
+ } else {
+ startSession(threadId, targetPackageName)
+ }
+ }
+
+ /** Starts a new live voice session for [threadId]. */
+ fun startSession(
+ threadId: String,
+ targetPackageName: String? = null,
+ ) {
+ val (mySessionId, previousJob) =
+ synchronized(stateLock) {
+ val nextId = ++activeSessionId
+ val prev = sessionJob
+ prev?.cancel()
+ _state.value = LiveSessionState.Connecting
+ _status.value = AgentStatus.Idle
+ nextId to prev
+ }
+
+ lateinit var launchedJob: Job
+ launchedJob =
+ scope.launch(ioDispatcher) {
+ try {
+ // Serialize full audio hardware teardown of any prior session before
+ // opening a new session so two AudioRecord instances never overlap.
+ previousJob?.cancelAndJoin()
+ sessionMutex.withLock {
+ if (!isActive || !isCurrentSession(mySessionId)) {
+ return@withLock
+ }
+ runSession(
+ sessionId = mySessionId,
+ threadId = threadId,
+ targetPackageName = targetPackageName,
+ )
+ }
+ } finally {
+ synchronized(stateLock) {
+ if (sessionJob === launchedJob) {
+ sessionJob = null
+ }
+ }
+ }
+ }
+
+ synchronized(stateLock) {
+ if (activeSessionId == mySessionId) {
+ sessionJob = launchedJob
+ } else {
+ launchedJob.cancel()
+ }
+ }
+ }
+
+ /**
+ * Stops any active live voice session without blocking the calling (UI) thread.
+ *
+ * Audio hardware teardown runs asynchronously on [ioDispatcher] inside the session
+ * coroutine's `finally` block under [sessionMutex].
+ */
+ fun stopSession() {
+ val jobToCancel =
+ synchronized(stateLock) {
+ activeSessionId++
+ _state.value = LiveSessionState.Idle
+ _status.value = AgentStatus.Idle
+ sessionJob
+ }
+ jobToCancel?.cancel()
+ }
+
+ private fun isCurrentSession(sessionId: Long): Boolean = synchronized(stateLock) { activeSessionId == sessionId }
+
+ private fun updateStateIfCurrent(
+ sessionId: Long,
+ newState: LiveSessionState,
+ ) {
+ synchronized(stateLock) {
+ if (activeSessionId == sessionId) {
+ _state.value = newState
+ }
+ }
+ }
+
+ private fun updateStatusIfCurrent(
+ sessionId: Long,
+ newStatus: AgentStatus,
+ ) {
+ synchronized(stateLock) {
+ if (activeSessionId == sessionId) {
+ _status.value = newStatus
+ }
+ }
+ }
+
+ private suspend fun getApiKey(provider: LlmProviderName): String? {
+ return when (provider) {
+ LlmProviderName.GEMINI -> settingsRepository.geminiApiKey.first()
+ }
+ }
+
+ private suspend fun runSession(
+ sessionId: Long,
+ threadId: String,
+ targetPackageName: String?,
+ ) {
+ val userTranscriptBuilder = StringBuilder()
+ val modelTranscriptBuilder = StringBuilder()
+ var localAudioSession: LiveAudioSession? = null
+
+ try {
+ val providerName = settingsRepository.selectedProvider.first()
+ val apiKey = getApiKey(providerName)?.takeIf { it.isNotBlank() }
+ if (!currentCoroutineContext().isActive || !isCurrentSession(sessionId)) {
+ return
+ }
+ if (apiKey == null) {
+ val errorMsg = "API key is missing for ${providerName.name}"
+ sendMessageUseCase(
+ threadId = threadId,
+ role = MessageRole.ASSISTANT,
+ textContent = errorMsg,
+ processingStatus = MessageProcessingStatus.FAILED,
+ )
+ updateStateIfCurrent(sessionId, LiveSessionState.Error(errorMsg))
+ return
+ }
+
+ val tools = getEnabledToolsUseCase(targetPackageName)
+ currentCoroutineContext().ensureActive()
+ if (!isCurrentSession(sessionId)) return
+
+ val liveProvider = llmProviderFactory.getLiveProvider(providerName)
+
+ liveProvider.connect(apiKey = apiKey, tools = tools) { session ->
+ currentCoroutineContext().ensureActive()
+ if (!isCurrentSession(sessionId)) return@connect
+
+ val audioSession = audioSessionFactory.create()
+ localAudioSession = audioSession
+ audioSession.start()
+
+ if (!currentCoroutineContext().isActive || !isCurrentSession(sessionId)) {
+ return@connect
+ }
+ updateStateIfCurrent(sessionId, LiveSessionState.Listening)
+
+ coroutineScope {
+ val sendAudioJob =
+ launch {
+ audioSession.listenToRecording().collect { pcmBytes ->
+ session.sendAudio(pcmBytes)
+ }
+ }
+
+ try {
+ session.events
+ .takeWhile {
+ currentCoroutineContext().isActive &&
+ isCurrentSession(sessionId)
+ }
+ .collect { event ->
+ handleLiveEvent(
+ event = event,
+ sessionId = sessionId,
+ session = session,
+ threadId = threadId,
+ tools = tools,
+ audioSession = audioSession,
+ userTranscriptBuilder = userTranscriptBuilder,
+ modelTranscriptBuilder = modelTranscriptBuilder,
+ )
+ }
+ } finally {
+ sendAudioJob.cancel()
+ }
+ }
+ }
+
+ if (currentCoroutineContext().isActive && isCurrentSession(sessionId)) {
+ flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder)
+ updateStateIfCurrent(sessionId, LiveSessionState.Idle)
+ } else {
+ flushRemainingTranscriptsAsync(
+ threadId = threadId,
+ userTranscriptBuilder = userTranscriptBuilder,
+ modelTranscriptBuilder = modelTranscriptBuilder,
+ )
+ }
+ } catch (e: CancellationException) {
+ flushRemainingTranscriptsAsync(
+ threadId = threadId,
+ userTranscriptBuilder = userTranscriptBuilder,
+ modelTranscriptBuilder = modelTranscriptBuilder,
+ )
+ throw e
+ } catch (e: Exception) {
+ if (!currentCoroutineContext().isActive || !isCurrentSession(sessionId)) {
+ flushRemainingTranscriptsAsync(
+ threadId = threadId,
+ userTranscriptBuilder = userTranscriptBuilder,
+ modelTranscriptBuilder = modelTranscriptBuilder,
+ )
+ return
+ }
+ Log.e(TAG, "Live session error", e)
+ flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder)
+ val errorText = "Live session error: ${e.message ?: "Connection failed"}"
+ runCatching {
+ sendMessageUseCase(
+ threadId = threadId,
+ role = MessageRole.ASSISTANT,
+ textContent = errorText,
+ processingStatus = MessageProcessingStatus.FAILED,
+ )
+ }
+ updateStateIfCurrent(sessionId, LiveSessionState.Error(errorText))
+ } finally {
+ updateStatusIfCurrent(sessionId, AgentStatus.Idle)
+ localAudioSession?.release()
+ localAudioSession = null
+ }
+ }
+
+ private suspend fun handleLiveEvent(
+ event: LiveLlmEvent,
+ sessionId: Long,
+ session: LiveLlmSession,
+ threadId: String,
+ tools: List,
+ audioSession: LiveAudioSession,
+ userTranscriptBuilder: StringBuilder,
+ modelTranscriptBuilder: StringBuilder,
+ ) {
+ when (event) {
+ LiveLlmEvent.Interrupted -> {
+ audioSession.stopAudio()
+ flushTranscript(threadId, MessageRole.ASSISTANT, modelTranscriptBuilder)
+ }
+
+ is LiveLlmEvent.InputTranscription -> {
+ userTranscriptBuilder.append(event.text)
+ }
+
+ is LiveLlmEvent.OutputTranscription -> {
+ // Flush user transcript before model response transcript starts.
+ flushTranscript(threadId, MessageRole.USER, userTranscriptBuilder)
+ modelTranscriptBuilder.append(event.text)
+ }
+
+ is LiveLlmEvent.AudioChunk -> {
+ audioSession.queueAudio(event.pcmBytes)
+ }
+
+ LiveLlmEvent.TurnComplete -> {
+ flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder)
+ }
+
+ is LiveLlmEvent.ToolCallRequest -> {
+ flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder)
+ if (event.calls.isNotEmpty()) {
+ audioSession.pauseRecording()
+ try {
+ val toolOutputs =
+ executeLiveToolCalls(
+ sessionId = sessionId,
+ threadId = threadId,
+ toolCalls = event.calls,
+ tools = tools,
+ )
+ session.sendToolResponses(toolOutputs)
+ } finally {
+ audioSession.resumeRecording()
+ }
+ }
+ }
+ }
+ }
+
+ private suspend fun executeLiveToolCalls(
+ sessionId: Long,
+ threadId: String,
+ toolCalls: List,
+ tools: List,
+ ): List {
+ val outputs = mutableListOf()
+ val capturedAttachments = mutableListOf()
+ try {
+ for (toolCall in toolCalls) {
+ updateStatusIfCurrent(
+ sessionId,
+ AgentStatus.InvokingTool(toolCall.functionId, toolCall.packageName),
+ )
+ when (
+ val result =
+ executeToolCallUseCase(
+ toolCall = toolCall,
+ tools = tools,
+ threadId = threadId,
+ )
+ ) {
+ is ToolCallExecutionResult.Success -> {
+ outputs.add(result.toolOutput)
+ result.attachment?.let { capturedAttachments.add(it) }
+ }
+
+ is ToolCallExecutionResult.PendingIntentRequired -> {
+ val launchResult = launchPendingIntentUseCase(result.pendingIntent)
+ if (launchResult.isSuccess) {
+ outputs.add(
+ ToolOutput(
+ functionId = toolCall.functionId,
+ callId = toolCall.callId,
+ result = PENDING_INTENT_LAUNCHED_RESULT,
+ ),
+ )
+ } else {
+ val reason =
+ launchResult.exceptionOrNull()?.message
+ ?: "PendingIntent canceled"
+ outputs.add(
+ ToolOutput(
+ functionId = toolCall.functionId,
+ callId = toolCall.callId,
+ result =
+ "Tool execution failed for ${toolCall.functionId}: $reason",
+ ),
+ )
+ }
+ }
+
+ is ToolCallExecutionResult.Failure -> {
+ outputs.add(
+ ToolOutput(
+ functionId = toolCall.functionId,
+ callId = toolCall.callId,
+ result = result.errorMessage,
+ ),
+ )
+ }
+ }
+ }
+
+ if (capturedAttachments.isNotEmpty()) {
+ sendMessageUseCase(
+ threadId = threadId,
+ role = MessageRole.ASSISTANT,
+ textContent = "",
+ processingStatus = MessageProcessingStatus.PROCESSED,
+ attachments = capturedAttachments,
+ )
+ }
+ } finally {
+ updateStatusIfCurrent(sessionId, AgentStatus.Idle)
+ }
+ return outputs
+ }
+
+ private fun flushRemainingTranscriptsAsync(
+ threadId: String,
+ userTranscriptBuilder: StringBuilder,
+ modelTranscriptBuilder: StringBuilder,
+ ) {
+ val remainingUserText = userTranscriptBuilder.toString().trim()
+ val remainingModelText = modelTranscriptBuilder.toString().trim()
+ userTranscriptBuilder.clear()
+ modelTranscriptBuilder.clear()
+ if (remainingUserText.isEmpty() && remainingModelText.isEmpty()) return
+
+ scope.launch(ioDispatcher) {
+ if (remainingUserText.isNotEmpty()) {
+ sendMessageUseCase(
+ threadId = threadId,
+ role = MessageRole.USER,
+ textContent = remainingUserText,
+ processingStatus = MessageProcessingStatus.PROCESSED,
+ )
+ }
+ if (remainingModelText.isNotEmpty()) {
+ sendMessageUseCase(
+ threadId = threadId,
+ role = MessageRole.ASSISTANT,
+ textContent = remainingModelText,
+ processingStatus = MessageProcessingStatus.PROCESSED,
+ )
+ }
+ }
+ }
+
+ private suspend fun flushTranscripts(
+ threadId: String,
+ userTranscriptBuilder: StringBuilder,
+ modelTranscriptBuilder: StringBuilder,
+ ) {
+ flushTranscript(threadId, MessageRole.USER, userTranscriptBuilder)
+ flushTranscript(threadId, MessageRole.ASSISTANT, modelTranscriptBuilder)
+ }
+
+ private suspend fun flushTranscript(
+ threadId: String,
+ role: MessageRole,
+ builder: StringBuilder,
+ ) {
+ val text = builder.toString().trim()
+ builder.clear()
+ if (text.isNotEmpty()) {
+ sendMessageUseCase(
+ threadId = threadId,
+ role = role,
+ textContent = text,
+ processingStatus = MessageProcessingStatus.PROCESSED,
+ )
+ }
+ }
+
+ companion object {
+ private const val TAG = "LiveAgentOrchestrator"
+ private const val PENDING_INTENT_LAUNCHED_RESULT =
+ """{"status":"launched","message":"The action was executed and the target app UI was opened."}"""
+ }
+ }
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAudioSession.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAudioSession.kt
new file mode 100644
index 00000000..a6209e12
--- /dev/null
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveAudioSession.kt
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.domain
+
+import kotlinx.coroutines.flow.Flow
+
+/** Abstraction over full-duplex audio recording and playback for a live voice session. */
+interface LiveAudioSession {
+ /** Acquires audio focus/routing and starts microphone capture and audio playback resources. */
+ fun start()
+
+ /** Streams captured PCM microphone audio chunks while recording is active and unpaused. */
+ fun listenToRecording(): Flow
+
+ /** Temporarily suppresses microphone emissions (e.g. while executing tools). */
+ fun pauseRecording()
+
+ /** Resumes microphone emissions after [pauseRecording]. */
+ fun resumeRecording()
+
+ /** Enqueues a PCM audio chunk from the assistant for speaker/headset playback. */
+ fun queueAudio(pcmBytes: ByteArray)
+
+ /** Immediately clears queued assistant audio and flushes active playback on interruption. */
+ fun stopAudio()
+
+ /** Releases all underlying audio recording, playback, and audio-focus resources. */
+ fun release()
+}
+
+/** Factory for creating a new [LiveAudioSession] per voice session. */
+fun interface LiveAudioSessionFactory {
+ fun create(): LiveAudioSession
+}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt
new file mode 100644
index 00000000..2b1c2025
--- /dev/null
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveLlmProvider.kt
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.domain
+
+import androidx.appfunctions.metadata.AppFunctionMetadata
+import kotlinx.coroutines.flow.Flow
+
+/** Interface for LLM providers supporting bidirectional realtime audio and tool sessions. */
+interface LiveLlmProvider {
+ /**
+ * Connects to the live LLM backend, completes session initialization with [tools], and invokes
+ * [block] with an active [LiveLlmSession] until [block] returns or is cancelled.
+ */
+ suspend fun connect(
+ apiKey: String,
+ tools: List,
+ block: suspend (LiveLlmSession) -> Unit,
+ )
+}
+
+/** Represents an active bidirectional voice and tool session with an LLM provider. */
+interface LiveLlmSession {
+ /** Sequential stream of server-originated events for this live session. */
+ val events: Flow
+
+ /** Sends a chunk of raw 16-bit 16kHz mono PCM user microphone audio to the model. */
+ suspend fun sendAudio(pcmBytes: ByteArray)
+
+ /** Sends tool execution [outputs] back to the model in response to a [LiveLlmEvent.ToolCallRequest]. */
+ suspend fun sendToolResponses(outputs: List)
+}
+
+/** Provider-agnostic events emitted by a [LiveLlmSession]. */
+sealed class LiveLlmEvent {
+ /** Indicates the user interrupted the model's current audio response. */
+ object Interrupted : LiveLlmEvent()
+
+ /** Incremental speech-to-text transcription of the user's audio input. */
+ data class InputTranscription(val text: String) : LiveLlmEvent()
+
+ /** Incremental transcription of the model's spoken response. */
+ data class OutputTranscription(val text: String) : LiveLlmEvent()
+
+ /** Raw PCM audio chunk generated by the model for playback. */
+ data class AudioChunk(val pcmBytes: ByteArray) : LiveLlmEvent()
+
+ /** Indicates the model finished its current conversational turn. */
+ object TurnComplete : LiveLlmEvent()
+
+ /** Requests execution of one or more [calls] before continuing the live conversation. */
+ data class ToolCallRequest(val calls: List) : LiveLlmEvent()
+}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveSessionState.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveSessionState.kt
new file mode 100644
index 00000000..a8893355
--- /dev/null
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveSessionState.kt
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.domain
+
+/** Represents the state of a live voice session. */
+sealed class LiveSessionState {
+ object Idle : LiveSessionState()
+
+ object Connecting : LiveSessionState()
+
+ object Listening : LiveSessionState()
+
+ data class Error(val message: String) : LiveSessionState()
+}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt
index 054c287a..c9ae94f8 100644
--- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LlmProviderFactory.kt
@@ -17,8 +17,11 @@ package com.example.appfunctions.agent.domain
import com.example.appfunctions.agent.data.LlmProviderName
-/** Factory interface to get the correct LlmProvider. */
+/** Factory interface to get the correct [LlmProvider] or [LiveLlmProvider]. */
interface LlmProviderFactory {
- /** Returns the LlmProvider instance for the given name. */
+ /** Returns the [LlmProvider] instance for the given [name]. */
fun getProvider(name: LlmProviderName): LlmProvider
+
+ /** Returns the [LiveLlmProvider] instance for the given [name]. */
+ fun getLiveProvider(name: LlmProviderName): LiveLlmProvider
}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ExecuteToolCallUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ExecuteToolCallUseCase.kt
new file mode 100644
index 00000000..2e14a187
--- /dev/null
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ExecuteToolCallUseCase.kt
@@ -0,0 +1,456 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.domain.appfunction
+
+import android.app.PendingIntent
+import android.content.Context
+import android.content.Intent
+import android.net.Uri
+import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata
+import androidx.appfunctions.metadata.AppFunctionDataTypeMetadata
+import androidx.appfunctions.metadata.AppFunctionMetadata
+import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata
+import androidx.appfunctions.metadata.AppFunctionParameterMetadata
+import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata
+import androidx.core.content.FileProvider
+import com.example.appfunctions.agent.data.AgentInternalTools
+import com.example.appfunctions.agent.data.db.entities.MessageAttachment
+import com.example.appfunctions.agent.domain.LlmResponsePart
+import com.example.appfunctions.agent.domain.ToolOutput
+import dagger.hilt.android.qualifiers.ApplicationContext
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import org.json.JSONObject
+import java.io.File
+import java.net.HttpURLConnection
+import java.net.URL
+import java.util.UUID
+import javax.inject.Inject
+
+/** Result of executing a single [LlmResponsePart.ToolCall]. */
+sealed class ToolCallExecutionResult {
+ data class Success(
+ val toolOutput: ToolOutput,
+ val attachment: MessageAttachment? = null,
+ ) : ToolCallExecutionResult()
+
+ data class PendingIntentRequired(
+ val pendingIntentId: String,
+ val pendingIntent: PendingIntent,
+ ) : ToolCallExecutionResult()
+
+ data class Failure(
+ val errorMessage: String,
+ val exception: Throwable? = null,
+ ) : ToolCallExecutionResult()
+}
+
+/**
+ * Shared pipeline that executes a single [LlmResponsePart.ToolCall] against internal agent tools or
+ * discovered Android AppFunctions, handling remote file reference downloads, recursive content URI
+ * permission grants, argument conversion, and [AppFunctionExceptionFormatter] formatting.
+ */
+class ExecuteToolCallUseCase
+ @Inject
+ constructor(
+ @ApplicationContext private val context: Context,
+ private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase,
+ private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase,
+ private val agentInternalTools: AgentInternalTools,
+ ) {
+ suspend operator fun invoke(
+ toolCall: LlmResponsePart.ToolCall,
+ tools: List,
+ threadId: String,
+ ): ToolCallExecutionResult {
+ if (toolCall.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) {
+ return executeInternalTool(toolCall)
+ .fold(
+ onSuccess = { jsonString ->
+ ToolCallExecutionResult.Success(
+ toolOutput =
+ ToolOutput(
+ functionId = toolCall.functionId,
+ callId = toolCall.callId,
+ result = jsonString,
+ ),
+ attachment = extractAttachment(jsonString),
+ )
+ },
+ onFailure = { exception ->
+ if (exception is CancellationException) {
+ throw exception
+ }
+ ToolCallExecutionResult.Failure(
+ errorMessage =
+ "Internal tool execution failed: ${exception.message}",
+ exception = exception,
+ )
+ },
+ )
+ }
+
+ val matchingTool =
+ tools.find {
+ it.packageName == toolCall.packageName && it.id == toolCall.functionId
+ }
+ ?: return ToolCallExecutionResult.Failure(
+ errorMessage = "Tool not found: ${toolCall.functionId}",
+ )
+
+ val rawConvertedInputs =
+ buildMap {
+ for ((key, value) in toolCall.arguments) {
+ if (value != null) {
+ put(key, value)
+ }
+ }
+ }
+
+ val convertedInputs =
+ try {
+ withContext(Dispatchers.IO) {
+ resolveRemoteFileReferencesRecursively(
+ context = context,
+ parametersMetadata = matchingTool.parameters,
+ inputs = rawConvertedInputs,
+ )
+ }
+ } catch (e: CancellationException) {
+ throw e
+ } catch (e: Exception) {
+ return ToolCallExecutionResult.Failure(
+ errorMessage = "Failed to download remote file reference: ${e.message}",
+ exception = e,
+ )
+ }
+
+ grantContentUriPermissionsRecursively(
+ context = context,
+ targetPackageName = toolCall.packageName,
+ value = convertedInputs,
+ )
+
+ val appFunctionDataResult =
+ withContext(Dispatchers.Default) {
+ convertInputToAppFunctionDataUseCase(
+ parameters = matchingTool.parameters,
+ components = matchingTool.components,
+ inputs = convertedInputs,
+ )
+ }
+ if (appFunctionDataResult.isFailure) {
+ val exception = appFunctionDataResult.exceptionOrNull()
+ if (exception is CancellationException) {
+ throw exception
+ }
+ return ToolCallExecutionResult.Failure(
+ errorMessage =
+ "Failed to convert arguments for ${toolCall.functionId}\n $exception",
+ exception = exception,
+ )
+ }
+
+ return when (
+ val executionResult =
+ executeAppFunctionUseCase(
+ function = matchingTool,
+ parameters = appFunctionDataResult.getOrThrow(),
+ threadId = threadId,
+ )
+ ) {
+ is ExecuteAppFunctionResult.Data -> {
+ ToolCallExecutionResult.Success(
+ toolOutput =
+ ToolOutput(
+ functionId = toolCall.functionId,
+ callId = toolCall.callId,
+ result = executionResult.formattedJson,
+ ),
+ attachment = extractAttachment(executionResult.formattedJson),
+ )
+ }
+
+ is ExecuteAppFunctionResult.PendingIntentAction -> {
+ ToolCallExecutionResult.PendingIntentRequired(
+ pendingIntentId = UUID.randomUUID().toString(),
+ pendingIntent = executionResult.pendingIntent,
+ )
+ }
+
+ is ExecuteAppFunctionResult.Error -> {
+ val exception = executionResult.exception
+ if (exception is CancellationException) {
+ throw exception
+ }
+ val appFunctionException =
+ AppFunctionExceptionFormatter.getAppFunctionException(exception)
+ if (appFunctionException != null) {
+ ToolCallExecutionResult.Success(
+ toolOutput =
+ ToolOutput(
+ functionId = toolCall.functionId,
+ callId = toolCall.callId,
+ result =
+ AppFunctionExceptionFormatter.format(
+ appFunctionException,
+ toolCall.functionId,
+ ),
+ ),
+ )
+ } else {
+ ToolCallExecutionResult.Failure(
+ errorMessage =
+ "Tool execution failed for ${toolCall.functionId}: ${exception.message}",
+ exception = exception,
+ )
+ }
+ }
+ }
+ }
+
+ private fun extractAttachment(jsonString: String): MessageAttachment? {
+ return runCatching {
+ val json = JSONObject(jsonString)
+ val uri = json.optString("imageUri")
+ if (uri.isNotBlank()) {
+ val mimeType =
+ json.optString("mimeType").takeIf { it.isNotBlank() } ?: "image/png"
+ MessageAttachment(uri = uri, mimeType = mimeType)
+ } else {
+ null
+ }
+ }.getOrNull()
+ }
+
+ private fun resolveRemoteFileReferencesRecursively(
+ context: Context,
+ parametersMetadata: List,
+ inputs: Map,
+ ): Map {
+ val paramMap = parametersMetadata.associateBy { it.name }
+ return inputs.mapValues { (key, value) ->
+ val paramMeta = paramMap[key]
+ resolveValueRecursively(context, paramMeta?.dataType, value, key)
+ }
+ }
+
+ private fun resolveValueRecursively(
+ context: Context,
+ dataType: AppFunctionDataTypeMetadata?,
+ value: Any,
+ paramName: String?,
+ ): Any {
+ return when (value) {
+ is String -> {
+ val shouldResolve =
+ isFileReferenceParameter(paramName) || isUriMetadata(dataType)
+ if (
+ shouldResolve &&
+ (value.startsWith("http://") || value.startsWith("https://"))
+ ) {
+ downloadRemoteFileToContentUri(context, value)
+ } else {
+ value
+ }
+ }
+ is Map<*, *> -> {
+ value.entries.associate { entry ->
+ val k = entry.key as String
+ val propType =
+ (dataType as? AppFunctionObjectTypeMetadata)?.properties?.get(k)
+ k to (
+ entry.value?.let {
+ resolveValueRecursively(
+ context,
+ propType,
+ it,
+ k,
+ )
+ } ?: ""
+ )
+ }
+ }
+ is List<*> -> {
+ val itemType = (dataType as? AppFunctionArrayTypeMetadata)?.itemType
+ value.mapNotNull { item ->
+ if (item != null) {
+ resolveValueRecursively(context, itemType, item, paramName)
+ } else {
+ null
+ }
+ }
+ }
+ else -> value
+ }
+ }
+
+ private fun downloadRemoteFileToContentUri(
+ context: Context,
+ urlString: String,
+ ): String {
+ val url = URL(urlString)
+ val connection =
+ (url.openConnection() as HttpURLConnection).apply {
+ connectTimeout = 10000
+ readTimeout = 15000
+ }
+ try {
+ connection.connect()
+ val contentType = connection.contentType ?: ""
+ val ext =
+ when {
+ contentType.contains("png", ignoreCase = true) -> "png"
+ contentType.contains("jpeg", ignoreCase = true) ||
+ contentType.contains("jpg", ignoreCase = true) -> "jpg"
+ contentType.contains("gif", ignoreCase = true) -> "gif"
+ contentType.contains("webp", ignoreCase = true) -> "webp"
+ urlString.substringAfterLast("/", "").contains(".") ->
+ urlString.substringAfterLast("/").substringAfterLast(".")
+ else -> "jpg"
+ }
+ val cacheDir = File(context.cacheDir, "file_references").apply { mkdirs() }
+ val file = File(cacheDir, "generated_${UUID.randomUUID()}.$ext")
+ connection.inputStream.use { input ->
+ file.outputStream().use { output ->
+ input.copyTo(output)
+ }
+ }
+ val contentUri =
+ FileProvider.getUriForFile(
+ context,
+ "${context.packageName}.fileprovider",
+ file,
+ )
+ return contentUri.toString()
+ } finally {
+ connection.disconnect()
+ }
+ }
+
+ private fun grantContentUriPermissionsRecursively(
+ context: Context,
+ targetPackageName: String,
+ value: Any?,
+ ) {
+ when (value) {
+ is String -> {
+ if (value.startsWith("content://")) {
+ runCatching {
+ val uri = Uri.parse(value)
+ context.grantUriPermission(
+ targetPackageName,
+ uri,
+ Intent.FLAG_GRANT_READ_URI_PERMISSION,
+ )
+ }
+ }
+ }
+ is Map<*, *> ->
+ value.values.forEach {
+ grantContentUriPermissionsRecursively(context, targetPackageName, it)
+ }
+ is List<*> ->
+ value.forEach {
+ grantContentUriPermissionsRecursively(context, targetPackageName, it)
+ }
+ }
+ }
+
+ private fun isFileReferenceParameter(parameterName: String?): Boolean {
+ if (parameterName == null) return false
+ if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true
+ return parameterName.endsWith("Uri", ignoreCase = true) ||
+ parameterName.endsWith("Uris", ignoreCase = true)
+ }
+
+ private fun isUriMetadata(dataType: AppFunctionDataTypeMetadata?): Boolean {
+ if (dataType == null) return false
+ if (dataType is AppFunctionObjectTypeMetadata && dataType.qualifiedName == "android.net.Uri") {
+ return true
+ }
+ if (dataType is AppFunctionReferenceTypeMetadata && dataType.referenceDataType == "android.net.Uri") {
+ return true
+ }
+ return false
+ }
+
+ private suspend fun executeInternalTool(toolCall: LlmResponsePart.ToolCall): Result {
+ return runCatching {
+ when (toolCall.functionId) {
+ "getCurrentLocation" -> {
+ val location = agentInternalTools.getCurrentLocation()
+ if (location != null) {
+ JSONObject().apply {
+ put("latitude", location.latitude)
+ put("longitude", location.longitude)
+ }.toString()
+ } else {
+ "null"
+ }
+ }
+ "geocodeAddress" -> {
+ val address =
+ toolCall.arguments["address"] as? String
+ ?: throw IllegalArgumentException(
+ "Missing required parameter 'address'",
+ )
+ val location = agentInternalTools.geocodeAddress(address)
+ if (location != null) {
+ JSONObject().apply {
+ put("latitude", location.latitude)
+ put("longitude", location.longitude)
+ }.toString()
+ } else {
+ "null"
+ }
+ }
+ "generateImage" -> {
+ val prompt =
+ toolCall.arguments["prompt"] as? String
+ ?: throw IllegalArgumentException(
+ "Missing required parameter 'prompt'",
+ )
+ val aspectRatio = toolCall.arguments["aspectRatio"] as? String
+ val result = agentInternalTools.generateImage(prompt, aspectRatio)
+ JSONObject().apply {
+ put("imageUri", result.imageUri)
+ put("mimeType", result.mimeType)
+ put("prompt", result.prompt)
+ }.toString()
+ }
+ else ->
+ throw IllegalArgumentException(
+ "Unknown internal tool: ${toolCall.functionId}",
+ )
+ }
+ }
+ }
+
+ companion object {
+ private val KNOWN_FILE_REFERENCE_PARAM_NAMES =
+ setOf(
+ "wallpaperUri",
+ "imageUri",
+ "attachmentUri",
+ "ringtoneUri",
+ "profilePictureUri",
+ "audioUri",
+ "voiceNoteUri",
+ )
+ }
+ }
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetEnabledToolsUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetEnabledToolsUseCase.kt
new file mode 100644
index 00000000..164624c8
--- /dev/null
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetEnabledToolsUseCase.kt
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.domain.appfunction
+
+import androidx.appfunctions.metadata.AppFunctionMetadata
+import com.example.appfunctions.agent.data.AgentInternalTools
+import com.example.appfunctions.agent.data.SettingsRepository
+import kotlinx.coroutines.flow.first
+import javax.inject.Inject
+
+/**
+ * Use case to discover all currently enabled AppFunction and internal agent tools, filtered by
+ * connected apps and optional [targetPackageName] scoping.
+ */
+class GetEnabledToolsUseCase
+ @Inject
+ constructor(
+ private val settingsRepository: SettingsRepository,
+ private val getAppFunctionsUseCase: GetAppFunctionsUseCase,
+ private val getAppFunctionStatesUseCase: GetAppFunctionStatesUseCase,
+ private val agentInternalTools: AgentInternalTools,
+ ) {
+ suspend operator fun invoke(targetPackageName: String? = null): List {
+ val disconnectedApps = settingsRepository.disconnectedApps.first()
+ val internalTools = agentInternalTools.getInternalToolsMetadata()
+ val allTools =
+ getAppFunctionsUseCase().first().values.flatten() +
+ internalTools.map { it.first }
+
+ val functionNames = allTools.map { it.name }
+ val states =
+ getAppFunctionStatesUseCase(functionNames) +
+ internalTools.map { it.second }
+ val stateMap = states.associateBy { it.functionName }
+
+ return allTools
+ .filter { metadata ->
+ val isEnabled =
+ if (metadata.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) {
+ true
+ } else {
+ stateMap[metadata.name]?.isEnabled ?: false
+ }
+ isEnabled &&
+ metadata.packageName !in disconnectedApps &&
+ (targetPackageName == null || metadata.packageName == targetPackageName)
+ }
+ .sortedByDescending { metadata -> metadata.id.startsWith(metadata.packageName) }
+ .distinctBy { metadata -> metadata.id }
+ }
+ }
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/contracts/SettingsScreenLayout.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/contracts/SettingsScreenLayout.kt
index 103c7720..5d0047e1 100644
--- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/contracts/SettingsScreenLayout.kt
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/contracts/SettingsScreenLayout.kt
@@ -18,6 +18,7 @@ package com.example.appfunctions.agent.ui.contracts
import androidx.compose.foundation.text.input.TextFieldState
import androidx.compose.runtime.Composable
import androidx.compose.ui.Modifier
+import com.example.appfunctions.agent.data.GeminiLiveVoice
import com.example.appfunctions.agent.data.ServiceTier
/**
@@ -33,5 +34,7 @@ interface SettingsScreenLayout {
onOpenLicenses: () -> Unit,
onNavigateToConnectedApps: () -> Unit,
modifier: Modifier,
+ geminiLiveVoice: GeminiLiveVoice = GeminiLiveVoice.DEFAULT,
+ onGeminiLiveVoiceSelected: (GeminiLiveVoice) -> Unit = {},
)
}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt
index b1c9f3b3..0966db4e 100644
--- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt
@@ -48,6 +48,7 @@ import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.automirrored.filled.Send
import androidx.compose.material.icons.filled.Add
import androidx.compose.material.icons.filled.ArrowDropDown
+import androidx.compose.material.icons.filled.GraphicEq
import androidx.compose.material.icons.filled.Menu
import androidx.compose.material.icons.filled.Warning
import androidx.compose.material3.ButtonDefaults
@@ -129,6 +130,7 @@ import com.example.appfunctions.agent.data.db.entities.ThreadEntity
import com.example.appfunctions.agent.domain.AgentStatus
import com.example.appfunctions.agent.domain.appfunction.AppInfo
import com.example.appfunctions.agent.ui.screens.debugging.LazyExposedDropdownMenu
+import com.example.appfunctions.agent.voice.rememberLiveVoiceSessionLauncher
import com.mikepenz.markdown.m3.Markdown
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.launch
@@ -136,8 +138,13 @@ import kotlinx.coroutines.launch
@Composable
fun AgentDemoScreen(viewModel: AgentDemoViewModel = hiltViewModel()) {
val uiState by viewModel.uiState.collectAsStateWithLifecycle()
+ val launchLiveSession = rememberLiveVoiceSessionLauncher()
- AgentDemoContent(uiState = uiState, onEvent = viewModel::onEvent)
+ AgentDemoContent(
+ uiState = uiState,
+ onEvent = viewModel::onEvent,
+ onLaunchLiveSession = launchLiveSession,
+ )
}
@OptIn(ExperimentalMaterial3Api::class)
@@ -146,6 +153,7 @@ fun AgentDemoContent(
uiState: AgentUiState,
onEvent: (AgentUiEvent) -> Unit,
initialSidePanelVisible: Boolean = false,
+ onLaunchLiveSession: (threadId: String, targetPackageName: String?) -> Unit = { _, _ -> },
) {
val context = LocalContext.current
val packageManager = context.packageManager
@@ -175,6 +183,7 @@ fun AgentDemoContent(
scope = scope,
packageManager = packageManager,
initialSidePanelVisible = initialSidePanelVisible,
+ onLaunchLiveSession = onLaunchLiveSession,
)
}
}
@@ -225,6 +234,7 @@ fun AgentDemoLoadedScreen(
scope: CoroutineScope,
packageManager: PackageManager,
initialSidePanelVisible: Boolean = false,
+ onLaunchLiveSession: (threadId: String, targetPackageName: String?) -> Unit = { _, _ -> },
) {
var messageText by remember { mutableStateOf(TextFieldValue("")) }
var isSidePanelVisible by remember { mutableStateOf(initialSidePanelVisible) }
@@ -440,17 +450,37 @@ fun AgentDemoLoadedScreen(
focusedBorderColor = Color.Transparent,
),
trailingIcon = {
- IconButton(
- onClick = sendMessage,
- enabled =
- messageText.text.isNotBlank() &&
- uiState.status == AgentStatus.Idle,
+ Row(
+ verticalAlignment = Alignment.CenterVertically,
+ modifier = Modifier.padding(end = 4.dp),
) {
- Icon(
- imageVector = Icons.AutoMirrored.Filled.Send,
- contentDescription =
- stringResource(R.string.agent_demo_send),
- )
+ IconButton(
+ onClick = {
+ onLaunchLiveSession(
+ uiState.currentThread.threadId,
+ selectedAppPackageName,
+ )
+ },
+ ) {
+ Icon(
+ imageVector = Icons.Default.GraphicEq,
+ contentDescription =
+ stringResource(R.string.agent_demo_live_start),
+ tint = MaterialTheme.colorScheme.primary,
+ )
+ }
+ IconButton(
+ onClick = sendMessage,
+ enabled =
+ messageText.text.isNotBlank() &&
+ uiState.status == AgentStatus.Idle,
+ ) {
+ Icon(
+ imageVector = Icons.AutoMirrored.Filled.Send,
+ contentDescription =
+ stringResource(R.string.agent_demo_send),
+ )
+ }
}
},
)
@@ -528,6 +558,7 @@ fun ModelDropdown(
onMenuClick: () -> Unit,
) {
var expanded by remember { mutableStateOf(false) }
+
ExposedDropdownMenuBox(
modifier = modifier,
expanded = expanded,
@@ -565,10 +596,7 @@ fun ModelDropdown(
Modifier
.weight(1f)
.fillMaxHeight()
- .menuAnchor(
- ExposedDropdownMenuAnchorType.PrimaryEditable,
- enabled = true,
- ),
+ .menuAnchor(ExposedDropdownMenuAnchorType.PrimaryEditable),
verticalAlignment = Alignment.CenterVertically,
) {
Column(modifier = Modifier.weight(1f)) {
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreen.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreen.kt
index b10ad052..63dbf6ab 100644
--- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreen.kt
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreen.kt
@@ -23,6 +23,7 @@ import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.consumeWindowInsets
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
+import androidx.compose.foundation.layout.heightIn
import androidx.compose.foundation.layout.imePadding
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.rememberScrollState
@@ -70,6 +71,7 @@ import androidx.hilt.navigation.compose.hiltViewModel
import androidx.lifecycle.compose.collectAsStateWithLifecycle
import com.example.appfunctions.agent.BuildConfig
import com.example.appfunctions.agent.R
+import com.example.appfunctions.agent.data.GeminiLiveVoice
import com.example.appfunctions.agent.data.ServiceTier
import com.google.android.gms.oss.licenses.v2.OssLicensesMenuActivity
import kotlin.OptIn
@@ -93,6 +95,8 @@ fun SettingsScreen(
geminiApiKeyState = viewModel.geminiApiKeyState,
serviceTier = uiState.serviceTier,
onServiceTierSelected = viewModel::setServiceTier,
+ geminiLiveVoice = uiState.geminiLiveVoice,
+ onGeminiLiveVoiceSelected = viewModel::setGeminiLiveVoice,
onOpenLicenses = onOpenLicensesClick,
onNavigateToConnectedApps = onNavigateToConnectedApps,
)
@@ -107,6 +111,8 @@ fun SettingsScreenContent(
onServiceTierSelected: (ServiceTier) -> Unit,
onOpenLicenses: () -> Unit,
onNavigateToConnectedApps: () -> Unit,
+ geminiLiveVoice: GeminiLiveVoice = GeminiLiveVoice.DEFAULT,
+ onGeminiLiveVoiceSelected: (GeminiLiveVoice) -> Unit = {},
) {
Scaffold(
modifier = Modifier.fillMaxSize(),
@@ -205,6 +211,24 @@ fun SettingsScreenContent(
)
}
+ Column(modifier = Modifier.padding(horizontal = 16.dp, vertical = 8.dp)) {
+ Text(
+ text = stringResource(id = R.string.settings_live_voice),
+ style = MaterialTheme.typography.bodyMedium,
+ modifier = Modifier.padding(bottom = 8.dp),
+ )
+ GeminiLiveVoiceDropdown(
+ selectedVoice = geminiLiveVoice,
+ onVoiceSelected = onGeminiLiveVoiceSelected,
+ )
+ Text(
+ text = stringResource(id = R.string.settings_live_voice_description),
+ style = MaterialTheme.typography.bodySmall,
+ color = MaterialTheme.colorScheme.onSurfaceVariant,
+ modifier = Modifier.padding(top = 4.dp),
+ )
+ }
+
ListItem(
headlineContent = {
Text(
@@ -325,3 +349,65 @@ private fun ServiceTier.labelRes(): Int =
ServiceTier.STANDARD -> R.string.settings_service_tier_standard
ServiceTier.PRIORITY -> R.string.settings_service_tier_priority
}
+
+/** Dropdown allowing the user to pick the [GeminiLiveVoice] for Gemini Live voice sessions. */
+@OptIn(ExperimentalMaterial3Api::class)
+@Composable
+private fun GeminiLiveVoiceDropdown(
+ selectedVoice: GeminiLiveVoice,
+ onVoiceSelected: (GeminiLiveVoice) -> Unit,
+) {
+ var expanded by remember { mutableStateOf(false) }
+ ExposedDropdownMenuBox(
+ expanded = expanded,
+ onExpandedChange = { expanded = it },
+ ) {
+ OutlinedTextField(
+ value =
+ stringResource(
+ id = R.string.settings_live_voice_option,
+ selectedVoice.apiName,
+ selectedVoice.style,
+ ),
+ onValueChange = {},
+ readOnly = true,
+ singleLine = true,
+ trailingIcon = {
+ ExposedDropdownMenuDefaults.TrailingIcon(expanded = expanded)
+ },
+ shape = CircleShape,
+ colors =
+ ExposedDropdownMenuDefaults.outlinedTextFieldColors(
+ unfocusedContainerColor = MaterialTheme.colorScheme.surfaceBright,
+ focusedContainerColor = MaterialTheme.colorScheme.surfaceBright,
+ ),
+ modifier =
+ Modifier.fillMaxWidth()
+ .menuAnchor(ExposedDropdownMenuAnchorType.PrimaryNotEditable, enabled = true),
+ )
+ ExposedDropdownMenu(
+ expanded = expanded,
+ onDismissRequest = { expanded = false },
+ modifier = Modifier.heightIn(max = 320.dp),
+ ) {
+ GeminiLiveVoice.entries.forEach { voice ->
+ DropdownMenuItem(
+ text = {
+ Text(
+ text =
+ stringResource(
+ id = R.string.settings_live_voice_option,
+ voice.apiName,
+ voice.style,
+ ),
+ )
+ },
+ onClick = {
+ onVoiceSelected(voice)
+ expanded = false
+ },
+ )
+ }
+ }
+ }
+}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsUiState.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsUiState.kt
index 3c6bada4..60dfaba6 100644
--- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsUiState.kt
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsUiState.kt
@@ -15,6 +15,7 @@
*/
package com.example.appfunctions.agent.ui.screens.agentdemo
+import com.example.appfunctions.agent.data.GeminiLiveVoice
import com.example.appfunctions.agent.data.LlmProviderName
import com.example.appfunctions.agent.data.ServiceTier
@@ -22,4 +23,5 @@ import com.example.appfunctions.agent.data.ServiceTier
data class SettingsUiState(
val selectedProvider: LlmProviderName = LlmProviderName.GEMINI,
val serviceTier: ServiceTier = ServiceTier.STANDARD,
+ val geminiLiveVoice: GeminiLiveVoice = GeminiLiveVoice.DEFAULT,
)
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModel.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModel.kt
index b9092e69..9c372f5e 100644
--- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModel.kt
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModel.kt
@@ -19,6 +19,7 @@ import androidx.compose.foundation.text.input.TextFieldState
import androidx.compose.foundation.text.input.setTextAndPlaceCursorAtEnd
import androidx.lifecycle.ViewModel
import androidx.lifecycle.viewModelScope
+import com.example.appfunctions.agent.data.GeminiLiveVoice
import com.example.appfunctions.agent.data.LlmProviderName
import com.example.appfunctions.agent.data.ServiceTier
import com.example.appfunctions.agent.data.SettingsRepository
@@ -46,8 +47,13 @@ class SettingsViewModel
combine(
settingsRepository.selectedProvider,
settingsRepository.serviceTier,
- ) { provider, serviceTier ->
- SettingsUiState(selectedProvider = provider, serviceTier = serviceTier)
+ settingsRepository.geminiLiveVoice,
+ ) { provider, serviceTier, geminiLiveVoice ->
+ SettingsUiState(
+ selectedProvider = provider,
+ serviceTier = serviceTier,
+ geminiLiveVoice = geminiLiveVoice,
+ )
}
.stateIn(
scope = viewModelScope,
@@ -79,4 +85,8 @@ class SettingsViewModel
fun setServiceTier(tier: ServiceTier) {
viewModelScope.launch { settingsRepository.setServiceTier(tier) }
}
+
+ fun setGeminiLiveVoice(voice: GeminiLiveVoice) {
+ viewModelScope.launch { settingsRepository.setGeminiLiveVoice(voice) }
+ }
}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentRecognitionService.kt b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentRecognitionService.kt
new file mode 100644
index 00000000..e1a7e355
--- /dev/null
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentRecognitionService.kt
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.voice
+
+import android.content.Intent
+import android.speech.RecognitionService
+import android.speech.SpeechRecognizer
+
+/**
+ * Companion [RecognitionService] required by Android's `VoiceInteractionServiceInfo` manifest
+ * metadata validation so [AgentVoiceInteractionService] is selectable as the device's Default
+ * Digital Assistant (`ROLE_ASSISTANT`).
+ */
+class AgentRecognitionService : RecognitionService() {
+ override fun onStartListening(
+ recognizerIntent: Intent?,
+ listener: Callback?,
+ ) {
+ runCatching { listener?.error(SpeechRecognizer.ERROR_CLIENT) }
+ }
+
+ override fun onCancel(listener: Callback?) = Unit
+
+ override fun onStopListening(listener: Callback?) = Unit
+}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionService.kt b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionService.kt
new file mode 100644
index 00000000..4a23a1d5
--- /dev/null
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionService.kt
@@ -0,0 +1,167 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.voice
+
+import android.app.Activity
+import android.content.ComponentName
+import android.content.Context
+import android.content.ContextWrapper
+import android.content.Intent
+import android.os.Bundle
+import android.provider.Settings
+import android.service.voice.VoiceInteractionService
+import android.util.Log
+import java.util.concurrent.atomic.AtomicReference
+
+/**
+ * Android [VoiceInteractionService] entry point for Gemini Live voice sessions.
+ *
+ * Running Live voice sessions inside a [VoiceInteractionService] / [AgentVoiceInteractionSession]
+ * keeps microphone recording active and renders a system-wide floating overlay even when
+ * `MainActivity` moves to the background (for example, when an AppFunction launches a confirmation
+ * `PendingIntent` or the user switches into a target app like `ChatApp`).
+ */
+class AgentVoiceInteractionService : VoiceInteractionService() {
+ @Volatile private var isServiceReady: Boolean = false
+
+ override fun onReady() {
+ super.onReady()
+ isServiceReady = true
+ activeInstance = this
+ Log.d(TAG, "AgentVoiceInteractionService onReady")
+ }
+
+ override fun onShutdown() {
+ isServiceReady = false
+ if (activeInstance === this) {
+ activeInstance = null
+ }
+ super.onShutdown()
+ }
+
+ override fun onDestroy() {
+ isServiceReady = false
+ if (activeInstance === this) {
+ activeInstance = null
+ }
+ super.onDestroy()
+ }
+
+ companion object {
+ private const val TAG = "AgentVoiceService"
+
+ const val EXTRA_THREAD_ID = "extra_thread_id"
+ const val EXTRA_TARGET_PACKAGE_NAME = "extra_target_package_name"
+
+ @Volatile private var activeInstance: AgentVoiceInteractionService? = null
+ private val pendingSessionArgs = AtomicReference(null)
+
+ internal var isActiveServiceChecker: (Context) -> Boolean = { context ->
+ isActiveService(
+ context,
+ ComponentName(context, AgentVoiceInteractionService::class.java),
+ )
+ }
+
+ /** Returns true if [AgentVoiceInteractionService] is currently the active assistant service. */
+ fun isVoiceServiceActive(context: Context): Boolean = isActiveServiceChecker(context)
+
+ /** Builds the session arguments [Bundle] for [threadId] and optional [targetPackageName]. */
+ fun createSessionArgs(
+ threadId: String?,
+ targetPackageName: String? = null,
+ ): Bundle =
+ Bundle().apply {
+ if (threadId != null) {
+ putString(EXTRA_THREAD_ID, threadId)
+ }
+ if (targetPackageName != null) {
+ putString(EXTRA_TARGET_PACKAGE_NAME, targetPackageName)
+ }
+ }
+
+ /** Clears any queued pending session request. */
+ internal fun clearPendingSession() {
+ pendingSessionArgs.set(null)
+ }
+
+ internal fun consumePendingSession(): Bundle? = pendingSessionArgs.getAndSet(null)
+
+ internal fun resetForTesting() {
+ activeInstance = null
+ pendingSessionArgs.set(null)
+ isActiveServiceChecker = { context ->
+ isActiveService(
+ context,
+ ComponentName(context, AgentVoiceInteractionService::class.java),
+ )
+ }
+ }
+
+ /**
+ * Triggers [showSession] if [AgentVoiceInteractionService] is currently the active and
+ * bound [VoiceInteractionService]. Returns `false` if the user must select this app in
+ * Digital Assistant settings.
+ */
+ fun launchLiveSession(
+ context: Context,
+ threadId: String,
+ targetPackageName: String? = null,
+ ): Boolean {
+ if (!isVoiceServiceActive(context)) {
+ Log.d(TAG, "AgentVoiceInteractionService is not the active digital assistant")
+ return false
+ }
+ val args = createSessionArgs(threadId, targetPackageName)
+ val instance = activeInstance
+ if (instance != null && instance.isServiceReady) {
+ clearPendingSession()
+ Log.d(TAG, "Showing live voice session via bound service")
+ instance.showSession(args, 0)
+ return true
+ }
+
+ pendingSessionArgs.set(args)
+ val showedViaActivity =
+ runCatching { context.findActivity()?.showAssist(args) == true }
+ .getOrDefault(false)
+ if (!showedViaActivity) {
+ clearPendingSession()
+ Log.w(TAG, "Voice service is selected in settings but not currently bound")
+ return false
+ }
+ return true
+ }
+
+ /**
+ * Creates an [Intent] opening Android's Digital Assistant picker settings
+ * ([Settings.ACTION_VOICE_INPUT_SETTINGS]).
+ *
+ * Note: `RoleManager.ROLE_ASSISTANT` is marked `requestable="false"` in AOSP
+ * `PermissionController`, so `RoleManager.createRequestRoleIntent` immediately finishes
+ * without showing UI; Digital Assistant selection must go through
+ * [Settings.ACTION_VOICE_INPUT_SETTINGS].
+ */
+ fun createAssistantSettingsIntent(): Intent = Intent(Settings.ACTION_VOICE_INPUT_SETTINGS)
+
+ private tailrec fun Context.findActivity(): Activity? =
+ when (this) {
+ is Activity -> this
+ is ContextWrapper -> baseContext.findActivity()
+ else -> null
+ }
+ }
+}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSession.kt b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSession.kt
new file mode 100644
index 00000000..81ac5ce7
--- /dev/null
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSession.kt
@@ -0,0 +1,317 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.voice
+
+import android.content.Context
+import android.content.Intent
+import android.graphics.Rect
+import android.os.Bundle
+import android.service.voice.VoiceInteractionSession
+import android.view.View
+import android.view.ViewGroup
+import android.view.WindowManager
+import androidx.compose.runtime.getValue
+import androidx.compose.ui.platform.ComposeView
+import androidx.compose.ui.platform.ViewCompositionStrategy
+import androidx.lifecycle.Lifecycle
+import androidx.lifecycle.LifecycleOwner
+import androidx.lifecycle.LifecycleRegistry
+import androidx.lifecycle.compose.collectAsStateWithLifecycle
+import androidx.lifecycle.setViewTreeLifecycleOwner
+import androidx.savedstate.SavedStateRegistry
+import androidx.savedstate.SavedStateRegistryController
+import androidx.savedstate.SavedStateRegistryOwner
+import androidx.savedstate.setViewTreeSavedStateRegistryOwner
+import com.example.appfunctions.agent.MainActivity
+import com.example.appfunctions.agent.data.LlmModel
+import com.example.appfunctions.agent.domain.LiveAgentOrchestrator
+import com.example.appfunctions.agent.domain.LiveSessionState
+import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase
+import com.example.appfunctions.agent.ui.theme.AppFunctionsAgentTheme
+import kotlinx.coroutines.CoroutineDispatcher
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.cancel
+import kotlinx.coroutines.flow.first
+import kotlinx.coroutines.launch
+
+/**
+ * System-wide [VoiceInteractionSession] hosting the floating Live voice session UI and driving
+ * [LiveAgentOrchestrator].
+ *
+ * Uses [Insets.TOUCHABLE_INSETS_REGION] and non-modal window flags so touches outside the floating
+ * Live pill pass directly through to underlying apps (such as `MainActivity` confirmation buttons
+ * or target apps like `ChatApp`).
+ */
+class AgentVoiceInteractionSession(
+ context: Context,
+ private val liveAgentOrchestrator: LiveAgentOrchestrator,
+ private val manageThreadsUseCase: ManageThreadsUseCase,
+ mainDispatcher: CoroutineDispatcher = Dispatchers.Main.immediate,
+) : VoiceInteractionSession(context), LifecycleOwner, SavedStateRegistryOwner {
+ private val lifecycleRegistry = LifecycleRegistry(this)
+ private val savedStateRegistryController = SavedStateRegistryController.create(this)
+ private val sessionScope = CoroutineScope(SupervisorJob() + mainDispatcher)
+
+ private val overlayBounds = Rect()
+
+ private var currentThreadId: String? = null
+ private var currentTargetPackageName: String? = null
+ private var startupJob: Job? = null
+ private var stateMonitorJob: Job? = null
+
+ override val lifecycle: Lifecycle
+ get() = lifecycleRegistry
+
+ override val savedStateRegistry: SavedStateRegistry
+ get() = savedStateRegistryController.savedStateRegistry
+
+ override fun onCreate() {
+ super.onCreate()
+ savedStateRegistryController.performRestore(null)
+ lifecycleRegistry.handleLifecycleEvent(Lifecycle.Event.ON_CREATE)
+ configurePassThroughWindow()
+ }
+
+ override fun onCreateContentView(): View {
+ configurePassThroughWindow()
+ window?.window?.decorView?.let { decorView ->
+ decorView.setViewTreeLifecycleOwner(this)
+ decorView.setViewTreeSavedStateRegistryOwner(this)
+ }
+
+ return ComposeView(context).apply {
+ setViewTreeLifecycleOwner(this@AgentVoiceInteractionSession)
+ setViewTreeSavedStateRegistryOwner(this@AgentVoiceInteractionSession)
+ setViewCompositionStrategy(ViewCompositionStrategy.DisposeOnViewTreeLifecycleDestroyed)
+ setContent {
+ AppFunctionsAgentTheme {
+ val liveState by
+ liveAgentOrchestrator.state.collectAsStateWithLifecycle()
+ val agentStatus by
+ liveAgentOrchestrator.status.collectAsStateWithLifecycle()
+
+ LiveVoiceSessionOverlay(
+ liveSessionState = liveState,
+ agentStatus = agentStatus,
+ onStopClick = ::stopAndHide,
+ onRetryClick = ::retrySession,
+ onOpenChatClick = ::openChat,
+ onOverlayBoundsChanged = ::updateOverlayBounds,
+ )
+ }
+ }
+ }
+ }
+
+ override fun onComputeInsets(outInsets: Insets) {
+ super.onComputeInsets(outInsets)
+ outInsets.contentInsets.setEmpty()
+ outInsets.touchableInsets = Insets.TOUCHABLE_INSETS_REGION
+ outInsets.touchableRegion.set(overlayBounds)
+ }
+
+ override fun onShow(
+ args: Bundle?,
+ showFlags: Int,
+ ) {
+ super.onShow(args, showFlags)
+ val effectiveArgs =
+ if (args?.containsKey(AgentVoiceInteractionService.EXTRA_THREAD_ID) == true) {
+ AgentVoiceInteractionService.clearPendingSession()
+ args
+ } else {
+ AgentVoiceInteractionService.consumePendingSession() ?: args
+ }
+ configurePassThroughWindow()
+ window?.window?.decorView?.let { decorView ->
+ decorView.setViewTreeLifecycleOwner(this)
+ decorView.setViewTreeSavedStateRegistryOwner(this)
+ }
+
+ if (lifecycleRegistry.currentState != Lifecycle.State.DESTROYED) {
+ lifecycleRegistry.handleLifecycleEvent(Lifecycle.Event.ON_START)
+ lifecycleRegistry.handleLifecycleEvent(Lifecycle.Event.ON_RESUME)
+ }
+
+ val requestedThreadId =
+ effectiveArgs?.getString(AgentVoiceInteractionService.EXTRA_THREAD_ID)
+ val requestedTargetPkg =
+ effectiveArgs?.getString(AgentVoiceInteractionService.EXTRA_TARGET_PACKAGE_NAME)
+
+ startOrAttachSession(
+ requestedThreadId = requestedThreadId,
+ requestedTargetPackageName = requestedTargetPkg,
+ )
+ }
+
+ override fun onCloseSystemDialogs() {
+ // Keep the floating Live pill visible when the user presses Home to switch to another app
+ // (e.g. ChatApp) while a voice conversation is active.
+ val state = liveAgentOrchestrator.state.value
+ val isActive =
+ state is LiveSessionState.Connecting || state is LiveSessionState.Listening
+ if (!isActive) {
+ super.onCloseSystemDialogs()
+ }
+ }
+
+ override fun onHide() {
+ stateMonitorJob?.cancel()
+ stateMonitorJob = null
+ startupJob?.cancel()
+ startupJob = null
+ liveAgentOrchestrator.stopSession()
+
+ if (lifecycleRegistry.currentState.isAtLeast(Lifecycle.State.STARTED)) {
+ if (lifecycleRegistry.currentState.isAtLeast(Lifecycle.State.RESUMED)) {
+ lifecycleRegistry.handleLifecycleEvent(Lifecycle.Event.ON_PAUSE)
+ }
+ lifecycleRegistry.handleLifecycleEvent(Lifecycle.Event.ON_STOP)
+ }
+ super.onHide()
+ }
+
+ override fun onDestroy() {
+ stateMonitorJob?.cancel()
+ stateMonitorJob = null
+ startupJob?.cancel()
+ startupJob = null
+ liveAgentOrchestrator.stopSession()
+ sessionScope.cancel()
+
+ if (lifecycleRegistry.currentState.isAtLeast(Lifecycle.State.CREATED)) {
+ lifecycleRegistry.handleLifecycleEvent(Lifecycle.Event.ON_DESTROY)
+ }
+ super.onDestroy()
+ }
+
+ internal fun stopAndHide() {
+ stateMonitorJob?.cancel()
+ stateMonitorJob = null
+ startupJob?.cancel()
+ startupJob = null
+ liveAgentOrchestrator.stopSession()
+ hide()
+ }
+
+ internal fun retrySession() {
+ startOrAttachSession(
+ requestedThreadId = currentThreadId,
+ requestedTargetPackageName = currentTargetPackageName,
+ )
+ }
+
+ internal fun openChat() {
+ val intent =
+ Intent(context, MainActivity::class.java).apply {
+ action = MainActivity.ACTION_OPEN_CHAT
+ addFlags(
+ Intent.FLAG_ACTIVITY_NEW_TASK or
+ Intent.FLAG_ACTIVITY_SINGLE_TOP or
+ Intent.FLAG_ACTIVITY_CLEAR_TOP,
+ )
+ currentThreadId?.let { putExtra(MainActivity.EXTRA_OPEN_THREAD_ID, it) }
+ }
+ context.startActivity(intent)
+ }
+
+ internal fun updateOverlayBounds(bounds: Rect) {
+ if (overlayBounds != bounds) {
+ overlayBounds.set(bounds)
+ window?.window?.decorView?.let { decorView ->
+ decorView.requestLayout()
+ decorView.invalidate()
+ }
+ }
+ }
+
+ private fun startOrAttachSession(
+ requestedThreadId: String?,
+ requestedTargetPackageName: String?,
+ ) {
+ stateMonitorJob?.cancel()
+ startupJob?.cancel()
+
+ val currentState = liveAgentOrchestrator.state.value
+ val isCurrentlyRunning =
+ currentState is LiveSessionState.Connecting ||
+ currentState is LiveSessionState.Listening
+ val matchesRunningSession =
+ requestedThreadId == null ||
+ (
+ requestedThreadId == currentThreadId &&
+ requestedTargetPackageName == currentTargetPackageName
+ )
+
+ if (isCurrentlyRunning && matchesRunningSession) {
+ monitorSessionState(initiallyActive = true)
+ return
+ }
+
+ startupJob =
+ sessionScope.launch {
+ val resolvedThreadId = requestedThreadId ?: resolveOrCreateDefaultThreadId()
+ currentThreadId = resolvedThreadId
+ currentTargetPackageName = requestedTargetPackageName
+ liveAgentOrchestrator.startSession(
+ threadId = resolvedThreadId,
+ targetPackageName = requestedTargetPackageName,
+ )
+ monitorSessionState(initiallyActive = true)
+ }
+ }
+
+ private fun monitorSessionState(initiallyActive: Boolean) {
+ stateMonitorJob?.cancel()
+ stateMonitorJob =
+ sessionScope.launch {
+ var sawActiveSession = initiallyActive
+ liveAgentOrchestrator.state.collect { state ->
+ if (
+ state is LiveSessionState.Connecting ||
+ state is LiveSessionState.Listening
+ ) {
+ sawActiveSession = true
+ } else if (sawActiveSession && state is LiveSessionState.Idle) {
+ hide()
+ }
+ }
+ }
+ }
+
+ private suspend fun resolveOrCreateDefaultThreadId(): String {
+ val existingThread = manageThreadsUseCase.getThreads().first().firstOrNull()
+ return existingThread?.threadId ?: manageThreadsUseCase.createThread(LlmModel.DEFAULT)
+ }
+
+ private fun configurePassThroughWindow() {
+ window?.window?.let { win ->
+ win.clearFlags(WindowManager.LayoutParams.FLAG_DIM_BEHIND)
+ win.addFlags(
+ WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL or
+ WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE,
+ )
+ win.setBackgroundDrawableResource(android.R.color.transparent)
+ win.setLayout(
+ ViewGroup.LayoutParams.MATCH_PARENT,
+ ViewGroup.LayoutParams.MATCH_PARENT,
+ )
+ }
+ }
+}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionService.kt b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionService.kt
new file mode 100644
index 00000000..8500f502
--- /dev/null
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionService.kt
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.voice
+
+import android.os.Bundle
+import android.service.voice.VoiceInteractionSession
+import android.service.voice.VoiceInteractionSessionService
+import com.example.appfunctions.agent.domain.LiveAgentOrchestrator
+import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase
+import dagger.hilt.android.AndroidEntryPoint
+import javax.inject.Inject
+
+/**
+ * Hilt-injected [VoiceInteractionSessionService] factory that creates [AgentVoiceInteractionSession]
+ * instances backed by [LiveAgentOrchestrator].
+ */
+@AndroidEntryPoint
+class AgentVoiceInteractionSessionService : VoiceInteractionSessionService() {
+ @Inject lateinit var liveAgentOrchestrator: LiveAgentOrchestrator
+
+ @Inject lateinit var manageThreadsUseCase: ManageThreadsUseCase
+
+ override fun onNewSession(args: Bundle?): VoiceInteractionSession {
+ return AgentVoiceInteractionSession(
+ context = this,
+ liveAgentOrchestrator = liveAgentOrchestrator,
+ manageThreadsUseCase = manageThreadsUseCase,
+ )
+ }
+}
diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/voice/LiveVoiceSessionOverlay.kt b/agent/app/src/main/java/com/example/appfunctions/agent/voice/LiveVoiceSessionOverlay.kt
new file mode 100644
index 00000000..2d8b0b7d
--- /dev/null
+++ b/agent/app/src/main/java/com/example/appfunctions/agent/voice/LiveVoiceSessionOverlay.kt
@@ -0,0 +1,386 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.voice
+
+import android.Manifest
+import android.content.Intent
+import android.content.pm.PackageManager
+import android.graphics.Rect
+import android.provider.Settings
+import android.widget.Toast
+import androidx.activity.compose.rememberLauncherForActivityResult
+import androidx.activity.result.contract.ActivityResultContracts
+import androidx.compose.animation.core.RepeatMode
+import androidx.compose.animation.core.animateFloat
+import androidx.compose.animation.core.infiniteRepeatable
+import androidx.compose.animation.core.rememberInfiniteTransition
+import androidx.compose.animation.core.tween
+import androidx.compose.foundation.Image
+import androidx.compose.foundation.layout.Arrangement
+import androidx.compose.foundation.layout.Box
+import androidx.compose.foundation.layout.Column
+import androidx.compose.foundation.layout.Row
+import androidx.compose.foundation.layout.Spacer
+import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.foundation.layout.fillMaxWidth
+import androidx.compose.foundation.layout.navigationBarsPadding
+import androidx.compose.foundation.layout.padding
+import androidx.compose.foundation.layout.size
+import androidx.compose.foundation.layout.width
+import androidx.compose.foundation.shape.CircleShape
+import androidx.compose.foundation.shape.RoundedCornerShape
+import androidx.compose.material.icons.Icons
+import androidx.compose.material.icons.filled.Close
+import androidx.compose.material.icons.filled.GraphicEq
+import androidx.compose.material.icons.filled.Warning
+import androidx.compose.material3.CircularProgressIndicator
+import androidx.compose.material3.FilledIconButton
+import androidx.compose.material3.Icon
+import androidx.compose.material3.IconButtonDefaults
+import androidx.compose.material3.MaterialTheme
+import androidx.compose.material3.Surface
+import androidx.compose.material3.Text
+import androidx.compose.material3.TextButton
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.getValue
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.remember
+import androidx.compose.runtime.saveable.rememberSaveable
+import androidx.compose.runtime.setValue
+import androidx.compose.ui.Alignment
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.draw.scale
+import androidx.compose.ui.graphics.asImageBitmap
+import androidx.compose.ui.layout.boundsInWindow
+import androidx.compose.ui.layout.onGloballyPositioned
+import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.res.stringResource
+import androidx.compose.ui.text.style.TextOverflow
+import androidx.compose.ui.unit.dp
+import androidx.core.content.ContextCompat
+import androidx.core.graphics.drawable.toBitmap
+import com.example.appfunctions.agent.R
+import com.example.appfunctions.agent.domain.AgentStatus
+import com.example.appfunctions.agent.domain.LiveSessionState
+import kotlin.math.roundToInt
+
+/**
+ * Floating system-overlay pill displayed inside [AgentVoiceInteractionSession] while a Live voice
+ * session is connecting, listening, invoking an AppFunction tool, or reporting an error.
+ */
+@Composable
+fun LiveVoiceSessionOverlay(
+ liveSessionState: LiveSessionState,
+ agentStatus: AgentStatus,
+ onStopClick: () -> Unit,
+ onRetryClick: () -> Unit,
+ onOpenChatClick: () -> Unit = {},
+ onOverlayBoundsChanged: (Rect) -> Unit = {},
+) {
+ val context = LocalContext.current
+ val packageManager = context.packageManager
+
+ Box(
+ modifier = Modifier.fillMaxSize(),
+ contentAlignment = Alignment.BottomCenter,
+ ) {
+ Surface(
+ onClick = onOpenChatClick,
+ modifier =
+ Modifier
+ .navigationBarsPadding()
+ .padding(horizontal = 16.dp, vertical = 16.dp)
+ .fillMaxWidth()
+ .onGloballyPositioned { coordinates ->
+ val bounds = coordinates.boundsInWindow()
+ onOverlayBoundsChanged(
+ Rect(
+ bounds.left.roundToInt(),
+ bounds.top.roundToInt(),
+ bounds.right.roundToInt(),
+ bounds.bottom.roundToInt(),
+ ),
+ )
+ },
+ shape = RoundedCornerShape(28.dp),
+ color = MaterialTheme.colorScheme.surfaceContainerHigh,
+ tonalElevation = 6.dp,
+ shadowElevation = 8.dp,
+ ) {
+ Row(
+ modifier = Modifier.padding(horizontal = 16.dp, vertical = 14.dp),
+ verticalAlignment = Alignment.CenterVertically,
+ horizontalArrangement = Arrangement.SpaceBetween,
+ ) {
+ Row(
+ modifier = Modifier.weight(1f),
+ verticalAlignment = Alignment.CenterVertically,
+ ) {
+ LiveStatusLeadingIcon(
+ liveSessionState = liveSessionState,
+ agentStatus = agentStatus,
+ packageManager = packageManager,
+ )
+ Spacer(modifier = Modifier.width(12.dp))
+ Column(modifier = Modifier.weight(1f)) {
+ Text(
+ text = stringResource(R.string.live_session_title),
+ style = MaterialTheme.typography.labelMedium,
+ color = MaterialTheme.colorScheme.primary,
+ maxLines = 1,
+ )
+ val subtitle =
+ when {
+ agentStatus is AgentStatus.InvokingTool -> {
+ val appLabel =
+ resolveAppLabel(packageManager, agentStatus.packageName)
+ stringResource(R.string.live_session_invoking_tool, appLabel)
+ }
+ liveSessionState is LiveSessionState.Connecting ->
+ stringResource(R.string.agent_demo_live_connecting)
+ liveSessionState is LiveSessionState.Listening ->
+ stringResource(R.string.agent_demo_live_listening)
+ liveSessionState is LiveSessionState.Error ->
+ liveSessionState.message
+ else -> stringResource(R.string.agent_demo_live_connecting)
+ }
+ val subtitleColor =
+ if (liveSessionState is LiveSessionState.Error) {
+ MaterialTheme.colorScheme.error
+ } else {
+ MaterialTheme.colorScheme.onSurface
+ }
+ Text(
+ text = subtitle,
+ style = MaterialTheme.typography.bodyMedium,
+ color = subtitleColor,
+ maxLines = 2,
+ overflow = TextOverflow.Ellipsis,
+ )
+ }
+ }
+
+ Row(verticalAlignment = Alignment.CenterVertically) {
+ if (liveSessionState is LiveSessionState.Error) {
+ TextButton(onClick = onRetryClick) {
+ Text(stringResource(R.string.live_session_retry))
+ }
+ }
+ FilledIconButton(
+ onClick = onStopClick,
+ shape = CircleShape,
+ colors =
+ IconButtonDefaults.filledIconButtonColors(
+ containerColor = MaterialTheme.colorScheme.errorContainer,
+ contentColor = MaterialTheme.colorScheme.onErrorContainer,
+ ),
+ ) {
+ Icon(
+ imageVector = Icons.Default.Close,
+ contentDescription = stringResource(R.string.agent_demo_live_stop),
+ )
+ }
+ }
+ }
+ }
+ }
+}
+
+@Composable
+private fun LiveStatusLeadingIcon(
+ liveSessionState: LiveSessionState,
+ agentStatus: AgentStatus,
+ packageManager: PackageManager,
+) {
+ when {
+ agentStatus is AgentStatus.InvokingTool -> {
+ val appIcon =
+ remember(agentStatus.packageName) {
+ runCatching {
+ packageManager.getApplicationIcon(agentStatus.packageName)
+ }.getOrNull()
+ }
+ if (appIcon != null) {
+ Image(
+ bitmap = appIcon.toBitmap().asImageBitmap(),
+ contentDescription = null,
+ modifier = Modifier.size(32.dp),
+ )
+ } else {
+ CircularProgressIndicator(
+ modifier = Modifier.size(24.dp),
+ strokeWidth = 2.5.dp,
+ )
+ }
+ }
+
+ liveSessionState is LiveSessionState.Listening -> {
+ val infiniteTransition = rememberInfiniteTransition(label = "live_pulse")
+ val scale by
+ infiniteTransition.animateFloat(
+ initialValue = 0.88f,
+ targetValue = 1.15f,
+ animationSpec =
+ infiniteRepeatable(
+ animation = tween(durationMillis = 700),
+ repeatMode = RepeatMode.Reverse,
+ ),
+ label = "live_pulse_scale",
+ )
+ Surface(
+ modifier = Modifier.size(36.dp).scale(scale),
+ shape = CircleShape,
+ color = MaterialTheme.colorScheme.primaryContainer,
+ ) {
+ Box(contentAlignment = Alignment.Center) {
+ Icon(
+ imageVector = Icons.Default.GraphicEq,
+ contentDescription = null,
+ tint = MaterialTheme.colorScheme.onPrimaryContainer,
+ modifier = Modifier.size(20.dp),
+ )
+ }
+ }
+ }
+
+ liveSessionState is LiveSessionState.Error -> {
+ Icon(
+ imageVector = Icons.Default.Warning,
+ contentDescription = null,
+ tint = MaterialTheme.colorScheme.error,
+ modifier = Modifier.size(28.dp),
+ )
+ }
+
+ else -> {
+ CircularProgressIndicator(
+ modifier = Modifier.size(24.dp),
+ strokeWidth = 2.5.dp,
+ )
+ }
+ }
+}
+
+private fun resolveAppLabel(
+ packageManager: PackageManager,
+ packageName: String,
+): String {
+ if (packageName.isBlank()) return "AppFunction"
+ return runCatching {
+ val appInfo = packageManager.getApplicationInfo(packageName, 0)
+ packageManager.getApplicationLabel(appInfo).toString()
+ }.getOrDefault(packageName)
+}
+
+/**
+ * Remembers a launcher callback that verifies `RECORD_AUDIO` permission and starts
+ * [AgentVoiceInteractionService] (opening Digital Assistant settings via
+ * [Settings.ACTION_VOICE_INPUT_SETTINGS] if this app is not yet the active
+ * [android.service.voice.VoiceInteractionService]).
+ */
+@Composable
+fun rememberLiveVoiceSessionLauncher(): (threadId: String, targetPackageName: String?) -> Unit {
+ val context = LocalContext.current
+ val selectAssistantPrompt = stringResource(R.string.live_session_select_assistant_prompt)
+ var pendingThreadId by rememberSaveable { mutableStateOf(null) }
+ var pendingTargetPackageName by rememberSaveable { mutableStateOf(null) }
+
+ val assistantSettingsLauncher =
+ rememberLauncherForActivityResult(
+ contract = ActivityResultContracts.StartActivityForResult(),
+ ) {
+ val threadId = pendingThreadId
+ val targetPackageName = pendingTargetPackageName
+ pendingThreadId = null
+ pendingTargetPackageName = null
+ if (threadId != null && AgentVoiceInteractionService.isVoiceServiceActive(context)) {
+ AgentVoiceInteractionService.launchLiveSession(
+ context = context,
+ threadId = threadId,
+ targetPackageName = targetPackageName,
+ )
+ }
+ }
+
+ val startVoiceServiceForRequest: (String, String?) -> Unit = { threadId, targetPackageName ->
+ val launched =
+ AgentVoiceInteractionService.launchLiveSession(
+ context = context,
+ threadId = threadId,
+ targetPackageName = targetPackageName,
+ )
+ if (!launched) {
+ pendingThreadId = threadId
+ pendingTargetPackageName = targetPackageName
+ Toast.makeText(
+ context,
+ selectAssistantPrompt,
+ Toast.LENGTH_LONG,
+ ).show()
+ runCatching {
+ assistantSettingsLauncher.launch(
+ AgentVoiceInteractionService.createAssistantSettingsIntent(),
+ )
+ }.onFailure {
+ runCatching {
+ assistantSettingsLauncher.launch(
+ Intent(Settings.ACTION_MANAGE_DEFAULT_APPS_SETTINGS),
+ )
+ }.onFailure {
+ pendingThreadId = null
+ pendingTargetPackageName = null
+ }
+ }
+ }
+ }
+
+ val audioPermissionLauncher =
+ rememberLauncherForActivityResult(
+ contract = ActivityResultContracts.RequestPermission(),
+ ) { isGranted ->
+ val threadId = pendingThreadId
+ val targetPackageName = pendingTargetPackageName
+ pendingThreadId = null
+ pendingTargetPackageName = null
+ if (isGranted && threadId != null) {
+ startVoiceServiceForRequest(threadId, targetPackageName)
+ }
+ }
+
+ return { threadId, targetPackageName ->
+ val hasRecordAudioPermission =
+ runCatching {
+ ContextCompat.checkSelfPermission(
+ context,
+ Manifest.permission.RECORD_AUDIO,
+ ) == PackageManager.PERMISSION_GRANTED
+ }.getOrDefault(true)
+
+ if (hasRecordAudioPermission) {
+ startVoiceServiceForRequest(threadId, targetPackageName)
+ } else {
+ pendingThreadId = threadId
+ pendingTargetPackageName = targetPackageName
+ runCatching {
+ audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO)
+ }.onFailure {
+ pendingThreadId = null
+ pendingTargetPackageName = null
+ startVoiceServiceForRequest(threadId, targetPackageName)
+ }
+ }
+ }
+}
diff --git a/agent/app/src/main/res/values/strings.xml b/agent/app/src/main/res/values/strings.xml
index 8a74b210..bd33f2bc 100644
--- a/agent/app/src/main/res/values/strings.xml
+++ b/agent/app/src/main/res/values/strings.xml
@@ -88,6 +88,9 @@
Priority lowers response latency but requires Tier 2+ billing and costs more.
Standard
Priority
+ Gemini Live voice
+ Spoken voice used for Gemini Live sessions. Changes apply to new sessions.
+ %1$s (%2$s)
Preferred Provider
Gemini
Open source licenses
@@ -110,6 +113,14 @@
Yes, go ahead
Done
I can perform this action for you. Should I proceed?
+ Start Live Voice Session
+ End Live Voice Session
+ Connecting…
+ Listening…
+ Gemini Live
+ Using %1$s…
+ Retry
+ Select AppFunction Testing Agent as your Digital Assistant to start Gemini Live
Copied value of %1$s
Copy value
diff --git a/agent/app/src/main/res/xml/voice_interaction_service.xml b/agent/app/src/main/res/xml/voice_interaction_service.xml
new file mode 100644
index 00000000..e78173de
--- /dev/null
+++ b/agent/app/src/main/res/xml/voice_interaction_service.xml
@@ -0,0 +1,23 @@
+
+
+
+
diff --git a/agent/app/src/screenshotTest/kotlin/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreenScreenshotTest.kt b/agent/app/src/screenshotTest/kotlin/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreenScreenshotTest.kt
index f06d860d..fa38d946 100644
--- a/agent/app/src/screenshotTest/kotlin/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreenScreenshotTest.kt
+++ b/agent/app/src/screenshotTest/kotlin/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsScreenScreenshotTest.kt
@@ -19,6 +19,7 @@ import androidx.compose.foundation.text.input.rememberTextFieldState
import androidx.compose.runtime.Composable
import androidx.compose.ui.tooling.preview.Preview
import com.android.tools.screenshot.PreviewTest
+import com.example.appfunctions.agent.data.GeminiLiveVoice
import com.example.appfunctions.agent.data.ServiceTier
/** Screenshot test for [SettingsScreen]. */
@@ -32,6 +33,10 @@ fun SettingsScreenScreenshotPreview() {
onServiceTierSelected = {
// no-op
},
+ geminiLiveVoice = GeminiLiveVoice.DEFAULT,
+ onGeminiLiveVoiceSelected = {
+ // no-op
+ },
onOpenLicenses = {
// no-op
},
diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelperTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelperTest.kt
new file mode 100644
index 00000000..fb8b97ee
--- /dev/null
+++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelperTest.kt
@@ -0,0 +1,330 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.data
+
+import android.content.Context
+import android.media.AudioAttributes
+import android.media.AudioDeviceInfo
+import android.media.AudioManager
+import android.media.AudioRecord
+import android.media.AudioTrack
+import androidx.test.core.app.ApplicationProvider
+import io.mockk.every
+import io.mockk.mockk
+import io.mockk.verify
+import kotlinx.coroutines.flow.toList
+import kotlinx.coroutines.runBlocking
+import org.junit.Assert.assertArrayEquals
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertNotNull
+import org.junit.Assert.assertNull
+import org.junit.Assert.assertSame
+import org.junit.Assert.assertTrue
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.robolectric.RobolectricTestRunner
+import org.robolectric.Shadows
+import org.robolectric.annotation.Config
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicInteger
+
+@RunWith(RobolectricTestRunner::class)
+@Config(sdk = [34])
+class GeminiLiveAudioHelperTest {
+ @Test
+ fun `start enters communication mode with voice communication focus and release restores state`() {
+ val context = ApplicationProvider.getApplicationContext()
+ val audioManager = context.getSystemService(AudioManager::class.java)
+ audioManager.mode = AudioManager.MODE_NORMAL
+ val shadowAudioManager = Shadows.shadowOf(audioManager)
+
+ val mockRecorder = createInitializedMockRecorder()
+ val mockTrack = createInitializedMockTrack()
+ var capturedAttributes: AudioAttributes? = null
+
+ val helper =
+ GeminiLiveAudioHelper(audioManager).apply {
+ audioRecordFactory = { mockRecorder }
+ audioTrackFactory = { attrs, _ ->
+ capturedAttributes = attrs
+ mockTrack
+ }
+ }
+
+ helper.start()
+
+ val focusReq = shadowAudioManager.lastAudioFocusRequest?.audioFocusRequest
+ assertNotNull(focusReq)
+ assertEquals(
+ AudioManager.AUDIOFOCUS_GAIN_TRANSIENT,
+ focusReq?.focusGain,
+ )
+ assertEquals(AudioAttributes.USAGE_VOICE_COMMUNICATION, capturedAttributes?.usage)
+ assertEquals(AudioAttributes.CONTENT_TYPE_SPEECH, capturedAttributes?.contentType)
+ assertEquals(AudioManager.MODE_IN_COMMUNICATION, audioManager.mode)
+ assertNull(shadowAudioManager.lastAbandonedAudioFocusRequest)
+
+ helper.release()
+
+ assertEquals(AudioManager.MODE_NORMAL, audioManager.mode)
+ assertSame(focusReq, shadowAudioManager.lastAbandonedAudioFocusRequest)
+ verify(exactly = 1) { mockRecorder.stop() }
+ verify(exactly = 1) { mockRecorder.release() }
+ verify(exactly = 1) { mockTrack.release() }
+ }
+
+ @Test
+ fun `start routes communication audio to speakerphone when no external headset is connected`() {
+ val earpiece =
+ mockk {
+ every { type } returns AudioDeviceInfo.TYPE_BUILTIN_EARPIECE
+ }
+ val speaker =
+ mockk {
+ every { type } returns AudioDeviceInfo.TYPE_BUILTIN_SPEAKER
+ }
+ val telephony =
+ mockk {
+ every { type } returns AudioDeviceInfo.TYPE_TELEPHONY
+ }
+ val audioManager =
+ mockk(relaxed = true) {
+ every { mode } returns AudioManager.MODE_NORMAL
+ every { requestAudioFocus(any()) } returns AudioManager.AUDIOFOCUS_REQUEST_GRANTED
+ every { communicationDevice } returns earpiece
+ every { availableCommunicationDevices } returns listOf(earpiece, speaker, telephony)
+ every { setCommunicationDevice(speaker) } returns true
+ }
+
+ val helper =
+ GeminiLiveAudioHelper(audioManager).apply {
+ audioRecordFactory = { createInitializedMockRecorder() }
+ audioTrackFactory = { _, _ -> createInitializedMockTrack() }
+ }
+
+ helper.start()
+ verify(exactly = 1) { audioManager.mode = AudioManager.MODE_IN_COMMUNICATION }
+ verify(exactly = 1) { audioManager.setCommunicationDevice(speaker) }
+
+ helper.release()
+ verify(exactly = 1) { audioManager.clearCommunicationDevice() }
+ verify(exactly = 1) { audioManager.mode = AudioManager.MODE_NORMAL }
+ }
+
+ @Test
+ fun `start preserves external headset routing when BLE or wired headset is available`() {
+ val speaker =
+ mockk {
+ every { type } returns AudioDeviceInfo.TYPE_BUILTIN_SPEAKER
+ }
+ val bleHeadset =
+ mockk {
+ every { type } returns AudioDeviceInfo.TYPE_BLE_HEADSET
+ }
+ val audioManager =
+ mockk(relaxed = true) {
+ every { mode } returns AudioManager.MODE_NORMAL
+ every { requestAudioFocus(any()) } returns AudioManager.AUDIOFOCUS_REQUEST_GRANTED
+ every { communicationDevice } returns null
+ every { availableCommunicationDevices } returns listOf(speaker, bleHeadset)
+ }
+
+ val helper =
+ GeminiLiveAudioHelper(audioManager).apply {
+ audioRecordFactory = { createInitializedMockRecorder() }
+ audioTrackFactory = { _, _ -> createInitializedMockTrack() }
+ }
+
+ helper.start()
+ verify(exactly = 0) { audioManager.setCommunicationDevice(any()) }
+
+ helper.release()
+ verify(exactly = 0) { audioManager.clearCommunicationDevice() }
+ }
+
+ @Test
+ fun `stopAudio flushes active AudioTrack and discards queued chunks`() {
+ val flushedLatch = CountDownLatch(1)
+ val mockTrack =
+ createInitializedMockTrack().apply {
+ every { flush() } answers {
+ flushedLatch.countDown()
+ }
+ }
+ val helper =
+ GeminiLiveAudioHelper().apply {
+ audioRecordFactory = { createInitializedMockRecorder() }
+ audioTrackFactory = { _, _ -> mockTrack }
+ }
+
+ helper.start()
+ helper.stopAudio()
+
+ assertTrue(flushedLatch.await(2, TimeUnit.SECONDS))
+ verify(atLeast = 1) { mockTrack.pause() }
+ verify(atLeast = 1) { mockTrack.flush() }
+ verify(atLeast = 2) { mockTrack.play() }
+
+ helper.release()
+ }
+
+ @Test
+ fun `pauseRecording and resumeRecording drain AudioRecord in software without stopping hardware`() =
+ runBlocking {
+ val mockRecorder = createInitializedMockRecorder()
+ val mockTrack = createInitializedMockTrack()
+ val helper =
+ GeminiLiveAudioHelper().apply {
+ audioRecordFactory = { mockRecorder }
+ audioTrackFactory = { _, _ -> mockTrack }
+ }
+
+ val readStep = AtomicInteger(0)
+ every { mockRecorder.read(any(), any(), any()) } answers {
+ val buf = firstArg()
+ when (readStep.incrementAndGet()) {
+ 1 -> {
+ buf[0] = 10
+ buf[1] = 11
+ 2
+ }
+ 2 -> {
+ // Simulate pauseRecording() active during tool call: read() still runs to
+ // drain the HAL ring buffer, but chunk 2 must be discarded.
+ helper.pauseRecording()
+ buf[0] = 20
+ buf[1] = 21
+ 2
+ }
+ 3 -> {
+ // Resume before delivering chunk 3
+ helper.resumeRecording()
+ buf[0] = 30
+ buf[1] = 31
+ 2
+ }
+ else -> {
+ helper.release()
+ -1
+ }
+ }
+ }
+
+ helper.start()
+ val emittedChunks = helper.listenToRecording().toList()
+
+ assertEquals(2, emittedChunks.size)
+ assertArrayEquals(byteArrayOf(10, 11), emittedChunks[0])
+ assertArrayEquals(byteArrayOf(30, 31), emittedChunks[1])
+
+ // Hardware AudioRecord was started only once in start(), and stop() was only called once
+ // during final release() — never during pauseRecording()/resumeRecording().
+ verify(exactly = 1) { mockRecorder.startRecording() }
+ verify(exactly = 1) { mockRecorder.stop() }
+ }
+
+ @Test
+ fun `audio focus loss pauses emission independently of tool-call pause state`() {
+ val helper = GeminiLiveAudioHelper()
+
+ assertFalse(helper.isPausedForTesting())
+
+ helper.pauseRecording()
+ assertTrue(helper.isPausedForTesting())
+
+ // Focus loss followed by focus gain must not clear an active tool-call pause.
+ helper.notifyAudioFocusChangeForTesting(AudioManager.AUDIOFOCUS_LOSS_TRANSIENT)
+ assertTrue(helper.isPausedForTesting())
+ helper.notifyAudioFocusChangeForTesting(AudioManager.AUDIOFOCUS_GAIN)
+ assertTrue(helper.isPausedForTesting())
+
+ helper.resumeRecording()
+ assertFalse(helper.isPausedForTesting())
+
+ // Focus loss while unpaused gates emission until AUDIOFOCUS_GAIN arrives.
+ helper.notifyAudioFocusChangeForTesting(AudioManager.AUDIOFOCUS_LOSS_TRANSIENT)
+ assertTrue(helper.isPausedForTesting())
+ helper.notifyAudioFocusChangeForTesting(AudioManager.AUDIOFOCUS_GAIN)
+ assertFalse(helper.isPausedForTesting())
+
+ helper.release()
+ }
+
+ @Test
+ fun `concurrent release during start cleans up resources without leaking active AudioRecord`() {
+ val context = ApplicationProvider.getApplicationContext()
+ val audioManager = context.getSystemService(AudioManager::class.java)
+ val shadowAudioManager = Shadows.shadowOf(audioManager)
+
+ val mockRecorder = createInitializedMockRecorder()
+ val mockTrack = createInitializedMockTrack()
+ val insideStartLatch = CountDownLatch(1)
+ val releaseIssuedLatch = CountDownLatch(1)
+
+ val helper =
+ GeminiLiveAudioHelper(audioManager).apply {
+ audioRecordFactory = {
+ insideStartLatch.countDown()
+ assertTrue(releaseIssuedLatch.await(2, TimeUnit.SECONDS))
+ mockRecorder
+ }
+ audioTrackFactory = { _, _ -> mockTrack }
+ }
+
+ val releaseThread =
+ Thread {
+ assertTrue(insideStartLatch.await(2, TimeUnit.SECONDS))
+ // Trigger release() from a second thread while start() is mid-initialization.
+ // release() flips isReleased=true immediately and then waits on stateLock.
+ val releaseCaller =
+ Thread {
+ helper.release()
+ }
+ releaseCaller.start()
+ // Spin briefly until releaseCaller flips isReleased (before blocking on stateLock)
+ Thread.sleep(30)
+ releaseIssuedLatch.countDown()
+ releaseCaller.join(2000)
+ }
+ releaseThread.start()
+
+ helper.start()
+ releaseThread.join(2000)
+
+ // Because release() was triggered mid-start(), AudioRecord must never be left recording.
+ verify(exactly = 0) { mockRecorder.startRecording() }
+ verify(exactly = 1) { mockRecorder.release() }
+ verify(exactly = 1) { mockTrack.release() }
+ val focusReq = shadowAudioManager.lastAudioFocusRequest?.audioFocusRequest
+ assertNotNull(focusReq)
+ assertSame(focusReq, shadowAudioManager.lastAbandonedAudioFocusRequest)
+ }
+
+ private fun createInitializedMockRecorder(): AudioRecord =
+ mockk(relaxed = true).apply {
+ every { state } returns AudioRecord.STATE_INITIALIZED
+ every { recordingState } returns AudioRecord.RECORDSTATE_RECORDING
+ every { audioSessionId } returns 42
+ }
+
+ private fun createInitializedMockTrack(): AudioTrack =
+ mockk(relaxed = true).apply {
+ every { state } returns AudioTrack.STATE_INITIALIZED
+ every { playState } returns AudioTrack.PLAYSTATE_PLAYING
+ }
+}
diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt
new file mode 100644
index 00000000..eaebf3ad
--- /dev/null
+++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiLiveProviderImplTest.kt
@@ -0,0 +1,396 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.data
+
+import android.util.Log
+import androidx.appfunctions.metadata.AppFunctionComponentsMetadata
+import androidx.appfunctions.metadata.AppFunctionMetadata
+import androidx.appfunctions.metadata.AppFunctionParameterMetadata
+import androidx.appfunctions.metadata.AppFunctionResponseMetadata
+import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata
+import com.example.appfunctions.agent.domain.LiveLlmEvent
+import com.example.appfunctions.agent.domain.ToolOutput
+import io.ktor.client.HttpClient
+import io.ktor.websocket.Frame
+import io.ktor.websocket.WebSocketSession
+import io.ktor.websocket.readText
+import io.mockk.coEvery
+import io.mockk.every
+import io.mockk.mockk
+import io.mockk.mockkStatic
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.flow.flowOf
+import kotlinx.coroutines.flow.toList
+import kotlinx.coroutines.test.runTest
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive
+import org.junit.Assert.assertArrayEquals
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertTrue
+import org.junit.Assert.fail
+import org.junit.Before
+import org.junit.Test
+import java.util.Base64
+
+class GeminiLiveProviderImplTest {
+ private lateinit var settingsRepository: SettingsRepository
+ private lateinit var provider: GeminiLiveProviderImpl
+
+ @Before
+ fun setUp() {
+ mockkStatic(Log::class)
+ every { Log.d(any(), any()) } returns 0
+ every { Log.w(any(), any()) } returns 0
+ every { Log.e(any(), any(), any()) } returns 0
+
+ settingsRepository = mockk(relaxed = true)
+ every { settingsRepository.geminiLiveVoice } returns flowOf(GeminiLiveVoice.DEFAULT)
+
+ provider =
+ GeminiLiveProviderImpl(
+ httpClient = mockk(relaxed = true),
+ toolConverter = GeminiToolConverter(),
+ settingsRepository = settingsRepository,
+ )
+ }
+
+ @Test
+ fun `buildSetupMessage configures gemini-3_8-live model, audio modality, transcriptions, and tools`() {
+ val tool = createSampleNoteTool()
+
+ val setupEnvelope = provider.buildSetupMessage(listOf(tool))
+ val setup = setupEnvelope["setup"]!!.jsonObject
+
+ assertEquals("models/gemini-3.8-live", setup["model"]!!.jsonPrimitive.content)
+
+ val modalities =
+ setup["generationConfig"]!!
+ .jsonObject["responseModalities"]!!
+ .jsonArray
+ .map { it.jsonPrimitive.content }
+ assertEquals(listOf("AUDIO"), modalities)
+
+ val voiceName =
+ setup["generationConfig"]!!
+ .jsonObject["speechConfig"]!!
+ .jsonObject["voiceConfig"]!!
+ .jsonObject["prebuiltVoiceConfig"]!!
+ .jsonObject["voiceName"]!!
+ .jsonPrimitive
+ .content
+ assertEquals("Zephyr", voiceName)
+
+ assertTrue(setup.containsKey("inputAudioTranscription"))
+ assertTrue(setup.containsKey("outputAudioTranscription"))
+ val systemInstructionText =
+ setup["systemInstruction"]!!
+ .jsonObject["parts"]!!
+ .jsonArray
+ .first()
+ .jsonObject["text"]!!
+ .jsonPrimitive
+ .content
+ assertTrue(
+ systemInstructionText.contains(
+ "Default to English for understanding and replying unless the user clearly " +
+ "speaks in or asks to switch to another language.",
+ ),
+ )
+ assertTrue(
+ systemInstructionText.contains(
+ "Execute user-requested actions directly without claiming an on-screen " +
+ "confirmation card or button is shown",
+ ),
+ )
+ assertFalse(systemInstructionText.contains("confirmation_required"))
+
+ val declarations =
+ setup["tools"]!!
+ .jsonArray
+ .first()
+ .jsonObject["functionDeclarations"]!!
+ .jsonArray
+ assertEquals(1, declarations.size)
+ assertEquals(
+ "com_example_notes_createNote",
+ declarations.first().jsonObject["name"]!!.jsonPrimitive.content,
+ )
+ }
+
+ @Test
+ fun `connect throws IllegalStateException when WebSocket closes before setupComplete`() =
+ runTest {
+ val incomingFrames = Channel(Channel.UNLIMITED)
+ incomingFrames.close()
+
+ provider.webSocketConnector = { _, block ->
+ coroutineScope {
+ createMockWebSocketSession(coroutineContext, incomingFrames).block()
+ }
+ }
+
+ try {
+ provider.connect(apiKey = "key-123", tools = emptyList()) {}
+ fail("Expected IllegalStateException")
+ } catch (e: IllegalStateException) {
+ assertTrue(e.message!!.contains("setupComplete"))
+ }
+ }
+
+ @Test
+ fun `connect sends selected GeminiLiveVoice from SettingsRepository in setup message`() =
+ runTest {
+ every { settingsRepository.geminiLiveVoice } returns flowOf(GeminiLiveVoice.AOEDE)
+
+ val incomingFrames = Channel(Channel.UNLIMITED)
+ incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}"""))
+ incomingFrames.close()
+
+ val sentFrames = mutableListOf()
+ provider.webSocketConnector = { _, block ->
+ coroutineScope {
+ createMockWebSocketSession(
+ context = coroutineContext,
+ incomingFrames = incomingFrames,
+ onSendText = { sentFrames.add(it) },
+ ).block()
+ }
+ }
+
+ provider.connect(apiKey = "key-123", tools = emptyList()) {}
+
+ val setupVoiceName =
+ Json.parseToJsonElement(sentFrames.first())
+ .jsonObject["setup"]!!
+ .jsonObject["generationConfig"]!!
+ .jsonObject["speechConfig"]!!
+ .jsonObject["voiceConfig"]!!
+ .jsonObject["prebuiltVoiceConfig"]!!
+ .jsonObject["voiceName"]!!
+ .jsonPrimitive
+ .content
+ assertEquals("Aoede", setupVoiceName)
+ }
+
+ @Test
+ fun `connect exchanges realtimeInput audio, parses server events, and sends mapped toolResponse`() =
+ runTest {
+ val tool = createSampleNoteTool()
+ val expectedPcm = byteArrayOf(10, 20, 30, 40)
+ val encodedPcm = Base64.getEncoder().encodeToString(expectedPcm)
+
+ val incomingFrames = Channel(Channel.UNLIMITED)
+ val sentFrames = mutableListOf()
+ var capturedWsUrl: String? = null
+
+ incomingFrames.trySend(Frame.Text("""{"setupComplete":{}}"""))
+ incomingFrames.trySend(
+ Frame.Text(
+ """
+ {
+ "serverContent": {
+ "interrupted": true,
+ "inputTranscription": { "text": "Create a note" },
+ "outputTranscription": { "text": "Creating note" },
+ "modelTurn": {
+ "parts": [
+ {
+ "inlineData": {
+ "mimeType": "audio/pcm;rate=24000",
+ "data": "$encodedPcm"
+ }
+ }
+ ]
+ },
+ "turnComplete": true
+ }
+ }
+ """.trimIndent(),
+ ),
+ )
+ incomingFrames.trySend(
+ Frame.Text(
+ """
+ {
+ "toolCall": {
+ "functionCalls": [
+ {
+ "id": "call-1",
+ "name": "com_example_notes_createNote",
+ "args": {
+ "title": "Shopping",
+ "meta": { "pinned": true, "priority": 2 }
+ }
+ },
+ {
+ "id": "call-unknown",
+ "name": "hallucinated_tool",
+ "args": {}
+ }
+ ]
+ }
+ }
+ """.trimIndent(),
+ ),
+ )
+ incomingFrames.close()
+
+ provider.webSocketConnector = { wsUrl, block ->
+ capturedWsUrl = wsUrl
+ coroutineScope {
+ val ws =
+ createMockWebSocketSession(
+ context = coroutineContext,
+ incomingFrames = incomingFrames,
+ onSendText = { sentFrames.add(it) },
+ )
+ ws.block()
+ }
+ }
+
+ val receivedEvents = mutableListOf()
+ provider.connect(apiKey = "secret-key", tools = listOf(tool)) { session ->
+ session.sendAudio(byteArrayOf(1, 2, 3))
+ receivedEvents.addAll(session.events.toList())
+ session.sendToolResponses(
+ listOf(
+ ToolOutput(
+ functionId = tool.id,
+ callId = "call-1",
+ result = """{"noteId":"n-1"}""",
+ ),
+ ToolOutput(
+ functionId = "hallucinated_tool",
+ callId = "call-unknown",
+ result = "Tool not found: hallucinated_tool",
+ ),
+ ),
+ )
+ }
+
+ assertTrue(capturedWsUrl!!.endsWith("?key=secret-key"))
+ assertEquals(3, sentFrames.size)
+
+ // Frame 0: setup
+ assertTrue(Json.parseToJsonElement(sentFrames[0]).jsonObject.containsKey("setup"))
+
+ // Frame 1: realtimeInput audio
+ val realtimeAudio =
+ Json.parseToJsonElement(sentFrames[1])
+ .jsonObject["realtimeInput"]!!
+ .jsonObject["audio"]!!
+ .jsonObject
+ assertEquals("audio/pcm;rate=16000", realtimeAudio["mimeType"]!!.jsonPrimitive.content)
+ assertArrayEquals(
+ byteArrayOf(1, 2, 3),
+ Base64.getDecoder().decode(realtimeAudio["data"]!!.jsonPrimitive.content),
+ )
+
+ // Received events
+ assertEquals(6, receivedEvents.size)
+ assertEquals(LiveLlmEvent.Interrupted, receivedEvents[0])
+ assertEquals(LiveLlmEvent.InputTranscription("Create a note"), receivedEvents[1])
+ assertEquals(LiveLlmEvent.OutputTranscription("Creating note"), receivedEvents[2])
+ assertArrayEquals(expectedPcm, (receivedEvents[3] as LiveLlmEvent.AudioChunk).pcmBytes)
+ assertEquals(LiveLlmEvent.TurnComplete, receivedEvents[4])
+
+ val toolCallReq = receivedEvents[5] as LiveLlmEvent.ToolCallRequest
+ assertEquals(2, toolCallReq.calls.size)
+ assertEquals("com.example.notes", toolCallReq.calls[0].packageName)
+ assertEquals("com.example.notes.createNote", toolCallReq.calls[0].functionId)
+ assertEquals("Shopping", toolCallReq.calls[0].arguments["title"])
+ @Suppress("UNCHECKED_CAST")
+ val meta = toolCallReq.calls[0].arguments["meta"] as Map
+ assertEquals(true, meta["pinned"])
+ assertEquals(2L, meta["priority"])
+
+ assertEquals("", toolCallReq.calls[1].packageName)
+ assertEquals("hallucinated_tool", toolCallReq.calls[1].functionId)
+
+ // Frame 2: toolResponse
+ val responses =
+ Json.parseToJsonElement(sentFrames[2])
+ .jsonObject["toolResponse"]!!
+ .jsonObject["functionResponses"]!!
+ .jsonArray
+ assertEquals(2, responses.size)
+ assertEquals(
+ "com_example_notes_createNote",
+ responses[0].jsonObject["name"]!!.jsonPrimitive.content,
+ )
+ assertEquals(
+ "n-1",
+ responses[0]
+ .jsonObject["response"]!!
+ .jsonObject["result"]!!
+ .jsonObject["noteId"]!!
+ .jsonPrimitive
+ .content,
+ )
+ assertEquals(
+ "hallucinated_tool",
+ responses[1].jsonObject["name"]!!.jsonPrimitive.content,
+ )
+ }
+
+ private fun createMockWebSocketSession(
+ context: kotlin.coroutines.CoroutineContext,
+ incomingFrames: Channel,
+ onSendText: (String) -> Unit = {},
+ ): WebSocketSession {
+ val outgoingFrames = Channel(Channel.UNLIMITED)
+ return mockk(relaxed = true).apply {
+ every { coroutineContext } returns context
+ every { incoming } returns incomingFrames
+ every { outgoing } returns outgoingFrames
+ coEvery { send(any()) } coAnswers {
+ val frame = firstArg()
+ if (frame is Frame.Text) {
+ onSendText(frame.readText())
+ }
+ }
+ }
+ }
+
+ private fun createSampleNoteTool(): AppFunctionMetadata =
+ AppFunctionMetadata(
+ id = "com.example.notes.createNote",
+ packageName = "com.example.notes",
+ isEnabled = true,
+ schema = null,
+ parameters =
+ listOf(
+ AppFunctionParameterMetadata(
+ name = "title",
+ isRequired = true,
+ dataType = AppFunctionStringTypeMetadata(isNullable = false),
+ description = "Note title",
+ ),
+ ),
+ response =
+ AppFunctionResponseMetadata(
+ valueType = AppFunctionStringTypeMetadata(isNullable = false),
+ description = "",
+ ),
+ components = AppFunctionComponentsMetadata(emptyMap()),
+ description = "Creates a note",
+ deprecation = null,
+ )
+}
diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt
index 62cb16ab..8388855d 100644
--- a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt
+++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt
@@ -520,4 +520,68 @@ class GeminiToolConverterTest {
assertEquals(expectedJson, schema)
}
+
+ @Test
+ fun convertForLiveApi_normalizesEnumsAndStripsFileReferenceFormat() {
+ val enumParam =
+ AppFunctionParameterMetadata(
+ name = "mode",
+ isRequired = true,
+ dataType =
+ AppFunctionStringTypeMetadata(isNullable = false, enumValues = setOf("A", "B")),
+ description = "Mode parameter",
+ )
+ val uriParam =
+ AppFunctionParameterMetadata(
+ name = "wallpaperUri",
+ isRequired = false,
+ dataType = AppFunctionStringTypeMetadata(isNullable = true),
+ description = "Wallpaper URI",
+ )
+ val tool =
+ AppFunctionMetadata(
+ id = "com.example.my_function",
+ packageName = "com.example",
+ isEnabled = true,
+ schema = null,
+ parameters = listOf(enumParam, uriParam),
+ response =
+ AppFunctionResponseMetadata(
+ valueType = AppFunctionStringTypeMetadata(isNullable = false),
+ description = "",
+ ),
+ components = AppFunctionComponentsMetadata(emptyMap()),
+ description = "Test function",
+ deprecation = null,
+ )
+
+ val liveSchema = converter.convertForLiveApi(tool)
+
+ val expectedJson =
+ Json.parseToJsonElement(
+ """
+ {
+ "name": "com_example_my_function",
+ "description": "Test function",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "mode": {
+ "type": "string",
+ "enum": ["A", "B"],
+ "description": "Mode parameter"
+ },
+ "wallpaperUri": {
+ "type": "string",
+ "description": "Wallpaper URI"
+ }
+ },
+ "required": ["mode"]
+ }
+ }
+ """,
+ )
+
+ assertEquals(expectedJson, liveSchema)
+ }
}
diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt
index ea05a1fb..b18b913c 100644
--- a/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt
+++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/SettingsRepositoryTest.kt
@@ -77,4 +77,19 @@ class SettingsRepositoryTest {
val value = repository.serviceTier.first()
assertEquals(ServiceTier.PRIORITY, value)
}
+
+ @Test
+ fun geminiLiveVoice_defaultsToZephyr() =
+ runTest {
+ val value = repository.geminiLiveVoice.first()
+ assertEquals(GeminiLiveVoice.ZEPHYR, value)
+ }
+
+ @Test
+ fun setGeminiLiveVoice_persistsValue() =
+ runTest {
+ repository.setGeminiLiveVoice(GeminiLiveVoice.AOEDE)
+ val value = repository.geminiLiveVoice.first()
+ assertEquals(GeminiLiveVoice.AOEDE, value)
+ }
}
diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt
index 8c26dbd8..5973bf5d 100644
--- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt
+++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt
@@ -34,8 +34,10 @@ import com.example.appfunctions.agent.data.db.entities.ThreadEntity
import com.example.appfunctions.agent.domain.appfunction.ConvertInputToAppFunctionDataUseCase
import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult
import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionUseCase
+import com.example.appfunctions.agent.domain.appfunction.ExecuteToolCallUseCase
import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionStatesUseCase
import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase
+import com.example.appfunctions.agent.domain.appfunction.GetEnabledToolsUseCase
import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase
import com.example.appfunctions.agent.domain.chat.ObservePendingMessagesUseCase
import com.example.appfunctions.agent.domain.chat.SendMessageUseCase
@@ -81,9 +83,23 @@ class AgentOrchestratorTest {
@Before
fun setUp() {
+ val getEnabledToolsUseCase =
+ GetEnabledToolsUseCase(
+ settingsRepository = settingsRepository,
+ getAppFunctionsUseCase = getAppFunctionsUseCase,
+ getAppFunctionStatesUseCase = getAppFunctionStatesUseCase,
+ agentInternalTools = agentInternalTools,
+ )
+ val executeToolCallUseCase =
+ ExecuteToolCallUseCase(
+ context = context,
+ convertInputToAppFunctionDataUseCase = convertInputToAppFunctionDataUseCase,
+ executeAppFunctionUseCase = executeAppFunctionUseCase,
+ agentInternalTools = agentInternalTools,
+ )
+
agentOrchestrator =
AgentOrchestrator(
- context = context,
manageThreadsUseCase = manageThreadsUseCase,
observePendingMessagesUseCase = observePendingMessagesUseCase,
sendMessageUseCase = sendMessageUseCase,
@@ -91,12 +107,9 @@ class AgentOrchestratorTest {
updateThreadUseCase = updateThreadUseCase,
llmProviderFactory = llmProviderFactory,
settingsRepository = settingsRepository,
- getAppFunctionsUseCase = getAppFunctionsUseCase,
- getAppFunctionStatesUseCase = getAppFunctionStatesUseCase,
- convertInputToAppFunctionDataUseCase = convertInputToAppFunctionDataUseCase,
- executeAppFunctionUseCase = executeAppFunctionUseCase,
+ getEnabledToolsUseCase = getEnabledToolsUseCase,
+ executeToolCallUseCase = executeToolCallUseCase,
savePendingIntentUseCase = savePendingIntentUseCase,
- agentInternalTools = agentInternalTools,
)
coEvery { getAppFunctionStatesUseCase(any()) } answers {
diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/LiveAgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/LiveAgentOrchestratorTest.kt
new file mode 100644
index 00000000..b4ab1e4f
--- /dev/null
+++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/LiveAgentOrchestratorTest.kt
@@ -0,0 +1,484 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.domain
+
+import android.app.PendingIntent
+import android.util.Log
+import androidx.appfunctions.metadata.AppFunctionComponentsMetadata
+import androidx.appfunctions.metadata.AppFunctionMetadata
+import androidx.appfunctions.metadata.AppFunctionParameterMetadata
+import androidx.appfunctions.metadata.AppFunctionResponseMetadata
+import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata
+import com.example.appfunctions.agent.data.LlmProviderName
+import com.example.appfunctions.agent.data.SettingsRepository
+import com.example.appfunctions.agent.data.db.entities.MessageAttachment
+import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus
+import com.example.appfunctions.agent.data.db.entities.MessageRole
+import com.example.appfunctions.agent.domain.appfunction.ExecuteToolCallUseCase
+import com.example.appfunctions.agent.domain.appfunction.GetEnabledToolsUseCase
+import com.example.appfunctions.agent.domain.appfunction.ToolCallExecutionResult
+import com.example.appfunctions.agent.domain.chat.SendMessageUseCase
+import com.example.appfunctions.agent.domain.pendingintent.LaunchPendingIntentUseCase
+import io.mockk.coEvery
+import io.mockk.coVerify
+import io.mockk.coVerifyOrder
+import io.mockk.every
+import io.mockk.mockk
+import io.mockk.mockkStatic
+import io.mockk.verify
+import io.mockk.verifyOrder
+import kotlinx.coroutines.CompletableDeferred
+import kotlinx.coroutines.ExperimentalCoroutinesApi
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.emptyFlow
+import kotlinx.coroutines.flow.flowOf
+import kotlinx.coroutines.flow.receiveAsFlow
+import kotlinx.coroutines.test.StandardTestDispatcher
+import kotlinx.coroutines.test.advanceUntilIdle
+import kotlinx.coroutines.test.runCurrent
+import kotlinx.coroutines.test.runTest
+import org.junit.Assert.assertArrayEquals
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertTrue
+import org.junit.Before
+import org.junit.Test
+import java.util.Collections
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicInteger
+
+@OptIn(ExperimentalCoroutinesApi::class)
+class LiveAgentOrchestratorTest {
+ private lateinit var llmProviderFactory: LlmProviderFactory
+ private lateinit var liveLlmProvider: LiveLlmProvider
+ private lateinit var settingsRepository: SettingsRepository
+ private lateinit var getEnabledToolsUseCase: GetEnabledToolsUseCase
+ private lateinit var executeToolCallUseCase: ExecuteToolCallUseCase
+ private lateinit var launchPendingIntentUseCase: LaunchPendingIntentUseCase
+ private lateinit var sendMessageUseCase: SendMessageUseCase
+ private var currentAudioFactory: () -> LiveAudioSession = { mockk(relaxed = true) }
+
+ private lateinit var orchestrator: LiveAgentOrchestrator
+
+ @Before
+ fun setUp() {
+ mockkStatic(Log::class)
+ every { Log.d(any(), any()) } returns 0
+ every { Log.w(any(), any()) } returns 0
+ every { Log.e(any(), any(), any()) } returns 0
+
+ llmProviderFactory = mockk()
+ liveLlmProvider = mockk()
+ settingsRepository = mockk(relaxed = true)
+ getEnabledToolsUseCase = mockk()
+ executeToolCallUseCase = mockk()
+ launchPendingIntentUseCase = mockk()
+ sendMessageUseCase = mockk(relaxed = true)
+
+ every { launchPendingIntentUseCase(any()) } returns Result.success(Unit)
+ every { settingsRepository.selectedProvider } returns flowOf(LlmProviderName.GEMINI)
+ every { settingsRepository.geminiApiKey } returns flowOf("test-api-key")
+ every { llmProviderFactory.getLiveProvider(LlmProviderName.GEMINI) } returns liveLlmProvider
+ coEvery { getEnabledToolsUseCase(any()) } returns emptyList()
+
+ orchestrator =
+ LiveAgentOrchestrator(
+ llmProviderFactory = llmProviderFactory,
+ settingsRepository = settingsRepository,
+ getEnabledToolsUseCase = getEnabledToolsUseCase,
+ executeToolCallUseCase = executeToolCallUseCase,
+ launchPendingIntentUseCase = launchPendingIntentUseCase,
+ sendMessageUseCase = sendMessageUseCase,
+ audioSessionFactory = LiveAudioSessionFactory { currentAudioFactory() },
+ )
+ }
+
+ @Test
+ fun `startSession reports Error and persists FAILED message when API key is missing`() =
+ runTest {
+ val ioDispatcher = StandardTestDispatcher(testScheduler)
+ orchestrator.ioDispatcher = ioDispatcher
+ every { settingsRepository.geminiApiKey } returns flowOf(null)
+
+ orchestrator.startSession("thread-1")
+ advanceUntilIdle()
+
+ assertEquals(
+ LiveSessionState.Error("API key is missing for GEMINI"),
+ orchestrator.state.value,
+ )
+ coVerify {
+ sendMessageUseCase(
+ threadId = "thread-1",
+ role = MessageRole.ASSISTANT,
+ textContent = "API key is missing for GEMINI",
+ processingStatus = MessageProcessingStatus.FAILED,
+ )
+ }
+ }
+
+ @Test
+ fun `stopSession does not call audioSession release synchronously on caller thread`() =
+ runTest {
+ val ioDispatcher = StandardTestDispatcher(testScheduler)
+ orchestrator.ioDispatcher = ioDispatcher
+
+ val mockAudio =
+ mockk(relaxed = true).apply {
+ every { listenToRecording() } returns emptyFlow()
+ }
+ currentAudioFactory = { mockAudio }
+
+ val eventChannel = Channel(Channel.UNLIMITED)
+ stubLiveConnection(eventChannel.receiveAsFlow())
+
+ orchestrator.startSession("thread-1")
+ runCurrent()
+ assertEquals(LiveSessionState.Listening, orchestrator.state.value)
+
+ orchestrator.stopSession()
+ assertEquals(LiveSessionState.Idle, orchestrator.state.value)
+ verify(exactly = 0) { mockAudio.release() }
+
+ advanceUntilIdle()
+ verify(exactly = 1) { mockAudio.release() }
+ }
+
+ @Test
+ fun `startSession waits for prior session audio teardown before starting new LiveAudioSession`() {
+ val events = Collections.synchronizedList(mutableListOf())
+ val session1Listening = CountDownLatch(1)
+ val release1Entered = CountDownLatch(1)
+ val allowRelease1Finish = CountDownLatch(1)
+ val session2Started = CountDownLatch(1)
+
+ val audio1 =
+ mockk(relaxed = true).apply {
+ every { listenToRecording() } returns emptyFlow()
+ every { start() } answers {
+ events.add("start1")
+ session1Listening.countDown()
+ }
+ every { release() } answers {
+ events.add("release1_begin")
+ release1Entered.countDown()
+ assertTrue(allowRelease1Finish.await(2, TimeUnit.SECONDS))
+ events.add("release1_end")
+ }
+ }
+ val audio2 =
+ mockk(relaxed = true).apply {
+ every { listenToRecording() } returns emptyFlow()
+ every { start() } answers {
+ events.add("start2")
+ session2Started.countDown()
+ }
+ every { release() } answers {
+ events.add("release2")
+ }
+ }
+
+ val counter = AtomicInteger(0)
+ currentAudioFactory = { if (counter.incrementAndGet() == 1) audio1 else audio2 }
+
+ val ch1 = Channel(Channel.UNLIMITED)
+ val ch2 = Channel(Channel.UNLIMITED)
+ val connCounter = AtomicInteger(0)
+ coEvery { liveLlmProvider.connect(any(), any(), any()) } coAnswers {
+ val block = thirdArg Unit>()
+ val flow = if (connCounter.incrementAndGet() == 1) ch1.receiveAsFlow() else ch2.receiveAsFlow()
+ block(FakeLiveLlmSession(flow))
+ }
+
+ orchestrator.startSession("thread-1")
+ assertTrue(session1Listening.await(2, TimeUnit.SECONDS))
+
+ orchestrator.stopSession()
+ assertTrue(release1Entered.await(2, TimeUnit.SECONDS))
+
+ orchestrator.startSession("thread-1")
+ Thread.sleep(50)
+ assertEquals(listOf("start1", "release1_begin"), events.toList())
+
+ allowRelease1Finish.countDown()
+ assertTrue(session2Started.await(2, TimeUnit.SECONDS))
+ assertEquals(
+ listOf("start1", "release1_begin", "release1_end", "start2"),
+ events.toList(),
+ )
+
+ orchestrator.stopSession()
+ }
+
+ @Test
+ fun `streams audio chunks, flushes transcriptions in order, and stops playback on interruption`() =
+ runTest {
+ val ioDispatcher = StandardTestDispatcher(testScheduler)
+ orchestrator.ioDispatcher = ioDispatcher
+
+ val micPcm = byteArrayOf(1, 2, 3, 4)
+ val modelPcm = byteArrayOf(9, 8, 7, 6)
+ val queuedChunks = mutableListOf()
+ val mockAudio =
+ mockk(relaxed = true).apply {
+ every { listenToRecording() } returns flowOf(micPcm)
+ every { queueAudio(any()) } answers { queuedChunks.add(firstArg()) }
+ }
+ currentAudioFactory = { mockAudio }
+
+ val eventChannel = Channel(Channel.UNLIMITED)
+ val fakeSession = stubLiveConnection(eventChannel.receiveAsFlow())
+
+ orchestrator.startSession("thread-1")
+ runCurrent()
+
+ eventChannel.trySend(LiveLlmEvent.InputTranscription("What is "))
+ eventChannel.trySend(LiveLlmEvent.InputTranscription("the weather?"))
+ eventChannel.trySend(LiveLlmEvent.OutputTranscription("It is sunny"))
+ eventChannel.trySend(LiveLlmEvent.AudioChunk(modelPcm))
+ eventChannel.trySend(LiveLlmEvent.Interrupted)
+ eventChannel.close()
+ advanceUntilIdle()
+
+ assertEquals(1, fakeSession.sentAudio.size)
+ assertArrayEquals(micPcm, fakeSession.sentAudio.first())
+ assertEquals(1, queuedChunks.size)
+ assertArrayEquals(modelPcm, queuedChunks.first())
+ verify(exactly = 1) { mockAudio.stopAudio() }
+ coVerifyOrder {
+ sendMessageUseCase(
+ threadId = "thread-1",
+ role = MessageRole.USER,
+ textContent = "What is the weather?",
+ processingStatus = MessageProcessingStatus.PROCESSED,
+ )
+ sendMessageUseCase(
+ threadId = "thread-1",
+ role = MessageRole.ASSISTANT,
+ textContent = "It is sunny",
+ processingStatus = MessageProcessingStatus.PROCESSED,
+ )
+ }
+ }
+
+ @Test
+ fun `ToolCallRequest gates recording, updates status, launches PendingIntent directly, saves attachments, and sends ToolOutputs`() =
+ runTest {
+ val ioDispatcher = StandardTestDispatcher(testScheduler)
+ orchestrator.ioDispatcher = ioDispatcher
+
+ val noteTool = createSampleNoteTool()
+ coEvery { getEnabledToolsUseCase("com.example.notes") } returns listOf(noteTool)
+
+ val noteCall =
+ LlmResponsePart.ToolCall(
+ packageName = "com.example.notes",
+ functionId = noteTool.id,
+ arguments = mapOf("title" to "Shopping"),
+ callId = "call-1",
+ )
+ val callAction =
+ LlmResponsePart.ToolCall(
+ packageName = "com.example.notes",
+ functionId = "makeCall",
+ arguments = emptyMap(),
+ callId = "call-2",
+ )
+ val pendingIntent = mockk()
+ val observedStatuses = mutableListOf()
+
+ coEvery {
+ executeToolCallUseCase(noteCall, listOf(noteTool), "thread-1")
+ } answers {
+ observedStatuses.add(orchestrator.status.value)
+ ToolCallExecutionResult.Success(
+ toolOutput = ToolOutput(noteTool.id, "call-1", """{"status":"ok"}"""),
+ attachment = MessageAttachment("content://img/1.png", "image/png"),
+ )
+ }
+ coEvery {
+ executeToolCallUseCase(callAction, listOf(noteTool), "thread-1")
+ } answers {
+ observedStatuses.add(orchestrator.status.value)
+ ToolCallExecutionResult.PendingIntentRequired("pi-123", pendingIntent)
+ }
+
+ val mockAudio =
+ mockk(relaxed = true).apply {
+ every { listenToRecording() } returns emptyFlow()
+ }
+ currentAudioFactory = { mockAudio }
+
+ val eventChannel = Channel(Channel.UNLIMITED)
+ eventChannel.trySend(LiveLlmEvent.ToolCallRequest(listOf(noteCall, callAction)))
+ val toolResponseSent = CompletableDeferred()
+ val fakeSession =
+ stubLiveConnection(
+ events = eventChannel.receiveAsFlow(),
+ onToolResponses = {
+ toolResponseSent.complete(Unit)
+ eventChannel.close()
+ },
+ )
+
+ orchestrator.startSession("thread-1", targetPackageName = "com.example.notes")
+ advanceUntilIdle()
+
+ assertTrue(toolResponseSent.isCompleted)
+ assertEquals(AgentStatus.Idle, orchestrator.status.value)
+ assertEquals(
+ listOf(
+ AgentStatus.InvokingTool(noteTool.id, "com.example.notes"),
+ AgentStatus.InvokingTool("makeCall", "com.example.notes"),
+ ),
+ observedStatuses,
+ )
+ verifyOrder {
+ mockAudio.start()
+ mockAudio.pauseRecording()
+ mockAudio.resumeRecording()
+ mockAudio.release()
+ }
+ verify(exactly = 1) { launchPendingIntentUseCase(pendingIntent) }
+ coVerify(exactly = 0) {
+ sendMessageUseCase(
+ threadId = any(),
+ role = any(),
+ textContent = any(),
+ processingStatus = any(),
+ pendingIntentId = "pi-123",
+ )
+ }
+ coVerify {
+ sendMessageUseCase(
+ threadId = "thread-1",
+ role = MessageRole.ASSISTANT,
+ textContent = "",
+ processingStatus = MessageProcessingStatus.PROCESSED,
+ attachments = listOf(MessageAttachment("content://img/1.png", "image/png")),
+ )
+ }
+ assertEquals(2, fakeSession.sentToolOutputs.single().size)
+ assertEquals("""{"status":"ok"}""", fakeSession.sentToolOutputs.single()[0].result)
+ assertTrue(
+ fakeSession.sentToolOutputs
+ .single()[1]
+ .result
+ .contains("\"status\":\"launched\""),
+ )
+ }
+
+ @Test
+ fun `ToolCallRequest reports failure ToolOutput when PendingIntent launch fails`() =
+ runTest {
+ val ioDispatcher = StandardTestDispatcher(testScheduler)
+ orchestrator.ioDispatcher = ioDispatcher
+
+ val pendingIntent = mockk()
+ every { launchPendingIntentUseCase(pendingIntent) } returns
+ Result.failure(PendingIntent.CanceledException())
+
+ val callAction =
+ LlmResponsePart.ToolCall(
+ packageName = "com.example.chat",
+ functionId = "makeCall",
+ arguments = emptyMap(),
+ callId = "call-err",
+ )
+ coEvery { executeToolCallUseCase(callAction, emptyList(), "thread-1") } returns
+ ToolCallExecutionResult.PendingIntentRequired("pi-err", pendingIntent)
+
+ val mockAudio =
+ mockk(relaxed = true).apply {
+ every { listenToRecording() } returns emptyFlow()
+ }
+ currentAudioFactory = { mockAudio }
+
+ val eventChannel = Channel(Channel.UNLIMITED)
+ eventChannel.trySend(LiveLlmEvent.ToolCallRequest(listOf(callAction)))
+ val fakeSession =
+ stubLiveConnection(
+ events = eventChannel.receiveAsFlow(),
+ onToolResponses = { eventChannel.close() },
+ )
+
+ orchestrator.startSession("thread-1")
+ advanceUntilIdle()
+
+ assertEquals(1, fakeSession.sentToolOutputs.single().size)
+ assertTrue(
+ fakeSession.sentToolOutputs
+ .single()[0]
+ .result
+ .contains("Tool execution failed for makeCall: PendingIntent canceled"),
+ )
+ }
+
+ private fun stubLiveConnection(
+ events: Flow,
+ onToolResponses: (List) -> Unit = {},
+ ): FakeLiveLlmSession {
+ val fakeSession = FakeLiveLlmSession(events, onToolResponses)
+ coEvery { liveLlmProvider.connect(any(), any(), any()) } coAnswers {
+ val block = thirdArg Unit>()
+ block(fakeSession)
+ }
+ return fakeSession
+ }
+
+ private class FakeLiveLlmSession(
+ override val events: Flow,
+ private val onToolResponses: (List) -> Unit = {},
+ ) : LiveLlmSession {
+ val sentAudio = mutableListOf()
+ val sentToolOutputs = mutableListOf>()
+
+ override suspend fun sendAudio(pcmBytes: ByteArray) {
+ sentAudio.add(pcmBytes)
+ }
+
+ override suspend fun sendToolResponses(outputs: List) {
+ sentToolOutputs.add(outputs)
+ onToolResponses(outputs)
+ }
+ }
+
+ private fun createSampleNoteTool(): AppFunctionMetadata =
+ AppFunctionMetadata(
+ id = "com.example.notes.createNote",
+ packageName = "com.example.notes",
+ isEnabled = true,
+ schema = null,
+ parameters =
+ listOf(
+ AppFunctionParameterMetadata(
+ name = "title",
+ isRequired = true,
+ dataType = AppFunctionStringTypeMetadata(isNullable = false),
+ description = "Note title",
+ ),
+ ),
+ response =
+ AppFunctionResponseMetadata(
+ valueType = AppFunctionStringTypeMetadata(isNullable = false),
+ description = "",
+ ),
+ components = AppFunctionComponentsMetadata(emptyMap()),
+ description = "Creates a note",
+ deprecation = null,
+ )
+}
diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteToolCallUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteToolCallUseCaseTest.kt
new file mode 100644
index 00000000..e94dc542
--- /dev/null
+++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteToolCallUseCaseTest.kt
@@ -0,0 +1,231 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.domain.appfunction
+
+import android.app.PendingIntent
+import android.content.Context
+import android.content.Intent
+import androidx.appfunctions.AppFunctionData
+import androidx.appfunctions.AppFunctionPermissionRequiredException
+import androidx.appfunctions.metadata.AppFunctionMetadata
+import com.example.appfunctions.agent.data.AgentInternalTools
+import com.example.appfunctions.agent.data.db.entities.MessageAttachment
+import com.example.appfunctions.agent.domain.LlmResponsePart
+import io.mockk.coEvery
+import io.mockk.every
+import io.mockk.mockk
+import io.mockk.verify
+import kotlinx.coroutines.test.runTest
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertSame
+import org.junit.Assert.assertTrue
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.robolectric.RobolectricTestRunner
+import org.robolectric.annotation.Config
+
+@RunWith(RobolectricTestRunner::class)
+@Config(sdk = [34])
+class ExecuteToolCallUseCaseTest {
+ private val context: Context = mockk(relaxed = true)
+ private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase = mockk()
+ private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase = mockk()
+ private val agentInternalTools: AgentInternalTools = mockk(relaxed = true)
+
+ private lateinit var useCase: ExecuteToolCallUseCase
+
+ @Before
+ fun setUp() {
+ useCase =
+ ExecuteToolCallUseCase(
+ context = context,
+ convertInputToAppFunctionDataUseCase = convertInputToAppFunctionDataUseCase,
+ executeAppFunctionUseCase = executeAppFunctionUseCase,
+ agentInternalTools = agentInternalTools,
+ )
+ }
+
+ @Test
+ fun `internal generateImage tool returns Success with extracted MessageAttachment`() =
+ runTest {
+ val imageResult =
+ AgentInternalTools.GeneratedImageResult(
+ imageUri = "content://com.example.appfunctions.agent.fileprovider/cache/cat.png",
+ mimeType = "image/png",
+ prompt = "a cat",
+ )
+ coEvery { agentInternalTools.generateImage("a cat", "1:1") } returns imageResult
+
+ val toolCall =
+ LlmResponsePart.ToolCall(
+ packageName = AgentInternalTools.INTERNAL_TOOL_PACKAGE,
+ functionId = "generateImage",
+ arguments = mapOf("prompt" to "a cat", "aspectRatio" to "1:1"),
+ callId = "call-1",
+ )
+
+ val result = useCase(toolCall = toolCall, tools = emptyList(), threadId = "thread-1")
+
+ assertTrue(result is ToolCallExecutionResult.Success)
+ val success = result as ToolCallExecutionResult.Success
+ assertEquals("call-1", success.toolOutput.callId)
+ assertEquals("generateImage", success.toolOutput.functionId)
+ assertEquals(
+ MessageAttachment(uri = imageResult.imageUri, mimeType = "image/png"),
+ success.attachment,
+ )
+ }
+
+ @Test
+ fun `missing tool metadata returns Failure`() =
+ runTest {
+ val toolCall =
+ LlmResponsePart.ToolCall(
+ packageName = "com.example.notes",
+ functionId = "nonexistent",
+ arguments = emptyMap(),
+ callId = "call-2",
+ )
+
+ val result = useCase(toolCall = toolCall, tools = emptyList(), threadId = "thread-1")
+
+ assertTrue(result is ToolCallExecutionResult.Failure)
+ assertEquals(
+ "Tool not found: nonexistent",
+ (result as ToolCallExecutionResult.Failure).errorMessage,
+ )
+ }
+
+ @Test
+ fun `AppFunction execution grants recursive URI permissions and returns Data output`() =
+ runTest {
+ val tool = createMockTool("com.example.notes", "attachImages")
+ coEvery {
+ convertInputToAppFunctionDataUseCase(any(), any(), any())
+ } returns Result.success(AppFunctionData.EMPTY)
+ coEvery {
+ executeAppFunctionUseCase(tool, any(), "thread-1")
+ } returns ExecuteAppFunctionResult.Data(AppFunctionData.EMPTY, """{"ok":true}""")
+
+ val nestedUri = "content://com.example.appfunctions.agent.fileprovider/cache/nested.jpg"
+ val toolCall =
+ LlmResponsePart.ToolCall(
+ packageName = "com.example.notes",
+ functionId = "attachImages",
+ arguments =
+ mapOf(
+ "payload" to mapOf("uris" to listOf(nestedUri)),
+ ),
+ callId = "call-3",
+ )
+
+ val result = useCase(toolCall = toolCall, tools = listOf(tool), threadId = "thread-1")
+
+ assertTrue(result is ToolCallExecutionResult.Success)
+ assertEquals(
+ """{"ok":true}""",
+ (result as ToolCallExecutionResult.Success).toolOutput.result,
+ )
+ verify(exactly = 1) {
+ context.grantUriPermission(
+ eq("com.example.notes"),
+ match { it.toString() == nestedUri },
+ eq(Intent.FLAG_GRANT_READ_URI_PERMISSION),
+ )
+ }
+ }
+
+ @Test
+ fun `AppFunction PendingIntentAction returns PendingIntentRequired`() =
+ runTest {
+ val tool = createMockTool("com.example.payments", "sendPayment")
+ val pendingIntent = mockk()
+ coEvery {
+ convertInputToAppFunctionDataUseCase(any(), any(), any())
+ } returns Result.success(AppFunctionData.EMPTY)
+ coEvery {
+ executeAppFunctionUseCase(tool, any(), "thread-1")
+ } returns ExecuteAppFunctionResult.PendingIntentAction(pendingIntent)
+
+ val toolCall =
+ LlmResponsePart.ToolCall(
+ packageName = "com.example.payments",
+ functionId = "sendPayment",
+ arguments = emptyMap(),
+ callId = "call-4",
+ )
+
+ val result = useCase(toolCall = toolCall, tools = listOf(tool), threadId = "thread-1")
+
+ assertTrue(result is ToolCallExecutionResult.PendingIntentRequired)
+ assertSame(
+ pendingIntent,
+ (result as ToolCallExecutionResult.PendingIntentRequired).pendingIntent,
+ )
+ }
+
+ @Test
+ fun `AppFunctionException returns formatted recoverable Success output while generic Exception returns Failure`() =
+ runTest {
+ val tool = createMockTool("com.example.calendar", "createEvent")
+ coEvery {
+ convertInputToAppFunctionDataUseCase(any(), any(), any())
+ } returns Result.success(AppFunctionData.EMPTY)
+
+ val appException = AppFunctionPermissionRequiredException("Calendar denied")
+ coEvery {
+ executeAppFunctionUseCase(tool, any(), "thread-1")
+ } returns ExecuteAppFunctionResult.Error(appException)
+
+ val toolCall =
+ LlmResponsePart.ToolCall(
+ packageName = "com.example.calendar",
+ functionId = "createEvent",
+ arguments = emptyMap(),
+ callId = "call-5",
+ )
+
+ val recoverable =
+ useCase(toolCall = toolCall, tools = listOf(tool), threadId = "thread-1")
+ assertTrue(recoverable is ToolCallExecutionResult.Success)
+ assertEquals(
+ "Tool execution failed for createEvent: Error: AppFunctionPermissionRequiredException - Calendar denied",
+ (recoverable as ToolCallExecutionResult.Success).toolOutput.result,
+ )
+
+ coEvery {
+ executeAppFunctionUseCase(tool, any(), "thread-1")
+ } returns ExecuteAppFunctionResult.Error(IllegalStateException("Binder died"))
+
+ val fatal = useCase(toolCall = toolCall, tools = listOf(tool), threadId = "thread-1")
+ assertTrue(fatal is ToolCallExecutionResult.Failure)
+ assertEquals(
+ "Tool execution failed for createEvent: Binder died",
+ (fatal as ToolCallExecutionResult.Failure).errorMessage,
+ )
+ }
+
+ private fun createMockTool(
+ packageName: String,
+ id: String,
+ ): AppFunctionMetadata =
+ mockk(relaxed = true).apply {
+ every { this@apply.packageName } returns packageName
+ every { this@apply.id } returns id
+ every { parameters } returns emptyList()
+ }
+}
diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetEnabledToolsUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetEnabledToolsUseCaseTest.kt
new file mode 100644
index 00000000..cc87d875
--- /dev/null
+++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetEnabledToolsUseCaseTest.kt
@@ -0,0 +1,119 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.domain.appfunction
+
+import androidx.appfunctions.AppFunctionName
+import androidx.appfunctions.AppFunctionState
+import androidx.appfunctions.metadata.AppFunctionMetadata
+import androidx.appfunctions.metadata.AppFunctionPackageMetadata
+import com.example.appfunctions.agent.data.AgentInternalTools
+import com.example.appfunctions.agent.data.SettingsRepository
+import io.mockk.coEvery
+import io.mockk.every
+import io.mockk.mockk
+import kotlinx.coroutines.flow.flowOf
+import kotlinx.coroutines.test.runTest
+import org.junit.Assert.assertEquals
+import org.junit.Before
+import org.junit.Test
+
+class GetEnabledToolsUseCaseTest {
+ private val settingsRepository: SettingsRepository = mockk()
+ private val getAppFunctionsUseCase: GetAppFunctionsUseCase = mockk()
+ private val getAppFunctionStatesUseCase: GetAppFunctionStatesUseCase = mockk()
+ private val agentInternalTools: AgentInternalTools = mockk()
+
+ private lateinit var useCase: GetEnabledToolsUseCase
+
+ @Before
+ fun setUp() {
+ every { settingsRepository.disconnectedApps } returns flowOf(emptySet())
+ every { agentInternalTools.getInternalToolsMetadata() } returns emptyList()
+ coEvery { getAppFunctionStatesUseCase(any()) } answers {
+ firstArg>().map { name ->
+ createState(name, isEnabled = true)
+ }
+ }
+
+ useCase =
+ GetEnabledToolsUseCase(
+ settingsRepository = settingsRepository,
+ getAppFunctionsUseCase = getAppFunctionsUseCase,
+ getAppFunctionStatesUseCase = getAppFunctionStatesUseCase,
+ agentInternalTools = agentInternalTools,
+ )
+ }
+
+ @Test
+ fun `invoke excludes disabled tools and disconnected apps while keeping internal tools`() =
+ runTest {
+ val enabledTool = createTool("com.example.notes", "com.example.notes.createNote")
+ val disabledTool = createTool("com.example.notes", "com.example.notes.deleteNote")
+ val disconnectedTool = createTool("com.example.chat", "com.example.chat.sendMessage")
+ val internalTool =
+ createTool(AgentInternalTools.INTERNAL_TOOL_PACKAGE, "generateImage")
+ val internalState = createState(internalTool.name, isEnabled = true)
+
+ val pkgMeta = mockk(relaxed = true)
+ every { getAppFunctionsUseCase() } returns
+ flowOf(mapOf(pkgMeta to listOf(enabledTool, disabledTool, disconnectedTool)))
+ every { settingsRepository.disconnectedApps } returns flowOf(setOf("com.example.chat"))
+ every { agentInternalTools.getInternalToolsMetadata() } returns
+ listOf(internalTool to internalState)
+ coEvery { getAppFunctionStatesUseCase(any()) } answers {
+ firstArg>().map { name ->
+ createState(name, isEnabled = name != disabledTool.name)
+ }
+ }
+
+ val result = useCase()
+
+ assertEquals(listOf(enabledTool, internalTool), result)
+ }
+
+ @Test
+ fun `invoke scopes returned tools to targetPackageName when specified`() =
+ runTest {
+ val notesTool = createTool("com.example.notes", "createNote")
+ val calendarTool = createTool("com.example.calendar", "createEvent")
+ val pkgMeta = mockk(relaxed = true)
+ every { getAppFunctionsUseCase() } returns
+ flowOf(mapOf(pkgMeta to listOf(notesTool, calendarTool)))
+
+ val result = useCase(targetPackageName = "com.example.calendar")
+
+ assertEquals(listOf(calendarTool), result)
+ }
+
+ private fun createTool(
+ packageName: String,
+ id: String,
+ ): AppFunctionMetadata =
+ mockk(relaxed = true).apply {
+ every { this@apply.packageName } returns packageName
+ every { this@apply.id } returns id
+ every { this@apply.name } returns AppFunctionName(packageName, id)
+ }
+
+ private fun createState(
+ name: AppFunctionName,
+ isEnabled: Boolean,
+ ): AppFunctionState =
+ mockk().apply {
+ every { functionName } returns name
+ every { this@apply.isEnabled } returns isEnabled
+ }
+}
diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt
index 27a48055..2aa588c0 100644
--- a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt
+++ b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt
@@ -98,7 +98,6 @@ class AgentDemoViewModelTest {
every { agentOrchestrator.status } returns agentStatusFlow
every { getChatHistoryUseCase(any()) } returns messagesFlow
every { observeActivePendingIntentsUseCase() } returns activePendingActionIdsFlow
- every { observeActivePendingIntentsUseCase() } returns activePendingActionIdsFlow
coEvery { agentOrchestrator.observeAndProcessMessages(any()) } returns Unit
}
@@ -123,20 +122,7 @@ class AgentDemoViewModelTest {
manageThreadsUseCase.updateThreadModel("thread-1", LlmModel.GEMINI_3_1_PRO_PREVIEW)
} returns Unit
- viewModel =
- AgentDemoViewModel(
- savedStateHandle,
- getChatHistoryUseCase,
- manageThreadsUseCase,
- sendMessageUseCase,
- agentOrchestrator,
- settingsRepository,
- observeActivePendingIntentsUseCase,
- launchPendingIntentUseCase,
- consumePendingIntentUseCase,
- getInstalledAppsUseCase,
- getAppFunctionsUseCase,
- )
+ viewModel = createViewModel()
viewModel.onEvent(AgentUiEvent.OnModelSelected(LlmModel.GEMINI_3_1_PRO_PREVIEW))
@@ -151,20 +137,7 @@ class AgentDemoViewModelTest {
threadsFlow.value = emptyList()
coEvery { manageThreadsUseCase.createThread(LlmModel.DEFAULT) } returns "thread-new"
- viewModel =
- AgentDemoViewModel(
- savedStateHandle,
- getChatHistoryUseCase,
- manageThreadsUseCase,
- sendMessageUseCase,
- agentOrchestrator,
- settingsRepository,
- observeActivePendingIntentsUseCase,
- launchPendingIntentUseCase,
- consumePendingIntentUseCase,
- getInstalledAppsUseCase,
- getAppFunctionsUseCase,
- )
+ viewModel = createViewModel()
coVerify { manageThreadsUseCase.createThread(LlmModel.DEFAULT) }
}
@@ -182,20 +155,7 @@ class AgentDemoViewModelTest {
threadsFlow.value = listOf(existingThread)
selectedProviderFlow.value = LlmProviderName.GEMINI
- viewModel =
- AgentDemoViewModel(
- savedStateHandle,
- getChatHistoryUseCase,
- manageThreadsUseCase,
- sendMessageUseCase,
- agentOrchestrator,
- settingsRepository,
- observeActivePendingIntentsUseCase,
- launchPendingIntentUseCase,
- consumePendingIntentUseCase,
- getInstalledAppsUseCase,
- getAppFunctionsUseCase,
- )
+ viewModel = createViewModel()
coVerify(exactly = 0) { manageThreadsUseCase.createThread(any()) }
assertEquals(
@@ -217,20 +177,7 @@ class AgentDemoViewModelTest {
threadsFlow.value = listOf(existingThread)
selectedProviderFlow.value = LlmProviderName.GEMINI
- viewModel =
- AgentDemoViewModel(
- savedStateHandle,
- getChatHistoryUseCase,
- manageThreadsUseCase,
- sendMessageUseCase,
- agentOrchestrator,
- settingsRepository,
- observeActivePendingIntentsUseCase,
- launchPendingIntentUseCase,
- consumePendingIntentUseCase,
- getInstalledAppsUseCase,
- getAppFunctionsUseCase,
- )
+ viewModel = createViewModel()
coEvery { sendMessageUseCase(any(), any(), any(), any()) } returns Unit
@@ -259,20 +206,7 @@ class AgentDemoViewModelTest {
threadsFlow.value = listOf(existingThread)
selectedProviderFlow.value = LlmProviderName.GEMINI
- viewModel =
- AgentDemoViewModel(
- savedStateHandle,
- getChatHistoryUseCase,
- manageThreadsUseCase,
- sendMessageUseCase,
- agentOrchestrator,
- settingsRepository,
- observeActivePendingIntentsUseCase,
- launchPendingIntentUseCase,
- consumePendingIntentUseCase,
- getInstalledAppsUseCase,
- getAppFunctionsUseCase,
- )
+ viewModel = createViewModel()
val message =
MessageEntity(
@@ -308,21 +242,23 @@ class AgentDemoViewModelTest {
threadsFlow.value = listOf(newThread, oldThread)
selectedProviderFlow.value = LlmProviderName.GEMINI
- viewModel =
- AgentDemoViewModel(
- savedStateHandle,
- getChatHistoryUseCase,
- manageThreadsUseCase,
- sendMessageUseCase,
- agentOrchestrator,
- settingsRepository,
- observeActivePendingIntentsUseCase,
- launchPendingIntentUseCase,
- consumePendingIntentUseCase,
- getInstalledAppsUseCase,
- getAppFunctionsUseCase,
- )
+ viewModel = createViewModel()
assertEquals(newThread, (viewModel.uiState.value as AgentUiState.Loaded).currentThread)
}
+
+ private fun createViewModel(): AgentDemoViewModel =
+ AgentDemoViewModel(
+ savedStateHandle,
+ getChatHistoryUseCase,
+ manageThreadsUseCase,
+ sendMessageUseCase,
+ agentOrchestrator,
+ settingsRepository,
+ observeActivePendingIntentsUseCase,
+ launchPendingIntentUseCase,
+ consumePendingIntentUseCase,
+ getInstalledAppsUseCase,
+ getAppFunctionsUseCase,
+ )
}
diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt
index 7ea3b0be..e36b55b0 100644
--- a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt
+++ b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/FakeSettingsRepository.kt
@@ -15,6 +15,7 @@
*/
package com.example.appfunctions.agent.ui.screens.agentdemo
+import com.example.appfunctions.agent.data.GeminiLiveVoice
import com.example.appfunctions.agent.data.LlmProviderName
import com.example.appfunctions.agent.data.ServiceTier
import com.example.appfunctions.agent.data.SettingsRepository
@@ -31,6 +32,9 @@ class FakeSettingsRepository : SettingsRepository {
private val _serviceTier = MutableStateFlow(ServiceTier.STANDARD)
override val serviceTier: Flow = _serviceTier
+ private val _geminiLiveVoice = MutableStateFlow(GeminiLiveVoice.DEFAULT)
+ override val geminiLiveVoice: Flow = _geminiLiveVoice
+
private val _pinnedApps = MutableStateFlow>(emptySet())
override val pinnedApps: Flow> = _pinnedApps
@@ -46,6 +50,10 @@ class FakeSettingsRepository : SettingsRepository {
_serviceTier.value = tier
}
+ override suspend fun setGeminiLiveVoice(voice: GeminiLiveVoice) {
+ _geminiLiveVoice.value = voice
+ }
+
override suspend fun setAppPinned(
packageName: String,
pinned: Boolean,
diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModelTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModelTest.kt
index d0b176a5..d9a33ea4 100644
--- a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModelTest.kt
+++ b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/SettingsViewModelTest.kt
@@ -16,6 +16,7 @@
package com.example.appfunctions.agent.ui.screens.agentdemo
import androidx.compose.runtime.snapshots.Snapshot
+import com.example.appfunctions.agent.data.GeminiLiveVoice
import com.example.appfunctions.agent.data.LlmProviderName
import com.example.appfunctions.agent.data.ServiceTier
import kotlinx.coroutines.Dispatchers
@@ -120,6 +121,23 @@ class SettingsViewModelTest {
val uiState = viewModel.uiState.value
assertEquals(ServiceTier.PRIORITY, uiState.serviceTier)
+ job.cancel()
+ }
+
+ @Test
+ fun setGeminiLiveVoice_updatesRepository() =
+ runTest {
+ val job = backgroundScope.launch { viewModel.uiState.collect {} }
+
+ assertEquals(GeminiLiveVoice.ZEPHYR, viewModel.uiState.value.geminiLiveVoice)
+
+ viewModel.setGeminiLiveVoice(GeminiLiveVoice.AOEDE)
+
+ testScheduler.advanceUntilIdle()
+
+ val uiState = viewModel.uiState.value
+ assertEquals(GeminiLiveVoice.AOEDE, uiState.geminiLiveVoice)
+
job.cancel()
}
}
diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionServiceTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionServiceTest.kt
new file mode 100644
index 00000000..8ab142e2
--- /dev/null
+++ b/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionServiceTest.kt
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.voice
+
+import android.content.Context
+import android.os.Bundle
+import android.provider.Settings
+import androidx.test.core.app.ApplicationProvider
+import io.mockk.Runs
+import io.mockk.every
+import io.mockk.just
+import io.mockk.slot
+import io.mockk.spyk
+import io.mockk.verify
+import org.junit.After
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertNull
+import org.junit.Assert.assertTrue
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.robolectric.Robolectric
+import org.robolectric.RobolectricTestRunner
+import org.robolectric.annotation.Config
+
+@RunWith(RobolectricTestRunner::class)
+@Config(sdk = [34])
+class AgentVoiceInteractionServiceTest {
+ private lateinit var context: Context
+
+ @Before
+ fun setUp() {
+ context = ApplicationProvider.getApplicationContext()
+ AgentVoiceInteractionService.resetForTesting()
+ }
+
+ @After
+ fun tearDown() {
+ AgentVoiceInteractionService.resetForTesting()
+ }
+
+ @Test
+ fun `launchLiveSession returns false when service is not active digital assistant`() {
+ AgentVoiceInteractionService.isActiveServiceChecker = { false }
+
+ val launched =
+ AgentVoiceInteractionService.launchLiveSession(
+ context = context,
+ threadId = "thread-1",
+ targetPackageName = "com.example.chat",
+ )
+
+ assertFalse(launched)
+ assertNull(AgentVoiceInteractionService.consumePendingSession())
+ }
+
+ @Test
+ fun `launchLiveSession calls showSession immediately when service is active and ready`() {
+ AgentVoiceInteractionService.isActiveServiceChecker = { true }
+ val service =
+ spyk(Robolectric.buildService(AgentVoiceInteractionService::class.java).create().get())
+ val argsSlot = slot()
+ every { service.showSession(capture(argsSlot), any()) } just Runs
+ service.onReady()
+
+ val launched =
+ AgentVoiceInteractionService.launchLiveSession(
+ context = context,
+ threadId = "thread-42",
+ targetPackageName = "com.example.notes",
+ )
+
+ assertTrue(launched)
+ verify(exactly = 1) { service.showSession(any(), 0) }
+ assertEquals(
+ "thread-42",
+ argsSlot.captured.getString(AgentVoiceInteractionService.EXTRA_THREAD_ID),
+ )
+ assertEquals(
+ "com.example.notes",
+ argsSlot.captured.getString(AgentVoiceInteractionService.EXTRA_TARGET_PACKAGE_NAME),
+ )
+ }
+
+ @Test
+ fun `launchLiveSession returns false when active in settings but unbound and showAssist fails`() {
+ AgentVoiceInteractionService.isActiveServiceChecker = { true }
+
+ val launched =
+ AgentVoiceInteractionService.launchLiveSession(
+ context = context,
+ threadId = "thread-1",
+ targetPackageName = null,
+ )
+
+ assertFalse(launched)
+ assertNull(AgentVoiceInteractionService.consumePendingSession())
+ }
+
+ @Test
+ fun `createAssistantSettingsIntent targets ACTION_VOICE_INPUT_SETTINGS`() {
+ val intent = AgentVoiceInteractionService.createAssistantSettingsIntent()
+
+ assertEquals(Settings.ACTION_VOICE_INPUT_SETTINGS, intent.action)
+ }
+
+ @Test
+ fun `onReady does not self-trigger showSession while settings UI is in foreground`() {
+ val service =
+ spyk(Robolectric.buildService(AgentVoiceInteractionService::class.java).create().get())
+ every { service.showSession(any(), any()) } just Runs
+
+ service.onReady()
+
+ verify(exactly = 0) { service.showSession(any(), any()) }
+ }
+}
diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionTest.kt
new file mode 100644
index 00000000..d290dfa1
--- /dev/null
+++ b/agent/app/src/test/java/com/example/appfunctions/agent/voice/AgentVoiceInteractionSessionTest.kt
@@ -0,0 +1,308 @@
+/*
+ * Copyright 2026 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.example.appfunctions.agent.voice
+
+import android.app.Application
+import android.content.Context
+import android.content.Intent
+import android.graphics.Rect
+import android.os.Looper
+import android.service.voice.VoiceInteractionSession
+import androidx.test.core.app.ApplicationProvider
+import com.example.appfunctions.agent.MainActivity
+import com.example.appfunctions.agent.data.LlmModel
+import com.example.appfunctions.agent.data.db.entities.ThreadEntity
+import com.example.appfunctions.agent.domain.AgentStatus
+import com.example.appfunctions.agent.domain.LiveAgentOrchestrator
+import com.example.appfunctions.agent.domain.LiveSessionState
+import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase
+import io.mockk.Runs
+import io.mockk.coEvery
+import io.mockk.coVerify
+import io.mockk.every
+import io.mockk.just
+import io.mockk.mockk
+import io.mockk.spyk
+import io.mockk.verify
+import kotlinx.coroutines.ExperimentalCoroutinesApi
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.flowOf
+import kotlinx.coroutines.test.StandardTestDispatcher
+import kotlinx.coroutines.test.advanceUntilIdle
+import kotlinx.coroutines.test.runTest
+import org.junit.After
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertNotNull
+import org.junit.Assert.assertTrue
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.robolectric.RobolectricTestRunner
+import org.robolectric.Shadows
+import org.robolectric.annotation.Config
+import org.robolectric.annotation.GraphicsMode
+
+@OptIn(ExperimentalCoroutinesApi::class)
+@RunWith(RobolectricTestRunner::class)
+@Config(sdk = [34])
+@GraphicsMode(GraphicsMode.Mode.NATIVE)
+class AgentVoiceInteractionSessionTest {
+ private lateinit var context: Context
+ private lateinit var liveAgentOrchestrator: LiveAgentOrchestrator
+ private lateinit var manageThreadsUseCase: ManageThreadsUseCase
+
+ private val liveStateFlow = MutableStateFlow(LiveSessionState.Idle)
+ private val agentStatusFlow = MutableStateFlow(AgentStatus.Idle)
+
+ @Before
+ fun setUp() {
+ context = ApplicationProvider.getApplicationContext()
+ AgentVoiceInteractionService.resetForTesting()
+
+ liveAgentOrchestrator = mockk(relaxed = true)
+ manageThreadsUseCase = mockk(relaxed = true)
+
+ every { liveAgentOrchestrator.state } returns liveStateFlow
+ every { liveAgentOrchestrator.status } returns agentStatusFlow
+ }
+
+ @After
+ fun tearDown() {
+ AgentVoiceInteractionService.resetForTesting()
+ }
+
+ @Test
+ fun `onShow starts LiveAgentOrchestrator with requested threadId and targetPackageName`() =
+ runTest {
+ val testDispatcher = StandardTestDispatcher(testScheduler)
+ val session =
+ spyk(
+ AgentVoiceInteractionSession(
+ context = context,
+ liveAgentOrchestrator = liveAgentOrchestrator,
+ manageThreadsUseCase = manageThreadsUseCase,
+ mainDispatcher = testDispatcher,
+ ),
+ )
+ every { session.hide() } just Runs
+ every { liveAgentOrchestrator.startSession(any(), any()) } answers {
+ liveStateFlow.value = LiveSessionState.Listening
+ }
+
+ val args =
+ AgentVoiceInteractionService.createSessionArgs(
+ threadId = "thread-123",
+ targetPackageName = "com.example.chat",
+ )
+ session.onShow(args, 0)
+ advanceUntilIdle()
+
+ verify(exactly = 1) {
+ liveAgentOrchestrator.startSession(
+ threadId = "thread-123",
+ targetPackageName = "com.example.chat",
+ )
+ }
+ session.onDestroy()
+ }
+
+ @Test
+ fun `onShow without threadId resolves default thread via ManageThreadsUseCase`() =
+ runTest {
+ val testDispatcher = StandardTestDispatcher(testScheduler)
+ every { manageThreadsUseCase.getThreads() } returns flowOf(emptyList())
+ coEvery { manageThreadsUseCase.createThread(LlmModel.DEFAULT) } returns "thread-created"
+
+ val session =
+ spyk(
+ AgentVoiceInteractionSession(
+ context = context,
+ liveAgentOrchestrator = liveAgentOrchestrator,
+ manageThreadsUseCase = manageThreadsUseCase,
+ mainDispatcher = testDispatcher,
+ ),
+ )
+ every { session.hide() } just Runs
+
+ session.onShow(null, 0)
+ advanceUntilIdle()
+
+ coVerify(exactly = 1) { manageThreadsUseCase.createThread(LlmModel.DEFAULT) }
+ verify(exactly = 1) {
+ liveAgentOrchestrator.startSession(
+ threadId = "thread-created",
+ targetPackageName = null,
+ )
+ }
+ session.onDestroy()
+ }
+
+ @Test
+ fun `onShow without threadId reuses newest existing thread when present`() =
+ runTest {
+ val testDispatcher = StandardTestDispatcher(testScheduler)
+ val existingThread =
+ ThreadEntity(
+ "thread-existing",
+ 1000L,
+ LlmModel.DEFAULT,
+ null,
+ )
+ every { manageThreadsUseCase.getThreads() } returns flowOf(listOf(existingThread))
+
+ val session =
+ spyk(
+ AgentVoiceInteractionSession(
+ context = context,
+ liveAgentOrchestrator = liveAgentOrchestrator,
+ manageThreadsUseCase = manageThreadsUseCase,
+ mainDispatcher = testDispatcher,
+ ),
+ )
+ every { session.hide() } just Runs
+
+ session.onShow(null, 0)
+ advanceUntilIdle()
+
+ coVerify(exactly = 0) { manageThreadsUseCase.createThread(any()) }
+ verify(exactly = 1) {
+ liveAgentOrchestrator.startSession(
+ threadId = "thread-existing",
+ targetPackageName = null,
+ )
+ }
+ session.onDestroy()
+ }
+
+ @Test
+ fun `onComputeInsets restricts touchable region to floating overlay bounds`() =
+ runTest {
+ val testDispatcher = StandardTestDispatcher(testScheduler)
+ val session =
+ AgentVoiceInteractionSession(
+ context = context,
+ liveAgentOrchestrator = liveAgentOrchestrator,
+ manageThreadsUseCase = manageThreadsUseCase,
+ mainDispatcher = testDispatcher,
+ )
+
+ val pillBounds = Rect(32, 1800, 1048, 1980)
+ session.updateOverlayBounds(pillBounds)
+ Shadows.shadowOf(Looper.getMainLooper()).idle()
+
+ val insets = VoiceInteractionSession.Insets()
+ session.onComputeInsets(insets)
+
+ assertEquals(VoiceInteractionSession.Insets.TOUCHABLE_INSETS_REGION, insets.touchableInsets)
+ assertEquals(pillBounds, insets.touchableRegion.bounds)
+ session.onDestroy()
+ }
+
+ @Test
+ fun `onCloseSystemDialogs keeps session visible while Listening and hides when Idle`() =
+ runTest {
+ val testDispatcher = StandardTestDispatcher(testScheduler)
+ val session =
+ spyk(
+ AgentVoiceInteractionSession(
+ context = context,
+ liveAgentOrchestrator = liveAgentOrchestrator,
+ manageThreadsUseCase = manageThreadsUseCase,
+ mainDispatcher = testDispatcher,
+ ),
+ )
+ every { session.hide() } just Runs
+
+ liveStateFlow.value = LiveSessionState.Listening
+ session.onCloseSystemDialogs()
+ verify(exactly = 0) { session.hide() }
+
+ liveStateFlow.value = LiveSessionState.Idle
+ session.onCloseSystemDialogs()
+ verify(exactly = 1) { session.hide() }
+
+ session.onDestroy()
+ }
+
+ @Test
+ fun `stopAndHide and onHide stop LiveAgentOrchestrator`() =
+ runTest {
+ val testDispatcher = StandardTestDispatcher(testScheduler)
+ val session =
+ spyk(
+ AgentVoiceInteractionSession(
+ context = context,
+ liveAgentOrchestrator = liveAgentOrchestrator,
+ manageThreadsUseCase = manageThreadsUseCase,
+ mainDispatcher = testDispatcher,
+ ),
+ )
+ every { session.hide() } just Runs
+
+ session.stopAndHide()
+ verify(exactly = 1) { liveAgentOrchestrator.stopSession() }
+ verify(exactly = 1) { session.hide() }
+
+ session.onHide()
+ verify(exactly = 2) { liveAgentOrchestrator.stopSession() }
+
+ session.onDestroy()
+ }
+
+ @Test
+ fun `openChat launches MainActivity with active threadId while keeping Live session running`() =
+ runTest {
+ val testDispatcher = StandardTestDispatcher(testScheduler)
+ val session =
+ spyk(
+ AgentVoiceInteractionSession(
+ context = context,
+ liveAgentOrchestrator = liveAgentOrchestrator,
+ manageThreadsUseCase = manageThreadsUseCase,
+ mainDispatcher = testDispatcher,
+ ),
+ )
+ every { session.hide() } just Runs
+ every { liveAgentOrchestrator.startSession(any(), any()) } answers {
+ liveStateFlow.value = LiveSessionState.Listening
+ }
+
+ val args =
+ AgentVoiceInteractionService.createSessionArgs(
+ threadId = "thread-live-42",
+ targetPackageName = null,
+ )
+ session.onShow(args, 0)
+ advanceUntilIdle()
+
+ session.openChat()
+
+ val startedIntent = Shadows.shadowOf(context as Application).nextStartedActivity
+ assertNotNull(startedIntent)
+ assertEquals(MainActivity::class.java.name, startedIntent.component?.className)
+ assertEquals(MainActivity.ACTION_OPEN_CHAT, startedIntent.action)
+ assertEquals(
+ "thread-live-42",
+ startedIntent.getStringExtra(MainActivity.EXTRA_OPEN_THREAD_ID),
+ )
+ assertTrue(startedIntent.flags and Intent.FLAG_ACTIVITY_NEW_TASK != 0)
+ assertTrue(startedIntent.flags and Intent.FLAG_ACTIVITY_SINGLE_TOP != 0)
+ verify(exactly = 0) { liveAgentOrchestrator.stopSession() }
+ verify(exactly = 0) { session.hide() }
+
+ session.onDestroy()
+ }
+}