From bbfbf2156a6da81a899e4282686ac48a63bfe653 Mon Sep 17 00:00:00 2001 From: Julian Toledo Date: Tue, 27 Oct 2026 20:55:20 +0100 Subject: [PATCH 1/2] refactor(agent): unify AppFunction discovery, input conversion, and execution Extract shared tool discovery, input conversion, and execution logic out of AgentOrchestrator into the core AppFunction use cases so chat, debugging, and future live voice orchestration share a single pipeline: - Evolve GetAppFunctionsUseCase with GetAppFunctionsParams (targetPackageNames, isEnabled, isConnected, includeInternalTools) and expose shared internal package metadata in AgentInternalTools. - Evolve ConvertInputToAppFunctionDataUseCase to resolve remote HTTP/HTTPS file reference URLs into local FileProvider content:// URIs during conversion to AppFunctionData. - Evolve ExecuteAppFunctionUseCase to dispatch internal agent tools directly from AppFunctionData and recursively grant content:// read URI permissions to target packages before executing external AppFunctions. - Update AgentOrchestrator and existing external-app-only callers (GetConnectedAppsUseCase, TroubleshootAppUseCase, AgentDemoViewModel, DebuggingViewModel) and expand unit/instrumented test coverage. --- .../domain/GetAppFunctionsUseCaseTest.kt | 13 +- .../agent/data/AgentInternalTools.kt | 24 +- .../agent/domain/AgentOrchestrator.kt | 438 +++----------- .../ConvertInputToAppFunctionDataUseCase.kt | 293 +++++++++- .../appfunction/ExecuteAppFunctionUseCase.kt | 346 ++++++++++- .../appfunction/GetAppFunctionsUseCase.kt | 136 ++++- .../appfunction/GetConnectedAppsUseCase.kt | 4 +- .../troubleshoot/TroubleshootAppUseCase.kt | 9 +- .../screens/agentdemo/AgentDemoViewModel.kt | 5 +- .../screens/debugging/DebuggingViewModel.kt | 5 +- .../agent/domain/AgentOrchestratorTest.kt | 155 ++++- ...onvertInputToAppFunctionDataUseCaseTest.kt | 545 +++++++++++------- .../ExecuteAppFunctionUseCaseTest.kt | 275 ++++++++- .../appfunction/GetAppFunctionsUseCaseTest.kt | 240 ++++++++ .../GetConnectedAppsUseCaseTest.kt | 8 +- .../agentdemo/AgentDemoViewModelTest.kt | 111 +--- .../debugging/DebuggingViewModelTest.kt | 33 +- 17 files changed, 1899 insertions(+), 741 deletions(-) create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCaseTest.kt diff --git a/agent/app/src/androidTest/java/com/example/appfunctions/agent/domain/GetAppFunctionsUseCaseTest.kt b/agent/app/src/androidTest/java/com/example/appfunctions/agent/domain/GetAppFunctionsUseCaseTest.kt index b83c075d..eae86a91 100644 --- a/agent/app/src/androidTest/java/com/example/appfunctions/agent/domain/GetAppFunctionsUseCaseTest.kt +++ b/agent/app/src/androidTest/java/com/example/appfunctions/agent/domain/GetAppFunctionsUseCaseTest.kt @@ -20,6 +20,10 @@ import androidx.appfunctions.AppFunctionManager import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.platform.app.InstrumentationRegistry +import com.example.appfunctions.agent.data.AgentInternalTools +import com.example.appfunctions.agent.data.DataStoreSettingsRepository +import com.example.appfunctions.agent.di.settingsDataStore +import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionStatesUseCase import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase import kotlinx.coroutines.flow.first import kotlinx.coroutines.runBlocking @@ -38,7 +42,14 @@ class GetAppFunctionsUseCaseTest { fun setup() { context = ApplicationProvider.getApplicationContext() appFunctionManager = AppFunctionManager.getInstance(context)!! - useCase = GetAppFunctionsUseCase(appFunctionManager) + val dataStore = context.settingsDataStore + useCase = + GetAppFunctionsUseCase( + appFunctionManager = appFunctionManager, + getAppFunctionStatesUseCase = GetAppFunctionStatesUseCase(appFunctionManager), + settingsRepository = DataStoreSettingsRepository(dataStore), + agentInternalTools = AgentInternalTools(context, dataStore), + ) // Adopt shell permission identity InstrumentationRegistry.getInstrumentation() diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/AgentInternalTools.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/AgentInternalTools.kt index 7ef75704..a58e9975 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/AgentInternalTools.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/AgentInternalTools.kt @@ -375,6 +375,12 @@ class AgentInternalTools qualifiedName = "com.example.appfunctions.agent.data.AgentInternalTools.GeneratedImageResult", ) + val internalPackageMetadata = + AppFunctionPackageMetadata( + packageName = INTERNAL_TOOL_PACKAGE, + components = AppFunctionComponentsMetadata(), + ) + val getCurrentLocationTool = AppFunctionMetadata( name = AppFunctionName(INTERNAL_TOOL_PACKAGE, "getCurrentLocation"), @@ -387,11 +393,7 @@ class AgentInternalTools ), description = "Retrieve the current latitude and longitude coordinates of the device.", deprecation = null, - packageMetadata = - AppFunctionPackageMetadata( - packageName = INTERNAL_TOOL_PACKAGE, - components = AppFunctionComponentsMetadata(), - ), + packageMetadata = internalPackageMetadata, ) val geocodeAddressTool = @@ -414,11 +416,7 @@ class AgentInternalTools ), description = "Geocode a physical address string into its latitude and longitude coordinates.", deprecation = null, - packageMetadata = - AppFunctionPackageMetadata( - packageName = INTERNAL_TOOL_PACKAGE, - components = AppFunctionComponentsMetadata(), - ), + packageMetadata = internalPackageMetadata, ) val generateImageTool = @@ -447,11 +445,7 @@ class AgentInternalTools ), description = "Generates an image from a text prompt and returns the remote image URI.", deprecation = null, - packageMetadata = - AppFunctionPackageMetadata( - packageName = INTERNAL_TOOL_PACKAGE, - components = AppFunctionComponentsMetadata(), - ), + packageMetadata = internalPackageMetadata, ) @SuppressLint("RestrictedApi") diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index d2af9ee1..9ef924ae 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -16,18 +16,8 @@ package com.example.appfunctions.agent.domain import android.app.PendingIntent -import android.content.Context -import android.content.Intent -import android.net.Uri import android.util.Log -import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata -import androidx.appfunctions.metadata.AppFunctionDataTypeMetadata import androidx.appfunctions.metadata.AppFunctionMetadata -import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata -import androidx.appfunctions.metadata.AppFunctionParameterMetadata -import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata -import androidx.core.content.FileProvider -import com.example.appfunctions.agent.data.AgentInternalTools import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.SettingsRepository import com.example.appfunctions.agent.data.db.entities.MessageAttachment @@ -39,7 +29,7 @@ import com.example.appfunctions.agent.domain.appfunction.AppFunctionExceptionFor import com.example.appfunctions.agent.domain.appfunction.ConvertInputToAppFunctionDataUseCase import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionUseCase -import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionStatesUseCase +import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsParams import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase import com.example.appfunctions.agent.domain.chat.ObservePendingMessagesUseCase @@ -49,9 +39,7 @@ import com.example.appfunctions.agent.domain.chat.UpdateMessageUseCase import com.example.appfunctions.agent.domain.chat.UpdateThreadParams import com.example.appfunctions.agent.domain.chat.UpdateThreadUseCase import com.example.appfunctions.agent.domain.pendingintent.SavePendingIntentUseCase -import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharingStarted @@ -60,11 +48,10 @@ import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.stateIn -import kotlinx.coroutines.withContext -import org.json.JSONObject -import java.io.File -import java.net.HttpURLConnection -import java.net.URL +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonPrimitive import java.util.UUID import javax.inject.Inject import javax.inject.Singleton @@ -74,7 +61,6 @@ import javax.inject.Singleton class AgentOrchestrator @Inject constructor( - @ApplicationContext private val context: Context, private val manageThreadsUseCase: ManageThreadsUseCase, private val observePendingMessagesUseCase: ObservePendingMessagesUseCase, private val sendMessageUseCase: SendMessageUseCase, @@ -83,11 +69,9 @@ class AgentOrchestrator private val llmProviderFactory: LlmProviderFactory, private val settingsRepository: SettingsRepository, private val getAppFunctionsUseCase: GetAppFunctionsUseCase, - private val getAppFunctionStatesUseCase: GetAppFunctionStatesUseCase, private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase, private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase, private val savePendingIntentUseCase: SavePendingIntentUseCase, - private val agentInternalTools: AgentInternalTools, ) { private val _status = MutableStateFlow(AgentStatus.Idle) @@ -132,27 +116,30 @@ class AgentOrchestrator return } - val disconnectedApps = settingsRepository.disconnectedApps.first() - val allTools = - getAppFunctionsUseCase().first().values.flatten() + - agentInternalTools.getInternalToolsMetadata().map { it.first } - - val targetPackageName = message.targetPackageName - val queryText = message.textContent - - val tools = filterTools(allTools, disconnectedApps, targetPackageName) + val tools = + getAppFunctionsUseCase( + GetAppFunctionsParams( + targetPackageNames = message.targetPackageName?.let { setOf(it) }, + isEnabled = true, + isConnected = true, + includeInternalTools = true, + ), + ).first().values.flatten() runInteractionLoop( message = message, thread = thread, apiKey = apiKey, tools = tools, - initialInput = queryText, + initialInput = message.textContent, ) _status.value = AgentStatus.Idle + } catch (e: CancellationException) { + _status.value = AgentStatus.Idle + throw e } catch (e: Exception) { - Log.e("AgentOrchestrator", "Error processing message", e) + Log.e(TAG, "Error processing message", e) completeMessageWithError( message.messageId, message.threadId, @@ -162,31 +149,6 @@ class AgentOrchestrator } } - private suspend fun filterTools( - allTools: List, - disconnectedApps: Set, - targetPackageName: String?, - ): List { - val functionNames = allTools.map { it.name } - val states = getAppFunctionStatesUseCase(functionNames) + agentInternalTools.getInternalToolsMetadata().map { it.second } - val stateMap = states.associateBy { it.functionName } - - return allTools - .filter { metadata -> - val isEnabled = - if (metadata.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) { - true - } else { - stateMap[metadata.name]?.isEnabled ?: false - } - isEnabled && - metadata.packageName !in disconnectedApps && - (targetPackageName == null || metadata.packageName == targetPackageName) - } - .sortedByDescending { metadata -> metadata.id.startsWith(metadata.packageName) } - .distinctBy { metadata -> metadata.id } - } - private suspend fun runInteractionLoop( message: MessageEntity, thread: ThreadEntity, @@ -202,7 +164,7 @@ class AgentOrchestrator var previousInteractionId = thread.latestInteractionId var currentToolOutputs = emptyList() var continueLoop = true - var currentInput = initialInput + val currentInput = initialInput val capturedAttachments = mutableListOf() while (continueLoop) { @@ -293,23 +255,11 @@ class AgentOrchestrator val textContent = textParts.joinToString("\n") { it.text } if (toolCalls.isNotEmpty()) { - when (val toolResult = executeToolCalls(toolCalls, tools, message)) { + when ( + val toolResult = + executeToolCalls(toolCalls, tools, message, capturedAttachments) + ) { is ExecuteToolCallsResult.Success -> { - for (output in toolResult.toolOutputs) { - runCatching { - val json = JSONObject(output.result) - val uri = json.optString("imageUri") - if (uri.isNotBlank()) { - val mimeType = - json.optString("mimeType") - .takeIf { it.isNotBlank() } - ?: "image/png" - capturedAttachments.add( - MessageAttachment(uri = uri, mimeType = mimeType), - ) - } - } - } if (textContent.isNotEmpty()) { sendMessageUseCase( threadId = message.threadId, @@ -358,7 +308,7 @@ class AgentOrchestrator } is LlmResponse.Error -> { - Log.e("AgentOrchestrator", "LLM Error: ${response.errorMessage}") + Log.e(TAG, "LLM Error: ${response.errorMessage}") completeMessageWithError( message.messageId, message.threadId, @@ -374,39 +324,16 @@ class AgentOrchestrator toolCalls: List, tools: List, message: MessageEntity, + capturedAttachments: MutableList, ): ExecuteToolCallsResult { val results = mutableListOf() for (toolCall in toolCalls) { _status.value = AgentStatus.InvokingTool(toolCall.functionId, toolCall.packageName) - if (toolCall.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) { - val executionResult = executeInternalTool(toolCall) - executionResult - .onSuccess { jsonString -> - results.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = jsonString, - ), - ) - } - .onFailure { exception -> - completeMessageWithError( - message.messageId, - message.threadId, - "Internal tool execution failed: ${exception.message}", - ) - return ExecuteToolCallsResult.Error - } - continue - } - val matchingTool = tools.find { it.packageName == toolCall.packageName && it.id == toolCall.functionId } - if (matchingTool == null) { completeMessageWithError( message.messageId, @@ -416,65 +343,39 @@ class AgentOrchestrator return ExecuteToolCallsResult.Error } - val rawConvertedInputs = toolCall.arguments.filterValues { it != null } as Map - val convertedInputs = - try { - withContext(Dispatchers.IO) { - resolveRemoteFileReferencesRecursively( - context = context, - parametersMetadata = matchingTool.parameters, - inputs = rawConvertedInputs, - ) - } - } catch (e: Exception) { - completeMessageWithError( - message.messageId, - message.threadId, - "Failed to download remote file reference: ${e.message}", - ) - return ExecuteToolCallsResult.Error - } - - grantContentUriPermissionsRecursively(context, toolCall.packageName, convertedInputs) - - for (value in convertedInputs.values) { - if (value is String && value.startsWith("content://")) { - runCatching { - val uri = Uri.parse(value) - context.grantUriPermission( - toolCall.packageName, - uri, - Intent.FLAG_GRANT_READ_URI_PERMISSION, - ) - } - } - } - + val inputs = nonNullArguments(toolCall.arguments) val appFunctionDataResult = - withContext(Dispatchers.Default) { - convertInputToAppFunctionDataUseCase( - parameters = matchingTool.parameters, - components = matchingTool.components, - inputs = convertedInputs, - ) - } + convertInputToAppFunctionDataUseCase( + parameters = matchingTool.parameters, + components = matchingTool.components, + inputs = inputs, + ) if (appFunctionDataResult.isFailure) { + val exception = appFunctionDataResult.exceptionOrNull() + if (exception is CancellationException) { + throw exception + } + val errorMessage = + "Failed to convert arguments for ${toolCall.functionId}\n $exception" + if (exception != null) { + Log.e(TAG, errorMessage, exception) + } completeMessageWithError( message.messageId, message.threadId, - "Failed to convert arguments for ${toolCall.functionId}\n ${appFunctionDataResult.exceptionOrNull()}", + errorMessage, ) return ExecuteToolCallsResult.Error } - val executionResult = - executeAppFunctionUseCase( - function = matchingTool, - parameters = appFunctionDataResult.getOrThrow(), - threadId = message.threadId, - ) - - when (executionResult) { + when ( + val executionResult = + executeAppFunctionUseCase( + function = matchingTool, + parameters = appFunctionDataResult.getOrThrow(), + threadId = message.threadId, + ) + ) { is ExecuteAppFunctionResult.Data -> { results.add( ToolOutput( @@ -483,13 +384,15 @@ class AgentOrchestrator result = executionResult.formattedJson, ), ) + extractMessageAttachment(executionResult.formattedJson)?.let { + capturedAttachments.add(it) + } } is ExecuteAppFunctionResult.PendingIntentAction -> { - val pendingIntentId = java.util.UUID.randomUUID().toString() return ExecuteToolCallsResult.PendingIntentAction( - pendingIntentId, - executionResult.pendingIntent, + pendingIntentId = UUID.randomUUID().toString(), + pendingIntent = executionResult.pendingIntent, ) } @@ -513,10 +416,15 @@ class AgentOrchestrator ), ) } else { - throw IllegalStateException( - "Tool execution failed for ${toolCall.functionId}: ${exception.message}", - exception, + val errorMessage = + "Tool execution failed for ${toolCall.functionId}: ${exception.message}" + Log.e(TAG, errorMessage, exception) + completeMessageWithError( + message.messageId, + message.threadId, + errorMessage, ) + return ExecuteToolCallsResult.Error } } } @@ -541,207 +449,31 @@ class AgentOrchestrator ) } - private fun resolveRemoteFileReferencesRecursively( - context: Context, - parametersMetadata: List, - inputs: Map, - ): Map { - val paramMap = parametersMetadata.associateBy { it.name } - return inputs.mapValues { (key, value) -> - val paramMeta = paramMap[key] - resolveValueRecursively(context, paramMeta?.dataType, value, key) - } - } - - private fun resolveValueRecursively( - context: Context, - dataType: AppFunctionDataTypeMetadata?, - value: Any, - paramName: String?, - ): Any { - return when (value) { - is String -> { - val shouldResolve = - isFileReferenceParameter(paramName) || isUriMetadata(dataType) - if (shouldResolve && ( - value.startsWith( - "http://", - ) || value.startsWith("https://") - ) - ) { - downloadRemoteFileToContentUri(context, value) - } else { - value - } - } - is Map<*, *> -> { - value.entries.associate { entry -> - val k = entry.key as String - val propType = - (dataType as? AppFunctionObjectTypeMetadata)?.properties?.get(k) - k to ( - entry.value?.let { - resolveValueRecursively( - context, - propType, - it, - k, - ) - } ?: "" - ) - } - } - is List<*> -> { - val itemType = (dataType as? AppFunctionArrayTypeMetadata)?.itemType - value.mapNotNull { item -> - if (item != null) resolveValueRecursively(context, itemType, item, paramName) else null - } - } - else -> value - } - } - - private fun downloadRemoteFileToContentUri( - context: Context, - urlString: String, - ): String { - val url = URL(urlString) - val connection = - (url.openConnection() as HttpURLConnection).apply { - connectTimeout = 10000 - readTimeout = 15000 - } - try { - connection.connect() - val contentType = connection.contentType ?: "" - val ext = - when { - contentType.contains("png", ignoreCase = true) -> "png" - contentType.contains("jpeg", ignoreCase = true) || - contentType.contains("jpg", ignoreCase = true) -> "jpg" - contentType.contains("gif", ignoreCase = true) -> "gif" - contentType.contains("webp", ignoreCase = true) -> "webp" - urlString.substringAfterLast("/", "").contains(".") -> - urlString.substringAfterLast("/").substringAfterLast(".") - else -> "jpg" - } - val cacheDir = File(context.cacheDir, "file_references").apply { mkdirs() } - val file = File(cacheDir, "generated_${UUID.randomUUID()}.$ext") - connection.inputStream.use { input -> - file.outputStream().use { output -> - input.copyTo(output) - } - } - val contentUri = - FileProvider.getUriForFile( - context, - "${context.packageName}.fileprovider", - file, - ) - return contentUri.toString() - } finally { - connection.disconnect() - } - } - - private fun grantContentUriPermissionsRecursively( - context: Context, - targetPackageName: String, - value: Any?, - ) { - when (value) { - is String -> { - if (value.startsWith("content://")) { - runCatching { - val uri = Uri.parse(value) - context.grantUriPermission( - targetPackageName, - uri, - Intent.FLAG_GRANT_READ_URI_PERMISSION, - ) - } - } - } - is Map<*, *> -> - value.values.forEach { - grantContentUriPermissionsRecursively(context, targetPackageName, it) - } - is List<*> -> - value.forEach { - grantContentUriPermissionsRecursively(context, targetPackageName, it) - } - } - } - - private fun isFileReferenceParameter(parameterName: String?): Boolean { - if (parameterName == null) return false - if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true - return parameterName.endsWith("Uri", ignoreCase = true) || - parameterName.endsWith("Uris", ignoreCase = true) - } - - private fun isUriMetadata(dataType: AppFunctionDataTypeMetadata?): Boolean { - if (dataType == null) return false - if (dataType is AppFunctionObjectTypeMetadata && dataType.qualifiedName == "android.net.Uri") return true - if (dataType is AppFunctionReferenceTypeMetadata && dataType.referenceDataType == "android.net.Uri") return true - return false + private companion object { + const val TAG = "AgentOrchestrator" } + } - private suspend fun executeInternalTool(toolCall: LlmResponsePart.ToolCall): Result { - return runCatching { - when (toolCall.functionId) { - "getCurrentLocation" -> { - val location = agentInternalTools.getCurrentLocation() - if (location != null) { - JSONObject().apply { - put("latitude", location.latitude) - put("longitude", location.longitude) - }.toString() - } else { - "null" - } - } - "geocodeAddress" -> { - val address = - toolCall.arguments["address"] as? String - ?: throw IllegalArgumentException("Missing required parameter 'address'") - val location = agentInternalTools.geocodeAddress(address) - if (location != null) { - JSONObject().apply { - put("latitude", location.latitude) - put("longitude", location.longitude) - }.toString() - } else { - "null" - } - } - "generateImage" -> { - val prompt = - toolCall.arguments["prompt"] as? String - ?: throw IllegalArgumentException("Missing required parameter 'prompt'") - val aspectRatio = toolCall.arguments["aspectRatio"] as? String - val result = agentInternalTools.generateImage(prompt, aspectRatio) - JSONObject().apply { - put("imageUri", result.imageUri) - put("mimeType", result.mimeType) - put("prompt", result.prompt) - }.toString() - } - else -> throw IllegalArgumentException("Unknown internal tool: ${toolCall.functionId}") - } +internal fun nonNullArguments(arguments: Map): Map = + buildMap { + for ((key, value) in arguments) { + if (value != null) { + put(key, value) } } + } - companion object { - private val KNOWN_FILE_REFERENCE_PARAM_NAMES = - setOf( - "wallpaperUri", - "imageUri", - "attachmentUri", - "ringtoneUri", - "profilePictureUri", - "audioUri", - "voiceNoteUri", - ) +internal fun extractMessageAttachment(jsonString: String): MessageAttachment? { + return runCatching { + val json = Json.parseToJsonElement(jsonString) as? JsonObject ?: return null + val uri = json["imageUri"]?.jsonPrimitive?.contentOrNull + if (!uri.isNullOrBlank()) { + val mimeType = + json["mimeType"]?.jsonPrimitive?.contentOrNull?.takeIf { it.isNotBlank() } + ?: "image/png" + MessageAttachment(uri = uri, mimeType = mimeType) + } else { + null } - } + }.getOrNull() +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt index 379c846a..c0c05be2 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt @@ -15,6 +15,7 @@ */ package com.example.appfunctions.agent.domain.appfunction +import android.content.Context import androidx.appfunctions.AppFunctionData import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata import androidx.appfunctions.metadata.AppFunctionBooleanTypeMetadata @@ -26,35 +27,207 @@ import androidx.appfunctions.metadata.AppFunctionIntTypeMetadata import androidx.appfunctions.metadata.AppFunctionLongTypeMetadata import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionParcelableTypeMetadata import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import androidx.core.content.FileProvider +import dagger.hilt.android.qualifiers.ApplicationContext +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.File +import java.io.IOException +import java.net.HttpURLConnection +import java.net.URL +import java.util.UUID import javax.inject.Inject -/** Use case to convert a map of raw inputs to type-safe [AppFunctionData]. */ +/** + * Use case to convert a map of raw inputs to type-safe [AppFunctionData], resolving remote + * `http://` and `https://` file reference URLs into local `content://` URIs when needed. + */ class ConvertInputToAppFunctionDataUseCase @Inject - constructor() { + constructor( + @ApplicationContext private val context: Context, + ) { + internal var remoteFileDownloader: suspend (Context, String) -> String = + ::downloadRemoteFileToContentUri + /** * Executes the use case. * * @param parameters The list of parameter metadata for the function. * @param components The component metadata for resolving references. - * @param inputs The map of raw inputs from the UI. + * @param inputs The map of raw inputs from the UI or LLM tool call. * @return The converted [AppFunctionData]. */ - operator fun invoke( + suspend operator fun invoke( parameters: List, components: AppFunctionComponentsMetadata, inputs: Map, ): Result = - runCatching { - val builder = AppFunctionData.Builder(parameters, components) - for (parameter in parameters) { - val value = inputs[parameter.name] ?: continue - setParameterValue(builder, parameter.name, parameter.dataType, value, components) + try { + val resolvedInputs = + resolveRemoteFileReferencesRecursively( + parametersMetadata = parameters, + components = components, + inputs = inputs, + ) + withContext(Dispatchers.Default) { + val builder = AppFunctionData.Builder(parameters, components) + for (parameter in parameters) { + val value = resolvedInputs[parameter.name] ?: continue + setParameterValue( + builder, + parameter.name, + parameter.dataType, + value, + components, + ) + } + Result.success(builder.build()) + } + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + Result.failure(e) + } + + private suspend fun resolveRemoteFileReferencesRecursively( + parametersMetadata: List, + components: AppFunctionComponentsMetadata, + inputs: Map, + ): Map { + val paramMap = parametersMetadata.associateBy { it.name } + return buildMap(inputs.size) { + for ((key, value) in inputs) { + val paramMeta = paramMap[key] + put( + key, + resolveValueRecursively( + dataType = paramMeta?.dataType, + components = components, + value = value, + paramName = key, + ), + ) + } + } + } + + private suspend fun resolveValueRecursively( + dataType: AppFunctionDataTypeMetadata?, + components: AppFunctionComponentsMetadata, + value: Any, + paramName: String?, + ): Any { + val resolvedType = + if (dataType is AppFunctionReferenceTypeMetadata) { + components.dataTypes[dataType.referenceDataType] ?: dataType + } else { + dataType } - builder.build() + + return when (value) { + is String -> { + val shouldResolve = + isFileReferenceParameter(paramName) || + isUriMetadata(dataType, components) + if ( + shouldResolve && + (value.startsWith("http://") || value.startsWith("https://")) + ) { + try { + remoteFileDownloader(context, value) + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + throw IOException( + "Failed to download remote file reference: ${e.message}", + e, + ) + } + } else { + value + } + } + is Map<*, *> -> { + val objectProperties = + (resolvedType as? AppFunctionObjectTypeMetadata)?.properties + buildMap(value.size) { + for (entry in value.entries) { + val k = entry.key as String + val propType = objectProperties?.get(k) + val resolvedProp = + entry.value?.let { + resolveValueRecursively( + dataType = propType, + components = components, + value = it, + paramName = k, + ) + } ?: "" + put(k, resolvedProp) + } + } + } + is List<*> -> { + val itemType = (resolvedType as? AppFunctionArrayTypeMetadata)?.itemType + buildList { + for (item in value) { + if (item != null) { + add( + resolveValueRecursively( + dataType = itemType, + components = components, + value = item, + paramName = paramName, + ), + ) + } + } + } + } + else -> value } + } + + private fun isFileReferenceParameter(parameterName: String?): Boolean { + if (parameterName == null) return false + if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true + return parameterName.endsWith("Uri", ignoreCase = true) || + parameterName.endsWith("Uris", ignoreCase = true) + } + + private fun isUriMetadata( + dataType: AppFunctionDataTypeMetadata?, + components: AppFunctionComponentsMetadata, + ): Boolean { + if (dataType == null) return false + if ( + dataType is AppFunctionObjectTypeMetadata && + dataType.qualifiedName == ANDROID_NET_URI + ) { + return true + } + if ( + dataType is AppFunctionParcelableTypeMetadata && + dataType.qualifiedName == ANDROID_NET_URI + ) { + return true + } + if (dataType is AppFunctionReferenceTypeMetadata) { + if (dataType.referenceDataType == ANDROID_NET_URI) { + return true + } + val resolved = components.dataTypes[dataType.referenceDataType] + if (resolved != null && resolved !== dataType) { + return isUriMetadata(resolved, components) + } + } + return false + } private fun setParameterValue( builder: AppFunctionData.Builder, @@ -82,19 +255,28 @@ class ConvertInputToAppFunctionDataUseCase is AppFunctionBooleanTypeMetadata -> { when (value) { is Boolean -> builder.setBoolean(name, value) - is String -> if (value.isNotEmpty()) builder.setBoolean(name, value.toBoolean()) + is String -> + if (value.isNotEmpty()) { + builder.setBoolean(name, value.toBoolean()) + } } } is AppFunctionDoubleTypeMetadata -> { when (value) { is Number -> builder.setDouble(name, value.toDouble()) - is String -> if (value.isNotEmpty()) builder.setDouble(name, value.toDouble()) + is String -> + if (value.isNotEmpty()) { + builder.setDouble(name, value.toDouble()) + } } } is AppFunctionFloatTypeMetadata -> { when (value) { is Number -> builder.setFloat(name, value.toFloat()) - is String -> if (value.isNotEmpty()) builder.setFloat(name, value.toFloat()) + is String -> + if (value.isNotEmpty()) { + builder.setFloat(name, value.toFloat()) + } } } is AppFunctionObjectTypeMetadata -> { @@ -121,7 +303,7 @@ class ConvertInputToAppFunctionDataUseCase value: Any, components: AppFunctionComponentsMetadata, ): AppFunctionData { - if (objectType.qualifiedName == "android.net.Uri" && value is String) { + if (objectType.qualifiedName == ANDROID_NET_URI && value is String) { val uriPropertyName = objectType.properties.keys.firstOrNull() ?: "uri" return AppFunctionData.Builder(objectType, components) .setString(uriPropertyName, value) @@ -155,11 +337,17 @@ class ConvertInputToAppFunctionDataUseCase builder.setStringList(name, values.map { it as String }) } is AppFunctionIntTypeMetadata -> { - val intArray = values.map { (it as String).toInt() }.toIntArray() + val intArray = + values + .map { if (it is Number) it.toInt() else (it as String).toInt() } + .toIntArray() builder.setIntArray(name, intArray) } is AppFunctionLongTypeMetadata -> { - val longArray = values.map { (it as String).toLong() }.toLongArray() + val longArray = + values + .map { if (it is Number) it.toLong() else (it as String).toLong() } + .toLongArray() builder.setLongArray(name, longArray) } is AppFunctionBooleanTypeMetadata -> { @@ -167,11 +355,21 @@ class ConvertInputToAppFunctionDataUseCase builder.setBooleanArray(name, booleanArray) } is AppFunctionDoubleTypeMetadata -> { - val doubleArray = values.map { (it as String).toDouble() }.toDoubleArray() + val doubleArray = + values + .map { + if (it is Number) it.toDouble() else (it as String).toDouble() + } + .toDoubleArray() builder.setDoubleArray(name, doubleArray) } is AppFunctionFloatTypeMetadata -> { - val floatArray = values.map { (it as String).toFloat() }.toFloatArray() + val floatArray = + values + .map { + if (it is Number) it.toFloat() else (it as String).toFloat() + } + .toFloatArray() builder.setFloatArray(name, floatArray) } is AppFunctionObjectTypeMetadata -> { @@ -195,4 +393,63 @@ class ConvertInputToAppFunctionDataUseCase } } } + + companion object { + private const val ANDROID_NET_URI = "android.net.Uri" + + private val KNOWN_FILE_REFERENCE_PARAM_NAMES = + setOf( + "wallpaperUri", + "imageUri", + "attachmentUri", + "ringtoneUri", + "profilePictureUri", + "audioUri", + "voiceNoteUri", + ) + + private suspend fun downloadRemoteFileToContentUri( + context: Context, + urlString: String, + ): String = + withContext(Dispatchers.IO) { + val url = URL(urlString) + val connection = + (url.openConnection() as HttpURLConnection).apply { + connectTimeout = 10000 + readTimeout = 15000 + } + try { + connection.connect() + val contentType = connection.contentType ?: "" + val ext = + when { + contentType.contains("png", ignoreCase = true) -> "png" + contentType.contains("jpeg", ignoreCase = true) || + contentType.contains("jpg", ignoreCase = true) -> "jpg" + contentType.contains("gif", ignoreCase = true) -> "gif" + contentType.contains("webp", ignoreCase = true) -> "webp" + urlString.substringAfterLast("/", "").contains(".") -> + urlString.substringAfterLast("/").substringAfterLast(".") + else -> "jpg" + } + val cacheDir = File(context.cacheDir, "file_references").apply { mkdirs() } + val file = File(cacheDir, "generated_${UUID.randomUUID()}.$ext") + connection.inputStream.use { input -> + file.outputStream().use { output -> + input.copyTo(output) + } + } + val contentUri = + FileProvider.getUriForFile( + context, + "${context.packageName}.fileprovider", + file, + ) + contentUri.toString() + } finally { + connection.disconnect() + } + } + } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCase.kt index 8bd1ef4b..1360b25f 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCase.kt @@ -17,42 +17,69 @@ package com.example.appfunctions.agent.domain.appfunction import android.app.AppInteractionAttribution import android.app.PendingIntent +import android.content.Context +import android.content.Intent +import android.net.Uri import android.os.Build import androidx.appfunctions.AppFunctionData import androidx.appfunctions.AppFunctionManager import androidx.appfunctions.ExecuteAppFunctionRequest import androidx.appfunctions.ExecuteAppFunctionResponse import androidx.appfunctions.ExecuteAppFunctionResponse.Success.Companion.PROPERTY_RETURN_VALUE +import androidx.appfunctions.metadata.AppFunctionAllOfTypeMetadata +import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionDataTypeMetadata import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata +import androidx.appfunctions.metadata.AppFunctionOneOfTypeMetadata import androidx.appfunctions.metadata.AppFunctionParcelableTypeMetadata +import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata import androidx.core.net.toUri +import com.example.appfunctions.agent.data.AgentInternalTools +import dagger.hilt.android.qualifiers.ApplicationContext +import kotlinx.coroutines.CancellationException +import org.json.JSONObject import javax.inject.Inject -/** Use case to execute an AppFunction. */ +/** + * Use case to execute an AppFunction (or internal agent tool), performing common pre-execution + * steps such as recursive content URI read permission grants. + */ class ExecuteAppFunctionUseCase @Inject constructor( + @ApplicationContext private val context: Context, private val appFunctionManager: AppFunctionManager?, private val convertAppFunctionDataToJsonUseCase: ConvertAppFunctionDataToJsonUseCase, + private val agentInternalTools: AgentInternalTools, ) { /** * Executes the use case. * * @param function The metadata of the function to execute. * @param parameters The input parameters for the function. - * @return A [ExecuteAppFunctionResult]. + * @param threadId Optional chat thread ID used for interaction attribution on API 37+. + * @return An [ExecuteAppFunctionResult]. */ suspend operator fun invoke( function: AppFunctionMetadata, parameters: AppFunctionData, threadId: String? = null, ): ExecuteAppFunctionResult { + if (function.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) { + return executeInternalTool(function, parameters) + } + if (appFunctionManager == null) { return ExecuteAppFunctionResult.Error( IllegalStateException("AppFunctionManager not available on this device"), ) } + grantContentUriPermissions(function, parameters) + val request = if (Build.VERSION.SDK_INT >= 37) { val uri = @@ -89,7 +116,10 @@ class ExecuteAppFunctionUseCase valueType.qualifiedName == PendingIntent::class.java.name ) { val pendingIntent = - data.getParcelable(PROPERTY_RETURN_VALUE, PendingIntent::class.java) + data.getParcelable( + PROPERTY_RETURN_VALUE, + PendingIntent::class.java, + ) if (pendingIntent != null) { ExecuteAppFunctionResult.PendingIntentAction(pendingIntent) } else { @@ -112,10 +142,320 @@ class ExecuteAppFunctionUseCase ExecuteAppFunctionResult.Error(response.error) } } + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + ExecuteAppFunctionResult.Error(e) + } + } + + private suspend fun executeInternalTool( + function: AppFunctionMetadata, + parameters: AppFunctionData, + ): ExecuteAppFunctionResult { + return try { + when (function.id) { + "getCurrentLocation" -> { + val location = agentInternalTools.getCurrentLocation() + buildLatLngResult(function, location) + } + "geocodeAddress" -> { + val address = + parameters.getString("address") + ?: throw IllegalArgumentException( + "Missing required parameter 'address'", + ) + val location = agentInternalTools.geocodeAddress(address) + buildLatLngResult(function, location) + } + "generateImage" -> { + val prompt = + parameters.getString("prompt") + ?: throw IllegalArgumentException( + "Missing required parameter 'prompt'", + ) + val aspectRatio = parameters.getString("aspectRatio") + val result = agentInternalTools.generateImage(prompt, aspectRatio) + buildGeneratedImageResult(function, result) + } + else -> + throw IllegalArgumentException( + "Unknown internal tool: ${function.id}", + ) + } + } catch (e: CancellationException) { + throw e } catch (e: Exception) { ExecuteAppFunctionResult.Error(e) } } + + private fun buildLatLngResult( + function: AppFunctionMetadata, + location: AgentInternalTools.LatLng?, + ): ExecuteAppFunctionResult.Data { + if (location == null) { + return ExecuteAppFunctionResult.Data( + data = AppFunctionData.EMPTY, + formattedJson = "null", + ) + } + val formattedJson = + JSONObject() + .apply { + put("latitude", location.latitude) + put("longitude", location.longitude) + } + .toString() + val valueType = function.response.valueType + val data = + if (valueType is AppFunctionObjectTypeMetadata) { + val locationData = + AppFunctionData.Builder(valueType, function.components) + .setDouble("latitude", location.latitude) + .setDouble("longitude", location.longitude) + .build() + AppFunctionData.Builder(function.response, function.components) + .setAppFunctionData(PROPERTY_RETURN_VALUE, locationData) + .build() + } else { + AppFunctionData.EMPTY + } + return ExecuteAppFunctionResult.Data(data = data, formattedJson = formattedJson) + } + + private fun buildGeneratedImageResult( + function: AppFunctionMetadata, + result: AgentInternalTools.GeneratedImageResult, + ): ExecuteAppFunctionResult.Data { + val formattedJson = + JSONObject() + .apply { + put("imageUri", result.imageUri) + put("mimeType", result.mimeType) + put("prompt", result.prompt) + } + .toString() + val valueType = function.response.valueType + val data = + if (valueType is AppFunctionObjectTypeMetadata) { + val imageData = + AppFunctionData.Builder(valueType, function.components) + .setString("imageUri", result.imageUri) + .setString("mimeType", result.mimeType) + .setString("prompt", result.prompt) + .build() + AppFunctionData.Builder(function.response, function.components) + .setAppFunctionData(PROPERTY_RETURN_VALUE, imageData) + .build() + } else { + AppFunctionData.EMPTY + } + return ExecuteAppFunctionResult.Data(data = data, formattedJson = formattedJson) + } + + private fun grantContentUriPermissions( + function: AppFunctionMetadata, + parameters: AppFunctionData, + ) { + for (parameter in function.parameters) { + grantUriPermissionsForProperty( + targetPackageName = function.packageName, + data = parameters, + name = parameter.name, + dataType = parameter.dataType, + components = function.components, + ) + } + } + + private fun grantUriPermissionsForProperty( + targetPackageName: String, + data: AppFunctionData, + name: String, + dataType: AppFunctionDataTypeMetadata, + components: AppFunctionComponentsMetadata, + ) { + when (dataType) { + is AppFunctionStringTypeMetadata -> { + runCatching { data.getString(name) } + .getOrNull() + ?.let { grantIfContentUri(targetPackageName, it) } + } + is AppFunctionParcelableTypeMetadata -> { + if (dataType.qualifiedName == Uri::class.java.name) { + runCatching { data.getParcelable(name, Uri::class.java) } + .getOrNull() + ?.let { grantIfContentUri(targetPackageName, it) } + } + } + is AppFunctionObjectTypeMetadata -> { + val nested = + runCatching { data.getAppFunctionData(name) }.getOrNull() ?: return + grantUriPermissionsForObject(targetPackageName, nested, dataType, components) + } + is AppFunctionAllOfTypeMetadata, + is AppFunctionOneOfTypeMetadata, + -> { + val nested = + runCatching { data.getAppFunctionData(name) }.getOrNull() ?: return + grantUriPermissionsForDataType(targetPackageName, nested, dataType, components) + } + is AppFunctionReferenceTypeMetadata -> { + val resolved = components.dataTypes[dataType.referenceDataType] ?: return + grantUriPermissionsForProperty( + targetPackageName = targetPackageName, + data = data, + name = name, + dataType = resolved, + components = components, + ) + } + is AppFunctionArrayTypeMetadata -> { + grantUriPermissionsForArray( + targetPackageName = targetPackageName, + data = data, + name = name, + itemType = dataType.itemType, + components = components, + ) + } + } + } + + private fun grantUriPermissionsForObject( + targetPackageName: String, + data: AppFunctionData, + objectType: AppFunctionObjectTypeMetadata, + components: AppFunctionComponentsMetadata, + ) { + if (objectType.qualifiedName == Uri::class.java.name && objectType.properties.isEmpty()) { + runCatching { data.getString("uri") } + .getOrNull() + ?.let { grantIfContentUri(targetPackageName, it) } + return + } + for ((propName, propType) in objectType.properties) { + grantUriPermissionsForProperty( + targetPackageName = targetPackageName, + data = data, + name = propName, + dataType = propType, + components = components, + ) + } + } + + private fun grantUriPermissionsForDataType( + targetPackageName: String, + data: AppFunctionData, + dataType: AppFunctionDataTypeMetadata, + components: AppFunctionComponentsMetadata, + ) { + when (dataType) { + is AppFunctionObjectTypeMetadata -> { + grantUriPermissionsForObject(targetPackageName, data, dataType, components) + } + is AppFunctionAllOfTypeMetadata -> { + for (subType in dataType.matchAll) { + grantUriPermissionsForDataType( + targetPackageName, + data, + subType, + components, + ) + } + } + is AppFunctionOneOfTypeMetadata -> { + for (candidate in dataType.matchOneOf) { + grantUriPermissionsForDataType( + targetPackageName, + data, + candidate, + components, + ) + } + } + is AppFunctionReferenceTypeMetadata -> { + val resolved = components.dataTypes[dataType.referenceDataType] ?: return + grantUriPermissionsForDataType(targetPackageName, data, resolved, components) + } + } + } + + private fun grantUriPermissionsForArray( + targetPackageName: String, + data: AppFunctionData, + name: String, + itemType: AppFunctionDataTypeMetadata, + components: AppFunctionComponentsMetadata, + ) { + when (itemType) { + is AppFunctionStringTypeMetadata -> { + runCatching { data.getStringList(name) } + .getOrNull() + ?.forEach { grantIfContentUri(targetPackageName, it) } + } + is AppFunctionParcelableTypeMetadata -> { + if (itemType.qualifiedName == Uri::class.java.name) { + runCatching { data.getParcelableList(name, Uri::class.java) } + .getOrNull() + ?.forEach { grantIfContentUri(targetPackageName, it) } + } + } + is AppFunctionReferenceTypeMetadata -> { + val resolved = components.dataTypes[itemType.referenceDataType] ?: return + grantUriPermissionsForArray( + targetPackageName = targetPackageName, + data = data, + name = name, + itemType = resolved, + components = components, + ) + } + is AppFunctionObjectTypeMetadata, + is AppFunctionAllOfTypeMetadata, + is AppFunctionOneOfTypeMetadata, + -> { + runCatching { data.getAppFunctionDataList(name) } + .getOrNull() + ?.forEach { itemData -> + grantUriPermissionsForDataType( + targetPackageName, + itemData, + itemType, + components, + ) + } + } + } + } + + private fun grantIfContentUri( + targetPackageName: String, + uriString: String, + ) { + if (uriString.startsWith("content://")) { + runCatching { + grantIfContentUri(targetPackageName, Uri.parse(uriString)) + } + } + } + + private fun grantIfContentUri( + targetPackageName: String, + uri: Uri, + ) { + if (uri.scheme == "content") { + runCatching { + context.grantUriPermission( + targetPackageName, + uri, + Intent.FLAG_GRANT_READ_URI_PERMISSION, + ) + } + } + } } /** Sealed class representing the result of an AppFunction execution. */ diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCase.kt index fa1e2848..095d6982 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCase.kt @@ -19,9 +19,12 @@ import androidx.appfunctions.AppFunctionManager import androidx.appfunctions.AppFunctionSearchSpec import androidx.appfunctions.metadata.AppFunctionMetadata import androidx.appfunctions.metadata.AppFunctionPackageMetadata +import com.example.appfunctions.agent.data.AgentInternalTools +import com.example.appfunctions.agent.data.SettingsRepository import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.FlowPreview import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.debounce import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.flow.mapLatest @@ -29,35 +32,136 @@ import kotlinx.coroutines.flow.onStart import javax.inject.Inject import kotlin.time.Duration.Companion.milliseconds -/** Use case to get all available AppFunctions grouped by package name. */ +/** + * Filter parameters for [GetAppFunctionsUseCase]. + * + * Nullable filter fields (`null`) indicate that the corresponding filter should not be applied. + */ +data class GetAppFunctionsParams( + val targetPackageNames: Set? = null, + val isEnabled: Boolean? = null, + val isConnected: Boolean? = null, + val includeInternalTools: Boolean = true, +) + +/** Use case to observe available AppFunctions grouped by package metadata with optional filters. */ class GetAppFunctionsUseCase @Inject constructor( private val appFunctionManager: AppFunctionManager?, + private val getAppFunctionStatesUseCase: GetAppFunctionStatesUseCase, + private val settingsRepository: SettingsRepository, + private val agentInternalTools: AgentInternalTools, ) { /** * Executes the use case. * - * @return A Flow emitting a map of package names to their list of AppFunctionMetadata. + * @param params Optional [GetAppFunctionsParams] filters to apply. + * @return A Flow emitting a map of [AppFunctionPackageMetadata] to their matching + * [AppFunctionMetadata] list. */ @OptIn(FlowPreview::class, ExperimentalCoroutinesApi::class) - operator fun invoke(): Flow>> { - if (appFunctionManager == null) { - return flowOf(emptyMap()) - } - return appFunctionManager.observeAppFunctions() - .debounce(500.milliseconds) - .mapLatest { _ -> - appFunctionManager.search() + operator fun invoke( + params: GetAppFunctionsParams = GetAppFunctionsParams(), + ): Flow>> { + val externalTargetPackages = + params.targetPackageNames?.minus(AgentInternalTools.INTERNAL_TOOL_PACKAGE) + val externalFunctionsFlow: Flow> = + if ( + appFunctionManager == null || + (externalTargetPackages != null && externalTargetPackages.isEmpty()) + ) { + flowOf(emptyList()) + } else { + val searchSpec = AppFunctionSearchSpec(packageNames = externalTargetPackages) + appFunctionManager + .observeAppFunctions() + .debounce(500.milliseconds) + .mapLatest { appFunctionManager.searchAppFunctions(searchSpec) } + .onStart { emit(appFunctionManager.searchAppFunctions(searchSpec)) } } - .onStart { - emit(appFunctionManager.search()) + + val disconnectedAppsFlow: Flow> = + if (params.isConnected != null) { + settingsRepository.disconnectedApps + } else { + flowOf(emptySet()) } + + return combine(externalFunctionsFlow, disconnectedAppsFlow) { + externalFunctions, + disconnectedApps, + -> + filterAndGroup( + externalFunctions = externalFunctions, + disconnectedApps = disconnectedApps, + params = params, + ) + } } - private suspend fun AppFunctionManager.search(): Map> { - return searchAppFunctions(AppFunctionSearchSpec()).groupBy( - AppFunctionMetadata::packageMetadata, - ) + private suspend fun filterAndGroup( + externalFunctions: List, + disconnectedApps: Set, + params: GetAppFunctionsParams, + ): Map> { + val internalEntries = + if (params.includeInternalTools) { + agentInternalTools.getInternalToolsMetadata() + } else { + emptyList() + } + val allCandidates = externalFunctions + internalEntries.map { it.first } + + val preFiltered = + allCandidates.filter { metadata -> + val matchesPackage = + params.targetPackageNames == null || + metadata.packageName in params.targetPackageNames + val matchesConnection = + if (params.isConnected == null) { + true + } else { + val connected = + metadata.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE || + metadata.packageName !in disconnectedApps + connected == params.isConnected + } + matchesPackage && matchesConnection + } + + val enabledFiltered = + if (params.isEnabled == null) { + preFiltered + } else { + val externalFunctionNames = + preFiltered + .filter { it.packageName != AgentInternalTools.INTERNAL_TOOL_PACKAGE } + .map { it.name } + val externalStates = + if (externalFunctionNames.isNotEmpty()) { + getAppFunctionStatesUseCase(externalFunctionNames) + } else { + emptyList() + } + val stateMap = + (externalStates + internalEntries.map { it.second }) + .associateBy { it.functionName } + + preFiltered.filter { metadata -> + val enabled = + if (metadata.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) { + stateMap[metadata.name]?.isEnabled ?: true + } else { + stateMap[metadata.name]?.isEnabled ?: false + } + enabled == params.isEnabled + } + } + + return enabledFiltered + .sortedByDescending { metadata -> metadata.id.startsWith(metadata.packageName) } + .distinctBy { metadata -> metadata.packageName to metadata.id } + .groupBy(AppFunctionMetadata::packageMetadata) } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetConnectedAppsUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetConnectedAppsUseCase.kt index 004896b3..e27660f2 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetConnectedAppsUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetConnectedAppsUseCase.kt @@ -35,7 +35,9 @@ class GetConnectedAppsUseCase ) { @OptIn(ExperimentalCoroutinesApi::class) operator fun invoke(): Flow> { - return getAppFunctionsUseCase().flatMapLatest { appFunctionsMap -> + return getAppFunctionsUseCase( + GetAppFunctionsParams(includeInternalTools = false), + ).flatMapLatest { appFunctionsMap -> val installedApps = getInstalledAppsUseCase() val installedAppsMap = installedApps.associateBy { it.packageName } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/troubleshoot/TroubleshootAppUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/troubleshoot/TroubleshootAppUseCase.kt index 7afbaab2..254b5d06 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/troubleshoot/TroubleshootAppUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/troubleshoot/TroubleshootAppUseCase.kt @@ -20,6 +20,7 @@ import android.content.Context import android.content.Intent import android.content.pm.PackageManager import com.example.appfunctions.agent.R +import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsParams import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.flow.first @@ -43,7 +44,13 @@ class TroubleshootAppUseCase val pm = context.packageManager // 1. Check if already visible - val appFunctionsMap = getAppFunctionsUseCase().first() + val appFunctionsMap = + getAppFunctionsUseCase( + GetAppFunctionsParams( + targetPackageNames = setOf(packageName), + includeInternalTools = false, + ), + ).first() val isVisible = appFunctionsMap.keys.any { it.packageName == packageName } if (isVisible) { steps.add( diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt index 2f92660d..32151b5a 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt @@ -28,6 +28,7 @@ import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentOrchestrator import com.example.appfunctions.agent.domain.AgentStatus import com.example.appfunctions.agent.domain.appfunction.AppInfo +import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsParams import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase import com.example.appfunctions.agent.domain.chat.GetChatHistoryUseCase @@ -74,7 +75,9 @@ class AgentDemoViewModel init { viewModelScope.launch { - getAppFunctionsUseCase().collect { toolsMap -> + getAppFunctionsUseCase( + GetAppFunctionsParams(includeInternalTools = false), + ).collect { toolsMap -> val allTools = toolsMap.values.flatten() val packagesWithTools = allTools.map { it.packageName }.toSet() val filteredApps = diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/DebuggingViewModel.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/DebuggingViewModel.kt index f929cd50..acfbdc36 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/DebuggingViewModel.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/DebuggingViewModel.kt @@ -31,6 +31,7 @@ import com.example.appfunctions.agent.domain.appfunction.ConvertInputToAppFuncti import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionUseCase import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionStatesUseCase +import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsParams import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase import com.example.appfunctions.agent.domain.pendingintent.LaunchPendingIntentUseCase @@ -92,7 +93,9 @@ class DebuggingViewModel private fun loadAppFunctions() { viewModelScope.launch { _uiState.update { it.copy(isLoading = true) } - getAppFunctionsUseCase().collect { appFunctionsMap -> + getAppFunctionsUseCase( + GetAppFunctionsParams(includeInternalTools = false), + ).collect { appFunctionsMap -> val allFunctionNames = appFunctionsMap.values.flatten().map { it.name } val appFunctionStates = getAppFunctionStatesUseCase(allFunctionNames) allAppFunctions = appFunctionsMap diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt index 8c26dbd8..7bf69940 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt @@ -17,10 +17,17 @@ package com.example.appfunctions.agent.domain import android.content.Intent import androidx.appfunctions.AppFunctionData +import androidx.appfunctions.AppFunctionManager import androidx.appfunctions.AppFunctionName import androidx.appfunctions.AppFunctionState +import androidx.appfunctions.ExecuteAppFunctionResponse +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata import androidx.appfunctions.metadata.AppFunctionMetadata import androidx.appfunctions.metadata.AppFunctionPackageMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionResponseMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import androidx.test.core.app.ApplicationProvider import com.example.appfunctions.agent.data.AgentInternalTools import com.example.appfunctions.agent.data.LlmModel import com.example.appfunctions.agent.data.LlmProviderName @@ -31,6 +38,7 @@ import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity +import com.example.appfunctions.agent.domain.appfunction.ConvertAppFunctionDataToJsonUseCase import com.example.appfunctions.agent.domain.appfunction.ConvertInputToAppFunctionDataUseCase import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionUseCase @@ -49,6 +57,7 @@ import io.mockk.coVerify import io.mockk.every import io.mockk.mockk import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.emptyFlow import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.test.runTest @@ -68,7 +77,7 @@ class AgentOrchestratorTest { private val manageThreadsUseCase: ManageThreadsUseCase = mockk(relaxed = true) private val settingsRepository: SettingsRepository = mockk() private val llmProviderFactory: LlmProviderFactory = mockk() - private val getAppFunctionsUseCase: GetAppFunctionsUseCase = mockk() + private val appFunctionManager: AppFunctionManager = mockk() private val getAppFunctionStatesUseCase: GetAppFunctionStatesUseCase = mockk() private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase = mockk() private val sendMessageUseCase: SendMessageUseCase = mockk(relaxed = true) @@ -77,13 +86,24 @@ class AgentOrchestratorTest { private val savePendingIntentUseCase: SavePendingIntentUseCase = mockk(relaxed = true) private val agentInternalTools: AgentInternalTools = mockk(relaxed = true) + private lateinit var getAppFunctionsUseCase: GetAppFunctionsUseCase private lateinit var agentOrchestrator: AgentOrchestrator @Before fun setUp() { + every { appFunctionManager.observeAppFunctions() } returns emptyFlow() + coEvery { appFunctionManager.searchAppFunctions(any()) } returns emptyList() + + getAppFunctionsUseCase = + GetAppFunctionsUseCase( + appFunctionManager = appFunctionManager, + getAppFunctionStatesUseCase = getAppFunctionStatesUseCase, + settingsRepository = settingsRepository, + agentInternalTools = agentInternalTools, + ) + agentOrchestrator = AgentOrchestrator( - context = context, manageThreadsUseCase = manageThreadsUseCase, observePendingMessagesUseCase = observePendingMessagesUseCase, sendMessageUseCase = sendMessageUseCase, @@ -92,11 +112,9 @@ class AgentOrchestratorTest { llmProviderFactory = llmProviderFactory, settingsRepository = settingsRepository, getAppFunctionsUseCase = getAppFunctionsUseCase, - getAppFunctionStatesUseCase = getAppFunctionStatesUseCase, convertInputToAppFunctionDataUseCase = convertInputToAppFunctionDataUseCase, executeAppFunctionUseCase = executeAppFunctionUseCase, savePendingIntentUseCase = savePendingIntentUseCase, - agentInternalTools = agentInternalTools, ) coEvery { getAppFunctionStatesUseCase(any()) } answers { @@ -148,11 +166,11 @@ class AgentOrchestratorTest { val llmProvider = mockk() setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) val errorMsg = "LLM failed" - coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any(), any()) } returns - LlmResponse.Error(errorMsg) + coEvery { + llmProvider.generateResponse(any(), any(), any(), any(), any(), any()) + } returns LlmResponse.Error(errorMsg) agentOrchestrator.observeAndProcessMessages(threadId) @@ -183,10 +201,11 @@ class AgentOrchestratorTest { val llmProvider = mockk() setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) val responseText = "Hi there" - coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any(), any()) } returns + coEvery { + llmProvider.generateResponse(any(), any(), any(), any(), any(), any()) + } returns LlmResponse.Success("interaction_123", listOf(LlmResponsePart.Text(responseText))) agentOrchestrator.observeAndProcessMessages(threadId) @@ -298,7 +317,10 @@ class AgentOrchestratorTest { convertInputToAppFunctionDataUseCase(any(), any(), any()) } returns Result.success(mockk()) - val appException = androidx.appfunctions.AppFunctionPermissionRequiredException("Calendar permission required") + val appException = + androidx.appfunctions.AppFunctionPermissionRequiredException( + "Calendar permission required", + ) coEvery { executeAppFunctionUseCase(any(), any(), any()) } returns ExecuteAppFunctionResult.Error(appException) @@ -319,10 +341,15 @@ class AgentOrchestratorTest { ) val expectedErrorOutput = - "Tool execution failed for create_event: Error: AppFunctionPermissionRequiredException - Calendar permission required" + "Tool execution failed for create_event: Error: " + + "AppFunctionPermissionRequiredException - Calendar permission required" coEvery { llmProvider.generateResponse(eq("interaction_1"), any(), any(), any(), any()) - } returns LlmResponse.Success("interaction_2", listOf(LlmResponsePart.Text("Permission needed."))) + } returns + LlmResponse.Success( + "interaction_2", + listOf(LlmResponsePart.Text("Permission needed.")), + ) agentOrchestrator.observeAndProcessMessages(threadId) @@ -378,16 +405,21 @@ class AgentOrchestratorTest { packageName: String, id: String, ): AppFunctionMetadata { + val pkgMetadata = + AppFunctionPackageMetadata( + packageName = packageName, + components = AppFunctionComponentsMetadata(), + ) val tool = mockk(relaxed = true) every { tool.packageName } returns packageName every { tool.id } returns id every { tool.name } returns AppFunctionName(packageName, id) + every { tool.packageMetadata } returns pkgMetadata return tool } private fun mockAppFunctions(tools: List) { - val packageMetadata = mockk(relaxed = true) - coEvery { getAppFunctionsUseCase() } returns flowOf(mapOf(packageMetadata to tools)) + coEvery { appFunctionManager.searchAppFunctions(any()) } returns tools } private fun setupDefaultMocks( @@ -464,10 +496,64 @@ class AgentOrchestratorTest { val thread = createThread(threadId) val llmProvider = mockk() - val targetTool = createMockTool("com.example.targetapp", "setWallpaper") + val components = AppFunctionComponentsMetadata() + val targetTool = + AppFunctionMetadata( + name = AppFunctionName("com.example.targetapp", "setWallpaper"), + schema = null, + parameters = + listOf( + AppFunctionParameterMetadata( + name = "uri", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(false), + ), + ), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(false), + description = "", + ), + description = "Set wallpaper", + deprecation = null, + packageMetadata = + AppFunctionPackageMetadata( + packageName = "com.example.targetapp", + components = components, + ), + ) mockAppFunctions(listOf(targetTool)) setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + val realConverter = + ConvertInputToAppFunctionDataUseCase(ApplicationProvider.getApplicationContext()) + val convertToJson = mockk() + every { convertToJson(any(), any(), any(), any()) } returns """{"success":true}""" + coEvery { appFunctionManager.executeAppFunction(any()) } returns + ExecuteAppFunctionResponse.Success(AppFunctionData.EMPTY) + val realExecutor = + ExecuteAppFunctionUseCase( + context = context, + appFunctionManager = appFunctionManager, + convertAppFunctionDataToJsonUseCase = convertToJson, + agentInternalTools = agentInternalTools, + ) + + val orchestratorWithRealPipeline = + AgentOrchestrator( + manageThreadsUseCase = manageThreadsUseCase, + observePendingMessagesUseCase = observePendingMessagesUseCase, + sendMessageUseCase = sendMessageUseCase, + updateMessageUseCase = updateMessageUseCase, + updateThreadUseCase = updateThreadUseCase, + llmProviderFactory = llmProviderFactory, + settingsRepository = settingsRepository, + getAppFunctionsUseCase = getAppFunctionsUseCase, + convertInputToAppFunctionDataUseCase = realConverter, + executeAppFunctionUseCase = realExecutor, + savePendingIntentUseCase = savePendingIntentUseCase, + ) + val contentUri = "content://com.example.appfunctions.agent.fileprovider/cache/img.jpg" val toolCall = LlmResponsePart.ToolCall( @@ -476,19 +562,36 @@ class AgentOrchestratorTest { arguments = mapOf("uri" to contentUri), callId = "call_2", ) - setupTwoStepToolCallAndExecution( - llmProvider = llmProvider, - toolCall = toolCall, - secondResponseText = "Wallpaper set", - toolResultJson = """{"success":true}""", - ) + coEvery { + llmProvider.generateResponse( + previousInteractionId = null, + input = any(), + tools = any(), + apiKey = any(), + modelName = any(), + ) + } returns LlmResponse.Success(interactionId = "interaction_1", parts = listOf(toolCall)) - agentOrchestrator.observeAndProcessMessages(threadId) + coEvery { + llmProvider.generateResponse( + previousInteractionId = "interaction_1", + input = any(), + tools = any(), + apiKey = any(), + modelName = any(), + ) + } returns + LlmResponse.Success( + interactionId = "interaction_2", + parts = listOf(LlmResponsePart.Text("Wallpaper set")), + ) + + orchestratorWithRealPipeline.observeAndProcessMessages(threadId) coVerify { context.grantUriPermission( eq("com.example.targetapp"), - any(), + match { it.toString() == contentUri }, eq(Intent.FLAG_GRANT_READ_URI_PERMISSION), ) } @@ -553,8 +656,10 @@ class AgentOrchestratorTest { val thread = createThread(threadId) val llmProvider = mockk() - val enabledTool = createMockTool("com.google.android.appfunctiontestingagent", "enabled_tool") - val disabledTool = createMockTool("com.google.android.appfunctiontestingagent", "disabled_tool") + val enabledTool = + createMockTool("com.google.android.appfunctiontestingagent", "enabled_tool") + val disabledTool = + createMockTool("com.google.android.appfunctiontestingagent", "disabled_tool") mockAppFunctions(listOf(enabledTool, disabledTool)) setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt index 4de6c269..ebee8411 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt @@ -27,200 +27,231 @@ import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata import androidx.appfunctions.metadata.AppFunctionParameterMetadata import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import androidx.test.core.app.ApplicationProvider +import kotlinx.coroutines.test.runTest import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue import org.junit.Test import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner import org.robolectric.annotation.Config +import java.io.IOException @RunWith(RobolectricTestRunner::class) @Config(sdk = [34]) class ConvertInputToAppFunctionDataUseCaseTest { - private val useCase = ConvertInputToAppFunctionDataUseCase() + private val useCase = + ConvertInputToAppFunctionDataUseCase(ApplicationProvider.getApplicationContext()) private val components = AppFunctionComponentsMetadata(emptyMap()) @Test - fun invoke_allPrimitiveTypes_convertsCorrectly() { - val parameters = - listOf( - AppFunctionParameterMetadata("str", true, AppFunctionStringTypeMetadata(false)), - AppFunctionParameterMetadata("int", true, AppFunctionIntTypeMetadata(false)), - AppFunctionParameterMetadata("long", true, AppFunctionLongTypeMetadata(false)), - AppFunctionParameterMetadata("bool", true, AppFunctionBooleanTypeMetadata(false)), - AppFunctionParameterMetadata("double", true, AppFunctionDoubleTypeMetadata(false)), - AppFunctionParameterMetadata("float", true, AppFunctionFloatTypeMetadata(false)), - ) - val inputs = - mapOf( - "str" to "value", - "int" to "123", - "long" to "456", - "bool" to true, - "double" to "1.23", - "float" to "4.56", - ) + fun invoke_allPrimitiveTypes_convertsCorrectly() = + runTest { + val parameters = + listOf( + AppFunctionParameterMetadata("str", true, AppFunctionStringTypeMetadata(false)), + AppFunctionParameterMetadata("int", true, AppFunctionIntTypeMetadata(false)), + AppFunctionParameterMetadata("long", true, AppFunctionLongTypeMetadata(false)), + AppFunctionParameterMetadata( + "bool", + true, + AppFunctionBooleanTypeMetadata(false), + ), + AppFunctionParameterMetadata( + "double", + true, + AppFunctionDoubleTypeMetadata(false), + ), + AppFunctionParameterMetadata( + "float", + true, + AppFunctionFloatTypeMetadata(false), + ), + ) + val inputs = + mapOf( + "str" to "value", + "int" to "123", + "long" to "456", + "bool" to true, + "double" to "1.23", + "float" to "4.56", + ) - val result = useCase(parameters, components, inputs).getOrThrow() + val result = useCase(parameters, components, inputs).getOrThrow() - assertEquals("value", result.getString("str")) - assertEquals(123, result.getInt("int")) - assertEquals(456L, result.getLong("long")) - assertEquals(true, result.getBoolean("bool")) - assertEquals(1.23, result.getDouble("double"), 0.001) - assertEquals(4.56f, result.getFloat("float"), 0.001f) - } + assertEquals("value", result.getString("str")) + assertEquals(123, result.getInt("int")) + assertEquals(456L, result.getLong("long")) + assertEquals(true, result.getBoolean("bool")) + assertEquals(1.23, result.getDouble("double"), 0.001) + assertEquals(4.56f, result.getFloat("float"), 0.001f) + } @Test - fun invoke_allPrimitiveArrays_convertsCorrectly() { - val parameters = - listOf( - AppFunctionParameterMetadata( - "strArray", - true, - AppFunctionArrayTypeMetadata(AppFunctionStringTypeMetadata(false), false), - ), - AppFunctionParameterMetadata( - "intArray", - true, - AppFunctionArrayTypeMetadata(AppFunctionIntTypeMetadata(false), false), - ), - AppFunctionParameterMetadata( - "longArray", - true, - AppFunctionArrayTypeMetadata(AppFunctionLongTypeMetadata(false), false), - ), - AppFunctionParameterMetadata( - "boolArray", - true, - AppFunctionArrayTypeMetadata(AppFunctionBooleanTypeMetadata(false), false), - ), - AppFunctionParameterMetadata( - "doubleArray", - true, - AppFunctionArrayTypeMetadata(AppFunctionDoubleTypeMetadata(false), false), - ), - AppFunctionParameterMetadata( - "floatArray", - true, - AppFunctionArrayTypeMetadata(AppFunctionFloatTypeMetadata(false), false), - ), - ) - val inputs = - mapOf( - "strArray" to listOf("v1", "v2"), - "intArray" to listOf("1", "2"), - "longArray" to listOf("3", "4"), - "boolArray" to listOf(true, false), - "doubleArray" to listOf("1.1", "2.2"), - "floatArray" to listOf("3.3", "4.4"), - ) + fun invoke_allPrimitiveArrays_convertsCorrectly() = + runTest { + val parameters = + listOf( + AppFunctionParameterMetadata( + "strArray", + true, + AppFunctionArrayTypeMetadata(AppFunctionStringTypeMetadata(false), false), + ), + AppFunctionParameterMetadata( + "intArray", + true, + AppFunctionArrayTypeMetadata(AppFunctionIntTypeMetadata(false), false), + ), + AppFunctionParameterMetadata( + "longArray", + true, + AppFunctionArrayTypeMetadata(AppFunctionLongTypeMetadata(false), false), + ), + AppFunctionParameterMetadata( + "boolArray", + true, + AppFunctionArrayTypeMetadata(AppFunctionBooleanTypeMetadata(false), false), + ), + AppFunctionParameterMetadata( + "doubleArray", + true, + AppFunctionArrayTypeMetadata(AppFunctionDoubleTypeMetadata(false), false), + ), + AppFunctionParameterMetadata( + "floatArray", + true, + AppFunctionArrayTypeMetadata(AppFunctionFloatTypeMetadata(false), false), + ), + ) + val inputs = + mapOf( + "strArray" to listOf("v1", "v2"), + "intArray" to listOf("1", "2"), + "longArray" to listOf("3", "4"), + "boolArray" to listOf(true, false), + "doubleArray" to listOf("1.1", "2.2"), + "floatArray" to listOf("3.3", "4.4"), + ) - val result = useCase(parameters, components, inputs).getOrThrow() + val result = useCase(parameters, components, inputs).getOrThrow() - assertEquals(listOf("v1", "v2"), result.getStringList("strArray")) - assertEquals(listOf(1, 2), result.getIntArray("intArray")?.toList()) - assertEquals(listOf(3L, 4L), result.getLongArray("longArray")?.toList()) - assertEquals(listOf(true, false), result.getBooleanArray("boolArray")?.toList()) - assertEquals(listOf(1.1, 2.2), result.getDoubleArray("doubleArray")?.toList()) - assertEquals(listOf(3.3f, 4.4f), result.getFloatArray("floatArray")?.toList()) - } + assertEquals(listOf("v1", "v2"), result.getStringList("strArray")) + assertEquals(listOf(1, 2), result.getIntArray("intArray")?.toList()) + assertEquals(listOf(3L, 4L), result.getLongArray("longArray")?.toList()) + assertEquals(listOf(true, false), result.getBooleanArray("boolArray")?.toList()) + assertEquals(listOf(1.1, 2.2), result.getDoubleArray("doubleArray")?.toList()) + assertEquals(listOf(3.3f, 4.4f), result.getFloatArray("floatArray")?.toList()) + } @Test - fun invoke_objectParameter_convertsCorrectly() { - val objectType = - AppFunctionObjectTypeMetadata( - properties = - mapOf( - "prop1" to AppFunctionStringTypeMetadata(false), - "prop2" to AppFunctionIntTypeMetadata(false), - ), - required = listOf("prop1", "prop2"), - qualifiedName = null, - isNullable = false, - ) - val parameters = - listOf( - AppFunctionParameterMetadata("obj", true, objectType), - ) - val inputs = - mapOf( - "obj" to mapOf("prop1" to "val1", "prop2" to "123"), - ) + fun invoke_objectParameter_convertsCorrectly() = + runTest { + val objectType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "prop1" to AppFunctionStringTypeMetadata(false), + "prop2" to AppFunctionIntTypeMetadata(false), + ), + required = listOf("prop1", "prop2"), + qualifiedName = null, + isNullable = false, + ) + val parameters = + listOf( + AppFunctionParameterMetadata("obj", true, objectType), + ) + val inputs = + mapOf( + "obj" to mapOf("prop1" to "val1", "prop2" to "123"), + ) - val result = useCase(parameters, components, inputs).getOrThrow() + val result = useCase(parameters, components, inputs).getOrThrow() - val objResult = result.getAppFunctionData("obj") - assertEquals("val1", objResult?.getString("prop1")) - assertEquals(123, objResult?.getInt("prop2")) - } + val objResult = result.getAppFunctionData("obj") + assertEquals("val1", objResult?.getString("prop1")) + assertEquals(123, objResult?.getInt("prop2")) + } @Test - fun invoke_referenceParameter_convertsCorrectly() { - val objectType = - AppFunctionObjectTypeMetadata( - properties = - mapOf( - "prop1" to AppFunctionStringTypeMetadata(false), + fun invoke_referenceParameter_convertsCorrectly() = + runTest { + val objectType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "prop1" to AppFunctionStringTypeMetadata(false), + ), + required = listOf("prop1"), + qualifiedName = null, + isNullable = false, + ) + val componentsWithTypes = + AppFunctionComponentsMetadata( + dataTypes = mapOf("RefType" to objectType), + ) + val parameters = + listOf( + AppFunctionParameterMetadata( + "ref", + true, + AppFunctionReferenceTypeMetadata("RefType", false), ), - required = listOf("prop1"), - qualifiedName = null, - isNullable = false, - ) - val componentsWithTypes = - AppFunctionComponentsMetadata( - dataTypes = mapOf("RefType" to objectType), - ) - val parameters = - listOf( - AppFunctionParameterMetadata( - "ref", - true, - AppFunctionReferenceTypeMetadata("RefType", false), - ), - ) - val inputs = - mapOf( - "ref" to mapOf("prop1" to "val1"), - ) + ) + val inputs = + mapOf( + "ref" to mapOf("prop1" to "val1"), + ) - val result = useCase(parameters, componentsWithTypes, inputs).getOrThrow() + val result = useCase(parameters, componentsWithTypes, inputs).getOrThrow() - val refResult = result.getAppFunctionData("ref") - assertEquals("val1", refResult?.getString("prop1")) - } + val refResult = result.getAppFunctionData("ref") + assertEquals("val1", refResult?.getString("prop1")) + } @Test - fun invoke_emptyStringNumericParameters_areSkipped() { - val parameters = - listOf( - AppFunctionParameterMetadata("int", false, AppFunctionIntTypeMetadata(false)), - AppFunctionParameterMetadata("long", false, AppFunctionLongTypeMetadata(false)), - AppFunctionParameterMetadata("double", false, AppFunctionDoubleTypeMetadata(false)), - AppFunctionParameterMetadata("float", false, AppFunctionFloatTypeMetadata(false)), - ) - val inputs = - mapOf( - "int" to "", - "long" to "", - "double" to "", - "float" to "", - ) + fun invoke_emptyStringNumericParameters_areSkipped() = + runTest { + val parameters = + listOf( + AppFunctionParameterMetadata("int", false, AppFunctionIntTypeMetadata(false)), + AppFunctionParameterMetadata("long", false, AppFunctionLongTypeMetadata(false)), + AppFunctionParameterMetadata( + "double", + false, + AppFunctionDoubleTypeMetadata(false), + ), + AppFunctionParameterMetadata( + "float", + false, + AppFunctionFloatTypeMetadata(false), + ), + ) + val inputs = + mapOf( + "int" to "", + "long" to "", + "double" to "", + "float" to "", + ) - val result = useCase(parameters, components, inputs) + val result = useCase(parameters, components, inputs) - // Verify that it doesn't throw NumberFormatException during conversion - assertEquals(true, result.isSuccess) - } + // Verify that it doesn't throw NumberFormatException during conversion + assertEquals(true, result.isSuccess) + } @Test - fun invoke_invalidNumericInputs_returnsFailure() { - assertFailureForInput(AppFunctionIntTypeMetadata(false), "not_an_int") - assertFailureForInput(AppFunctionLongTypeMetadata(false), "not_a_long") - assertFailureForInput(AppFunctionDoubleTypeMetadata(false), "not_a_double") - assertFailureForInput(AppFunctionFloatTypeMetadata(false), "not_a_float") - } + fun invoke_invalidNumericInputs_returnsFailure() = + runTest { + assertFailureForInput(AppFunctionIntTypeMetadata(false), "not_an_int") + assertFailureForInput(AppFunctionLongTypeMetadata(false), "not_a_long") + assertFailureForInput(AppFunctionDoubleTypeMetadata(false), "not_a_double") + assertFailureForInput(AppFunctionFloatTypeMetadata(false), "not_a_float") + } - private fun assertFailureForInput( + private suspend fun assertFailureForInput( dataType: AppFunctionDataTypeMetadata, value: Any, ) { @@ -240,62 +271,160 @@ class ConvertInputToAppFunctionDataUseCaseTest { } @Test - fun convert_uriObjectTypeWithStringInput_buildsUriAppFunctionData() { - val uriObjectType = - AppFunctionObjectTypeMetadata( - properties = - mapOf( - "uri" to AppFunctionStringTypeMetadata(false), + fun convert_uriObjectTypeWithStringInput_buildsUriAppFunctionData() = + runTest { + val uriObjectType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "uri" to AppFunctionStringTypeMetadata(false), + ), + required = listOf("uri"), + qualifiedName = "android.net.Uri", + isNullable = false, + ) + val parameters = + listOf( + AppFunctionParameterMetadata( + name = "wallpaperUri", + isRequired = true, + dataType = uriObjectType, ), - required = listOf("uri"), - qualifiedName = "android.net.Uri", - isNullable = false, - ) - val parameters = - listOf( - AppFunctionParameterMetadata( - name = "wallpaperUri", - isRequired = true, - dataType = uriObjectType, - ), - ) - val inputs = mapOf("wallpaperUri" to "content://com.example/file.jpg") + ) + val inputs = mapOf("wallpaperUri" to "content://com.example/file.jpg") - val result = useCase(parameters, components, inputs).getOrThrow() + val result = useCase(parameters, components, inputs).getOrThrow() - val uriData = result.getAppFunctionData("wallpaperUri") - assertEquals("content://com.example/file.jpg", uriData?.getString("uri")) - } + val uriData = result.getAppFunctionData("wallpaperUri") + assertEquals("content://com.example/file.jpg", uriData?.getString("uri")) + } @Test - fun convert_uriObjectTypeArrayWithStringInput_buildsUriObjectTypeArray() { - val uriObjectType = - AppFunctionObjectTypeMetadata( - properties = mapOf("uri" to AppFunctionStringTypeMetadata(false)), - required = listOf("uri"), - qualifiedName = "android.net.Uri", - isNullable = false, + fun convert_uriObjectTypeArrayWithStringInput_buildsUriObjectTypeArray() = + runTest { + val uriObjectType = + AppFunctionObjectTypeMetadata( + properties = mapOf("uri" to AppFunctionStringTypeMetadata(false)), + required = listOf("uri"), + qualifiedName = "android.net.Uri", + isNullable = false, + ) + val arrayType = + AppFunctionArrayTypeMetadata( + itemType = uriObjectType, + isNullable = false, + ) + val parameters = + listOf( + AppFunctionParameterMetadata( + name = "attachments", + isRequired = true, + dataType = arrayType, + ), + ) + val inputs = mapOf("attachments" to listOf("content://com.example/file.jpg")) + + val result = useCase(parameters, components, inputs).getOrThrow() + + val dataList = result.getAppFunctionDataList("attachments") + assertEquals(1, dataList?.size) + val uriData = dataList?.first() + assertEquals("content://com.example/file.jpg", uriData?.getString("uri")) + } + + @Test + fun invoke_resolvesRemoteFileReferencesWhileLeavingRegularUrlParametersUntouched() = + runTest { + val downloadedUrls = mutableListOf() + val testUseCase = + ConvertInputToAppFunctionDataUseCase(ApplicationProvider.getApplicationContext()) + .apply { + remoteFileDownloader = { _, url -> + downloadedUrls.add(url) + "content://com.example.appfunctions.agent.fileprovider/cache/resolved.png" + } + } + + val uriObjectType = + AppFunctionObjectTypeMetadata( + properties = mapOf("uri" to AppFunctionStringTypeMetadata(false)), + required = listOf("uri"), + qualifiedName = "android.net.Uri", + isNullable = false, + ) + val parameters = + listOf( + AppFunctionParameterMetadata( + name = "imageUri", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(false), + ), + AppFunctionParameterMetadata( + name = "attachment", + isRequired = true, + dataType = uriObjectType, + ), + AppFunctionParameterMetadata( + name = "websiteUrl", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(false), + ), + ) + val inputs = + mapOf( + "imageUri" to "https://example.com/cat.png", + "attachment" to "https://example.com/doc.png", + "websiteUrl" to "https://example.com/home", + ) + + val result = testUseCase(parameters, components, inputs).getOrThrow() + + assertEquals( + listOf("https://example.com/cat.png", "https://example.com/doc.png"), + downloadedUrls, ) - val arrayType = - AppFunctionArrayTypeMetadata( - itemType = uriObjectType, - isNullable = false, + assertEquals( + "content://com.example.appfunctions.agent.fileprovider/cache/resolved.png", + result.getString("imageUri"), ) - val parameters = - listOf( - AppFunctionParameterMetadata( - name = "attachments", - isRequired = true, - dataType = arrayType, - ), + assertEquals( + "content://com.example.appfunctions.agent.fileprovider/cache/resolved.png", + result.getAppFunctionData("attachment")?.getString("uri"), ) - val inputs = mapOf("attachments" to listOf("content://com.example/file.jpg")) + assertEquals("https://example.com/home", result.getString("websiteUrl")) + } - val result = useCase(parameters, components, inputs).getOrThrow() + @Test + fun invoke_returnsFailureWhenRemoteFileDownloadFails() = + runTest { + val testUseCase = + ConvertInputToAppFunctionDataUseCase(ApplicationProvider.getApplicationContext()) + .apply { + remoteFileDownloader = { _, _ -> + throw IOException("HTTP 404") + } + } - val dataList = result.getAppFunctionDataList("attachments") - assertEquals(1, dataList?.size) - val uriData = dataList?.first() - assertEquals("content://com.example/file.jpg", uriData?.getString("uri")) - } + val parameters = + listOf( + AppFunctionParameterMetadata( + name = "wallpaperUri", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(false), + ), + ) + val result = + testUseCase( + parameters, + components, + mapOf("wallpaperUri" to "https://example.com/missing.png"), + ) + + assertTrue(result.isFailure) + assertTrue( + result.exceptionOrNull()?.message?.contains( + "Failed to download remote file reference: HTTP 404", + ) == true, + ) + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt index 9c660ff5..b94dceec 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt @@ -15,15 +15,28 @@ */ package com.example.appfunctions.agent.domain.appfunction +import android.content.Context +import android.content.Intent import androidx.appfunctions.AppFunctionData import androidx.appfunctions.AppFunctionManager +import androidx.appfunctions.AppFunctionName import androidx.appfunctions.AppFunctionPermissionRequiredException import androidx.appfunctions.ExecuteAppFunctionResponse +import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata +import androidx.appfunctions.metadata.AppFunctionPackageMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionResponseMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import com.example.appfunctions.agent.data.AgentInternalTools import io.mockk.coEvery import io.mockk.every import io.mockk.mockk +import io.mockk.verify import kotlinx.coroutines.test.runTest +import org.json.JSONObject import org.junit.Assert.assertEquals import org.junit.Assert.assertTrue import org.junit.Test @@ -34,9 +47,17 @@ import org.robolectric.annotation.Config @RunWith(RobolectricTestRunner::class) @Config(sdk = [34]) class ExecuteAppFunctionUseCaseTest { + private val context = mockk(relaxed = true) private val appFunctionManager = mockk() private val convertAppFunctionDataToJsonUseCase = mockk() - private val useCase = ExecuteAppFunctionUseCase(appFunctionManager, convertAppFunctionDataToJsonUseCase) + private val agentInternalTools = mockk(relaxed = true) + private val useCase = + ExecuteAppFunctionUseCase( + context = context, + appFunctionManager = appFunctionManager, + convertAppFunctionDataToJsonUseCase = convertAppFunctionDataToJsonUseCase, + agentInternalTools = agentInternalTools, + ) @Test fun `invoke returns Error with original AppFunctionException when response is Error`() = @@ -44,10 +65,12 @@ class ExecuteAppFunctionUseCaseTest { val function = mockk(relaxed = true) every { function.packageName } returns "com.example.app" every { function.id } returns "test_function" + every { function.parameters } returns emptyList() val parameters = mockk() val appException = AppFunctionPermissionRequiredException("Permission needed") - coEvery { appFunctionManager.executeAppFunction(any()) } returns ExecuteAppFunctionResponse.Error(appException) + coEvery { appFunctionManager.executeAppFunction(any()) } returns + ExecuteAppFunctionResponse.Error(appException) val result = useCase(function, parameters) @@ -62,6 +85,7 @@ class ExecuteAppFunctionUseCaseTest { val function = mockk(relaxed = true) every { function.packageName } returns "com.example.app" every { function.id } returns "test_function" + every { function.parameters } returns emptyList() val parameters = mockk() val runtimeException = RuntimeException("Unexpected crash") @@ -73,4 +97,251 @@ class ExecuteAppFunctionUseCaseTest { val errorResult = result as ExecuteAppFunctionResult.Error assertEquals(runtimeException, errorResult.exception) } + + @Test + fun `invoke grants recursive read URI permissions for content URIs in AppFunctionData`() = + runTest { + val uriObjectType = + AppFunctionObjectTypeMetadata( + properties = mapOf("uri" to AppFunctionStringTypeMetadata(false)), + required = listOf("uri"), + qualifiedName = "android.net.Uri", + isNullable = false, + ) + val payloadObjectType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "nestedUris" to + AppFunctionArrayTypeMetadata( + itemType = AppFunctionStringTypeMetadata(false), + isNullable = false, + ), + ), + required = listOf("nestedUris"), + qualifiedName = null, + isNullable = false, + ) + val parametersMeta = + listOf( + AppFunctionParameterMetadata( + name = "directUri", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(false), + ), + AppFunctionParameterMetadata( + name = "uriObj", + isRequired = true, + dataType = uriObjectType, + ), + AppFunctionParameterMetadata( + name = "payload", + isRequired = true, + dataType = payloadObjectType, + ), + AppFunctionParameterMetadata( + name = "plainText", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(false), + ), + ) + val components = AppFunctionComponentsMetadata() + val function = + AppFunctionMetadata( + name = AppFunctionName("com.example.notes", "attachMedia"), + schema = null, + parameters = parametersMeta, + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(false), + description = "", + ), + description = "Attach media", + deprecation = null, + packageMetadata = + AppFunctionPackageMetadata( + packageName = "com.example.notes", + components = components, + ), + ) + + val directUri = "content://com.example.appfunctions.agent.fileprovider/cache/direct.png" + val objectUri = "content://com.example.appfunctions.agent.fileprovider/cache/object.png" + val nestedUri = "content://com.example.appfunctions.agent.fileprovider/cache/nested.jpg" + + val uriObjData = + AppFunctionData.Builder(uriObjectType, components) + .setString("uri", objectUri) + .build() + val payloadData = + AppFunctionData.Builder(payloadObjectType, components) + .setStringList("nestedUris", listOf(nestedUri, "https://example.com/ignore")) + .build() + val inputData = + AppFunctionData.Builder(parametersMeta, components) + .setString("directUri", directUri) + .setAppFunctionData("uriObj", uriObjData) + .setAppFunctionData("payload", payloadData) + .setString("plainText", "hello world") + .build() + + coEvery { appFunctionManager.executeAppFunction(any()) } returns + ExecuteAppFunctionResponse.Success(AppFunctionData.EMPTY) + every { + convertAppFunctionDataToJsonUseCase(any(), any(), any(), any()) + } returns """{"ok":true}""" + + val result = useCase(function, inputData) + + assertTrue(result is ExecuteAppFunctionResult.Data) + assertEquals("""{"ok":true}""", (result as ExecuteAppFunctionResult.Data).formattedJson) + + for (expectedUri in listOf(directUri, objectUri, nestedUri)) { + verify(exactly = 1) { + context.grantUriPermission( + eq("com.example.notes"), + match { it.toString() == expectedUri }, + eq(Intent.FLAG_GRANT_READ_URI_PERMISSION), + ) + } + } + } + + @Test + fun `invoke dispatches internal generateImage tool even when AppFunctionManager is null`() = + runTest { + val nullManagerUseCase = + ExecuteAppFunctionUseCase( + context = context, + appFunctionManager = null, + convertAppFunctionDataToJsonUseCase = convertAppFunctionDataToJsonUseCase, + agentInternalTools = agentInternalTools, + ) + val imageResult = + AgentInternalTools.GeneratedImageResult( + imageUri = "content://com.example.appfunctions.agent.fileprovider/cache/cat.png", + mimeType = "image/png", + prompt = "a cat", + ) + coEvery { agentInternalTools.generateImage("a cat", "1:1") } returns imageResult + + val parametersMeta = + listOf( + AppFunctionParameterMetadata( + name = "prompt", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(false), + ), + AppFunctionParameterMetadata( + name = "aspectRatio", + isRequired = false, + dataType = AppFunctionStringTypeMetadata(true), + ), + ) + val components = AppFunctionComponentsMetadata() + val imageResultType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "imageUri" to AppFunctionStringTypeMetadata(false), + "mimeType" to AppFunctionStringTypeMetadata(false), + "prompt" to AppFunctionStringTypeMetadata(false), + ), + required = listOf("imageUri", "mimeType", "prompt"), + isNullable = false, + qualifiedName = + "com.example.appfunctions.agent.data.AgentInternalTools.GeneratedImageResult", + ) + val function = + AppFunctionMetadata( + name = + AppFunctionName( + AgentInternalTools.INTERNAL_TOOL_PACKAGE, + "generateImage", + ), + schema = null, + parameters = parametersMeta, + response = + AppFunctionResponseMetadata( + valueType = imageResultType, + description = "", + ), + description = "Generate image", + deprecation = null, + packageMetadata = + AppFunctionPackageMetadata( + packageName = AgentInternalTools.INTERNAL_TOOL_PACKAGE, + components = components, + ), + ) + val inputData = + AppFunctionData.Builder(parametersMeta, components) + .setString("prompt", "a cat") + .setString("aspectRatio", "1:1") + .build() + + val result = nullManagerUseCase(function, inputData) + + assertTrue(result is ExecuteAppFunctionResult.Data) + val json = JSONObject((result as ExecuteAppFunctionResult.Data).formattedJson) + assertEquals(imageResult.imageUri, json.getString("imageUri")) + assertEquals("image/png", json.getString("mimeType")) + assertEquals("a cat", json.getString("prompt")) + } + + @Test + fun `invoke dispatches internal geocodeAddress tool and returns Error on missing required parameter`() = + runTest { + coEvery { agentInternalTools.geocodeAddress("London") } returns + AgentInternalTools.LatLng(51.5074, -0.1278) + + val parametersMeta = + listOf( + AppFunctionParameterMetadata( + name = "address", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(false), + ), + ) + val components = AppFunctionComponentsMetadata() + val function = + AppFunctionMetadata( + name = + AppFunctionName( + AgentInternalTools.INTERNAL_TOOL_PACKAGE, + "geocodeAddress", + ), + schema = null, + parameters = parametersMeta, + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(true), + description = "", + ), + description = "Geocode", + deprecation = null, + packageMetadata = + AppFunctionPackageMetadata( + packageName = AgentInternalTools.INTERNAL_TOOL_PACKAGE, + components = components, + ), + ) + val validData = + AppFunctionData.Builder(parametersMeta, components) + .setString("address", "London") + .build() + + val okResult = useCase(function, validData) + assertTrue(okResult is ExecuteAppFunctionResult.Data) + val json = JSONObject((okResult as ExecuteAppFunctionResult.Data).formattedJson) + assertEquals(51.5074, json.getDouble("latitude"), 0.0001) + assertEquals(-0.1278, json.getDouble("longitude"), 0.0001) + + val missingResult = useCase(function, AppFunctionData.EMPTY) + assertTrue(missingResult is ExecuteAppFunctionResult.Error) + assertEquals( + "Missing required parameter 'address'", + (missingResult as ExecuteAppFunctionResult.Error).exception.message, + ) + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCaseTest.kt new file mode 100644 index 00000000..0fda397a --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCaseTest.kt @@ -0,0 +1,240 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain.appfunction + +import androidx.appfunctions.AppFunctionManager +import androidx.appfunctions.AppFunctionName +import androidx.appfunctions.AppFunctionState +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionPackageMetadata +import com.example.appfunctions.agent.data.AgentInternalTools +import com.example.appfunctions.agent.data.SettingsRepository +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.emptyFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test + +class GetAppFunctionsUseCaseTest { + private val appFunctionManager: AppFunctionManager = mockk() + private val getAppFunctionStatesUseCase: GetAppFunctionStatesUseCase = mockk() + private val settingsRepository: SettingsRepository = mockk() + private val agentInternalTools: AgentInternalTools = mockk() + + private val disconnectedAppsFlow = MutableStateFlow>(emptySet()) + + private lateinit var useCase: GetAppFunctionsUseCase + + @Before + fun setUp() { + every { appFunctionManager.observeAppFunctions() } returns emptyFlow() + every { settingsRepository.disconnectedApps } returns disconnectedAppsFlow + every { agentInternalTools.getInternalToolsMetadata() } returns emptyList() + coEvery { getAppFunctionStatesUseCase(any()) } answers { + firstArg>().map { name -> + createState(name, isEnabled = true) + } + } + + useCase = + GetAppFunctionsUseCase( + appFunctionManager = appFunctionManager, + getAppFunctionStatesUseCase = getAppFunctionStatesUseCase, + settingsRepository = settingsRepository, + agentInternalTools = agentInternalTools, + ) + } + + @Test + fun `default params return all external functions and internal tools without state or connection filtering`() = + runTest { + val notesPkg = createPackage("com.example.notes") + val chatPkg = createPackage("com.example.chat") + val internalPkg = createPackage(AgentInternalTools.INTERNAL_TOOL_PACKAGE) + + val enabledTool = createTool(notesPkg, "com.example.notes.createNote") + val disabledTool = createTool(notesPkg, "com.example.notes.deleteNote") + val disconnectedTool = createTool(chatPkg, "com.example.chat.sendMessage") + val internalTool = createTool(internalPkg, "generateImage") + + disconnectedAppsFlow.value = setOf("com.example.chat") + coEvery { appFunctionManager.searchAppFunctions(any()) } returns + listOf(enabledTool, disabledTool, disconnectedTool) + every { agentInternalTools.getInternalToolsMetadata() } returns + listOf(internalTool to createState(internalTool.name, isEnabled = true)) + + val result = useCase().first() + + assertEquals(listOf(enabledTool, disabledTool), result[notesPkg]) + assertEquals(listOf(disconnectedTool), result[chatPkg]) + assertEquals(listOf(internalTool), result[internalPkg]) + coVerify(exactly = 0) { getAppFunctionStatesUseCase(any()) } + } + + @Test + fun `includeInternalTools false excludes internal tools`() = + runTest { + val notesPkg = createPackage("com.example.notes") + val internalPkg = createPackage(AgentInternalTools.INTERNAL_TOOL_PACKAGE) + val notesTool = createTool(notesPkg, "createNote") + val internalTool = createTool(internalPkg, "generateImage") + + coEvery { appFunctionManager.searchAppFunctions(any()) } returns listOf(notesTool) + every { agentInternalTools.getInternalToolsMetadata() } returns + listOf(internalTool to createState(internalTool.name, isEnabled = true)) + + val result = + useCase(GetAppFunctionsParams(includeInternalTools = false)).first() + + assertEquals(mapOf(notesPkg to listOf(notesTool)), result) + } + + @Test + fun `isEnabled and isConnected filters exclude disabled tools and disconnected apps while keeping internal tools`() = + runTest { + val notesPkg = createPackage("com.example.notes") + val chatPkg = createPackage("com.example.chat") + val internalPkg = createPackage(AgentInternalTools.INTERNAL_TOOL_PACKAGE) + + val enabledTool = createTool(notesPkg, "com.example.notes.createNote") + val disabledTool = createTool(notesPkg, "com.example.notes.deleteNote") + val disconnectedTool = createTool(chatPkg, "com.example.chat.sendMessage") + val internalTool = createTool(internalPkg, "generateImage") + + disconnectedAppsFlow.value = setOf("com.example.chat") + coEvery { appFunctionManager.searchAppFunctions(any()) } returns + listOf(enabledTool, disabledTool, disconnectedTool) + every { agentInternalTools.getInternalToolsMetadata() } returns + listOf(internalTool to createState(internalTool.name, isEnabled = true)) + coEvery { getAppFunctionStatesUseCase(any()) } answers { + firstArg>().map { name -> + createState(name, isEnabled = name != disabledTool.name) + } + } + + val enabledConnected = + useCase( + GetAppFunctionsParams( + isEnabled = true, + isConnected = true, + includeInternalTools = true, + ), + ).first() + + assertEquals( + mapOf( + notesPkg to listOf(enabledTool), + internalPkg to listOf(internalTool), + ), + enabledConnected, + ) + + val disabledOnly = + useCase(GetAppFunctionsParams(isEnabled = false)).first() + assertEquals(mapOf(notesPkg to listOf(disabledTool)), disabledOnly) + + val disconnectedOnly = + useCase(GetAppFunctionsParams(isConnected = false)).first() + assertEquals(mapOf(chatPkg to listOf(disconnectedTool)), disconnectedOnly) + } + + @Test + fun `targetPackageNames scopes returned functions strictly to requested packages`() = + runTest { + val notesPkg = createPackage("com.example.notes") + val calendarPkg = createPackage("com.example.calendar") + val internalPkg = createPackage(AgentInternalTools.INTERNAL_TOOL_PACKAGE) + + val notesTool = createTool(notesPkg, "createNote") + val calendarTool = createTool(calendarPkg, "createEvent") + val internalTool = createTool(internalPkg, "geocodeAddress") + + coEvery { appFunctionManager.searchAppFunctions(any()) } returns + listOf(notesTool, calendarTool) + every { agentInternalTools.getInternalToolsMetadata() } returns + listOf(internalTool to createState(internalTool.name, isEnabled = true)) + + val result = + useCase( + GetAppFunctionsParams( + targetPackageNames = setOf("com.example.calendar"), + isEnabled = true, + isConnected = true, + includeInternalTools = true, + ), + ).first() + + assertEquals(mapOf(calendarPkg to listOf(calendarTool)), result) + } + + @Test + fun `null AppFunctionManager still emits internal tools when includeInternalTools is true`() = + runTest { + val internalPkg = createPackage(AgentInternalTools.INTERNAL_TOOL_PACKAGE) + val internalTool = createTool(internalPkg, "getCurrentLocation") + every { agentInternalTools.getInternalToolsMetadata() } returns + listOf(internalTool to createState(internalTool.name, isEnabled = true)) + + val nullManagerUseCase = + GetAppFunctionsUseCase( + appFunctionManager = null, + getAppFunctionStatesUseCase = getAppFunctionStatesUseCase, + settingsRepository = settingsRepository, + agentInternalTools = agentInternalTools, + ) + + val withInternal = nullManagerUseCase().first() + assertEquals(mapOf(internalPkg to listOf(internalTool)), withInternal) + + val withoutInternal = + nullManagerUseCase(GetAppFunctionsParams(includeInternalTools = false)).first() + assertTrue(withoutInternal.isEmpty()) + } + + private fun createPackage(packageName: String): AppFunctionPackageMetadata = + AppFunctionPackageMetadata( + packageName = packageName, + components = AppFunctionComponentsMetadata(), + ) + + private fun createTool( + packageMetadata: AppFunctionPackageMetadata, + id: String, + ): AppFunctionMetadata = + mockk(relaxed = true).apply { + every { this@apply.packageName } returns packageMetadata.packageName + every { this@apply.id } returns id + every { this@apply.name } returns AppFunctionName(packageMetadata.packageName, id) + every { this@apply.packageMetadata } returns packageMetadata + } + + private fun createState( + name: AppFunctionName, + isEnabled: Boolean, + ): AppFunctionState = + mockk().apply { + every { functionName } returns name + every { this@apply.isEnabled } returns isEnabled + } +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetConnectedAppsUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetConnectedAppsUseCaseTest.kt index 4faa69b5..5b944842 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetConnectedAppsUseCaseTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetConnectedAppsUseCaseTest.kt @@ -61,7 +61,9 @@ class GetConnectedAppsUseCaseTest { app2Metadata to emptyList(), ) - every { mockGetAppFunctionsUseCase() } returns flowOf(appFunctionsMap) + every { + mockGetAppFunctionsUseCase(GetAppFunctionsParams(includeInternalTools = false)) + } returns flowOf(appFunctionsMap) fakeRepository.setAppConnected("com.example.app2", false) val appInfo1 = AppInfo("com.example.app1", "App 1", null) @@ -87,7 +89,9 @@ class GetConnectedAppsUseCaseTest { val appFunctionsMap = mapOf(app1Metadata to emptyList()) - every { mockGetAppFunctionsUseCase() } returns flowOf(appFunctionsMap) + every { + mockGetAppFunctionsUseCase(GetAppFunctionsParams(includeInternalTools = false)) + } returns flowOf(appFunctionsMap) fakeRepository.setAppConnected("com.example.app1", true) every { mockGetInstalledAppsUseCase() } returns emptyList() diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt index 27a48055..a4aba8a0 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt @@ -25,6 +25,7 @@ import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentOrchestrator import com.example.appfunctions.agent.domain.AgentStatus +import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsParams import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase import com.example.appfunctions.agent.domain.chat.GetChatHistoryUseCase @@ -91,14 +92,15 @@ class AgentDemoViewModelTest { getAppFunctionsUseCase = mockk() every { getInstalledAppsUseCase() } returns emptyList() - every { getAppFunctionsUseCase() } returns flowOf(emptyMap()) + every { + getAppFunctionsUseCase(GetAppFunctionsParams(includeInternalTools = false)) + } returns flowOf(emptyMap()) every { manageThreadsUseCase.getThreads() } returns threadsFlow every { settingsRepository.selectedProvider } returns selectedProviderFlow every { agentOrchestrator.status } returns agentStatusFlow every { getChatHistoryUseCase(any()) } returns messagesFlow every { observeActivePendingIntentsUseCase() } returns activePendingActionIdsFlow - every { observeActivePendingIntentsUseCase() } returns activePendingActionIdsFlow coEvery { agentOrchestrator.observeAndProcessMessages(any()) } returns Unit } @@ -123,20 +125,7 @@ class AgentDemoViewModelTest { manageThreadsUseCase.updateThreadModel("thread-1", LlmModel.GEMINI_3_1_PRO_PREVIEW) } returns Unit - viewModel = - AgentDemoViewModel( - savedStateHandle, - getChatHistoryUseCase, - manageThreadsUseCase, - sendMessageUseCase, - agentOrchestrator, - settingsRepository, - observeActivePendingIntentsUseCase, - launchPendingIntentUseCase, - consumePendingIntentUseCase, - getInstalledAppsUseCase, - getAppFunctionsUseCase, - ) + viewModel = createViewModel() viewModel.onEvent(AgentUiEvent.OnModelSelected(LlmModel.GEMINI_3_1_PRO_PREVIEW)) @@ -151,20 +140,7 @@ class AgentDemoViewModelTest { threadsFlow.value = emptyList() coEvery { manageThreadsUseCase.createThread(LlmModel.DEFAULT) } returns "thread-new" - viewModel = - AgentDemoViewModel( - savedStateHandle, - getChatHistoryUseCase, - manageThreadsUseCase, - sendMessageUseCase, - agentOrchestrator, - settingsRepository, - observeActivePendingIntentsUseCase, - launchPendingIntentUseCase, - consumePendingIntentUseCase, - getInstalledAppsUseCase, - getAppFunctionsUseCase, - ) + viewModel = createViewModel() coVerify { manageThreadsUseCase.createThread(LlmModel.DEFAULT) } } @@ -182,20 +158,7 @@ class AgentDemoViewModelTest { threadsFlow.value = listOf(existingThread) selectedProviderFlow.value = LlmProviderName.GEMINI - viewModel = - AgentDemoViewModel( - savedStateHandle, - getChatHistoryUseCase, - manageThreadsUseCase, - sendMessageUseCase, - agentOrchestrator, - settingsRepository, - observeActivePendingIntentsUseCase, - launchPendingIntentUseCase, - consumePendingIntentUseCase, - getInstalledAppsUseCase, - getAppFunctionsUseCase, - ) + viewModel = createViewModel() coVerify(exactly = 0) { manageThreadsUseCase.createThread(any()) } assertEquals( @@ -217,20 +180,7 @@ class AgentDemoViewModelTest { threadsFlow.value = listOf(existingThread) selectedProviderFlow.value = LlmProviderName.GEMINI - viewModel = - AgentDemoViewModel( - savedStateHandle, - getChatHistoryUseCase, - manageThreadsUseCase, - sendMessageUseCase, - agentOrchestrator, - settingsRepository, - observeActivePendingIntentsUseCase, - launchPendingIntentUseCase, - consumePendingIntentUseCase, - getInstalledAppsUseCase, - getAppFunctionsUseCase, - ) + viewModel = createViewModel() coEvery { sendMessageUseCase(any(), any(), any(), any()) } returns Unit @@ -259,20 +209,7 @@ class AgentDemoViewModelTest { threadsFlow.value = listOf(existingThread) selectedProviderFlow.value = LlmProviderName.GEMINI - viewModel = - AgentDemoViewModel( - savedStateHandle, - getChatHistoryUseCase, - manageThreadsUseCase, - sendMessageUseCase, - agentOrchestrator, - settingsRepository, - observeActivePendingIntentsUseCase, - launchPendingIntentUseCase, - consumePendingIntentUseCase, - getInstalledAppsUseCase, - getAppFunctionsUseCase, - ) + viewModel = createViewModel() val message = MessageEntity( @@ -308,21 +245,23 @@ class AgentDemoViewModelTest { threadsFlow.value = listOf(newThread, oldThread) selectedProviderFlow.value = LlmProviderName.GEMINI - viewModel = - AgentDemoViewModel( - savedStateHandle, - getChatHistoryUseCase, - manageThreadsUseCase, - sendMessageUseCase, - agentOrchestrator, - settingsRepository, - observeActivePendingIntentsUseCase, - launchPendingIntentUseCase, - consumePendingIntentUseCase, - getInstalledAppsUseCase, - getAppFunctionsUseCase, - ) + viewModel = createViewModel() assertEquals(newThread, (viewModel.uiState.value as AgentUiState.Loaded).currentThread) } + + private fun createViewModel(): AgentDemoViewModel = + AgentDemoViewModel( + savedStateHandle, + getChatHistoryUseCase, + manageThreadsUseCase, + sendMessageUseCase, + agentOrchestrator, + settingsRepository, + observeActivePendingIntentsUseCase, + launchPendingIntentUseCase, + consumePendingIntentUseCase, + getInstalledAppsUseCase, + getAppFunctionsUseCase, + ) } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/debugging/DebuggingViewModelTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/debugging/DebuggingViewModelTest.kt index fb64fb53..fa40f3ca 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/debugging/DebuggingViewModelTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/debugging/DebuggingViewModelTest.kt @@ -32,6 +32,7 @@ import com.example.appfunctions.agent.domain.appfunction.AppInfo import com.example.appfunctions.agent.domain.appfunction.ConvertInputToAppFunctionDataUseCase import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionUseCase import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionStatesUseCase +import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsParams import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase import com.example.appfunctions.agent.domain.pendingintent.LaunchPendingIntentUseCase @@ -117,7 +118,9 @@ class DebuggingViewModelTest { val mockPackageMetadata = mockk() every { mockPackageMetadata.packageName } returns "com.example.app" val expectedAppInfo = AppInfo("com.example.app", "com.example.app", null) - every { mockGetAppFunctionsUseCase() } returns + every { + mockGetAppFunctionsUseCase(GetAppFunctionsParams(includeInternalTools = false)) + } returns flowOf(mapOf(mockPackageMetadata to listOf(mockMetadata))) every { mockGetInstalledAppsUseCase() } returns listOf(expectedAppInfo) @@ -153,7 +156,9 @@ class DebuggingViewModelTest { val mockPackageMetadata = mockk() val packageName = "com.example.app.resolved" every { mockPackageMetadata.packageName } returns packageName - every { mockGetAppFunctionsUseCase() } returns + every { + mockGetAppFunctionsUseCase(GetAppFunctionsParams(includeInternalTools = false)) + } returns flowOf(mapOf(mockPackageMetadata to listOf(mockMetadata))) val pm = context.packageManager @@ -216,7 +221,9 @@ class DebuggingViewModelTest { every { mockPackageMetadata1.packageName } returns "com.example.app1" val mockPackageMetadata2 = mockk() every { mockPackageMetadata2.packageName } returns "com.example.app2" - every { mockGetAppFunctionsUseCase() } returns + every { + mockGetAppFunctionsUseCase(GetAppFunctionsParams(includeInternalTools = false)) + } returns flowOf( mapOf( mockPackageMetadata1 to listOf(mockMetadata), @@ -261,7 +268,9 @@ class DebuggingViewModelTest { } val mockPackageMetadata = mockk() every { mockPackageMetadata.packageName } returns "com.example.app" - every { mockGetAppFunctionsUseCase() } returns + every { + mockGetAppFunctionsUseCase(GetAppFunctionsParams(includeInternalTools = false)) + } returns flowOf(mapOf(mockPackageMetadata to listOf(mockMetadata))) viewModel = @@ -297,7 +306,9 @@ class DebuggingViewModelTest { } val mockPackageMetadata = mockk() every { mockPackageMetadata.packageName } returns "com.example.app" - every { mockGetAppFunctionsUseCase() } returns + every { + mockGetAppFunctionsUseCase(GetAppFunctionsParams(includeInternalTools = false)) + } returns flowOf(mapOf(mockPackageMetadata to listOf(mockMetadata))) viewModel = @@ -341,7 +352,9 @@ class DebuggingViewModelTest { } val mockPackageMetadata = mockk() every { mockPackageMetadata.packageName } returns "com.example.app" - every { mockGetAppFunctionsUseCase() } returns + every { + mockGetAppFunctionsUseCase(GetAppFunctionsParams(includeInternalTools = false)) + } returns flowOf(mapOf(mockPackageMetadata to listOf(mockMetadata))) viewModel = @@ -380,7 +393,9 @@ class DebuggingViewModelTest { } val mockPackageMetadata = mockk() every { mockPackageMetadata.packageName } returns "com.example.app" - every { mockGetAppFunctionsUseCase() } returns + every { + mockGetAppFunctionsUseCase(GetAppFunctionsParams(includeInternalTools = false)) + } returns flowOf(mapOf(mockPackageMetadata to listOf(mockMetadata))) val pendingIntent = mockk() @@ -436,7 +451,9 @@ class DebuggingViewModelTest { val mockPackageMetadata = mockk() every { mockPackageMetadata.packageName } returns "com.example.app" - every { mockGetAppFunctionsUseCase() } returns + every { + mockGetAppFunctionsUseCase(GetAppFunctionsParams(includeInternalTools = false)) + } returns flowOf( mapOf( mockPackageMetadata to From 9e3f776c27846afe5e13c95dc1f87fc73512f95c Mon Sep 17 00:00:00 2001 From: Julian Toledo Date: Thu, 29 Oct 2026 15:08:57 +0100 Subject: [PATCH 2/2] refactor(agent): extract file download & URI permission use cases per PR review - Extract ResolveRemoteFileToLocalUriUseCase and FileDownloadHelper / FileDownloadHelperImpl out of ConvertInputToAppFunctionDataUseCase, chaining them in AgentOrchestrator. - Move executeInternalTool into AgentInternalTools so internal tool mapping stays encapsulated. - Extract GrantAppFunctionUriPermissionsUseCase out of ExecuteAppFunctionUseCase, removing runCatching around AppFunctionData getters, replacing .let with explicit locals, and using try/catch for grantUriPermission. - Refactor GetAppFunctionsUseCase into Flow operators (observeExternalFunctions, filterByConnection, filterByEnabledState) merging getInternalFunctions at the end. - Move nonNullArguments and extractMessageAttachment into AgentOrchestrator and update AgentOrchestratorTest to mock GetAppFunctionsUseCase directly. --- .../agent/data/AgentInternalTools.kt | 113 ++++ .../agent/data/FileDownloadHelper.kt | 86 +++ .../appfunctions/agent/di/DataModule.kt | 6 + .../agent/domain/AgentOrchestrator.kt | 73 ++- .../ConvertInputToAppFunctionDataUseCase.kt | 256 +------- .../appfunction/ExecuteAppFunctionUseCase.kt | 327 +---------- .../appfunction/GetAppFunctionsUseCase.kt | 147 ++--- .../GrantAppFunctionUriPermissionsUseCase.kt | 263 +++++++++ .../ResolveRemoteFileToLocalUriUseCase.kt | 200 +++++++ .../agent/data/AgentInternalToolsTest.kt | 183 ++++++ .../agent/domain/AgentOrchestratorTest.kt | 383 ++++-------- ...onvertInputToAppFunctionDataUseCaseTest.kt | 553 +++++++----------- .../ExecuteAppFunctionUseCaseTest.kt | 262 +-------- .../appfunction/GetAppFunctionsUseCaseTest.kt | 7 +- ...antAppFunctionUriPermissionsUseCaseTest.kt | 141 +++++ .../ResolveRemoteFileToLocalUriUseCaseTest.kt | 163 ++++++ 16 files changed, 1658 insertions(+), 1505 deletions(-) create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/data/FileDownloadHelper.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GrantAppFunctionUriPermissionsUseCase.kt create mode 100644 agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ResolveRemoteFileToLocalUriUseCase.kt create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/data/AgentInternalToolsTest.kt create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GrantAppFunctionUriPermissionsUseCaseTest.kt create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ResolveRemoteFileToLocalUriUseCaseTest.kt diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/AgentInternalTools.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/AgentInternalTools.kt index a58e9975..c52784da 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/AgentInternalTools.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/AgentInternalTools.kt @@ -24,8 +24,10 @@ import android.location.Geocoder import android.location.LocationManager import android.util.Base64 import androidx.annotation.RequiresApi +import androidx.appfunctions.AppFunctionData import androidx.appfunctions.AppFunctionName import androidx.appfunctions.AppFunctionState +import androidx.appfunctions.ExecuteAppFunctionResponse.Success.Companion.PROPERTY_RETURN_VALUE import androidx.appfunctions.metadata.AppFunctionComponentsMetadata import androidx.appfunctions.metadata.AppFunctionDoubleTypeMetadata import androidx.appfunctions.metadata.AppFunctionMetadata @@ -40,7 +42,9 @@ import androidx.datastore.core.DataStore import androidx.datastore.preferences.core.Preferences import androidx.datastore.preferences.core.stringPreferencesKey import com.example.appfunctions.agent.BuildConfig +import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult import dagger.hilt.android.qualifiers.ApplicationContext +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.first import kotlinx.coroutines.withContext @@ -456,6 +460,115 @@ class AgentInternalTools ) } + /** + * Executes an internal agent tool identified by [function] using [parameters] and maps the + * output into an [ExecuteAppFunctionResult]. + */ + suspend fun executeInternalTool( + function: AppFunctionMetadata, + parameters: AppFunctionData, + ): ExecuteAppFunctionResult { + return try { + when (function.id) { + "getCurrentLocation" -> { + val location = getCurrentLocation() + buildLatLngResult(function, location) + } + "geocodeAddress" -> { + val address = + parameters.getString("address") + ?: throw IllegalArgumentException( + "Missing required parameter 'address'", + ) + val location = geocodeAddress(address) + buildLatLngResult(function, location) + } + "generateImage" -> { + val prompt = + parameters.getString("prompt") + ?: throw IllegalArgumentException( + "Missing required parameter 'prompt'", + ) + val aspectRatio = parameters.getString("aspectRatio") + val result = generateImage(prompt, aspectRatio) + buildGeneratedImageResult(function, result) + } + else -> + throw IllegalArgumentException( + "Unknown internal tool: ${function.id}", + ) + } + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + ExecuteAppFunctionResult.Error(e) + } + } + + private fun buildLatLngResult( + function: AppFunctionMetadata, + location: LatLng?, + ): ExecuteAppFunctionResult.Data { + if (location == null) { + return ExecuteAppFunctionResult.Data( + data = AppFunctionData.EMPTY, + formattedJson = "null", + ) + } + val formattedJson = + JSONObject() + .apply { + put("latitude", location.latitude) + put("longitude", location.longitude) + } + .toString() + val valueType = function.response.valueType + val data = + if (valueType is AppFunctionObjectTypeMetadata) { + val locationData = + AppFunctionData.Builder(valueType, function.components) + .setDouble("latitude", location.latitude) + .setDouble("longitude", location.longitude) + .build() + AppFunctionData.Builder(function.response, function.components) + .setAppFunctionData(PROPERTY_RETURN_VALUE, locationData) + .build() + } else { + AppFunctionData.EMPTY + } + return ExecuteAppFunctionResult.Data(data = data, formattedJson = formattedJson) + } + + private fun buildGeneratedImageResult( + function: AppFunctionMetadata, + result: GeneratedImageResult, + ): ExecuteAppFunctionResult.Data { + val formattedJson = + JSONObject() + .apply { + put("imageUri", result.imageUri) + put("mimeType", result.mimeType) + put("prompt", result.prompt) + } + .toString() + val valueType = function.response.valueType + val data = + if (valueType is AppFunctionObjectTypeMetadata) { + val imageData = + AppFunctionData.Builder(valueType, function.components) + .setString("imageUri", result.imageUri) + .setString("mimeType", result.mimeType) + .setString("prompt", result.prompt) + .build() + AppFunctionData.Builder(function.response, function.components) + .setAppFunctionData(PROPERTY_RETURN_VALUE, imageData) + .build() + } else { + AppFunctionData.EMPTY + } + return ExecuteAppFunctionResult.Data(data = data, formattedJson = formattedJson) + } + /** Represents the latitude and longitude coordinates. */ data class LatLng( val latitude: Double, diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/FileDownloadHelper.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/FileDownloadHelper.kt new file mode 100644 index 00000000..6658d135 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/FileDownloadHelper.kt @@ -0,0 +1,86 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import android.content.Context +import androidx.core.content.FileProvider +import dagger.hilt.android.qualifiers.ApplicationContext +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.File +import java.net.HttpURLConnection +import java.net.URL +import java.util.UUID +import javax.inject.Inject +import javax.inject.Singleton + +/** Helper for downloading remote files into local cache storage. */ +interface FileDownloadHelper { + /** + * Downloads a remote `http://` or `https://` file at [url] into the local cache directory and + * returns its `content://` URI string. + */ + suspend fun downloadToLocalUri(url: String): String +} + +/** Default [FileDownloadHelper] implementation backed by [HttpURLConnection] and [FileProvider]. */ +@Singleton +class FileDownloadHelperImpl + @Inject + constructor( + @ApplicationContext private val context: Context, + ) : FileDownloadHelper { + override suspend fun downloadToLocalUri(url: String): String = + withContext(Dispatchers.IO) { + val parsedUrl = URL(url) + val connection = + (parsedUrl.openConnection() as HttpURLConnection).apply { + connectTimeout = 10000 + readTimeout = 15000 + } + try { + connection.connect() + val contentType = connection.contentType ?: "" + val ext = + when { + contentType.contains("png", ignoreCase = true) -> "png" + contentType.contains("jpeg", ignoreCase = true) || + contentType.contains("jpg", ignoreCase = true) -> "jpg" + contentType.contains("gif", ignoreCase = true) -> "gif" + contentType.contains("webp", ignoreCase = true) -> "webp" + url.substringAfterLast("/", "").contains(".") -> + url.substringAfterLast("/").substringAfterLast(".") + else -> "jpg" + } + val cacheDir = File(context.cacheDir, "file_references").apply { mkdirs() } + val file = File(cacheDir, "generated_${UUID.randomUUID()}.$ext") + connection.inputStream.use { input -> + file.outputStream().use { output -> + input.copyTo(output) + } + } + val contentUri = + FileProvider.getUriForFile( + context, + "${context.packageName}.fileprovider", + file, + ) + contentUri.toString() + } finally { + connection.disconnect() + } + } + } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt index 3757f9df..f92fae70 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt @@ -22,6 +22,8 @@ import androidx.datastore.preferences.preferencesDataStore import androidx.room.Room import com.example.appfunctions.agent.data.ChatRepository import com.example.appfunctions.agent.data.DataStoreSettingsRepository +import com.example.appfunctions.agent.data.FileDownloadHelper +import com.example.appfunctions.agent.data.FileDownloadHelperImpl import com.example.appfunctions.agent.data.InMemoryPendingIntentRepository import com.example.appfunctions.agent.data.PendingIntentRepository import com.example.appfunctions.agent.data.RoomChatRepository @@ -66,6 +68,10 @@ abstract class DataModule { impl: com.example.appfunctions.agent.data.LlmProviderFactoryImpl, ): com.example.appfunctions.agent.domain.LlmProviderFactory + @Binds + @Singleton + abstract fun bindFileDownloadHelper(impl: FileDownloadHelperImpl): FileDownloadHelper + companion object { @Provides @Singleton diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index 9ef924ae..5e059f26 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -31,6 +31,7 @@ import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResul import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionUseCase import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsParams import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase +import com.example.appfunctions.agent.domain.appfunction.ResolveRemoteFileToLocalUriUseCase import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase import com.example.appfunctions.agent.domain.chat.ObservePendingMessagesUseCase import com.example.appfunctions.agent.domain.chat.SendMessageUseCase @@ -69,6 +70,7 @@ class AgentOrchestrator private val llmProviderFactory: LlmProviderFactory, private val settingsRepository: SettingsRepository, private val getAppFunctionsUseCase: GetAppFunctionsUseCase, + private val resolveRemoteFileToLocalUriUseCase: ResolveRemoteFileToLocalUriUseCase, private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase, private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase, private val savePendingIntentUseCase: SavePendingIntentUseCase, @@ -116,10 +118,18 @@ class AgentOrchestrator return } + val targetPackage = message.targetPackageName + val targetPackageNames = + if (targetPackage != null) { + setOf(targetPackage) + } else { + null + } + val tools = getAppFunctionsUseCase( GetAppFunctionsParams( - targetPackageNames = message.targetPackageName?.let { setOf(it) }, + targetPackageNames = targetPackageNames, isEnabled = true, isConnected = true, includeInternalTools = true, @@ -344,11 +354,17 @@ class AgentOrchestrator } val inputs = nonNullArguments(toolCall.arguments) + val convertedLocalFileInput = + resolveRemoteFileToLocalUriUseCase( + parameters = matchingTool.parameters, + components = matchingTool.components, + inputs = inputs, + ) val appFunctionDataResult = convertInputToAppFunctionDataUseCase( parameters = matchingTool.parameters, components = matchingTool.components, - inputs = inputs, + inputs = convertedLocalFileInput, ) if (appFunctionDataResult.isFailure) { val exception = appFunctionDataResult.exceptionOrNull() @@ -384,8 +400,9 @@ class AgentOrchestrator result = executionResult.formattedJson, ), ) - extractMessageAttachment(executionResult.formattedJson)?.let { - capturedAttachments.add(it) + val attachment = extractMessageAttachment(executionResult.formattedJson) + if (attachment != null) { + capturedAttachments.add(attachment) } } @@ -449,31 +466,33 @@ class AgentOrchestrator ) } - private companion object { - const val TAG = "AgentOrchestrator" - } - } + private fun nonNullArguments(arguments: Map): Map = + buildMap { + for ((key, value) in arguments) { + if (value != null) { + put(key, value) + } + } + } -internal fun nonNullArguments(arguments: Map): Map = - buildMap { - for ((key, value) in arguments) { - if (value != null) { - put(key, value) + private fun extractMessageAttachment(jsonString: String): MessageAttachment? { + return try { + val json = Json.parseToJsonElement(jsonString) as? JsonObject ?: return null + val uri = json["imageUri"]?.jsonPrimitive?.contentOrNull + if (!uri.isNullOrBlank()) { + val mimeType = + json["mimeType"]?.jsonPrimitive?.contentOrNull?.takeIf { it.isNotBlank() } + ?: "image/png" + MessageAttachment(uri = uri, mimeType = mimeType) + } else { + null + } + } catch (e: Exception) { + null } } - } -internal fun extractMessageAttachment(jsonString: String): MessageAttachment? { - return runCatching { - val json = Json.parseToJsonElement(jsonString) as? JsonObject ?: return null - val uri = json["imageUri"]?.jsonPrimitive?.contentOrNull - if (!uri.isNullOrBlank()) { - val mimeType = - json["mimeType"]?.jsonPrimitive?.contentOrNull?.takeIf { it.isNotBlank() } - ?: "image/png" - MessageAttachment(uri = uri, mimeType = mimeType) - } else { - null + private companion object { + const val TAG = "AgentOrchestrator" } - }.getOrNull() -} + } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt index c0c05be2..fe1d9fc1 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt @@ -15,7 +15,6 @@ */ package com.example.appfunctions.agent.domain.appfunction -import android.content.Context import androidx.appfunctions.AppFunctionData import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata import androidx.appfunctions.metadata.AppFunctionBooleanTypeMetadata @@ -27,33 +26,14 @@ import androidx.appfunctions.metadata.AppFunctionIntTypeMetadata import androidx.appfunctions.metadata.AppFunctionLongTypeMetadata import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata import androidx.appfunctions.metadata.AppFunctionParameterMetadata -import androidx.appfunctions.metadata.AppFunctionParcelableTypeMetadata import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata -import androidx.core.content.FileProvider -import dagger.hilt.android.qualifiers.ApplicationContext -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext -import java.io.File -import java.io.IOException -import java.net.HttpURLConnection -import java.net.URL -import java.util.UUID import javax.inject.Inject -/** - * Use case to convert a map of raw inputs to type-safe [AppFunctionData], resolving remote - * `http://` and `https://` file reference URLs into local `content://` URIs when needed. - */ +/** Use case to convert a map of raw inputs to type-safe [AppFunctionData]. */ class ConvertInputToAppFunctionDataUseCase @Inject - constructor( - @ApplicationContext private val context: Context, - ) { - internal var remoteFileDownloader: suspend (Context, String) -> String = - ::downloadRemoteFileToContentUri - + constructor() { /** * Executes the use case. * @@ -62,172 +42,25 @@ class ConvertInputToAppFunctionDataUseCase * @param inputs The map of raw inputs from the UI or LLM tool call. * @return The converted [AppFunctionData]. */ - suspend operator fun invoke( + operator fun invoke( parameters: List, components: AppFunctionComponentsMetadata, inputs: Map, ): Result = - try { - val resolvedInputs = - resolveRemoteFileReferencesRecursively( - parametersMetadata = parameters, - components = components, - inputs = inputs, - ) - withContext(Dispatchers.Default) { - val builder = AppFunctionData.Builder(parameters, components) - for (parameter in parameters) { - val value = resolvedInputs[parameter.name] ?: continue - setParameterValue( - builder, - parameter.name, - parameter.dataType, - value, - components, - ) - } - Result.success(builder.build()) - } - } catch (e: CancellationException) { - throw e - } catch (e: Exception) { - Result.failure(e) - } - - private suspend fun resolveRemoteFileReferencesRecursively( - parametersMetadata: List, - components: AppFunctionComponentsMetadata, - inputs: Map, - ): Map { - val paramMap = parametersMetadata.associateBy { it.name } - return buildMap(inputs.size) { - for ((key, value) in inputs) { - val paramMeta = paramMap[key] - put( - key, - resolveValueRecursively( - dataType = paramMeta?.dataType, - components = components, - value = value, - paramName = key, - ), + runCatching { + val builder = AppFunctionData.Builder(parameters, components) + for (parameter in parameters) { + val value = inputs[parameter.name] ?: continue + setParameterValue( + builder, + parameter.name, + parameter.dataType, + value, + components, ) } + builder.build() } - } - - private suspend fun resolveValueRecursively( - dataType: AppFunctionDataTypeMetadata?, - components: AppFunctionComponentsMetadata, - value: Any, - paramName: String?, - ): Any { - val resolvedType = - if (dataType is AppFunctionReferenceTypeMetadata) { - components.dataTypes[dataType.referenceDataType] ?: dataType - } else { - dataType - } - - return when (value) { - is String -> { - val shouldResolve = - isFileReferenceParameter(paramName) || - isUriMetadata(dataType, components) - if ( - shouldResolve && - (value.startsWith("http://") || value.startsWith("https://")) - ) { - try { - remoteFileDownloader(context, value) - } catch (e: CancellationException) { - throw e - } catch (e: Exception) { - throw IOException( - "Failed to download remote file reference: ${e.message}", - e, - ) - } - } else { - value - } - } - is Map<*, *> -> { - val objectProperties = - (resolvedType as? AppFunctionObjectTypeMetadata)?.properties - buildMap(value.size) { - for (entry in value.entries) { - val k = entry.key as String - val propType = objectProperties?.get(k) - val resolvedProp = - entry.value?.let { - resolveValueRecursively( - dataType = propType, - components = components, - value = it, - paramName = k, - ) - } ?: "" - put(k, resolvedProp) - } - } - } - is List<*> -> { - val itemType = (resolvedType as? AppFunctionArrayTypeMetadata)?.itemType - buildList { - for (item in value) { - if (item != null) { - add( - resolveValueRecursively( - dataType = itemType, - components = components, - value = item, - paramName = paramName, - ), - ) - } - } - } - } - else -> value - } - } - - private fun isFileReferenceParameter(parameterName: String?): Boolean { - if (parameterName == null) return false - if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true - return parameterName.endsWith("Uri", ignoreCase = true) || - parameterName.endsWith("Uris", ignoreCase = true) - } - - private fun isUriMetadata( - dataType: AppFunctionDataTypeMetadata?, - components: AppFunctionComponentsMetadata, - ): Boolean { - if (dataType == null) return false - if ( - dataType is AppFunctionObjectTypeMetadata && - dataType.qualifiedName == ANDROID_NET_URI - ) { - return true - } - if ( - dataType is AppFunctionParcelableTypeMetadata && - dataType.qualifiedName == ANDROID_NET_URI - ) { - return true - } - if (dataType is AppFunctionReferenceTypeMetadata) { - if (dataType.referenceDataType == ANDROID_NET_URI) { - return true - } - val resolved = components.dataTypes[dataType.referenceDataType] - if (resolved != null && resolved !== dataType) { - return isUriMetadata(resolved, components) - } - } - return false - } private fun setParameterValue( builder: AppFunctionData.Builder, @@ -303,7 +136,7 @@ class ConvertInputToAppFunctionDataUseCase value: Any, components: AppFunctionComponentsMetadata, ): AppFunctionData { - if (objectType.qualifiedName == ANDROID_NET_URI && value is String) { + if (objectType.qualifiedName == "android.net.Uri" && value is String) { val uriPropertyName = objectType.properties.keys.firstOrNull() ?: "uri" return AppFunctionData.Builder(objectType, components) .setString(uriPropertyName, value) @@ -393,63 +226,4 @@ class ConvertInputToAppFunctionDataUseCase } } } - - companion object { - private const val ANDROID_NET_URI = "android.net.Uri" - - private val KNOWN_FILE_REFERENCE_PARAM_NAMES = - setOf( - "wallpaperUri", - "imageUri", - "attachmentUri", - "ringtoneUri", - "profilePictureUri", - "audioUri", - "voiceNoteUri", - ) - - private suspend fun downloadRemoteFileToContentUri( - context: Context, - urlString: String, - ): String = - withContext(Dispatchers.IO) { - val url = URL(urlString) - val connection = - (url.openConnection() as HttpURLConnection).apply { - connectTimeout = 10000 - readTimeout = 15000 - } - try { - connection.connect() - val contentType = connection.contentType ?: "" - val ext = - when { - contentType.contains("png", ignoreCase = true) -> "png" - contentType.contains("jpeg", ignoreCase = true) || - contentType.contains("jpg", ignoreCase = true) -> "jpg" - contentType.contains("gif", ignoreCase = true) -> "gif" - contentType.contains("webp", ignoreCase = true) -> "webp" - urlString.substringAfterLast("/", "").contains(".") -> - urlString.substringAfterLast("/").substringAfterLast(".") - else -> "jpg" - } - val cacheDir = File(context.cacheDir, "file_references").apply { mkdirs() } - val file = File(cacheDir, "generated_${UUID.randomUUID()}.$ext") - connection.inputStream.use { input -> - file.outputStream().use { output -> - input.copyTo(output) - } - } - val contentUri = - FileProvider.getUriForFile( - context, - "${context.packageName}.fileprovider", - file, - ) - contentUri.toString() - } finally { - connection.disconnect() - } - } - } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCase.kt index 1360b25f..a8c8e71f 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCase.kt @@ -17,30 +17,17 @@ package com.example.appfunctions.agent.domain.appfunction import android.app.AppInteractionAttribution import android.app.PendingIntent -import android.content.Context -import android.content.Intent -import android.net.Uri import android.os.Build import androidx.appfunctions.AppFunctionData import androidx.appfunctions.AppFunctionManager import androidx.appfunctions.ExecuteAppFunctionRequest import androidx.appfunctions.ExecuteAppFunctionResponse import androidx.appfunctions.ExecuteAppFunctionResponse.Success.Companion.PROPERTY_RETURN_VALUE -import androidx.appfunctions.metadata.AppFunctionAllOfTypeMetadata -import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata -import androidx.appfunctions.metadata.AppFunctionComponentsMetadata -import androidx.appfunctions.metadata.AppFunctionDataTypeMetadata import androidx.appfunctions.metadata.AppFunctionMetadata -import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata -import androidx.appfunctions.metadata.AppFunctionOneOfTypeMetadata import androidx.appfunctions.metadata.AppFunctionParcelableTypeMetadata -import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata -import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata import androidx.core.net.toUri import com.example.appfunctions.agent.data.AgentInternalTools -import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.CancellationException -import org.json.JSONObject import javax.inject.Inject /** @@ -50,10 +37,10 @@ import javax.inject.Inject class ExecuteAppFunctionUseCase @Inject constructor( - @ApplicationContext private val context: Context, private val appFunctionManager: AppFunctionManager?, private val convertAppFunctionDataToJsonUseCase: ConvertAppFunctionDataToJsonUseCase, private val agentInternalTools: AgentInternalTools, + private val grantAppFunctionUriPermissionsUseCase: GrantAppFunctionUriPermissionsUseCase, ) { /** * Executes the use case. @@ -69,7 +56,7 @@ class ExecuteAppFunctionUseCase threadId: String? = null, ): ExecuteAppFunctionResult { if (function.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) { - return executeInternalTool(function, parameters) + return agentInternalTools.executeInternalTool(function, parameters) } if (appFunctionManager == null) { @@ -78,7 +65,7 @@ class ExecuteAppFunctionUseCase ) } - grantContentUriPermissions(function, parameters) + grantAppFunctionUriPermissionsUseCase(function, parameters) val request = if (Build.VERSION.SDK_INT >= 37) { @@ -148,314 +135,6 @@ class ExecuteAppFunctionUseCase ExecuteAppFunctionResult.Error(e) } } - - private suspend fun executeInternalTool( - function: AppFunctionMetadata, - parameters: AppFunctionData, - ): ExecuteAppFunctionResult { - return try { - when (function.id) { - "getCurrentLocation" -> { - val location = agentInternalTools.getCurrentLocation() - buildLatLngResult(function, location) - } - "geocodeAddress" -> { - val address = - parameters.getString("address") - ?: throw IllegalArgumentException( - "Missing required parameter 'address'", - ) - val location = agentInternalTools.geocodeAddress(address) - buildLatLngResult(function, location) - } - "generateImage" -> { - val prompt = - parameters.getString("prompt") - ?: throw IllegalArgumentException( - "Missing required parameter 'prompt'", - ) - val aspectRatio = parameters.getString("aspectRatio") - val result = agentInternalTools.generateImage(prompt, aspectRatio) - buildGeneratedImageResult(function, result) - } - else -> - throw IllegalArgumentException( - "Unknown internal tool: ${function.id}", - ) - } - } catch (e: CancellationException) { - throw e - } catch (e: Exception) { - ExecuteAppFunctionResult.Error(e) - } - } - - private fun buildLatLngResult( - function: AppFunctionMetadata, - location: AgentInternalTools.LatLng?, - ): ExecuteAppFunctionResult.Data { - if (location == null) { - return ExecuteAppFunctionResult.Data( - data = AppFunctionData.EMPTY, - formattedJson = "null", - ) - } - val formattedJson = - JSONObject() - .apply { - put("latitude", location.latitude) - put("longitude", location.longitude) - } - .toString() - val valueType = function.response.valueType - val data = - if (valueType is AppFunctionObjectTypeMetadata) { - val locationData = - AppFunctionData.Builder(valueType, function.components) - .setDouble("latitude", location.latitude) - .setDouble("longitude", location.longitude) - .build() - AppFunctionData.Builder(function.response, function.components) - .setAppFunctionData(PROPERTY_RETURN_VALUE, locationData) - .build() - } else { - AppFunctionData.EMPTY - } - return ExecuteAppFunctionResult.Data(data = data, formattedJson = formattedJson) - } - - private fun buildGeneratedImageResult( - function: AppFunctionMetadata, - result: AgentInternalTools.GeneratedImageResult, - ): ExecuteAppFunctionResult.Data { - val formattedJson = - JSONObject() - .apply { - put("imageUri", result.imageUri) - put("mimeType", result.mimeType) - put("prompt", result.prompt) - } - .toString() - val valueType = function.response.valueType - val data = - if (valueType is AppFunctionObjectTypeMetadata) { - val imageData = - AppFunctionData.Builder(valueType, function.components) - .setString("imageUri", result.imageUri) - .setString("mimeType", result.mimeType) - .setString("prompt", result.prompt) - .build() - AppFunctionData.Builder(function.response, function.components) - .setAppFunctionData(PROPERTY_RETURN_VALUE, imageData) - .build() - } else { - AppFunctionData.EMPTY - } - return ExecuteAppFunctionResult.Data(data = data, formattedJson = formattedJson) - } - - private fun grantContentUriPermissions( - function: AppFunctionMetadata, - parameters: AppFunctionData, - ) { - for (parameter in function.parameters) { - grantUriPermissionsForProperty( - targetPackageName = function.packageName, - data = parameters, - name = parameter.name, - dataType = parameter.dataType, - components = function.components, - ) - } - } - - private fun grantUriPermissionsForProperty( - targetPackageName: String, - data: AppFunctionData, - name: String, - dataType: AppFunctionDataTypeMetadata, - components: AppFunctionComponentsMetadata, - ) { - when (dataType) { - is AppFunctionStringTypeMetadata -> { - runCatching { data.getString(name) } - .getOrNull() - ?.let { grantIfContentUri(targetPackageName, it) } - } - is AppFunctionParcelableTypeMetadata -> { - if (dataType.qualifiedName == Uri::class.java.name) { - runCatching { data.getParcelable(name, Uri::class.java) } - .getOrNull() - ?.let { grantIfContentUri(targetPackageName, it) } - } - } - is AppFunctionObjectTypeMetadata -> { - val nested = - runCatching { data.getAppFunctionData(name) }.getOrNull() ?: return - grantUriPermissionsForObject(targetPackageName, nested, dataType, components) - } - is AppFunctionAllOfTypeMetadata, - is AppFunctionOneOfTypeMetadata, - -> { - val nested = - runCatching { data.getAppFunctionData(name) }.getOrNull() ?: return - grantUriPermissionsForDataType(targetPackageName, nested, dataType, components) - } - is AppFunctionReferenceTypeMetadata -> { - val resolved = components.dataTypes[dataType.referenceDataType] ?: return - grantUriPermissionsForProperty( - targetPackageName = targetPackageName, - data = data, - name = name, - dataType = resolved, - components = components, - ) - } - is AppFunctionArrayTypeMetadata -> { - grantUriPermissionsForArray( - targetPackageName = targetPackageName, - data = data, - name = name, - itemType = dataType.itemType, - components = components, - ) - } - } - } - - private fun grantUriPermissionsForObject( - targetPackageName: String, - data: AppFunctionData, - objectType: AppFunctionObjectTypeMetadata, - components: AppFunctionComponentsMetadata, - ) { - if (objectType.qualifiedName == Uri::class.java.name && objectType.properties.isEmpty()) { - runCatching { data.getString("uri") } - .getOrNull() - ?.let { grantIfContentUri(targetPackageName, it) } - return - } - for ((propName, propType) in objectType.properties) { - grantUriPermissionsForProperty( - targetPackageName = targetPackageName, - data = data, - name = propName, - dataType = propType, - components = components, - ) - } - } - - private fun grantUriPermissionsForDataType( - targetPackageName: String, - data: AppFunctionData, - dataType: AppFunctionDataTypeMetadata, - components: AppFunctionComponentsMetadata, - ) { - when (dataType) { - is AppFunctionObjectTypeMetadata -> { - grantUriPermissionsForObject(targetPackageName, data, dataType, components) - } - is AppFunctionAllOfTypeMetadata -> { - for (subType in dataType.matchAll) { - grantUriPermissionsForDataType( - targetPackageName, - data, - subType, - components, - ) - } - } - is AppFunctionOneOfTypeMetadata -> { - for (candidate in dataType.matchOneOf) { - grantUriPermissionsForDataType( - targetPackageName, - data, - candidate, - components, - ) - } - } - is AppFunctionReferenceTypeMetadata -> { - val resolved = components.dataTypes[dataType.referenceDataType] ?: return - grantUriPermissionsForDataType(targetPackageName, data, resolved, components) - } - } - } - - private fun grantUriPermissionsForArray( - targetPackageName: String, - data: AppFunctionData, - name: String, - itemType: AppFunctionDataTypeMetadata, - components: AppFunctionComponentsMetadata, - ) { - when (itemType) { - is AppFunctionStringTypeMetadata -> { - runCatching { data.getStringList(name) } - .getOrNull() - ?.forEach { grantIfContentUri(targetPackageName, it) } - } - is AppFunctionParcelableTypeMetadata -> { - if (itemType.qualifiedName == Uri::class.java.name) { - runCatching { data.getParcelableList(name, Uri::class.java) } - .getOrNull() - ?.forEach { grantIfContentUri(targetPackageName, it) } - } - } - is AppFunctionReferenceTypeMetadata -> { - val resolved = components.dataTypes[itemType.referenceDataType] ?: return - grantUriPermissionsForArray( - targetPackageName = targetPackageName, - data = data, - name = name, - itemType = resolved, - components = components, - ) - } - is AppFunctionObjectTypeMetadata, - is AppFunctionAllOfTypeMetadata, - is AppFunctionOneOfTypeMetadata, - -> { - runCatching { data.getAppFunctionDataList(name) } - .getOrNull() - ?.forEach { itemData -> - grantUriPermissionsForDataType( - targetPackageName, - itemData, - itemType, - components, - ) - } - } - } - } - - private fun grantIfContentUri( - targetPackageName: String, - uriString: String, - ) { - if (uriString.startsWith("content://")) { - runCatching { - grantIfContentUri(targetPackageName, Uri.parse(uriString)) - } - } - } - - private fun grantIfContentUri( - targetPackageName: String, - uri: Uri, - ) { - if (uri.scheme == "content") { - runCatching { - context.grantUriPermission( - targetPackageName, - uri, - Intent.FLAG_GRANT_READ_URI_PERMISSION, - ) - } - } - } } /** Sealed class representing the result of an AppFunction execution. */ diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCase.kt index 095d6982..d96318c8 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCase.kt @@ -27,6 +27,7 @@ import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.debounce import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.mapLatest import kotlinx.coroutines.flow.onStart import javax.inject.Inject @@ -60,108 +61,78 @@ class GetAppFunctionsUseCase * @return A Flow emitting a map of [AppFunctionPackageMetadata] to their matching * [AppFunctionMetadata] list. */ - @OptIn(FlowPreview::class, ExperimentalCoroutinesApi::class) operator fun invoke( params: GetAppFunctionsParams = GetAppFunctionsParams(), ): Flow>> { - val externalTargetPackages = - params.targetPackageNames?.minus(AgentInternalTools.INTERNAL_TOOL_PACKAGE) - val externalFunctionsFlow: Flow> = - if ( - appFunctionManager == null || - (externalTargetPackages != null && externalTargetPackages.isEmpty()) - ) { - flowOf(emptyList()) - } else { - val searchSpec = AppFunctionSearchSpec(packageNames = externalTargetPackages) - appFunctionManager - .observeAppFunctions() - .debounce(500.milliseconds) - .mapLatest { appFunctionManager.searchAppFunctions(searchSpec) } - .onStart { emit(appFunctionManager.searchAppFunctions(searchSpec)) } + val internalFunctions = getInternalFunctions(params) + return observeExternalFunctions(params) + .filterByConnection(params.isConnected) + .filterByEnabledState(params.isEnabled) + .map { externalFunctions -> + (externalFunctions + internalFunctions) + .sortedByDescending { it.id.startsWith(it.packageName) } + .distinctBy { it.packageName to it.id } + .groupBy(AppFunctionMetadata::packageMetadata) } + } - val disconnectedAppsFlow: Flow> = - if (params.isConnected != null) { - settingsRepository.disconnectedApps - } else { - flowOf(emptySet()) + private fun getInternalFunctions(params: GetAppFunctionsParams): List { + if (!params.includeInternalTools || params.isConnected == false) { + return emptyList() + } + return agentInternalTools + .getInternalToolsMetadata() + .filter { (metadata, state) -> + val matchesPackage = + params.targetPackageNames == null || + metadata.packageName in params.targetPackageNames + val matchesEnabled = + params.isEnabled == null || state.isEnabled == params.isEnabled + matchesPackage && matchesEnabled } + .map { it.first } + } - return combine(externalFunctionsFlow, disconnectedAppsFlow) { - externalFunctions, - disconnectedApps, - -> - filterAndGroup( - externalFunctions = externalFunctions, - disconnectedApps = disconnectedApps, - params = params, - ) + @OptIn(FlowPreview::class, ExperimentalCoroutinesApi::class) + private fun observeExternalFunctions(params: GetAppFunctionsParams): Flow> { + if (appFunctionManager == null) { + return flowOf(emptyList()) } + val searchSpec = AppFunctionSearchSpec(packageNames = params.targetPackageNames) + return appFunctionManager + .observeAppFunctions() + .debounce(500.milliseconds) + .mapLatest { appFunctionManager.searchAppFunctions(searchSpec) } + .onStart { emit(appFunctionManager.searchAppFunctions(searchSpec)) } } - private suspend fun filterAndGroup( - externalFunctions: List, - disconnectedApps: Set, - params: GetAppFunctionsParams, - ): Map> { - val internalEntries = - if (params.includeInternalTools) { - agentInternalTools.getInternalToolsMetadata() - } else { - emptyList() - } - val allCandidates = externalFunctions + internalEntries.map { it.first } - - val preFiltered = - allCandidates.filter { metadata -> - val matchesPackage = - params.targetPackageNames == null || - metadata.packageName in params.targetPackageNames - val matchesConnection = - if (params.isConnected == null) { - true - } else { - val connected = - metadata.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE || - metadata.packageName !in disconnectedApps - connected == params.isConnected - } - matchesPackage && matchesConnection + private fun Flow>.filterByConnection(isConnected: Boolean?): Flow> { + if (isConnected == null) { + return this + } + return combine(this, settingsRepository.disconnectedApps) { functions, disconnectedApps -> + functions.filter { metadata -> + val connected = metadata.packageName !in disconnectedApps + connected == isConnected } + } + } - val enabledFiltered = - if (params.isEnabled == null) { - preFiltered + private fun Flow>.filterByEnabledState(isEnabled: Boolean?): Flow> { + if (isEnabled == null) { + return this + } + return map { functions -> + if (functions.isEmpty()) { + emptyList() } else { - val externalFunctionNames = - preFiltered - .filter { it.packageName != AgentInternalTools.INTERNAL_TOOL_PACKAGE } - .map { it.name } - val externalStates = - if (externalFunctionNames.isNotEmpty()) { - getAppFunctionStatesUseCase(externalFunctionNames) - } else { - emptyList() - } - val stateMap = - (externalStates + internalEntries.map { it.second }) - .associateBy { it.functionName } - - preFiltered.filter { metadata -> - val enabled = - if (metadata.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) { - stateMap[metadata.name]?.isEnabled ?: true - } else { - stateMap[metadata.name]?.isEnabled ?: false - } - enabled == params.isEnabled + val states = getAppFunctionStatesUseCase(functions.map { it.name }) + val stateMap = states.associateBy { it.functionName } + functions.filter { metadata -> + val enabled = stateMap[metadata.name]?.isEnabled ?: false + enabled == isEnabled } } - - return enabledFiltered - .sortedByDescending { metadata -> metadata.id.startsWith(metadata.packageName) } - .distinctBy { metadata -> metadata.packageName to metadata.id } - .groupBy(AppFunctionMetadata::packageMetadata) + } } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GrantAppFunctionUriPermissionsUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GrantAppFunctionUriPermissionsUseCase.kt new file mode 100644 index 00000000..bee3c76b --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/GrantAppFunctionUriPermissionsUseCase.kt @@ -0,0 +1,263 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain.appfunction + +import android.content.Context +import android.content.Intent +import android.net.Uri +import android.util.Log +import androidx.appfunctions.AppFunctionData +import androidx.appfunctions.metadata.AppFunctionAllOfTypeMetadata +import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionDataTypeMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata +import androidx.appfunctions.metadata.AppFunctionOneOfTypeMetadata +import androidx.appfunctions.metadata.AppFunctionParcelableTypeMetadata +import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import dagger.hilt.android.qualifiers.ApplicationContext +import javax.inject.Inject + +/** + * Use case to recursively traverse [AppFunctionData] parameters and grant read URI permissions on + * any `content://` URIs to the target AppFunction package. + */ +class GrantAppFunctionUriPermissionsUseCase + @Inject + constructor( + @ApplicationContext private val context: Context, + ) { + /** + * Grants `FLAG_GRANT_READ_URI_PERMISSION` to [function]'s target package for all + * `content://` URIs contained in [parameters]. + */ + operator fun invoke( + function: AppFunctionMetadata, + parameters: AppFunctionData, + ) { + for (parameter in function.parameters) { + grantUriPermissionsForProperty( + targetPackageName = function.packageName, + data = parameters, + name = parameter.name, + dataType = parameter.dataType, + components = function.components, + ) + } + } + + private fun grantUriPermissionsForProperty( + targetPackageName: String, + data: AppFunctionData, + name: String, + dataType: AppFunctionDataTypeMetadata, + components: AppFunctionComponentsMetadata, + ) { + when (dataType) { + is AppFunctionStringTypeMetadata -> { + val uriString = data.getString(name) + if (uriString != null) { + grantIfContentUri(targetPackageName, uriString) + } + } + is AppFunctionParcelableTypeMetadata -> { + if (dataType.qualifiedName == Uri::class.java.name) { + val uri = data.getParcelable(name, Uri::class.java) + if (uri != null) { + grantIfContentUri(targetPackageName, uri) + } + } + } + is AppFunctionObjectTypeMetadata -> { + val nested = data.getAppFunctionData(name) ?: return + grantUriPermissionsForObject(targetPackageName, nested, dataType, components) + } + is AppFunctionAllOfTypeMetadata, + is AppFunctionOneOfTypeMetadata, + -> { + val nested = data.getAppFunctionData(name) ?: return + grantUriPermissionsForDataType(targetPackageName, nested, dataType, components) + } + is AppFunctionReferenceTypeMetadata -> { + val resolved = components.dataTypes[dataType.referenceDataType] ?: return + grantUriPermissionsForProperty( + targetPackageName = targetPackageName, + data = data, + name = name, + dataType = resolved, + components = components, + ) + } + is AppFunctionArrayTypeMetadata -> { + grantUriPermissionsForArray( + targetPackageName = targetPackageName, + data = data, + name = name, + itemType = dataType.itemType, + components = components, + ) + } + } + } + + private fun grantUriPermissionsForObject( + targetPackageName: String, + data: AppFunctionData, + objectType: AppFunctionObjectTypeMetadata, + components: AppFunctionComponentsMetadata, + ) { + if (objectType.qualifiedName == Uri::class.java.name && objectType.properties.isEmpty()) { + val uriString = data.getString("uri") + if (uriString != null) { + grantIfContentUri(targetPackageName, uriString) + } + return + } + for ((propName, propType) in objectType.properties) { + grantUriPermissionsForProperty( + targetPackageName = targetPackageName, + data = data, + name = propName, + dataType = propType, + components = components, + ) + } + } + + private fun grantUriPermissionsForDataType( + targetPackageName: String, + data: AppFunctionData, + dataType: AppFunctionDataTypeMetadata, + components: AppFunctionComponentsMetadata, + ) { + when (dataType) { + is AppFunctionObjectTypeMetadata -> { + grantUriPermissionsForObject(targetPackageName, data, dataType, components) + } + is AppFunctionAllOfTypeMetadata -> { + for (subType in dataType.matchAll) { + grantUriPermissionsForDataType( + targetPackageName, + data, + subType, + components, + ) + } + } + is AppFunctionOneOfTypeMetadata -> { + for (candidate in dataType.matchOneOf) { + grantUriPermissionsForDataType( + targetPackageName, + data, + candidate, + components, + ) + } + } + is AppFunctionReferenceTypeMetadata -> { + val resolved = components.dataTypes[dataType.referenceDataType] ?: return + grantUriPermissionsForDataType(targetPackageName, data, resolved, components) + } + } + } + + private fun grantUriPermissionsForArray( + targetPackageName: String, + data: AppFunctionData, + name: String, + itemType: AppFunctionDataTypeMetadata, + components: AppFunctionComponentsMetadata, + ) { + when (itemType) { + is AppFunctionStringTypeMetadata -> { + val uriStrings = data.getStringList(name) + if (uriStrings != null) { + for (uriString in uriStrings) { + grantIfContentUri(targetPackageName, uriString) + } + } + } + is AppFunctionParcelableTypeMetadata -> { + if (itemType.qualifiedName == Uri::class.java.name) { + val uris = data.getParcelableList(name, Uri::class.java) + if (uris != null) { + for (uri in uris) { + grantIfContentUri(targetPackageName, uri) + } + } + } + } + is AppFunctionReferenceTypeMetadata -> { + val resolved = components.dataTypes[itemType.referenceDataType] ?: return + grantUriPermissionsForArray( + targetPackageName = targetPackageName, + data = data, + name = name, + itemType = resolved, + components = components, + ) + } + is AppFunctionObjectTypeMetadata, + is AppFunctionAllOfTypeMetadata, + is AppFunctionOneOfTypeMetadata, + -> { + val items = data.getAppFunctionDataList(name) + if (items != null) { + for (itemData in items) { + grantUriPermissionsForDataType( + targetPackageName, + itemData, + itemType, + components, + ) + } + } + } + } + } + + private fun grantIfContentUri( + targetPackageName: String, + uriString: String, + ) { + if (uriString.startsWith("content://")) { + grantIfContentUri(targetPackageName, Uri.parse(uriString)) + } + } + + private fun grantIfContentUri( + targetPackageName: String, + uri: Uri, + ) { + if (uri.scheme == "content") { + try { + context.grantUriPermission( + targetPackageName, + uri, + Intent.FLAG_GRANT_READ_URI_PERMISSION, + ) + } catch (e: Exception) { + Log.w(TAG, "Failed to grant URI permission for $uri to $targetPackageName", e) + } + } + } + + private companion object { + const val TAG = "GrantAppFunctionUriPermissions" + } + } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ResolveRemoteFileToLocalUriUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ResolveRemoteFileToLocalUriUseCase.kt new file mode 100644 index 00000000..8fcc4c25 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ResolveRemoteFileToLocalUriUseCase.kt @@ -0,0 +1,200 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain.appfunction + +import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionDataTypeMetadata +import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionParcelableTypeMetadata +import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata +import com.example.appfunctions.agent.data.FileDownloadHelper +import kotlinx.coroutines.CancellationException +import java.io.IOException +import javax.inject.Inject + +/** + * Use case to resolve remote `http://` and `https://` file reference URLs in raw input maps into + * local `content://` URIs via [FileDownloadHelper]. + */ +class ResolveRemoteFileToLocalUriUseCase + @Inject + constructor( + private val fileDownloadHelper: FileDownloadHelper, + ) { + /** + * Resolves remote file reference URLs in [inputs] to local `content://` URIs. + * + * @param parameters The list of parameter metadata for the function. + * @param components The component metadata for resolving references. + * @param inputs The map of raw inputs. + * @return A new input map with remote file reference URLs replaced by local `content://` + * URIs. + */ + suspend operator fun invoke( + parameters: List, + components: AppFunctionComponentsMetadata, + inputs: Map, + ): Map { + val paramMap = parameters.associateBy { it.name } + return buildMap(inputs.size) { + for ((key, value) in inputs) { + val paramMeta = paramMap[key] + put( + key, + resolveValueRecursively( + dataType = paramMeta?.dataType, + components = components, + value = value, + paramName = key, + ), + ) + } + } + } + + private suspend fun resolveValueRecursively( + dataType: AppFunctionDataTypeMetadata?, + components: AppFunctionComponentsMetadata, + value: Any, + paramName: String?, + ): Any { + val resolvedType = + if (dataType is AppFunctionReferenceTypeMetadata) { + components.dataTypes[dataType.referenceDataType] ?: dataType + } else { + dataType + } + + return when (value) { + is String -> { + val shouldResolve = + isFileReferenceParameter(paramName) || + isUriMetadata(dataType, components) + if ( + shouldResolve && + (value.startsWith("http://") || value.startsWith("https://")) + ) { + try { + fileDownloadHelper.downloadToLocalUri(value) + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + throw IOException( + "Failed to download remote file reference: ${e.message}", + e, + ) + } + } else { + value + } + } + is Map<*, *> -> { + val objectProperties = + (resolvedType as? AppFunctionObjectTypeMetadata)?.properties + buildMap(value.size) { + for (entry in value.entries) { + val key = entry.key as String + val propType = objectProperties?.get(key) + val entryValue = entry.value + val resolvedProp = + if (entryValue != null) { + resolveValueRecursively( + dataType = propType, + components = components, + value = entryValue, + paramName = key, + ) + } else { + "" + } + put(key, resolvedProp) + } + } + } + is List<*> -> { + val itemType = (resolvedType as? AppFunctionArrayTypeMetadata)?.itemType + buildList { + for (item in value) { + if (item != null) { + add( + resolveValueRecursively( + dataType = itemType, + components = components, + value = item, + paramName = paramName, + ), + ) + } + } + } + } + else -> value + } + } + + private fun isFileReferenceParameter(parameterName: String?): Boolean { + if (parameterName == null) return false + if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true + return parameterName.endsWith("Uri", ignoreCase = true) || + parameterName.endsWith("Uris", ignoreCase = true) + } + + private fun isUriMetadata( + dataType: AppFunctionDataTypeMetadata?, + components: AppFunctionComponentsMetadata, + ): Boolean { + if (dataType == null) return false + if ( + dataType is AppFunctionObjectTypeMetadata && + dataType.qualifiedName == ANDROID_NET_URI + ) { + return true + } + if ( + dataType is AppFunctionParcelableTypeMetadata && + dataType.qualifiedName == ANDROID_NET_URI + ) { + return true + } + if (dataType is AppFunctionReferenceTypeMetadata) { + if (dataType.referenceDataType == ANDROID_NET_URI) { + return true + } + val resolved = components.dataTypes[dataType.referenceDataType] + if (resolved != null && resolved !== dataType) { + return isUriMetadata(resolved, components) + } + } + return false + } + + companion object { + private const val ANDROID_NET_URI = "android.net.Uri" + + private val KNOWN_FILE_REFERENCE_PARAM_NAMES = + setOf( + "wallpaperUri", + "imageUri", + "attachmentUri", + "ringtoneUri", + "profilePictureUri", + "audioUri", + "voiceNoteUri", + ) + } + } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/AgentInternalToolsTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/AgentInternalToolsTest.kt new file mode 100644 index 00000000..db8e9f8b --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/AgentInternalToolsTest.kt @@ -0,0 +1,183 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import androidx.appfunctions.AppFunctionData +import androidx.appfunctions.AppFunctionName +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata +import androidx.appfunctions.metadata.AppFunctionPackageMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionResponseMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import androidx.test.core.app.ApplicationProvider +import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult +import io.mockk.coEvery +import io.mockk.mockk +import io.mockk.spyk +import kotlinx.coroutines.test.runTest +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [36]) +class AgentInternalToolsTest { + private val agentInternalTools = + spyk( + AgentInternalTools( + context = ApplicationProvider.getApplicationContext(), + settingsDataStore = mockk(relaxed = true), + ), + ) + + @Test + fun `executeInternalTool dispatches generateImage and maps result`() = + runTest { + val imageResult = + AgentInternalTools.GeneratedImageResult( + imageUri = "content://com.example.appfunctions.agent.fileprovider/cache/cat.png", + mimeType = "image/png", + prompt = "a cat", + ) + coEvery { agentInternalTools.generateImage("a cat", "1:1") } returns imageResult + + val parametersMeta = + listOf( + AppFunctionParameterMetadata( + name = "prompt", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(false), + ), + AppFunctionParameterMetadata( + name = "aspectRatio", + isRequired = false, + dataType = AppFunctionStringTypeMetadata(true), + ), + ) + val components = AppFunctionComponentsMetadata() + val imageResultType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "imageUri" to AppFunctionStringTypeMetadata(false), + "mimeType" to AppFunctionStringTypeMetadata(false), + "prompt" to AppFunctionStringTypeMetadata(false), + ), + required = listOf("imageUri", "mimeType", "prompt"), + isNullable = false, + qualifiedName = + "com.example.appfunctions.agent.data.AgentInternalTools.GeneratedImageResult", + ) + val function = + AppFunctionMetadata( + name = + AppFunctionName( + AgentInternalTools.INTERNAL_TOOL_PACKAGE, + "generateImage", + ), + schema = null, + parameters = parametersMeta, + response = + AppFunctionResponseMetadata( + valueType = imageResultType, + description = "", + ), + description = "Generate image", + deprecation = null, + packageMetadata = + AppFunctionPackageMetadata( + packageName = AgentInternalTools.INTERNAL_TOOL_PACKAGE, + components = components, + ), + ) + val inputData = + AppFunctionData.Builder(parametersMeta, components) + .setString("prompt", "a cat") + .setString("aspectRatio", "1:1") + .build() + + val result = agentInternalTools.executeInternalTool(function, inputData) + + assertTrue(result is ExecuteAppFunctionResult.Data) + val json = JSONObject((result as ExecuteAppFunctionResult.Data).formattedJson) + assertEquals(imageResult.imageUri, json.getString("imageUri")) + assertEquals("image/png", json.getString("mimeType")) + assertEquals("a cat", json.getString("prompt")) + } + + @Test + fun `executeInternalTool dispatches geocodeAddress and returns Error on missing required parameter`() = + runTest { + coEvery { agentInternalTools.geocodeAddress("London") } returns + AgentInternalTools.LatLng(51.5074, -0.1278) + + val parametersMeta = + listOf( + AppFunctionParameterMetadata( + name = "address", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(false), + ), + ) + val components = AppFunctionComponentsMetadata() + val function = + AppFunctionMetadata( + name = + AppFunctionName( + AgentInternalTools.INTERNAL_TOOL_PACKAGE, + "geocodeAddress", + ), + schema = null, + parameters = parametersMeta, + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(true), + description = "", + ), + description = "Geocode", + deprecation = null, + packageMetadata = + AppFunctionPackageMetadata( + packageName = AgentInternalTools.INTERNAL_TOOL_PACKAGE, + components = components, + ), + ) + val validData = + AppFunctionData.Builder(parametersMeta, components) + .setString("address", "London") + .build() + + val okResult = agentInternalTools.executeInternalTool(function, validData) + assertTrue(okResult is ExecuteAppFunctionResult.Data) + val json = JSONObject((okResult as ExecuteAppFunctionResult.Data).formattedJson) + assertEquals(51.5074, json.getDouble("latitude"), 0.0001) + assertEquals(-0.1278, json.getDouble("longitude"), 0.0001) + + val missingResult = + agentInternalTools.executeInternalTool(function, AppFunctionData.EMPTY) + assertTrue(missingResult is ExecuteAppFunctionResult.Error) + assertEquals( + "Missing required parameter 'address'", + (missingResult as ExecuteAppFunctionResult.Error).exception.message, + ) + } +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt index 7bf69940..ffb34207 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt @@ -15,20 +15,11 @@ */ package com.example.appfunctions.agent.domain -import android.content.Intent import androidx.appfunctions.AppFunctionData -import androidx.appfunctions.AppFunctionManager import androidx.appfunctions.AppFunctionName -import androidx.appfunctions.AppFunctionState -import androidx.appfunctions.ExecuteAppFunctionResponse import androidx.appfunctions.metadata.AppFunctionComponentsMetadata import androidx.appfunctions.metadata.AppFunctionMetadata import androidx.appfunctions.metadata.AppFunctionPackageMetadata -import androidx.appfunctions.metadata.AppFunctionParameterMetadata -import androidx.appfunctions.metadata.AppFunctionResponseMetadata -import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata -import androidx.test.core.app.ApplicationProvider -import com.example.appfunctions.agent.data.AgentInternalTools import com.example.appfunctions.agent.data.LlmModel import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.ServiceTier @@ -38,12 +29,12 @@ import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity -import com.example.appfunctions.agent.domain.appfunction.ConvertAppFunctionDataToJsonUseCase import com.example.appfunctions.agent.domain.appfunction.ConvertInputToAppFunctionDataUseCase import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionUseCase -import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionStatesUseCase +import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsParams import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase +import com.example.appfunctions.agent.domain.appfunction.ResolveRemoteFileToLocalUriUseCase import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase import com.example.appfunctions.agent.domain.chat.ObservePendingMessagesUseCase import com.example.appfunctions.agent.domain.chat.SendMessageUseCase @@ -56,8 +47,8 @@ import io.mockk.coEvery import io.mockk.coVerify import io.mockk.every import io.mockk.mockk +import io.mockk.verify import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.emptyFlow import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.test.runTest @@ -77,30 +68,19 @@ class AgentOrchestratorTest { private val manageThreadsUseCase: ManageThreadsUseCase = mockk(relaxed = true) private val settingsRepository: SettingsRepository = mockk() private val llmProviderFactory: LlmProviderFactory = mockk() - private val appFunctionManager: AppFunctionManager = mockk() - private val getAppFunctionStatesUseCase: GetAppFunctionStatesUseCase = mockk() + private val getAppFunctionsUseCase: GetAppFunctionsUseCase = mockk() + private val resolveRemoteFileToLocalUriUseCase: ResolveRemoteFileToLocalUriUseCase = mockk() + private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase = mockk() private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase = mockk() private val sendMessageUseCase: SendMessageUseCase = mockk(relaxed = true) - private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase = mockk() - private val context: android.content.Context = mockk(relaxed = true) private val savePendingIntentUseCase: SavePendingIntentUseCase = mockk(relaxed = true) - private val agentInternalTools: AgentInternalTools = mockk(relaxed = true) - private lateinit var getAppFunctionsUseCase: GetAppFunctionsUseCase private lateinit var agentOrchestrator: AgentOrchestrator @Before fun setUp() { - every { appFunctionManager.observeAppFunctions() } returns emptyFlow() - coEvery { appFunctionManager.searchAppFunctions(any()) } returns emptyList() - - getAppFunctionsUseCase = - GetAppFunctionsUseCase( - appFunctionManager = appFunctionManager, - getAppFunctionStatesUseCase = getAppFunctionStatesUseCase, - settingsRepository = settingsRepository, - agentInternalTools = agentInternalTools, - ) + every { getAppFunctionsUseCase(any()) } returns flowOf(emptyMap()) + coEvery { resolveRemoteFileToLocalUriUseCase(any(), any(), any()) } answers { thirdArg() } agentOrchestrator = AgentOrchestrator( @@ -112,20 +92,11 @@ class AgentOrchestratorTest { llmProviderFactory = llmProviderFactory, settingsRepository = settingsRepository, getAppFunctionsUseCase = getAppFunctionsUseCase, + resolveRemoteFileToLocalUriUseCase = resolveRemoteFileToLocalUriUseCase, convertInputToAppFunctionDataUseCase = convertInputToAppFunctionDataUseCase, executeAppFunctionUseCase = executeAppFunctionUseCase, savePendingIntentUseCase = savePendingIntentUseCase, ) - - coEvery { getAppFunctionStatesUseCase(any()) } answers { - val names = firstArg>() - names.map { name -> - val state = mockk() - every { state.functionName } returns name - every { state.isEnabled } returns true - state - } - } } @Test @@ -139,10 +110,8 @@ class AgentOrchestratorTest { agentOrchestrator.observeAndProcessMessages(threadId) - // Verify behavior (state) assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) - // Verify interactions coVerify { updateMessageUseCase( message.messageId, @@ -174,10 +143,8 @@ class AgentOrchestratorTest { agentOrchestrator.observeAndProcessMessages(threadId) - // Verify behavior (state) assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) - // Verify interactions coVerify { updateMessageUseCase( message.messageId, @@ -210,10 +177,8 @@ class AgentOrchestratorTest { agentOrchestrator.observeAndProcessMessages(threadId) - // Verify behavior (state) assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) - // Verify interactions coVerify { updateThreadUseCase(threadId, UpdateThreadParams(interactionId = "interaction_123")) sendMessageUseCase( @@ -233,19 +198,18 @@ class AgentOrchestratorTest { fun `observeAndProcessMessages scopes tools when targetPackageName is set`() = runTest { val threadId = "thread_1" + val targetPkg = "com.google.android.appfunctiontestingagent" val message = createUserMessage( threadId = threadId, textContent = "run geo code address for n1c4ag", - targetPackageName = "com.google.android.appfunctiontestingagent", + targetPackageName = targetPkg, ) val thread = createThread(threadId) val llmProvider = mockk() - val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") - val tool2 = - createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") - mockAppFunctions(listOf(tool1, tool2)) + val tool1 = createMockTool(targetPkg, "run_geo_code") + mockAppFunctions(listOf(tool1)) setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) @@ -255,6 +219,16 @@ class AgentOrchestratorTest { agentOrchestrator.observeAndProcessMessages(threadId) + verify { + getAppFunctionsUseCase( + GetAppFunctionsParams( + targetPackageNames = setOf(targetPkg), + isEnabled = true, + isConnected = true, + includeInternalTools = true, + ), + ) + } coVerify { llmProvider.generateResponse( previousInteractionId = null, @@ -287,6 +261,16 @@ class AgentOrchestratorTest { agentOrchestrator.observeAndProcessMessages(threadId) + verify { + getAppFunctionsUseCase( + GetAppFunctionsParams( + targetPackageNames = null, + isEnabled = true, + isConnected = true, + includeInternalTools = true, + ), + ) + } coVerify { llmProvider.generateResponse( previousInteractionId = null, @@ -313,7 +297,7 @@ class AgentOrchestratorTest { setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - coEvery { + every { convertInputToAppFunctionDataUseCase(any(), any(), any()) } returns Result.success(mockk()) @@ -375,6 +359,94 @@ class AgentOrchestratorTest { } } + @Test + fun `observeAndProcessMessages chains resolveRemoteFileToLocalUriUseCase before converting input`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "set wallpaper") + val thread = createThread(threadId) + val llmProvider = mockk() + + val targetTool = createMockTool("com.example.targetapp", "setWallpaper") + mockAppFunctions(listOf(targetTool)) + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + + val rawArgs = mapOf("wallpaperUri" to "https://example.com/wallpaper.png") + val resolvedArgs = mapOf("wallpaperUri" to "content://com.example/resolved.png") + coEvery { + resolveRemoteFileToLocalUriUseCase(any(), any(), rawArgs) + } returns resolvedArgs + + val toolCall = + LlmResponsePart.ToolCall( + packageName = "com.example.targetapp", + functionId = "setWallpaper", + arguments = rawArgs, + callId = "call_1", + ) + setupTwoStepToolCallAndExecution( + llmProvider = llmProvider, + toolCall = toolCall, + secondResponseText = "Wallpaper set", + toolResultJson = """{"ok":true}""", + ) + + agentOrchestrator.observeAndProcessMessages(threadId) + + coVerify(exactly = 1) { + resolveRemoteFileToLocalUriUseCase(any(), any(), rawArgs) + } + verify(exactly = 1) { + convertInputToAppFunctionDataUseCase(any(), any(), resolvedArgs) + } + } + + @Test + fun `observeAndProcessMessages extracts attachments when tool returns imageUri and mimeType`() { + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "generate image of a dog") + val thread = createThread(threadId) + val llmProvider = mockk() + + val generateTool = createMockTool("com.example.appfunctions.agent", "generateImage") + mockAppFunctions(listOf(generateTool)) + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + + val toolCall = + LlmResponsePart.ToolCall( + packageName = "com.example.appfunctions.agent", + functionId = "generateImage", + arguments = mapOf("prompt" to "dog"), + callId = "call_1", + ) + setupTwoStepToolCallAndExecution( + llmProvider = llmProvider, + toolCall = toolCall, + secondResponseText = "Here is your image!", + toolResultJson = + """{"imageUri":"content://com.example.appfunctions.agent.fileprovider/cache/test.jpg",""" + + """"mimeType":"image/jpeg","prompt":"dog"}""", + ) + + agentOrchestrator.observeAndProcessMessages(threadId) + + val testUri = + "content://com.example.appfunctions.agent.fileprovider/cache/test.jpg" + coVerify { + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = "Here is your image!", + processingStatus = MessageProcessingStatus.PROCESSED, + pendingIntentId = null, + targetPackageName = null, + attachments = listOf(MessageAttachment(uri = testUri, mimeType = "image/jpeg")), + ) + } + } + } + private fun createUserMessage( threadId: String, textContent: String, @@ -405,21 +477,25 @@ class AgentOrchestratorTest { packageName: String, id: String, ): AppFunctionMetadata { + val components = AppFunctionComponentsMetadata() val pkgMetadata = AppFunctionPackageMetadata( packageName = packageName, - components = AppFunctionComponentsMetadata(), + components = components, ) val tool = mockk(relaxed = true) every { tool.packageName } returns packageName every { tool.id } returns id every { tool.name } returns AppFunctionName(packageName, id) + every { tool.parameters } returns emptyList() + every { tool.components } returns components every { tool.packageMetadata } returns pkgMetadata return tool } private fun mockAppFunctions(tools: List) { - coEvery { appFunctionManager.searchAppFunctions(any()) } returns tools + every { getAppFunctionsUseCase(any()) } returns + flowOf(tools.groupBy(AppFunctionMetadata::packageMetadata)) } private fun setupDefaultMocks( @@ -427,7 +503,6 @@ class AgentOrchestratorTest { message: MessageEntity, thread: ThreadEntity, apiKey: String? = "dummy_key", - disconnectedApps: Set = emptySet(), llmProvider: LlmProvider = mockk(), ) { coEvery { observePendingMessagesUseCase(threadId) } returns @@ -437,167 +512,10 @@ class AgentOrchestratorTest { } coEvery { manageThreadsUseCase.getThread(threadId) } returns flowOf(thread) coEvery { settingsRepository.geminiApiKey } returns flowOf(apiKey) - coEvery { settingsRepository.disconnectedApps } returns flowOf(disconnectedApps) coEvery { settingsRepository.serviceTier } returns flowOf(ServiceTier.STANDARD) coEvery { llmProviderFactory.getProvider(LlmProviderName.GEMINI) } returns llmProvider } - @Test - fun `observeAndProcessMessages extracts attachments when tool returns imageUri and mimeType`() { - runTest { - val threadId = "thread_1" - val message = createUserMessage(threadId, "generate image of a dog") - val thread = createThread(threadId) - val llmProvider = mockk() - - val generateTool = createMockTool("com.example.appfunctions.agent", "generateImage") - mockAppFunctions(listOf(generateTool)) - setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - - val toolCall = - LlmResponsePart.ToolCall( - packageName = "com.example.appfunctions.agent", - functionId = "generateImage", - arguments = mapOf("prompt" to "dog"), - callId = "call_1", - ) - setupTwoStepToolCallAndExecution( - llmProvider = llmProvider, - toolCall = toolCall, - secondResponseText = "Here is your image!", - toolResultJson = - """{"imageUri":"content://com.example.appfunctions.agent.fileprovider/cache/test.jpg",""" + - """"mimeType":"image/jpeg","prompt":"dog"}""", - ) - - agentOrchestrator.observeAndProcessMessages(threadId) - - val testUri = - "content://com.example.appfunctions.agent.fileprovider/cache/test.jpg" - coVerify { - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = "Here is your image!", - processingStatus = MessageProcessingStatus.PROCESSED, - pendingIntentId = null, - targetPackageName = null, - attachments = listOf(MessageAttachment(uri = testUri, mimeType = "image/jpeg")), - ) - } - } - } - - @Test - fun `observeAndProcessMessages grants URI read permission when tool is called with content URI argument`() { - runTest { - val threadId = "thread_1" - val message = createUserMessage(threadId, "set wallpaper") - val thread = createThread(threadId) - val llmProvider = mockk() - - val components = AppFunctionComponentsMetadata() - val targetTool = - AppFunctionMetadata( - name = AppFunctionName("com.example.targetapp", "setWallpaper"), - schema = null, - parameters = - listOf( - AppFunctionParameterMetadata( - name = "uri", - isRequired = true, - dataType = AppFunctionStringTypeMetadata(false), - ), - ), - response = - AppFunctionResponseMetadata( - valueType = AppFunctionStringTypeMetadata(false), - description = "", - ), - description = "Set wallpaper", - deprecation = null, - packageMetadata = - AppFunctionPackageMetadata( - packageName = "com.example.targetapp", - components = components, - ), - ) - mockAppFunctions(listOf(targetTool)) - setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - - val realConverter = - ConvertInputToAppFunctionDataUseCase(ApplicationProvider.getApplicationContext()) - val convertToJson = mockk() - every { convertToJson(any(), any(), any(), any()) } returns """{"success":true}""" - coEvery { appFunctionManager.executeAppFunction(any()) } returns - ExecuteAppFunctionResponse.Success(AppFunctionData.EMPTY) - val realExecutor = - ExecuteAppFunctionUseCase( - context = context, - appFunctionManager = appFunctionManager, - convertAppFunctionDataToJsonUseCase = convertToJson, - agentInternalTools = agentInternalTools, - ) - - val orchestratorWithRealPipeline = - AgentOrchestrator( - manageThreadsUseCase = manageThreadsUseCase, - observePendingMessagesUseCase = observePendingMessagesUseCase, - sendMessageUseCase = sendMessageUseCase, - updateMessageUseCase = updateMessageUseCase, - updateThreadUseCase = updateThreadUseCase, - llmProviderFactory = llmProviderFactory, - settingsRepository = settingsRepository, - getAppFunctionsUseCase = getAppFunctionsUseCase, - convertInputToAppFunctionDataUseCase = realConverter, - executeAppFunctionUseCase = realExecutor, - savePendingIntentUseCase = savePendingIntentUseCase, - ) - - val contentUri = "content://com.example.appfunctions.agent.fileprovider/cache/img.jpg" - val toolCall = - LlmResponsePart.ToolCall( - packageName = "com.example.targetapp", - functionId = "setWallpaper", - arguments = mapOf("uri" to contentUri), - callId = "call_2", - ) - coEvery { - llmProvider.generateResponse( - previousInteractionId = null, - input = any(), - tools = any(), - apiKey = any(), - modelName = any(), - ) - } returns LlmResponse.Success(interactionId = "interaction_1", parts = listOf(toolCall)) - - coEvery { - llmProvider.generateResponse( - previousInteractionId = "interaction_1", - input = any(), - tools = any(), - apiKey = any(), - modelName = any(), - ) - } returns - LlmResponse.Success( - interactionId = "interaction_2", - parts = listOf(LlmResponsePart.Text("Wallpaper set")), - ) - - orchestratorWithRealPipeline.observeAndProcessMessages(threadId) - - coVerify { - context.grantUriPermission( - eq("com.example.targetapp"), - match { it.toString() == contentUri }, - eq(Intent.FLAG_GRANT_READ_URI_PERMISSION), - ) - } - } - } - private fun setupTwoStepToolCallAndExecution( llmProvider: LlmProvider, toolCall: LlmResponsePart.ToolCall, @@ -635,7 +553,7 @@ class AgentOrchestratorTest { ) } returns secondResponse - coEvery { + every { convertInputToAppFunctionDataUseCase(any(), any(), any()) } returns Result.success(AppFunctionData.EMPTY) @@ -647,47 +565,4 @@ class AgentOrchestratorTest { formattedJson = toolResultJson, ) } - - @Test - fun `observeAndProcessMessages filters out disabled tools`() = - runTest { - val threadId = "thread_1" - val message = createUserMessage(threadId, "run geo code address for n1c4ag") - val thread = createThread(threadId) - val llmProvider = mockk() - - val enabledTool = - createMockTool("com.google.android.appfunctiontestingagent", "enabled_tool") - val disabledTool = - createMockTool("com.google.android.appfunctiontestingagent", "disabled_tool") - mockAppFunctions(listOf(enabledTool, disabledTool)) - - setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - - coEvery { getAppFunctionStatesUseCase(any()) } answers { - val names = firstArg>() - names.map { name -> - val state = mockk() - every { state.functionName } returns name - every { state.isEnabled } returns (name.toString().contains("enabled_tool")) - state - } - } - - coEvery { - llmProvider.generateResponse(any(), any(), any(), any(), any(), any()) - } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) - - agentOrchestrator.observeAndProcessMessages(threadId) - - coVerify { - llmProvider.generateResponse( - previousInteractionId = null, - input = eq(LlmInput.UserMessage("run geo code address for n1c4ag")), - tools = listOf(enabledTool), - apiKey = "dummy_key", - modelName = any(), - ) - } - } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt index ebee8411..c3305efb 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt @@ -27,231 +27,208 @@ import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata import androidx.appfunctions.metadata.AppFunctionParameterMetadata import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata -import androidx.test.core.app.ApplicationProvider -import kotlinx.coroutines.test.runTest import org.junit.Assert.assertEquals -import org.junit.Assert.assertTrue import org.junit.Test import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner import org.robolectric.annotation.Config -import java.io.IOException @RunWith(RobolectricTestRunner::class) @Config(sdk = [34]) class ConvertInputToAppFunctionDataUseCaseTest { - private val useCase = - ConvertInputToAppFunctionDataUseCase(ApplicationProvider.getApplicationContext()) + private val useCase = ConvertInputToAppFunctionDataUseCase() private val components = AppFunctionComponentsMetadata(emptyMap()) @Test - fun invoke_allPrimitiveTypes_convertsCorrectly() = - runTest { - val parameters = - listOf( - AppFunctionParameterMetadata("str", true, AppFunctionStringTypeMetadata(false)), - AppFunctionParameterMetadata("int", true, AppFunctionIntTypeMetadata(false)), - AppFunctionParameterMetadata("long", true, AppFunctionLongTypeMetadata(false)), - AppFunctionParameterMetadata( - "bool", - true, - AppFunctionBooleanTypeMetadata(false), - ), - AppFunctionParameterMetadata( - "double", - true, - AppFunctionDoubleTypeMetadata(false), - ), - AppFunctionParameterMetadata( - "float", - true, - AppFunctionFloatTypeMetadata(false), - ), - ) - val inputs = - mapOf( - "str" to "value", - "int" to "123", - "long" to "456", - "bool" to true, - "double" to "1.23", - "float" to "4.56", - ) + fun invoke_allPrimitiveTypes_convertsCorrectly() { + val parameters = + listOf( + AppFunctionParameterMetadata("str", true, AppFunctionStringTypeMetadata(false)), + AppFunctionParameterMetadata("int", true, AppFunctionIntTypeMetadata(false)), + AppFunctionParameterMetadata("long", true, AppFunctionLongTypeMetadata(false)), + AppFunctionParameterMetadata("bool", true, AppFunctionBooleanTypeMetadata(false)), + AppFunctionParameterMetadata( + "double", + true, + AppFunctionDoubleTypeMetadata(false), + ), + AppFunctionParameterMetadata("float", true, AppFunctionFloatTypeMetadata(false)), + ) + val inputs = + mapOf( + "str" to "value", + "int" to "123", + "long" to "456", + "bool" to true, + "double" to "1.23", + "float" to "4.56", + ) - val result = useCase(parameters, components, inputs).getOrThrow() + val result = useCase(parameters, components, inputs).getOrThrow() - assertEquals("value", result.getString("str")) - assertEquals(123, result.getInt("int")) - assertEquals(456L, result.getLong("long")) - assertEquals(true, result.getBoolean("bool")) - assertEquals(1.23, result.getDouble("double"), 0.001) - assertEquals(4.56f, result.getFloat("float"), 0.001f) - } + assertEquals("value", result.getString("str")) + assertEquals(123, result.getInt("int")) + assertEquals(456L, result.getLong("long")) + assertEquals(true, result.getBoolean("bool")) + assertEquals(1.23, result.getDouble("double"), 0.001) + assertEquals(4.56f, result.getFloat("float"), 0.001f) + } @Test - fun invoke_allPrimitiveArrays_convertsCorrectly() = - runTest { - val parameters = - listOf( - AppFunctionParameterMetadata( - "strArray", - true, - AppFunctionArrayTypeMetadata(AppFunctionStringTypeMetadata(false), false), - ), - AppFunctionParameterMetadata( - "intArray", - true, - AppFunctionArrayTypeMetadata(AppFunctionIntTypeMetadata(false), false), - ), - AppFunctionParameterMetadata( - "longArray", - true, - AppFunctionArrayTypeMetadata(AppFunctionLongTypeMetadata(false), false), - ), - AppFunctionParameterMetadata( - "boolArray", - true, - AppFunctionArrayTypeMetadata(AppFunctionBooleanTypeMetadata(false), false), - ), - AppFunctionParameterMetadata( - "doubleArray", - true, - AppFunctionArrayTypeMetadata(AppFunctionDoubleTypeMetadata(false), false), - ), - AppFunctionParameterMetadata( - "floatArray", - true, - AppFunctionArrayTypeMetadata(AppFunctionFloatTypeMetadata(false), false), - ), - ) - val inputs = - mapOf( - "strArray" to listOf("v1", "v2"), - "intArray" to listOf("1", "2"), - "longArray" to listOf("3", "4"), - "boolArray" to listOf(true, false), - "doubleArray" to listOf("1.1", "2.2"), - "floatArray" to listOf("3.3", "4.4"), - ) + fun invoke_allPrimitiveArrays_convertsCorrectly() { + val parameters = + listOf( + AppFunctionParameterMetadata( + "strArray", + true, + AppFunctionArrayTypeMetadata(AppFunctionStringTypeMetadata(false), false), + ), + AppFunctionParameterMetadata( + "intArray", + true, + AppFunctionArrayTypeMetadata(AppFunctionIntTypeMetadata(false), false), + ), + AppFunctionParameterMetadata( + "longArray", + true, + AppFunctionArrayTypeMetadata(AppFunctionLongTypeMetadata(false), false), + ), + AppFunctionParameterMetadata( + "boolArray", + true, + AppFunctionArrayTypeMetadata(AppFunctionBooleanTypeMetadata(false), false), + ), + AppFunctionParameterMetadata( + "doubleArray", + true, + AppFunctionArrayTypeMetadata(AppFunctionDoubleTypeMetadata(false), false), + ), + AppFunctionParameterMetadata( + "floatArray", + true, + AppFunctionArrayTypeMetadata(AppFunctionFloatTypeMetadata(false), false), + ), + ) + val inputs = + mapOf( + "strArray" to listOf("v1", "v2"), + "intArray" to listOf("1", 2), + "longArray" to listOf("3", 4L), + "boolArray" to listOf(true, false), + "doubleArray" to listOf("1.1", 2.2), + "floatArray" to listOf("3.3", 4.4f), + ) - val result = useCase(parameters, components, inputs).getOrThrow() + val result = useCase(parameters, components, inputs).getOrThrow() - assertEquals(listOf("v1", "v2"), result.getStringList("strArray")) - assertEquals(listOf(1, 2), result.getIntArray("intArray")?.toList()) - assertEquals(listOf(3L, 4L), result.getLongArray("longArray")?.toList()) - assertEquals(listOf(true, false), result.getBooleanArray("boolArray")?.toList()) - assertEquals(listOf(1.1, 2.2), result.getDoubleArray("doubleArray")?.toList()) - assertEquals(listOf(3.3f, 4.4f), result.getFloatArray("floatArray")?.toList()) - } + assertEquals(listOf("v1", "v2"), result.getStringList("strArray")) + assertEquals(listOf(1, 2), result.getIntArray("intArray")?.toList()) + assertEquals(listOf(3L, 4L), result.getLongArray("longArray")?.toList()) + assertEquals(listOf(true, false), result.getBooleanArray("boolArray")?.toList()) + assertEquals(listOf(1.1, 2.2), result.getDoubleArray("doubleArray")?.toList()) + assertEquals(listOf(3.3f, 4.4f), result.getFloatArray("floatArray")?.toList()) + } @Test - fun invoke_objectParameter_convertsCorrectly() = - runTest { - val objectType = - AppFunctionObjectTypeMetadata( - properties = - mapOf( - "prop1" to AppFunctionStringTypeMetadata(false), - "prop2" to AppFunctionIntTypeMetadata(false), - ), - required = listOf("prop1", "prop2"), - qualifiedName = null, - isNullable = false, - ) - val parameters = - listOf( - AppFunctionParameterMetadata("obj", true, objectType), - ) - val inputs = - mapOf( - "obj" to mapOf("prop1" to "val1", "prop2" to "123"), - ) + fun invoke_objectParameter_convertsCorrectly() { + val objectType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "prop1" to AppFunctionStringTypeMetadata(false), + "prop2" to AppFunctionIntTypeMetadata(false), + ), + required = listOf("prop1", "prop2"), + qualifiedName = null, + isNullable = false, + ) + val parameters = + listOf( + AppFunctionParameterMetadata("obj", true, objectType), + ) + val inputs = + mapOf( + "obj" to mapOf("prop1" to "val1", "prop2" to "123"), + ) - val result = useCase(parameters, components, inputs).getOrThrow() + val result = useCase(parameters, components, inputs).getOrThrow() - val objResult = result.getAppFunctionData("obj") - assertEquals("val1", objResult?.getString("prop1")) - assertEquals(123, objResult?.getInt("prop2")) - } + val objResult = result.getAppFunctionData("obj") + assertEquals("val1", objResult?.getString("prop1")) + assertEquals(123, objResult?.getInt("prop2")) + } @Test - fun invoke_referenceParameter_convertsCorrectly() = - runTest { - val objectType = - AppFunctionObjectTypeMetadata( - properties = - mapOf( - "prop1" to AppFunctionStringTypeMetadata(false), - ), - required = listOf("prop1"), - qualifiedName = null, - isNullable = false, - ) - val componentsWithTypes = - AppFunctionComponentsMetadata( - dataTypes = mapOf("RefType" to objectType), - ) - val parameters = - listOf( - AppFunctionParameterMetadata( - "ref", - true, - AppFunctionReferenceTypeMetadata("RefType", false), + fun invoke_referenceParameter_convertsCorrectly() { + val objectType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "prop1" to AppFunctionStringTypeMetadata(false), ), - ) - val inputs = - mapOf( - "ref" to mapOf("prop1" to "val1"), - ) + required = listOf("prop1"), + qualifiedName = null, + isNullable = false, + ) + val componentsWithTypes = + AppFunctionComponentsMetadata( + dataTypes = mapOf("RefType" to objectType), + ) + val parameters = + listOf( + AppFunctionParameterMetadata( + "ref", + true, + AppFunctionReferenceTypeMetadata("RefType", false), + ), + ) + val inputs = + mapOf( + "ref" to mapOf("prop1" to "val1"), + ) - val result = useCase(parameters, componentsWithTypes, inputs).getOrThrow() + val result = useCase(parameters, componentsWithTypes, inputs).getOrThrow() - val refResult = result.getAppFunctionData("ref") - assertEquals("val1", refResult?.getString("prop1")) - } + val refResult = result.getAppFunctionData("ref") + assertEquals("val1", refResult?.getString("prop1")) + } @Test - fun invoke_emptyStringNumericParameters_areSkipped() = - runTest { - val parameters = - listOf( - AppFunctionParameterMetadata("int", false, AppFunctionIntTypeMetadata(false)), - AppFunctionParameterMetadata("long", false, AppFunctionLongTypeMetadata(false)), - AppFunctionParameterMetadata( - "double", - false, - AppFunctionDoubleTypeMetadata(false), - ), - AppFunctionParameterMetadata( - "float", - false, - AppFunctionFloatTypeMetadata(false), - ), - ) - val inputs = - mapOf( - "int" to "", - "long" to "", - "double" to "", - "float" to "", - ) + fun invoke_emptyStringNumericParameters_areSkipped() { + val parameters = + listOf( + AppFunctionParameterMetadata("int", false, AppFunctionIntTypeMetadata(false)), + AppFunctionParameterMetadata("long", false, AppFunctionLongTypeMetadata(false)), + AppFunctionParameterMetadata( + "double", + false, + AppFunctionDoubleTypeMetadata(false), + ), + AppFunctionParameterMetadata("float", false, AppFunctionFloatTypeMetadata(false)), + ) + val inputs = + mapOf( + "int" to "", + "long" to "", + "double" to "", + "float" to "", + ) - val result = useCase(parameters, components, inputs) + val result = useCase(parameters, components, inputs) - // Verify that it doesn't throw NumberFormatException during conversion - assertEquals(true, result.isSuccess) - } + // Verify that it doesn't throw NumberFormatException during conversion + assertEquals(true, result.isSuccess) + } @Test - fun invoke_invalidNumericInputs_returnsFailure() = - runTest { - assertFailureForInput(AppFunctionIntTypeMetadata(false), "not_an_int") - assertFailureForInput(AppFunctionLongTypeMetadata(false), "not_a_long") - assertFailureForInput(AppFunctionDoubleTypeMetadata(false), "not_a_double") - assertFailureForInput(AppFunctionFloatTypeMetadata(false), "not_a_float") - } + fun invoke_invalidNumericInputs_returnsFailure() { + assertFailureForInput(AppFunctionIntTypeMetadata(false), "not_an_int") + assertFailureForInput(AppFunctionLongTypeMetadata(false), "not_a_long") + assertFailureForInput(AppFunctionDoubleTypeMetadata(false), "not_a_double") + assertFailureForInput(AppFunctionFloatTypeMetadata(false), "not_a_float") + } - private suspend fun assertFailureForInput( + private fun assertFailureForInput( dataType: AppFunctionDataTypeMetadata, value: Any, ) { @@ -271,160 +248,62 @@ class ConvertInputToAppFunctionDataUseCaseTest { } @Test - fun convert_uriObjectTypeWithStringInput_buildsUriAppFunctionData() = - runTest { - val uriObjectType = - AppFunctionObjectTypeMetadata( - properties = - mapOf( - "uri" to AppFunctionStringTypeMetadata(false), - ), - required = listOf("uri"), - qualifiedName = "android.net.Uri", - isNullable = false, - ) - val parameters = - listOf( - AppFunctionParameterMetadata( - name = "wallpaperUri", - isRequired = true, - dataType = uriObjectType, + fun convert_uriObjectTypeWithStringInput_buildsUriAppFunctionData() { + val uriObjectType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "uri" to AppFunctionStringTypeMetadata(false), ), - ) - val inputs = mapOf("wallpaperUri" to "content://com.example/file.jpg") - - val result = useCase(parameters, components, inputs).getOrThrow() - - val uriData = result.getAppFunctionData("wallpaperUri") - assertEquals("content://com.example/file.jpg", uriData?.getString("uri")) - } - - @Test - fun convert_uriObjectTypeArrayWithStringInput_buildsUriObjectTypeArray() = - runTest { - val uriObjectType = - AppFunctionObjectTypeMetadata( - properties = mapOf("uri" to AppFunctionStringTypeMetadata(false)), - required = listOf("uri"), - qualifiedName = "android.net.Uri", - isNullable = false, - ) - val arrayType = - AppFunctionArrayTypeMetadata( - itemType = uriObjectType, - isNullable = false, - ) - val parameters = - listOf( - AppFunctionParameterMetadata( - name = "attachments", - isRequired = true, - dataType = arrayType, - ), - ) - val inputs = mapOf("attachments" to listOf("content://com.example/file.jpg")) + required = listOf("uri"), + qualifiedName = "android.net.Uri", + isNullable = false, + ) + val parameters = + listOf( + AppFunctionParameterMetadata( + name = "wallpaperUri", + isRequired = true, + dataType = uriObjectType, + ), + ) + val inputs = mapOf("wallpaperUri" to "content://com.example/file.jpg") - val result = useCase(parameters, components, inputs).getOrThrow() + val result = useCase(parameters, components, inputs).getOrThrow() - val dataList = result.getAppFunctionDataList("attachments") - assertEquals(1, dataList?.size) - val uriData = dataList?.first() - assertEquals("content://com.example/file.jpg", uriData?.getString("uri")) - } + val uriData = result.getAppFunctionData("wallpaperUri") + assertEquals("content://com.example/file.jpg", uriData?.getString("uri")) + } @Test - fun invoke_resolvesRemoteFileReferencesWhileLeavingRegularUrlParametersUntouched() = - runTest { - val downloadedUrls = mutableListOf() - val testUseCase = - ConvertInputToAppFunctionDataUseCase(ApplicationProvider.getApplicationContext()) - .apply { - remoteFileDownloader = { _, url -> - downloadedUrls.add(url) - "content://com.example.appfunctions.agent.fileprovider/cache/resolved.png" - } - } - - val uriObjectType = - AppFunctionObjectTypeMetadata( - properties = mapOf("uri" to AppFunctionStringTypeMetadata(false)), - required = listOf("uri"), - qualifiedName = "android.net.Uri", - isNullable = false, - ) - val parameters = - listOf( - AppFunctionParameterMetadata( - name = "imageUri", - isRequired = true, - dataType = AppFunctionStringTypeMetadata(false), - ), - AppFunctionParameterMetadata( - name = "attachment", - isRequired = true, - dataType = uriObjectType, - ), - AppFunctionParameterMetadata( - name = "websiteUrl", - isRequired = true, - dataType = AppFunctionStringTypeMetadata(false), - ), - ) - val inputs = - mapOf( - "imageUri" to "https://example.com/cat.png", - "attachment" to "https://example.com/doc.png", - "websiteUrl" to "https://example.com/home", - ) - - val result = testUseCase(parameters, components, inputs).getOrThrow() - - assertEquals( - listOf("https://example.com/cat.png", "https://example.com/doc.png"), - downloadedUrls, + fun convert_uriObjectTypeArrayWithStringInput_buildsUriObjectTypeArray() { + val uriObjectType = + AppFunctionObjectTypeMetadata( + properties = mapOf("uri" to AppFunctionStringTypeMetadata(false)), + required = listOf("uri"), + qualifiedName = "android.net.Uri", + isNullable = false, ) - assertEquals( - "content://com.example.appfunctions.agent.fileprovider/cache/resolved.png", - result.getString("imageUri"), + val arrayType = + AppFunctionArrayTypeMetadata( + itemType = uriObjectType, + isNullable = false, ) - assertEquals( - "content://com.example.appfunctions.agent.fileprovider/cache/resolved.png", - result.getAppFunctionData("attachment")?.getString("uri"), + val parameters = + listOf( + AppFunctionParameterMetadata( + name = "attachments", + isRequired = true, + dataType = arrayType, + ), ) - assertEquals("https://example.com/home", result.getString("websiteUrl")) - } + val inputs = mapOf("attachments" to listOf("content://com.example/file.jpg")) - @Test - fun invoke_returnsFailureWhenRemoteFileDownloadFails() = - runTest { - val testUseCase = - ConvertInputToAppFunctionDataUseCase(ApplicationProvider.getApplicationContext()) - .apply { - remoteFileDownloader = { _, _ -> - throw IOException("HTTP 404") - } - } - - val parameters = - listOf( - AppFunctionParameterMetadata( - name = "wallpaperUri", - isRequired = true, - dataType = AppFunctionStringTypeMetadata(false), - ), - ) - val result = - testUseCase( - parameters, - components, - mapOf("wallpaperUri" to "https://example.com/missing.png"), - ) + val result = useCase(parameters, components, inputs).getOrThrow() - assertTrue(result.isFailure) - assertTrue( - result.exceptionOrNull()?.message?.contains( - "Failed to download remote file reference: HTTP 404", - ) == true, - ) - } + val dataList = result.getAppFunctionDataList("attachments") + assertEquals(1, dataList?.size) + val uriData = dataList?.first() + assertEquals("content://com.example/file.jpg", uriData?.getString("uri")) + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt index b94dceec..5c3de98c 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt @@ -15,28 +15,19 @@ */ package com.example.appfunctions.agent.domain.appfunction -import android.content.Context -import android.content.Intent import androidx.appfunctions.AppFunctionData import androidx.appfunctions.AppFunctionManager -import androidx.appfunctions.AppFunctionName import androidx.appfunctions.AppFunctionPermissionRequiredException import androidx.appfunctions.ExecuteAppFunctionResponse -import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata -import androidx.appfunctions.metadata.AppFunctionComponentsMetadata import androidx.appfunctions.metadata.AppFunctionMetadata -import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata -import androidx.appfunctions.metadata.AppFunctionPackageMetadata -import androidx.appfunctions.metadata.AppFunctionParameterMetadata -import androidx.appfunctions.metadata.AppFunctionResponseMetadata import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata import com.example.appfunctions.agent.data.AgentInternalTools import io.mockk.coEvery +import io.mockk.coVerify import io.mockk.every import io.mockk.mockk import io.mockk.verify import kotlinx.coroutines.test.runTest -import org.json.JSONObject import org.junit.Assert.assertEquals import org.junit.Assert.assertTrue import org.junit.Test @@ -47,16 +38,17 @@ import org.robolectric.annotation.Config @RunWith(RobolectricTestRunner::class) @Config(sdk = [34]) class ExecuteAppFunctionUseCaseTest { - private val context = mockk(relaxed = true) private val appFunctionManager = mockk() private val convertAppFunctionDataToJsonUseCase = mockk() private val agentInternalTools = mockk(relaxed = true) + private val grantAppFunctionUriPermissionsUseCase = + mockk(relaxed = true) private val useCase = ExecuteAppFunctionUseCase( - context = context, appFunctionManager = appFunctionManager, convertAppFunctionDataToJsonUseCase = convertAppFunctionDataToJsonUseCase, agentInternalTools = agentInternalTools, + grantAppFunctionUriPermissionsUseCase = grantAppFunctionUriPermissionsUseCase, ) @Test @@ -77,6 +69,7 @@ class ExecuteAppFunctionUseCaseTest { assertTrue(result is ExecuteAppFunctionResult.Error) val errorResult = result as ExecuteAppFunctionResult.Error assertEquals(appException, errorResult.exception) + verify(exactly = 1) { grantAppFunctionUriPermissionsUseCase(function, parameters) } } @Test @@ -99,91 +92,13 @@ class ExecuteAppFunctionUseCaseTest { } @Test - fun `invoke grants recursive read URI permissions for content URIs in AppFunctionData`() = + fun `invoke grants URI permissions before executing external AppFunction`() = runTest { - val uriObjectType = - AppFunctionObjectTypeMetadata( - properties = mapOf("uri" to AppFunctionStringTypeMetadata(false)), - required = listOf("uri"), - qualifiedName = "android.net.Uri", - isNullable = false, - ) - val payloadObjectType = - AppFunctionObjectTypeMetadata( - properties = - mapOf( - "nestedUris" to - AppFunctionArrayTypeMetadata( - itemType = AppFunctionStringTypeMetadata(false), - isNullable = false, - ), - ), - required = listOf("nestedUris"), - qualifiedName = null, - isNullable = false, - ) - val parametersMeta = - listOf( - AppFunctionParameterMetadata( - name = "directUri", - isRequired = true, - dataType = AppFunctionStringTypeMetadata(false), - ), - AppFunctionParameterMetadata( - name = "uriObj", - isRequired = true, - dataType = uriObjectType, - ), - AppFunctionParameterMetadata( - name = "payload", - isRequired = true, - dataType = payloadObjectType, - ), - AppFunctionParameterMetadata( - name = "plainText", - isRequired = true, - dataType = AppFunctionStringTypeMetadata(false), - ), - ) - val components = AppFunctionComponentsMetadata() - val function = - AppFunctionMetadata( - name = AppFunctionName("com.example.notes", "attachMedia"), - schema = null, - parameters = parametersMeta, - response = - AppFunctionResponseMetadata( - valueType = AppFunctionStringTypeMetadata(false), - description = "", - ), - description = "Attach media", - deprecation = null, - packageMetadata = - AppFunctionPackageMetadata( - packageName = "com.example.notes", - components = components, - ), - ) - - val directUri = "content://com.example.appfunctions.agent.fileprovider/cache/direct.png" - val objectUri = "content://com.example.appfunctions.agent.fileprovider/cache/object.png" - val nestedUri = "content://com.example.appfunctions.agent.fileprovider/cache/nested.jpg" - - val uriObjData = - AppFunctionData.Builder(uriObjectType, components) - .setString("uri", objectUri) - .build() - val payloadData = - AppFunctionData.Builder(payloadObjectType, components) - .setStringList("nestedUris", listOf(nestedUri, "https://example.com/ignore")) - .build() - val inputData = - AppFunctionData.Builder(parametersMeta, components) - .setString("directUri", directUri) - .setAppFunctionData("uriObj", uriObjData) - .setAppFunctionData("payload", payloadData) - .setString("plainText", "hello world") - .build() + val function = mockk(relaxed = true) + every { function.packageName } returns "com.example.notes" + every { function.id } returns "attachMedia" + every { function.response.valueType } returns AppFunctionStringTypeMetadata(false) + val parameters = AppFunctionData.EMPTY coEvery { appFunctionManager.executeAppFunction(any()) } returns ExecuteAppFunctionResponse.Success(AppFunctionData.EMPTY) @@ -191,157 +106,40 @@ class ExecuteAppFunctionUseCaseTest { convertAppFunctionDataToJsonUseCase(any(), any(), any(), any()) } returns """{"ok":true}""" - val result = useCase(function, inputData) + val result = useCase(function, parameters) assertTrue(result is ExecuteAppFunctionResult.Data) assertEquals("""{"ok":true}""", (result as ExecuteAppFunctionResult.Data).formattedJson) - - for (expectedUri in listOf(directUri, objectUri, nestedUri)) { - verify(exactly = 1) { - context.grantUriPermission( - eq("com.example.notes"), - match { it.toString() == expectedUri }, - eq(Intent.FLAG_GRANT_READ_URI_PERMISSION), - ) - } - } + verify(exactly = 1) { grantAppFunctionUriPermissionsUseCase(function, parameters) } } @Test - fun `invoke dispatches internal generateImage tool even when AppFunctionManager is null`() = + fun `invoke delegates internal tools to AgentInternalTools even when AppFunctionManager is null`() = runTest { val nullManagerUseCase = ExecuteAppFunctionUseCase( - context = context, appFunctionManager = null, convertAppFunctionDataToJsonUseCase = convertAppFunctionDataToJsonUseCase, agentInternalTools = agentInternalTools, + grantAppFunctionUriPermissionsUseCase = grantAppFunctionUriPermissionsUseCase, ) - val imageResult = - AgentInternalTools.GeneratedImageResult( - imageUri = "content://com.example.appfunctions.agent.fileprovider/cache/cat.png", - mimeType = "image/png", - prompt = "a cat", - ) - coEvery { agentInternalTools.generateImage("a cat", "1:1") } returns imageResult - - val parametersMeta = - listOf( - AppFunctionParameterMetadata( - name = "prompt", - isRequired = true, - dataType = AppFunctionStringTypeMetadata(false), - ), - AppFunctionParameterMetadata( - name = "aspectRatio", - isRequired = false, - dataType = AppFunctionStringTypeMetadata(true), - ), - ) - val components = AppFunctionComponentsMetadata() - val imageResultType = - AppFunctionObjectTypeMetadata( - properties = - mapOf( - "imageUri" to AppFunctionStringTypeMetadata(false), - "mimeType" to AppFunctionStringTypeMetadata(false), - "prompt" to AppFunctionStringTypeMetadata(false), - ), - required = listOf("imageUri", "mimeType", "prompt"), - isNullable = false, - qualifiedName = - "com.example.appfunctions.agent.data.AgentInternalTools.GeneratedImageResult", - ) - val function = - AppFunctionMetadata( - name = - AppFunctionName( - AgentInternalTools.INTERNAL_TOOL_PACKAGE, - "generateImage", - ), - schema = null, - parameters = parametersMeta, - response = - AppFunctionResponseMetadata( - valueType = imageResultType, - description = "", - ), - description = "Generate image", - deprecation = null, - packageMetadata = - AppFunctionPackageMetadata( - packageName = AgentInternalTools.INTERNAL_TOOL_PACKAGE, - components = components, - ), - ) - val inputData = - AppFunctionData.Builder(parametersMeta, components) - .setString("prompt", "a cat") - .setString("aspectRatio", "1:1") - .build() - - val result = nullManagerUseCase(function, inputData) - - assertTrue(result is ExecuteAppFunctionResult.Data) - val json = JSONObject((result as ExecuteAppFunctionResult.Data).formattedJson) - assertEquals(imageResult.imageUri, json.getString("imageUri")) - assertEquals("image/png", json.getString("mimeType")) - assertEquals("a cat", json.getString("prompt")) - } - - @Test - fun `invoke dispatches internal geocodeAddress tool and returns Error on missing required parameter`() = - runTest { - coEvery { agentInternalTools.geocodeAddress("London") } returns - AgentInternalTools.LatLng(51.5074, -0.1278) - - val parametersMeta = - listOf( - AppFunctionParameterMetadata( - name = "address", - isRequired = true, - dataType = AppFunctionStringTypeMetadata(false), - ), - ) - val components = AppFunctionComponentsMetadata() - val function = - AppFunctionMetadata( - name = - AppFunctionName( - AgentInternalTools.INTERNAL_TOOL_PACKAGE, - "geocodeAddress", - ), - schema = null, - parameters = parametersMeta, - response = - AppFunctionResponseMetadata( - valueType = AppFunctionStringTypeMetadata(true), - description = "", - ), - description = "Geocode", - deprecation = null, - packageMetadata = - AppFunctionPackageMetadata( - packageName = AgentInternalTools.INTERNAL_TOOL_PACKAGE, - components = components, - ), + val function = mockk(relaxed = true) + every { function.packageName } returns AgentInternalTools.INTERNAL_TOOL_PACKAGE + every { function.id } returns "generateImage" + val parameters = AppFunctionData.EMPTY + val expectedResult = + ExecuteAppFunctionResult.Data( + data = AppFunctionData.EMPTY, + formattedJson = """{"imageUri":"content://test/cat.png"}""", ) - val validData = - AppFunctionData.Builder(parametersMeta, components) - .setString("address", "London") - .build() + coEvery { + agentInternalTools.executeInternalTool(function, parameters) + } returns expectedResult - val okResult = useCase(function, validData) - assertTrue(okResult is ExecuteAppFunctionResult.Data) - val json = JSONObject((okResult as ExecuteAppFunctionResult.Data).formattedJson) - assertEquals(51.5074, json.getDouble("latitude"), 0.0001) - assertEquals(-0.1278, json.getDouble("longitude"), 0.0001) + val result = nullManagerUseCase(function, parameters) - val missingResult = useCase(function, AppFunctionData.EMPTY) - assertTrue(missingResult is ExecuteAppFunctionResult.Error) - assertEquals( - "Missing required parameter 'address'", - (missingResult as ExecuteAppFunctionResult.Error).exception.message, - ) + assertEquals(expectedResult, result) + coVerify(exactly = 1) { agentInternalTools.executeInternalTool(function, parameters) } + verify(exactly = 0) { grantAppFunctionUriPermissionsUseCase(any(), any()) } } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCaseTest.kt index 0fda397a..51417e95 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCaseTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GetAppFunctionsUseCaseTest.kt @@ -170,8 +170,11 @@ class GetAppFunctionsUseCaseTest { val calendarTool = createTool(calendarPkg, "createEvent") val internalTool = createTool(internalPkg, "geocodeAddress") - coEvery { appFunctionManager.searchAppFunctions(any()) } returns - listOf(notesTool, calendarTool) + coEvery { + appFunctionManager.searchAppFunctions( + match { it.packageNames == setOf("com.example.calendar") }, + ) + } returns listOf(calendarTool) every { agentInternalTools.getInternalToolsMetadata() } returns listOf(internalTool to createState(internalTool.name, isEnabled = true)) diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GrantAppFunctionUriPermissionsUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GrantAppFunctionUriPermissionsUseCaseTest.kt new file mode 100644 index 00000000..53db31bf --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/GrantAppFunctionUriPermissionsUseCaseTest.kt @@ -0,0 +1,141 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain.appfunction + +import android.content.Context +import android.content.Intent +import androidx.appfunctions.AppFunctionData +import androidx.appfunctions.AppFunctionName +import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata +import androidx.appfunctions.metadata.AppFunctionPackageMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionResponseMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import io.mockk.mockk +import io.mockk.verify +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +class GrantAppFunctionUriPermissionsUseCaseTest { + private val context = mockk(relaxed = true) + private val useCase = GrantAppFunctionUriPermissionsUseCase(context) + + @Test + fun `invoke grants recursive read URI permissions for content URIs in AppFunctionData`() { + val uriObjectType = + AppFunctionObjectTypeMetadata( + properties = mapOf("uri" to AppFunctionStringTypeMetadata(false)), + required = listOf("uri"), + qualifiedName = "android.net.Uri", + isNullable = false, + ) + val payloadObjectType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "nestedUris" to + AppFunctionArrayTypeMetadata( + itemType = AppFunctionStringTypeMetadata(false), + isNullable = false, + ), + ), + required = listOf("nestedUris"), + qualifiedName = null, + isNullable = false, + ) + val parametersMeta = + listOf( + AppFunctionParameterMetadata( + name = "directUri", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(false), + ), + AppFunctionParameterMetadata( + name = "uriObj", + isRequired = true, + dataType = uriObjectType, + ), + AppFunctionParameterMetadata( + name = "payload", + isRequired = true, + dataType = payloadObjectType, + ), + AppFunctionParameterMetadata( + name = "plainText", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(false), + ), + ) + val components = AppFunctionComponentsMetadata() + val function = + AppFunctionMetadata( + name = AppFunctionName("com.example.notes", "attachMedia"), + schema = null, + parameters = parametersMeta, + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(false), + description = "", + ), + description = "Attach media", + deprecation = null, + packageMetadata = + AppFunctionPackageMetadata( + packageName = "com.example.notes", + components = components, + ), + ) + + val directUri = "content://com.example.appfunctions.agent.fileprovider/cache/direct.png" + val objectUri = "content://com.example.appfunctions.agent.fileprovider/cache/object.png" + val nestedUri = "content://com.example.appfunctions.agent.fileprovider/cache/nested.jpg" + + val uriObjData = + AppFunctionData.Builder(uriObjectType, components) + .setString("uri", objectUri) + .build() + val payloadData = + AppFunctionData.Builder(payloadObjectType, components) + .setStringList("nestedUris", listOf(nestedUri, "https://example.com/ignore")) + .build() + val inputData = + AppFunctionData.Builder(parametersMeta, components) + .setString("directUri", directUri) + .setAppFunctionData("uriObj", uriObjData) + .setAppFunctionData("payload", payloadData) + .setString("plainText", "hello world") + .build() + + useCase(function, inputData) + + for (expectedUri in listOf(directUri, objectUri, nestedUri)) { + verify(exactly = 1) { + context.grantUriPermission( + eq("com.example.notes"), + match { it.toString() == expectedUri }, + eq(Intent.FLAG_GRANT_READ_URI_PERMISSION), + ) + } + } + } +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ResolveRemoteFileToLocalUriUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ResolveRemoteFileToLocalUriUseCaseTest.kt new file mode 100644 index 00000000..0d674986 --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ResolveRemoteFileToLocalUriUseCaseTest.kt @@ -0,0 +1,163 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain.appfunction + +import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import com.example.appfunctions.agent.data.FileDownloadHelper +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Assert.fail +import org.junit.Test +import java.io.IOException + +class ResolveRemoteFileToLocalUriUseCaseTest { + private val components = AppFunctionComponentsMetadata(emptyMap()) + + private class FakeFileDownloadHelper( + private val onDownload: (String) -> String = { + "content://com.example.appfunctions.agent.fileprovider/cache/resolved.png" + }, + ) : FileDownloadHelper { + val downloadedUrls = mutableListOf() + + override suspend fun downloadToLocalUri(url: String): String { + downloadedUrls.add(url) + return onDownload(url) + } + } + + @Test + fun invoke_resolvesRemoteFileReferencesWhileLeavingRegularUrlParametersUntouched() = + runTest { + val fakeHelper = FakeFileDownloadHelper() + val useCase = ResolveRemoteFileToLocalUriUseCase(fakeHelper) + + val uriObjectType = + AppFunctionObjectTypeMetadata( + properties = mapOf("uri" to AppFunctionStringTypeMetadata(false)), + required = listOf("uri"), + qualifiedName = "android.net.Uri", + isNullable = false, + ) + val payloadObjectType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "nestedUris" to + AppFunctionArrayTypeMetadata( + itemType = AppFunctionStringTypeMetadata(false), + isNullable = false, + ), + ), + required = listOf("nestedUris"), + qualifiedName = null, + isNullable = false, + ) + val componentsWithRef = + AppFunctionComponentsMetadata( + dataTypes = mapOf("android.net.Uri" to uriObjectType), + ) + val parameters = + listOf( + AppFunctionParameterMetadata( + name = "imageUri", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(false), + ), + AppFunctionParameterMetadata( + name = "attachment", + isRequired = true, + dataType = AppFunctionReferenceTypeMetadata("android.net.Uri", false), + ), + AppFunctionParameterMetadata( + name = "payload", + isRequired = true, + dataType = payloadObjectType, + ), + AppFunctionParameterMetadata( + name = "websiteUrl", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(false), + ), + ) + val inputs = + mapOf( + "imageUri" to "https://example.com/cat.png", + "attachment" to "https://example.com/doc.png", + "payload" to mapOf("nestedUris" to listOf("https://example.com/nested.png")), + "websiteUrl" to "https://example.com/home", + ) + + val resolved = useCase(parameters, componentsWithRef, inputs) + + assertEquals( + listOf( + "https://example.com/cat.png", + "https://example.com/doc.png", + "https://example.com/nested.png", + ), + fakeHelper.downloadedUrls, + ) + val expectedLocalUri = + "content://com.example.appfunctions.agent.fileprovider/cache/resolved.png" + assertEquals(expectedLocalUri, resolved["imageUri"]) + assertEquals(expectedLocalUri, resolved["attachment"]) + assertEquals( + mapOf("nestedUris" to listOf(expectedLocalUri)), + resolved["payload"], + ) + assertEquals("https://example.com/home", resolved["websiteUrl"]) + } + + @Test + fun invoke_throwsIOExceptionWhenRemoteFileDownloadFails() = + runTest { + val fakeHelper = + FakeFileDownloadHelper { + throw IOException("HTTP 404") + } + val useCase = ResolveRemoteFileToLocalUriUseCase(fakeHelper) + + val parameters = + listOf( + AppFunctionParameterMetadata( + name = "wallpaperUri", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(false), + ), + ) + + try { + useCase( + parameters, + components, + mapOf("wallpaperUri" to "https://example.com/missing.png"), + ) + fail("Expected IOException to be thrown") + } catch (e: IOException) { + assertTrue( + e.message?.contains("Failed to download remote file reference: HTTP 404") == + true, + ) + } + } +}