From 95b2471d325891be9edf867555e67b9bb2c674f7 Mon Sep 17 00:00:00 2001 From: Jebran Syed Date: Wed, 2 Sep 2026 11:22:33 -0700 Subject: [PATCH] Add Tier-2 androidDevice actions to the mobile-2 sample Extends the `androidDevice` client agent with the five medium-risk actions from the Android feature plan. Every one stays a compose/open intent that the user confirms, so no new Android permissions are needed and a hallucinated action cannot complete on its own. ## Actions - `composeEmail` - `ACTION_SENDTO` with a bare `mailto:` URI. `SENDTO` rather than `SEND` so only mail apps resolve, not every share target. Recipients ride as `EXTRA_EMAIL`/`EXTRA_CC`/`EXTRA_BCC` arrays rather than being spliced into the URI. One unusable address fails the whole action: a draft addressed to fewer people than asked for would look like success. - `shareText` - `ACTION_SEND` (`text/plain`) inside `createChooser`, so the user picks the destination. The inner intent is resolved before wrapping, because the chooser is a system activity that always resolves and would otherwise report a false success on a device with no text handler. - `openSettings` - the screen comes from a closed `AndroidSettingsScreen` enum mapped exhaustively to `Settings.ACTION_*`; the model never supplies a raw action string, and `appInfo` is pinned to this app's own package. Screens only display settings; nothing is toggled. - `createCalendarEvent` - `ACTION_INSERT` on `CalendarContract.Events`, which opens the calendar app's pre-filled new-event editor, so no `WRITE_CALENDAR` permission is involved. ISO-8601 times are resolved without `java.time` (minSdk 24, no core-library desugaring). - `playMusicFromSearch` - `MEDIA_PLAY_FROM_SEARCH`. What actually plays is up to the installed app, so the action reports what it dispatched. ## Invariants - Every action reaches the OS through `launchExternalIntent`, keeping the existing resolveActivity / foreground-lifecycle / exception funnel, and completes its callback exactly once. - Each new implicit intent has a matching `` entry; without one Android 11+ package visibility makes `resolveActivity` return null and the action falsely reports that no app is available. - Parsers read strings via `opt(name) as? String`, never `optString`, because Android's `org.json` renders a JSON null as the string "null". - Ambiguity fails the action rather than being guessed at: an unknown music `focus`, an unknown settings screen, an all-day event that also carries a time of day, and a span over 366 days are all rejected. - Calendar offsets are built from a raw integer offset via `SimpleTimeZone`, not by formatting a `GMT+hh:mm` string. `String.format` is locale-sensitive, and under a locale with non-Latin digits `TimeZone.getTimeZone` cannot read the result and silently falls back to GMT - turning `+05:30` into a five-and-a-half-hour shift with no error. - All-day events are anchored at UTC midnight as `CalendarContract` requires, so a user in UTC+10 does not see them land a day early. - `AndroidDeviceSchemaAssetTest` now checks that the settings-screen and music-focus unions in `androidDeviceSchema.ts` match the Kotlin enums, so the two cannot drift apart silently. ## Validation - `gradlew testDebugUnitTest` - 234 tests, 0 failures (was 179). - `gradlew assembleDebug lintDebug` - clean; lint reports only pre-existing informational items. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- android/samples/mobile-2/README.md | 5 + .../mobile-2/app/src/main/AndroidManifest.xml | 71 +++ .../assets/typeagent/androidDeviceSchema.ts | 125 +++++- .../typeagentchat/AndroidDeviceAgent.kt | 62 ++- .../example/typeagentchat/ChatViewModel.kt | 66 +++ .../typeagentchat/ComposeEmailActionParser.kt | 135 ++++++ .../CreateCalendarEventActionParser.kt | 256 +++++++++++ .../com/example/typeagentchat/MainActivity.kt | 282 ++++++++++++ .../typeagentchat/OpenSettingsActionParser.kt | 73 ++++ .../PlayMusicFromSearchActionParser.kt | 77 ++++ .../typeagentchat/ShareTextActionParser.kt | 50 +++ .../example/typeagentchat/WebSocketManager.kt | 35 ++ .../app/src/main/res/values/strings.xml | 2 + .../typeagentchat/AndroidDeviceAgentTest.kt | 88 ++++ .../AndroidDeviceSchemaAssetTest.kt | 30 ++ .../ComposeEmailActionParserTest.kt | 137 ++++++ .../CreateCalendarEventActionParserTest.kt | 406 ++++++++++++++++++ .../OpenSettingsActionParserTest.kt | 63 +++ .../PlayMusicFromSearchActionParserTest.kt | 82 ++++ .../ShareTextActionParserTest.kt | 64 +++ 20 files changed, 2106 insertions(+), 3 deletions(-) create mode 100644 android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ComposeEmailActionParser.kt create mode 100644 android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/CreateCalendarEventActionParser.kt create mode 100644 android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/OpenSettingsActionParser.kt create mode 100644 android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/PlayMusicFromSearchActionParser.kt create mode 100644 android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ShareTextActionParser.kt create mode 100644 android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ComposeEmailActionParserTest.kt create mode 100644 android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/CreateCalendarEventActionParserTest.kt create mode 100644 android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/OpenSettingsActionParserTest.kt create mode 100644 android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/PlayMusicFromSearchActionParserTest.kt create mode 100644 android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ShareTextActionParserTest.kt diff --git a/android/samples/mobile-2/README.md b/android/samples/mobile-2/README.md index 5358258673..4ecb83b60c 100644 --- a/android/samples/mobile-2/README.md +++ b/android/samples/mobile-2/README.md @@ -106,6 +106,11 @@ interaction traffic. | `composeSms` | `Intent.ACTION_SENDTO` with an `smsto:` URI and `sms_body` | Opens a pre-filled draft — the user still presses send, so no `SEND_SMS` permission is needed. With no recipient the draft opens with an empty To field; an *unusable* recipient is rejected rather than silently dropped. | | `webSearch` | `Intent.ACTION_WEB_SEARCH` with `SearchManager.QUERY` | The query travels as an extra rather than being spliced into a URL, so it needs no encoding. | | `openWebPage` | `Intent.ACTION_VIEW` with an `http`/`https` URI | The scheme allowlist is the load-bearing check: `ACTION_VIEW` would otherwise follow `market:`, `file:` or any app's own deep-link scheme, turning "open this page" into an arbitrary-app launcher driven by text the model read. URLs containing whitespace are refused rather than repaired into a different host. | +| `composeEmail` | `Intent.ACTION_SENDTO` with a bare `mailto:` URI plus `EXTRA_EMAIL`/`EXTRA_CC`/`EXTRA_BCC` | Opens a draft — the user still presses send, so nothing leaves the device unattended. `ACTION_SENDTO` is used rather than `ACTION_SEND` so only mail apps resolve, not every share target. Recipients ride as extras rather than being spliced into the URI, which keeps encoding out of the picture. One unusable address fails the whole action: a draft addressed to fewer people than asked for looks like success. | +| `shareText` | `Intent.ACTION_SEND` (`text/plain`) wrapped in `Intent.createChooser` | The user picks the destination app, so the model never chooses where the text goes. `createChooser` always resolves, so the inner `ACTION_SEND` intent is resolved first — otherwise a device with no text handler would report a false success. Newlines survive here (unlike the URI-bound actions) because shared text is a document, not a query. | +| `openSettings` | `Settings.ACTION_*` for a fixed screen | The model picks from a closed `AndroidSettingsScreen` enum, never a raw action string, so it cannot be steered into an arbitrary system activity. Screens only *display* settings; nothing is toggled. `appInfo` is pinned to this app's own package. | +| `createCalendarEvent` | `Intent.ACTION_INSERT` on `CalendarContract.Events.CONTENT_URI` | Opens the calendar app's pre-filled *new event* editor — the user still saves it, so no `WRITE_CALENDAR` permission is needed. Times are ISO-8601 and are resolved without `java.time` (minSdk 24). All-day events are anchored at UTC midnight as `CalendarContract` requires; an all-day event that also carries a time of day is rejected rather than guessed at. Spans longer than 366 days are refused. | +| `playMusicFromSearch` | `MediaStore.INTENT_ACTION_MEDIA_PLAY_FROM_SEARCH` | Asks whichever music app claims the intent to play the best match. What actually plays is up to that app, so the action reports what it dispatched, not what started. An unrecognised `focus` fails rather than falling back to `any`, which would quietly search for something broader than asked. | All actions require the app to be in the foreground: Android 10+ silently refuses background activity starts (no exception is thrown), so the app checks its own diff --git a/android/samples/mobile-2/app/src/main/AndroidManifest.xml b/android/samples/mobile-2/app/src/main/AndroidManifest.xml index d77717baf9..2c5719a2d9 100644 --- a/android/samples/mobile-2/app/src/main/AndroidManifest.xml +++ b/android/samples/mobile-2/app/src/main/AndroidManifest.xml @@ -47,6 +47,77 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/android/samples/mobile-2/app/src/main/assets/typeagent/androidDeviceSchema.ts b/android/samples/mobile-2/app/src/main/assets/typeagent/androidDeviceSchema.ts index 581e340b85..f8573d97ff 100644 --- a/android/samples/mobile-2/app/src/main/assets/typeagent/androidDeviceSchema.ts +++ b/android/samples/mobile-2/app/src/main/assets/typeagent/androidDeviceSchema.ts @@ -11,7 +11,12 @@ export type AndroidDeviceAction = | DialPhoneNumberAction | ComposeSmsAction | WebSearchAction - | OpenWebPageAction; + | OpenWebPageAction + | ComposeEmailAction + | ShareTextAction + | OpenSettingsAction + | CreateCalendarEventAction + | PlayMusicFromSearchAction; // A day of the week an alarm can repeat on. export type AlarmRepeatDay = @@ -140,3 +145,121 @@ export type OpenWebPageAction = { url: string; }; }; + +// Opens the Android device's email app on a pre-filled draft message. +// The user still has to press send, so this never sends mail by itself. +// Use when the user asks to email someone. +export type ComposeEmailAction = { + actionName: "composeEmail"; + parameters: { + // The original user request. + originalRequest: string; + // Recipient email addresses. Omit when the user did not name anyone; + // the email app then opens with an empty To field. + to?: string[]; + // Carbon-copy recipients. + cc?: string[]; + // Blind carbon-copy recipients. + bcc?: string[]; + // The subject line. + subject?: string; + // The message body. + body?: string; + }; +}; + +// Offers a piece of text to the Android share sheet so the user can pass it to +// another app. Use only when the user explicitly asks to share, send or post +// some text they have named; the user picks the destination app and confirms. +export type ShareTextAction = { + actionName: "shareText"; + parameters: { + // The original user request. + originalRequest: string; + // The exact text to share. + text: string; + // An optional title or subject offered to apps that can use one. + subject?: string; + }; +}; + +// A settings screen this device agent is allowed to open. +export type AndroidSettingsScreen = + | "settings" + | "wifi" + | "bluetooth" + | "display" + | "sound" + | "location" + | "battery" + | "airplaneMode" + | "dateAndTime" + | "storage" + | "accessibility" + | "security" + | "appInfo"; + +// Opens one of the Android device's settings screens. +// Use when the user asks to change a device setting - the app cannot toggle +// settings directly, so it takes the user to the right screen instead. +export type OpenSettingsAction = { + actionName: "openSettings"; + parameters: { + // The original user request. + originalRequest: string; + // Which settings screen to open. Only these screens are supported; + // "settings" is the top-level settings app and "appInfo" is this chat + // app's own details page. + screen: AndroidSettingsScreen; + }; +}; + +// Opens the Android device's calendar app on a pre-filled new event. +// The user still has to save it, so this never writes to a calendar by itself. +// Use when the user asks to create, schedule or add a calendar event. +export type CreateCalendarEventAction = { + actionName: "createCalendarEvent"; + parameters: { + // The original user request. + originalRequest: string; + // The event title. + title: string; + // Local start time as ISO-8601 without a time zone, for example + // "2026-08-24T15:00". For an all-day event use a date only, + // "2026-08-24". Times are interpreted in the device's own time zone, + // so never convert to UTC and never append "Z" or an offset. + start: string; + // Local end time in the same format as start. Omit for a one hour + // event, or a single day when allDay is true. + end?: string; + // True when the event covers whole days rather than a time of day. + allDay?: boolean; + // Where the event takes place. + location?: string; + // Longer notes about the event. + description?: string; + }; +}; + +// What a music search query names. +export type MusicSearchFocus = + | "any" + | "artist" + | "album" + | "song" + | "playlist"; + +// Asks the Android device's music app to play something matching a search. +// Use when the user asks to play music, an artist, an album or a song. +export type PlayMusicFromSearchAction = { + actionName: "playMusicFromSearch"; + parameters: { + // The original user request. + originalRequest: string; + // What to search for, for example "Kind of Blue" or "Miles Davis". + query: string; + // What the query names. Omit or use "any" when it is unclear, which + // lets the music app decide. + focus?: MusicSearchFocus; + }; +}; diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AndroidDeviceAgent.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AndroidDeviceAgent.kt index 42a39e6565..31868ea644 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AndroidDeviceAgent.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AndroidDeviceAgent.kt @@ -10,8 +10,9 @@ internal object AndroidDeviceAgent { private const val AGENT_DESCRIPTION = "Acts on this Android device: sets alarms and countdown timers, shows the " + "alarm and timer lists, searches for nearby places, shows a place on the " + - "map, opens the dialer or a text message draft, runs a web search and " + - "opens web pages." + "map, opens the dialer or a text message draft, runs a web search, " + + "opens web pages, drafts email, shares text with another app, opens " + + "device settings screens, drafts calendar events and plays music." fun createRegistrationParams( conversationId: String, @@ -143,6 +144,56 @@ internal object AndroidDeviceAgent { AndroidDeviceActionParseResult.Success(AndroidDeviceAction.OpenWebPage(parsed)) } + "composeEmail" -> { + val parsed = parseComposeEmailActionPayload(parameters) + ?: return AndroidDeviceActionParseResult.ActionError( + "Invalid composeEmail parameters: every recipient must be a " + + "valid email address and the draft cannot be empty." + ) + AndroidDeviceActionParseResult.Success(AndroidDeviceAction.ComposeEmail(parsed)) + } + + "shareText" -> { + val parsed = parseShareTextActionPayload(parameters) + ?: return AndroidDeviceActionParseResult.ActionError( + "Invalid shareText parameters: text is required." + ) + AndroidDeviceActionParseResult.Success(AndroidDeviceAction.ShareText(parsed)) + } + + "openSettings" -> { + val parsed = parseOpenSettingsActionPayload(parameters) + ?: return AndroidDeviceActionParseResult.ActionError( + "Invalid openSettings parameters: screen must be one of " + + AndroidSettingsScreen.entries.joinToString { it.schemaName } + "." + ) + AndroidDeviceActionParseResult.Success(AndroidDeviceAction.OpenSettings(parsed)) + } + + "createCalendarEvent" -> { + val parsed = parseCreateCalendarEventActionPayload(parameters) + ?: return AndroidDeviceActionParseResult.ActionError( + "Invalid createCalendarEvent parameters: title is required and " + + "start/end must be local ISO-8601 values such as " + + "2026-08-24T15:00, with end after start." + ) + AndroidDeviceActionParseResult.Success( + AndroidDeviceAction.CreateCalendarEvent(parsed) + ) + } + + "playMusicFromSearch" -> { + val parsed = parsePlayMusicFromSearchActionPayload(parameters) + ?: return AndroidDeviceActionParseResult.ActionError( + "Invalid playMusicFromSearch parameters: query is required and " + + "focus must be one of " + + MusicSearchFocus.entries.joinToString { it.schemaName } + "." + ) + AndroidDeviceActionParseResult.Success( + AndroidDeviceAction.PlayMusicFromSearch(parsed) + ) + } + else -> AndroidDeviceActionParseResult.ActionError( "Unsupported Android agent action: $actionName" ) @@ -175,6 +226,13 @@ internal sealed interface AndroidDeviceAction { data class ComposeSms(val action: ComposeSmsAction) : AndroidDeviceAction data class WebSearch(val action: WebSearchAction) : AndroidDeviceAction data class OpenWebPage(val action: OpenWebPageAction) : AndroidDeviceAction + data class ComposeEmail(val action: ComposeEmailAction) : AndroidDeviceAction + data class ShareText(val action: ShareTextAction) : AndroidDeviceAction + data class OpenSettings(val action: OpenSettingsAction) : AndroidDeviceAction + data class CreateCalendarEvent(val action: CreateCalendarEventAction) : AndroidDeviceAction + data class PlayMusicFromSearch( + val action: PlayMusicFromSearchAction + ) : AndroidDeviceAction } internal sealed interface AndroidDeviceActionParseResult { diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ChatViewModel.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ChatViewModel.kt index ddc8859518..3126d474bf 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ChatViewModel.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ChatViewModel.kt @@ -77,6 +77,31 @@ internal sealed interface ClientAction { val action: OpenWebPageAction, val completion: (AndroidDeviceExecutionResult) -> Unit ) : ClientAction + + data class ComposeEmail( + val action: ComposeEmailAction, + val completion: (AndroidDeviceExecutionResult) -> Unit + ) : ClientAction + + data class ShareText( + val action: ShareTextAction, + val completion: (AndroidDeviceExecutionResult) -> Unit + ) : ClientAction + + data class OpenSettings( + val action: OpenSettingsAction, + val completion: (AndroidDeviceExecutionResult) -> Unit + ) : ClientAction + + data class CreateCalendarEvent( + val action: CreateCalendarEventAction, + val completion: (AndroidDeviceExecutionResult) -> Unit + ) : ClientAction + + data class PlayMusicFromSearch( + val action: PlayMusicFromSearchAction, + val completion: (AndroidDeviceExecutionResult) -> Unit + ) : ClientAction } /** @@ -207,6 +232,47 @@ class ChatViewModel(application: Application) : AndroidViewModel(application) { ) { dispatchClientAction(ClientAction.OpenWebPage(action, completion), completion) } + + override fun onComposeEmail( + action: ComposeEmailAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + dispatchClientAction(ClientAction.ComposeEmail(action, completion), completion) + } + + override fun onShareText( + action: ShareTextAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + dispatchClientAction(ClientAction.ShareText(action, completion), completion) + } + + override fun onOpenSettings( + action: OpenSettingsAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + dispatchClientAction(ClientAction.OpenSettings(action, completion), completion) + } + + override fun onCreateCalendarEvent( + action: CreateCalendarEventAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + dispatchClientAction( + ClientAction.CreateCalendarEvent(action, completion), + completion + ) + } + + override fun onPlayMusicFromSearch( + action: PlayMusicFromSearchAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + dispatchClientAction( + ClientAction.PlayMusicFromSearch(action, completion), + completion + ) + } }) webSocketManager.setStaleConversationHandler { diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ComposeEmailActionParser.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ComposeEmailActionParser.kt new file mode 100644 index 0000000000..8fcf6b0efc --- /dev/null +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ComposeEmailActionParser.kt @@ -0,0 +1,135 @@ +package com.example.typeagentchat + +import org.json.JSONArray +import org.json.JSONObject + +internal data class ComposeEmailAction( + val originalRequest: String, + val to: List, + val cc: List, + val bcc: List, + val subject: String, + val body: String +) + +/** RFC 5321 caps a path at 256 octets including the angle brackets. */ +private const val MAX_EMAIL_ADDRESS_CHARS = 254 + +/** + * A single draft cannot usefully name more people than this, and the cap keeps + * a runaway model from building an extra that approaches the binder budget. + */ +private const val MAX_EMAIL_RECIPIENTS = 32 + +/** + * An email body is the longest field any action carries. Still three orders of + * magnitude below the ~1 MB binder transaction limit. + */ +private const val MAX_EMAIL_BODY_CHARS = 8_000 + +/** + * Deliberately loose: the point is to reject values that are obviously not + * addresses - a name, a sentence, a phone number - not to re-derive RFC 5322, + * which no practical regex captures. The receiving email app does the real + * validation, and the user sees the draft before anything is sent. + */ +private val emailAddressRegex = Regex("""^[^\s@,;<>]+@[^\s@,;<>]+\.[^\s@,;<>]+$""") + +/** + * Parses the `parameters` of the `composeEmail` action declared by + * `androidDeviceSchema.ts`: + * + * ```ts + * parameters: { + * originalRequest: string; to?: string[]; cc?: string[]; bcc?: string[]; + * subject?: string; body?: string; + * } + * ``` + * + * The result is used with `Intent.ACTION_SENDTO` and a `mailto:` URI, which + * opens the email app on a draft - the user still has to press send. `mailto:` + * rather than `ACTION_SEND` on purpose: `ACTION_SEND` would offer the message + * to every share target on the device, so a draft meant for an inbox could end + * up in a social app instead. + * + * Every address is validated and a bad one fails the whole action. Dropping it + * would be worse: the user would get a draft that quietly reaches fewer people + * than they asked for, and the model would never learn it got the address wrong. + */ +internal fun parseComposeEmailActionPayload(data: Any?): ComposeEmailAction? { + val payload = data as? JSONObject ?: return null + + val to = payload.parseEmailAddressList("to") ?: return null + val cc = payload.parseEmailAddressList("cc") ?: return null + val bcc = payload.parseEmailAddressList("bcc") ?: return null + val subject = payload.sanitizedActionText("subject") + val body = payload.sanitizedActionText("body", MAX_EMAIL_BODY_CHARS) + + // An entirely empty draft is not something a user ever asks for, so it is + // treated as a failed translation rather than silently opening a blank + // compose window the user then has to dismiss. + if (to.isEmpty() && cc.isEmpty() && bcc.isEmpty() && subject.isEmpty() && body.isEmpty()) { + return null + } + + return ComposeEmailAction( + originalRequest = payload.sanitizedActionText("originalRequest"), + to = to, + cc = cc, + bcc = bcc, + subject = subject, + body = body + ) +} + +/** + * Reads one optional address array. + * + * @return the validated addresses, an empty list when the field is absent, or + * null when the field is present but unusable - which fails the action. + */ +private fun JSONObject.parseEmailAddressList(name: String): List? { + val raw = opt(name) + if (raw == null || raw == JSONObject.NULL) { + return emptyList() + } + // A lone string where the schema says string[] is a common model slip and + // costs nothing to accept. + if (raw is String) { + val address = normalizeEmailAddress(raw) ?: return null + return listOf(address) + } + val array = raw as? JSONArray ?: return null + if (array.length() > MAX_EMAIL_RECIPIENTS) { + return null + } + + val addresses = mutableListOf() + for (index in 0 until array.length()) { + val entry = array.opt(index) as? String ?: return null + val address = normalizeEmailAddress(entry) ?: return null + if (address !in addresses) { + addresses.add(address) + } + } + return addresses +} + +/** + * Trims and validates a single address. + * + * Whitespace is only trimmed at the edges - an interior space means two + * addresses were run together or the value is not an address at all, and either + * way repairing it would guess at who the user meant. + */ +internal fun normalizeEmailAddress(raw: String): String? { + val address = raw.trim() + if (address.isEmpty() || address.length > MAX_EMAIL_ADDRESS_CHARS) { + return null + } + return if (emailAddressRegex.matches(address)) address else null +} + +/** True when [address] is usable as an email recipient. */ +internal fun isSupportedEmailAddress(address: String): Boolean = + normalizeEmailAddress(address) != null diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/CreateCalendarEventActionParser.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/CreateCalendarEventActionParser.kt new file mode 100644 index 0000000000..b76168bda3 --- /dev/null +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/CreateCalendarEventActionParser.kt @@ -0,0 +1,256 @@ +package com.example.typeagentchat + +import org.json.JSONObject +import java.util.Calendar +import java.util.GregorianCalendar +import java.util.SimpleTimeZone +import java.util.TimeZone + +internal data class CreateCalendarEventAction( + val originalRequest: String, + val title: String, + /** Epoch milliseconds for `CalendarContract.EXTRA_EVENT_BEGIN_TIME`. */ + val startMillis: Long, + /** Epoch milliseconds for `CalendarContract.EXTRA_EVENT_END_TIME`, always after [startMillis]. */ + val endMillis: Long, + val allDay: Boolean, + val location: String, + val description: String +) + +private const val MAX_EVENT_TITLE_CHARS = 256 +private const val MAX_EVENT_LOCATION_CHARS = 256 +private const val MAX_EVENT_DESCRIPTION_CHARS = 4_000 + +private const val MILLIS_PER_MINUTE = 60_000L +private const val MILLIS_PER_HOUR = 60L * MILLIS_PER_MINUTE +private const val MILLIS_PER_DAY = 24L * MILLIS_PER_HOUR + +/** Default length of an event whose end the user did not give. */ +private const val DEFAULT_EVENT_MILLIS = MILLIS_PER_HOUR + +/** + * Anything longer is far likelier to be a mis-parsed year than a real event, + * and a multi-decade block dropped into someone's calendar is annoying to undo. + */ +private const val MAX_EVENT_SPAN_MILLIS = 366L * MILLIS_PER_DAY + +/** Guards against a mistyped or hallucinated year landing an event in year 9999. */ +private val supportedYears = 1970..2200 + +/** + * `YYYY-MM-DD` on its own, or with a `T`-separated local time and an optional + * `Z`/`+hh:mm` offset. + * + * A space is accepted in place of `T` and lower case `t`/`z` are allowed + * because models produce both; neither changes the meaning. + * + * Fractional seconds are accepted (RFC 3339 permits them and models emit them) + * but discarded: calendar UIs are minute-granular, so sub-second precision is + * not something the user could observe, and rejecting an otherwise valid + * timestamp over it would fail the action for no benefit. + */ +private val isoDateTimeRegex = Regex( + """^(\d{4})-(\d{2})-(\d{2})(?:[Tt ](\d{2}):(\d{2})(?::(\d{2})(?:[.,]\d{1,9})?)?\s*([Zz]|[+-]\d{2}(?::?\d{2})?)?)?$""" +) + +/** + * A parsed ISO-8601 value, before it is turned into an instant. + * + * The time zone is kept separate from the fields because the same wall-clock + * value means different instants depending on whose clock it is: an all-day + * event is anchored to UTC midnight by `CalendarContract`, a floating local time + * belongs to the device's zone, and an explicit offset overrides both. + */ +private data class IsoDateTimeParts( + val year: Int, + val month: Int, + val day: Int, + val hour: Int, + val minute: Int, + val second: Int, + val hasTimeOfDay: Boolean, + val explicitZone: TimeZone? +) + +/** + * Parses the `parameters` of the `createCalendarEvent` action declared by + * `androidDeviceSchema.ts`: + * + * ```ts + * parameters: { + * originalRequest: string; title: string; start: string; end?: string; + * allDay?: boolean; location?: string; description?: string; + * } + * ``` + * + * The result is used with `Intent.ACTION_INSERT` on the calendar provider, + * which opens the calendar app's new-event editor pre-filled - the user still + * has to save. No calendar permission is involved, because the calendar app + * does the write. + * + * Date handling is the whole risk in this action. `CalendarContract` wants epoch + * milliseconds, and the difference between "3pm" in the device's zone and 3pm + * UTC is a silently wrong entry in someone's calendar, so: + * - a value with no offset is read in [timeZone], the device's own zone; + * - a value carrying `Z` or `+hh:mm` is honoured at that offset rather than + * being rejected, since models emit those routinely; + * - an all-day event is anchored to UTC midnight, which is what the provider + * documents and what every calendar app expects. + * + * @param timeZone the zone a value without an explicit offset is read in. + * Injectable so the tests do not depend on the machine's zone. + */ +internal fun parseCreateCalendarEventActionPayload( + data: Any?, + timeZone: TimeZone = TimeZone.getDefault() +): CreateCalendarEventAction? { + val payload = data as? JSONObject ?: return null + + val title = payload.sanitizedActionText("title", MAX_EVENT_TITLE_CHARS) + if (title.isEmpty()) { + return null + } + + val allDay = when (val raw = payload.opt("allDay")) { + null, JSONObject.NULL -> false + is Boolean -> raw + // "true"/"false" as a string is a common model slip and is unambiguous. + is String -> raw.trim().lowercase().toBooleanStrictOrNull() ?: return null + else -> return null + } + + val start = parseIsoDateTime(payload.sanitizedActionText("start")) ?: return null + // The schema asks for a date only when allDay is set. A time of day here + // means the model contradicted itself, and either reading of it - honour + // the time, or drop it - would put something in the calendar the user did + // not ask for, so the action fails and says so instead. + if (allDay && start.hasTimeOfDay) { + return null + } + + val rawEnd = payload.sanitizedActionText("end") + val end = if (rawEnd.isEmpty()) null else parseIsoDateTime(rawEnd) ?: return null + if (end != null && allDay && end.hasTimeOfDay) { + return null + } + + val startMillis = start.toEpochMillis(timeZone, allDay) ?: return null + val endMillis = when { + end != null -> { + val parsed = end.toEpochMillis(timeZone, allDay) ?: return null + // For an all-day event the user names the last day they mean, while + // the provider wants an exclusive end, so the final day is added on. + if (allDay) parsed + MILLIS_PER_DAY else parsed + } + + allDay -> startMillis + MILLIS_PER_DAY + else -> startMillis + DEFAULT_EVENT_MILLIS + } + + if (endMillis <= startMillis || endMillis - startMillis > MAX_EVENT_SPAN_MILLIS) { + return null + } + + return CreateCalendarEventAction( + originalRequest = payload.sanitizedActionText("originalRequest"), + title = title, + startMillis = startMillis, + endMillis = endMillis, + allDay = allDay, + location = payload.sanitizedActionText("location", MAX_EVENT_LOCATION_CHARS), + description = payload.sanitizedActionText("description", MAX_EVENT_DESCRIPTION_CHARS) + ) +} + +/** Parses an ISO-8601 local date, or date and time, into its fields. */ +private fun parseIsoDateTime(value: String): IsoDateTimeParts? { + val match = isoDateTimeRegex.matchEntire(value.trim()) ?: return null + val (year, month, day, hour, minute, second, offset) = match.destructured + + val explicitZone = if (offset.isEmpty()) null else parseOffsetZone(offset) ?: return null + + return IsoDateTimeParts( + year = year.toInt(), + month = month.toInt(), + day = day.toInt(), + hour = if (hour.isEmpty()) 0 else hour.toInt(), + minute = if (minute.isEmpty()) 0 else minute.toInt(), + second = if (second.isEmpty()) 0 else second.toInt(), + hasTimeOfDay = hour.isNotEmpty(), + explicitZone = explicitZone + ) +} + +/** + * Turns `Z` or `+hh:mm` into a fixed-offset zone. + * + * The zone is built from a raw offset in milliseconds rather than by handing + * `TimeZone.getTimeZone` a `GMT+hh:mm` string. Two reasons: that method falls + * back to GMT for anything it cannot read rather than failing, and formatting + * the string with `"%02d"` uses the default locale, which in locales with + * non-Latin digits emits characters `getTimeZone` cannot read - so a valid + * `+05:30` would silently become UTC and move the event five and a half hours. + * A raw integer offset cannot be misread. + * + * The offset is still range-checked, since `+99:00` is not a real zone. + */ +private fun parseOffsetZone(offset: String): TimeZone? { + if (offset.equals("Z", ignoreCase = true)) { + return TimeZone.getTimeZone("UTC") + } + val sign = if (offset[0] == '-') -1 else 1 + val digits = offset.substring(1).replace(":", "") + // ISO-8601 allows the minutes to be omitted, so "+05" means "+05:00". + val minutePart = when (digits.length) { + 2 -> "00" + 4 -> digits.substring(2, 4) + else -> return null + } + val hours = digits.substring(0, 2).toInt() + val minutes = minutePart.toInt() + if (hours > 18 || minutes > 59) { + return null + } + val offsetMillis = (sign * (hours * MILLIS_PER_HOUR + minutes * MILLIS_PER_MINUTE)).toInt() + if (offsetMillis == 0) { + return TimeZone.getTimeZone("UTC") + } + // SimpleTimeZone with no DST rule is exactly an ISO-8601 fixed offset. + return SimpleTimeZone(offsetMillis, "UTC$offset") +} + +/** + * Resolves the fields to an instant, or null when they do not describe a real + * date - 31 February and 25 o'clock included. + * + * @param allDay anchors the value to UTC midnight, which is how + * `CalendarContract` stores all-day events; without it a user in UTC+10 would + * see the event land on the previous day. + */ +private fun IsoDateTimeParts.toEpochMillis(timeZone: TimeZone, allDay: Boolean): Long? { + if (year !in supportedYears) { + return null + } + val zone = when { + allDay -> TimeZone.getTimeZone("UTC") + else -> explicitZone ?: timeZone + } + val calendar = GregorianCalendar(zone).apply { + // Non-lenient so 2026-02-31 is rejected instead of rolling into March, + // which would put the event on a day the user never named. + isLenient = false + clear() + set(Calendar.YEAR, year) + set(Calendar.MONTH, month - 1) + set(Calendar.DAY_OF_MONTH, day) + set(Calendar.HOUR_OF_DAY, hour) + set(Calendar.MINUTE, minute) + set(Calendar.SECOND, second) + } + return try { + calendar.timeInMillis + } catch (_: IllegalArgumentException) { + null + } +} diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt index 04e24d12e6..3e0f821a80 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/MainActivity.kt @@ -10,6 +10,9 @@ import android.net.Uri import android.os.Build import android.os.Bundle import android.provider.AlarmClock +import android.provider.CalendarContract +import android.provider.MediaStore +import android.provider.Settings import android.speech.RecognizerIntent import android.speech.SpeechRecognizer import android.util.Log @@ -79,6 +82,9 @@ import com.example.typeagentchat.ui.theme.TypeAgentChatTheme import kotlinx.coroutines.CancellationException import kotlinx.coroutines.launch import kotlinx.coroutines.withTimeoutOrNull +import java.text.DateFormat +import java.util.Date +import java.util.TimeZone class MainActivity : ComponentActivity() { @@ -132,6 +138,16 @@ class MainActivity : ComponentActivity() { launchWebSearchIntent(action.action, action.completion) is ClientAction.OpenWebPage -> launchOpenWebPageIntent(action.action, action.completion) + is ClientAction.ComposeEmail -> + launchComposeEmailIntent(action.action, action.completion) + is ClientAction.ShareText -> + launchShareTextIntent(action.action, action.completion) + is ClientAction.OpenSettings -> + launchOpenSettingsIntent(action.action, action.completion) + is ClientAction.CreateCalendarEvent -> + launchCreateCalendarEventIntent(action.action, action.completion) + is ClientAction.PlayMusicFromSearch -> + launchPlayMusicFromSearchIntent(action.action, action.completion) } } catch (cancellation: CancellationException) { // The action was already taken off the channel, so no other @@ -197,6 +213,11 @@ class MainActivity : ComponentActivity() { is ClientAction.ComposeSms -> completion(result) is ClientAction.WebSearch -> completion(result) is ClientAction.OpenWebPage -> completion(result) + is ClientAction.ComposeEmail -> completion(result) + is ClientAction.ShareText -> completion(result) + is ClientAction.OpenSettings -> completion(result) + is ClientAction.CreateCalendarEvent -> completion(result) + is ClientAction.PlayMusicFromSearch -> completion(result) } } @@ -480,6 +501,267 @@ class MainActivity : ComponentActivity() { ) } + /** + * Handles the `composeEmail` action by opening a pre-filled draft. + * + * `ACTION_SENDTO` with a bare `mailto:` URI rather than `ACTION_SEND`: only + * email apps claim `mailto:`, so a message meant for an inbox cannot be + * routed into a social or messaging app by mistake. Recipients ride as + * extras, which is what mail apps read when the URI carries no address. + */ + private fun launchComposeEmailIntent( + action: ComposeEmailAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + val intent = Intent(Intent.ACTION_SENDTO, Uri.parse("mailto:")).apply { + if (action.to.isNotEmpty()) { + putExtra(Intent.EXTRA_EMAIL, action.to.toTypedArray()) + } + if (action.cc.isNotEmpty()) { + putExtra(Intent.EXTRA_CC, action.cc.toTypedArray()) + } + if (action.bcc.isNotEmpty()) { + putExtra(Intent.EXTRA_BCC, action.bcc.toTypedArray()) + } + if (action.subject.isNotEmpty()) { + putExtra(Intent.EXTRA_SUBJECT, action.subject) + } + if (action.body.isNotEmpty()) { + putExtra(Intent.EXTRA_TEXT, action.body) + } + } + val recipient = action.to.firstOrNull() ?: "a new message" + launchExternalIntent( + intent = intent, + actionName = "compose-email", + detail = "to=${action.to.size} cc=${action.cc.size} bcc=${action.bcc.size} " + + "bodyChars=${action.body.length}", + successMessage = "Email draft opened for $recipient", + missingAppMessage = "No email app is available on this device.", + deniedMessage = "This app is not allowed to open the email app.", + backgroundMessage = "Could not open the email app while the app was in the background.", + completion = completion + ) + } + + /** + * Handles the `shareText` action by opening the system share sheet. + * + * The chooser is what makes this safe: the destination is every app on the + * device, so the user - not the model - picks where the text goes. + * + * The inner `ACTION_SEND` intent is resolved before the chooser is built. + * `createChooser` always resolves, since the chooser itself is a system + * activity, so without this check a device with no text-sharing app would + * get an empty share sheet while the agent was told the action succeeded. + */ + private fun launchShareTextIntent( + action: ShareTextAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + val shareIntent = Intent(Intent.ACTION_SEND).apply { + type = "text/plain" + putExtra(Intent.EXTRA_TEXT, action.text) + if (action.subject.isNotEmpty()) { + // Read by email and note apps; ignored by everything else. + putExtra(Intent.EXTRA_SUBJECT, action.subject) + } + } + val missingAppMessage = "No app on this device can accept shared text." + if (shareIntent.resolveActivity(packageManager) == null) { + Log.e(TAG, "No app available to handle share-text intent") + Toast.makeText(this, missingAppMessage, Toast.LENGTH_SHORT).show() + completion(AndroidDeviceExecutionResult.Failure(missingAppMessage)) + return + } + + launchExternalIntent( + intent = Intent.createChooser(shareIntent, getString(R.string.share_chooser_title)), + actionName = "share-text", + detail = "textChars=${action.text.length}", + successMessage = "Share sheet opened", + missingAppMessage = missingAppMessage, + deniedMessage = "This app is not allowed to open the share sheet.", + backgroundMessage = + "Could not open the share sheet while the app was in the background.", + completion = completion + ) + } + + /** + * Handles the `openSettings` action by opening one settings screen. + * + * An ordinary app cannot toggle wifi, Bluetooth or Do Not Disturb - the + * platform removed those APIs - so taking the user to the right screen is + * the honest version of "turn on wifi for me". + */ + private fun launchOpenSettingsIntent( + action: OpenSettingsAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + val screen = action.screen + launchExternalIntent( + intent = buildSettingsIntent(screen), + actionName = "open-settings", + detail = "screen=${screen.schemaName}", + successMessage = "Opening ${screen.displayName}", + missingAppMessage = "This device has no ${screen.displayName} screen.", + deniedMessage = "This app is not allowed to open ${screen.displayName}.", + backgroundMessage = + "Could not open ${screen.displayName} while the app was in the background.", + completion = completion + ) + } + + /** + * Maps the closed [AndroidSettingsScreen] set onto platform intents. + * + * Exhaustive on purpose: the model never supplies an action string, so + * adding a screen means adding a branch here rather than widening what an + * `openSettings` request can start. + */ + private fun buildSettingsIntent(screen: AndroidSettingsScreen): Intent = when (screen) { + AndroidSettingsScreen.Settings -> Intent(Settings.ACTION_SETTINGS) + AndroidSettingsScreen.Wifi -> Intent(Settings.ACTION_WIFI_SETTINGS) + AndroidSettingsScreen.Bluetooth -> Intent(Settings.ACTION_BLUETOOTH_SETTINGS) + AndroidSettingsScreen.Display -> Intent(Settings.ACTION_DISPLAY_SETTINGS) + AndroidSettingsScreen.Sound -> Intent(Settings.ACTION_SOUND_SETTINGS) + AndroidSettingsScreen.Location -> Intent(Settings.ACTION_LOCATION_SOURCE_SETTINGS) + AndroidSettingsScreen.Battery -> Intent(Settings.ACTION_BATTERY_SAVER_SETTINGS) + AndroidSettingsScreen.AirplaneMode -> Intent(Settings.ACTION_AIRPLANE_MODE_SETTINGS) + AndroidSettingsScreen.DateAndTime -> Intent(Settings.ACTION_DATE_SETTINGS) + AndroidSettingsScreen.Storage -> Intent(Settings.ACTION_INTERNAL_STORAGE_SETTINGS) + AndroidSettingsScreen.Accessibility -> Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS) + AndroidSettingsScreen.Security -> Intent(Settings.ACTION_SECURITY_SETTINGS) + // Pinned to this app's own package. Taking a package name from the + // model would turn this into "open any installed app's settings". + AndroidSettingsScreen.AppInfo -> Intent( + Settings.ACTION_APPLICATION_DETAILS_SETTINGS, + Uri.fromParts("package", packageName, null) + ) + } + + /** + * Handles the `createCalendarEvent` action by opening the calendar app's + * new-event editor pre-filled. + * + * `ACTION_INSERT` on the provider's events URI, never a direct write: the + * calendar app owns the insert, so no calendar permission is needed and the + * user sees the event before it is saved. + */ + private fun launchCreateCalendarEventIntent( + action: CreateCalendarEventAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + val intent = Intent(Intent.ACTION_INSERT) + .setData(CalendarContract.Events.CONTENT_URI) + .putExtra(CalendarContract.Events.TITLE, action.title) + .putExtra(CalendarContract.EXTRA_EVENT_BEGIN_TIME, action.startMillis) + .putExtra(CalendarContract.EXTRA_EVENT_END_TIME, action.endMillis) + .putExtra(CalendarContract.EXTRA_EVENT_ALL_DAY, action.allDay) + if (action.location.isNotEmpty()) { + intent.putExtra(CalendarContract.Events.EVENT_LOCATION, action.location) + } + if (action.description.isNotEmpty()) { + intent.putExtra(CalendarContract.Events.DESCRIPTION, action.description) + } + + val whenLabel = formatEventStart(action) + launchExternalIntent( + intent = intent, + actionName = "create-calendar-event", + detail = "allDay=${action.allDay} start=${action.startMillis} end=${action.endMillis}", + successMessage = "Calendar draft opened for \"${action.title}\" on $whenLabel", + missingAppMessage = "No calendar app is available on this device.", + deniedMessage = "This app is not allowed to open the calendar app.", + backgroundMessage = + "Could not open the calendar app while the app was in the background.", + completion = completion + ) + } + + /** + * Formats the start of an event for the confirmation toast. + * + * All-day events are stored at UTC midnight, so they are formatted in UTC + * too - reading them in the device's zone would show the previous day for + * anyone west of Greenwich. + */ + private fun formatEventStart(action: CreateCalendarEventAction): String { + val format = if (action.allDay) { + DateFormat.getDateInstance(DateFormat.MEDIUM).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + } else { + DateFormat.getDateTimeInstance(DateFormat.MEDIUM, DateFormat.SHORT) + } + return format.format(Date(action.startMillis)) + } + + /** + * Handles the `playMusicFromSearch` action by asking the device's music app + * to play the best match for a query. + * + * What actually plays is entirely the music app's decision, so the toast + * and the agent result both say what was asked for, not what is playing. + */ + @Suppress("DEPRECATION") + private fun launchPlayMusicFromSearchIntent( + action: PlayMusicFromSearchAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) { + val intent = Intent(MediaStore.INTENT_ACTION_MEDIA_PLAY_FROM_SEARCH).apply { + putExtra(SearchManager.QUERY, action.query) + // EXTRA_MEDIA_FOCUS tells the music app how to read the query. It is + // left off for "any", which is the documented way to say + // "unstructured search - you decide". + when (action.focus) { + MusicSearchFocus.Any -> Unit + MusicSearchFocus.Artist -> { + putExtra( + MediaStore.EXTRA_MEDIA_FOCUS, + MediaStore.Audio.Artists.ENTRY_CONTENT_TYPE + ) + putExtra(MediaStore.EXTRA_MEDIA_ARTIST, action.query) + } + + MusicSearchFocus.Album -> { + putExtra( + MediaStore.EXTRA_MEDIA_FOCUS, + MediaStore.Audio.Albums.ENTRY_CONTENT_TYPE + ) + putExtra(MediaStore.EXTRA_MEDIA_ALBUM, action.query) + } + + MusicSearchFocus.Song -> { + putExtra( + MediaStore.EXTRA_MEDIA_FOCUS, + MediaStore.Audio.Media.ENTRY_CONTENT_TYPE + ) + putExtra(MediaStore.EXTRA_MEDIA_TITLE, action.query) + } + + MusicSearchFocus.Playlist -> { + putExtra( + MediaStore.EXTRA_MEDIA_FOCUS, + MediaStore.Audio.Playlists.ENTRY_CONTENT_TYPE + ) + putExtra(MediaStore.EXTRA_MEDIA_PLAYLIST, action.query) + } + } + } + launchExternalIntent( + intent = intent, + actionName = "play-music-from-search", + detail = "focus=${action.focus.schemaName} query=${action.query}", + successMessage = "Asked your music app to play ${action.query}", + missingAppMessage = "No music app on this device can play from a search.", + deniedMessage = "This app is not allowed to start playback.", + backgroundMessage = "Could not start playback while the app was in the background.", + completion = completion + ) + } + /** * Starts an intent handled by another app and reports the outcome * truthfully. diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/OpenSettingsActionParser.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/OpenSettingsActionParser.kt new file mode 100644 index 0000000000..4792cecf64 --- /dev/null +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/OpenSettingsActionParser.kt @@ -0,0 +1,73 @@ +package com.example.typeagentchat + +import org.json.JSONObject + +/** + * The closed set of settings screens the agent may open. + * + * This enum *is* the security control for `openSettings`. The model names a + * member; it never supplies an intent action string. Accepting a free-form + * action would turn one convenience action into "start any exported system + * activity", which is exactly the shape of bug prompt injection looks for. + * Anything unrecognised fails the action instead of being passed through. + * + * Mirrors `AndroidSettingsScreen` in `androidDeviceSchema.ts`; the two lists + * are kept in step by `AndroidDeviceSchemaAssetTest`. + */ +internal enum class AndroidSettingsScreen(val schemaName: String, val displayName: String) { + Settings("settings", "settings"), + Wifi("wifi", "wifi settings"), + Bluetooth("bluetooth", "Bluetooth settings"), + Display("display", "display settings"), + Sound("sound", "sound settings"), + Location("location", "location settings"), + Battery("battery", "battery settings"), + AirplaneMode("airplaneMode", "airplane mode settings"), + DateAndTime("dateAndTime", "date and time settings"), + Storage("storage", "storage settings"), + Accessibility("accessibility", "accessibility settings"), + Security("security", "security settings"), + + /** This app's own details page - never another app's. */ + AppInfo("appInfo", "this app's settings"); + + companion object { + /** + * Matched case-insensitively because the schema name is the contract but + * models routinely emit "Wifi" or "WIFI"; the mapping stays closed + * either way. + */ + fun fromSchemaName(name: String): AndroidSettingsScreen? = + entries.firstOrNull { it.schemaName.equals(name, ignoreCase = true) } + } +} + +internal data class OpenSettingsAction( + val originalRequest: String, + val screen: AndroidSettingsScreen +) + +/** + * Parses the `parameters` of the `openSettings` action declared by + * `androidDeviceSchema.ts`: + * + * ```ts + * parameters: { originalRequest: string; screen: AndroidSettingsScreen } + * ``` + * + * Opening a settings screen is the only thing an ordinary app can do about + * wifi, Bluetooth or Do Not Disturb - the platform blocks direct toggling - so + * this action hands the user to the right screen rather than pretending to + * change anything itself. + */ +internal fun parseOpenSettingsActionPayload(data: Any?): OpenSettingsAction? { + val payload = data as? JSONObject ?: return null + val screen = AndroidSettingsScreen + .fromSchemaName(payload.sanitizedActionText("screen")) + ?: return null + + return OpenSettingsAction( + originalRequest = payload.sanitizedActionText("originalRequest"), + screen = screen + ) +} diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/PlayMusicFromSearchActionParser.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/PlayMusicFromSearchActionParser.kt new file mode 100644 index 0000000000..5225d770a6 --- /dev/null +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/PlayMusicFromSearchActionParser.kt @@ -0,0 +1,77 @@ +package com.example.typeagentchat + +import org.json.JSONObject + +/** + * What a music search query names. + * + * Mirrors `MusicSearchFocus` in `androidDeviceSchema.ts`; the two lists are + * kept in step by `AndroidDeviceSchemaAssetTest`. + */ +internal enum class MusicSearchFocus(val schemaName: String) { + /** Let the music app decide what the query means. */ + Any("any"), + Artist("artist"), + Album("album"), + Song("song"), + Playlist("playlist"); + + companion object { + fun fromSchemaName(name: String): MusicSearchFocus? = + entries.firstOrNull { it.schemaName.equals(name, ignoreCase = true) } + } +} + +internal data class PlayMusicFromSearchAction( + val originalRequest: String, + val query: String, + val focus: MusicSearchFocus +) + +/** + * Parses the `parameters` of the `playMusicFromSearch` action declared by + * `androidDeviceSchema.ts`: + * + * ```ts + * parameters: { originalRequest: string; query: string; focus?: MusicSearchFocus } + * ``` + * + * The result is used with `MediaStore.INTENT_ACTION_MEDIA_PLAY_FROM_SEARCH`, + * which asks whichever music app claims it to play the best match. How well + * that works is entirely up to the installed app - several streaming apps + * handle the intent poorly or not at all - so the action reports what it + * dispatched, not what ended up playing. + * + * An unrecognised focus fails the action rather than falling back to [Any]: the + * fallback would look like success while searching for something broader than + * the user asked for. + */ +internal fun parsePlayMusicFromSearchActionPayload(data: Any?): PlayMusicFromSearchAction? { + val payload = data as? JSONObject ?: return null + val query = payload.sanitizedActionText("query") + if (query.isEmpty()) { + return null + } + + // Read `focus` through `opt` rather than `sanitizedActionText` so that a + // wrong-typed value (a number, an object) is rejected instead of collapsing + // to the empty string and being mistaken for an omitted field. + val focus = when (val rawFocus = payload.opt("focus")) { + null, JSONObject.NULL -> MusicSearchFocus.Any + is String -> { + val sanitized = sanitizeActionText(rawFocus) + if (sanitized.isEmpty()) { + MusicSearchFocus.Any + } else { + MusicSearchFocus.fromSchemaName(sanitized) ?: return null + } + } + else -> return null + } + + return PlayMusicFromSearchAction( + originalRequest = payload.sanitizedActionText("originalRequest"), + query = query, + focus = focus + ) +} diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ShareTextActionParser.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ShareTextActionParser.kt new file mode 100644 index 0000000000..1c1616a688 --- /dev/null +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/ShareTextActionParser.kt @@ -0,0 +1,50 @@ +package com.example.typeagentchat + +import org.json.JSONObject + +internal data class ShareTextAction( + val originalRequest: String, + val text: String, + val subject: String +) + +/** + * Shared text can reasonably be a paragraph or two, but it is still handed to + * another app through a binder transaction, so it is capped like every other + * free-text field. + */ +private const val MAX_SHARE_TEXT_CHARS = 4_000 + +/** + * Parses the `parameters` of the `shareText` action declared by + * `androidDeviceSchema.ts`: + * + * ```ts + * parameters: { originalRequest: string; text: string; subject?: string } + * ``` + * + * The result is used with `Intent.ACTION_SEND` inside `Intent.createChooser`, + * so the destination app is chosen by the user, not by the model. That matters + * more here than for the other actions: the set of possible destinations is + * every app on the device, so the chooser is what keeps this from being a way + * to push model-authored text into an arbitrary app unseen. + * + * The text itself is required and never defaulted. There is no sensible "share + * whatever was on screen" behaviour to fall back to, and inventing one would + * risk sharing conversation content the user never pointed at. + */ +internal fun parseShareTextActionPayload(data: Any?): ShareTextAction? { + val payload = data as? JSONObject ?: return null + // Newlines survive here, unlike in the URI-bound actions: shared text is + // carried as an extra, and a multi-line note is a normal thing to share. + val text = payload.optActionString("text").trim().take(MAX_SHARE_TEXT_CHARS).trim() + if (text.isEmpty()) { + return null + } + + return ShareTextAction( + originalRequest = payload.sanitizedActionText("originalRequest"), + text = text, + subject = payload.sanitizedActionText("subject") + ) +} diff --git a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt index 25a88bfcd7..fce81ece4a 100644 --- a/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt +++ b/android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/WebSocketManager.kt @@ -818,6 +818,16 @@ class WebSocketManager internal constructor( handler.onWebSearch(action.action, completion) is AndroidDeviceAction.OpenWebPage -> handler.onOpenWebPage(action.action, completion) + is AndroidDeviceAction.ComposeEmail -> + handler.onComposeEmail(action.action, completion) + is AndroidDeviceAction.ShareText -> + handler.onShareText(action.action, completion) + is AndroidDeviceAction.OpenSettings -> + handler.onOpenSettings(action.action, completion) + is AndroidDeviceAction.CreateCalendarEvent -> + handler.onCreateCalendarEvent(action.action, completion) + is AndroidDeviceAction.PlayMusicFromSearch -> + handler.onPlayMusicFromSearch(action.action, completion) } } @@ -1675,6 +1685,31 @@ class WebSocketManager internal constructor( action: OpenWebPageAction, completion: (AndroidDeviceExecutionResult) -> Unit ) + + fun onComposeEmail( + action: ComposeEmailAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) + + fun onShareText( + action: ShareTextAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) + + fun onOpenSettings( + action: OpenSettingsAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) + + fun onCreateCalendarEvent( + action: CreateCalendarEventAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) + + fun onPlayMusicFromSearch( + action: PlayMusicFromSearchAction, + completion: (AndroidDeviceExecutionResult) -> Unit + ) } } diff --git a/android/samples/mobile-2/app/src/main/res/values/strings.xml b/android/samples/mobile-2/app/src/main/res/values/strings.xml index 24283ceb54..f8feb17514 100644 --- a/android/samples/mobile-2/app/src/main/res/values/strings.xml +++ b/android/samples/mobile-2/app/src/main/res/values/strings.xml @@ -1,3 +1,5 @@ TypeAgent Android Chat Sample + + Share via \ No newline at end of file diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceAgentTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceAgentTest.kt index d1d3a03cba..be7d738289 100644 --- a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceAgentTest.kt +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceAgentTest.kt @@ -272,6 +272,94 @@ class AndroidDeviceAgentTest { assertEquals(listOf(Calendar.MONDAY, Calendar.TUESDAY), parsed.action.days) } + @Test + fun parsesComposeEmailExecuteAction() { + val parsed = parseSuccess( + "composeEmail", + JSONObject() + .put("originalRequest", "Email Ada the notes") + .put("to", JSONArray(listOf("ada@example.com"))) + .put("subject", "Notes") + ) + + assertEquals(listOf("ada@example.com"), parsed.action.to) + assertTrue( + parse("composeEmail", JSONObject().put("to", JSONArray(listOf("not an address")))) + is AndroidDeviceActionParseResult.ActionError + ) + } + + @Test + fun parsesShareTextExecuteAction() { + val parsed = parseSuccess( + "shareText", + JSONObject() + .put("originalRequest", "Share the address") + .put("text", "1 Microsoft Way, Redmond WA") + ) + + assertEquals("1 Microsoft Way, Redmond WA", parsed.action.text) + assertTrue( + parse("shareText", JSONObject().put("text", " ")) + is AndroidDeviceActionParseResult.ActionError + ) + } + + @Test + fun parsesOpenSettingsExecuteAction() { + val parsed = parseSuccess( + "openSettings", + JSONObject() + .put("originalRequest", "Turn on wifi") + .put("screen", "wifi") + ) + + assertEquals(AndroidSettingsScreen.Wifi, parsed.action.screen) + // A raw intent action must never be accepted here. + assertTrue( + parse("openSettings", JSONObject().put("screen", "android.settings.SETTINGS")) + is AndroidDeviceActionParseResult.ActionError + ) + } + + @Test + fun parsesCreateCalendarEventExecuteAction() { + val parsed = parseSuccess( + "createCalendarEvent", + JSONObject() + .put("originalRequest", "Add lunch on the 24th") + .put("title", "Lunch") + .put("start", "2026-08-24T12:00") + ) + + assertEquals("Lunch", parsed.action.title) + assertTrue(parsed.action.endMillis > parsed.action.startMillis) + assertTrue( + parse( + "createCalendarEvent", + JSONObject().put("title", "Lunch").put("start", "next Tuesday at 3") + ) is AndroidDeviceActionParseResult.ActionError + ) + } + + @Test + fun parsesPlayMusicFromSearchExecuteAction() { + val parsed = parseSuccess( + "playMusicFromSearch", + JSONObject() + .put("originalRequest", "Play Miles Davis") + .put("query", "Miles Davis") + .put("focus", "artist") + ) + + assertEquals("Miles Davis", parsed.action.query) + assertEquals(MusicSearchFocus.Artist, parsed.action.focus) + assertTrue( + parse("playMusicFromSearch", JSONObject().put("query", " ")) + is AndroidDeviceActionParseResult.ActionError + ) + } + @Test fun serializesActionResults() { val success = AndroidDeviceAgent.createSuccessResult("Timer request sent for 30 seconds") val failure = AndroidDeviceAgent.createErrorResult("No timer app") diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceSchemaAssetTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceSchemaAssetTest.kt index 4dc1b858e2..ac08069b38 100644 --- a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceSchemaAssetTest.kt +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/AndroidDeviceSchemaAssetTest.kt @@ -111,6 +111,36 @@ class AndroidDeviceSchemaAssetTest { assertTrue("AlarmActionParser rejects the day names $rejected", rejected.isEmpty()) } + @Test + fun `the settings screens the schema offers match the ones the parser accepts`() { + // The Kotlin enum is the allowlist. A screen the schema offers but the + // enum does not know fails the action at the moment a user asks for it, + // and a screen the enum has but the schema hides is unreachable. + assertEquals( + AndroidSettingsScreen.entries.map { it.schemaName }.sorted(), + schemaUnionMembers("AndroidSettingsScreen").sorted() + ) + } + + @Test + fun `the music search focus values match the ones the parser accepts`() { + assertEquals( + MusicSearchFocus.entries.map { it.schemaName }.sorted(), + schemaUnionMembers("MusicSearchFocus").sorted() + ) + } + + /** Reads the string literals of a named string-union type alias. */ + private fun schemaUnionMembers(typeName: String): List = + Regex("""export type $typeName =([^;]*);""") + .find(schema) + ?.groupValues + ?.get(1) + ?.split("|") + ?.map { it.trim().trim('"') } + ?.filter { it.isNotEmpty() } + .orEmpty() + private fun isAlarmDayAccepted(day: String): Boolean { val parameters = JSONObject() .put("originalRequest", "wake me up") diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ComposeEmailActionParserTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ComposeEmailActionParserTest.kt new file mode 100644 index 0000000000..69b2d968f7 --- /dev/null +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ComposeEmailActionParserTest.kt @@ -0,0 +1,137 @@ +package com.example.typeagentchat + +import org.json.JSONArray +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class ComposeEmailActionParserTest { + private fun payload(vararg fields: Pair): JSONObject { + val json = JSONObject().put("originalRequest", "Email the team") + fields.forEach { (key, value) -> json.put(key, value) } + return json + } + + @Test + fun parsesAFullDraft() { + val parsed = parseComposeEmailActionPayload( + payload( + "to" to JSONArray().put("ada@example.com"), + "cc" to JSONArray().put("grace@example.com"), + "bcc" to JSONArray().put("alan@example.com"), + "subject" to "Status", + "body" to "All good." + ) + ) + + assertEquals(listOf("ada@example.com"), parsed?.to) + assertEquals(listOf("grace@example.com"), parsed?.cc) + assertEquals(listOf("alan@example.com"), parsed?.bcc) + assertEquals("Status", parsed?.subject) + assertEquals("All good.", parsed?.body) + } + + @Test + fun acceptsASingleAddressWhereTheSchemaAsksForAnArray() { + val parsed = parseComposeEmailActionPayload(payload("to" to "ada@example.com")) + + assertEquals(listOf("ada@example.com"), parsed?.to) + } + + @Test + fun allowsADraftWithNoRecipientSoTheUserCanFillItIn() { + val parsed = parseComposeEmailActionPayload( + payload("subject" to "Notes", "body" to "Draft this for me") + ) + + assertEquals(emptyList(), parsed?.to) + assertEquals("Notes", parsed?.subject) + } + + @Test + fun rejectsAnEmptyDraft() { + // Nothing to show the user, so it is a failed translation rather than a + // blank compose window they have to dismiss. + assertNull(parseComposeEmailActionPayload(payload())) + assertNull(parseComposeEmailActionPayload(payload("to" to JSONArray()))) + } + + @Test + fun rejectsTheWholeActionWhenAnyAddressIsUnusable() { + // Dropping the bad one would send a draft to fewer people than the user + // asked for, and the model would never learn it got the address wrong. + assertNull( + parseComposeEmailActionPayload( + payload("to" to JSONArray().put("ada@example.com").put("not an address")) + ) + ) + assertNull(parseComposeEmailActionPayload(payload("cc" to JSONArray().put("ada@")))) + assertNull(parseComposeEmailActionPayload(payload("bcc" to JSONArray().put("@example.com")))) + } + + @Test + fun rejectsAddressesThatAreReallyTwoAddressesRunTogether() { + assertFalse(isSupportedEmailAddress("ada@example.com, grace@example.com")) + assertFalse(isSupportedEmailAddress("ada@example.com grace@example.com")) + assertFalse(isSupportedEmailAddress("Ada ")) + } + + @Test + fun acceptsOrdinaryAddressesAndTrimsThem() { + assertTrue(isSupportedEmailAddress("ada.lovelace+news@sub.example.co.uk")) + assertEquals("ada@example.com", normalizeEmailAddress(" ada@example.com \n")) + } + + @Test + fun rejectsAddressesWithoutADottedDomain() { + assertFalse(isSupportedEmailAddress("ada@localhost")) + assertFalse(isSupportedEmailAddress("ada.example.com")) + } + + @Test + fun deduplicatesRepeatedRecipients() { + val parsed = parseComposeEmailActionPayload( + payload("to" to JSONArray().put("ada@example.com").put("ada@example.com")) + ) + + assertEquals(listOf("ada@example.com"), parsed?.to) + } + + @Test + fun rejectsTooManyRecipients() { + val many = JSONArray() + repeat(40) { many.put("user$it@example.com") } + + assertNull(parseComposeEmailActionPayload(payload("to" to many))) + } + + @Test + fun rejectsNonStringAndMalformedPayloads() { + assertNull(parseComposeEmailActionPayload(null)) + assertNull(parseComposeEmailActionPayload("to: ada@example.com")) + assertNull(parseComposeEmailActionPayload(payload("to" to 42))) + assertNull(parseComposeEmailActionPayload(payload("to" to JSONArray().put(42)))) + } + + @Test + fun treatsJsonNullAsAnAbsentField() { + // org.json renders a JSON null as the literal string "null", which would + // otherwise be validated as an address and fail the action. + val parsed = parseComposeEmailActionPayload( + payload("to" to JSONObject.NULL, "subject" to "Hi") + ) + + assertEquals(emptyList(), parsed?.to) + assertEquals("Hi", parsed?.subject) + } + + @Test + fun capsAnOverlongBodyRatherThanFailing() { + val parsed = parseComposeEmailActionPayload(payload("body" to "x".repeat(20_000))) + + assertEquals(8_000, parsed?.body?.length) + } +} diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/CreateCalendarEventActionParserTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/CreateCalendarEventActionParserTest.kt new file mode 100644 index 0000000000..d81475da67 --- /dev/null +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/CreateCalendarEventActionParserTest.kt @@ -0,0 +1,406 @@ +package com.example.typeagentchat + +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import java.util.Locale +import java.util.TimeZone + +class CreateCalendarEventActionParserTest { + + private val utc = TimeZone.getTimeZone("UTC") + + /** UTC+10, and never on daylight saving, so the offset is stable. */ + private val brisbane = TimeZone.getTimeZone("Australia/Brisbane") + + /** 2026-08-24T15:00:00Z */ + private val august24At3pmUtc = 1_787_583_600_000L + + /** 2026-08-24T00:00:00Z */ + private val august24MidnightUtc = 1_787_529_600_000L + + private val hour = 3_600_000L + private val day = 86_400_000L + + private fun payload(vararg fields: Pair): JSONObject { + val json = JSONObject() + .put("originalRequest", "Put lunch in my calendar") + .put("title", "Lunch") + fields.forEach { (key, value) -> json.put(key, value) } + return json + } + + @Test + fun readsALocalTimeInTheDeviceTimeZone() { + val parsed = parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T15:00"), + timeZone = utc + ) + + assertEquals(august24At3pmUtc, parsed?.startMillis) + assertFalse(parsed?.allDay ?: true) + } + + @Test + fun theSameWallClockValueMeansADifferentInstantInADifferentZone() { + // This is the whole risk in the action: 3pm in Brisbane is not 3pm UTC, + // and getting it wrong writes a silently wrong entry into a calendar. + val inBrisbane = parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T15:00"), + timeZone = brisbane + ) + + assertEquals(august24At3pmUtc - 10 * hour, inBrisbane?.startMillis) + } + + @Test + fun honoursAnExplicitOffsetInsteadOfRejectingIt() { + // Models emit "Z" and "+05:30" routinely; reading them at face value in + // the device zone would shift the event by the offset. + assertEquals( + august24At3pmUtc, + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T15:00Z"), + timeZone = brisbane + )?.startMillis + ) + assertEquals( + august24At3pmUtc, + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T20:30+05:30"), + timeZone = brisbane + )?.startMillis + ) + assertEquals( + august24At3pmUtc, + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T08:00-0700"), + timeZone = brisbane + )?.startMillis + ) + } + + @Test + fun readsAnOffsetIdenticallyUnderALocaleWithNonLatinDigits() { + // The offset zone used to be built by formatting "GMT+%02d:%02d", which + // uses the default locale. Under ar-EG that emits Arabic-Indic digits, + // TimeZone.getTimeZone cannot read them, and it falls back to GMT + // without complaining - silently moving the event by the offset. + val original = Locale.getDefault() + try { + Locale.setDefault(Locale.forLanguageTag("ar-EG-u-nu-arab")) + assertEquals( + august24At3pmUtc, + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T20:30+05:30"), + timeZone = brisbane + )?.startMillis + ) + } finally { + Locale.setDefault(original) + } + } + + @Test + fun acceptsFractionalSecondsAndTruncatesThemToTheSecond() { + // RFC 3339 permits them and models emit them. Calendar UIs are + // minute-granular, so dropping the fraction is unobservable, whereas + // rejecting the timestamp would fail the whole action. + assertEquals( + august24At3pmUtc + 30_000L, + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T15:00:30.123Z"), + timeZone = brisbane + )?.startMillis + ) + assertEquals( + august24At3pmUtc + 30_000L, + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T15:00:30,123456789Z"), + timeZone = brisbane + )?.startMillis + ) + } + + @Test + fun stillRejectsAMalformedFraction() { + // A trailing separator with no digits is not a timestamp, and accepting + // it would mean the regex had been loosened further than intended. + assertNull( + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T15:00:30.Z"), + timeZone = utc + ) + ) + assertNull( + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T15:00.500"), + timeZone = utc + ) + ) + } + + @Test + fun acceptsAnHoursOnlyOffset() { + // ISO-8601 permits "+05" as shorthand for "+05:00"; rejecting it would + // fail the action over a form the standard allows. + assertEquals( + august24At3pmUtc, + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T20:00+05"), + timeZone = brisbane + )?.startMillis + ) + assertEquals( + august24At3pmUtc, + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T08:00-07"), + timeZone = brisbane + )?.startMillis + ) + } + + @Test + fun rejectsAnImpossibleOffset() { + // TimeZone.getTimeZone quietly falls back to GMT for anything it cannot + // read, so an unchecked "+99:00" would become UTC. + assertNull( + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T15:00+99:00"), + timeZone = utc + ) + ) + } + + @Test + fun defaultsATimedEventToOneHour() { + val parsed = parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T15:00"), + timeZone = utc + ) + + assertEquals(august24At3pmUtc + hour, parsed?.endMillis) + } + + @Test + fun usesAnExplicitEnd() { + val parsed = parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T15:00", "end" to "2026-08-24T17:30"), + timeZone = utc + ) + + assertEquals(august24At3pmUtc + 2 * hour + 30 * 60_000L, parsed?.endMillis) + } + + @Test + fun anchorsAnAllDayEventToUtcMidnight() { + // CalendarContract stores all-day events at UTC midnight. Using the + // device zone would land the event on the previous day for a user in + // UTC+10. + val parsed = parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24", "allDay" to true), + timeZone = brisbane + ) + + assertTrue(parsed?.allDay ?: false) + assertEquals(august24MidnightUtc, parsed?.startMillis) + assertEquals(august24MidnightUtc + day, parsed?.endMillis) + } + + @Test + fun treatsAnAllDayEndAsTheLastDayTheUserNamed() { + // "24th to the 26th" means three days, and the provider wants an + // exclusive end, so the final day is added on. + val parsed = parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24", "end" to "2026-08-26", "allDay" to true), + timeZone = utc + ) + + assertEquals(august24MidnightUtc + 3 * day, parsed?.endMillis) + } + + @Test + fun readsADateOnlyValueAsLocalMidnightWhenItIsNotAllDay() { + val parsed = parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24"), + timeZone = brisbane + ) + + assertEquals(august24MidnightUtc - 10 * hour, parsed?.startMillis) + } + + @Test + fun rejectsAnAllDayEventThatAlsoCarriesATimeOfDay() { + // The model contradicted itself; either reading would put something in + // the calendar the user did not ask for. + assertNull( + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T09:00", "allDay" to true), + timeZone = utc + ) + ) + } + + @Test + fun acceptsBooleansSentAsStrings() { + assertTrue( + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24", "allDay" to "true"), + timeZone = utc + )?.allDay ?: false + ) + assertFalse( + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T15:00", "allDay" to "False"), + timeZone = utc + )?.allDay ?: true + ) + assertNull( + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24", "allDay" to "yes please"), + timeZone = utc + ) + ) + } + + @Test + fun rejectsAnEndThatIsNotAfterTheStart() { + assertNull( + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T15:00", "end" to "2026-08-24T15:00"), + timeZone = utc + ) + ) + assertNull( + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T15:00", "end" to "2026-08-24T09:00"), + timeZone = utc + ) + ) + } + + @Test + fun rejectsAnAbsurdlyLongEvent() { + assertNull( + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T15:00", "end" to "2030-08-24T15:00"), + timeZone = utc + ) + ) + } + + @Test + fun rejectsDatesThatDoNotExist() { + // Non-lenient parsing: 31 February would otherwise roll into March and + // land on a day the user never named. + assertNull( + parseCreateCalendarEventActionPayload( + payload("start" to "2026-02-31"), + timeZone = utc + ) + ) + assertNull( + parseCreateCalendarEventActionPayload( + payload("start" to "2026-13-01"), + timeZone = utc + ) + ) + assertNull( + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24T25:00"), + timeZone = utc + ) + ) + } + + @Test + fun acceptsLeapDaysThatDoExist() { + assertEquals( + true, + parseCreateCalendarEventActionPayload( + payload("start" to "2028-02-29T09:00"), + timeZone = utc + ) != null + ) + } + + @Test + fun rejectsYearsOutsideTheSupportedRange() { + assertNull( + parseCreateCalendarEventActionPayload( + payload("start" to "1969-08-24T15:00"), + timeZone = utc + ) + ) + assertNull( + parseCreateCalendarEventActionPayload( + payload("start" to "9999-08-24T15:00"), + timeZone = utc + ) + ) + } + + @Test + fun rejectsFormatsTheSchemaDoesNotAskFor() { + listOf( + "24/08/2026", + "August 24 2026", + "next Tuesday at 3", + "2026-8-4T15:00", + "15:00", + "" + ).forEach { value -> + assertNull( + "Should not accept $value", + parseCreateCalendarEventActionPayload( + payload("start" to value), + timeZone = utc + ) + ) + } + } + + @Test + fun acceptsSecondsAndASpaceSeparator() { + assertEquals( + august24At3pmUtc + 30_000L, + parseCreateCalendarEventActionPayload( + payload("start" to "2026-08-24 15:00:30"), + timeZone = utc + )?.startMillis + ) + } + + @Test + fun requiresATitle() { + val noTitle = JSONObject() + .put("originalRequest", "Put lunch in my calendar") + .put("start", "2026-08-24T15:00") + + assertNull(parseCreateCalendarEventActionPayload(noTitle, timeZone = utc)) + } + + @Test + fun keepsOptionalDetails() { + val parsed = parseCreateCalendarEventActionPayload( + payload( + "start" to "2026-08-24T15:00", + "location" to "Cafe Rio", + "description" to "Bring the deck" + ), + timeZone = utc + ) + + assertEquals("Cafe Rio", parsed?.location) + assertEquals("Bring the deck", parsed?.description) + } + + @Test + fun rejectsNonObjectPayloads() { + assertNull(parseCreateCalendarEventActionPayload(null, timeZone = utc)) + assertNull(parseCreateCalendarEventActionPayload("2026-08-24T15:00", timeZone = utc)) + } +} diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/OpenSettingsActionParserTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/OpenSettingsActionParserTest.kt new file mode 100644 index 0000000000..da0e9cf679 --- /dev/null +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/OpenSettingsActionParserTest.kt @@ -0,0 +1,63 @@ +package com.example.typeagentchat + +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class OpenSettingsActionParserTest { + private fun payload(screen: Any?): JSONObject = + JSONObject() + .put("originalRequest", "Turn on wifi") + .put("screen", screen) + + @Test + fun parsesEverySchemaNameTheEnumDeclares() { + AndroidSettingsScreen.entries.forEach { screen -> + assertEquals( + "Screen ${screen.schemaName} should parse", + screen, + parseOpenSettingsActionPayload(payload(screen.schemaName))?.screen + ) + } + } + + @Test + fun acceptsDifferentCasingFromTheModel() { + assertEquals( + AndroidSettingsScreen.Wifi, + parseOpenSettingsActionPayload(payload("WiFi"))?.screen + ) + assertEquals( + AndroidSettingsScreen.AirplaneMode, + parseOpenSettingsActionPayload(payload("airplanemode"))?.screen + ) + } + + @Test + fun rejectsAnythingOutsideTheAllowlist() { + // The enum is the security control: a free-form value must never reach + // an Intent action, or openSettings becomes "start any system activity". + assertNull(parseOpenSettingsActionPayload(payload("android.settings.SETTINGS"))) + assertNull(parseOpenSettingsActionPayload(payload("developerOptions"))) + assertNull(parseOpenSettingsActionPayload(payload("android.intent.action.CALL"))) + assertNull(parseOpenSettingsActionPayload(payload(""))) + } + + @Test + fun rejectsMissingBlankAndNonStringScreens() { + assertNull(parseOpenSettingsActionPayload(JSONObject())) + assertNull(parseOpenSettingsActionPayload(payload(" "))) + assertNull(parseOpenSettingsActionPayload(payload(JSONObject.NULL))) + assertNull(parseOpenSettingsActionPayload(payload(7))) + assertNull(parseOpenSettingsActionPayload(null)) + assertNull(parseOpenSettingsActionPayload("wifi")) + } + + @Test + fun schemaNamesAreUnique() { + val names = AndroidSettingsScreen.entries.map { it.schemaName } + + assertEquals(names.size, names.toSet().size) + } +} diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/PlayMusicFromSearchActionParserTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/PlayMusicFromSearchActionParserTest.kt new file mode 100644 index 0000000000..562d500937 --- /dev/null +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/PlayMusicFromSearchActionParserTest.kt @@ -0,0 +1,82 @@ +package com.example.typeagentchat + +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class PlayMusicFromSearchActionParserTest { + private fun payload(query: Any?, focus: Any? = null): JSONObject = + JSONObject() + .put("originalRequest", "Play some music") + .put("query", query) + .apply { if (focus != null) put("focus", focus) } + + @Test + fun parsesAQueryAndDefaultsTheFocusToAny() { + val parsed = parsePlayMusicFromSearchActionPayload(payload("Kind of Blue")) + + assertEquals("Kind of Blue", parsed?.query) + assertEquals(MusicSearchFocus.Any, parsed?.focus) + } + + @Test + fun parsesEverySchemaNameTheEnumDeclares() { + MusicSearchFocus.entries.forEach { focus -> + assertEquals( + "Focus ${focus.schemaName} should parse", + focus, + parsePlayMusicFromSearchActionPayload(payload("anything", focus.schemaName))?.focus + ) + } + } + + @Test + fun acceptsDifferentCasingFromTheModel() { + assertEquals( + MusicSearchFocus.Artist, + parsePlayMusicFromSearchActionPayload(payload("Miles Davis", "ARTIST"))?.focus + ) + } + + @Test + fun rejectsAnUnknownFocusRatherThanFallingBackToAny() { + // Falling back would look like success while searching for something + // broader than the user asked for. + assertNull(parsePlayMusicFromSearchActionPayload(payload("Miles Davis", "composer"))) + assertNull(parsePlayMusicFromSearchActionPayload(payload("Miles Davis", 3))) + } + + @Test + fun treatsAnAbsentOrNullFocusAsAny() { + assertEquals( + MusicSearchFocus.Any, + parsePlayMusicFromSearchActionPayload(payload("jazz", JSONObject.NULL))?.focus + ) + } + + @Test + fun requiresAQuery() { + assertNull(parsePlayMusicFromSearchActionPayload(payload(""))) + assertNull(parsePlayMusicFromSearchActionPayload(payload(" "))) + assertNull(parsePlayMusicFromSearchActionPayload(payload(JSONObject.NULL))) + assertNull(parsePlayMusicFromSearchActionPayload(JSONObject())) + assertNull(parsePlayMusicFromSearchActionPayload(null)) + assertNull(parsePlayMusicFromSearchActionPayload("jazz")) + } + + @Test + fun collapsesControlCharactersInTheQuery() { + val parsed = parsePlayMusicFromSearchActionPayload(payload("Kind\nof\tBlue")) + + assertEquals("Kind of Blue", parsed?.query) + } + + @Test + fun capsAnOverlongQuery() { + val parsed = parsePlayMusicFromSearchActionPayload(payload("x".repeat(1_000))) + + assertTrue((parsed?.query?.length ?: 0) <= MAX_ACTION_TEXT_CHARS) + } +} diff --git a/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ShareTextActionParserTest.kt b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ShareTextActionParserTest.kt new file mode 100644 index 0000000000..e9a2d50feb --- /dev/null +++ b/android/samples/mobile-2/app/src/test/java/com/example/typeagentchat/ShareTextActionParserTest.kt @@ -0,0 +1,64 @@ +package com.example.typeagentchat + +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class ShareTextActionParserTest { + private fun payload(text: Any?, subject: Any? = null): JSONObject = + JSONObject() + .put("originalRequest", "Share this") + .put("text", text) + .apply { if (subject != null) put("subject", subject) } + + @Test + fun parsesTextAndSubject() { + val parsed = parseShareTextActionPayload(payload("Meet at 6", "Plans")) + + assertEquals("Meet at 6", parsed?.text) + assertEquals("Plans", parsed?.subject) + } + + @Test + fun keepsLineBreaksInsideSharedText() { + // Unlike the URI-bound actions, shared text rides as an extra, so a + // multi-line note survives intact. + val parsed = parseShareTextActionPayload(payload("line one\nline two")) + + assertEquals("line one\nline two", parsed?.text) + } + + @Test + fun trimsSurroundingWhitespace() { + assertEquals("hello", parseShareTextActionPayload(payload(" hello \n"))?.text) + } + + @Test + fun defaultsTheSubjectToEmpty() { + assertEquals("", parseShareTextActionPayload(payload("hello"))?.subject) + } + + @Test + fun requiresText() { + // There is no safe default here: falling back to "whatever was on + // screen" could share conversation content the user never pointed at. + assertNull(parseShareTextActionPayload(payload(""))) + assertNull(parseShareTextActionPayload(payload(" "))) + assertNull(parseShareTextActionPayload(payload(JSONObject.NULL))) + assertNull(parseShareTextActionPayload(JSONObject())) + } + + @Test + fun capsOverlongText() { + val parsed = parseShareTextActionPayload(payload("x".repeat(9_000))) + + assertEquals(4_000, parsed?.text?.length) + } + + @Test + fun rejectsNonObjectPayloads() { + assertNull(parseShareTextActionPayload(null)) + assertNull(parseShareTextActionPayload("hello")) + } +}