From 46166907a4b5b33a020e962cf834988362db5724 Mon Sep 17 00:00:00 2001
From: finalerock44 <ethan@devicecloud.dev>
Date: Wed, 11 Nov 2026 12:51:17 +0000
Subject: [PATCH 1/2] docs(agent-devices): typed actions: tap_on, input_text,
 swipe, press_key and the dcd device subcommands

The hosted MCP server gains tap_on, input_text, swipe and press_key, and
dcd device gains tap, type, swipe and press-key, all over the typed
POST /device/sessions/:id/actions route, which is performed on the device
directly and answers once the screen has settled. The pages say when to
use them and when to use run, and the API row notes mode: "maestro".
---
 api/device-sessions.md     |  2 +-
 cli/dcd-device.md          | 27 +++++++++++++++++++++++
 mcp/agent-devices-skill.md |  4 ++--
 mcp/agent-devices-tools.md | 44 +++++++++++++++++++++++++++++++++++++-
 mcp/agent-devices.md       |  2 +-
 5 files changed, 74 insertions(+), 5 deletions(-)

diff --git a/api/device-sessions.md b/api/device-sessions.md
index 7c68e0c..51d5929 100644
--- a/api/device-sessions.md
+++ b/api/device-sessions.md
@@ -20,7 +20,7 @@ A session is named by its `id` (for example `brave-teal-otter`) in every path. E
 |---|---|
 | `POST /device/sessions/:id/install` | Install a build: `{ "binaryUploadId": "..." }`, or `{ "url": "https://..." }` for an `.apk` or a zipped simulator `.app` at a link (up to 500 MB). It installs in the background. A command sent meanwhile waits up to 45 s for it, then answers `LIVE_NOT_READY` |
 | `POST /device/sessions/:id/launch` | Launch an app: `{ "appId"?: "...", "clearState"?: true }`. Without `appId`, the installed build |
-| `POST /device/sessions/:id/actions` | One action: `tap` (`x`/`y` percent, `text` or `id`), `type` (`text`, and `sensitive: true` to keep it secret), `swipe` (`direction`: `up`, `down`, `left` or `right`, or `from`/`to` points in percent), `press_key` (`key`: `back`, `home`, `enter`, `backspace`, `tab`, `volume_up`, `volume_down`, `lock` or `power`) |
+| `POST /device/sessions/:id/actions` | One action: `tap` (`x`/`y` percent, `text` or `id`), `type` (`text`, and `sensitive: true` to keep it secret), `swipe` (`direction`: `up`, `down`, `left` or `right`, or `from`/`to` points in percent), `press_key` (`key`: `back`, `home`, `enter`, `backspace`, `tab`, `volume_up`, `volume_down`, `lock` or `power`). Performed on the device directly and answered once the screen has settled, usually well under a second; `text` and `id` are matched whole, ignoring case. `mode: "maestro"` runs it as a Maestro flow instead. A `tap` on something that is not on screen waits about 17 s before it fails |
 | `POST /device/sessions/:id/run` | Run Maestro: `{ "yaml": "..." }` for one flow, or `{ "flowUploadId": "...", "flow"?: "..." }` for a [flow bundle](#flow-bundles). `env` passes [secrets and variables](#secrets). Waits up to 110 s, or `waitSeconds` (0 to 300), then answers `running` with a `command_id`. A failed run's result includes `screen_text`, the app's visible text after the failure |
 | `GET /device/sessions/:id/commands/:commandId` | The status and result of a `run` that answered `running`. `?waitSeconds=` (0 to 300) waits for it to finish instead of answering at once |
 | `GET /device/sessions/:id/inspect` | The screen as compact JSON. The on-screen keyboard and the System UI are left out unless you add `?all=true` |
diff --git a/cli/dcd-device.md b/cli/dcd-device.md
index 94539a4..ae88ad8 100644
--- a/cli/dcd-device.md
+++ b/cli/dcd-device.md
@@ -16,6 +16,8 @@ A typical session:
 dcd device start --platform ios --app ./build/MyApp.app --wait
 dcd device launch
 dcd device inspect
+dcd device tap --text "Log in"
+dcd device type --text "ann@example.com"
 dcd device run flows/login.yaml
 dcd device screenshot -o after-login.jpg
 dcd device stop
@@ -95,6 +97,31 @@ The screen as compact JSON: each element with its text, id, accessibility label
 
 Save a screenshot. `-o <file>` sets where (default `./<session>-<time>.jpg`). With `--json`, `data` gives the file's path.
 
+## `dcd device tap [id]`
+
+Tap one thing on the screen, by exactly one target. Answers once the screen has settled, usually well under a second. A target that is not on screen waits about 17 seconds before it fails, so `dcd device inspect` first. A failed tap exits `2`.
+
+| Flag | Description |
+|------|-------------|
+| `--text <text>` | The element's text, label or hint, matched whole and ignoring case (not a regex) |
+| `--rid <id>` | The element's resource id, `rid` in `dcd device inspect` |
+| `--x <n> --y <n>` | A point, as percentages of the screen (0 to 100) |
+| `--maestro` | Run the tap as a Maestro flow instead of directly on the device |
+
+## `dcd device type [id] --text <text>`
+
+Type into the focused field (tap it first). `--sensitive` keeps the text secret: it is stored encrypted, never shown, and masked in the output. `--maestro` as above.
+
+## `dcd device swipe [id] <up|down|left|right>`
+
+Swipe across most of the screen in a direction (`up` scrolls the content up, down the page), or from one point to another with `--from x,y --to x,y`, in percentages. `--maestro` as above.
+
+## `dcd device press-key [id] <key>`
+
+Press a key: `back`, `home`, `enter`, `backspace`, `tab`, `volume_up`, `volume_down`, `lock` or `power`. `enter` submits the focused field; `back` closes a keyboard or goes back on Android. `--maestro` as above.
+
+Each of these sends one [typed action](../api/device-sessions.md). For several steps, an assertion, scrolling until something is visible, or launching an app, use `run`.
+
 ## `dcd device run [id] <flow.yaml|directory>`
 
 Run Maestro on the device and wait for the result. A failed flow exits `2`. Maestro's output is printed either way, and after a failure, so is the app's visible text.
diff --git a/mcp/agent-devices-skill.md b/mcp/agent-devices-skill.md
index 465ff94..57636ca 100644
--- a/mcp/agent-devices-skill.md
+++ b/mcp/agent-devices-skill.md
@@ -14,7 +14,7 @@ Use the DeviceCloud MCP tools to check app changes on a real iOS simulator or An
 1. Call `cheat_sheet` once for Maestro syntax.
 2. `start_device` with the platform (`list_devices` shows the models). Give the user the session's `monitor_url`, where they can watch. A queued session is normal: it starts on its own. Until it is ready, call `get_device_status` with `wait: true`.
 3. `install_app`, then `get_device_status` with `wait: true` until the install is done, then `launch_app`.
-4. Loop: `inspect_screen` to see what is on screen, then `run` with Maestro commands to act, then `inspect_screen` again. Target elements by their text or id from `inspect_screen`, not by coordinates. Use `take_screenshot` when the layout matters.
+4. Loop: `inspect_screen` to see what is on screen, then act: `tap_on`, `input_text`, `swipe` or `press_key` for one step (the fastest path), or `run` with Maestro commands for several steps or an assertion, then `inspect_screen` again. Target elements by their text or id from `inspect_screen`, not by coordinates. Use `take_screenshot` when the layout matters.
 5. If a step needs a person (a 2FA code, a CAPTCHA, a payment sheet), call `request_human` with what they should do, tell the user, and poll `get_human_status` until it is handed back.
 6. Always `stop_device` when you are done, even after a failure. A session bills by the minute.
 
@@ -37,7 +37,7 @@ description: Check app changes on a real cloud iOS simulator or Android emulator
 1. Call `cheat_sheet` once for Maestro syntax.
 2. `start_device` with the platform, and give the user the session's `monitor_url`. A queued session starts on its own: call `get_device_status` with `wait: true` until it is ready.
 3. `install_app` (with `app_path` for a local build when using `dcd mcp`), wait with `get_device_status` until it is installed, then `launch_app`.
-4. Loop: `inspect_screen`, then `run` with Maestro commands, then `inspect_screen` again. Target elements by text or id, not coordinates.
+4. Loop: `inspect_screen`, then `tap_on`, `input_text`, `swipe` or `press_key` for one step, or `run` with Maestro commands for several, then `inspect_screen` again. Target elements by text or id, not coordinates.
 5. If a person is needed (2FA, CAPTCHA, payment), call `request_human`, tell the user, and poll `get_human_status` until it is handed back.
 6. Always `stop_device` when done, even after a failure.
 
diff --git a/mcp/agent-devices-tools.md b/mcp/agent-devices-tools.md
index 47cfaaf..53fd03d 100644
--- a/mcp/agent-devices-tools.md
+++ b/mcp/agent-devices-tools.md
@@ -116,9 +116,51 @@ Read-only.
 |-------|------|----------|-------------|
 | `session_id` | string | No | The session id from start_device, e.g. "brave-teal-otter". Optional when your organization has exactly one open session. |
 
+## `tap_on`
+
+Tap one thing on the screen: an element by its text (txt or a11y from inspect_screen, matched whole and ignoring case, not as a regex), by its id (rid), or a point as x and y percentages of the screen. Give exactly one target. The fastest way to act: it answers once the screen has settled, usually well under a second. A target that is not on screen waits about 17 seconds before it fails, so inspect_screen first. For several steps or an assertion, use run.
+
+| Input | Type | Required | Description |
+|-------|------|----------|-------------|
+| `session_id` | string | No | The session id from start_device, e.g. "brave-teal-otter". Optional when your organization has exactly one open session. |
+| `text` | string | No | The element's text, label or hint, copied from inspect_screen. |
+| `id` | string | No | The element's resource id (rid in inspect_screen). |
+| `x` | number, 0 to 100 | No | x as a percentage of the screen, 0 to 100. |
+| `y` | number, 0 to 100 | No | y as a percentage of the screen, 0 to 100. |
+
+## `input_text`
+
+Type into the focused field (tap_on the field first). Answers once the screen has settled. With sensitive: true the text is stored encrypted, never shown and masked in the output: use it for passwords and codes.
+
+| Input | Type | Required | Description |
+|-------|------|----------|-------------|
+| `session_id` | string | No | The session id from start_device, e.g. "brave-teal-otter". Optional when your organization has exactly one open session. |
+| `text` | string | Yes | What to type, as the user would. |
+| `sensitive` | boolean | No | The text is a secret: a password, a code. Default false. |
+
+## `swipe`
+
+Swipe across the screen: a direction (up scrolls the content up, i.e. down the page), or from one point to another as x and y percentages. Give either direction, or from and to. Answers once the screen has settled.
+
+| Input | Type | Required | Description |
+|-------|------|----------|-------------|
+| `session_id` | string | No | The session id from start_device, e.g. "brave-teal-otter". Optional when your organization has exactly one open session. |
+| `direction` | `up`, `down`, `left` or `right` | No | A swipe across most of the screen. |
+| `from` | object `{ x, y }` | No | Where the swipe starts, in percentages of the screen. |
+| `to` | object `{ x, y }` | No | Where it ends. |
+
+## `press_key`
+
+Press a device key: back (Android), home, enter, backspace, tab, volume_up, volume_down, lock, power. enter submits the focused field; back closes a keyboard or goes back. Answers once the screen has settled.
+
+| Input | Type | Required | Description |
+|-------|------|----------|-------------|
+| `session_id` | string | No | The session id from start_device, e.g. "brave-teal-otter". Optional when your organization has exactly one open session. |
+| `key` | `back`, `home`, `enter`, `backspace`, `tab`, `volume_up`, `volume_down`, `lock` or `power` | Yes | Which key. |
+
 ## `run`
 
-Run Maestro on the device: `yaml` is one flow (a header naming the app, `---`, then commands), run whole. Or `flow_upload_id` runs an uploaded zip of flows (`flow` names the entry file; omitted, all of them run). Waits up to 110s (wait_seconds changes it); a longer flow answers status "running" with a command_id: call get_command_status with wait: true. A failed run's result has screen_text, the app's visible text after the failure, and failed_at: the file, line and command that failed. See cheat_sheet for syntax.
+Run Maestro on the device: `yaml` is one flow (a header naming the app, `---`, then commands), run whole. Or `flow_upload_id` runs an uploaded zip of flows (`flow` names the entry file; omitted, all of them run). For a single tap, swipe, key or typed text prefer tap_on, swipe, press_key and input_text, which are faster; use run for several steps, assertions, scrolling until something is visible, or launching an app. Waits up to 110s (wait_seconds changes it); a longer flow answers status "running" with a command_id: call get_command_status with wait: true. A failed run's result has screen_text, the app's visible text after the failure, and failed_at: the file, line and command that failed. See cheat_sheet for syntax.
 
 | Input | Type | Required | Description |
 |-------|------|----------|-------------|
diff --git a/mcp/agent-devices.md b/mcp/agent-devices.md
index 9b02f20..b183bc4 100644
--- a/mcp/agent-devices.md
+++ b/mcp/agent-devices.md
@@ -108,7 +108,7 @@ Every client you sign in shows up under **Settings → Account → Connected app
 1. `list_devices` shows the devices it can start, and any sessions already open.
 2. `start_device` leases a device and waits up to a minute for it. If the fleet is busy, the session is queued, which isn't an error: it starts on its own, and the agent waits for it with `get_device_status`.
 3. `install_app` installs a build you uploaded, or one at an HTTPS link. `launch_app` starts it.
-4. The agent loops: `inspect_screen` to see what's on screen, then `run` to act with Maestro commands, then `inspect_screen` again. `take_screenshot` shows the screen as an image, and `get_logs` reads the app's device log.
+4. The agent loops: `inspect_screen` to see what's on screen, then `tap_on`, `input_text`, `swipe` or `press_key` for one step, or `run` for several Maestro commands, then `inspect_screen` again. `take_screenshot` shows the screen as an image, and `get_logs` reads the app's device log.
 5. `stop_device` releases the device when it's done.
 
 Every response carries a short **session** block: the session's state, how long it has left, its cost so far, its page in the console, and plain-English warnings, such as an upcoming idle timeout or the app having crashed. The agent never needs a separate call to stay oriented. Agents are told to give you the session's console link as soon as they have it, so you can watch.

From 31b29023a45120959b8123f5cda1ef632afe9b56 Mon Sep 17 00:00:00 2001
From: finalerock44 <ethan@devicecloud.dev>
Date: Wed, 11 Nov 2026 13:07:35 +0000
Subject: [PATCH 2/2] docs(openapi): typed actions are performed on the device
 and settle; mode: maestro

The performAction description said each action becomes a Maestro command
that waits up to 120 s. It now says the device performs it directly and
answers once the screen has settled, that mode: "maestro" runs it as a
Maestro flow instead, and that a tap on something not on screen waits
about 17 s. Each action schema gains the optional mode field (ActionMode),
and TapAction says text and id are matched whole, ignoring case.
---
 public/openapi.yaml | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/public/openapi.yaml b/public/openapi.yaml
index 19f48cc..331a14f 100644
--- a/public/openapi.yaml
+++ b/public/openapi.yaml
@@ -1422,11 +1422,13 @@ paths:
       operationId: performAction
       summary: Perform one action
       description: >-
-        Sends one action to the device: `tap`, `type`, `swipe` or `press_key`. Each becomes one Maestro
-        command. The request waits for it to finish, up to 120 s, after waiting up to 45 s for a device that
-        is still starting or installing. An action that fails on the device still answers 201, with
-        `result.success` false. A `type` action with `sensitive: true` sends its text as a secret: it never
-        appears in the timeline and is masked in `inspect`.
+        Sends one action to the device: `tap`, `type`, `swipe` or `press_key`. The device performs it
+        directly and the request answers once the screen has settled, usually well under a second (up to
+        120 s, after waiting up to 45 s for a device that is still starting or installing). `mode: "maestro"`
+        runs the action as a one-command Maestro flow instead. A `tap` on something that is not on screen
+        waits about 17 s before it fails, so `inspect` first. An action that fails on the device still answers
+        201, with `result.success` false. A `type` action with `sensitive: true` sends its text as a secret: it
+        never appears in the timeline and is masked in `inspect`.
       parameters:
         - name: x-dcd-source
           in: header
@@ -4073,12 +4075,15 @@ components:
       type: object
       description: >-
         Tap one target: a point (`x` and `y`), an element's `text`, or its `id`. Exactly one. Text and ids are
-        matched literally, not as regular expressions.
+        matched whole and ignoring case, literally, not as regular expressions. A target that is not on
+        screen waits about 17 s before the tap fails.
       required: [type]
       properties:
         type:
           type: string
           const: tap
+        mode:
+          $ref: '#/components/schemas/ActionMode'
         x:
           type: number
           minimum: 0
@@ -4112,6 +4117,8 @@ components:
         type:
           type: string
           const: type
+        mode:
+          $ref: '#/components/schemas/ActionMode'
         text:
           type: string
           minLength: 1
@@ -4131,6 +4138,8 @@ components:
         type:
           type: string
           const: swipe
+        mode:
+          $ref: '#/components/schemas/ActionMode'
         direction:
           type: string
           enum: [up, down, left, right]
@@ -4150,10 +4159,19 @@ components:
         type:
           type: string
           const: press_key
+        mode:
+          $ref: '#/components/schemas/ActionMode'
         key:
           type: string
           enum: [back, home, enter, backspace, tab, volume_up, volume_down, lock, power]
 
+    ActionMode:
+      type: string
+      enum: [maestro]
+      description: >-
+        `maestro` runs the action as a one-command Maestro flow instead of directly on the device: slower,
+        with Maestro's own waits. Leave it out otherwise.
+
     ScreenPoint:
       type: object
       description: A point on the screen, in percent. Each value is rounded to a whole number.