diff --git a/scripts/help-conformance-cases.mjs b/scripts/help-conformance-cases.mjs index 84c5b0a54c..92a4a56dbc 100644 --- a/scripts/help-conformance-cases.mjs +++ b/scripts/help-conformance-cases.mjs @@ -152,6 +152,28 @@ export const CASES = [ { id: 'runsPlatformAudit', pattern: /\bagent-device\s+audit\b/i }, ], }, + { + id: 'accessibility-review', + docs: ['--help:first30', 'accessibility'], + task: 'Review the accessibility of the installed iOS shop app com.example.shop. Cover the Home and Cart tabs, including large text, and capture evidence for each finding. The simulator starts at text size large in light mode. Plan commands only.', + allowedExternalCommands: ['mkdir'], + expectations: ['validPlanCommands', 'fullPrefix', 'usesSnapshotI', 'opensAndCloses'], + matchers: [ + { id: 'runsAudit', pattern: /\bagent-device\s+audit\b/i }, + // More than one screen: at least two audit runs, because the topic runs audit on + // each screen. A screen change without a second audit does not count. + { + id: 'auditsMoreThanOneScreen', + test: ({ joined }) => (joined.match(/\bagent-device\s+audit\b/gi) ?? []).length >= 2, + }, + { id: 'changesTextSize', pattern: /\bagent-device\s+settings\s+text-size\s+accessibility-/i }, + { + id: 'relaunchesAfterTextSize', + pattern: + /\bsettings\s+text-size\s+accessibility-[\s\S]*\bagent-device\s+open\s+[^\n]*--relaunch\b/i, + }, + ], + }, { id: 'engineering-validate-mode', docs: ['--help:first30', 'validate'], diff --git a/skills/agent-device/SKILL.md b/skills/agent-device/SKILL.md index 948a189165..603796cd69 100644 --- a/skills/agent-device/SKILL.md +++ b/skills/agent-device/SKILL.md @@ -15,7 +15,9 @@ That starts the session and returns the initial interactive snapshot with `@refs Loop: act with `press|click|fill|longpress ... --settle`, `scroll --settle`, or `back --settle`; continue from the printed diff, verify the named expectation with a read-only command (`wait text "..."`, `is`, or `get`; bare `find` clicks, so check with `find text "..." exists`), then run `agent-device close`, or `agent-device close --shutdown` if your session booted a local simulator or emulator, unless the user asks to keep it running. -For exploratory QA or accessibility checks, read `agent-device help dogfood`; its platform checks run `agent-device audit`. +For exploratory QA, read `agent-device help dogfood`; its platform checks run `agent-device audit`. + +For an accessibility review of a whole app, read `agent-device help accessibility`. Reaching an off-screen target is one command, not a scroll-and-check loop: `scroll down --until ` scrolls until that element is on screen, and `scroll bottom` runs to the end of the content. Repeated bare `scroll down` calls are the slow way to find something. diff --git a/src/commands/schema/cli-help-first-screen.test.ts b/src/commands/schema/cli-help-first-screen.test.ts index cbd3e89f8a..461db0bfa3 100644 --- a/src/commands/schema/cli-help-first-screen.test.ts +++ b/src/commands/schema/cli-help-first-screen.test.ts @@ -136,6 +136,13 @@ const FIRST_SCREEN_RULES: Record = { 'Follow Apple', 'React Native warning/error overlays can be real findings or test blockers. Capture them, use react-native dismiss-overlay', ], + accessibility: [ + 'Audit findings are advisory heuristics', + 'This tool does not measure color contrast or focus order, and does not give a compliance verdict. Do not claim compliance', + '2. Run agent-device audit, then agent-device snapshot -i', + 'Visit every tab, pushed screen, sheet, dialog, and empty, error, and loading state', + 'Large text: first run agent-device settings text-size to record the current size.', + ], validate: [ 'Prove the changed behavior through public agent-device surfaces. Do not validate against stale dist output', 'For a TypeScript runtime or CLI output change, start with pnpm build', diff --git a/src/commands/schema/cli-help-overview.ts b/src/commands/schema/cli-help-overview.ts index 51fa29cf6c..4bcc75a2c3 100644 --- a/src/commands/schema/cli-help-overview.ts +++ b/src/commands/schema/cli-help-overview.ts @@ -54,7 +54,7 @@ More commands (exact shapes: agent-device help ): Guides (agent-device help ): workflow full refs, selectors, waits, recovery, and platform limits - manual-qa / dogfood / validate / debugging / scripting / gestures + manual-qa / dogfood / accessibility / validate / debugging / scripting / gestures react-native / react-devtools / cdp / tv / web / macos / remote physical-device / ios-system-ui / maestro foldable iPhone Duo panels, the fold command, and why refs die on a pose change diff --git a/src/commands/schema/cli-help-workflows.test.ts b/src/commands/schema/cli-help-workflows.test.ts index 926a25e8cf..613ef7a2b5 100644 --- a/src/commands/schema/cli-help-workflows.test.ts +++ b/src/commands/schema/cli-help-workflows.test.ts @@ -85,6 +85,23 @@ test('usageForCommand resolves validate help topic', async () => { assert.match(help, /Runner reuse is authorized only by the cache metadata's content manifest/); }); +test('usageForCommand resolves accessibility help topic', async () => { + const help = await usageForCommand('accessibility'); + if (help === null) throw new Error('Expected accessibility help text'); + assert.match(help, /^agent-device \S+ — accessibility/); + assert.match(help, /Do not claim compliance/); + assert.match(help, /Audit captures the screen itself, so it goes first/); + assert.match(help, /agent-device settings text-size accessibility-extra-extra-extra-large/); + assert.match(help, /agent-device open --relaunch/); + assert.match(help, /run agent-device settings text-size to record the current size/); + assert.match(help, /set the recorded size again/); + assert.match(help, /agent-device settings appearance dark/); + assert.match(help, /set the noted mode again/); + assert.doesNotMatch(help, /|/); + assert.match(help, /agent-device scroll down 0\.8, then run audit again/); + assert.match(help, /Group findings by screen/); +}); + test('usageForCommand resolves dogfood help topic', async () => { const help = await usageForCommand('dogfood'); if (help === null) throw new Error('Expected dogfood help text'); diff --git a/src/commands/schema/cli-help-workflows.ts b/src/commands/schema/cli-help-workflows.ts index 0c36d11a98..507fd0ea81 100644 --- a/src/commands/schema/cli-help-workflows.ts +++ b/src/commands/schema/cli-help-workflows.ts @@ -199,6 +199,48 @@ React Native internals: } as const; export const qaReportHelpTopics = { + accessibility: { + summary: 'Whole-app accessibility review with audit and device-setting checks', + body: `agent-device help accessibility + +Use this when asked for an accessibility review, a11y audit, or a screen-reader or touch-target check of an app. + +Scope: + Audit findings are advisory heuristics. Audit checks touch-target size, missing accessible names, and covered controls. + This tool does not measure color contrast or focus order, and does not give a compliance verdict. Do not claim compliance. + +Loop for each screen: + 1. Open the app: agent-device open . Run mkdir -p ./a11y-output once. + 2. Run agent-device audit, then agent-device snapshot -i. Audit captures the screen itself, so it goes first. + 3. Record each finding with its kind, role, and size. Audit's pinned refs (@eN~sG) stay valid only until the next snapshot. + To act on a finding, use the ref from that snapshot -i. The ref numbers can differ, so do not match by ref number. + Find the element in the snapshot by role and position. To compare positions, use the rect from agent-device audit --json and from agent-device snapshot -i --json. + 4. Capture evidence: agent-device screenshot ./a11y-output/.png --overlay-refs + +Coverage: + Visit every tab, pushed screen, sheet, dialog, and empty, error, and loading state. + Audit covers the visible screen only. On a long list, run agent-device scroll down 0.8, then run audit again. + +Checks audit cannot do: + Large text: first run agent-device settings text-size to record the current size. Then run agent-device settings text-size accessibility-extra-extra-extra-large, then agent-device open --relaunch. The app adopts the size only after a relaunch. + Then run audit and screenshot each key screen. Look for clipped, truncated, or overlapping text. + At the end, set the recorded size again. For example, if the recorded size was large, run agent-device settings text-size large. + Dark mode: first note the current mode from a screenshot. Then run agent-device settings appearance dark, then screenshot each key screen. Look for low-contrast text and icons by eye. + At the end, set the noted mode again. For example, if the noted mode was light, run agent-device settings appearance light. + If settings refuses on this device, such as a physical iPhone, list large text and dark mode as not done. + Screen reader order and labels: read agent-device snapshot (without -i) from top to bottom. The order must match the visual order. + A label must be specific and unique: "Save Lemon Herb Rice", not "Save". A control must expose its state (selected, checked, disabled). + Gesture-only actions: every swipe or long-press action needs another way to reach it, such as a visible control or an accessibility action. + On the iOS simulator, agent-device snapshot -i --actions names the custom actions that a merged element hides. + +Fix loop (only when the user asks to fix): + Change the app source, then reload (Fast Refresh, or agent-device open --relaunch). + Run audit again on the same screen and confirm the finding is gone. Report the before and after audit output. + +Report: + Group findings by screen. For each finding, give the kind, element name, ref, screenshot path, and suggested fix. + List the checks you could not do. Include audit's limitation lines in that list.`, + }, dogfood: { summary: 'Exploratory QA workflow with reproducible evidence', body: `agent-device help dogfood @@ -219,7 +261,8 @@ Loop: Keep stateful commands serial within the same session. Parallel runs can pollute text fields, focus, alerts, and navigation state. Platform checks: - On the first screen, and after a screen changes meaningfully, run audit, then snapshot -i. Audit captures the screen itself, so it goes first. Act on the refs from that snapshot, or on the pinned refs (@eN~sG) audit prints. Report audit's findings as accessibility findings. + On the first screen, and after a screen changes meaningfully, run audit, then snapshot -i. Audit captures the screen itself, so it goes first. Audit's pinned refs (@eN~sG) stay valid only until the next snapshot. To act on a finding, use the ref from that snapshot -i. Find the element by role and position, not by ref number. Report audit's findings as accessibility findings. + For a whole-app accessibility review, read help accessibility. On iOS, test Back and the edge-back swipe where a screen was pushed. On Android, test system Back and applicable sheet dismissal. Where a sheet is dismissible, swipe it down with no edits and expect dismiss. Dismiss it after an edit and expect confirm or save. Then press explicit Cancel. When the app autosaves, discards explicitly, requires the flow, or presents the surface as nondismissible, skip that probe. diff --git a/website/docs/docs/agent-setup.md b/website/docs/docs/agent-setup.md index 1179415ec2..eb47545b01 100644 --- a/website/docs/docs/agent-setup.md +++ b/website/docs/docs/agent-setup.md @@ -33,7 +33,8 @@ Add this rule to your agent client's project rules or custom instructions. [Set Use agent-device only for app/device automation tasks. For a normal app-driving task, start immediately. Do not probe first with `--help`, `--version`, `devices`, `appstate`, `snapshot`, or `screenshot`; open the requested app in the foreground and continue from its initial interactive snapshot. For TV, Fire TV, or Vega OS tasks, read `agent-device help tv`. -For exploratory QA or accessibility checks, read `agent-device help dogfood`; its platform checks run `agent-device audit`. +For exploratory QA, read `agent-device help dogfood`; its platform checks run `agent-device audit`. +For an accessibility review of a whole app, read `agent-device help accessibility`. For logs, network, audio, traces, or runtime failures, read `agent-device help debugging`. For React Native component internals (tree, props, state, hooks) and performance profiling (slow renders, rerenders, and their causes), read `agent-device help react-devtools`. For React Native JavaScript heap growth, heap snapshots, allocation hotspots, or retained-object leaks, read `agent-device help cdp`.