From 38860ef1ca210ea8b3557c3605a884ccb17f42c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 19:41:01 +0100 Subject: [PATCH 01/20] feat(audit): name findings and explain platform guidance --- .../snapshot/__tests__/snapshot-audit.test.ts | 97 +++++++++++++++++++ .../src/snapshot/snapshot-audit.ts | 51 +++++++++- .../src/capture/audit-output.test.ts | 37 ++++++- .../src/capture/audit-output.ts | 23 +++-- .../command-output/src/cli-output.test.ts | 1 + .../command-surface/src/capture/snapshot.ts | 14 ++- packages/contracts/src/audit.ts | 4 + .../__tests__/support/command-surface-fake.ts | 1 + website/docs/docs/commands.md | 4 +- 9 files changed, 214 insertions(+), 18 deletions(-) diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts index 2b864f0f31..de1cbdab98 100644 --- a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts @@ -321,3 +321,100 @@ describe('scope', () => { expect(result.limitations).toEqual([expect.stringContaining('web rects')]); }); }); + +test('publishes the captured label and identifier but omits a missing-name label', () => { + const findings = audit([ + node(1, { label: 'Save Lemon Herb Rice', rect: { x: 0, y: 0, width: 22, height: 22 } }), + node(2, { identifier: 'save', label: 'com.example:id/save' }), + node(3, { label: 'Pay', interactionBlocked: 'covered', hittable: false }), + ]).findings; + expect(findings[0]).toMatchObject({ label: 'Save Lemon Herb Rice' }); + expect(findings[1]).toMatchObject({ kind: 'missing-name', identifier: 'save' }); + expect(findings[1]).not.toHaveProperty('label'); + expect(findings[2]).toMatchObject({ label: 'Pay' }); +}); + +test.each(['tab-bar', 'navigation-bar', 'toolbar', 'bottomnavigation', 'bottomnavigationview'])( + 'ignores %s over a scrolled row but retains it over a fixed control', + (kind) => { + const covered = { + interactionBlocked: 'covered', + hittable: false, + coveredBy: { kind, label: 'Not a chrome label' }, + } as const; + const findings = audit([ + node(0, { type: 'ScrollView', hittable: false }), + node(1, { type: 'Other', parentIndex: 0 }), + node(2, { parentIndex: 1, label: 'Scrolled row', ...covered }), + node(3, { label: 'Fixed control', ...covered }), + ]).findings; + expect(summarize(findings)).toEqual(['covered-control @e3']); + }, +); + +test.each(['sheet', 'dialog', 'banner', 'group', 'floating-view'])( + 'retains %s over a control inside a scroll container', + (kind) => { + const findings = audit([ + node(0, { type: 'ScrollView', hittable: false }), + node(1, { + parentIndex: 0, + label: 'Pay', + interactionBlocked: 'covered', + hittable: false, + coveredBy: { kind, label: 'Tab Bar' }, + }), + ]).findings; + expect(summarize(findings)).toEqual(['covered-control @e1']); + }, +); + +test.each([ + [IOS, 'Apple HIG recommends 44×44pt.'], + [ANDROID, 'Material recommends 48×48dp.'], +] as const)('returns guidance only for present kinds on %j', (context, size) => { + const result = audit( + [ + node(1, { label: 'Save', rect: { x: 0, y: 0, width: 22, height: 22 } }), + node(2, { label: 'Save again', rect: { x: 0, y: 0, width: 22, height: 22 } }), + node(3), + node(4, { interactionBlocked: 'covered', hittable: false }), + ], + context, + ); + expect(result.guidance).toEqual({ + 'small-target': `${size} Enlarge the touch area with padding, a minimum size, or hitSlop.`, + 'missing-name': + 'Set an accessibility label (iOS accessibilityLabel, Android contentDescription, React Native accessibilityLabel).', + 'covered-control': + 'Move the control out from under the overlay, or dismiss the overlay before you use it.', + }); + expect( + audit([node(1, { label: 'Large', rect: { x: 0, y: 0, width: 100, height: 100 } })], context) + .guidance, + ).toEqual({}); + expect( + Object.keys( + audit([node(1, { rect: { x: 0, y: 0, width: 100, height: 100 } })], context).guidance, + ), + ).toEqual(['missing-name']); +}); + +test('keeps a child under an overlay when its covered row is excluded as scroll chrome', () => { + const findings = audit([ + node(0, { type: 'ScrollView', hittable: false }), + node(1, { + type: 'Cell', + parentIndex: 0, + interactionBlocked: 'covered', + coveredBy: { kind: 'tab-bar' }, + }), + node(2, { + parentIndex: 1, + label: 'Pay', + interactionBlocked: 'covered', + coveredBy: { kind: 'sheet' }, + }), + ]).findings; + expect(summarize(findings)).toEqual(['covered-control @e2']); +}); diff --git a/packages/capture-kit/src/snapshot/snapshot-audit.ts b/packages/capture-kit/src/snapshot/snapshot-audit.ts index 948474e2d2..a8ad12ba9c 100644 --- a/packages/capture-kit/src/snapshot/snapshot-audit.ts +++ b/packages/capture-kit/src/snapshot/snapshot-audit.ts @@ -2,10 +2,11 @@ import type { AuditChecks, AuditContext, AuditFinding, + AuditFindingKind, AuditResult, AuditSizeUnit, } from '@agent-device/contracts/capture'; -import { normalizeType } from '@agent-device/contracts/snapshot'; +import { isScrollableNodeLike, normalizeType } from '@agent-device/contracts/snapshot'; import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; import { formatRole, type SnapshotNode } from '@agent-device/kernel/snapshot'; import { @@ -65,6 +66,7 @@ export function auditSnapshot(nodes: readonly SnapshotNode[], context: AuditCont const findings = nodes.flatMap((node) => auditNode(node, scan, rule)); return { findings, + guidance: auditGuidance(findings, context), checks: { snapshot: snapshotCompleteness(context) }, limitations: collectLimitations(context, rule), }; @@ -217,7 +219,8 @@ function smallTargetFinding( } function missingNameFinding(node: SnapshotNode, role: string): AuditFinding { - return { ...findingBase(node, role), kind: 'missing-name', severity: 'warning' }; + const { label: _label, ...base } = findingBase(node, role); + return { ...base, kind: 'missing-name', severity: 'warning' }; } /** @@ -227,6 +230,7 @@ function missingNameFinding(node: SnapshotNode, role: string): AuditFinding { function coveredFinding(node: SnapshotNode, scan: AuditScan): AuditFinding | undefined { const role = coveredRole(node); if (node.interactionBlocked !== 'covered' || role === undefined) return undefined; + if (isScrollableChromeOcclusion(node, scan)) return undefined; if (isOnlyChildOfReportedCover(node, scan)) return undefined; return { ...findingBase(node, role), kind: 'covered-control', severity: 'observation' }; } @@ -243,7 +247,9 @@ function isRow(node: SnapshotNode): boolean { function isOnlyChildOfReportedCover(node: SnapshotNode, scan: AuditScan): boolean { const parent = parentOf(node, scan); if (!parent || parent.ref === undefined || coveredRole(parent) === undefined) return false; - return scan.children.get(parent.index)?.length === 1; + return ( + !isScrollableChromeOcclusion(parent, scan) && scan.children.get(parent.index)?.length === 1 + ); } function findingBase(node: SnapshotNode, role: string) { @@ -251,6 +257,8 @@ function findingBase(node: SnapshotNode, role: string) { ref: `@${node.ref}`, ...(node.rect ? { rect: node.rect } : {}), role, + ...(node.label?.trim() ? { label: node.label.trim() } : {}), + ...(node.identifier?.trim() ? { identifier: node.identifier.trim() } : {}), }; } @@ -287,3 +295,40 @@ function typedContent(node: SnapshotNode): string | undefined { function roundTenth(value: number): number { return Math.round(value * 10) / 10; } + +const PERSISTENT_CHROME_KINDS = new Set([ + 'tabbar', + 'navigationbar', + 'toolbar', + 'bottomnavigation', + 'bottomnavigationview', +]); + +function isScrollableChromeOcclusion(node: SnapshotNode, scan: AuditScan): boolean { + if (!PERSISTENT_CHROME_KINDS.has(normalizeType(node.coveredBy?.kind ?? '').replaceAll('-', ''))) + return false; + const visited = new Set(); + let ancestor = parentOf(node, scan); + while (ancestor && !visited.has(ancestor.index)) { + if (isScrollableNodeLike(ancestor)) return true; + visited.add(ancestor.index); + ancestor = parentOf(ancestor, scan); + } + return false; +} + +function auditGuidance( + findings: readonly AuditFinding[], + context: AuditContext, +): AuditResult['guidance'] { + const size = + context.platform === 'ios' ? 'Apple HIG recommends 44×44pt.' : 'Material recommends 48×48dp.'; + const sentences: Record = { + 'small-target': `${size} Enlarge the touch area with padding, a minimum size, or hitSlop.`, + 'missing-name': + 'Set an accessibility label (iOS accessibilityLabel, Android contentDescription, React Native accessibilityLabel).', + 'covered-control': + 'Move the control out from under the overlay, or dismiss the overlay before you use it.', + }; + return Object.fromEntries(findings.map(({ kind }) => [kind, sentences[kind]])); +} diff --git a/packages/command-output/src/capture/audit-output.test.ts b/packages/command-output/src/capture/audit-output.test.ts index 240575819b..8811220c7e 100644 --- a/packages/command-output/src/capture/audit-output.test.ts +++ b/packages/command-output/src/capture/audit-output.test.ts @@ -12,6 +12,7 @@ test('renders warnings and observations as aligned lines with the advisory wordi severity: 'warning', ref: '@e7', role: 'button', + label: 'Save Lemon Herb Rice', detail: { unit: 'pt', width: 22, height: 22.5, recommendedSize: 44 }, }, { @@ -19,6 +20,7 @@ test('renders warnings and observations as aligned lines with the advisory wordi severity: 'warning', ref: '@e9', role: 'text-field', + identifier: 'email', }, { kind: 'covered-control', @@ -27,6 +29,7 @@ test('renders warnings and observations as aligned lines with the advisory wordi role: 'button', }, ], + guidance: {}, checks: CHECKS, limitations: [], refsGeneration: 3, @@ -35,9 +38,9 @@ test('renders warnings and observations as aligned lines with the advisory wordi [ 'agent-device audit', '2 warnings, 1 observation — current screen', - '@e7~s3 small-target 22×22.5pt; review touch area', - '@e9~s3 missing-name Text field has no accessible name', - '@e12~s3 covered-control Accessible action is unavailable to touch', + '@e7~s3 small-target button "Save Lemon Herb Rice" 22×22.5pt, below 44×44pt', + '@e9~s3 missing-name text-field id="email" No accessible name', + '@e12~s3 covered-control button Accessible action is unavailable to touch', '', ].join('\n'), ); @@ -46,6 +49,7 @@ test('renders warnings and observations as aligned lines with the advisory wordi test('prints plain refs when the result carries no generation to pin them to', () => { const output = auditCliOutput({ findings: [{ kind: 'missing-name', severity: 'warning', ref: '@e9', role: 'button' }], + guidance: {}, checks: CHECKS, limitations: [], }); @@ -56,6 +60,7 @@ test('prints plain refs when the result carries no generation to pin them to', ( test('says no findings and lists each limitation', () => { const output = auditCliOutput({ findings: [], + guidance: {}, checks: CHECKS, limitations: ['Target size was not checked: tv targets are focus-driven.'], }); @@ -71,6 +76,30 @@ test('says no findings and lists each limitation', () => { }); test('keeps the structured result as the JSON payload', () => { - const result: AuditSnapshotCommandResult = { findings: [], checks: CHECKS, limitations: [] }; + const result: AuditSnapshotCommandResult = { + findings: [], + guidance: {}, + checks: CHECKS, + limitations: [], + }; expect(auditCliOutput(result).data).toBe(result); }); + +test('prints each guidance entry once after the findings and preserves it in JSON', () => { + const result: AuditSnapshotCommandResult = { + findings: [1, 2].map((index) => ({ + kind: 'missing-name', + severity: 'warning', + ref: `@e${index}`, + role: 'button', + })), + guidance: { 'missing-name': 'Set an accessibility label.' }, + checks: CHECKS, + limitations: [], + }; + const output = auditCliOutput(result); + expect(output.text).toContain('button No accessible name'); + expect(output.text!.split('missing-name: Set an accessibility label.')).toHaveLength(2); + expect(output.text!.indexOf('missing-name:')).toBeGreaterThan(output.text!.indexOf('@e2')); + expect(output.data).toBe(result); +}); diff --git a/packages/command-output/src/capture/audit-output.ts b/packages/command-output/src/capture/audit-output.ts index 3e4971e6d7..71e977acd6 100644 --- a/packages/command-output/src/capture/audit-output.ts +++ b/packages/command-output/src/capture/audit-output.ts @@ -15,6 +15,9 @@ function formatAuditText(result: AuditSnapshotCommandResult): string { summaryLine(result.findings), ...findingLines(result.findings, result.refsGeneration), ]; + const guidance = Object.entries(result.guidance); + if (guidance.length > 0) + lines.push('', ...guidance.map(([kind, sentence]) => `${kind}: ${sentence}`)); if (result.limitations.length > 0) { lines.push('', ...result.limitations.map((limitation) => `Limitation: ${limitation}`)); } @@ -44,25 +47,31 @@ function findingLines( const kindWidth = Math.max(0, ...findings.map((finding) => finding.kind.length)); return findings.map( (finding, index) => - `${refs[index]!.padEnd(refWidth)} ${finding.kind.padEnd(kindWidth)} ${findingText(finding)}`, + `${refs[index]!.padEnd(refWidth)} ${finding.kind.padEnd(kindWidth)} ${elementText(finding)} ${findingText(finding)}`, ); } function findingText(finding: AuditFinding): string { switch (finding.kind) { case 'small-target': - return `${sizeText(finding.detail)}; review touch area`; + return sizeText(finding.detail); case 'missing-name': - return `${capitalize(finding.role.replaceAll('-', ' '))} has no accessible name`; + return 'No accessible name'; case 'covered-control': return 'Accessible action is unavailable to touch'; } } -function sizeText({ width, height, unit }: AuditSizeDetail): string { - return `${width}×${height}${unit}`; +function sizeText({ width, height, unit, recommendedSize }: AuditSizeDetail): string { + return `${width}×${height}${unit}, below ${recommendedSize}×${recommendedSize}${unit}`; } -function capitalize(text: string): string { - return `${text.charAt(0).toUpperCase()}${text.slice(1)}`; +function elementText(finding: AuditFinding): string { + const name = finding.kind === 'missing-name' ? undefined : finding.label; + const identity = name + ? JSON.stringify(name) + : finding.identifier + ? `id=${JSON.stringify(finding.identifier)}` + : undefined; + return identity ? `${finding.role} ${identity}` : finding.role; } diff --git a/packages/command-output/src/cli-output.test.ts b/packages/command-output/src/cli-output.test.ts index 49bfc1ca4a..afb93ff772 100644 --- a/packages/command-output/src/cli-output.test.ts +++ b/packages/command-output/src/cli-output.test.ts @@ -102,6 +102,7 @@ const RESPONSE_FIXTURES: Partial> = { apps: ['com.example.app'], artifacts: { source: 'daemon', artifacts: [], message: 'No daemon artifacts.' }, audit: { + guidance: {}, findings: [], checks: { snapshot: 'complete' }, limitations: [], diff --git a/packages/command-surface/src/capture/snapshot.ts b/packages/command-surface/src/capture/snapshot.ts index bd456dde9e..34d2f6660c 100644 --- a/packages/command-surface/src/capture/snapshot.ts +++ b/packages/command-surface/src/capture/snapshot.ts @@ -126,6 +126,8 @@ export const AUDIT_COMMAND_OUTPUT_SCHEMAS = { ['x', 'y', 'width', 'height'], ), role: stringSchema(), + label: stringSchema(), + identifier: stringSchema(), detail: objectSchema( { unit: enumSchema(['pt', 'dp']), @@ -139,6 +141,14 @@ export const AUDIT_COMMAND_OUTPUT_SCHEMAS = { ['kind', 'severity', 'ref', 'role'], ), }, + guidance: objectSchema( + { + 'small-target': stringSchema(), + 'missing-name': stringSchema(), + 'covered-control': stringSchema(), + }, + [], + ), checks: objectSchema( { snapshot: enumSchema( @@ -154,12 +164,12 @@ export const AUDIT_COMMAND_OUTPUT_SCHEMAS = { ), warnings: stringArraySchema(), }, - ['findings', 'checks', 'limitations'], + ['findings', 'guidance', 'checks', 'limitations'], ), } satisfies Pick, 'audit'>; const auditCommandDescription = - 'Report advisory accessibility findings from one fresh capture of the current screen: touch targets below the platform recommendation, controls without an accessible name, and controls the snapshot already marks covered. Findings are review prompts, not failures, and the audit changes nothing on the device.'; + 'Report advisory accessibility findings from one fresh capture of the current screen: touch targets below the platform recommendation, controls without an accessible name, and covered controls. Each finding shows its role and accessible label when available. Guidance follows once per finding kind. Controls in scroll containers under persistent bars are excluded. Findings are review prompts, not failures, and the audit changes nothing on the device.'; export const auditCommandFacet = defineParameterlessCommandFacet({ name: AUDIT_COMMAND_NAME, diff --git a/packages/contracts/src/audit.ts b/packages/contracts/src/audit.ts index 73e0fedfa8..80776be126 100644 --- a/packages/contracts/src/audit.ts +++ b/packages/contracts/src/audit.ts @@ -23,6 +23,9 @@ type AuditFindingBase = { rect?: Rect; /** Control role as the snapshot presents it, such as `button`. */ role: string; + /** Accessible label from the captured node. */ + label?: string; + identifier?: string; }; export type AuditFinding = @@ -50,6 +53,7 @@ export type AuditChecks = { export type AuditResult = { findings: AuditFinding[]; + guidance: Partial>; checks: AuditChecks; /** Checks the audit could not run or could only partly run, with the reason. */ limitations: string[]; diff --git a/packages/daemon/src/__tests__/support/command-surface-fake.ts b/packages/daemon/src/__tests__/support/command-surface-fake.ts index c4d4ec1eca..9fcd5a21d9 100644 --- a/packages/daemon/src/__tests__/support/command-surface-fake.ts +++ b/packages/daemon/src/__tests__/support/command-surface-fake.ts @@ -27,6 +27,7 @@ const fakeCommandSurface: DaemonCommandSurface = { })), auditSnapshot: vi.fn(async () => ({ findings: [], + guidance: {}, checks: { snapshot: 'complete' as const }, limitations: [], })), diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 15887d1736..e285bafe5f 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -433,11 +433,11 @@ agent-device audit - `small-target`: an enabled, visible control below 44pt on iOS or 48dp on Android, with a 0.5-unit tolerance for layout rounding. Android pixels convert to dp with the captured display density; without it the check reports a limitation instead. TV and desktop targets skip the check. A link inside a text container is exempt. - `missing-name`: a control with no accessible name. An identifier, a resource id, or a field value is not a name; a label, an Android content description, a placeholder, or a named child is. -- `covered-control`: a control or row the snapshot already marks `[covered]`. +- `covered-control`: a control or row the snapshot already marks `[covered]`. Controls inside scroll containers under tab bars, navigation bars, toolbars, or bottom navigation bars are excluded. Scroll to reveal these controls. Controls under sheets, dialogs, and other overlays remain findings. Controls are recognized by role (button, link, switch, checkbox, radio, slider, menu item, text and search fields). Generic clickable containers on Android are not evaluated. iOS visibility relies on the `hittable` fact. System bars and keyboard keys are skipped. When a control is the only child of a control with the same role, the child drops a finding only if the parent reports the same kind of finding. -Like `find`, `audit` publishes only the refs it returns. The text prints each as `@eN~s`; press that pinned ref. A plain ref from before the audit stays refused after the frame expired, and an audit with no findings issues no refs. An audit with findings also ends the snapshot's complete frame, so take a new snapshot before you use refs that the audit did not return. `--json` returns `findings`, `checks`, `limitations`, and `refsGeneration` (only with findings). `checks.snapshot` is `unknown` when the capture cannot report truncation, such as Appium page source. +Like `find`, `audit` publishes only the refs it returns. The text prints each as `@eN~s`; press that pinned ref. A plain ref from before the audit stays refused after the frame expired, and an audit with no findings issues no refs. An audit with findings also ends the snapshot's complete frame, so take a new snapshot before you use refs that the audit did not return. Each finding shows its role and accessible label. An unnamed control shows its identifier when available. Guidance follows once per finding kind, with Apple HIG sizes on iOS and Material sizes on Android. `--json` returns `findings` (with `label` when available), `guidance` keyed by finding kind, `checks`, `limitations`, and `refsGeneration` (only with findings). `checks.snapshot` is `unknown` when the capture cannot report truncation, such as Appium page source. ## Wait and alerts From 2bae5b60d8bf9d46df607e08d4c14cbba9642f84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 19:50:57 +0100 Subject: [PATCH 02/20] fix(audit): exclude UIKit controls from target size findings --- .../snapshot/__tests__/snapshot-audit.test.ts | 34 +++++++++++++++ .../src/snapshot/snapshot-audit.ts | 42 ++++++++++++++----- .../command-surface/src/capture/snapshot.ts | 2 +- website/docs/docs/commands.md | 2 +- 4 files changed, 67 insertions(+), 13 deletions(-) diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts index de1cbdab98..325aeb23d6 100644 --- a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts @@ -418,3 +418,37 @@ test('keeps a child under an overlay when its covered row is excluded as scroll ]).findings; expect(summarize(findings)).toEqual(['covered-control @e2']); }); + +test('skips UIKit sheet grabbers by captured class without matching their label', () => { + const small = { x: 0, y: 0, width: 73, height: 24 }; + const findings = audit([ + node(1, { role: '_UIGrabber', label: 'Sheet Grabber', value: 'Expanded', rect: small }), + node(2, { role: '_UIGrabber', label: 'Poignée', rect: small }), + node(3, { subrole: '_UIGrabber', rect: small }), + node(4, { label: 'Sheet Grabber', rect: small }), + ]).findings; + expect(summarize(findings)).toEqual(['small-target @e4']); +}); + +test.each(['NavigationBar', 'Toolbar', 'TabBar'])( + 'skips iOS %s target size but keeps missing names and app content', + (type) => { + const small = { x: 0, y: 0, width: 42.2, height: 42.2 }; + const nodes = [ + node(0, { type, hittable: false }), + node(1, { type: 'Other', parentIndex: 0 }), + node(2, { parentIndex: 1, label: 'Cancel', rect: { ...small, width: 50.6 } }), + node(3, { parentIndex: 1, label: 'Add', rect: small }), + node(4, { parentIndex: 1, identifier: 'unnamed-bar-item', rect: small }), + node(5, { label: 'App action', rect: small }), + ]; + expect(summarize(audit(nodes).findings)).toEqual(['missing-name @e4', 'small-target @e5']); + expect(summarize(audit(nodes, ANDROID).findings)).toEqual([ + 'small-target @e2', + 'small-target @e3', + 'small-target @e4', + 'missing-name @e4', + 'small-target @e5', + ]); + }, +); diff --git a/packages/capture-kit/src/snapshot/snapshot-audit.ts b/packages/capture-kit/src/snapshot/snapshot-audit.ts index a8ad12ba9c..b0fce7fb30 100644 --- a/packages/capture-kit/src/snapshot/snapshot-audit.ts +++ b/packages/capture-kit/src/snapshot/snapshot-audit.ts @@ -49,6 +49,7 @@ type SizeRule = | { kind: 'unchecked'; limitation: string }; type AuditScan = { + platform: AuditContext['platform']; byIndex: ReadonlyMap; children: ChildrenByParent; }; @@ -59,6 +60,7 @@ type AuditScan = { */ export function auditSnapshot(nodes: readonly SnapshotNode[], context: AuditContext): AuditResult { const scan: AuditScan = { + platform: context.platform, byIndex: new Map(nodes.map((node) => [node.index, node])), children: buildChildrenByParent(nodes), }; @@ -95,8 +97,12 @@ function controlFindings( scan: AuditScan, rule: SizeRule, ): AuditFinding[] { - if (node.ref === undefined || !isActionable(node)) return []; - const small = isInlineLink(node, role, scan) ? undefined : smallTargetFinding(node, role, rule); + const ownership = controlOwnership(node, scan); + if (node.ref === undefined || !isActionable(node, ownership)) return []; + const small = + ownership === 'platform-sized' || isInlineLink(node, role, scan) + ? undefined + : smallTargetFinding(node, role, rule); const unnamed = hasAccessibleName(node, role, scan) ? undefined : missingNameFinding(node, role); return [small, unnamed].filter((finding) => finding !== undefined); } @@ -124,15 +130,26 @@ function controlRole(node: SnapshotNode): string | undefined { return CONTROL_ROLES.find(([suffix]) => type.endsWith(suffix))?.[1]; } -/** - * Visible, enabled, not already proven unreachable by position, and owned by the app: system bars - * and keyboard keys are the platform's controls. - */ -function isActionable(node: SnapshotNode): boolean { +type ControlOwnership = 'app' | 'platform-sized' | 'system'; + +function controlOwnership(node: SnapshotNode, scan: AuditScan): ControlOwnership { + if (node.systemBar !== undefined || node.keyboardOwned === true) return 'system'; + if (scan.platform !== 'ios') return 'app'; + if (node.role === '_UIGrabber' || node.subrole === '_UIGrabber') return 'system'; + const visited = new Set(); + let ancestor = parentOf(node, scan); + while (ancestor && !visited.has(ancestor.index)) { + if (isPersistentChromeKind(ancestor.type)) return 'platform-sized'; + visited.add(ancestor.index); + ancestor = parentOf(ancestor, scan); + } + return 'app'; +} + +function isActionable(node: SnapshotNode, ownership: ControlOwnership): boolean { return ( isPositiveFiniteRect(node.rect) && - node.systemBar === undefined && - node.keyboardOwned !== true && + ownership !== 'system' && node.enabled !== false && node.visibleToUser !== false && node.hittable !== false && @@ -304,9 +321,12 @@ const PERSISTENT_CHROME_KINDS = new Set([ 'bottomnavigationview', ]); +function isPersistentChromeKind(kind: string | undefined): boolean { + return PERSISTENT_CHROME_KINDS.has(normalizeType(kind ?? '').replaceAll('-', '')); +} + function isScrollableChromeOcclusion(node: SnapshotNode, scan: AuditScan): boolean { - if (!PERSISTENT_CHROME_KINDS.has(normalizeType(node.coveredBy?.kind ?? '').replaceAll('-', ''))) - return false; + if (!isPersistentChromeKind(node.coveredBy?.kind)) return false; const visited = new Set(); let ancestor = parentOf(node, scan); while (ancestor && !visited.has(ancestor.index)) { diff --git a/packages/command-surface/src/capture/snapshot.ts b/packages/command-surface/src/capture/snapshot.ts index 34d2f6660c..d6911e3d89 100644 --- a/packages/command-surface/src/capture/snapshot.ts +++ b/packages/command-surface/src/capture/snapshot.ts @@ -169,7 +169,7 @@ export const AUDIT_COMMAND_OUTPUT_SCHEMAS = { } satisfies Pick, 'audit'>; const auditCommandDescription = - 'Report advisory accessibility findings from one fresh capture of the current screen: touch targets below the platform recommendation, controls without an accessible name, and covered controls. Each finding shows its role and accessible label when available. Guidance follows once per finding kind. Controls in scroll containers under persistent bars are excluded. Findings are review prompts, not failures, and the audit changes nothing on the device.'; + 'Report advisory accessibility findings from one fresh capture of the current screen: touch targets below the platform recommendation, controls without an accessible name, and covered controls. Each finding shows its role and accessible label when available. Guidance follows once per finding kind. Controls in scroll containers under persistent bars are excluded. UIKit sheet grabbers are skipped. iOS bar controls skip size checks but keep name checks. Findings are review prompts, not failures, and the audit changes nothing on the device.'; export const auditCommandFacet = defineParameterlessCommandFacet({ name: AUDIT_COMMAND_NAME, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index e285bafe5f..2aadeb1ce7 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -435,7 +435,7 @@ agent-device audit - `missing-name`: a control with no accessible name. An identifier, a resource id, or a field value is not a name; a label, an Android content description, a placeholder, or a named child is. - `covered-control`: a control or row the snapshot already marks `[covered]`. Controls inside scroll containers under tab bars, navigation bars, toolbars, or bottom navigation bars are excluded. Scroll to reveal these controls. Controls under sheets, dialogs, and other overlays remain findings. -Controls are recognized by role (button, link, switch, checkbox, radio, slider, menu item, text and search fields). Generic clickable containers on Android are not evaluated. iOS visibility relies on the `hittable` fact. System bars and keyboard keys are skipped. When a control is the only child of a control with the same role, the child drops a finding only if the parent reports the same kind of finding. +Controls are recognized by role (button, link, switch, checkbox, radio, slider, menu item, text and search fields). Generic clickable containers on Android are not evaluated. iOS visibility relies on the `hittable` fact. System bars, keyboard keys, and UIKit sheet grabbers are skipped. On iOS, controls inside navigation bars, toolbars, and tab bars skip target-size checks because the platform sets their size. These bar controls still receive missing-name checks. A system-generated accessible name, such as "add" or "remove" for an icon-only React Native Pressable on iOS 26, prevents a missing-name finding. The audit cannot determine whether the app supplied that name. When a control is the only child of a control with the same role, the child drops a finding only if the parent reports the same kind of finding. Like `find`, `audit` publishes only the refs it returns. The text prints each as `@eN~s`; press that pinned ref. A plain ref from before the audit stays refused after the frame expired, and an audit with no findings issues no refs. An audit with findings also ends the snapshot's complete frame, so take a new snapshot before you use refs that the audit did not return. Each finding shows its role and accessible label. An unnamed control shows its identifier when available. Guidance follows once per finding kind, with Apple HIG sizes on iOS and Material sizes on Android. `--json` returns `findings` (with `label` when available), `guidance` keyed by finding kind, `checks`, `limitations`, and `refsGeneration` (only with findings). `checks.snapshot` is `unknown` when the capture cannot report truncation, such as Appium page source. From aee392e8fbb7787756e7cfd08004fda94cd8502b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 20:24:05 +0100 Subject: [PATCH 03/20] fix(audit): bound bar ownership and type audit fixtures --- .../snapshot/__tests__/snapshot-audit.test.ts | 21 +++++++++++++++---- .../src/snapshot/snapshot-audit.ts | 10 ++------- .../command-output/src/cli-output.test.ts | 3 ++- .../command-surface/src/capture/snapshot.ts | 2 +- src/mcp/__tests__/tool-ref-pins.test.ts | 14 +++++++++++-- website/docs/docs/commands.md | 2 +- 6 files changed, 35 insertions(+), 17 deletions(-) diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts index 325aeb23d6..ddce9cbdcf 100644 --- a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts @@ -436,10 +436,9 @@ test.each(['NavigationBar', 'Toolbar', 'TabBar'])( const small = { x: 0, y: 0, width: 42.2, height: 42.2 }; const nodes = [ node(0, { type, hittable: false }), - node(1, { type: 'Other', parentIndex: 0 }), - node(2, { parentIndex: 1, label: 'Cancel', rect: { ...small, width: 50.6 } }), - node(3, { parentIndex: 1, label: 'Add', rect: small }), - node(4, { parentIndex: 1, identifier: 'unnamed-bar-item', rect: small }), + node(2, { parentIndex: 0, label: 'Cancel', rect: { ...small, width: 50.6 } }), + node(3, { parentIndex: 0, label: 'Add', rect: small }), + node(4, { parentIndex: 0, identifier: 'unnamed-bar-item', rect: small }), node(5, { label: 'App action', rect: small }), ]; expect(summarize(audit(nodes).findings)).toEqual(['missing-name @e4', 'small-target @e5']); @@ -452,3 +451,17 @@ test.each(['NavigationBar', 'Toolbar', 'TabBar'])( ]); }, ); + +test.each(['NavigationBar', 'Toolbar', 'TabBar', 'BottomNavigationView'])( + 'checks app content inside a %s-shaped container', + (type) => { + for (const container of ['ScrollView', 'Other', 'Button']) { + const result = audit([ + node(0, { type, hittable: false }), + node(1, { type: container, label: 'Content', parentIndex: 0 }), + node(2, { label: 'Save', parentIndex: 1, rect: { x: 0, y: 0, width: 22, height: 22 } }), + ]); + expect(summarize(result.findings), container).toEqual(['small-target @e2']); + } + }, +); diff --git a/packages/capture-kit/src/snapshot/snapshot-audit.ts b/packages/capture-kit/src/snapshot/snapshot-audit.ts index b0fce7fb30..ea605d652d 100644 --- a/packages/capture-kit/src/snapshot/snapshot-audit.ts +++ b/packages/capture-kit/src/snapshot/snapshot-audit.ts @@ -136,14 +136,8 @@ function controlOwnership(node: SnapshotNode, scan: AuditScan): ControlOwnership if (node.systemBar !== undefined || node.keyboardOwned === true) return 'system'; if (scan.platform !== 'ios') return 'app'; if (node.role === '_UIGrabber' || node.subrole === '_UIGrabber') return 'system'; - const visited = new Set(); - let ancestor = parentOf(node, scan); - while (ancestor && !visited.has(ancestor.index)) { - if (isPersistentChromeKind(ancestor.type)) return 'platform-sized'; - visited.add(ancestor.index); - ancestor = parentOf(ancestor, scan); - } - return 'app'; + const parent = parentOf(node, scan); + return parent && isPersistentChromeKind(parent.type) ? 'platform-sized' : 'app'; } function isActionable(node: SnapshotNode, ownership: ControlOwnership): boolean { diff --git a/packages/command-output/src/cli-output.test.ts b/packages/command-output/src/cli-output.test.ts index afb93ff772..ab345e1903 100644 --- a/packages/command-output/src/cli-output.test.ts +++ b/packages/command-output/src/cli-output.test.ts @@ -1,3 +1,4 @@ +import type { AuditSnapshotCommandResult } from '@agent-device/contracts/capture'; import { bindTestCommandHost } from '@agent-device/test-kit'; import { describe, expect, test } from 'vitest'; import { findCommandMetadata, listCommandMetadata } from '@agent-device/command-surface'; @@ -106,7 +107,7 @@ const RESPONSE_FIXTURES: Partial> = { findings: [], checks: { snapshot: 'complete' }, limitations: [], - }, + } satisfies AuditSnapshotCommandResult, batch: { executed: 0, total: 0, totalDurationMs: 5, results: [] }, capabilities: { device: { name: 'iPhone' }, availableCommands: [] }, debug: { diff --git a/packages/command-surface/src/capture/snapshot.ts b/packages/command-surface/src/capture/snapshot.ts index d6911e3d89..9848c6eb09 100644 --- a/packages/command-surface/src/capture/snapshot.ts +++ b/packages/command-surface/src/capture/snapshot.ts @@ -169,7 +169,7 @@ export const AUDIT_COMMAND_OUTPUT_SCHEMAS = { } satisfies Pick, 'audit'>; const auditCommandDescription = - 'Report advisory accessibility findings from one fresh capture of the current screen: touch targets below the platform recommendation, controls without an accessible name, and covered controls. Each finding shows its role and accessible label when available. Guidance follows once per finding kind. Controls in scroll containers under persistent bars are excluded. UIKit sheet grabbers are skipped. iOS bar controls skip size checks but keep name checks. Findings are review prompts, not failures, and the audit changes nothing on the device.'; + 'Report advisory accessibility findings from one fresh capture of the current screen: touch targets below the platform recommendation, controls without an accessible name, and covered controls. Each finding shows its role and accessible label when available. Guidance follows once per finding kind. Controls in scroll containers under persistent bars are excluded. UIKit sheet grabbers are skipped. Direct iOS bar items skip size checks but keep name checks. Nested app content remains checked. Findings are review prompts, not failures, and the audit changes nothing on the device.'; export const auditCommandFacet = defineParameterlessCommandFacet({ name: AUDIT_COMMAND_NAME, diff --git a/src/mcp/__tests__/tool-ref-pins.test.ts b/src/mcp/__tests__/tool-ref-pins.test.ts index 9f8f7900eb..23acdfced5 100644 --- a/src/mcp/__tests__/tool-ref-pins.test.ts +++ b/src/mcp/__tests__/tool-ref-pins.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; +import type { AuditSnapshotCommandResult } from '@agent-device/contracts/capture'; import { createToolRefPinStore } from '../tool-ref-pins.ts'; function makeStore() { @@ -479,10 +480,14 @@ test('ref-pin store pins audit finding refs to the audit generation and keeps ot 'audit', { findings: [{ kind: 'missing-name', severity: 'warning', ref: '@e2', role: 'button' }], + guidance: { + 'missing-name': + 'Set an accessibility label (iOS accessibilityLabel, Android contentDescription, React Native accessibilityLabel).', + }, checks, limitations: [], refsGeneration: 9, - }, + } satisfies AuditSnapshotCommandResult, undefined, 'demo', ); @@ -508,7 +513,12 @@ test('ref-pin store leaves pins untouched for an audit without findings', () => ); pins.mergeCommandResult( 'audit', - { findings: [], checks: { snapshot: 'complete' }, limitations: [] }, + { + findings: [], + guidance: {}, + checks: { snapshot: 'complete' }, + limitations: [], + } satisfies AuditSnapshotCommandResult, undefined, 'demo', ); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 2aadeb1ce7..5ada17b9b8 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -435,7 +435,7 @@ agent-device audit - `missing-name`: a control with no accessible name. An identifier, a resource id, or a field value is not a name; a label, an Android content description, a placeholder, or a named child is. - `covered-control`: a control or row the snapshot already marks `[covered]`. Controls inside scroll containers under tab bars, navigation bars, toolbars, or bottom navigation bars are excluded. Scroll to reveal these controls. Controls under sheets, dialogs, and other overlays remain findings. -Controls are recognized by role (button, link, switch, checkbox, radio, slider, menu item, text and search fields). Generic clickable containers on Android are not evaluated. iOS visibility relies on the `hittable` fact. System bars, keyboard keys, and UIKit sheet grabbers are skipped. On iOS, controls inside navigation bars, toolbars, and tab bars skip target-size checks because the platform sets their size. These bar controls still receive missing-name checks. A system-generated accessible name, such as "add" or "remove" for an icon-only React Native Pressable on iOS 26, prevents a missing-name finding. The audit cannot determine whether the app supplied that name. When a control is the only child of a control with the same role, the child drops a finding only if the parent reports the same kind of finding. +Controls are recognized by role (button, link, switch, checkbox, radio, slider, menu item, text and search fields). Generic clickable containers on Android are not evaluated. iOS visibility relies on the `hittable` fact. System bars, keyboard keys, and UIKit sheet grabbers are skipped. On iOS, direct child controls of navigation bars, toolbars, and tab bars skip target-size checks because the platform sets their size. Controls inside nested content containers remain checked. These bar controls still receive missing-name checks. A system-generated accessible name, such as "add" or "remove" for an icon-only React Native Pressable on iOS 26, prevents a missing-name finding. The audit cannot determine whether the app supplied that name. When a control is the only child of a control with the same role, the child drops a finding only if the parent reports the same kind of finding. Like `find`, `audit` publishes only the refs it returns. The text prints each as `@eN~s`; press that pinned ref. A plain ref from before the audit stays refused after the frame expired, and an audit with no findings issues no refs. An audit with findings also ends the snapshot's complete frame, so take a new snapshot before you use refs that the audit did not return. Each finding shows its role and accessible label. An unnamed control shows its identifier when available. Guidance follows once per finding kind, with Apple HIG sizes on iOS and Material sizes on Android. `--json` returns `findings` (with `label` when available), `guidance` keyed by finding kind, `checks`, `limitations`, and `refsGeneration` (only with findings). `checks.snapshot` is `unknown` when the capture cannot report truncation, such as Appium page source. From 8b1de769bcd265f8b7c7393f057542ff4f8efa70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 20:38:42 +0100 Subject: [PATCH 04/20] fix(audit): reuse occlusion bar identity for size exemptions --- .../capture-kit/src/snapshot-occlusion.ts | 28 ++++--------------- .../snapshot/__tests__/snapshot-audit.test.ts | 19 +++++++++++++ .../src/snapshot/snapshot-audit.ts | 8 ++++-- packages/contracts/src/facades/snapshot.ts | 2 ++ packages/contracts/src/snapshot-text.ts | 27 ++++++++++++++++++ 5 files changed, 59 insertions(+), 25 deletions(-) diff --git a/packages/capture-kit/src/snapshot-occlusion.ts b/packages/capture-kit/src/snapshot-occlusion.ts index 822f352067..1641dd0f9b 100644 --- a/packages/capture-kit/src/snapshot-occlusion.ts +++ b/packages/capture-kit/src/snapshot-occlusion.ts @@ -8,7 +8,8 @@ import { areRectsApproximatelyEqual, normalizeRect } from '@agent-device/kernel/ import { containsPoint, rectContains } from '@agent-device/kernel/rect'; import { isScrollableNodeLike, - normalizeType, + isSnapshotBarNode, + snapshotNodeKindIncludesAny as nodeKindIncludesAny, isViewportRootNode, } from '@agent-device/contracts/snapshot'; @@ -25,7 +26,6 @@ const OVERLAY_KIND_FRAGMENTS = [ 'popover', 'menu', ]; -const VIEWPORT_CHROME_KIND_FRAGMENTS = ['tabbar', 'toolbar', 'navigationbar']; const FLOATING_BAR_CONTAINER_ROLE = '_UIFloatingBarContainerView'; /** * How far below the viewport top a full-viewport chrome container may start, before the 0.5pt rect @@ -380,7 +380,7 @@ function isSiblingSizedChromeContainer( node: RawSnapshotNode, neighbourhood: NeighbourhoodIndex, ): boolean { - if (!nodeKindIncludesAny(node, VIEWPORT_CHROME_KIND_FRAGMENTS)) return false; + if (!isSnapshotBarNode(node)) return false; if (typeof node.parentIndex !== 'number') return false; const rect = positiveRect(node.rect); if (!rect) return false; @@ -539,7 +539,7 @@ function isNavigationBarAround(bar: RawSnapshotNode, rect: Rect): boolean { /** A bar or a scroll container inside `rect`; a scroll indicator is not a container. */ function isEnclosedHostedRegion(rect: Rect, node: RawSnapshotNode): boolean { const isHostedRegion = - nodeKindIncludesAny(node, VIEWPORT_CHROME_KIND_FRAGMENTS) || + isSnapshotBarNode(node) || (isScrollableNodeLike(node) && !nodeKindIncludesAny(node, SCROLL_INDICATOR_KIND_FRAGMENTS)); const nodeRect = positiveRect(node.rect); return Boolean(isHostedRegion && nodeRect && rectContains(rect, nodeRect)); @@ -549,7 +549,7 @@ function isFullViewportChromeContainer( node: RawSnapshotNode, neighbourhood: NeighbourhoodIndex, ): boolean { - if (!nodeKindIncludesAny(node, VIEWPORT_CHROME_KIND_FRAGMENTS)) return false; + if (!isSnapshotBarNode(node)) return false; const rect = positiveRect(node.rect); if (!rect) return false; @@ -665,24 +665,6 @@ function isSemanticTouchNode(node: RawSnapshotNode): boolean { return nodeKindIncludesAny(node, SEMANTIC_TOUCH_KIND_FRAGMENTS); } -function nodeKindIncludesAny( - node: Pick, - fragments: readonly string[], -): boolean { - const normalized = normalizeNodeKind(node); - return fragments.some((fragment) => normalized.includes(fragment)); -} - -// Mutation-lane note: the `?? ''` fallback's exact replacement text is not -// observable through this module's public API — every caller only checks -// substring membership against a fixed, known fragment list, and no -// plausible filler text coincides with any of them, so no test can -// distinguish `''` from another non-matching filler here without reaching -// into this private function directly. -function normalizeNodeKind(node: Pick): string { - return [node.type, node.role, node.subrole].map((value) => normalizeType(value ?? '')).join(' '); -} - function areRelatedSnapshotNodes( left: RawSnapshotNode, right: RawSnapshotNode, diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts index ddce9cbdcf..8d14a218f6 100644 --- a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts @@ -465,3 +465,22 @@ test.each(['NavigationBar', 'Toolbar', 'TabBar', 'BottomNavigationView'])( } }, ); + +test.each([{ role: 'UITabBar' }, { subrole: 'UIToolbar' }, { subrole: 'AXToolbar' }])( + 'recognizes a direct bar item through its parent class %j', + (barClass) => { + const item = node(1, { + parentIndex: 0, + label: 'Save', + rect: { x: 0, y: 0, width: 36, height: 36 }, + }); + const parent = node(0, { type: 'Other', hittable: false, ...barClass }); + expect(audit([parent, item]).findings).toEqual([]); + expect(summarize(audit([parent, { ...item, label: undefined }]).findings)).toEqual([ + 'missing-name @e1', + ]); + expect( + summarize(audit([{ ...parent, role: 'UIView', subrole: undefined }, item]).findings), + ).toEqual(['small-target @e1']); + }, +); diff --git a/packages/capture-kit/src/snapshot/snapshot-audit.ts b/packages/capture-kit/src/snapshot/snapshot-audit.ts index ea605d652d..27399dc5c7 100644 --- a/packages/capture-kit/src/snapshot/snapshot-audit.ts +++ b/packages/capture-kit/src/snapshot/snapshot-audit.ts @@ -6,7 +6,11 @@ import type { AuditResult, AuditSizeUnit, } from '@agent-device/contracts/capture'; -import { isScrollableNodeLike, normalizeType } from '@agent-device/contracts/snapshot'; +import { + isScrollableNodeLike, + isSnapshotBarNode, + normalizeType, +} from '@agent-device/contracts/snapshot'; import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; import { formatRole, type SnapshotNode } from '@agent-device/kernel/snapshot'; import { @@ -137,7 +141,7 @@ function controlOwnership(node: SnapshotNode, scan: AuditScan): ControlOwnership if (scan.platform !== 'ios') return 'app'; if (node.role === '_UIGrabber' || node.subrole === '_UIGrabber') return 'system'; const parent = parentOf(node, scan); - return parent && isPersistentChromeKind(parent.type) ? 'platform-sized' : 'app'; + return parent && isSnapshotBarNode(parent) ? 'platform-sized' : 'app'; } function isActionable(node: SnapshotNode, ownership: ControlOwnership): boolean { diff --git a/packages/contracts/src/facades/snapshot.ts b/packages/contracts/src/facades/snapshot.ts index cb1922bc69..de1deb0371 100644 --- a/packages/contracts/src/facades/snapshot.ts +++ b/packages/contracts/src/facades/snapshot.ts @@ -45,4 +45,6 @@ export { isMeaningfulLabel, isMeaningfulSignal, normalizeType, + isSnapshotBarNode, + snapshotNodeKindIncludesAny, } from '../snapshot-text.ts'; diff --git a/packages/contracts/src/snapshot-text.ts b/packages/contracts/src/snapshot-text.ts index 446575ce0d..e277e40e50 100644 --- a/packages/contracts/src/snapshot-text.ts +++ b/packages/contracts/src/snapshot-text.ts @@ -14,6 +14,33 @@ export function normalizeType(type: string): string { return normalized; } +const SNAPSHOT_BAR_KIND_FRAGMENTS = ['tabbar', 'toolbar', 'navigationbar']; + +/** Reads bar identity from the captured type, role, and subrole. */ +export function isSnapshotBarNode( + node: Pick, +): boolean { + return snapshotNodeKindIncludesAny(node, SNAPSHOT_BAR_KIND_FRAGMENTS); +} + +export function snapshotNodeKindIncludesAny( + node: Pick, + fragments: readonly string[], +): boolean { + const normalized = normalizeNodeKind(node); + return fragments.some((fragment) => normalized.includes(fragment)); +} + +// Mutation-lane note: the `?? ''` fallback's exact replacement text is not +// observable through this module's public API — every caller only checks +// substring membership against a fixed, known fragment list, and no +// plausible filler text coincides with any of them, so no test can +// distinguish `''` from another non-matching filler here without reaching +// into this private function directly. +function normalizeNodeKind(node: Pick): string { + return [node.type, node.role, node.subrole].map((value) => normalizeType(value ?? '')).join(' '); +} + export function isFillableType(type: string, platform: Platform | PublicPlatform): boolean { const normalized = normalizeType(type); if (!normalized) return true; From 7cefe5423b0060edd8e3d5f616c775e15507a75e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 10:15:56 +0100 Subject: [PATCH 05/20] fix(audit): exclude scroll rows under bars through a typed persistentBar cover fact describeCoveringNode now sets persistentBar from the cover's type, role, and subrole, and publishes an unnamed bar of a generic type. The audit reads that fact instead of matching the display kind against a copied list. Co-Authored-By: Claude --- .../capture-kit/src/snapshot-occlusion.ts | 14 +- .../snapshot/__tests__/snapshot-audit.test.ts | 130 +++++++++++++----- .../src/snapshot/snapshot-audit.ts | 14 +- .../runtime/snapshot-unchanged.test.ts | 3 + .../capture/runtime/snapshot-unchanged.ts | 1 + .../runtime/target-visibility-stages.test.ts | 1 + packages/contracts/src/audit.ts | 1 + packages/kernel/src/snapshot.ts | 5 + website/docs/docs/commands.md | 2 +- website/docs/docs/snapshots.md | 5 +- 10 files changed, 124 insertions(+), 52 deletions(-) diff --git a/packages/capture-kit/src/snapshot-occlusion.ts b/packages/capture-kit/src/snapshot-occlusion.ts index 1641dd0f9b..831340e324 100644 --- a/packages/capture-kit/src/snapshot-occlusion.ts +++ b/packages/capture-kit/src/snapshot-occlusion.ts @@ -141,6 +141,7 @@ export function annotateCoveredSnapshotNodes( /** Presented roles that name a box rather than a surface a reader would recognize and dismiss. */ const ANONYMOUS_CONTAINER_KINDS = new Set(['element', 'group', 'other']); +const BOTTOM_NAVIGATION_KIND_FRAGMENTS = ['bottomnavigation']; /** * The bounded, non-actionable description of the cover the scan found, or nothing when the cover has @@ -152,22 +153,29 @@ const ANONYMOUS_CONTAINER_KINDS = new Set(['element', 'group', 'other']); * the surface to dismiss. Strip that label and the same node would say `the group at (...)`, which * points at a box nobody would recognize; an anonymous container of that shape therefore publishes * no cover, and the refusal keeps its generic recovery. A named kind (`dialog`, `tab-bar`, `sheet`) - * still describes itself with no label to lean on. + * still describes itself with no label to lean on. A persistent bar always publishes, because its + * `persistentBar` fact reaches consumers even when its type is generic and the bar is unnamed. */ function describeCoveringNode(node: CoveringNodeEvidence): SnapshotCoveringNode | null { const label = node.label?.trim(); const kind = formatRole(node.type ?? 'Element'); - if (!label && ANONYMOUS_CONTAINER_KINDS.has(kind)) return null; + const persistentBar = + isSnapshotBarNode(node) || nodeKindIncludesAny(node, BOTTOM_NAVIGATION_KIND_FRAGMENTS); + if (!label && !persistentBar && ANONYMOUS_CONTAINER_KINDS.has(kind)) return null; const rect = positiveRect(node.rect); return { kind, ...(label ? { label: label.slice(0, SNAPSHOT_COVERING_LABEL_MAX_LENGTH) } : {}), ...(rect ? { rect } : {}), + ...(persistentBar ? { persistentBar: true } : {}), }; } /** What a backend hit test observed on top of a node, described like a scanned cover. */ -export type CoveringNodeEvidence = Pick; +export type CoveringNodeEvidence = Pick< + RawSnapshotNode, + 'type' | 'role' | 'subrole' | 'label' | 'rect' +>; /** * Applies backend-owned covered decisions through the same publication annotation contract. A diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts index 8d14a218f6..9351fc30d8 100644 --- a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from 'vitest'; import type { AuditContext, AuditFinding } from '@agent-device/contracts/capture'; -import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot'; +import { annotateCoveredSnapshotNodes } from '../../snapshot-occlusion.ts'; import { auditSnapshot } from '../snapshot-audit.ts'; const IOS: AuditContext = { platform: 'ios', truncated: false }; @@ -334,40 +335,101 @@ test('publishes the captured label and identifier but omits a missing-name label expect(findings[2]).toMatchObject({ label: 'Pay' }); }); -test.each(['tab-bar', 'navigation-bar', 'toolbar', 'bottomnavigation', 'bottomnavigationview'])( - 'ignores %s over a scrolled row but retains it over a fixed control', - (kind) => { - const covered = { - interactionBlocked: 'covered', - hittable: false, - coveredBy: { kind, label: 'Not a chrome label' }, - } as const; - const findings = audit([ - node(0, { type: 'ScrollView', hittable: false }), - node(1, { type: 'Other', parentIndex: 0 }), - node(2, { parentIndex: 1, label: 'Scrolled row', ...covered }), - node(3, { label: 'Fixed control', ...covered }), - ]).findings; - expect(summarize(findings)).toEqual(['covered-control @e3']); - }, -); +type CoverPublication = { + name: string; + type: string; + role?: string; + subrole?: string; + label?: string; +}; + +/** + * Window > screen holding a scroll view (one row inside) and a fixed control, with the cover listed + * last so it draws on top. The row and the fixed control both sit under the cover. The occlusion scan + * produces each `coveredBy`, so the audit reads exactly what a capture publishes. + */ +function scannedCoverTree(cover: CoverPublication): SnapshotNode[] { + const raw: RawSnapshotNode[] = [ + { index: 0, depth: 0, type: 'Window', rect: { x: 0, y: 0, width: 402, height: 874 } }, + { + index: 1, + parentIndex: 0, + depth: 1, + type: 'Other', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 2, + parentIndex: 1, + depth: 2, + type: 'ScrollView', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 3, + parentIndex: 2, + depth: 3, + type: 'Button', + label: 'Scrolled row', + hittable: true, + rect: { x: 0, y: 800, width: 402, height: 60 }, + }, + { + index: 4, + parentIndex: 1, + depth: 2, + type: 'Button', + label: 'Fixed control', + hittable: true, + rect: { x: 300, y: 800, width: 60, height: 60 }, + }, + { + index: 5, + parentIndex: 1, + depth: 2, + type: cover.type, + ...(cover.role ? { role: cover.role } : {}), + ...(cover.subrole ? { subrole: cover.subrole } : {}), + ...(cover.label ? { label: cover.label } : {}), + hittable: true, + rect: { x: 0, y: 791, width: 402, height: 83 }, + }, + ]; + return annotateCoveredSnapshotNodes(raw).map((scanned) => ({ + ...scanned, + ref: `e${scanned.index}`, + })); +} -test.each(['sheet', 'dialog', 'banner', 'group', 'floating-view'])( - 'retains %s over a control inside a scroll container', - (kind) => { - const findings = audit([ - node(0, { type: 'ScrollView', hittable: false }), - node(1, { - parentIndex: 0, - label: 'Pay', - interactionBlocked: 'covered', - hittable: false, - coveredBy: { kind, label: 'Tab Bar' }, - }), - ]).findings; - expect(summarize(findings)).toEqual(['covered-control @e1']); +test.for([ + { name: 'type only', type: 'TabBar' }, + { name: 'navigation bar type', type: 'NavigationBar', label: 'Inbox' }, + { name: 'toolbar type', type: 'Toolbar' }, + { name: 'UIKit class in role beside a generic type', type: 'Other', role: 'UITabBar' }, + { name: 'subrole only', type: 'Other', subrole: 'AXToolbar' }, + { + name: 'Android bottom navigation', + type: 'com.google.android.material.bottomnavigation.BottomNavigationView', }, -); +])('ignores a scanned bar from the $name publication over a scrolled row only', (cover) => { + const nodes = scannedCoverTree(cover); + expect(nodes[3]).toMatchObject({ + interactionBlocked: 'covered', + coveredBy: { persistentBar: true }, + }); + expect(summarize(audit(nodes).findings)).toEqual(['covered-control @e4']); +}); + +test.for([ + { name: 'sheet', type: 'Sheet' }, + { name: 'dialog', type: 'Dialog', label: 'Tab Bar' }, + { name: 'labelled alert group', type: 'Alert', label: 'Confirm' }, +])('retains a scanned $name over a control inside a scroll container', (cover) => { + const nodes = scannedCoverTree(cover); + expect(nodes[3]).toMatchObject({ interactionBlocked: 'covered' }); + expect(nodes[3]?.coveredBy?.persistentBar).toBeUndefined(); + expect(summarize(audit(nodes).findings)).toEqual(['covered-control @e3', 'covered-control @e4']); +}); test.each([ [IOS, 'Apple HIG recommends 44×44pt.'], @@ -407,7 +469,7 @@ test('keeps a child under an overlay when its covered row is excluded as scroll type: 'Cell', parentIndex: 0, interactionBlocked: 'covered', - coveredBy: { kind: 'tab-bar' }, + coveredBy: { kind: 'tab-bar', persistentBar: true }, }), node(2, { parentIndex: 1, diff --git a/packages/capture-kit/src/snapshot/snapshot-audit.ts b/packages/capture-kit/src/snapshot/snapshot-audit.ts index 27399dc5c7..63ebad7a1f 100644 --- a/packages/capture-kit/src/snapshot/snapshot-audit.ts +++ b/packages/capture-kit/src/snapshot/snapshot-audit.ts @@ -311,20 +311,8 @@ function roundTenth(value: number): number { return Math.round(value * 10) / 10; } -const PERSISTENT_CHROME_KINDS = new Set([ - 'tabbar', - 'navigationbar', - 'toolbar', - 'bottomnavigation', - 'bottomnavigationview', -]); - -function isPersistentChromeKind(kind: string | undefined): boolean { - return PERSISTENT_CHROME_KINDS.has(normalizeType(kind ?? '').replaceAll('-', '')); -} - function isScrollableChromeOcclusion(node: SnapshotNode, scan: AuditScan): boolean { - if (!isPersistentChromeKind(node.coveredBy?.kind)) return false; + if (node.coveredBy?.persistentBar !== true) return false; const visited = new Set(); let ancestor = parentOf(node, scan); while (ancestor && !visited.has(ancestor.index)) { diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts index 5d1b1d304b..d937b6db39 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts @@ -241,6 +241,9 @@ test('unchanged metadata compares covering-node descriptions by value', () => { current.nodes[0]!.coveredBy = { ...cover, rect: { ...cover.rect, y: 700 } }; expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined(); + current.nodes[0]!.coveredBy = { ...cover, persistentBar: true }; + expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined(); + current.nodes[0]!.coveredBy = undefined; expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined(); }); diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts index b71f23dc82..5734828f65 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts @@ -169,6 +169,7 @@ function areCoveringNodesEquivalent( return ( previous.kind === current.kind && previous.label === current.label && + previous.persistentBar === current.persistentBar && areRectsEquivalent(previous.rect, current.rect) ); } diff --git a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts index fcd481d5ff..8f7bc5b348 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts @@ -175,6 +175,7 @@ test('runtime covered refusal names the covering element the occlusion scan foun kind: 'tab-bar', label: 'Media', rect: { x: 0, y: 760, width: 390, height: 84 }, + persistentBar: true, }); assert.doesNotMatch(String(details?.hint), /@e\d/); assert.match(String(details?.hint), /Covered by the tab-bar "Media" at \(0, 760 390x84\)/); diff --git a/packages/contracts/src/audit.ts b/packages/contracts/src/audit.ts index 80776be126..1ad5be8403 100644 --- a/packages/contracts/src/audit.ts +++ b/packages/contracts/src/audit.ts @@ -25,6 +25,7 @@ type AuditFindingBase = { role: string; /** Accessible label from the captured node. */ label?: string; + /** Accessibility identifier or resource id from the captured node. */ identifier?: string; }; diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 7b915c5b0f..8687de146a 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -269,6 +269,11 @@ export type SnapshotCoveringNode = { label?: string; /** The cover's positive rect as the scan measured it. */ rect?: Rect; + /** + * Set when the cover is persistent bar chrome (tab bar, toolbar, navigation bar, bottom + * navigation), read from the cover's type, role, and subrole. Absent for any other cover. + */ + persistentBar?: true; }; /** Upper bound for a covering-node label carried in diagnostics; longer names are clipped. */ diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 5ada17b9b8..14cdcc29d0 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -437,7 +437,7 @@ agent-device audit Controls are recognized by role (button, link, switch, checkbox, radio, slider, menu item, text and search fields). Generic clickable containers on Android are not evaluated. iOS visibility relies on the `hittable` fact. System bars, keyboard keys, and UIKit sheet grabbers are skipped. On iOS, direct child controls of navigation bars, toolbars, and tab bars skip target-size checks because the platform sets their size. Controls inside nested content containers remain checked. These bar controls still receive missing-name checks. A system-generated accessible name, such as "add" or "remove" for an icon-only React Native Pressable on iOS 26, prevents a missing-name finding. The audit cannot determine whether the app supplied that name. When a control is the only child of a control with the same role, the child drops a finding only if the parent reports the same kind of finding. -Like `find`, `audit` publishes only the refs it returns. The text prints each as `@eN~s`; press that pinned ref. A plain ref from before the audit stays refused after the frame expired, and an audit with no findings issues no refs. An audit with findings also ends the snapshot's complete frame, so take a new snapshot before you use refs that the audit did not return. Each finding shows its role and accessible label. An unnamed control shows its identifier when available. Guidance follows once per finding kind, with Apple HIG sizes on iOS and Material sizes on Android. `--json` returns `findings` (with `label` when available), `guidance` keyed by finding kind, `checks`, `limitations`, and `refsGeneration` (only with findings). `checks.snapshot` is `unknown` when the capture cannot report truncation, such as Appium page source. +Like `find`, `audit` publishes only the refs it returns. The text prints each as `@eN~s`; press that pinned ref. A plain ref from before the audit stays refused after the frame expired, and an audit with no findings issues no refs. An audit with findings also ends the snapshot's complete frame, so take a new snapshot before you use refs that the audit did not return. Each finding shows its role and accessible label. An unnamed control shows its identifier when available. Guidance follows once per finding kind, with Apple HIG sizes on iOS and Material sizes on Android. `--json` returns `findings` (with `label` and `identifier` when available), `guidance` keyed by finding kind, `checks`, `limitations`, and `refsGeneration` (only with findings). `checks.snapshot` is `unknown` when the capture cannot report truncation, such as Appium page source. ## Wait and alerts diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 3057e0db55..26145faee1 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -162,11 +162,14 @@ when it has one, and its `rect`: `coveredBy.kind` is the same presented role any other node carries, so a cover can be a generic container — an alert's container often arrives as `group` — and the label is what makes it nameable. +When the cover is a persistent bar (tab bar, toolbar, navigation bar, or bottom navigation), +`coveredBy` also carries `persistentBar: true`. The scan reads that fact from the cover's type, role, +and subrole, so a bar with a generic kind and no name still publishes `coveredBy`. A `click`/`press`/`fill`/`find ... focus` refused on that node repeats the same description in `error.details.coveredBy` and names it in `error.hint`. `coveredBy` is diagnosis only: it carries no ref, so the cover is not an actionable target. It is absent when nothing nameable covers — an -anonymous container with no accessible name, or a backend policy that reports only the boolean, such +anonymous container with no accessible name that is not a persistent bar, or a backend policy that reports only the boolean, such as an Android replacement surface — and those refusals say only that the target is covered. `role=` selectors (and `find role=...`) match the `kind` vocabulary: a node whose `kind` is `text` From a5928902931f6475aba49e484e8b9942cf2b66be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 10:23:18 +0100 Subject: [PATCH 06/20] test(find): expect the persistentBar fact on a scanned tab-bar cover Co-Authored-By: Claude --- .../src/execution/interaction/internal/__tests__/find.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/daemon/src/execution/interaction/internal/__tests__/find.test.ts b/packages/daemon/src/execution/interaction/internal/__tests__/find.test.ts index 83beed2136..d26719f703 100644 --- a/packages/daemon/src/execution/interaction/internal/__tests__/find.test.ts +++ b/packages/daemon/src/execution/interaction/internal/__tests__/find.test.ts @@ -556,6 +556,7 @@ test('handleFindCommands focus rejects covered matches before dispatching coordi kind: 'tab-bar', label: 'Media', rect: { x: 0, y: 760, width: 390, height: 84 }, + persistentBar: true, }); expect(response.error.details?.hint).toContain('Covered by the tab-bar "Media"'); expect(response.error.details?.hint).not.toMatch(/@e\d/); From 670a93ffe03342449fa1b9e23eb3d579ed6babc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 12:20:22 +0100 Subject: [PATCH 07/20] fix(occlusion): name an unnamed generic-typed bar cover by its bar kind The shared bar predicate now returns the matched bar kind. describeCoveringNode publishes that kind when the type presents as an anonymous container, and the covered refusal hint for a persistent bar tells the agent to scroll, not dismiss. Co-Authored-By: Claude --- .../capture-kit/src/snapshot-occlusion.ts | 25 ++++++++----- .../runtime/__tests__/test-utils/index.ts | 36 +++++++++++++++++++ .../runtime/target-visibility-stages.test.ts | 26 ++++++++++++++ packages/contracts/src/facades/snapshot.ts | 1 + packages/contracts/src/snapshot-text.ts | 19 ++++++++-- .../selectors/src/interaction-error.test.ts | 18 ++++++++++ packages/selectors/src/interaction-error.ts | 7 ++++ website/docs/docs/snapshots.md | 6 ++-- 8 files changed, 126 insertions(+), 12 deletions(-) diff --git a/packages/capture-kit/src/snapshot-occlusion.ts b/packages/capture-kit/src/snapshot-occlusion.ts index 831340e324..b6188c5c59 100644 --- a/packages/capture-kit/src/snapshot-occlusion.ts +++ b/packages/capture-kit/src/snapshot-occlusion.ts @@ -9,6 +9,7 @@ import { containsPoint, rectContains } from '@agent-device/kernel/rect'; import { isScrollableNodeLike, isSnapshotBarNode, + snapshotBarKind, snapshotNodeKindIncludesAny as nodeKindIncludesAny, isViewportRootNode, } from '@agent-device/contracts/snapshot'; @@ -153,24 +154,32 @@ const BOTTOM_NAVIGATION_KIND_FRAGMENTS = ['bottomnavigation']; * the surface to dismiss. Strip that label and the same node would say `the group at (...)`, which * points at a box nobody would recognize; an anonymous container of that shape therefore publishes * no cover, and the refusal keeps its generic recovery. A named kind (`dialog`, `tab-bar`, `sheet`) - * still describes itself with no label to lean on. A persistent bar always publishes, because its - * `persistentBar` fact reaches consumers even when its type is generic and the bar is unnamed. + * still describes itself with no label to lean on. A persistent bar always publishes, and when its + * type presents as an anonymous container, its kind comes from the bar identity its role or subrole + * carries (`Other` with role `UITabBar` publishes `tab-bar`), so the cover names a recognizable bar. */ function describeCoveringNode(node: CoveringNodeEvidence): SnapshotCoveringNode | null { const label = node.label?.trim(); - const kind = formatRole(node.type ?? 'Element'); - const persistentBar = - isSnapshotBarNode(node) || nodeKindIncludesAny(node, BOTTOM_NAVIGATION_KIND_FRAGMENTS); - if (!label && !persistentBar && ANONYMOUS_CONTAINER_KINDS.has(kind)) return null; + const typeKind = formatRole(node.type ?? 'Element'); + const barKind = persistentBarKind(node); + const anonymous = ANONYMOUS_CONTAINER_KINDS.has(typeKind); + if (!label && !barKind && anonymous) return null; const rect = positiveRect(node.rect); return { - kind, + kind: barKind && anonymous ? barKind : typeKind, ...(label ? { label: label.slice(0, SNAPSHOT_COVERING_LABEL_MAX_LENGTH) } : {}), ...(rect ? { rect } : {}), - ...(persistentBar ? { persistentBar: true } : {}), + ...(barKind ? { persistentBar: true } : {}), }; } +function persistentBarKind(node: CoveringNodeEvidence): string | undefined { + return ( + snapshotBarKind(node) ?? + (nodeKindIncludesAny(node, BOTTOM_NAVIGATION_KIND_FRAGMENTS) ? 'bottom-navigation' : undefined) + ); +} + /** What a backend hit test observed on top of a node, described like a scanned cover. */ export type CoveringNodeEvidence = Pick< RawSnapshotNode, diff --git a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts index b646a045bc..e0ee2f0c9f 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts @@ -220,6 +220,42 @@ export function coveredByScanSnapshot(): SnapshotState { ); } +/** + * An unnamed bar published as type `Other` with its UIKit class in `role`, run through the real + * annotation pass: the scan reads the bar identity from the role, not the generic type. + */ +export function coveredByAnonymousTabBarScanSnapshot(): SnapshotState { + return makeSnapshotState( + annotateCoveredSnapshotNodes([ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Save draft', + rect: { x: 16, y: 800, width: 140, height: 44 }, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'Other', + role: 'UITabBar', + rect: { x: 0, y: 791, width: 402, height: 83 }, + hittable: true, + }, + ]), + ); +} + export function duplicateCoveredLabelSnapshot(): SnapshotState { return makeSnapshotState([ { diff --git a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts index 8f7bc5b348..8ad0eeb2db 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts @@ -5,6 +5,7 @@ import { throwIfOffscreenInteractionTarget } from './target-visibility-stages.ts import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import type { Point } from '@agent-device/kernel/snapshot'; import { + coveredByAnonymousTabBarScanSnapshot, coveredByScanSnapshot, coveredByTabBarSnapshot, createInteractionDevice, @@ -185,6 +186,31 @@ test('runtime covered refusal names the covering element the occlusion scan foun assert.deepEqual(calls, []); }); +test('runtime covered refusal names an unnamed generic-typed bar and never says dismiss', async () => { + const device = createInteractionDevice(coveredByAnonymousTabBarScanSnapshot(), { + tap: async () => {}, + }); + + await assert.rejects( + () => device.interactions.click(ref('@e2'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_covered'); + assert.deepEqual(details?.coveredBy, { + kind: 'tab-bar', + rect: { x: 0, y: 791, width: 402, height: 83 }, + persistentBar: true, + }); + assert.match( + String(details?.hint), + /^Covered by the tab-bar at \(0, 791 402x83\)\. Scroll the target clear of it/, + ); + assert.doesNotMatch(String(details?.hint), /dismiss|close|the other/i); + return true; + }, + ); +}); + test('runtime covered refusal keeps the generic hint when the cover is a policy boolean', async () => { // `coveredByTabBarSnapshot` hand-annotates the covered boolean the way the Android // replacement-surface lane publishes it: no cover identity, so nothing to name. diff --git a/packages/contracts/src/facades/snapshot.ts b/packages/contracts/src/facades/snapshot.ts index de1deb0371..888d173ae8 100644 --- a/packages/contracts/src/facades/snapshot.ts +++ b/packages/contracts/src/facades/snapshot.ts @@ -46,5 +46,6 @@ export { isMeaningfulSignal, normalizeType, isSnapshotBarNode, + snapshotBarKind, snapshotNodeKindIncludesAny, } from '../snapshot-text.ts'; diff --git a/packages/contracts/src/snapshot-text.ts b/packages/contracts/src/snapshot-text.ts index e277e40e50..44ddb0942c 100644 --- a/packages/contracts/src/snapshot-text.ts +++ b/packages/contracts/src/snapshot-text.ts @@ -14,13 +14,28 @@ export function normalizeType(type: string): string { return normalized; } -const SNAPSHOT_BAR_KIND_FRAGMENTS = ['tabbar', 'toolbar', 'navigationbar']; +/** Presented bar kind, keyed by the normalized kind fragment that identifies it. */ +const SNAPSHOT_BAR_KINDS = [ + ['tabbar', 'tab-bar'], + ['toolbar', 'toolbar'], + ['navigationbar', 'navigation-bar'], +] as const; + +export type SnapshotBarKind = (typeof SNAPSHOT_BAR_KINDS)[number][1]; + +/** Reads the bar kind from the captured type, role, and subrole, or nothing for a non-bar node. */ +export function snapshotBarKind( + node: Pick, +): SnapshotBarKind | undefined { + const normalized = normalizeNodeKind(node); + return SNAPSHOT_BAR_KINDS.find(([fragment]) => normalized.includes(fragment))?.[1]; +} /** Reads bar identity from the captured type, role, and subrole. */ export function isSnapshotBarNode( node: Pick, ): boolean { - return snapshotNodeKindIncludesAny(node, SNAPSHOT_BAR_KIND_FRAGMENTS); + return snapshotBarKind(node) !== undefined; } export function snapshotNodeKindIncludesAny( diff --git a/packages/selectors/src/interaction-error.test.ts b/packages/selectors/src/interaction-error.test.ts index da42a6c1f1..08abb31947 100644 --- a/packages/selectors/src/interaction-error.test.ts +++ b/packages/selectors/src/interaction-error.test.ts @@ -84,6 +84,24 @@ test('the hint falls back to the bare kind when the label sanitizes to nothing', assert.match(hint, /^Covered by the tab-bar at \(0, 760 390x84\)/); }); +test('the hint for a persistent bar names scrolling, not dismissing', () => { + const hint = String( + coveredInteractionDetails({ + ref: 'e2', + interactionBlocked: 'covered', + coveredBy: { + kind: 'navigation-bar', + rect: { x: 0, y: 0, width: 390, height: 96 }, + persistentBar: true, + }, + }).hint, + ); + assert.equal( + hint, + 'Covered by the navigation-bar at (0, 0 390x96). Scroll the target clear of it, or inspect with snapshot/screenshot before retrying.', + ); +}); + test('the hint names a cover the capture published without a rect', () => { const hint = String( coveredInteractionDetails({ diff --git a/packages/selectors/src/interaction-error.ts b/packages/selectors/src/interaction-error.ts index 0e8f3950ab..82185a0429 100644 --- a/packages/selectors/src/interaction-error.ts +++ b/packages/selectors/src/interaction-error.ts @@ -88,6 +88,13 @@ function coveredInteractionHint(coveredBy: SnapshotCoveringNode | undefined): st if (!coveredBy) { return 'Use a different visible target, scroll it clear of the overlay, or inspect with snapshot/screenshot before retrying.'; } + // Persistent bar chrome cannot be dismissed, so its hint names only the recoveries that work. + if (coveredBy.persistentBar) { + return ( + `Covered by ${describeCoverForHint(coveredBy)}. Scroll the target clear of it, ` + + 'or inspect with snapshot/screenshot before retrying.' + ); + } return ( `Covered by ${describeCoverForHint(coveredBy)}. Dismiss or close that element, ` + 'scroll the target clear of it, or inspect with snapshot/screenshot before retrying.' diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 26145faee1..bf4dcc04ee 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -164,10 +164,12 @@ when it has one, and its `rect`: container — an alert's container often arrives as `group` — and the label is what makes it nameable. When the cover is a persistent bar (tab bar, toolbar, navigation bar, or bottom navigation), `coveredBy` also carries `persistentBar: true`. The scan reads that fact from the cover's type, role, -and subrole, so a bar with a generic kind and no name still publishes `coveredBy`. +and subrole, so a bar with a generic type and no name still publishes `coveredBy`. Its `kind` is then +the bar kind (`tab-bar`, `toolbar`, `navigation-bar`, or `bottom-navigation`). A `click`/`press`/`fill`/`find ... focus` refused on that node repeats the same description in -`error.details.coveredBy` and names it in `error.hint`. `coveredBy` is diagnosis only: it carries no +`error.details.coveredBy` and names it in `error.hint`. A persistent bar cannot be dismissed, so its +hint tells you to scroll the target clear of it. `coveredBy` is diagnosis only: it carries no ref, so the cover is not an actionable target. It is absent when nothing nameable covers — an anonymous container with no accessible name that is not a persistent bar, or a backend policy that reports only the boolean, such as an Android replacement surface — and those refusals say only that the target is covered. From 94484e531c435eb58bf72ffba3b0ad2ec3def38f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 13:28:40 +0100 Subject: [PATCH 08/20] fix(contracts): keep the bar kind type private to its module Co-Authored-By: Claude --- packages/contracts/src/snapshot-text.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/contracts/src/snapshot-text.ts b/packages/contracts/src/snapshot-text.ts index 44ddb0942c..628134b24f 100644 --- a/packages/contracts/src/snapshot-text.ts +++ b/packages/contracts/src/snapshot-text.ts @@ -21,7 +21,7 @@ const SNAPSHOT_BAR_KINDS = [ ['navigationbar', 'navigation-bar'], ] as const; -export type SnapshotBarKind = (typeof SNAPSHOT_BAR_KINDS)[number][1]; +type SnapshotBarKind = (typeof SNAPSHOT_BAR_KINDS)[number][1]; /** Reads the bar kind from the captured type, role, and subrole, or nothing for a non-bar node. */ export function snapshotBarKind( From 8d521f139b6bcc77b7d2c728c72c439ee5adde17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 15:45:41 +0100 Subject: [PATCH 09/20] fix(occlusion): suggest a scroll past a persistent bar only for scrollable targets The occlusion scan now marks a cover with targetInScrollableContent when the covered node has a scrollable ancestor (findNearestScrollableAncestor). The persistent-bar refusal hint suggests a scroll only with that fact, and points a fixed target to a different visible target or snapshot inspection. The audit reads the same fact instead of walking ancestors itself. Co-Authored-By: Claude --- .../capture-kit/src/snapshot-occlusion.ts | 11 ++++++- .../snapshot/__tests__/snapshot-audit.test.ts | 6 ++-- .../src/snapshot/snapshot-audit.ts | 27 +++++----------- .../runtime/snapshot-unchanged.test.ts | 3 ++ .../capture/runtime/snapshot-unchanged.ts | 1 + .../runtime/__tests__/test-utils/index.ts | 29 +++++++++++++---- .../runtime/target-visibility-stages.test.ts | 32 ++++++++++++++++++- packages/kernel/src/snapshot.ts | 5 +++ .../selectors/src/interaction-error.test.ts | 26 +++++++-------- packages/selectors/src/interaction-error.ts | 7 ++-- website/docs/docs/snapshots.md | 7 ++-- 11 files changed, 108 insertions(+), 46 deletions(-) diff --git a/packages/capture-kit/src/snapshot-occlusion.ts b/packages/capture-kit/src/snapshot-occlusion.ts index b6188c5c59..21aefb427b 100644 --- a/packages/capture-kit/src/snapshot-occlusion.ts +++ b/packages/capture-kit/src/snapshot-occlusion.ts @@ -7,6 +7,7 @@ import { import { areRectsApproximatelyEqual, normalizeRect } from '@agent-device/kernel/rect-center'; import { containsPoint, rectContains } from '@agent-device/kernel/rect'; import { + findNearestScrollableAncestor, isScrollableNodeLike, isSnapshotBarNode, snapshotBarKind, @@ -210,14 +211,22 @@ function annotateCoveredPositions( coveredPositions: ReadonlyArray<{ position: number; coveredBy?: SnapshotCoveringNode }>, ): RawSnapshotNode[] { if (coveredPositions.length === 0) return nodes; + const byIndex = new Map(nodes.map((node) => [node.index, node])); const annotated = [...nodes]; for (const { position, coveredBy } of coveredPositions) { const node = nodes[position]!; + const inScrollableContent = findNearestScrollableAncestor(node, byIndex) !== null; annotated[position] = { ...node, hittable: false, interactionBlocked: 'covered' as const, - ...(coveredBy ? { coveredBy } : {}), + ...(coveredBy + ? { + coveredBy: inScrollableContent + ? { ...coveredBy, targetInScrollableContent: true as const } + : coveredBy, + } + : {}), presentationHints: mergeCoveredHint(node.presentationHints), }; } diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts index 9351fc30d8..a957160e65 100644 --- a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts @@ -415,8 +415,10 @@ test.for([ const nodes = scannedCoverTree(cover); expect(nodes[3]).toMatchObject({ interactionBlocked: 'covered', - coveredBy: { persistentBar: true }, + coveredBy: { persistentBar: true, targetInScrollableContent: true }, }); + expect(nodes[4]?.coveredBy).toMatchObject({ persistentBar: true }); + expect(nodes[4]?.coveredBy?.targetInScrollableContent).toBeUndefined(); expect(summarize(audit(nodes).findings)).toEqual(['covered-control @e4']); }); @@ -469,7 +471,7 @@ test('keeps a child under an overlay when its covered row is excluded as scroll type: 'Cell', parentIndex: 0, interactionBlocked: 'covered', - coveredBy: { kind: 'tab-bar', persistentBar: true }, + coveredBy: { kind: 'tab-bar', persistentBar: true, targetInScrollableContent: true }, }), node(2, { parentIndex: 1, diff --git a/packages/capture-kit/src/snapshot/snapshot-audit.ts b/packages/capture-kit/src/snapshot/snapshot-audit.ts index 63ebad7a1f..252b3d51b9 100644 --- a/packages/capture-kit/src/snapshot/snapshot-audit.ts +++ b/packages/capture-kit/src/snapshot/snapshot-audit.ts @@ -6,11 +6,7 @@ import type { AuditResult, AuditSizeUnit, } from '@agent-device/contracts/capture'; -import { - isScrollableNodeLike, - isSnapshotBarNode, - normalizeType, -} from '@agent-device/contracts/snapshot'; +import { isSnapshotBarNode, normalizeType } from '@agent-device/contracts/snapshot'; import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; import { formatRole, type SnapshotNode } from '@agent-device/kernel/snapshot'; import { @@ -245,7 +241,7 @@ function missingNameFinding(node: SnapshotNode, role: string): AuditFinding { function coveredFinding(node: SnapshotNode, scan: AuditScan): AuditFinding | undefined { const role = coveredRole(node); if (node.interactionBlocked !== 'covered' || role === undefined) return undefined; - if (isScrollableChromeOcclusion(node, scan)) return undefined; + if (isScrollableChromeOcclusion(node)) return undefined; if (isOnlyChildOfReportedCover(node, scan)) return undefined; return { ...findingBase(node, role), kind: 'covered-control', severity: 'observation' }; } @@ -262,9 +258,7 @@ function isRow(node: SnapshotNode): boolean { function isOnlyChildOfReportedCover(node: SnapshotNode, scan: AuditScan): boolean { const parent = parentOf(node, scan); if (!parent || parent.ref === undefined || coveredRole(parent) === undefined) return false; - return ( - !isScrollableChromeOcclusion(parent, scan) && scan.children.get(parent.index)?.length === 1 - ); + return !isScrollableChromeOcclusion(parent) && scan.children.get(parent.index)?.length === 1; } function findingBase(node: SnapshotNode, role: string) { @@ -311,16 +305,11 @@ function roundTenth(value: number): number { return Math.round(value * 10) / 10; } -function isScrollableChromeOcclusion(node: SnapshotNode, scan: AuditScan): boolean { - if (node.coveredBy?.persistentBar !== true) return false; - const visited = new Set(); - let ancestor = parentOf(node, scan); - while (ancestor && !visited.has(ancestor.index)) { - if (isScrollableNodeLike(ancestor)) return true; - visited.add(ancestor.index); - ancestor = parentOf(ancestor, scan); - } - return false; +/** Persistent bar chrome over scrollable content is a scroll position, not a covered control. */ +function isScrollableChromeOcclusion(node: SnapshotNode): boolean { + return ( + node.coveredBy?.persistentBar === true && node.coveredBy.targetInScrollableContent === true + ); } function auditGuidance( diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts index d937b6db39..154094ea18 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts @@ -244,6 +244,9 @@ test('unchanged metadata compares covering-node descriptions by value', () => { current.nodes[0]!.coveredBy = { ...cover, persistentBar: true }; expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined(); + current.nodes[0]!.coveredBy = { ...cover, targetInScrollableContent: true }; + expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined(); + current.nodes[0]!.coveredBy = undefined; expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined(); }); diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts index 5734828f65..8636bbf5af 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts @@ -170,6 +170,7 @@ function areCoveringNodesEquivalent( previous.kind === current.kind && previous.label === current.label && previous.persistentBar === current.persistentBar && + previous.targetInScrollableContent === current.targetInScrollableContent && areRectsEquivalent(previous.rect, current.rect) ); } diff --git a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts index e0ee2f0c9f..0e9aa3f207 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts @@ -2,6 +2,7 @@ import type { AgentDeviceBackend, BackendSnapshotResult } from '@agent-device/co import { annotateCoveredSnapshotNodes } from '@agent-device/capture-kit/snapshot-occlusion'; import { createLocalArtifactAdapter } from '@agent-device/host-kit/local-artifact-adapter'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { createAgentDevice } from '../../../../../runtime.ts'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; import type { CommandSessionStore } from '@agent-device/contracts/runtime-contract'; @@ -222,9 +223,24 @@ export function coveredByScanSnapshot(): SnapshotState { /** * An unnamed bar published as type `Other` with its UIKit class in `role`, run through the real - * annotation pass: the scan reads the bar identity from the role, not the generic type. + * annotation pass: the scan reads the bar identity from the role, not the generic type. The Save draft + * button is fixed beside the bar by default; `scrollable` puts it inside a scroll view instead. */ -export function coveredByAnonymousTabBarScanSnapshot(): SnapshotState { +export function coveredByAnonymousTabBarScanSnapshot( + options: { scrollable?: boolean } = {}, +): SnapshotState { + const scroller: RawSnapshotNode[] = options.scrollable + ? [ + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'ScrollView', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + ] + : []; + const buttonIndex = scroller.length + 1; return makeSnapshotState( annotateCoveredSnapshotNodes([ { @@ -234,17 +250,18 @@ export function coveredByAnonymousTabBarScanSnapshot(): SnapshotState { label: 'Example', rect: { x: 0, y: 0, width: 402, height: 874 }, }, + ...scroller, { - index: 1, - depth: 1, - parentIndex: 0, + index: buttonIndex, + depth: scroller.length + 1, + parentIndex: scroller.length > 0 ? 1 : 0, type: 'Button', label: 'Save draft', rect: { x: 16, y: 800, width: 140, height: 44 }, hittable: true, }, { - index: 2, + index: buttonIndex + 1, depth: 1, parentIndex: 0, type: 'Other', diff --git a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts index 8ad0eeb2db..b1f6b846d4 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts @@ -201,11 +201,41 @@ test('runtime covered refusal names an unnamed generic-typed bar and never says rect: { x: 0, y: 791, width: 402, height: 83 }, persistentBar: true, }); + assert.match( + String(details?.hint), + /^Covered by the tab-bar at \(0, 791 402x83\)\. Use a different visible target/, + ); + // The fixed button has no scrollable ancestor, so no scroll can move it clear of the bar. + assert.doesNotMatch(String(details?.hint), /dismiss|close|scroll|the other/i); + return true; + }, + ); +}); + +test('runtime covered refusal suggests a scroll for a scrollable target under a scanned bar', async () => { + const device = createInteractionDevice( + coveredByAnonymousTabBarScanSnapshot({ scrollable: true }), + { + tap: async () => {}, + }, + ); + + await assert.rejects( + () => device.interactions.click(ref('@e3'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_covered'); + assert.deepEqual(details?.coveredBy, { + kind: 'tab-bar', + rect: { x: 0, y: 791, width: 402, height: 83 }, + persistentBar: true, + targetInScrollableContent: true, + }); assert.match( String(details?.hint), /^Covered by the tab-bar at \(0, 791 402x83\)\. Scroll the target clear of it/, ); - assert.doesNotMatch(String(details?.hint), /dismiss|close|the other/i); + assert.doesNotMatch(String(details?.hint), /dismiss|close/i); return true; }, ); diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 8687de146a..5ae2b78585 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -274,6 +274,11 @@ export type SnapshotCoveringNode = { * navigation), read from the cover's type, role, and subrole. Absent for any other cover. */ persistentBar?: true; + /** + * Set when the covered node sits inside scrollable content, so a scroll can move it out from under + * the cover. Absent for a fixed node. + */ + targetInScrollableContent?: true; }; /** Upper bound for a covering-node label carried in diagnostics; longer names are clipped. */ diff --git a/packages/selectors/src/interaction-error.test.ts b/packages/selectors/src/interaction-error.test.ts index 08abb31947..fd9e5f1273 100644 --- a/packages/selectors/src/interaction-error.test.ts +++ b/packages/selectors/src/interaction-error.test.ts @@ -84,22 +84,22 @@ test('the hint falls back to the bare kind when the label sanitizes to nothing', assert.match(hint, /^Covered by the tab-bar at \(0, 760 390x84\)/); }); -test('the hint for a persistent bar names scrolling, not dismissing', () => { - const hint = String( - coveredInteractionDetails({ - ref: 'e2', - interactionBlocked: 'covered', - coveredBy: { - kind: 'navigation-bar', - rect: { x: 0, y: 0, width: 390, height: 96 }, - persistentBar: true, - }, - }).hint, - ); +test('the hint for a persistent bar suggests a scroll only for a target in scrollable content', () => { + const bar = { + kind: 'navigation-bar', + rect: { x: 0, y: 0, width: 390, height: 96 }, + persistentBar: true, + } as const; + const hint = (coveredBy: typeof bar & { targetInScrollableContent?: true }) => + String(coveredInteractionDetails({ ref: 'e2', interactionBlocked: 'covered', coveredBy }).hint); assert.equal( - hint, + hint({ ...bar, targetInScrollableContent: true }), 'Covered by the navigation-bar at (0, 0 390x96). Scroll the target clear of it, or inspect with snapshot/screenshot before retrying.', ); + assert.equal( + hint(bar), + 'Covered by the navigation-bar at (0, 0 390x96). Use a different visible target, or inspect with snapshot/screenshot before retrying.', + ); }); test('the hint names a cover the capture published without a rect', () => { diff --git a/packages/selectors/src/interaction-error.ts b/packages/selectors/src/interaction-error.ts index 82185a0429..a4bebb234e 100644 --- a/packages/selectors/src/interaction-error.ts +++ b/packages/selectors/src/interaction-error.ts @@ -88,10 +88,13 @@ function coveredInteractionHint(coveredBy: SnapshotCoveringNode | undefined): st if (!coveredBy) { return 'Use a different visible target, scroll it clear of the overlay, or inspect with snapshot/screenshot before retrying.'; } - // Persistent bar chrome cannot be dismissed, so its hint names only the recoveries that work. + // Persistent bar chrome cannot be dismissed, and a scroll moves only a target in scrollable content. if (coveredBy.persistentBar) { + const recovery = coveredBy.targetInScrollableContent + ? 'Scroll the target clear of it' + : 'Use a different visible target'; return ( - `Covered by ${describeCoverForHint(coveredBy)}. Scroll the target clear of it, ` + + `Covered by ${describeCoverForHint(coveredBy)}. ${recovery}, ` + 'or inspect with snapshot/screenshot before retrying.' ); } diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index bf4dcc04ee..3e078e4523 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -168,8 +168,11 @@ and subrole, so a bar with a generic type and no name still publishes `coveredBy the bar kind (`tab-bar`, `toolbar`, `navigation-bar`, or `bottom-navigation`). A `click`/`press`/`fill`/`find ... focus` refused on that node repeats the same description in -`error.details.coveredBy` and names it in `error.hint`. A persistent bar cannot be dismissed, so its -hint tells you to scroll the target clear of it. `coveredBy` is diagnosis only: it carries no +`error.details.coveredBy` and names it in `error.hint`. When the covered node sits inside scrollable +content, `coveredBy` also carries `targetInScrollableContent: true`. A persistent bar cannot be +dismissed. Its hint tells you to scroll the target clear of it only when the target is in scrollable +content. For a fixed target, the hint tells you to use a different visible target or to inspect with +snapshot or screenshot. `coveredBy` is diagnosis only: it carries no ref, so the cover is not an actionable target. It is absent when nothing nameable covers — an anonymous container with no accessible name that is not a persistent bar, or a backend policy that reports only the boolean, such as an Android replacement surface — and those refusals say only that the target is covered. From 5ba9a2397e74d5825ab71374f28c0f351a7a9be0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 16:00:39 +0100 Subject: [PATCH 10/20] fix(audit): covered-control guidance dismisses only dismissable covers --- .../capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts | 2 +- packages/capture-kit/src/snapshot/snapshot-audit.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts index a957160e65..2bbf103b6a 100644 --- a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts @@ -451,7 +451,7 @@ test.each([ 'missing-name': 'Set an accessibility label (iOS accessibilityLabel, Android contentDescription, React Native accessibilityLabel).', 'covered-control': - 'Move the control out from under the overlay, or dismiss the overlay before you use it.', + 'Move the control out from under the cover. If the cover can be dismissed, such as a sheet or dialog, dismiss it before you use the control.', }); expect( audit([node(1, { label: 'Large', rect: { x: 0, y: 0, width: 100, height: 100 } })], context) diff --git a/packages/capture-kit/src/snapshot/snapshot-audit.ts b/packages/capture-kit/src/snapshot/snapshot-audit.ts index 252b3d51b9..61caf3771e 100644 --- a/packages/capture-kit/src/snapshot/snapshot-audit.ts +++ b/packages/capture-kit/src/snapshot/snapshot-audit.ts @@ -323,7 +323,7 @@ function auditGuidance( 'missing-name': 'Set an accessibility label (iOS accessibilityLabel, Android contentDescription, React Native accessibilityLabel).', 'covered-control': - 'Move the control out from under the overlay, or dismiss the overlay before you use it.', + 'Move the control out from under the cover. If the cover can be dismissed, such as a sheet or dialog, dismiss it before you use the control.', }; return Object.fromEntries(findings.map(({ kind }) => [kind, sentences[kind]])); } From 66aadf548ab818e3e2c79876979b26c67befe80c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 17:08:43 +0100 Subject: [PATCH 11/20] fix(help,contracts): align manual-qa covered recovery with the error hint; own full bar identity in contracts - cli-help-workflows.ts: the manual-qa recovery for target_covered no longer tells the reader to dismiss or scroll clear any cover. It now points at the production hint's recovery: dismiss only dismissible covers (sheet, dialog, alert); under a persistent bar use a different visible target or inspect with snapshot/screenshot, scrolling past the bar only when coveredBy.targetInScrollableContent is true. cli-help-workflows.test.ts pins the new wording and the absence of the old. - contracts/snapshot-text.ts: SNAPSHOT_BAR_KINDS gains bottomnavigation -> bottom-navigation, so snapshotBarKind/isSnapshotBarNode own the full bar identity. capture-kit/snapshot-occlusion.ts drops its separate BOTTOM_NAVIGATION_KIND_FRAGMENTS match and selects the three existing kinds (tab-bar, toolbar, navigation-bar) for the viewport-chrome exemptions via a local VIEWPORT_CHROME_BAR_KINDS set, so an Android BottomNavigationView still covers content instead of being exempted. New tests pin the bar identity in contracts and the bottom-navigation occlusion behavior in capture-kit. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .../src/snapshot-occlusion.test.ts | 31 ++++++++++++++++ .../capture-kit/src/snapshot-occlusion.ts | 36 +++++++++++------- packages/contracts/src/snapshot-text.test.ts | 37 ++++++++++++++++++- packages/contracts/src/snapshot-text.ts | 11 ++++-- .../schema/cli-help-workflows.test.ts | 14 +++++++ src/commands/schema/cli-help-workflows.ts | 2 +- 6 files changed, 113 insertions(+), 18 deletions(-) diff --git a/packages/capture-kit/src/snapshot-occlusion.test.ts b/packages/capture-kit/src/snapshot-occlusion.test.ts index 69a0589a72..95a698e448 100644 --- a/packages/capture-kit/src/snapshot-occlusion.test.ts +++ b/packages/capture-kit/src/snapshot-occlusion.test.ts @@ -707,6 +707,37 @@ test('a full-viewport toolbar container does not cover its application siblings' assert.equal(annotated[1]?.interactionBlocked, undefined); }); +test('a full-viewport bottom navigation bar is not viewport chrome: it still covers and publishes as a persistent bar', () => { + // Bar identity (including bottom navigation) is owned by contracts; the viewport exemption is + // this scan's own choice of the three chrome kinds, so a viewport-sized BottomNavigationView + // keeps covering the content behind it instead of being exempted as a chrome container. + const application: RawSnapshotNode = { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }; + const target: RawSnapshotNode = { + index: 1, + type: 'button', + role: 'button', + parentIndex: 0, + label: 'Room', + rect: { x: 20, y: 100, width: 120, height: 50 }, + }; + const bottomNavigation: RawSnapshotNode = { + index: 2, + type: 'com.google.android.material.bottomnavigation.BottomNavigationView', + parentIndex: 0, + rect: { x: 0, y: 0, width: 400, height: 800 }, + }; + + const annotated = annotateCoveredSnapshotNodes([application, target, bottomNavigation]); + + assert.equal(annotated[1]?.interactionBlocked, 'covered'); + assert.equal(annotated[1]?.coveredBy?.persistentBar, true); + assert.match(annotated[1]?.coveredBy?.kind ?? '', /bottomnavigation/); +}); + test.for([ // #3883: on iOS 27 an app's 2pt top bar pushes the floating-bar container to y=2. { diff --git a/packages/capture-kit/src/snapshot-occlusion.ts b/packages/capture-kit/src/snapshot-occlusion.ts index 21aefb427b..aa12bd8e6a 100644 --- a/packages/capture-kit/src/snapshot-occlusion.ts +++ b/packages/capture-kit/src/snapshot-occlusion.ts @@ -9,7 +9,6 @@ import { containsPoint, rectContains } from '@agent-device/kernel/rect'; import { findNearestScrollableAncestor, isScrollableNodeLike, - isSnapshotBarNode, snapshotBarKind, snapshotNodeKindIncludesAny as nodeKindIncludesAny, isViewportRootNode, @@ -143,7 +142,25 @@ export function annotateCoveredSnapshotNodes( /** Presented roles that name a box rather than a surface a reader would recognize and dismiss. */ const ANONYMOUS_CONTAINER_KINDS = new Set(['element', 'group', 'other']); -const BOTTOM_NAVIGATION_KIND_FRAGMENTS = ['bottomnavigation']; + +/** + * The bar kinds this scan exempts as viewport chrome. Bar identity itself — including bottom + * navigation — is owned by `snapshotBarKind` in contracts; the choice of which kinds get a + * viewport exemption stays here, and bottom navigation is not one of them: an Android + * BottomNavigationView keeps covering the content behind it. + */ +const VIEWPORT_CHROME_BAR_KINDS: ReadonlySet = new Set([ + 'tab-bar', + 'toolbar', + 'navigation-bar', +]); + +function isViewportChromeBarNode( + node: Pick, +): boolean { + const kind = snapshotBarKind(node); + return kind !== undefined && VIEWPORT_CHROME_BAR_KINDS.has(kind); +} /** * The bounded, non-actionable description of the cover the scan found, or nothing when the cover has @@ -162,7 +179,7 @@ const BOTTOM_NAVIGATION_KIND_FRAGMENTS = ['bottomnavigation']; function describeCoveringNode(node: CoveringNodeEvidence): SnapshotCoveringNode | null { const label = node.label?.trim(); const typeKind = formatRole(node.type ?? 'Element'); - const barKind = persistentBarKind(node); + const barKind = snapshotBarKind(node); const anonymous = ANONYMOUS_CONTAINER_KINDS.has(typeKind); if (!label && !barKind && anonymous) return null; const rect = positiveRect(node.rect); @@ -174,13 +191,6 @@ function describeCoveringNode(node: CoveringNodeEvidence): SnapshotCoveringNode }; } -function persistentBarKind(node: CoveringNodeEvidence): string | undefined { - return ( - snapshotBarKind(node) ?? - (nodeKindIncludesAny(node, BOTTOM_NAVIGATION_KIND_FRAGMENTS) ? 'bottom-navigation' : undefined) - ); -} - /** What a backend hit test observed on top of a node, described like a scanned cover. */ export type CoveringNodeEvidence = Pick< RawSnapshotNode, @@ -406,7 +416,7 @@ function isSiblingSizedChromeContainer( node: RawSnapshotNode, neighbourhood: NeighbourhoodIndex, ): boolean { - if (!isSnapshotBarNode(node)) return false; + if (!isViewportChromeBarNode(node)) return false; if (typeof node.parentIndex !== 'number') return false; const rect = positiveRect(node.rect); if (!rect) return false; @@ -565,7 +575,7 @@ function isNavigationBarAround(bar: RawSnapshotNode, rect: Rect): boolean { /** A bar or a scroll container inside `rect`; a scroll indicator is not a container. */ function isEnclosedHostedRegion(rect: Rect, node: RawSnapshotNode): boolean { const isHostedRegion = - isSnapshotBarNode(node) || + isViewportChromeBarNode(node) || (isScrollableNodeLike(node) && !nodeKindIncludesAny(node, SCROLL_INDICATOR_KIND_FRAGMENTS)); const nodeRect = positiveRect(node.rect); return Boolean(isHostedRegion && nodeRect && rectContains(rect, nodeRect)); @@ -575,7 +585,7 @@ function isFullViewportChromeContainer( node: RawSnapshotNode, neighbourhood: NeighbourhoodIndex, ): boolean { - if (!isSnapshotBarNode(node)) return false; + if (!isViewportChromeBarNode(node)) return false; const rect = positiveRect(node.rect); if (!rect) return false; diff --git a/packages/contracts/src/snapshot-text.test.ts b/packages/contracts/src/snapshot-text.test.ts index 1370f5e92b..741cf2bf10 100644 --- a/packages/contracts/src/snapshot-text.test.ts +++ b/packages/contracts/src/snapshot-text.test.ts @@ -1,6 +1,11 @@ import { readFileSync } from 'node:fs'; import { expect, test } from 'vitest'; -import { androidFieldValue, isFillableType } from './snapshot-text.ts'; +import { + androidFieldValue, + isFillableType, + isSnapshotBarNode, + snapshotBarKind, +} from './snapshot-text.ts'; const fieldValues = JSON.parse( readFileSync( @@ -25,3 +30,33 @@ test('a web text input is fillable by its ARIA role, a button is not', () => { expect(isFillableType('searchbox', 'web')).toBe(true); expect(isFillableType('button', 'web')).toBe(false); }); + +// The full bar identity lives here, bottom navigation included, so consumers that publish a +// persistent-bar fact read one vocabulary instead of matching platform role strings themselves. +test.each([ + { name: 'an iOS tab bar type', node: { type: 'TabBar' }, kind: 'tab-bar' }, + { + name: 'a UIKit class in role beside a generic type', + node: { type: 'Other', role: 'UINavigationBar' }, + kind: 'navigation-bar', + }, + { + name: 'a toolbar in subrole only', + node: { type: 'Other', subrole: 'AXToolbar' }, + kind: 'toolbar', + }, + { + name: 'an Android bottom navigation view', + node: { type: 'com.google.android.material.bottomnavigation.BottomNavigationView' }, + kind: 'bottom-navigation', + }, +])('snapshotBarKind reads $name', ({ node, kind }) => { + expect(snapshotBarKind(node)).toBe(kind); + expect(isSnapshotBarNode(node)).toBe(true); +}); + +test('a non-bar node reads no bar kind', () => { + const node = { type: 'Button', role: 'UIButton' }; + expect(snapshotBarKind(node)).toBeUndefined(); + expect(isSnapshotBarNode(node)).toBe(false); +}); diff --git a/packages/contracts/src/snapshot-text.ts b/packages/contracts/src/snapshot-text.ts index 628134b24f..c263894b47 100644 --- a/packages/contracts/src/snapshot-text.ts +++ b/packages/contracts/src/snapshot-text.ts @@ -14,16 +14,21 @@ export function normalizeType(type: string): string { return normalized; } -/** Presented bar kind, keyed by the normalized kind fragment that identifies it. */ +/** + * Presented bar kind, keyed by the normalized kind fragment that identifies it. This module owns + * the full bar identity, including Android bottom navigation, so every consumer reads one + * vocabulary; which kinds a scan treats as viewport chrome is the consumer's own choice. + */ const SNAPSHOT_BAR_KINDS = [ ['tabbar', 'tab-bar'], ['toolbar', 'toolbar'], ['navigationbar', 'navigation-bar'], + ['bottomnavigation', 'bottom-navigation'], ] as const; type SnapshotBarKind = (typeof SNAPSHOT_BAR_KINDS)[number][1]; -/** Reads the bar kind from the captured type, role, and subrole, or nothing for a non-bar node. */ +/** Reads the bar kind, including bottom navigation, from the captured type, role, and subrole, or nothing for a non-bar node. */ export function snapshotBarKind( node: Pick, ): SnapshotBarKind | undefined { @@ -31,7 +36,7 @@ export function snapshotBarKind( return SNAPSHOT_BAR_KINDS.find(([fragment]) => normalized.includes(fragment))?.[1]; } -/** Reads bar identity from the captured type, role, and subrole. */ +/** Reads bar identity, including bottom navigation, from the captured type, role, and subrole. */ export function isSnapshotBarNode( node: Pick, ): boolean { diff --git a/src/commands/schema/cli-help-workflows.test.ts b/src/commands/schema/cli-help-workflows.test.ts index 613ef7a2b5..a796683acd 100644 --- a/src/commands/schema/cli-help-workflows.test.ts +++ b/src/commands/schema/cli-help-workflows.test.ts @@ -66,6 +66,20 @@ test('usageForCommand resolves manual QA help topic', async () => { assert.match(help, /wait_target_absent: readable capture, no match/); assert.match(help, /wait_target_present: absence timeout/); assert.match(help, /wait_capture_stalled: no readable capture by deadline/); + assert.match(help, /Target covered by other visible UI: refused with target_covered/); + assert.match( + help, + /dismiss or close the cover only when it is dismissible \(sheet, dialog, alert\)/, + ); + assert.match( + help, + /under a persistent bar \(tab bar, toolbar, navigation bar, bottom navigation\) use a different visible target or inspect with snapshot\/screenshot/, + ); + assert.match( + help, + /scrolling past the bar only when coveredBy\.targetInScrollableContent is true/, + ); + assert.doesNotMatch(help, /Dismiss\/close it or scroll the target clear/); }); test('usageForCommand resolves validate help topic', async () => { diff --git a/src/commands/schema/cli-help-workflows.ts b/src/commands/schema/cli-help-workflows.ts index 81f2fdbd75..979c8ac7dd 100644 --- a/src/commands/schema/cli-help-workflows.ts +++ b/src/commands/schema/cli-help-workflows.ts @@ -53,7 +53,7 @@ Recovery: Network/typeahead result missing: wait text "Expected result" or wait . A target that should be gone/disappear: use wait absent ; wait exists ... is rejected in favor of the plain selector wait. Keyboard visible over the next target: a tap point under a measured keyboard frame returns tap_keyboard_occludes_target. Press a visible Hide Keyboard key or the app's dismissal control. In apps that support drag dismissal, use gesture pan from inside the scrollable content. Use keyboard enter only to send Return. Missing keyboard nodes never prove absence. A raw coordinate behind the keys taps anyway and reports a warning. - Target covered by other visible UI: refused with target_covered. When the occlusion scan identified the covering element, error.details.coveredBy and the hint name it (role, label, rect) so you know what to dismiss; the cover is diagnosis only -- never target its ref. Dismiss/close it or scroll the target clear, then retry. + Target covered by other visible UI: refused with target_covered. When the occlusion scan identified the covering element, error.details.coveredBy and the hint name it (role, label, rect); the cover is diagnosis only -- never target its ref. Follow the hint's recovery: dismiss or close the cover only when it is dismissible (sheet, dialog, alert); under a persistent bar (tab bar, toolbar, navigation bar, bottom navigation) use a different visible target or inspect with snapshot/screenshot, scrolling past the bar only when coveredBy.targetInScrollableContent is true; then retry. Sparse or recovered accessibility snapshot: use screenshot as visual truth, leave the bad screen if needed, then retry snapshot -i. Non-hittable success hint: verify with the settled diff or snapshot; retarget by a better ref/selector if the UI did not change. From acd8238a228cc2989c188445758e6901dd9f63c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 17:47:12 +0100 Subject: [PATCH 12/20] fix(contracts): resolve a BottomNavigationBar name to bottom-navigation --- packages/contracts/src/snapshot-text.test.ts | 5 +++++ packages/contracts/src/snapshot-text.ts | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/contracts/src/snapshot-text.test.ts b/packages/contracts/src/snapshot-text.test.ts index 741cf2bf10..54f652b5d5 100644 --- a/packages/contracts/src/snapshot-text.test.ts +++ b/packages/contracts/src/snapshot-text.test.ts @@ -50,6 +50,11 @@ test.each([ node: { type: 'com.google.android.material.bottomnavigation.BottomNavigationView' }, kind: 'bottom-navigation', }, + { + name: 'a bottom navigation bar named in role only', + node: { type: 'Other', role: 'BottomNavigationBar' }, + kind: 'bottom-navigation', + }, ])('snapshotBarKind reads $name', ({ node, kind }) => { expect(snapshotBarKind(node)).toBe(kind); expect(isSnapshotBarNode(node)).toBe(true); diff --git a/packages/contracts/src/snapshot-text.ts b/packages/contracts/src/snapshot-text.ts index c263894b47..38e7873075 100644 --- a/packages/contracts/src/snapshot-text.ts +++ b/packages/contracts/src/snapshot-text.ts @@ -20,10 +20,11 @@ export function normalizeType(type: string): string { * vocabulary; which kinds a scan treats as viewport chrome is the consumer's own choice. */ const SNAPSHOT_BAR_KINDS = [ + // First, so a BottomNavigationBar name does not resolve to navigation-bar. + ['bottomnavigation', 'bottom-navigation'], ['tabbar', 'tab-bar'], ['toolbar', 'toolbar'], ['navigationbar', 'navigation-bar'], - ['bottomnavigation', 'bottom-navigation'], ] as const; type SnapshotBarKind = (typeof SNAPSHOT_BAR_KINDS)[number][1]; From f070be284c39df164f79be888b190a302710dd2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 18:34:23 +0100 Subject: [PATCH 13/20] fix(audit): limit target size exemptions to native iOS bars --- .../src/snapshot/__tests__/snapshot-audit.test.ts | 15 +++++++++++++++ .../capture-kit/src/snapshot/snapshot-audit.ts | 4 ++-- packages/contracts/src/facades/snapshot.ts | 2 +- packages/contracts/src/snapshot-text.test.ts | 6 +++--- packages/contracts/src/snapshot-text.ts | 11 ++++++++--- 5 files changed, 29 insertions(+), 9 deletions(-) diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts index 2bbf103b6a..061fa71391 100644 --- a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts @@ -530,6 +530,21 @@ test.each(['NavigationBar', 'Toolbar', 'TabBar', 'BottomNavigationView'])( }, ); +test.each([{ type: 'HomeToolbarView' }, { type: 'Other', role: 'BottomTabBar' }])( + 'checks the size of controls in the custom iOS bar %j', + (container) => { + const result = audit([ + node(0, { ...container, hittable: false }), + node(1, { + parentIndex: 0, + label: 'Save', + rect: { x: 0, y: 0, width: 24, height: 24 }, + }), + ]); + expect(summarize(result.findings)).toEqual(['small-target @e1']); + }, +); + test.each([{ role: 'UITabBar' }, { subrole: 'UIToolbar' }, { subrole: 'AXToolbar' }])( 'recognizes a direct bar item through its parent class %j', (barClass) => { diff --git a/packages/capture-kit/src/snapshot/snapshot-audit.ts b/packages/capture-kit/src/snapshot/snapshot-audit.ts index 61caf3771e..cf354659d4 100644 --- a/packages/capture-kit/src/snapshot/snapshot-audit.ts +++ b/packages/capture-kit/src/snapshot/snapshot-audit.ts @@ -6,7 +6,7 @@ import type { AuditResult, AuditSizeUnit, } from '@agent-device/contracts/capture'; -import { isSnapshotBarNode, normalizeType } from '@agent-device/contracts/snapshot'; +import { isNativeIosSnapshotBarNode, normalizeType } from '@agent-device/contracts/snapshot'; import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; import { formatRole, type SnapshotNode } from '@agent-device/kernel/snapshot'; import { @@ -137,7 +137,7 @@ function controlOwnership(node: SnapshotNode, scan: AuditScan): ControlOwnership if (scan.platform !== 'ios') return 'app'; if (node.role === '_UIGrabber' || node.subrole === '_UIGrabber') return 'system'; const parent = parentOf(node, scan); - return parent && isSnapshotBarNode(parent) ? 'platform-sized' : 'app'; + return parent && isNativeIosSnapshotBarNode(parent) ? 'platform-sized' : 'app'; } function isActionable(node: SnapshotNode, ownership: ControlOwnership): boolean { diff --git a/packages/contracts/src/facades/snapshot.ts b/packages/contracts/src/facades/snapshot.ts index 888d173ae8..ce0fc1b87a 100644 --- a/packages/contracts/src/facades/snapshot.ts +++ b/packages/contracts/src/facades/snapshot.ts @@ -45,7 +45,7 @@ export { isMeaningfulLabel, isMeaningfulSignal, normalizeType, - isSnapshotBarNode, + isNativeIosSnapshotBarNode, snapshotBarKind, snapshotNodeKindIncludesAny, } from '../snapshot-text.ts'; diff --git a/packages/contracts/src/snapshot-text.test.ts b/packages/contracts/src/snapshot-text.test.ts index 54f652b5d5..e613c4e033 100644 --- a/packages/contracts/src/snapshot-text.test.ts +++ b/packages/contracts/src/snapshot-text.test.ts @@ -3,7 +3,7 @@ import { expect, test } from 'vitest'; import { androidFieldValue, isFillableType, - isSnapshotBarNode, + isNativeIosSnapshotBarNode, snapshotBarKind, } from './snapshot-text.ts'; @@ -57,11 +57,11 @@ test.each([ }, ])('snapshotBarKind reads $name', ({ node, kind }) => { expect(snapshotBarKind(node)).toBe(kind); - expect(isSnapshotBarNode(node)).toBe(true); + expect(isNativeIosSnapshotBarNode(node)).toBe(kind !== 'bottom-navigation'); }); test('a non-bar node reads no bar kind', () => { const node = { type: 'Button', role: 'UIButton' }; expect(snapshotBarKind(node)).toBeUndefined(); - expect(isSnapshotBarNode(node)).toBe(false); + expect(isNativeIosSnapshotBarNode(node)).toBe(false); }); diff --git a/packages/contracts/src/snapshot-text.ts b/packages/contracts/src/snapshot-text.ts index 38e7873075..2bb4e1efa5 100644 --- a/packages/contracts/src/snapshot-text.ts +++ b/packages/contracts/src/snapshot-text.ts @@ -37,11 +37,16 @@ export function snapshotBarKind( return SNAPSHOT_BAR_KINDS.find(([fragment]) => normalized.includes(fragment))?.[1]; } -/** Reads bar identity, including bottom navigation, from the captured type, role, and subrole. */ -export function isSnapshotBarNode( +/** Identifies native iOS bars whose direct items use platform sizing. */ +export function isNativeIosSnapshotBarNode( node: Pick, ): boolean { - return snapshotBarKind(node) !== undefined; + return [node.type, node.role, node.subrole].some((value) => { + const name = normalizeType(value ?? '').replace(/^ui/, ''); + return SNAPSHOT_BAR_KINDS.some( + ([fragment, kind]) => kind !== 'bottom-navigation' && name === fragment, + ); + }); } export function snapshotNodeKindIncludesAny( From 7d61884d69e7ad018166a61536c41b575b5d6216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 18:37:08 +0100 Subject: [PATCH 14/20] fix(occlusion): require viewport evidence for scroll recovery --- ...snapshot-occlusion-scroll-recovery.test.ts | 66 +++++++++++++++++++ .../capture-kit/src/snapshot-occlusion.ts | 53 +++++++++++++-- .../snapshot/__tests__/snapshot-audit.test.ts | 49 +++++++++++++- .../src/snapshot/snapshot-audit.ts | 6 +- .../runtime/snapshot-unchanged.test.ts | 2 +- .../capture/runtime/snapshot-unchanged.ts | 2 +- .../runtime/__tests__/test-utils/index.ts | 35 ++++++---- .../runtime/target-visibility-stages.test.ts | 34 +++++++++- packages/kernel/src/snapshot.ts | 7 +- .../selectors/src/interaction-error.test.ts | 6 +- packages/selectors/src/interaction-error.ts | 5 +- .../schema/cli-help-workflows.test.ts | 5 +- src/commands/schema/cli-help-workflows.ts | 2 +- website/docs/docs/snapshots.md | 9 ++- 14 files changed, 229 insertions(+), 52 deletions(-) create mode 100644 packages/capture-kit/src/snapshot-occlusion-scroll-recovery.test.ts diff --git a/packages/capture-kit/src/snapshot-occlusion-scroll-recovery.test.ts b/packages/capture-kit/src/snapshot-occlusion-scroll-recovery.test.ts new file mode 100644 index 0000000000..cedad1e2a7 --- /dev/null +++ b/packages/capture-kit/src/snapshot-occlusion-scroll-recovery.test.ts @@ -0,0 +1,66 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { annotateCoveredSnapshotNodes } from './snapshot-occlusion.ts'; + +test.each([ + { + name: 'a horizontal viewport wholly contained by a bottom bar', + scrollerType: 'HorizontalScrollView', + scrollerRect: { x: 0, y: 725, width: 400, height: 65 }, + barRect: { x: 0, y: 725, width: 400, height: 75 }, + expected: undefined, + }, + { + name: 'a viewport contained by a bar after clipping to the screen', + scrollerType: 'ScrollView', + scrollerRect: { x: 0, y: 730, width: 400, height: 150 }, + barRect: { x: 0, y: 725, width: 400, height: 75 }, + expected: undefined, + }, + { + name: 'a horizontal tap path entirely behind a full-width bar', + scrollerType: 'HorizontalScrollView', + scrollerRect: { x: 0, y: 700, width: 400, height: 100 }, + barRect: { x: 0, y: 725, width: 400, height: 75 }, + expected: undefined, + }, + { + name: 'a horizontal tap path that extends past the bar', + scrollerType: 'HorizontalScrollView', + scrollerRect: { x: 0, y: 700, width: 400, height: 100 }, + barRect: { x: 300, y: 700, width: 100, height: 100 }, + expected: true, + }, + { + name: 'a vertical tap path that extends above the bar', + scrollerType: 'VerticalScrollView', + scrollerRect: { x: 0, y: 700, width: 400, height: 100 }, + barRect: { x: 0, y: 725, width: 400, height: 75 }, + expected: true, + }, +])('reports scroll recovery for $name', ({ scrollerType, scrollerRect, barRect, expected }) => { + const nodes: RawSnapshotNode[] = [ + { index: 0, type: 'Window', rect: { x: 0, y: 0, width: 400, height: 800 } }, + { index: 1, parentIndex: 0, type: `android.widget.${scrollerType}`, rect: scrollerRect }, + { + index: 2, + parentIndex: 1, + type: 'Button', + hittable: true, + label: 'Buy', + rect: { x: 320, y: 740, width: 60, height: 40 }, + }, + { + index: 3, + parentIndex: 0, + type: 'com.google.android.material.bottomnavigation.BottomNavigationView', + rect: barRect, + }, + ]; + + const target = annotateCoveredSnapshotNodes(nodes)[2]; + assert.equal(target?.interactionBlocked, 'covered'); + assert.equal(target?.coveredBy?.persistentBar, true); + assert.equal(target?.coveredBy?.scrollCanReveal, expected); +}); diff --git a/packages/capture-kit/src/snapshot-occlusion.ts b/packages/capture-kit/src/snapshot-occlusion.ts index aa12bd8e6a..6dcdc5f46d 100644 --- a/packages/capture-kit/src/snapshot-occlusion.ts +++ b/packages/capture-kit/src/snapshot-occlusion.ts @@ -5,9 +5,11 @@ import { SNAPSHOT_COVERING_LABEL_MAX_LENGTH, } from '@agent-device/kernel/snapshot'; import { areRectsApproximatelyEqual, normalizeRect } from '@agent-device/kernel/rect-center'; -import { containsPoint, rectContains } from '@agent-device/kernel/rect'; +import { containsPoint, intersectRects, rectContains } from '@agent-device/kernel/rect'; import { + collectViewportRects, findNearestScrollableAncestor, + resolveViewportRect, isScrollableNodeLike, snapshotBarKind, snapshotNodeKindIncludesAny as nodeKindIncludesAny, @@ -223,19 +225,25 @@ function annotateCoveredPositions( if (coveredPositions.length === 0) return nodes; const byIndex = new Map(nodes.map((node) => [node.index, node])); const annotated = [...nodes]; + let viewportRects: Rect[] | undefined; for (const { position, coveredBy } of coveredPositions) { const node = nodes[position]!; - const inScrollableContent = findNearestScrollableAncestor(node, byIndex) !== null; + const scrollCanReveal = + coveredBy?.persistentBar && + coveredBy.rect && + canScrollingRevealTarget( + node, + coveredBy.rect, + byIndex, + nodes, + (viewportRects ??= collectViewportRects(nodes)), + ); annotated[position] = { ...node, hittable: false, interactionBlocked: 'covered' as const, ...(coveredBy - ? { - coveredBy: inScrollableContent - ? { ...coveredBy, targetInScrollableContent: true as const } - : coveredBy, - } + ? { coveredBy: scrollCanReveal ? { ...coveredBy, scrollCanReveal: true } : coveredBy } : {}), presentationHints: mergeCoveredHint(node.presentationHints), }; @@ -243,6 +251,37 @@ function annotateCoveredPositions( return annotated; } +function canScrollingRevealTarget( + target: RawSnapshotNode, + cover: Rect, + byIndex: ReadonlyMap, + nodes: RawSnapshotNode[], + viewportRects: readonly Rect[], +): boolean { + const scroller = findNearestScrollableAncestor(target, byIndex); + const scrollRect = positiveRect(scroller?.rect); + const targetRect = positiveRect(target.rect); + if (!scroller || !scrollRect || !targetRect) return false; + + const screen = + viewportRects.length > 0 ? resolveViewportRect(nodes, targetRect, viewportRects) : null; + const viewport = screen ? intersectRects(scrollRect, screen) : scrollRect; + if (!viewport) return false; + + const horizontal = nodeKindIncludesAny(scroller, ['horizontal']); + const vertical = nodeKindIncludesAny(scroller, ['vertical']); + if (horizontal === vertical) return !rectContains(cover, viewport); + + // The tap point's perpendicular coordinate stays fixed in a one-axis scroller. + const { x, y } = centerOfRect(targetRect); + if (horizontal) { + if (y < viewport.y || y > viewport.y + viewport.height) return false; + return !rectContains(cover, { ...viewport, y, height: 0 }); + } + if (x < viewport.x || x > viewport.x + viewport.width) return false; + return !rectContains(cover, { ...viewport, x, width: 0 }); +} + export function isSnapshotNodeInteractionBlocked( node: Pick, ): boolean { diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts index 061fa71391..50c339bf75 100644 --- a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts @@ -415,13 +415,56 @@ test.for([ const nodes = scannedCoverTree(cover); expect(nodes[3]).toMatchObject({ interactionBlocked: 'covered', - coveredBy: { persistentBar: true, targetInScrollableContent: true }, + coveredBy: { persistentBar: true, scrollCanReveal: true }, }); expect(nodes[4]?.coveredBy).toMatchObject({ persistentBar: true }); - expect(nodes[4]?.coveredBy?.targetInScrollableContent).toBeUndefined(); + expect(nodes[4]?.coveredBy?.scrollCanReveal).toBeUndefined(); expect(summarize(audit(nodes).findings)).toEqual(['covered-control @e4']); }); +test('keeps the covered-control finding for a button in a horizontal scroller wholly behind a bottom bar', () => { + // The bar contains the scroller's whole usable viewport (400x65 inside the 400x75 bar), so no + // scroll can reveal the button: the audit keeps the finding instead of excluding it as scroll + // chrome. The occlusion scan produces the `coveredBy`, as in a real capture. + const raw: RawSnapshotNode[] = [ + { index: 0, depth: 0, type: 'Window', rect: { x: 0, y: 0, width: 400, height: 800 } }, + { + index: 1, + parentIndex: 0, + depth: 1, + type: 'android.widget.HorizontalScrollView', + rect: { x: 0, y: 725, width: 400, height: 65 }, + }, + { + index: 2, + parentIndex: 1, + depth: 2, + type: 'Button', + label: 'Buy now', + hittable: true, + rect: { x: 20, y: 735, width: 120, height: 50 }, + }, + { + index: 3, + parentIndex: 0, + depth: 1, + type: 'com.google.android.material.bottomnavigation.BottomNavigationView', + hittable: true, + rect: { x: 0, y: 725, width: 400, height: 75 }, + }, + ]; + const nodes = annotateCoveredSnapshotNodes(raw).map((scanned) => ({ + ...scanned, + ref: `e${scanned.index}`, + })); + expect(nodes[2]).toMatchObject({ + interactionBlocked: 'covered', + coveredBy: { persistentBar: true }, + }); + expect(nodes[2]?.coveredBy?.scrollCanReveal).toBeUndefined(); + expect(summarize(audit(nodes, ANDROID).findings)).toEqual(['covered-control @e2']); +}); + test.for([ { name: 'sheet', type: 'Sheet' }, { name: 'dialog', type: 'Dialog', label: 'Tab Bar' }, @@ -471,7 +514,7 @@ test('keeps a child under an overlay when its covered row is excluded as scroll type: 'Cell', parentIndex: 0, interactionBlocked: 'covered', - coveredBy: { kind: 'tab-bar', persistentBar: true, targetInScrollableContent: true }, + coveredBy: { kind: 'tab-bar', persistentBar: true, scrollCanReveal: true }, }), node(2, { parentIndex: 1, diff --git a/packages/capture-kit/src/snapshot/snapshot-audit.ts b/packages/capture-kit/src/snapshot/snapshot-audit.ts index cf354659d4..ff621f4980 100644 --- a/packages/capture-kit/src/snapshot/snapshot-audit.ts +++ b/packages/capture-kit/src/snapshot/snapshot-audit.ts @@ -305,11 +305,9 @@ function roundTenth(value: number): number { return Math.round(value * 10) / 10; } -/** Persistent bar chrome over scrollable content is a scroll position, not a covered control. */ +/** Persistent bar chrome over a target a scroll can reveal is a scroll position, not a covered control. */ function isScrollableChromeOcclusion(node: SnapshotNode): boolean { - return ( - node.coveredBy?.persistentBar === true && node.coveredBy.targetInScrollableContent === true - ); + return node.coveredBy?.persistentBar === true && node.coveredBy.scrollCanReveal === true; } function auditGuidance( diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts index 154094ea18..0f71fd8e72 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts @@ -244,7 +244,7 @@ test('unchanged metadata compares covering-node descriptions by value', () => { current.nodes[0]!.coveredBy = { ...cover, persistentBar: true }; expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined(); - current.nodes[0]!.coveredBy = { ...cover, targetInScrollableContent: true }; + current.nodes[0]!.coveredBy = { ...cover, scrollCanReveal: true }; expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined(); current.nodes[0]!.coveredBy = undefined; diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts index 8636bbf5af..2b10757c36 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts @@ -170,7 +170,7 @@ function areCoveringNodesEquivalent( previous.kind === current.kind && previous.label === current.label && previous.persistentBar === current.persistentBar && - previous.targetInScrollableContent === current.targetInScrollableContent && + previous.scrollCanReveal === current.scrollCanReveal && areRectsEquivalent(previous.rect, current.rect) ); } diff --git a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts index 0e9aa3f207..2380d7e096 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts @@ -224,22 +224,27 @@ export function coveredByScanSnapshot(): SnapshotState { /** * An unnamed bar published as type `Other` with its UIKit class in `role`, run through the real * annotation pass: the scan reads the bar identity from the role, not the generic type. The Save draft - * button is fixed beside the bar by default; `scrollable` puts it inside a scroll view instead. + * button is fixed beside the bar by default; `scrollable` puts it inside a scroll view instead; + * `horizontal` puts it inside a horizontal scroller whose viewport the bar contains entirely, so no + * scroll can reveal it. */ export function coveredByAnonymousTabBarScanSnapshot( - options: { scrollable?: boolean } = {}, + options: { scrollable?: boolean; horizontal?: boolean } = {}, ): SnapshotState { - const scroller: RawSnapshotNode[] = options.scrollable - ? [ - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'ScrollView', - rect: { x: 0, y: 0, width: 402, height: 874 }, - }, - ] - : []; + const scroller: RawSnapshotNode[] = + options.scrollable || options.horizontal + ? [ + { + index: 1, + depth: 1, + parentIndex: 0, + type: options.horizontal ? 'android.widget.HorizontalScrollView' : 'ScrollView', + rect: options.horizontal + ? { x: 0, y: 800, width: 402, height: 65 } + : { x: 0, y: 0, width: 402, height: 874 }, + }, + ] + : []; const buttonIndex = scroller.length + 1; return makeSnapshotState( annotateCoveredSnapshotNodes([ @@ -257,7 +262,9 @@ export function coveredByAnonymousTabBarScanSnapshot( parentIndex: scroller.length > 0 ? 1 : 0, type: 'Button', label: 'Save draft', - rect: { x: 16, y: 800, width: 140, height: 44 }, + rect: options.horizontal + ? { x: 16, y: 810, width: 140, height: 44 } + : { x: 16, y: 800, width: 140, height: 44 }, hittable: true, }, { diff --git a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts index b1f6b846d4..268afb7d75 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts @@ -212,7 +212,7 @@ test('runtime covered refusal names an unnamed generic-typed bar and never says ); }); -test('runtime covered refusal suggests a scroll for a scrollable target under a scanned bar', async () => { +test('runtime covered refusal suggests a scroll for a scroll-revealable target under a scanned bar', async () => { const device = createInteractionDevice( coveredByAnonymousTabBarScanSnapshot({ scrollable: true }), { @@ -229,7 +229,7 @@ test('runtime covered refusal suggests a scroll for a scrollable target under a kind: 'tab-bar', rect: { x: 0, y: 791, width: 402, height: 83 }, persistentBar: true, - targetInScrollableContent: true, + scrollCanReveal: true, }); assert.match( String(details?.hint), @@ -241,6 +241,36 @@ test('runtime covered refusal suggests a scroll for a scrollable target under a ); }); +test('runtime covered refusal keeps the fixed-target recovery for a horizontal scroller wholly behind a scanned bar', async () => { + // The bar contains the horizontal scroller's whole 402x65 viewport, so no scroll can reveal the + // button: the hint must not suggest scrolling, which could never clear the bar. + const device = createInteractionDevice( + coveredByAnonymousTabBarScanSnapshot({ horizontal: true }), + { + tap: async () => {}, + }, + ); + + await assert.rejects( + () => device.interactions.click(ref('@e3'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_covered'); + assert.deepEqual(details?.coveredBy, { + kind: 'tab-bar', + rect: { x: 0, y: 791, width: 402, height: 83 }, + persistentBar: true, + }); + assert.equal( + String(details?.hint), + 'Covered by the tab-bar at (0, 791 402x83). Use a different visible target, or inspect with snapshot/screenshot before retrying.', + ); + assert.doesNotMatch(String(details?.hint), /scroll/i); + return true; + }, + ); +}); + test('runtime covered refusal keeps the generic hint when the cover is a policy boolean', async () => { // `coveredByTabBarSnapshot` hand-annotates the covered boolean the way the Android // replacement-surface lane publishes it: no cover identity, so nothing to name. diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 5ae2b78585..e7f5a533be 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -274,11 +274,8 @@ export type SnapshotCoveringNode = { * navigation), read from the cover's type, role, and subrole. Absent for any other cover. */ persistentBar?: true; - /** - * Set when the covered node sits inside scrollable content, so a scroll can move it out from under - * the cover. Absent for a fixed node. - */ - targetInScrollableContent?: true; + /** The target's scroll viewport and axis leave a possible tap point clear of the persistent bar. */ + scrollCanReveal?: true; }; /** Upper bound for a covering-node label carried in diagnostics; longer names are clipped. */ diff --git a/packages/selectors/src/interaction-error.test.ts b/packages/selectors/src/interaction-error.test.ts index fd9e5f1273..d36fef3ddf 100644 --- a/packages/selectors/src/interaction-error.test.ts +++ b/packages/selectors/src/interaction-error.test.ts @@ -84,16 +84,16 @@ test('the hint falls back to the bare kind when the label sanitizes to nothing', assert.match(hint, /^Covered by the tab-bar at \(0, 760 390x84\)/); }); -test('the hint for a persistent bar suggests a scroll only for a target in scrollable content', () => { +test('the hint for a persistent bar suggests a scroll only when the scan proved scrolling can reveal the target', () => { const bar = { kind: 'navigation-bar', rect: { x: 0, y: 0, width: 390, height: 96 }, persistentBar: true, } as const; - const hint = (coveredBy: typeof bar & { targetInScrollableContent?: true }) => + const hint = (coveredBy: typeof bar & { scrollCanReveal?: true }) => String(coveredInteractionDetails({ ref: 'e2', interactionBlocked: 'covered', coveredBy }).hint); assert.equal( - hint({ ...bar, targetInScrollableContent: true }), + hint({ ...bar, scrollCanReveal: true }), 'Covered by the navigation-bar at (0, 0 390x96). Scroll the target clear of it, or inspect with snapshot/screenshot before retrying.', ); assert.equal( diff --git a/packages/selectors/src/interaction-error.ts b/packages/selectors/src/interaction-error.ts index a4bebb234e..5a7cb85a5d 100644 --- a/packages/selectors/src/interaction-error.ts +++ b/packages/selectors/src/interaction-error.ts @@ -88,9 +88,10 @@ function coveredInteractionHint(coveredBy: SnapshotCoveringNode | undefined): st if (!coveredBy) { return 'Use a different visible target, scroll it clear of the overlay, or inspect with snapshot/screenshot before retrying.'; } - // Persistent bar chrome cannot be dismissed, and a scroll moves only a target in scrollable content. + // Persistent bar chrome cannot be dismissed, and only a target the scan proved scroll-revealable + // can be scrolled clear of it. if (coveredBy.persistentBar) { - const recovery = coveredBy.targetInScrollableContent + const recovery = coveredBy.scrollCanReveal ? 'Scroll the target clear of it' : 'Use a different visible target'; return ( diff --git a/src/commands/schema/cli-help-workflows.test.ts b/src/commands/schema/cli-help-workflows.test.ts index a796683acd..b23888edb3 100644 --- a/src/commands/schema/cli-help-workflows.test.ts +++ b/src/commands/schema/cli-help-workflows.test.ts @@ -75,10 +75,7 @@ test('usageForCommand resolves manual QA help topic', async () => { help, /under a persistent bar \(tab bar, toolbar, navigation bar, bottom navigation\) use a different visible target or inspect with snapshot\/screenshot/, ); - assert.match( - help, - /scrolling past the bar only when coveredBy\.targetInScrollableContent is true/, - ); + assert.match(help, /scrolling past the bar only when coveredBy\.scrollCanReveal is true/); assert.doesNotMatch(help, /Dismiss\/close it or scroll the target clear/); }); diff --git a/src/commands/schema/cli-help-workflows.ts b/src/commands/schema/cli-help-workflows.ts index 979c8ac7dd..976c38f59f 100644 --- a/src/commands/schema/cli-help-workflows.ts +++ b/src/commands/schema/cli-help-workflows.ts @@ -53,7 +53,7 @@ Recovery: Network/typeahead result missing: wait text "Expected result" or wait . A target that should be gone/disappear: use wait absent ; wait exists ... is rejected in favor of the plain selector wait. Keyboard visible over the next target: a tap point under a measured keyboard frame returns tap_keyboard_occludes_target. Press a visible Hide Keyboard key or the app's dismissal control. In apps that support drag dismissal, use gesture pan from inside the scrollable content. Use keyboard enter only to send Return. Missing keyboard nodes never prove absence. A raw coordinate behind the keys taps anyway and reports a warning. - Target covered by other visible UI: refused with target_covered. When the occlusion scan identified the covering element, error.details.coveredBy and the hint name it (role, label, rect); the cover is diagnosis only -- never target its ref. Follow the hint's recovery: dismiss or close the cover only when it is dismissible (sheet, dialog, alert); under a persistent bar (tab bar, toolbar, navigation bar, bottom navigation) use a different visible target or inspect with snapshot/screenshot, scrolling past the bar only when coveredBy.targetInScrollableContent is true; then retry. + Target covered by other visible UI: refused with target_covered. When the occlusion scan identified the covering element, error.details.coveredBy and the hint name it (role, label, rect); the cover is diagnosis only -- never target its ref. Follow the hint's recovery: dismiss or close the cover only when it is dismissible (sheet, dialog, alert); under a persistent bar (tab bar, toolbar, navigation bar, bottom navigation) use a different visible target or inspect with snapshot/screenshot, scrolling past the bar only when coveredBy.scrollCanReveal is true; then retry. Sparse or recovered accessibility snapshot: use screenshot as visual truth, leave the bad screen if needed, then retry snapshot -i. Non-hittable success hint: verify with the settled diff or snapshot; retarget by a better ref/selector if the UI did not change. diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 3e078e4523..d064c9213a 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -168,11 +168,10 @@ and subrole, so a bar with a generic type and no name still publishes `coveredBy the bar kind (`tab-bar`, `toolbar`, `navigation-bar`, or `bottom-navigation`). A `click`/`press`/`fill`/`find ... focus` refused on that node repeats the same description in -`error.details.coveredBy` and names it in `error.hint`. When the covered node sits inside scrollable -content, `coveredBy` also carries `targetInScrollableContent: true`. A persistent bar cannot be -dismissed. Its hint tells you to scroll the target clear of it only when the target is in scrollable -content. For a fixed target, the hint tells you to use a different visible target or to inspect with -snapshot or screenshot. `coveredBy` is diagnosis only: it carries no +`error.details.coveredBy` and names it in `error.hint`. For a persistent bar, follow the scroll +suggestion when `coveredBy.scrollCanReveal` is true. If the bar covers the whole scroll viewport +or blocks every reachable tap point, use a different visible target or inspect with a snapshot or +screenshot. The same recovery applies to a fixed target. `coveredBy` is diagnostic and has no ref, so the cover is not an actionable target. It is absent when nothing nameable covers — an anonymous container with no accessible name that is not a persistent bar, or a backend policy that reports only the boolean, such as an Android replacement surface — and those refusals say only that the target is covered. From 0a7e56ecd93eaa83abe6aa8837f29a72d491cc81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 19:03:34 +0100 Subject: [PATCH 15/20] fix(occlusion): require scroll direction evidence for persistent bars --- ...snapshot-occlusion-scroll-recovery.test.ts | 92 ++++++++++++++----- .../src/snapshot-occlusion.test.ts | 2 +- .../capture-kit/src/snapshot-occlusion.ts | 36 +++++--- .../snapshot/__tests__/snapshot-audit.test.ts | 1 + .../runtime/__tests__/test-utils/index.ts | 1 + website/docs/docs/snapshots.md | 16 ++-- 6 files changed, 103 insertions(+), 45 deletions(-) diff --git a/packages/capture-kit/src/snapshot-occlusion-scroll-recovery.test.ts b/packages/capture-kit/src/snapshot-occlusion-scroll-recovery.test.ts index cedad1e2a7..770db9957f 100644 --- a/packages/capture-kit/src/snapshot-occlusion-scroll-recovery.test.ts +++ b/packages/capture-kit/src/snapshot-occlusion-scroll-recovery.test.ts @@ -25,6 +25,32 @@ test.each([ barRect: { x: 0, y: 725, width: 400, height: 75 }, expected: undefined, }, + { + name: 'an iOS scroll view with unknown axis behind a tab bar', + scrollerType: 'ScrollView', + barType: 'TabBar', + scrollerRect: { x: 0, y: 700, width: 400, height: 100 }, + barRect: { x: 0, y: 725, width: 400, height: 75 }, + expected: undefined, + }, + { + name: 'an iOS scroll view with hidden content below a tab bar', + scrollerType: 'ScrollView', + barType: 'TabBar', + hiddenContentBelow: true, + scrollerRect: { x: 0, y: 700, width: 400, height: 100 }, + barRect: { x: 0, y: 725, width: 400, height: 75 }, + expected: true, + }, + { + name: 'an iOS scroll view with hidden content only above a tab bar', + scrollerType: 'ScrollView', + barType: 'TabBar', + hiddenContentAbove: true, + scrollerRect: { x: 0, y: 700, width: 400, height: 100 }, + barRect: { x: 0, y: 725, width: 400, height: 75 }, + expected: undefined, + }, { name: 'a horizontal tap path that extends past the bar', scrollerType: 'HorizontalScrollView', @@ -39,28 +65,46 @@ test.each([ barRect: { x: 0, y: 725, width: 400, height: 75 }, expected: true, }, -])('reports scroll recovery for $name', ({ scrollerType, scrollerRect, barRect, expected }) => { - const nodes: RawSnapshotNode[] = [ - { index: 0, type: 'Window', rect: { x: 0, y: 0, width: 400, height: 800 } }, - { index: 1, parentIndex: 0, type: `android.widget.${scrollerType}`, rect: scrollerRect }, - { - index: 2, - parentIndex: 1, - type: 'Button', - hittable: true, - label: 'Buy', - rect: { x: 320, y: 740, width: 60, height: 40 }, - }, - { - index: 3, - parentIndex: 0, - type: 'com.google.android.material.bottomnavigation.BottomNavigationView', - rect: barRect, - }, - ]; +])( + 'reports scroll recovery for $name', + ({ + scrollerType, + barType, + hiddenContentAbove, + hiddenContentBelow, + scrollerRect, + barRect, + expected, + }) => { + const nodes: RawSnapshotNode[] = [ + { index: 0, type: 'Window', rect: { x: 0, y: 0, width: 400, height: 800 } }, + { + index: 1, + parentIndex: 0, + type: barType ? scrollerType : `android.widget.${scrollerType}`, + rect: scrollerRect, + hiddenContentAbove, + hiddenContentBelow, + }, + { + index: 2, + parentIndex: 1, + type: 'Button', + hittable: true, + label: 'Buy', + rect: { x: 320, y: 740, width: 60, height: 40 }, + }, + { + index: 3, + parentIndex: 0, + type: barType ?? 'com.google.android.material.bottomnavigation.BottomNavigationView', + rect: barRect, + }, + ]; - const target = annotateCoveredSnapshotNodes(nodes)[2]; - assert.equal(target?.interactionBlocked, 'covered'); - assert.equal(target?.coveredBy?.persistentBar, true); - assert.equal(target?.coveredBy?.scrollCanReveal, expected); -}); + const target = annotateCoveredSnapshotNodes(nodes)[2]; + assert.equal(target?.interactionBlocked, 'covered'); + assert.equal(target?.coveredBy?.persistentBar, true); + assert.equal(target?.coveredBy?.scrollCanReveal, expected); + }, +); diff --git a/packages/capture-kit/src/snapshot-occlusion.test.ts b/packages/capture-kit/src/snapshot-occlusion.test.ts index 95a698e448..c33bac4a7d 100644 --- a/packages/capture-kit/src/snapshot-occlusion.test.ts +++ b/packages/capture-kit/src/snapshot-occlusion.test.ts @@ -735,7 +735,7 @@ test('a full-viewport bottom navigation bar is not viewport chrome: it still cov assert.equal(annotated[1]?.interactionBlocked, 'covered'); assert.equal(annotated[1]?.coveredBy?.persistentBar, true); - assert.match(annotated[1]?.coveredBy?.kind ?? '', /bottomnavigation/); + assert.equal(annotated[1]?.coveredBy?.kind, 'bottom-navigation'); }); test.for([ diff --git a/packages/capture-kit/src/snapshot-occlusion.ts b/packages/capture-kit/src/snapshot-occlusion.ts index 6dcdc5f46d..c372c90191 100644 --- a/packages/capture-kit/src/snapshot-occlusion.ts +++ b/packages/capture-kit/src/snapshot-occlusion.ts @@ -174,9 +174,8 @@ function isViewportChromeBarNode( * the surface to dismiss. Strip that label and the same node would say `the group at (...)`, which * points at a box nobody would recognize; an anonymous container of that shape therefore publishes * no cover, and the refusal keeps its generic recovery. A named kind (`dialog`, `tab-bar`, `sheet`) - * still describes itself with no label to lean on. A persistent bar always publishes, and when its - * type presents as an anonymous container, its kind comes from the bar identity its role or subrole - * carries (`Other` with role `UITabBar` publishes `tab-bar`), so the cover names a recognizable bar. + * still describes itself with no label to lean on. Persistent bars always publish their canonical + * bar kind, including when the type is generic and only the role or subrole identifies the bar. */ function describeCoveringNode(node: CoveringNodeEvidence): SnapshotCoveringNode | null { const label = node.label?.trim(); @@ -186,7 +185,7 @@ function describeCoveringNode(node: CoveringNodeEvidence): SnapshotCoveringNode if (!label && !barKind && anonymous) return null; const rect = positiveRect(node.rect); return { - kind: barKind && anonymous ? barKind : typeKind, + kind: barKind ?? typeKind, ...(label ? { label: label.slice(0, SNAPSHOT_COVERING_LABEL_MAX_LENGTH) } : {}), ...(rect ? { rect } : {}), ...(barKind ? { persistentBar: true } : {}), @@ -268,18 +267,33 @@ function canScrollingRevealTarget( const viewport = screen ? intersectRects(scrollRect, screen) : scrollRect; if (!viewport) return false; + const path = scrollTapPath(scroller, viewport, centerOfRect(targetRect)); + return path !== undefined && !rectContains(cover, path); +} + +function scrollTapPath( + scroller: RawSnapshotNode, + viewport: Rect, + { x, y }: Pick, +): Rect | undefined { const horizontal = nodeKindIncludesAny(scroller, ['horizontal']); const vertical = nodeKindIncludesAny(scroller, ['vertical']); - if (horizontal === vertical) return !rectContains(cover, viewport); // The tap point's perpendicular coordinate stays fixed in a one-axis scroller. - const { x, y } = centerOfRect(targetRect); - if (horizontal) { - if (y < viewport.y || y > viewport.y + viewport.height) return false; - return !rectContains(cover, { ...viewport, y, height: 0 }); + if (horizontal && !vertical) { + if (y < viewport.y || y > viewport.y + viewport.height) return undefined; + return { ...viewport, y, height: 0 }; } - if (x < viewport.x || x > viewport.x + viewport.width) return false; - return !rectContains(cover, { ...viewport, x, width: 0 }); + if (vertical && !horizontal) { + if (x < viewport.x || x > viewport.x + viewport.width) return undefined; + return { ...viewport, x, width: 0 }; + } + + if (!containsPoint(viewport, x, y)) return undefined; + // Revealing content below moves the target up, and revealing content above moves it down. + const top = scroller.hiddenContentBelow ? viewport.y : y; + const bottom = scroller.hiddenContentAbove ? viewport.y + viewport.height : y; + return bottom > top ? { x, y: top, width: 0, height: bottom - top } : undefined; } export function isSnapshotNodeInteractionBlocked( diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts index 50c339bf75..26472e6b09 100644 --- a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts @@ -363,6 +363,7 @@ function scannedCoverTree(cover: CoverPublication): SnapshotNode[] { parentIndex: 1, depth: 2, type: 'ScrollView', + hiddenContentBelow: true, rect: { x: 0, y: 0, width: 402, height: 874 }, }, { diff --git a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts index 2380d7e096..b102b6085f 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts @@ -239,6 +239,7 @@ export function coveredByAnonymousTabBarScanSnapshot( depth: 1, parentIndex: 0, type: options.horizontal ? 'android.widget.HorizontalScrollView' : 'ScrollView', + hiddenContentBelow: options.scrollable ? true : undefined, rect: options.horizontal ? { x: 0, y: 800, width: 402, height: 65 } : { x: 0, y: 0, width: 402, height: 874 }, diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index d064c9213a..f8f392dd35 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -160,18 +160,16 @@ when it has one, and its `rect`: } ``` -`coveredBy.kind` is the same presented role any other node carries, so a cover can be a generic -container — an alert's container often arrives as `group` — and the label is what makes it nameable. -When the cover is a persistent bar (tab bar, toolbar, navigation bar, or bottom navigation), -`coveredBy` also carries `persistentBar: true`. The scan reads that fact from the cover's type, role, -and subrole, so a bar with a generic type and no name still publishes `coveredBy`. Its `kind` is then -the bar kind (`tab-bar`, `toolbar`, `navigation-bar`, or `bottom-navigation`). +A persistent bar publishes `persistentBar: true`. Its `coveredBy.kind` is `tab-bar`, `toolbar`, +`navigation-bar`, or `bottom-navigation`. The cover can be identified by its type, +role, or subrole. Other covers use their presented role. For example, an alert's container may +publish as `group` with the alert's accessible name. A `click`/`press`/`fill`/`find ... focus` refused on that node repeats the same description in `error.details.coveredBy` and names it in `error.hint`. For a persistent bar, follow the scroll -suggestion when `coveredBy.scrollCanReveal` is true. If the bar covers the whole scroll viewport -or blocks every reachable tap point, use a different visible target or inspect with a snapshot or -screenshot. The same recovery applies to a fixed target. `coveredBy` is diagnostic and has no +suggestion when `coveredBy.scrollCanReveal` is true. Otherwise, use a different visible target +or inspect with a snapshot or screenshot. This includes fixed targets, targets with no available +scroll direction, and targets whose reachable tap points all sit behind the bar. `coveredBy` is diagnostic and has no ref, so the cover is not an actionable target. It is absent when nothing nameable covers — an anonymous container with no accessible name that is not a persistent bar, or a backend policy that reports only the boolean, such as an Android replacement surface — and those refusals say only that the target is covered. From 95758cb3abec4b33c0aa33f946bd204baefc8ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 19:20:38 +0100 Subject: [PATCH 16/20] docs(audit): explain scroll recovery evidence and limits --- website/docs/docs/commands.md | 2 +- website/docs/docs/snapshots.md | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 14cdcc29d0..40e5878578 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -433,7 +433,7 @@ agent-device audit - `small-target`: an enabled, visible control below 44pt on iOS or 48dp on Android, with a 0.5-unit tolerance for layout rounding. Android pixels convert to dp with the captured display density; without it the check reports a limitation instead. TV and desktop targets skip the check. A link inside a text container is exempt. - `missing-name`: a control with no accessible name. An identifier, a resource id, or a field value is not a name; a label, an Android content description, a placeholder, or a named child is. -- `covered-control`: a control or row the snapshot already marks `[covered]`. Controls inside scroll containers under tab bars, navigation bars, toolbars, or bottom navigation bars are excluded. Scroll to reveal these controls. Controls under sheets, dialogs, and other overlays remain findings. +- `covered-control`: a control or row the snapshot already marks `[covered]`. Audit omits controls under persistent bars only when the snapshot indicates scrolling could clear the bar. Controls with no such scroll path stay in the findings, as do controls under sheets, dialogs, and other overlays. Controls are recognized by role (button, link, switch, checkbox, radio, slider, menu item, text and search fields). Generic clickable containers on Android are not evaluated. iOS visibility relies on the `hittable` fact. System bars, keyboard keys, and UIKit sheet grabbers are skipped. On iOS, direct child controls of navigation bars, toolbars, and tab bars skip target-size checks because the platform sets their size. Controls inside nested content containers remain checked. These bar controls still receive missing-name checks. A system-generated accessible name, such as "add" or "remove" for an icon-only React Native Pressable on iOS 26, prevents a missing-name finding. The audit cannot determine whether the app supplied that name. When a control is the only child of a control with the same role, the child drops a finding only if the parent reports the same kind of finding. diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index f8f392dd35..8be450d12b 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -167,9 +167,10 @@ publish as `group` with the alert's accessible name. A `click`/`press`/`fill`/`find ... focus` refused on that node repeats the same description in `error.details.coveredBy` and names it in `error.hint`. For a persistent bar, follow the scroll -suggestion when `coveredBy.scrollCanReveal` is true. Otherwise, use a different visible target -or inspect with a snapshot or screenshot. This includes fixed targets, targets with no available -scroll direction, and targets whose reachable tap points all sit behind the bar. `coveredBy` is diagnostic and has no +suggestion when `coveredBy.scrollCanReveal` is true. This flag describes a possible path based on +the captured viewport and scroll direction. Snapshots do not measure the remaining scroll distance. +If the flag is absent or scrolling leaves the bar in the way, use a different visible target or +inspect with a snapshot or screenshot. `coveredBy` is diagnostic and has no ref, so the cover is not an actionable target. It is absent when nothing nameable covers — an anonymous container with no accessible name that is not a persistent bar, or a backend policy that reports only the boolean, such as an Android replacement surface — and those refusals say only that the target is covered. From ea1bbc878bae45635219598ae9adb9f86f7f8fb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 19:46:26 +0100 Subject: [PATCH 17/20] refactor(audit): separate scroll clearance from finding output --- ...snapshot-occlusion-scroll-recovery.test.ts | 110 ------------------ .../capture-kit/src/snapshot-occlusion.ts | 67 +---------- .../snapshot/__tests__/snapshot-audit.test.ts | 27 +---- .../src/snapshot/snapshot-audit.ts | 8 +- .../runtime/snapshot-unchanged.test.ts | 3 - .../capture/runtime/snapshot-unchanged.ts | 1 - .../runtime/target-visibility-stages.test.ts | 7 +- .../command-surface/src/capture/snapshot.ts | 2 +- packages/kernel/src/snapshot.ts | 4 +- .../selectors/src/interaction-error.test.ts | 11 +- packages/selectors/src/interaction-error.ts | 7 +- .../schema/cli-help-workflows.test.ts | 1 - src/commands/schema/cli-help-workflows.ts | 2 +- website/docs/docs/commands.md | 6 +- website/docs/docs/snapshots.md | 8 +- 15 files changed, 22 insertions(+), 242 deletions(-) delete mode 100644 packages/capture-kit/src/snapshot-occlusion-scroll-recovery.test.ts diff --git a/packages/capture-kit/src/snapshot-occlusion-scroll-recovery.test.ts b/packages/capture-kit/src/snapshot-occlusion-scroll-recovery.test.ts deleted file mode 100644 index 770db9957f..0000000000 --- a/packages/capture-kit/src/snapshot-occlusion-scroll-recovery.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { annotateCoveredSnapshotNodes } from './snapshot-occlusion.ts'; - -test.each([ - { - name: 'a horizontal viewport wholly contained by a bottom bar', - scrollerType: 'HorizontalScrollView', - scrollerRect: { x: 0, y: 725, width: 400, height: 65 }, - barRect: { x: 0, y: 725, width: 400, height: 75 }, - expected: undefined, - }, - { - name: 'a viewport contained by a bar after clipping to the screen', - scrollerType: 'ScrollView', - scrollerRect: { x: 0, y: 730, width: 400, height: 150 }, - barRect: { x: 0, y: 725, width: 400, height: 75 }, - expected: undefined, - }, - { - name: 'a horizontal tap path entirely behind a full-width bar', - scrollerType: 'HorizontalScrollView', - scrollerRect: { x: 0, y: 700, width: 400, height: 100 }, - barRect: { x: 0, y: 725, width: 400, height: 75 }, - expected: undefined, - }, - { - name: 'an iOS scroll view with unknown axis behind a tab bar', - scrollerType: 'ScrollView', - barType: 'TabBar', - scrollerRect: { x: 0, y: 700, width: 400, height: 100 }, - barRect: { x: 0, y: 725, width: 400, height: 75 }, - expected: undefined, - }, - { - name: 'an iOS scroll view with hidden content below a tab bar', - scrollerType: 'ScrollView', - barType: 'TabBar', - hiddenContentBelow: true, - scrollerRect: { x: 0, y: 700, width: 400, height: 100 }, - barRect: { x: 0, y: 725, width: 400, height: 75 }, - expected: true, - }, - { - name: 'an iOS scroll view with hidden content only above a tab bar', - scrollerType: 'ScrollView', - barType: 'TabBar', - hiddenContentAbove: true, - scrollerRect: { x: 0, y: 700, width: 400, height: 100 }, - barRect: { x: 0, y: 725, width: 400, height: 75 }, - expected: undefined, - }, - { - name: 'a horizontal tap path that extends past the bar', - scrollerType: 'HorizontalScrollView', - scrollerRect: { x: 0, y: 700, width: 400, height: 100 }, - barRect: { x: 300, y: 700, width: 100, height: 100 }, - expected: true, - }, - { - name: 'a vertical tap path that extends above the bar', - scrollerType: 'VerticalScrollView', - scrollerRect: { x: 0, y: 700, width: 400, height: 100 }, - barRect: { x: 0, y: 725, width: 400, height: 75 }, - expected: true, - }, -])( - 'reports scroll recovery for $name', - ({ - scrollerType, - barType, - hiddenContentAbove, - hiddenContentBelow, - scrollerRect, - barRect, - expected, - }) => { - const nodes: RawSnapshotNode[] = [ - { index: 0, type: 'Window', rect: { x: 0, y: 0, width: 400, height: 800 } }, - { - index: 1, - parentIndex: 0, - type: barType ? scrollerType : `android.widget.${scrollerType}`, - rect: scrollerRect, - hiddenContentAbove, - hiddenContentBelow, - }, - { - index: 2, - parentIndex: 1, - type: 'Button', - hittable: true, - label: 'Buy', - rect: { x: 320, y: 740, width: 60, height: 40 }, - }, - { - index: 3, - parentIndex: 0, - type: barType ?? 'com.google.android.material.bottomnavigation.BottomNavigationView', - rect: barRect, - }, - ]; - - const target = annotateCoveredSnapshotNodes(nodes)[2]; - assert.equal(target?.interactionBlocked, 'covered'); - assert.equal(target?.coveredBy?.persistentBar, true); - assert.equal(target?.coveredBy?.scrollCanReveal, expected); - }, -); diff --git a/packages/capture-kit/src/snapshot-occlusion.ts b/packages/capture-kit/src/snapshot-occlusion.ts index c372c90191..c778e9f3e2 100644 --- a/packages/capture-kit/src/snapshot-occlusion.ts +++ b/packages/capture-kit/src/snapshot-occlusion.ts @@ -5,11 +5,8 @@ import { SNAPSHOT_COVERING_LABEL_MAX_LENGTH, } from '@agent-device/kernel/snapshot'; import { areRectsApproximatelyEqual, normalizeRect } from '@agent-device/kernel/rect-center'; -import { containsPoint, intersectRects, rectContains } from '@agent-device/kernel/rect'; +import { containsPoint, rectContains } from '@agent-device/kernel/rect'; import { - collectViewportRects, - findNearestScrollableAncestor, - resolveViewportRect, isScrollableNodeLike, snapshotBarKind, snapshotNodeKindIncludesAny as nodeKindIncludesAny, @@ -222,80 +219,20 @@ function annotateCoveredPositions( coveredPositions: ReadonlyArray<{ position: number; coveredBy?: SnapshotCoveringNode }>, ): RawSnapshotNode[] { if (coveredPositions.length === 0) return nodes; - const byIndex = new Map(nodes.map((node) => [node.index, node])); const annotated = [...nodes]; - let viewportRects: Rect[] | undefined; for (const { position, coveredBy } of coveredPositions) { const node = nodes[position]!; - const scrollCanReveal = - coveredBy?.persistentBar && - coveredBy.rect && - canScrollingRevealTarget( - node, - coveredBy.rect, - byIndex, - nodes, - (viewportRects ??= collectViewportRects(nodes)), - ); annotated[position] = { ...node, hittable: false, interactionBlocked: 'covered' as const, - ...(coveredBy - ? { coveredBy: scrollCanReveal ? { ...coveredBy, scrollCanReveal: true } : coveredBy } - : {}), + ...(coveredBy ? { coveredBy } : {}), presentationHints: mergeCoveredHint(node.presentationHints), }; } return annotated; } -function canScrollingRevealTarget( - target: RawSnapshotNode, - cover: Rect, - byIndex: ReadonlyMap, - nodes: RawSnapshotNode[], - viewportRects: readonly Rect[], -): boolean { - const scroller = findNearestScrollableAncestor(target, byIndex); - const scrollRect = positiveRect(scroller?.rect); - const targetRect = positiveRect(target.rect); - if (!scroller || !scrollRect || !targetRect) return false; - - const screen = - viewportRects.length > 0 ? resolveViewportRect(nodes, targetRect, viewportRects) : null; - const viewport = screen ? intersectRects(scrollRect, screen) : scrollRect; - if (!viewport) return false; - - const path = scrollTapPath(scroller, viewport, centerOfRect(targetRect)); - return path !== undefined && !rectContains(cover, path); -} - -function scrollTapPath( - scroller: RawSnapshotNode, - viewport: Rect, - { x, y }: Pick, -): Rect | undefined { - const horizontal = nodeKindIncludesAny(scroller, ['horizontal']); - const vertical = nodeKindIncludesAny(scroller, ['vertical']); - - // The tap point's perpendicular coordinate stays fixed in a one-axis scroller. - if (horizontal && !vertical) { - if (y < viewport.y || y > viewport.y + viewport.height) return undefined; - return { ...viewport, y, height: 0 }; - } - if (vertical && !horizontal) { - if (x < viewport.x || x > viewport.x + viewport.width) return undefined; - return { ...viewport, x, width: 0 }; - } - - if (!containsPoint(viewport, x, y)) return undefined; - // Revealing content below moves the target up, and revealing content above moves it down. - const top = scroller.hiddenContentBelow ? viewport.y : y; - const bottom = scroller.hiddenContentAbove ? viewport.y + viewport.height : y; - return bottom > top ? { x, y: top, width: 0, height: bottom - top } : undefined; -} - export function isSnapshotNodeInteractionBlocked( node: Pick, ): boolean { diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts index 26472e6b09..acf0bbaa85 100644 --- a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts @@ -412,15 +412,14 @@ test.for([ name: 'Android bottom navigation', type: 'com.google.android.material.bottomnavigation.BottomNavigationView', }, -])('ignores a scanned bar from the $name publication over a scrolled row only', (cover) => { +])('reports both covered controls under a scanned bar from the $name publication', (cover) => { const nodes = scannedCoverTree(cover); expect(nodes[3]).toMatchObject({ interactionBlocked: 'covered', - coveredBy: { persistentBar: true, scrollCanReveal: true }, + coveredBy: { persistentBar: true }, }); expect(nodes[4]?.coveredBy).toMatchObject({ persistentBar: true }); - expect(nodes[4]?.coveredBy?.scrollCanReveal).toBeUndefined(); - expect(summarize(audit(nodes).findings)).toEqual(['covered-control @e4']); + expect(summarize(audit(nodes).findings)).toEqual(['covered-control @e3', 'covered-control @e4']); }); test('keeps the covered-control finding for a button in a horizontal scroller wholly behind a bottom bar', () => { @@ -462,7 +461,6 @@ test('keeps the covered-control finding for a button in a horizontal scroller wh interactionBlocked: 'covered', coveredBy: { persistentBar: true }, }); - expect(nodes[2]?.coveredBy?.scrollCanReveal).toBeUndefined(); expect(summarize(audit(nodes, ANDROID).findings)).toEqual(['covered-control @e2']); }); @@ -508,25 +506,6 @@ test.each([ ).toEqual(['missing-name']); }); -test('keeps a child under an overlay when its covered row is excluded as scroll chrome', () => { - const findings = audit([ - node(0, { type: 'ScrollView', hittable: false }), - node(1, { - type: 'Cell', - parentIndex: 0, - interactionBlocked: 'covered', - coveredBy: { kind: 'tab-bar', persistentBar: true, scrollCanReveal: true }, - }), - node(2, { - parentIndex: 1, - label: 'Pay', - interactionBlocked: 'covered', - coveredBy: { kind: 'sheet' }, - }), - ]).findings; - expect(summarize(findings)).toEqual(['covered-control @e2']); -}); - test('skips UIKit sheet grabbers by captured class without matching their label', () => { const small = { x: 0, y: 0, width: 73, height: 24 }; const findings = audit([ diff --git a/packages/capture-kit/src/snapshot/snapshot-audit.ts b/packages/capture-kit/src/snapshot/snapshot-audit.ts index ff621f4980..78d40e6d47 100644 --- a/packages/capture-kit/src/snapshot/snapshot-audit.ts +++ b/packages/capture-kit/src/snapshot/snapshot-audit.ts @@ -241,7 +241,6 @@ function missingNameFinding(node: SnapshotNode, role: string): AuditFinding { function coveredFinding(node: SnapshotNode, scan: AuditScan): AuditFinding | undefined { const role = coveredRole(node); if (node.interactionBlocked !== 'covered' || role === undefined) return undefined; - if (isScrollableChromeOcclusion(node)) return undefined; if (isOnlyChildOfReportedCover(node, scan)) return undefined; return { ...findingBase(node, role), kind: 'covered-control', severity: 'observation' }; } @@ -258,7 +257,7 @@ function isRow(node: SnapshotNode): boolean { function isOnlyChildOfReportedCover(node: SnapshotNode, scan: AuditScan): boolean { const parent = parentOf(node, scan); if (!parent || parent.ref === undefined || coveredRole(parent) === undefined) return false; - return !isScrollableChromeOcclusion(parent) && scan.children.get(parent.index)?.length === 1; + return scan.children.get(parent.index)?.length === 1; } function findingBase(node: SnapshotNode, role: string) { @@ -305,11 +304,6 @@ function roundTenth(value: number): number { return Math.round(value * 10) / 10; } -/** Persistent bar chrome over a target a scroll can reveal is a scroll position, not a covered control. */ -function isScrollableChromeOcclusion(node: SnapshotNode): boolean { - return node.coveredBy?.persistentBar === true && node.coveredBy.scrollCanReveal === true; -} - function auditGuidance( findings: readonly AuditFinding[], context: AuditContext, diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts index 0f71fd8e72..d937b6db39 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts @@ -244,9 +244,6 @@ test('unchanged metadata compares covering-node descriptions by value', () => { current.nodes[0]!.coveredBy = { ...cover, persistentBar: true }; expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined(); - current.nodes[0]!.coveredBy = { ...cover, scrollCanReveal: true }; - expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined(); - current.nodes[0]!.coveredBy = undefined; expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined(); }); diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts index 2b10757c36..5734828f65 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts @@ -170,7 +170,6 @@ function areCoveringNodesEquivalent( previous.kind === current.kind && previous.label === current.label && previous.persistentBar === current.persistentBar && - previous.scrollCanReveal === current.scrollCanReveal && areRectsEquivalent(previous.rect, current.rect) ); } diff --git a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts index 268afb7d75..2570329975 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts @@ -212,7 +212,7 @@ test('runtime covered refusal names an unnamed generic-typed bar and never says ); }); -test('runtime covered refusal suggests a scroll for a scroll-revealable target under a scanned bar', async () => { +test('runtime covered refusal suggests inspection for a covered target in a scroll view', async () => { const device = createInteractionDevice( coveredByAnonymousTabBarScanSnapshot({ scrollable: true }), { @@ -229,13 +229,12 @@ test('runtime covered refusal suggests a scroll for a scroll-revealable target u kind: 'tab-bar', rect: { x: 0, y: 791, width: 402, height: 83 }, persistentBar: true, - scrollCanReveal: true, }); assert.match( String(details?.hint), - /^Covered by the tab-bar at \(0, 791 402x83\)\. Scroll the target clear of it/, + /^Covered by the tab-bar at \(0, 791 402x83\)\. Use a different visible target/, ); - assert.doesNotMatch(String(details?.hint), /dismiss|close/i); + assert.doesNotMatch(String(details?.hint), /dismiss|close|scroll/i); return true; }, ); diff --git a/packages/command-surface/src/capture/snapshot.ts b/packages/command-surface/src/capture/snapshot.ts index 9848c6eb09..7610ceaae4 100644 --- a/packages/command-surface/src/capture/snapshot.ts +++ b/packages/command-surface/src/capture/snapshot.ts @@ -169,7 +169,7 @@ export const AUDIT_COMMAND_OUTPUT_SCHEMAS = { } satisfies Pick, 'audit'>; const auditCommandDescription = - 'Report advisory accessibility findings from one fresh capture of the current screen: touch targets below the platform recommendation, controls without an accessible name, and covered controls. Each finding shows its role and accessible label when available. Guidance follows once per finding kind. Controls in scroll containers under persistent bars are excluded. UIKit sheet grabbers are skipped. Direct iOS bar items skip size checks but keep name checks. Nested app content remains checked. Findings are review prompts, not failures, and the audit changes nothing on the device.'; + 'Report advisory accessibility findings from one fresh capture of the current screen: touch targets below the platform recommendation, controls without an accessible name, and covered controls. Each finding shows its role and includes the captured label when available. Guidance follows once per finding kind. Covered controls remain findings inside scroll containers. UIKit sheet grabbers are skipped. Direct items of recognized native iOS bars skip size checks and keep name checks. Custom bar children and nested app content remain subject to size checks. Findings are review prompts, not failures, and the audit changes nothing on the device.'; export const auditCommandFacet = defineParameterlessCommandFacet({ name: AUDIT_COMMAND_NAME, diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index e7f5a533be..8fae1b0ee8 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -263,7 +263,7 @@ export type SnapshotPresentation = SnapshotOptionFields< * issues refs for, and naming it is diagnosis, not an invitation to target it. */ export type SnapshotCoveringNode = { - /** The cover's presented role, from the same table the text presenter reads (`formatRole`). */ + /** The canonical persistent-bar kind or the cover's presented role. */ kind: string; /** The cover's accessible name, trimmed and clipped to {@link SNAPSHOT_COVERING_LABEL_MAX_LENGTH}. */ label?: string; @@ -274,8 +274,6 @@ export type SnapshotCoveringNode = { * navigation), read from the cover's type, role, and subrole. Absent for any other cover. */ persistentBar?: true; - /** The target's scroll viewport and axis leave a possible tap point clear of the persistent bar. */ - scrollCanReveal?: true; }; /** Upper bound for a covering-node label carried in diagnostics; longer names are clipped. */ diff --git a/packages/selectors/src/interaction-error.test.ts b/packages/selectors/src/interaction-error.test.ts index d36fef3ddf..cce44dae4c 100644 --- a/packages/selectors/src/interaction-error.test.ts +++ b/packages/selectors/src/interaction-error.test.ts @@ -84,20 +84,15 @@ test('the hint falls back to the bare kind when the label sanitizes to nothing', assert.match(hint, /^Covered by the tab-bar at \(0, 760 390x84\)/); }); -test('the hint for a persistent bar suggests a scroll only when the scan proved scrolling can reveal the target', () => { +test('the hint for a persistent bar suggests a visible target or inspection', () => { const bar = { kind: 'navigation-bar', rect: { x: 0, y: 0, width: 390, height: 96 }, persistentBar: true, } as const; - const hint = (coveredBy: typeof bar & { scrollCanReveal?: true }) => - String(coveredInteractionDetails({ ref: 'e2', interactionBlocked: 'covered', coveredBy }).hint); assert.equal( - hint({ ...bar, scrollCanReveal: true }), - 'Covered by the navigation-bar at (0, 0 390x96). Scroll the target clear of it, or inspect with snapshot/screenshot before retrying.', - ); - assert.equal( - hint(bar), + coveredInteractionDetails({ ref: 'e2', interactionBlocked: 'covered', coveredBy: bar }).hint, + 'Covered by the navigation-bar at (0, 0 390x96). Use a different visible target, or inspect with snapshot/screenshot before retrying.', ); }); diff --git a/packages/selectors/src/interaction-error.ts b/packages/selectors/src/interaction-error.ts index 5a7cb85a5d..a49f3e8f54 100644 --- a/packages/selectors/src/interaction-error.ts +++ b/packages/selectors/src/interaction-error.ts @@ -88,14 +88,9 @@ function coveredInteractionHint(coveredBy: SnapshotCoveringNode | undefined): st if (!coveredBy) { return 'Use a different visible target, scroll it clear of the overlay, or inspect with snapshot/screenshot before retrying.'; } - // Persistent bar chrome cannot be dismissed, and only a target the scan proved scroll-revealable - // can be scrolled clear of it. if (coveredBy.persistentBar) { - const recovery = coveredBy.scrollCanReveal - ? 'Scroll the target clear of it' - : 'Use a different visible target'; return ( - `Covered by ${describeCoverForHint(coveredBy)}. ${recovery}, ` + + `Covered by ${describeCoverForHint(coveredBy)}. Use a different visible target, ` + 'or inspect with snapshot/screenshot before retrying.' ); } diff --git a/src/commands/schema/cli-help-workflows.test.ts b/src/commands/schema/cli-help-workflows.test.ts index b23888edb3..99871d1305 100644 --- a/src/commands/schema/cli-help-workflows.test.ts +++ b/src/commands/schema/cli-help-workflows.test.ts @@ -75,7 +75,6 @@ test('usageForCommand resolves manual QA help topic', async () => { help, /under a persistent bar \(tab bar, toolbar, navigation bar, bottom navigation\) use a different visible target or inspect with snapshot\/screenshot/, ); - assert.match(help, /scrolling past the bar only when coveredBy\.scrollCanReveal is true/); assert.doesNotMatch(help, /Dismiss\/close it or scroll the target clear/); }); diff --git a/src/commands/schema/cli-help-workflows.ts b/src/commands/schema/cli-help-workflows.ts index 976c38f59f..ea4bfe63d4 100644 --- a/src/commands/schema/cli-help-workflows.ts +++ b/src/commands/schema/cli-help-workflows.ts @@ -53,7 +53,7 @@ Recovery: Network/typeahead result missing: wait text "Expected result" or wait . A target that should be gone/disappear: use wait absent ; wait exists ... is rejected in favor of the plain selector wait. Keyboard visible over the next target: a tap point under a measured keyboard frame returns tap_keyboard_occludes_target. Press a visible Hide Keyboard key or the app's dismissal control. In apps that support drag dismissal, use gesture pan from inside the scrollable content. Use keyboard enter only to send Return. Missing keyboard nodes never prove absence. A raw coordinate behind the keys taps anyway and reports a warning. - Target covered by other visible UI: refused with target_covered. When the occlusion scan identified the covering element, error.details.coveredBy and the hint name it (role, label, rect); the cover is diagnosis only -- never target its ref. Follow the hint's recovery: dismiss or close the cover only when it is dismissible (sheet, dialog, alert); under a persistent bar (tab bar, toolbar, navigation bar, bottom navigation) use a different visible target or inspect with snapshot/screenshot, scrolling past the bar only when coveredBy.scrollCanReveal is true; then retry. + Target covered by other visible UI: refused with target_covered. When the occlusion scan identified the covering element, error.details.coveredBy and the hint name it (role, label, rect); the cover is diagnosis only -- never target its ref. Follow the hint's recovery: dismiss or close the cover only when it is dismissible (sheet, dialog, alert); under a persistent bar (tab bar, toolbar, navigation bar, bottom navigation) use a different visible target or inspect with snapshot/screenshot before retrying. Sparse or recovered accessibility snapshot: use screenshot as visual truth, leave the bad screen if needed, then retry snapshot -i. Non-hittable success hint: verify with the settled diff or snapshot; retarget by a better ref/selector if the UI did not change. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 40e5878578..0d21404564 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -433,11 +433,11 @@ agent-device audit - `small-target`: an enabled, visible control below 44pt on iOS or 48dp on Android, with a 0.5-unit tolerance for layout rounding. Android pixels convert to dp with the captured display density; without it the check reports a limitation instead. TV and desktop targets skip the check. A link inside a text container is exempt. - `missing-name`: a control with no accessible name. An identifier, a resource id, or a field value is not a name; a label, an Android content description, a placeholder, or a named child is. -- `covered-control`: a control or row the snapshot already marks `[covered]`. Audit omits controls under persistent bars only when the snapshot indicates scrolling could clear the bar. Controls with no such scroll path stay in the findings, as do controls under sheets, dialogs, and other overlays. +- `covered-control`: a control or row the snapshot already marks `[covered]`. Audit reports covered controls inside scroll containers, including those under persistent bars, sheets, and dialogs. -Controls are recognized by role (button, link, switch, checkbox, radio, slider, menu item, text and search fields). Generic clickable containers on Android are not evaluated. iOS visibility relies on the `hittable` fact. System bars, keyboard keys, and UIKit sheet grabbers are skipped. On iOS, direct child controls of navigation bars, toolbars, and tab bars skip target-size checks because the platform sets their size. Controls inside nested content containers remain checked. These bar controls still receive missing-name checks. A system-generated accessible name, such as "add" or "remove" for an icon-only React Native Pressable on iOS 26, prevents a missing-name finding. The audit cannot determine whether the app supplied that name. When a control is the only child of a control with the same role, the child drops a finding only if the parent reports the same kind of finding. +Controls are recognized by role (button, link, switch, checkbox, radio, slider, menu item, text and search fields). Generic clickable containers on Android are not evaluated. iOS visibility relies on the `hittable` fact. System bars, keyboard keys, and UIKit sheet grabbers are skipped. Direct items of recognized native iOS navigation bars, toolbars, and tab bars skip target-size checks and still receive missing-name checks. Custom bar children and items inside nested content containers remain subject to size checks. A system-generated accessible name, such as "add" or "remove" for an icon-only React Native Pressable on iOS 26, prevents a missing-name finding. The audit cannot determine whether the app supplied that name. When a control is the only child of a control with the same role, the child drops a finding only if the parent reports the same kind of finding. -Like `find`, `audit` publishes only the refs it returns. The text prints each as `@eN~s`; press that pinned ref. A plain ref from before the audit stays refused after the frame expired, and an audit with no findings issues no refs. An audit with findings also ends the snapshot's complete frame, so take a new snapshot before you use refs that the audit did not return. Each finding shows its role and accessible label. An unnamed control shows its identifier when available. Guidance follows once per finding kind, with Apple HIG sizes on iOS and Material sizes on Android. `--json` returns `findings` (with `label` and `identifier` when available), `guidance` keyed by finding kind, `checks`, `limitations`, and `refsGeneration` (only with findings). `checks.snapshot` is `unknown` when the capture cannot report truncation, such as Appium page source. +Like `find`, `audit` publishes only the refs it returns. The text prints each as `@eN~s`; press that pinned ref. A plain ref from before the audit stays refused after the frame expired, and an audit with no findings issues no refs. An audit with findings also ends the snapshot's complete frame, so take a new snapshot before you use refs that the audit did not return. Each finding shows its role and includes the captured label when available. For example, a control may get its name from a child or placeholder while its own label is absent. An unnamed control shows its identifier when available. Guidance follows once per finding kind, with Apple HIG sizes on iOS and Material sizes on Android. `--json` returns `findings` (with `label` and `identifier` when available), `guidance` keyed by finding kind, `checks`, `limitations`, and `refsGeneration` (only with findings). `checks.snapshot` is `unknown` when the capture cannot report truncation, such as Appium page source. ## Wait and alerts diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 8be450d12b..5c69b8274b 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -166,11 +166,9 @@ role, or subrole. Other covers use their presented role. For example, an alert's publish as `group` with the alert's accessible name. A `click`/`press`/`fill`/`find ... focus` refused on that node repeats the same description in -`error.details.coveredBy` and names it in `error.hint`. For a persistent bar, follow the scroll -suggestion when `coveredBy.scrollCanReveal` is true. This flag describes a possible path based on -the captured viewport and scroll direction. Snapshots do not measure the remaining scroll distance. -If the flag is absent or scrolling leaves the bar in the way, use a different visible target or -inspect with a snapshot or screenshot. `coveredBy` is diagnostic and has no +`error.details.coveredBy` and names it in `error.hint`. For a persistent bar, use a different visible target or inspect with a snapshot or screenshot +before retrying. Audit reports covered controls inside scroll containers. +`coveredBy` is diagnostic and has no ref, so the cover is not an actionable target. It is absent when nothing nameable covers — an anonymous container with no accessible name that is not a persistent bar, or a backend policy that reports only the boolean, such as an Android replacement surface — and those refusals say only that the target is covered. From 6d872cc8406c35a658a4d5c88f3b11abe1b84604 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 20:08:15 +0100 Subject: [PATCH 18/20] fix(occlusion): require exact persistent bar identity --- .../capture-kit/src/snapshot-occlusion.ts | 23 ++++++++---------- .../runtime/__tests__/test-utils/index.ts | 4 ++-- .../runtime/target-visibility-stages.test.ts | 21 ++++++++++++++++ packages/contracts/src/snapshot-text.test.ts | 7 ++++-- packages/contracts/src/snapshot-text.ts | 24 +++++++------------ 5 files changed, 47 insertions(+), 32 deletions(-) diff --git a/packages/capture-kit/src/snapshot-occlusion.ts b/packages/capture-kit/src/snapshot-occlusion.ts index c778e9f3e2..ea76d4ec44 100644 --- a/packages/capture-kit/src/snapshot-occlusion.ts +++ b/packages/capture-kit/src/snapshot-occlusion.ts @@ -142,23 +142,20 @@ export function annotateCoveredSnapshotNodes( /** Presented roles that name a box rather than a surface a reader would recognize and dismiss. */ const ANONYMOUS_CONTAINER_KINDS = new Set(['element', 'group', 'other']); -/** - * The bar kinds this scan exempts as viewport chrome. Bar identity itself — including bottom - * navigation — is owned by `snapshotBarKind` in contracts; the choice of which kinds get a - * viewport exemption stays here, and bottom navigation is not one of them: an Android - * BottomNavigationView keeps covering the content behind it. - */ -const VIEWPORT_CHROME_BAR_KINDS: ReadonlySet = new Set([ - 'tab-bar', - 'toolbar', - 'navigation-bar', -]); +const VIEWPORT_CHROME_KIND_FRAGMENTS = [ + ...TAB_BAR_KIND_FRAGMENTS, + ...TOOLBAR_HOST_KIND_FRAGMENTS, + ...NAVIGATION_BAR_KIND_FRAGMENTS, +]; +/** Viewport hosts can expose internal classes such as UIKit tab platters. */ function isViewportChromeBarNode( node: Pick, ): boolean { - const kind = snapshotBarKind(node); - return kind !== undefined && VIEWPORT_CHROME_BAR_KINDS.has(kind); + return ( + !nodeKindIncludesAny(node, ['bottomnavigation']) && + nodeKindIncludesAny(node, VIEWPORT_CHROME_KIND_FRAGMENTS) + ); } /** diff --git a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts index b102b6085f..c7b0f8168e 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts @@ -189,7 +189,7 @@ export function coveredByTabBarSnapshot(): SnapshotState { * The same tab-bar-over-button shape run through the real annotation pass, so the covered node * carries the scan's `coveredBy` description exactly as a capture publishes it. */ -export function coveredByScanSnapshot(): SnapshotState { +export function coveredByScanSnapshot(coverType = 'TabBar'): SnapshotState { return makeSnapshotState( annotateCoveredSnapshotNodes([ { @@ -212,7 +212,7 @@ export function coveredByScanSnapshot(): SnapshotState { index: 2, depth: 1, parentIndex: 0, - type: 'TabBar', + type: coverType, label: 'Media', rect: { x: 0, y: 760, width: 390, height: 84 }, hittable: true, diff --git a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts index 2570329975..bf7b9f34f1 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts @@ -186,6 +186,27 @@ test('runtime covered refusal names the covering element the occlusion scan foun assert.deepEqual(calls, []); }); +test('runtime covered refusal keeps dismissal recovery for a custom toolbar overlay', async () => { + const device = createInteractionDevice(coveredByScanSnapshot('HomeToolbarView'), { + tap: async () => {}, + }); + + await assert.rejects( + () => device.interactions.click(ref('@e2'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_covered'); + assert.deepEqual(details?.coveredBy, { + kind: 'hometoolbarview', + label: 'Media', + rect: { x: 0, y: 760, width: 390, height: 84 }, + }); + assert.match(String(details?.hint), /Dismiss or close that element/); + return true; + }, + ); +}); + test('runtime covered refusal names an unnamed generic-typed bar and never says dismiss', async () => { const device = createInteractionDevice(coveredByAnonymousTabBarScanSnapshot(), { tap: async () => {}, diff --git a/packages/contracts/src/snapshot-text.test.ts b/packages/contracts/src/snapshot-text.test.ts index e613c4e033..b68a658bc6 100644 --- a/packages/contracts/src/snapshot-text.test.ts +++ b/packages/contracts/src/snapshot-text.test.ts @@ -60,8 +60,11 @@ test.each([ expect(isNativeIosSnapshotBarNode(node)).toBe(kind !== 'bottom-navigation'); }); -test('a non-bar node reads no bar kind', () => { - const node = { type: 'Button', role: 'UIButton' }; +test.each([ + { type: 'Button', role: 'UIButton' }, + { type: 'HomeToolbarView' }, + { type: 'Other', role: 'BottomTabBar' }, +])('a non-bar node reads no bar kind for %j', (node) => { expect(snapshotBarKind(node)).toBeUndefined(); expect(isNativeIosSnapshotBarNode(node)).toBe(false); }); diff --git a/packages/contracts/src/snapshot-text.ts b/packages/contracts/src/snapshot-text.ts index 2bb4e1efa5..0e13b8dc4c 100644 --- a/packages/contracts/src/snapshot-text.ts +++ b/packages/contracts/src/snapshot-text.ts @@ -14,14 +14,10 @@ export function normalizeType(type: string): string { return normalized; } -/** - * Presented bar kind, keyed by the normalized kind fragment that identifies it. This module owns - * the full bar identity, including Android bottom navigation, so every consumer reads one - * vocabulary; which kinds a scan treats as viewport chrome is the consumer's own choice. - */ const SNAPSHOT_BAR_KINDS = [ - // First, so a BottomNavigationBar name does not resolve to navigation-bar. ['bottomnavigation', 'bottom-navigation'], + ['bottomnavigationbar', 'bottom-navigation'], + ['bottomnavigationview', 'bottom-navigation'], ['tabbar', 'tab-bar'], ['toolbar', 'toolbar'], ['navigationbar', 'navigation-bar'], @@ -29,24 +25,22 @@ const SNAPSHOT_BAR_KINDS = [ type SnapshotBarKind = (typeof SNAPSHOT_BAR_KINDS)[number][1]; -/** Reads the bar kind, including bottom navigation, from the captured type, role, and subrole, or nothing for a non-bar node. */ +/** Reads an exact bar identity from the captured type, role, or subrole. */ export function snapshotBarKind( node: Pick, ): SnapshotBarKind | undefined { - const normalized = normalizeNodeKind(node); - return SNAPSHOT_BAR_KINDS.find(([fragment]) => normalized.includes(fragment))?.[1]; + const names = [node.type, node.role, node.subrole].map((value) => + normalizeType(value ?? '').replace(/^ui/, ''), + ); + return SNAPSHOT_BAR_KINDS.find(([name]) => names.includes(name))?.[1]; } /** Identifies native iOS bars whose direct items use platform sizing. */ export function isNativeIosSnapshotBarNode( node: Pick, ): boolean { - return [node.type, node.role, node.subrole].some((value) => { - const name = normalizeType(value ?? '').replace(/^ui/, ''); - return SNAPSHOT_BAR_KINDS.some( - ([fragment, kind]) => kind !== 'bottom-navigation' && name === fragment, - ); - }); + const kind = snapshotBarKind(node); + return kind !== undefined && kind !== 'bottom-navigation'; } export function snapshotNodeKindIncludesAny( From b584745f7b03d95fc280f984b8d3aebe9147fd89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 20:18:01 +0100 Subject: [PATCH 19/20] docs(audit): clarify grabber checks and rejected labels --- packages/command-surface/src/capture/snapshot.ts | 2 +- website/docs/docs/commands.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/command-surface/src/capture/snapshot.ts b/packages/command-surface/src/capture/snapshot.ts index 7610ceaae4..29d61122b3 100644 --- a/packages/command-surface/src/capture/snapshot.ts +++ b/packages/command-surface/src/capture/snapshot.ts @@ -169,7 +169,7 @@ export const AUDIT_COMMAND_OUTPUT_SCHEMAS = { } satisfies Pick, 'audit'>; const auditCommandDescription = - 'Report advisory accessibility findings from one fresh capture of the current screen: touch targets below the platform recommendation, controls without an accessible name, and covered controls. Each finding shows its role and includes the captured label when available. Guidance follows once per finding kind. Covered controls remain findings inside scroll containers. UIKit sheet grabbers are skipped. Direct items of recognized native iOS bars skip size checks and keep name checks. Custom bar children and nested app content remain subject to size checks. Findings are review prompts, not failures, and the audit changes nothing on the device.'; + 'Report advisory accessibility findings from one fresh capture of the current screen: touch targets below the platform recommendation, controls without an accessible name, and covered controls. Each finding shows its role. Missing-name findings omit rejected labels and can show the captured identifier. Other findings include the captured label when available. Guidance follows once per finding kind. Covered controls remain findings inside scroll containers. UIKit sheet grabbers are exempt from size and name checks. Direct items of recognized native iOS bars skip size checks and keep name checks. Custom bar children and nested app content remain subject to size checks. Findings are review prompts, not failures, and the audit changes nothing on the device.'; export const auditCommandFacet = defineParameterlessCommandFacet({ name: AUDIT_COMMAND_NAME, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 0d21404564..1d6f8c1941 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -435,9 +435,9 @@ agent-device audit - `missing-name`: a control with no accessible name. An identifier, a resource id, or a field value is not a name; a label, an Android content description, a placeholder, or a named child is. - `covered-control`: a control or row the snapshot already marks `[covered]`. Audit reports covered controls inside scroll containers, including those under persistent bars, sheets, and dialogs. -Controls are recognized by role (button, link, switch, checkbox, radio, slider, menu item, text and search fields). Generic clickable containers on Android are not evaluated. iOS visibility relies on the `hittable` fact. System bars, keyboard keys, and UIKit sheet grabbers are skipped. Direct items of recognized native iOS navigation bars, toolbars, and tab bars skip target-size checks and still receive missing-name checks. Custom bar children and items inside nested content containers remain subject to size checks. A system-generated accessible name, such as "add" or "remove" for an icon-only React Native Pressable on iOS 26, prevents a missing-name finding. The audit cannot determine whether the app supplied that name. When a control is the only child of a control with the same role, the child drops a finding only if the parent reports the same kind of finding. +Controls are recognized by role (button, link, switch, checkbox, radio, slider, menu item, text and search fields). Generic clickable containers on Android are not evaluated. iOS visibility relies on the `hittable` fact. Size and name checks skip system bars, keyboard keys, and UIKit sheet grabbers. Direct items of recognized native iOS navigation bars, toolbars, and tab bars skip target-size checks and still receive missing-name checks. Custom bar children and items inside nested content containers remain subject to size checks. A system-generated accessible name, such as "add" or "remove" for an icon-only React Native Pressable on iOS 26, prevents a missing-name finding. The audit cannot determine whether the app supplied that name. When a control is the only child of a control with the same role, the child drops a finding only if the parent reports the same kind of finding. -Like `find`, `audit` publishes only the refs it returns. The text prints each as `@eN~s`; press that pinned ref. A plain ref from before the audit stays refused after the frame expired, and an audit with no findings issues no refs. An audit with findings also ends the snapshot's complete frame, so take a new snapshot before you use refs that the audit did not return. Each finding shows its role and includes the captured label when available. For example, a control may get its name from a child or placeholder while its own label is absent. An unnamed control shows its identifier when available. Guidance follows once per finding kind, with Apple HIG sizes on iOS and Material sizes on Android. `--json` returns `findings` (with `label` and `identifier` when available), `guidance` keyed by finding kind, `checks`, `limitations`, and `refsGeneration` (only with findings). `checks.snapshot` is `unknown` when the capture cannot report truncation, such as Appium page source. +Like `find`, `audit` publishes only the refs it returns. The text prints each as `@eN~s`; press that pinned ref. A plain ref from before the audit stays refused after the frame expired, and an audit with no findings issues no refs. An audit with findings also ends the snapshot's complete frame, so take a new snapshot before you use refs that the audit did not return. Each finding shows its role. Missing-name findings omit labels rejected by the name check and show the captured identifier when available. Other findings include the captured label when available. A control can get its name from a child or placeholder even when it has no label of its own. Guidance follows once per finding kind, with Apple HIG sizes on iOS and Material sizes on Android. `--json` returns `findings` (with `label` and `identifier` when available), `guidance` keyed by finding kind, `checks`, `limitations`, and `refsGeneration` (only with findings). `checks.snapshot` is `unknown` when the capture cannot report truncation, such as Appium page source. ## Wait and alerts From c70a9dd57c0a485d76a591360be06b8cfa625a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 18 Nov 2026 21:02:54 +0100 Subject: [PATCH 20/20] fix(occlusion): qualify persistent bar evidence by capture source --- .../src/snapshot-occlusion.test.ts | 74 +++++++---- .../capture-kit/src/snapshot-occlusion.ts | 13 +- packages/capture-kit/src/snapshot-state.ts | 5 +- .../snapshot/__tests__/snapshot-audit.test.ts | 19 ++- .../src/snapshot/snapshot-audit.ts | 4 +- .../src/commands/capture/runtime/snapshot.ts | 1 + .../runtime/__tests__/test-utils/index.ts | 120 +++++++++--------- packages/contracts/src/audit.ts | 2 + packages/contracts/src/snapshot-text.test.ts | 15 +-- packages/contracts/src/snapshot-text.ts | 57 ++++++--- .../__tests__/find-touch-runtime-fixture.ts | 4 +- .../internal/__tests__/find.test.ts | 10 +- 12 files changed, 194 insertions(+), 130 deletions(-) diff --git a/packages/capture-kit/src/snapshot-occlusion.test.ts b/packages/capture-kit/src/snapshot-occlusion.test.ts index c33bac4a7d..a83854e336 100644 --- a/packages/capture-kit/src/snapshot-occlusion.test.ts +++ b/packages/capture-kit/src/snapshot-occlusion.test.ts @@ -6,6 +6,7 @@ import { annotateCoveredSnapshotNodes, annotateSnapshotNodesCoveredByPolicy, } from './snapshot-occlusion.ts'; +import { buildSnapshotState } from './snapshot-state.ts'; // #1478 P5 codec-extraction regression: `annotateCoveredSnapshotNodes` marks // a touch candidate `interactionBlocked: 'covered'` when a later, floating @@ -258,6 +259,21 @@ test('annotateSnapshotNodesCoveredByPolicy publishes the cover a backend hit tes assert.deepEqual(result?.coveredBy, { kind: 'button', label: 'Top story', rect: card.rect }); }); +test('a labelled ARIA toolbar keeps its hit-test cover without claiming persistence', () => { + const rect = { x: 0, y: 100, width: 300, height: 60 }; + const target: RawSnapshotNode = { index: 0, type: 'button', rect, hittable: true }; + + const [result] = annotateSnapshotNodesCoveredByPolicy([target], () => ({ + type: 'toolbar', + role: 'toolbar', + label: 'Editing actions', + rect, + })); + + assert.equal(result?.interactionBlocked, 'covered'); + assert.deepEqual(result?.coveredBy, { kind: 'toolbar', label: 'Editing actions', rect }); +}); + test('annotateSnapshotNodesCoveredByPolicy names no cover when the hit element is an anonymous box', () => { const target: RawSnapshotNode = { index: 0, @@ -707,35 +723,39 @@ test('a full-viewport toolbar container does not cover its application siblings' assert.equal(annotated[1]?.interactionBlocked, undefined); }); -test('a full-viewport bottom navigation bar is not viewport chrome: it still covers and publishes as a persistent bar', () => { - // Bar identity (including bottom navigation) is owned by contracts; the viewport exemption is - // this scan's own choice of the three chrome kinds, so a viewport-sized BottomNavigationView - // keeps covering the content behind it instead of being exempted as a chrome container. - const application: RawSnapshotNode = { - index: 0, - type: 'application', - rect: { x: 0, y: 0, width: 400, height: 800 }, - }; - const target: RawSnapshotNode = { - index: 1, - type: 'button', - role: 'button', - parentIndex: 0, - label: 'Room', - rect: { x: 20, y: 100, width: 120, height: 50 }, - }; - const bottomNavigation: RawSnapshotNode = { - index: 2, +test.each([ + { type: 'com.google.android.material.bottomnavigation.BottomNavigationView', - parentIndex: 0, - rect: { x: 0, y: 0, width: 400, height: 800 }, - }; - - const annotated = annotateCoveredSnapshotNodes([application, target, bottomNavigation]); + cover: { kind: 'bottom-navigation', persistentBar: true }, + }, + { + type: 'com.example.BottomNavigationView', + cover: { kind: 'bottomnavigationview' }, + }, +])('Android snapshot assembly preserves qualified cover identity for $type', ({ type, cover }) => { + const rect = { x: 0, y: 0, width: 400, height: 800 }; + const state = buildSnapshotState( + { + backend: 'android', + producer: 'android-uiautomator', + nodes: [ + { index: 0, type: 'application', rect }, + { + index: 1, + type: 'button', + role: 'button', + parentIndex: 0, + label: 'Room', + rect: { x: 20, y: 100, width: 120, height: 50 }, + }, + { index: 2, type, parentIndex: 0, label: 'Actions', rect }, + ], + }, + undefined, + ); - assert.equal(annotated[1]?.interactionBlocked, 'covered'); - assert.equal(annotated[1]?.coveredBy?.persistentBar, true); - assert.equal(annotated[1]?.coveredBy?.kind, 'bottom-navigation'); + assert.equal(state.nodes[1]?.interactionBlocked, 'covered'); + assert.deepEqual(state.nodes[1]?.coveredBy, { ...cover, label: 'Actions', rect }); }); test.for([ diff --git a/packages/capture-kit/src/snapshot-occlusion.ts b/packages/capture-kit/src/snapshot-occlusion.ts index ea76d4ec44..3b17b8c32a 100644 --- a/packages/capture-kit/src/snapshot-occlusion.ts +++ b/packages/capture-kit/src/snapshot-occlusion.ts @@ -8,6 +8,7 @@ import { areRectsApproximatelyEqual, normalizeRect } from '@agent-device/kernel/ import { containsPoint, rectContains } from '@agent-device/kernel/rect'; import { isScrollableNodeLike, + type SnapshotProducer, snapshotBarKind, snapshotNodeKindIncludesAny as nodeKindIncludesAny, isViewportRootNode, @@ -97,6 +98,7 @@ function collectNeighbourhoodIndex(nodes: readonly RawSnapshotNode[]): Neighbour } export type SnapshotOcclusionOptions = { + producer?: SnapshotProducer; /** Backend-declared overlay roots that use the shared geometric scan (for example Android IME). */ isAdditionalOverlayNode?: (node: RawSnapshotNode) => boolean; }; @@ -133,7 +135,10 @@ export function annotateCoveredSnapshotNodes( if (!isCandidateTouchNode(node)) continue; const coveringNode = findCoveringNode(scan, position, node, options); if (!coveringNode) continue; - const coveredBy = describeCoveringNode(coveringNode); + const coveredBy = describeCoveringNode( + coveringNode, + snapshotBarKind(coveringNode, options.producer), + ); coveredPositions.push(coveredBy ? { position, coveredBy } : { position }); } return annotateCoveredPositions(nodes, coveredPositions); @@ -171,10 +176,12 @@ function isViewportChromeBarNode( * still describes itself with no label to lean on. Persistent bars always publish their canonical * bar kind, including when the type is generic and only the role or subrole identifies the bar. */ -function describeCoveringNode(node: CoveringNodeEvidence): SnapshotCoveringNode | null { +function describeCoveringNode( + node: CoveringNodeEvidence, + barKind?: ReturnType, +): SnapshotCoveringNode | null { const label = node.label?.trim(); const typeKind = formatRole(node.type ?? 'Element'); - const barKind = snapshotBarKind(node); const anonymous = ANONYMOUS_CONTAINER_KINDS.has(typeKind); if (!label && !barKind && anonymous) return null; const rect = positiveRect(node.rect); diff --git a/packages/capture-kit/src/snapshot-state.ts b/packages/capture-kit/src/snapshot-state.ts index 044b2b3b2c..a03693b917 100644 --- a/packages/capture-kit/src/snapshot-state.ts +++ b/packages/capture-kit/src/snapshot-state.ts @@ -109,7 +109,10 @@ export function buildSnapshotState( snapshotRaw ? scopedNodes : annotateUnmeasuredSnapshotNodes( - annotateCoveredSnapshotNodes(scopedNodes, overlayOptionsForBackend(data?.backend)), + annotateCoveredSnapshotNodes(scopedNodes, { + ...overlayOptionsForBackend(data?.backend), + producer: data?.producer, + }), ), snapshotRefMinting(data), ); diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts index acf0bbaa85..4bca0227d5 100644 --- a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts @@ -4,7 +4,7 @@ import type { RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapsho import { annotateCoveredSnapshotNodes } from '../../snapshot-occlusion.ts'; import { auditSnapshot } from '../snapshot-audit.ts'; -const IOS: AuditContext = { platform: 'ios', truncated: false }; +const IOS: AuditContext = { platform: 'ios', producer: 'apple-runner', truncated: false }; const ANDROID: AuditContext = { platform: 'android', pixelDensity: 2, truncated: false }; function node(index: number, fields: Partial = {}): SnapshotNode { @@ -396,7 +396,9 @@ function scannedCoverTree(cover: CoverPublication): SnapshotNode[] { rect: { x: 0, y: 791, width: 402, height: 83 }, }, ]; - return annotateCoveredSnapshotNodes(raw).map((scanned) => ({ + return annotateCoveredSnapshotNodes(raw, { + producer: cover.type.startsWith('com.google.android.') ? 'android-uiautomator' : 'apple-runner', + }).map((scanned) => ({ ...scanned, ref: `e${scanned.index}`, })); @@ -453,10 +455,12 @@ test('keeps the covered-control finding for a button in a horizontal scroller wh rect: { x: 0, y: 725, width: 400, height: 75 }, }, ]; - const nodes = annotateCoveredSnapshotNodes(raw).map((scanned) => ({ - ...scanned, - ref: `e${scanned.index}`, - })); + const nodes = annotateCoveredSnapshotNodes(raw, { producer: 'android-uiautomator' }).map( + (scanned) => ({ + ...scanned, + ref: `e${scanned.index}`, + }), + ); expect(nodes[2]).toMatchObject({ interactionBlocked: 'covered', coveredBy: { persistentBar: true }, @@ -578,6 +582,9 @@ test.each([{ role: 'UITabBar' }, { subrole: 'UIToolbar' }, { subrole: 'AXToolbar }); const parent = node(0, { type: 'Other', hittable: false, ...barClass }); expect(audit([parent, item]).findings).toEqual([]); + expect( + summarize(audit([parent, item], { ...IOS, producer: 'appium-source' }).findings), + ).toEqual(['small-target @e1']); expect(summarize(audit([parent, { ...item, label: undefined }]).findings)).toEqual([ 'missing-name @e1', ]); diff --git a/packages/capture-kit/src/snapshot/snapshot-audit.ts b/packages/capture-kit/src/snapshot/snapshot-audit.ts index 78d40e6d47..9f6af648a4 100644 --- a/packages/capture-kit/src/snapshot/snapshot-audit.ts +++ b/packages/capture-kit/src/snapshot/snapshot-audit.ts @@ -50,6 +50,7 @@ type SizeRule = type AuditScan = { platform: AuditContext['platform']; + producer: AuditContext['producer']; byIndex: ReadonlyMap; children: ChildrenByParent; }; @@ -61,6 +62,7 @@ type AuditScan = { export function auditSnapshot(nodes: readonly SnapshotNode[], context: AuditContext): AuditResult { const scan: AuditScan = { platform: context.platform, + producer: context.producer, byIndex: new Map(nodes.map((node) => [node.index, node])), children: buildChildrenByParent(nodes), }; @@ -137,7 +139,7 @@ function controlOwnership(node: SnapshotNode, scan: AuditScan): ControlOwnership if (scan.platform !== 'ios') return 'app'; if (node.role === '_UIGrabber' || node.subrole === '_UIGrabber') return 'system'; const parent = parentOf(node, scan); - return parent && isNativeIosSnapshotBarNode(parent) ? 'platform-sized' : 'app'; + return parent && isNativeIosSnapshotBarNode(parent, scan.producer) ? 'platform-sized' : 'app'; } function isActionable(node: SnapshotNode, ownership: ControlOwnership): boolean { diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts index d1c76bc7a4..69723bcd96 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts @@ -197,6 +197,7 @@ export const auditSnapshotCommand: RuntimeCommand< const capture = await captureRuntimeSnapshot(runtime, options); const audit = auditSnapshot(capture.snapshot.nodes, { platform: runtime.backend.platform, + producer: capture.snapshot.producer, target: options.target, pixelDensity: capture.annotations.androidSnapshot?.pixelDensity, truncated: snapshotTruncationForResult(capture.snapshot), diff --git a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts index c7b0f8168e..b965fe6f37 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts @@ -191,33 +191,36 @@ export function coveredByTabBarSnapshot(): SnapshotState { */ export function coveredByScanSnapshot(coverType = 'TabBar'): SnapshotState { return makeSnapshotState( - annotateCoveredSnapshotNodes([ - { - index: 0, - depth: 0, - type: 'Application', - label: 'Example', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Save draft', - rect: { x: 16, y: 790, width: 140, height: 44 }, - hittable: true, - }, - { - index: 2, - depth: 1, - parentIndex: 0, - type: coverType, - label: 'Media', - rect: { x: 0, y: 760, width: 390, height: 84 }, - hittable: true, - }, - ]), + annotateCoveredSnapshotNodes( + [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Save draft', + rect: { x: 16, y: 790, width: 140, height: 44 }, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: coverType, + label: 'Media', + rect: { x: 0, y: 760, width: 390, height: 84 }, + hittable: true, + }, + ], + { producer: 'apple-runner' }, + ), ); } @@ -248,36 +251,39 @@ export function coveredByAnonymousTabBarScanSnapshot( : []; const buttonIndex = scroller.length + 1; return makeSnapshotState( - annotateCoveredSnapshotNodes([ - { - index: 0, - depth: 0, - type: 'Application', - label: 'Example', - rect: { x: 0, y: 0, width: 402, height: 874 }, - }, - ...scroller, - { - index: buttonIndex, - depth: scroller.length + 1, - parentIndex: scroller.length > 0 ? 1 : 0, - type: 'Button', - label: 'Save draft', - rect: options.horizontal - ? { x: 16, y: 810, width: 140, height: 44 } - : { x: 16, y: 800, width: 140, height: 44 }, - hittable: true, - }, - { - index: buttonIndex + 1, - depth: 1, - parentIndex: 0, - type: 'Other', - role: 'UITabBar', - rect: { x: 0, y: 791, width: 402, height: 83 }, - hittable: true, - }, - ]), + annotateCoveredSnapshotNodes( + [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + ...scroller, + { + index: buttonIndex, + depth: scroller.length + 1, + parentIndex: scroller.length > 0 ? 1 : 0, + type: 'Button', + label: 'Save draft', + rect: options.horizontal + ? { x: 16, y: 810, width: 140, height: 44 } + : { x: 16, y: 800, width: 140, height: 44 }, + hittable: true, + }, + { + index: buttonIndex + 1, + depth: 1, + parentIndex: 0, + type: 'Other', + role: 'UITabBar', + rect: { x: 0, y: 791, width: 402, height: 83 }, + hittable: true, + }, + ], + { producer: 'apple-runner' }, + ), ); } diff --git a/packages/contracts/src/audit.ts b/packages/contracts/src/audit.ts index 1ad5be8403..d92673eef7 100644 --- a/packages/contracts/src/audit.ts +++ b/packages/contracts/src/audit.ts @@ -1,5 +1,6 @@ import type { DeviceTarget, PublicPlatform } from '@agent-device/kernel/device'; import type { Rect } from '@agent-device/kernel/snapshot'; +import type { SnapshotProducer } from './snapshot-producers.ts'; export type AuditFindingKind = 'small-target' | 'missing-name' | 'covered-control'; @@ -36,6 +37,7 @@ export type AuditFinding = export type AuditContext = { platform: PublicPlatform; + producer?: SnapshotProducer; /** Absent means a mobile target. */ target?: DeviceTarget; /** Physical pixels per dp, from the Android capture. */ diff --git a/packages/contracts/src/snapshot-text.test.ts b/packages/contracts/src/snapshot-text.test.ts index b68a658bc6..0a64fe7b69 100644 --- a/packages/contracts/src/snapshot-text.test.ts +++ b/packages/contracts/src/snapshot-text.test.ts @@ -50,14 +50,11 @@ test.each([ node: { type: 'com.google.android.material.bottomnavigation.BottomNavigationView' }, kind: 'bottom-navigation', }, - { - name: 'a bottom navigation bar named in role only', - node: { type: 'Other', role: 'BottomNavigationBar' }, - kind: 'bottom-navigation', - }, ])('snapshotBarKind reads $name', ({ node, kind }) => { - expect(snapshotBarKind(node)).toBe(kind); - expect(isNativeIosSnapshotBarNode(node)).toBe(kind !== 'bottom-navigation'); + const producer = kind === 'bottom-navigation' ? 'android-uiautomator' : 'apple-runner'; + expect(snapshotBarKind(node, producer)).toBe(kind); + expect(isNativeIosSnapshotBarNode(node, producer)).toBe(kind !== 'bottom-navigation'); + expect(snapshotBarKind(node, 'cdp-dom')).toBeUndefined(); }); test.each([ @@ -65,6 +62,6 @@ test.each([ { type: 'HomeToolbarView' }, { type: 'Other', role: 'BottomTabBar' }, ])('a non-bar node reads no bar kind for %j', (node) => { - expect(snapshotBarKind(node)).toBeUndefined(); - expect(isNativeIosSnapshotBarNode(node)).toBe(false); + expect(snapshotBarKind(node, 'apple-runner')).toBeUndefined(); + expect(isNativeIosSnapshotBarNode(node, 'apple-runner')).toBe(false); }); diff --git a/packages/contracts/src/snapshot-text.ts b/packages/contracts/src/snapshot-text.ts index 0e13b8dc4c..e567da3792 100644 --- a/packages/contracts/src/snapshot-text.ts +++ b/packages/contracts/src/snapshot-text.ts @@ -1,5 +1,6 @@ import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import type { SnapshotProducer } from './snapshot-producers.ts'; export function normalizeType(type: string): string { let normalized = type @@ -14,33 +15,55 @@ export function normalizeType(type: string): string { return normalized; } -const SNAPSHOT_BAR_KINDS = [ - ['bottomnavigation', 'bottom-navigation'], - ['bottomnavigationbar', 'bottom-navigation'], - ['bottomnavigationview', 'bottom-navigation'], - ['tabbar', 'tab-bar'], - ['toolbar', 'toolbar'], - ['navigationbar', 'navigation-bar'], +const APPLE_BAR_KINDS = [ + ['TabBar', 'tab-bar'], + ['Toolbar', 'toolbar'], + ['NavigationBar', 'navigation-bar'], ] as const; -type SnapshotBarKind = (typeof SNAPSHOT_BAR_KINDS)[number][1]; +const ANDROID_BAR_KINDS = [ + ['com.google.android.material.bottomnavigation.BottomNavigationView', 'bottom-navigation'], + ['android.widget.Toolbar', 'toolbar'], + ['androidx.appcompat.widget.Toolbar', 'toolbar'], +] as const; + +type SnapshotBarNode = Pick; +type SnapshotBarKind = (typeof APPLE_BAR_KINDS | typeof ANDROID_BAR_KINDS)[number][1]; -/** Reads an exact bar identity from the captured type, role, or subrole. */ +/** Reads native bar identity only from captures that publish native type or class names. */ export function snapshotBarKind( - node: Pick, + node: SnapshotBarNode, + producer: SnapshotProducer | undefined, ): SnapshotBarKind | undefined { - const names = [node.type, node.role, node.subrole].map((value) => - normalizeType(value ?? '').replace(/^ui/, ''), - ); - return SNAPSHOT_BAR_KINDS.find(([name]) => names.includes(name))?.[1]; + if ( + producer === 'apple-runner' || + producer === 'simulator-ax-bridge' || + producer === 'macos-helper' + ) { + const names = [node.type, node.role, node.subrole].map((name) => + name + ?.trim() + .replace(/^UIKit\./, '') + .replace(/^(?:XCUIElementType|UI|AX)/, ''), + ); + return APPLE_BAR_KINDS.find(([name]) => names.includes(name))?.[1]; + } + if (producer === 'android-uiautomator') { + const names = [node.type, node.role, node.subrole]; + return ANDROID_BAR_KINDS.find(([name]) => names.includes(name))?.[1]; + } + return undefined; } /** Identifies native iOS bars whose direct items use platform sizing. */ export function isNativeIosSnapshotBarNode( - node: Pick, + node: SnapshotBarNode, + producer: SnapshotProducer | undefined, ): boolean { - const kind = snapshotBarKind(node); - return kind !== undefined && kind !== 'bottom-navigation'; + return ( + (producer === 'apple-runner' || producer === 'simulator-ax-bridge') && + snapshotBarKind(node, producer) !== undefined + ); } export function snapshotNodeKindIncludesAny( diff --git a/packages/daemon/src/execution/interaction/internal/__tests__/find-touch-runtime-fixture.ts b/packages/daemon/src/execution/interaction/internal/__tests__/find-touch-runtime-fixture.ts index 7d10afdb83..678e40fa0b 100644 --- a/packages/daemon/src/execution/interaction/internal/__tests__/find-touch-runtime-fixture.ts +++ b/packages/daemon/src/execution/interaction/internal/__tests__/find-touch-runtime-fixture.ts @@ -1,3 +1,4 @@ +import type { SnapshotCaptureProvenance } from '@agent-device/contracts/snapshot'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; import { expect } from 'vitest'; import { legacyDispatchCapture } from '../../../../__tests__/support/legacy-snapshot-capture-fixture.ts'; @@ -60,6 +61,7 @@ export function resetFindTouchRuntimeFixture(): void { export async function runFindClickScenario(options: { positionals: string[]; nodes?: Array>; + snapshotProvenance?: SnapshotCaptureProvenance; flags?: DaemonRequest['flags']; session?: SessionState; createCommandSurface?: CreateDaemonCommandSurface; @@ -77,7 +79,7 @@ export async function runFindClickScenario(options: { if (options.nodes !== undefined) { mockDispatch.mockImplementation(async (_device, command) => { if (command === 'snapshot') { - return { nodes: options.nodes }; + return { nodes: options.nodes, ...options.snapshotProvenance }; } return {}; }); diff --git a/packages/daemon/src/execution/interaction/internal/__tests__/find.test.ts b/packages/daemon/src/execution/interaction/internal/__tests__/find.test.ts index d26719f703..8b524a2a6c 100644 --- a/packages/daemon/src/execution/interaction/internal/__tests__/find.test.ts +++ b/packages/daemon/src/execution/interaction/internal/__tests__/find.test.ts @@ -509,6 +509,7 @@ test('handleFindCommands focus uses the promoted actionable node center', async test('handleFindCommands focus rejects covered matches before dispatching coordinates', async () => { const { response } = await runFindClickScenario({ positionals: ['Save draft', 'focus'], + snapshotProvenance: { backend: 'xctest', producer: 'apple-runner' }, nodes: [ { index: 0, @@ -521,14 +522,7 @@ test('handleFindCommands focus rejects covered matches before dispatching coordi ref: 'e2', type: 'Button', label: 'Save draft', - hittable: false, - interactionBlocked: 'covered', - coveredBy: { - kind: 'tab-bar', - label: 'Media', - rect: { x: 0, y: 760, width: 390, height: 84 }, - }, - presentationHints: ['covered'], + hittable: true, rect: { x: 16, y: 790, width: 140, height: 44 }, parentIndex: 0, },