diff --git a/packages/capture-kit/src/snapshot-occlusion.test.ts b/packages/capture-kit/src/snapshot-occlusion.test.ts index 69a0589a72..a83854e336 100644 --- a/packages/capture-kit/src/snapshot-occlusion.test.ts +++ b/packages/capture-kit/src/snapshot-occlusion.test.ts @@ -6,6 +6,7 @@ import { annotateCoveredSnapshotNodes, annotateSnapshotNodesCoveredByPolicy, } from './snapshot-occlusion.ts'; +import { buildSnapshotState } from './snapshot-state.ts'; // #1478 P5 codec-extraction regression: `annotateCoveredSnapshotNodes` marks // a touch candidate `interactionBlocked: 'covered'` when a later, floating @@ -258,6 +259,21 @@ test('annotateSnapshotNodesCoveredByPolicy publishes the cover a backend hit tes assert.deepEqual(result?.coveredBy, { kind: 'button', label: 'Top story', rect: card.rect }); }); +test('a labelled ARIA toolbar keeps its hit-test cover without claiming persistence', () => { + const rect = { x: 0, y: 100, width: 300, height: 60 }; + const target: RawSnapshotNode = { index: 0, type: 'button', rect, hittable: true }; + + const [result] = annotateSnapshotNodesCoveredByPolicy([target], () => ({ + type: 'toolbar', + role: 'toolbar', + label: 'Editing actions', + rect, + })); + + assert.equal(result?.interactionBlocked, 'covered'); + assert.deepEqual(result?.coveredBy, { kind: 'toolbar', label: 'Editing actions', rect }); +}); + test('annotateSnapshotNodesCoveredByPolicy names no cover when the hit element is an anonymous box', () => { const target: RawSnapshotNode = { index: 0, @@ -707,6 +723,41 @@ test('a full-viewport toolbar container does not cover its application siblings' assert.equal(annotated[1]?.interactionBlocked, undefined); }); +test.each([ + { + type: 'com.google.android.material.bottomnavigation.BottomNavigationView', + cover: { kind: 'bottom-navigation', persistentBar: true }, + }, + { + type: 'com.example.BottomNavigationView', + cover: { kind: 'bottomnavigationview' }, + }, +])('Android snapshot assembly preserves qualified cover identity for $type', ({ type, cover }) => { + const rect = { x: 0, y: 0, width: 400, height: 800 }; + const state = buildSnapshotState( + { + backend: 'android', + producer: 'android-uiautomator', + nodes: [ + { index: 0, type: 'application', rect }, + { + index: 1, + type: 'button', + role: 'button', + parentIndex: 0, + label: 'Room', + rect: { x: 20, y: 100, width: 120, height: 50 }, + }, + { index: 2, type, parentIndex: 0, label: 'Actions', rect }, + ], + }, + undefined, + ); + + assert.equal(state.nodes[1]?.interactionBlocked, 'covered'); + assert.deepEqual(state.nodes[1]?.coveredBy, { ...cover, label: 'Actions', rect }); +}); + test.for([ // #3883: on iOS 27 an app's 2pt top bar pushes the floating-bar container to y=2. { diff --git a/packages/capture-kit/src/snapshot-occlusion.ts b/packages/capture-kit/src/snapshot-occlusion.ts index 822f352067..3b17b8c32a 100644 --- a/packages/capture-kit/src/snapshot-occlusion.ts +++ b/packages/capture-kit/src/snapshot-occlusion.ts @@ -8,7 +8,9 @@ import { areRectsApproximatelyEqual, normalizeRect } from '@agent-device/kernel/ import { containsPoint, rectContains } from '@agent-device/kernel/rect'; import { isScrollableNodeLike, - normalizeType, + type SnapshotProducer, + snapshotBarKind, + snapshotNodeKindIncludesAny as nodeKindIncludesAny, isViewportRootNode, } from '@agent-device/contracts/snapshot'; @@ -25,7 +27,6 @@ const OVERLAY_KIND_FRAGMENTS = [ 'popover', 'menu', ]; -const VIEWPORT_CHROME_KIND_FRAGMENTS = ['tabbar', 'toolbar', 'navigationbar']; const FLOATING_BAR_CONTAINER_ROLE = '_UIFloatingBarContainerView'; /** * How far below the viewport top a full-viewport chrome container may start, before the 0.5pt rect @@ -97,6 +98,7 @@ function collectNeighbourhoodIndex(nodes: readonly RawSnapshotNode[]): Neighbour } export type SnapshotOcclusionOptions = { + producer?: SnapshotProducer; /** Backend-declared overlay roots that use the shared geometric scan (for example Android IME). */ isAdditionalOverlayNode?: (node: RawSnapshotNode) => boolean; }; @@ -133,7 +135,10 @@ export function annotateCoveredSnapshotNodes( if (!isCandidateTouchNode(node)) continue; const coveringNode = findCoveringNode(scan, position, node, options); if (!coveringNode) continue; - const coveredBy = describeCoveringNode(coveringNode); + const coveredBy = describeCoveringNode( + coveringNode, + snapshotBarKind(coveringNode, options.producer), + ); coveredPositions.push(coveredBy ? { position, coveredBy } : { position }); } return annotateCoveredPositions(nodes, coveredPositions); @@ -142,6 +147,22 @@ export function annotateCoveredSnapshotNodes( /** Presented roles that name a box rather than a surface a reader would recognize and dismiss. */ const ANONYMOUS_CONTAINER_KINDS = new Set(['element', 'group', 'other']); +const VIEWPORT_CHROME_KIND_FRAGMENTS = [ + ...TAB_BAR_KIND_FRAGMENTS, + ...TOOLBAR_HOST_KIND_FRAGMENTS, + ...NAVIGATION_BAR_KIND_FRAGMENTS, +]; + +/** Viewport hosts can expose internal classes such as UIKit tab platters. */ +function isViewportChromeBarNode( + node: Pick, +): boolean { + return ( + !nodeKindIncludesAny(node, ['bottomnavigation']) && + nodeKindIncludesAny(node, VIEWPORT_CHROME_KIND_FRAGMENTS) + ); +} + /** * The bounded, non-actionable description of the cover the scan found, or nothing when the cover has * no name. No ref is ever carried — see {@link SnapshotCoveringNode}. @@ -152,22 +173,31 @@ const ANONYMOUS_CONTAINER_KINDS = new Set(['element', 'group', 'other']); * the surface to dismiss. Strip that label and the same node would say `the group at (...)`, which * points at a box nobody would recognize; an anonymous container of that shape therefore publishes * no cover, and the refusal keeps its generic recovery. A named kind (`dialog`, `tab-bar`, `sheet`) - * still describes itself with no label to lean on. + * still describes itself with no label to lean on. Persistent bars always publish their canonical + * bar kind, including when the type is generic and only the role or subrole identifies the bar. */ -function describeCoveringNode(node: CoveringNodeEvidence): SnapshotCoveringNode | null { +function describeCoveringNode( + node: CoveringNodeEvidence, + barKind?: ReturnType, +): SnapshotCoveringNode | null { const label = node.label?.trim(); - const kind = formatRole(node.type ?? 'Element'); - if (!label && ANONYMOUS_CONTAINER_KINDS.has(kind)) return null; + const typeKind = formatRole(node.type ?? 'Element'); + const anonymous = ANONYMOUS_CONTAINER_KINDS.has(typeKind); + if (!label && !barKind && anonymous) return null; const rect = positiveRect(node.rect); return { - kind, + kind: barKind ?? typeKind, ...(label ? { label: label.slice(0, SNAPSHOT_COVERING_LABEL_MAX_LENGTH) } : {}), ...(rect ? { rect } : {}), + ...(barKind ? { persistentBar: true } : {}), }; } /** What a backend hit test observed on top of a node, described like a scanned cover. */ -export type CoveringNodeEvidence = Pick; +export type CoveringNodeEvidence = Pick< + RawSnapshotNode, + 'type' | 'role' | 'subrole' | 'label' | 'rect' +>; /** * Applies backend-owned covered decisions through the same publication annotation contract. A @@ -380,7 +410,7 @@ function isSiblingSizedChromeContainer( node: RawSnapshotNode, neighbourhood: NeighbourhoodIndex, ): boolean { - if (!nodeKindIncludesAny(node, VIEWPORT_CHROME_KIND_FRAGMENTS)) return false; + if (!isViewportChromeBarNode(node)) return false; if (typeof node.parentIndex !== 'number') return false; const rect = positiveRect(node.rect); if (!rect) return false; @@ -539,7 +569,7 @@ function isNavigationBarAround(bar: RawSnapshotNode, rect: Rect): boolean { /** A bar or a scroll container inside `rect`; a scroll indicator is not a container. */ function isEnclosedHostedRegion(rect: Rect, node: RawSnapshotNode): boolean { const isHostedRegion = - nodeKindIncludesAny(node, VIEWPORT_CHROME_KIND_FRAGMENTS) || + isViewportChromeBarNode(node) || (isScrollableNodeLike(node) && !nodeKindIncludesAny(node, SCROLL_INDICATOR_KIND_FRAGMENTS)); const nodeRect = positiveRect(node.rect); return Boolean(isHostedRegion && nodeRect && rectContains(rect, nodeRect)); @@ -549,7 +579,7 @@ function isFullViewportChromeContainer( node: RawSnapshotNode, neighbourhood: NeighbourhoodIndex, ): boolean { - if (!nodeKindIncludesAny(node, VIEWPORT_CHROME_KIND_FRAGMENTS)) return false; + if (!isViewportChromeBarNode(node)) return false; const rect = positiveRect(node.rect); if (!rect) return false; @@ -665,24 +695,6 @@ function isSemanticTouchNode(node: RawSnapshotNode): boolean { return nodeKindIncludesAny(node, SEMANTIC_TOUCH_KIND_FRAGMENTS); } -function nodeKindIncludesAny( - node: Pick, - fragments: readonly string[], -): boolean { - const normalized = normalizeNodeKind(node); - return fragments.some((fragment) => normalized.includes(fragment)); -} - -// Mutation-lane note: the `?? ''` fallback's exact replacement text is not -// observable through this module's public API — every caller only checks -// substring membership against a fixed, known fragment list, and no -// plausible filler text coincides with any of them, so no test can -// distinguish `''` from another non-matching filler here without reaching -// into this private function directly. -function normalizeNodeKind(node: Pick): string { - return [node.type, node.role, node.subrole].map((value) => normalizeType(value ?? '')).join(' '); -} - function areRelatedSnapshotNodes( left: RawSnapshotNode, right: RawSnapshotNode, diff --git a/packages/capture-kit/src/snapshot-state.ts b/packages/capture-kit/src/snapshot-state.ts index 044b2b3b2c..a03693b917 100644 --- a/packages/capture-kit/src/snapshot-state.ts +++ b/packages/capture-kit/src/snapshot-state.ts @@ -109,7 +109,10 @@ export function buildSnapshotState( snapshotRaw ? scopedNodes : annotateUnmeasuredSnapshotNodes( - annotateCoveredSnapshotNodes(scopedNodes, overlayOptionsForBackend(data?.backend)), + annotateCoveredSnapshotNodes(scopedNodes, { + ...overlayOptionsForBackend(data?.backend), + producer: data?.producer, + }), ), snapshotRefMinting(data), ); diff --git a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts index 2b864f0f31..4bca0227d5 100644 --- a/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/snapshot-audit.test.ts @@ -1,9 +1,10 @@ import { describe, expect, test } from 'vitest'; import type { AuditContext, AuditFinding } from '@agent-device/contracts/capture'; -import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot'; +import { annotateCoveredSnapshotNodes } from '../../snapshot-occlusion.ts'; import { auditSnapshot } from '../snapshot-audit.ts'; -const IOS: AuditContext = { platform: 'ios', truncated: false }; +const IOS: AuditContext = { platform: 'ios', producer: 'apple-runner', truncated: false }; const ANDROID: AuditContext = { platform: 'android', pixelDensity: 2, truncated: false }; function node(index: number, fields: Partial = {}): SnapshotNode { @@ -321,3 +322,274 @@ describe('scope', () => { expect(result.limitations).toEqual([expect.stringContaining('web rects')]); }); }); + +test('publishes the captured label and identifier but omits a missing-name label', () => { + const findings = audit([ + node(1, { label: 'Save Lemon Herb Rice', rect: { x: 0, y: 0, width: 22, height: 22 } }), + node(2, { identifier: 'save', label: 'com.example:id/save' }), + node(3, { label: 'Pay', interactionBlocked: 'covered', hittable: false }), + ]).findings; + expect(findings[0]).toMatchObject({ label: 'Save Lemon Herb Rice' }); + expect(findings[1]).toMatchObject({ kind: 'missing-name', identifier: 'save' }); + expect(findings[1]).not.toHaveProperty('label'); + expect(findings[2]).toMatchObject({ label: 'Pay' }); +}); + +type CoverPublication = { + name: string; + type: string; + role?: string; + subrole?: string; + label?: string; +}; + +/** + * Window > screen holding a scroll view (one row inside) and a fixed control, with the cover listed + * last so it draws on top. The row and the fixed control both sit under the cover. The occlusion scan + * produces each `coveredBy`, so the audit reads exactly what a capture publishes. + */ +function scannedCoverTree(cover: CoverPublication): SnapshotNode[] { + const raw: RawSnapshotNode[] = [ + { index: 0, depth: 0, type: 'Window', rect: { x: 0, y: 0, width: 402, height: 874 } }, + { + index: 1, + parentIndex: 0, + depth: 1, + type: 'Other', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 2, + parentIndex: 1, + depth: 2, + type: 'ScrollView', + hiddenContentBelow: true, + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 3, + parentIndex: 2, + depth: 3, + type: 'Button', + label: 'Scrolled row', + hittable: true, + rect: { x: 0, y: 800, width: 402, height: 60 }, + }, + { + index: 4, + parentIndex: 1, + depth: 2, + type: 'Button', + label: 'Fixed control', + hittable: true, + rect: { x: 300, y: 800, width: 60, height: 60 }, + }, + { + index: 5, + parentIndex: 1, + depth: 2, + type: cover.type, + ...(cover.role ? { role: cover.role } : {}), + ...(cover.subrole ? { subrole: cover.subrole } : {}), + ...(cover.label ? { label: cover.label } : {}), + hittable: true, + rect: { x: 0, y: 791, width: 402, height: 83 }, + }, + ]; + return annotateCoveredSnapshotNodes(raw, { + producer: cover.type.startsWith('com.google.android.') ? 'android-uiautomator' : 'apple-runner', + }).map((scanned) => ({ + ...scanned, + ref: `e${scanned.index}`, + })); +} + +test.for([ + { name: 'type only', type: 'TabBar' }, + { name: 'navigation bar type', type: 'NavigationBar', label: 'Inbox' }, + { name: 'toolbar type', type: 'Toolbar' }, + { name: 'UIKit class in role beside a generic type', type: 'Other', role: 'UITabBar' }, + { name: 'subrole only', type: 'Other', subrole: 'AXToolbar' }, + { + name: 'Android bottom navigation', + type: 'com.google.android.material.bottomnavigation.BottomNavigationView', + }, +])('reports both covered controls under a scanned bar from the $name publication', (cover) => { + const nodes = scannedCoverTree(cover); + expect(nodes[3]).toMatchObject({ + interactionBlocked: 'covered', + coveredBy: { persistentBar: true }, + }); + expect(nodes[4]?.coveredBy).toMatchObject({ persistentBar: true }); + expect(summarize(audit(nodes).findings)).toEqual(['covered-control @e3', 'covered-control @e4']); +}); + +test('keeps the covered-control finding for a button in a horizontal scroller wholly behind a bottom bar', () => { + // The bar contains the scroller's whole usable viewport (400x65 inside the 400x75 bar), so no + // scroll can reveal the button: the audit keeps the finding instead of excluding it as scroll + // chrome. The occlusion scan produces the `coveredBy`, as in a real capture. + const raw: RawSnapshotNode[] = [ + { index: 0, depth: 0, type: 'Window', rect: { x: 0, y: 0, width: 400, height: 800 } }, + { + index: 1, + parentIndex: 0, + depth: 1, + type: 'android.widget.HorizontalScrollView', + rect: { x: 0, y: 725, width: 400, height: 65 }, + }, + { + index: 2, + parentIndex: 1, + depth: 2, + type: 'Button', + label: 'Buy now', + hittable: true, + rect: { x: 20, y: 735, width: 120, height: 50 }, + }, + { + index: 3, + parentIndex: 0, + depth: 1, + type: 'com.google.android.material.bottomnavigation.BottomNavigationView', + hittable: true, + rect: { x: 0, y: 725, width: 400, height: 75 }, + }, + ]; + const nodes = annotateCoveredSnapshotNodes(raw, { producer: 'android-uiautomator' }).map( + (scanned) => ({ + ...scanned, + ref: `e${scanned.index}`, + }), + ); + expect(nodes[2]).toMatchObject({ + interactionBlocked: 'covered', + coveredBy: { persistentBar: true }, + }); + expect(summarize(audit(nodes, ANDROID).findings)).toEqual(['covered-control @e2']); +}); + +test.for([ + { name: 'sheet', type: 'Sheet' }, + { name: 'dialog', type: 'Dialog', label: 'Tab Bar' }, + { name: 'labelled alert group', type: 'Alert', label: 'Confirm' }, +])('retains a scanned $name over a control inside a scroll container', (cover) => { + const nodes = scannedCoverTree(cover); + expect(nodes[3]).toMatchObject({ interactionBlocked: 'covered' }); + expect(nodes[3]?.coveredBy?.persistentBar).toBeUndefined(); + expect(summarize(audit(nodes).findings)).toEqual(['covered-control @e3', 'covered-control @e4']); +}); + +test.each([ + [IOS, 'Apple HIG recommends 44×44pt.'], + [ANDROID, 'Material recommends 48×48dp.'], +] as const)('returns guidance only for present kinds on %j', (context, size) => { + const result = audit( + [ + node(1, { label: 'Save', rect: { x: 0, y: 0, width: 22, height: 22 } }), + node(2, { label: 'Save again', rect: { x: 0, y: 0, width: 22, height: 22 } }), + node(3), + node(4, { interactionBlocked: 'covered', hittable: false }), + ], + context, + ); + expect(result.guidance).toEqual({ + 'small-target': `${size} Enlarge the touch area with padding, a minimum size, or hitSlop.`, + 'missing-name': + 'Set an accessibility label (iOS accessibilityLabel, Android contentDescription, React Native accessibilityLabel).', + 'covered-control': + 'Move the control out from under the cover. If the cover can be dismissed, such as a sheet or dialog, dismiss it before you use the control.', + }); + expect( + audit([node(1, { label: 'Large', rect: { x: 0, y: 0, width: 100, height: 100 } })], context) + .guidance, + ).toEqual({}); + expect( + Object.keys( + audit([node(1, { rect: { x: 0, y: 0, width: 100, height: 100 } })], context).guidance, + ), + ).toEqual(['missing-name']); +}); + +test('skips UIKit sheet grabbers by captured class without matching their label', () => { + const small = { x: 0, y: 0, width: 73, height: 24 }; + const findings = audit([ + node(1, { role: '_UIGrabber', label: 'Sheet Grabber', value: 'Expanded', rect: small }), + node(2, { role: '_UIGrabber', label: 'Poignée', rect: small }), + node(3, { subrole: '_UIGrabber', rect: small }), + node(4, { label: 'Sheet Grabber', rect: small }), + ]).findings; + expect(summarize(findings)).toEqual(['small-target @e4']); +}); + +test.each(['NavigationBar', 'Toolbar', 'TabBar'])( + 'skips iOS %s target size but keeps missing names and app content', + (type) => { + const small = { x: 0, y: 0, width: 42.2, height: 42.2 }; + const nodes = [ + node(0, { type, hittable: false }), + node(2, { parentIndex: 0, label: 'Cancel', rect: { ...small, width: 50.6 } }), + node(3, { parentIndex: 0, label: 'Add', rect: small }), + node(4, { parentIndex: 0, identifier: 'unnamed-bar-item', rect: small }), + node(5, { label: 'App action', rect: small }), + ]; + expect(summarize(audit(nodes).findings)).toEqual(['missing-name @e4', 'small-target @e5']); + expect(summarize(audit(nodes, ANDROID).findings)).toEqual([ + 'small-target @e2', + 'small-target @e3', + 'small-target @e4', + 'missing-name @e4', + 'small-target @e5', + ]); + }, +); + +test.each(['NavigationBar', 'Toolbar', 'TabBar', 'BottomNavigationView'])( + 'checks app content inside a %s-shaped container', + (type) => { + for (const container of ['ScrollView', 'Other', 'Button']) { + const result = audit([ + node(0, { type, hittable: false }), + node(1, { type: container, label: 'Content', parentIndex: 0 }), + node(2, { label: 'Save', parentIndex: 1, rect: { x: 0, y: 0, width: 22, height: 22 } }), + ]); + expect(summarize(result.findings), container).toEqual(['small-target @e2']); + } + }, +); + +test.each([{ type: 'HomeToolbarView' }, { type: 'Other', role: 'BottomTabBar' }])( + 'checks the size of controls in the custom iOS bar %j', + (container) => { + const result = audit([ + node(0, { ...container, hittable: false }), + node(1, { + parentIndex: 0, + label: 'Save', + rect: { x: 0, y: 0, width: 24, height: 24 }, + }), + ]); + expect(summarize(result.findings)).toEqual(['small-target @e1']); + }, +); + +test.each([{ role: 'UITabBar' }, { subrole: 'UIToolbar' }, { subrole: 'AXToolbar' }])( + 'recognizes a direct bar item through its parent class %j', + (barClass) => { + const item = node(1, { + parentIndex: 0, + label: 'Save', + rect: { x: 0, y: 0, width: 36, height: 36 }, + }); + const parent = node(0, { type: 'Other', hittable: false, ...barClass }); + expect(audit([parent, item]).findings).toEqual([]); + expect( + summarize(audit([parent, item], { ...IOS, producer: 'appium-source' }).findings), + ).toEqual(['small-target @e1']); + expect(summarize(audit([parent, { ...item, label: undefined }]).findings)).toEqual([ + 'missing-name @e1', + ]); + expect( + summarize(audit([{ ...parent, role: 'UIView', subrole: undefined }, item]).findings), + ).toEqual(['small-target @e1']); + }, +); diff --git a/packages/capture-kit/src/snapshot/snapshot-audit.ts b/packages/capture-kit/src/snapshot/snapshot-audit.ts index 948474e2d2..9f6af648a4 100644 --- a/packages/capture-kit/src/snapshot/snapshot-audit.ts +++ b/packages/capture-kit/src/snapshot/snapshot-audit.ts @@ -2,10 +2,11 @@ import type { AuditChecks, AuditContext, AuditFinding, + AuditFindingKind, AuditResult, AuditSizeUnit, } from '@agent-device/contracts/capture'; -import { normalizeType } from '@agent-device/contracts/snapshot'; +import { isNativeIosSnapshotBarNode, normalizeType } from '@agent-device/contracts/snapshot'; import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; import { formatRole, type SnapshotNode } from '@agent-device/kernel/snapshot'; import { @@ -48,6 +49,8 @@ type SizeRule = | { kind: 'unchecked'; limitation: string }; type AuditScan = { + platform: AuditContext['platform']; + producer: AuditContext['producer']; byIndex: ReadonlyMap; children: ChildrenByParent; }; @@ -58,6 +61,8 @@ type AuditScan = { */ export function auditSnapshot(nodes: readonly SnapshotNode[], context: AuditContext): AuditResult { const scan: AuditScan = { + platform: context.platform, + producer: context.producer, byIndex: new Map(nodes.map((node) => [node.index, node])), children: buildChildrenByParent(nodes), }; @@ -65,6 +70,7 @@ export function auditSnapshot(nodes: readonly SnapshotNode[], context: AuditCont const findings = nodes.flatMap((node) => auditNode(node, scan, rule)); return { findings, + guidance: auditGuidance(findings, context), checks: { snapshot: snapshotCompleteness(context) }, limitations: collectLimitations(context, rule), }; @@ -93,8 +99,12 @@ function controlFindings( scan: AuditScan, rule: SizeRule, ): AuditFinding[] { - if (node.ref === undefined || !isActionable(node)) return []; - const small = isInlineLink(node, role, scan) ? undefined : smallTargetFinding(node, role, rule); + const ownership = controlOwnership(node, scan); + if (node.ref === undefined || !isActionable(node, ownership)) return []; + const small = + ownership === 'platform-sized' || isInlineLink(node, role, scan) + ? undefined + : smallTargetFinding(node, role, rule); const unnamed = hasAccessibleName(node, role, scan) ? undefined : missingNameFinding(node, role); return [small, unnamed].filter((finding) => finding !== undefined); } @@ -122,15 +132,20 @@ function controlRole(node: SnapshotNode): string | undefined { return CONTROL_ROLES.find(([suffix]) => type.endsWith(suffix))?.[1]; } -/** - * Visible, enabled, not already proven unreachable by position, and owned by the app: system bars - * and keyboard keys are the platform's controls. - */ -function isActionable(node: SnapshotNode): boolean { +type ControlOwnership = 'app' | 'platform-sized' | 'system'; + +function controlOwnership(node: SnapshotNode, scan: AuditScan): ControlOwnership { + if (node.systemBar !== undefined || node.keyboardOwned === true) return 'system'; + if (scan.platform !== 'ios') return 'app'; + if (node.role === '_UIGrabber' || node.subrole === '_UIGrabber') return 'system'; + const parent = parentOf(node, scan); + return parent && isNativeIosSnapshotBarNode(parent, scan.producer) ? 'platform-sized' : 'app'; +} + +function isActionable(node: SnapshotNode, ownership: ControlOwnership): boolean { return ( isPositiveFiniteRect(node.rect) && - node.systemBar === undefined && - node.keyboardOwned !== true && + ownership !== 'system' && node.enabled !== false && node.visibleToUser !== false && node.hittable !== false && @@ -217,7 +232,8 @@ function smallTargetFinding( } function missingNameFinding(node: SnapshotNode, role: string): AuditFinding { - return { ...findingBase(node, role), kind: 'missing-name', severity: 'warning' }; + const { label: _label, ...base } = findingBase(node, role); + return { ...base, kind: 'missing-name', severity: 'warning' }; } /** @@ -251,6 +267,8 @@ function findingBase(node: SnapshotNode, role: string) { ref: `@${node.ref}`, ...(node.rect ? { rect: node.rect } : {}), role, + ...(node.label?.trim() ? { label: node.label.trim() } : {}), + ...(node.identifier?.trim() ? { identifier: node.identifier.trim() } : {}), }; } @@ -287,3 +305,19 @@ function typedContent(node: SnapshotNode): string | undefined { function roundTenth(value: number): number { return Math.round(value * 10) / 10; } + +function auditGuidance( + findings: readonly AuditFinding[], + context: AuditContext, +): AuditResult['guidance'] { + const size = + context.platform === 'ios' ? 'Apple HIG recommends 44×44pt.' : 'Material recommends 48×48dp.'; + const sentences: Record = { + 'small-target': `${size} Enlarge the touch area with padding, a minimum size, or hitSlop.`, + 'missing-name': + 'Set an accessibility label (iOS accessibilityLabel, Android contentDescription, React Native accessibilityLabel).', + 'covered-control': + 'Move the control out from under the cover. If the cover can be dismissed, such as a sheet or dialog, dismiss it before you use the control.', + }; + return Object.fromEntries(findings.map(({ kind }) => [kind, sentences[kind]])); +} diff --git a/packages/command-output/src/capture/audit-output.test.ts b/packages/command-output/src/capture/audit-output.test.ts index 240575819b..8811220c7e 100644 --- a/packages/command-output/src/capture/audit-output.test.ts +++ b/packages/command-output/src/capture/audit-output.test.ts @@ -12,6 +12,7 @@ test('renders warnings and observations as aligned lines with the advisory wordi severity: 'warning', ref: '@e7', role: 'button', + label: 'Save Lemon Herb Rice', detail: { unit: 'pt', width: 22, height: 22.5, recommendedSize: 44 }, }, { @@ -19,6 +20,7 @@ test('renders warnings and observations as aligned lines with the advisory wordi severity: 'warning', ref: '@e9', role: 'text-field', + identifier: 'email', }, { kind: 'covered-control', @@ -27,6 +29,7 @@ test('renders warnings and observations as aligned lines with the advisory wordi role: 'button', }, ], + guidance: {}, checks: CHECKS, limitations: [], refsGeneration: 3, @@ -35,9 +38,9 @@ test('renders warnings and observations as aligned lines with the advisory wordi [ 'agent-device audit', '2 warnings, 1 observation — current screen', - '@e7~s3 small-target 22×22.5pt; review touch area', - '@e9~s3 missing-name Text field has no accessible name', - '@e12~s3 covered-control Accessible action is unavailable to touch', + '@e7~s3 small-target button "Save Lemon Herb Rice" 22×22.5pt, below 44×44pt', + '@e9~s3 missing-name text-field id="email" No accessible name', + '@e12~s3 covered-control button Accessible action is unavailable to touch', '', ].join('\n'), ); @@ -46,6 +49,7 @@ test('renders warnings and observations as aligned lines with the advisory wordi test('prints plain refs when the result carries no generation to pin them to', () => { const output = auditCliOutput({ findings: [{ kind: 'missing-name', severity: 'warning', ref: '@e9', role: 'button' }], + guidance: {}, checks: CHECKS, limitations: [], }); @@ -56,6 +60,7 @@ test('prints plain refs when the result carries no generation to pin them to', ( test('says no findings and lists each limitation', () => { const output = auditCliOutput({ findings: [], + guidance: {}, checks: CHECKS, limitations: ['Target size was not checked: tv targets are focus-driven.'], }); @@ -71,6 +76,30 @@ test('says no findings and lists each limitation', () => { }); test('keeps the structured result as the JSON payload', () => { - const result: AuditSnapshotCommandResult = { findings: [], checks: CHECKS, limitations: [] }; + const result: AuditSnapshotCommandResult = { + findings: [], + guidance: {}, + checks: CHECKS, + limitations: [], + }; expect(auditCliOutput(result).data).toBe(result); }); + +test('prints each guidance entry once after the findings and preserves it in JSON', () => { + const result: AuditSnapshotCommandResult = { + findings: [1, 2].map((index) => ({ + kind: 'missing-name', + severity: 'warning', + ref: `@e${index}`, + role: 'button', + })), + guidance: { 'missing-name': 'Set an accessibility label.' }, + checks: CHECKS, + limitations: [], + }; + const output = auditCliOutput(result); + expect(output.text).toContain('button No accessible name'); + expect(output.text!.split('missing-name: Set an accessibility label.')).toHaveLength(2); + expect(output.text!.indexOf('missing-name:')).toBeGreaterThan(output.text!.indexOf('@e2')); + expect(output.data).toBe(result); +}); diff --git a/packages/command-output/src/capture/audit-output.ts b/packages/command-output/src/capture/audit-output.ts index 3e4971e6d7..71e977acd6 100644 --- a/packages/command-output/src/capture/audit-output.ts +++ b/packages/command-output/src/capture/audit-output.ts @@ -15,6 +15,9 @@ function formatAuditText(result: AuditSnapshotCommandResult): string { summaryLine(result.findings), ...findingLines(result.findings, result.refsGeneration), ]; + const guidance = Object.entries(result.guidance); + if (guidance.length > 0) + lines.push('', ...guidance.map(([kind, sentence]) => `${kind}: ${sentence}`)); if (result.limitations.length > 0) { lines.push('', ...result.limitations.map((limitation) => `Limitation: ${limitation}`)); } @@ -44,25 +47,31 @@ function findingLines( const kindWidth = Math.max(0, ...findings.map((finding) => finding.kind.length)); return findings.map( (finding, index) => - `${refs[index]!.padEnd(refWidth)} ${finding.kind.padEnd(kindWidth)} ${findingText(finding)}`, + `${refs[index]!.padEnd(refWidth)} ${finding.kind.padEnd(kindWidth)} ${elementText(finding)} ${findingText(finding)}`, ); } function findingText(finding: AuditFinding): string { switch (finding.kind) { case 'small-target': - return `${sizeText(finding.detail)}; review touch area`; + return sizeText(finding.detail); case 'missing-name': - return `${capitalize(finding.role.replaceAll('-', ' '))} has no accessible name`; + return 'No accessible name'; case 'covered-control': return 'Accessible action is unavailable to touch'; } } -function sizeText({ width, height, unit }: AuditSizeDetail): string { - return `${width}×${height}${unit}`; +function sizeText({ width, height, unit, recommendedSize }: AuditSizeDetail): string { + return `${width}×${height}${unit}, below ${recommendedSize}×${recommendedSize}${unit}`; } -function capitalize(text: string): string { - return `${text.charAt(0).toUpperCase()}${text.slice(1)}`; +function elementText(finding: AuditFinding): string { + const name = finding.kind === 'missing-name' ? undefined : finding.label; + const identity = name + ? JSON.stringify(name) + : finding.identifier + ? `id=${JSON.stringify(finding.identifier)}` + : undefined; + return identity ? `${finding.role} ${identity}` : finding.role; } diff --git a/packages/command-output/src/cli-output.test.ts b/packages/command-output/src/cli-output.test.ts index 49bfc1ca4a..ab345e1903 100644 --- a/packages/command-output/src/cli-output.test.ts +++ b/packages/command-output/src/cli-output.test.ts @@ -1,3 +1,4 @@ +import type { AuditSnapshotCommandResult } from '@agent-device/contracts/capture'; import { bindTestCommandHost } from '@agent-device/test-kit'; import { describe, expect, test } from 'vitest'; import { findCommandMetadata, listCommandMetadata } from '@agent-device/command-surface'; @@ -102,10 +103,11 @@ const RESPONSE_FIXTURES: Partial> = { apps: ['com.example.app'], artifacts: { source: 'daemon', artifacts: [], message: 'No daemon artifacts.' }, audit: { + guidance: {}, findings: [], checks: { snapshot: 'complete' }, limitations: [], - }, + } satisfies AuditSnapshotCommandResult, batch: { executed: 0, total: 0, totalDurationMs: 5, results: [] }, capabilities: { device: { name: 'iPhone' }, availableCommands: [] }, debug: { diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts index 5d1b1d304b..d937b6db39 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.test.ts @@ -241,6 +241,9 @@ test('unchanged metadata compares covering-node descriptions by value', () => { current.nodes[0]!.coveredBy = { ...cover, rect: { ...cover.rect, y: 700 } }; expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined(); + current.nodes[0]!.coveredBy = { ...cover, persistentBar: true }; + expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined(); + current.nodes[0]!.coveredBy = undefined; expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined(); }); diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts index b71f23dc82..5734828f65 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts @@ -169,6 +169,7 @@ function areCoveringNodesEquivalent( return ( previous.kind === current.kind && previous.label === current.label && + previous.persistentBar === current.persistentBar && areRectsEquivalent(previous.rect, current.rect) ); } diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts index d1c76bc7a4..69723bcd96 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot.ts @@ -197,6 +197,7 @@ export const auditSnapshotCommand: RuntimeCommand< const capture = await captureRuntimeSnapshot(runtime, options); const audit = auditSnapshot(capture.snapshot.nodes, { platform: runtime.backend.platform, + producer: capture.snapshot.producer, target: options.target, pixelDensity: capture.annotations.androidSnapshot?.pixelDensity, truncated: snapshotTruncationForResult(capture.snapshot), diff --git a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts index b646a045bc..b965fe6f37 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/__tests__/test-utils/index.ts @@ -2,6 +2,7 @@ import type { AgentDeviceBackend, BackendSnapshotResult } from '@agent-device/co import { annotateCoveredSnapshotNodes } from '@agent-device/capture-kit/snapshot-occlusion'; import { createLocalArtifactAdapter } from '@agent-device/host-kit/local-artifact-adapter'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { createAgentDevice } from '../../../../../runtime.ts'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; import type { CommandSessionStore } from '@agent-device/contracts/runtime-contract'; @@ -188,35 +189,101 @@ export function coveredByTabBarSnapshot(): SnapshotState { * The same tab-bar-over-button shape run through the real annotation pass, so the covered node * carries the scan's `coveredBy` description exactly as a capture publishes it. */ -export function coveredByScanSnapshot(): SnapshotState { +export function coveredByScanSnapshot(coverType = 'TabBar'): SnapshotState { return makeSnapshotState( - annotateCoveredSnapshotNodes([ - { - index: 0, - depth: 0, - type: 'Application', - label: 'Example', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'Button', - label: 'Save draft', - rect: { x: 16, y: 790, width: 140, height: 44 }, - hittable: true, - }, - { - index: 2, - depth: 1, - parentIndex: 0, - type: 'TabBar', - label: 'Media', - rect: { x: 0, y: 760, width: 390, height: 84 }, - hittable: true, - }, - ]), + annotateCoveredSnapshotNodes( + [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Save draft', + rect: { x: 16, y: 790, width: 140, height: 44 }, + hittable: true, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: coverType, + label: 'Media', + rect: { x: 0, y: 760, width: 390, height: 84 }, + hittable: true, + }, + ], + { producer: 'apple-runner' }, + ), + ); +} + +/** + * An unnamed bar published as type `Other` with its UIKit class in `role`, run through the real + * annotation pass: the scan reads the bar identity from the role, not the generic type. The Save draft + * button is fixed beside the bar by default; `scrollable` puts it inside a scroll view instead; + * `horizontal` puts it inside a horizontal scroller whose viewport the bar contains entirely, so no + * scroll can reveal it. + */ +export function coveredByAnonymousTabBarScanSnapshot( + options: { scrollable?: boolean; horizontal?: boolean } = {}, +): SnapshotState { + const scroller: RawSnapshotNode[] = + options.scrollable || options.horizontal + ? [ + { + index: 1, + depth: 1, + parentIndex: 0, + type: options.horizontal ? 'android.widget.HorizontalScrollView' : 'ScrollView', + hiddenContentBelow: options.scrollable ? true : undefined, + rect: options.horizontal + ? { x: 0, y: 800, width: 402, height: 65 } + : { x: 0, y: 0, width: 402, height: 874 }, + }, + ] + : []; + const buttonIndex = scroller.length + 1; + return makeSnapshotState( + annotateCoveredSnapshotNodes( + [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + ...scroller, + { + index: buttonIndex, + depth: scroller.length + 1, + parentIndex: scroller.length > 0 ? 1 : 0, + type: 'Button', + label: 'Save draft', + rect: options.horizontal + ? { x: 16, y: 810, width: 140, height: 44 } + : { x: 16, y: 800, width: 140, height: 44 }, + hittable: true, + }, + { + index: buttonIndex + 1, + depth: 1, + parentIndex: 0, + type: 'Other', + role: 'UITabBar', + rect: { x: 0, y: 791, width: 402, height: 83 }, + hittable: true, + }, + ], + { producer: 'apple-runner' }, + ), ); } diff --git a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts index fcd481d5ff..bf7b9f34f1 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/target-visibility-stages.test.ts @@ -5,6 +5,7 @@ import { throwIfOffscreenInteractionTarget } from './target-visibility-stages.ts import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import type { Point } from '@agent-device/kernel/snapshot'; import { + coveredByAnonymousTabBarScanSnapshot, coveredByScanSnapshot, coveredByTabBarSnapshot, createInteractionDevice, @@ -175,6 +176,7 @@ test('runtime covered refusal names the covering element the occlusion scan foun kind: 'tab-bar', label: 'Media', rect: { x: 0, y: 760, width: 390, height: 84 }, + persistentBar: true, }); assert.doesNotMatch(String(details?.hint), /@e\d/); assert.match(String(details?.hint), /Covered by the tab-bar "Media" at \(0, 760 390x84\)/); @@ -184,6 +186,111 @@ test('runtime covered refusal names the covering element the occlusion scan foun assert.deepEqual(calls, []); }); +test('runtime covered refusal keeps dismissal recovery for a custom toolbar overlay', async () => { + const device = createInteractionDevice(coveredByScanSnapshot('HomeToolbarView'), { + tap: async () => {}, + }); + + await assert.rejects( + () => device.interactions.click(ref('@e2'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_covered'); + assert.deepEqual(details?.coveredBy, { + kind: 'hometoolbarview', + label: 'Media', + rect: { x: 0, y: 760, width: 390, height: 84 }, + }); + assert.match(String(details?.hint), /Dismiss or close that element/); + return true; + }, + ); +}); + +test('runtime covered refusal names an unnamed generic-typed bar and never says dismiss', async () => { + const device = createInteractionDevice(coveredByAnonymousTabBarScanSnapshot(), { + tap: async () => {}, + }); + + await assert.rejects( + () => device.interactions.click(ref('@e2'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_covered'); + assert.deepEqual(details?.coveredBy, { + kind: 'tab-bar', + rect: { x: 0, y: 791, width: 402, height: 83 }, + persistentBar: true, + }); + assert.match( + String(details?.hint), + /^Covered by the tab-bar at \(0, 791 402x83\)\. Use a different visible target/, + ); + // The fixed button has no scrollable ancestor, so no scroll can move it clear of the bar. + assert.doesNotMatch(String(details?.hint), /dismiss|close|scroll|the other/i); + return true; + }, + ); +}); + +test('runtime covered refusal suggests inspection for a covered target in a scroll view', async () => { + const device = createInteractionDevice( + coveredByAnonymousTabBarScanSnapshot({ scrollable: true }), + { + tap: async () => {}, + }, + ); + + await assert.rejects( + () => device.interactions.click(ref('@e3'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_covered'); + assert.deepEqual(details?.coveredBy, { + kind: 'tab-bar', + rect: { x: 0, y: 791, width: 402, height: 83 }, + persistentBar: true, + }); + assert.match( + String(details?.hint), + /^Covered by the tab-bar at \(0, 791 402x83\)\. Use a different visible target/, + ); + assert.doesNotMatch(String(details?.hint), /dismiss|close|scroll/i); + return true; + }, + ); +}); + +test('runtime covered refusal keeps the fixed-target recovery for a horizontal scroller wholly behind a scanned bar', async () => { + // The bar contains the horizontal scroller's whole 402x65 viewport, so no scroll can reveal the + // button: the hint must not suggest scrolling, which could never clear the bar. + const device = createInteractionDevice( + coveredByAnonymousTabBarScanSnapshot({ horizontal: true }), + { + tap: async () => {}, + }, + ); + + await assert.rejects( + () => device.interactions.click(ref('@e3'), { session: 'default' }), + (error: unknown) => { + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'target_covered'); + assert.deepEqual(details?.coveredBy, { + kind: 'tab-bar', + rect: { x: 0, y: 791, width: 402, height: 83 }, + persistentBar: true, + }); + assert.equal( + String(details?.hint), + 'Covered by the tab-bar at (0, 791 402x83). Use a different visible target, or inspect with snapshot/screenshot before retrying.', + ); + assert.doesNotMatch(String(details?.hint), /scroll/i); + return true; + }, + ); +}); + test('runtime covered refusal keeps the generic hint when the cover is a policy boolean', async () => { // `coveredByTabBarSnapshot` hand-annotates the covered boolean the way the Android // replacement-surface lane publishes it: no cover identity, so nothing to name. diff --git a/packages/command-surface/src/capture/snapshot.ts b/packages/command-surface/src/capture/snapshot.ts index bd456dde9e..29d61122b3 100644 --- a/packages/command-surface/src/capture/snapshot.ts +++ b/packages/command-surface/src/capture/snapshot.ts @@ -126,6 +126,8 @@ export const AUDIT_COMMAND_OUTPUT_SCHEMAS = { ['x', 'y', 'width', 'height'], ), role: stringSchema(), + label: stringSchema(), + identifier: stringSchema(), detail: objectSchema( { unit: enumSchema(['pt', 'dp']), @@ -139,6 +141,14 @@ export const AUDIT_COMMAND_OUTPUT_SCHEMAS = { ['kind', 'severity', 'ref', 'role'], ), }, + guidance: objectSchema( + { + 'small-target': stringSchema(), + 'missing-name': stringSchema(), + 'covered-control': stringSchema(), + }, + [], + ), checks: objectSchema( { snapshot: enumSchema( @@ -154,12 +164,12 @@ export const AUDIT_COMMAND_OUTPUT_SCHEMAS = { ), warnings: stringArraySchema(), }, - ['findings', 'checks', 'limitations'], + ['findings', 'guidance', 'checks', 'limitations'], ), } satisfies Pick, 'audit'>; const auditCommandDescription = - 'Report advisory accessibility findings from one fresh capture of the current screen: touch targets below the platform recommendation, controls without an accessible name, and controls the snapshot already marks covered. Findings are review prompts, not failures, and the audit changes nothing on the device.'; + 'Report advisory accessibility findings from one fresh capture of the current screen: touch targets below the platform recommendation, controls without an accessible name, and covered controls. Each finding shows its role. Missing-name findings omit rejected labels and can show the captured identifier. Other findings include the captured label when available. Guidance follows once per finding kind. Covered controls remain findings inside scroll containers. UIKit sheet grabbers are exempt from size and name checks. Direct items of recognized native iOS bars skip size checks and keep name checks. Custom bar children and nested app content remain subject to size checks. Findings are review prompts, not failures, and the audit changes nothing on the device.'; export const auditCommandFacet = defineParameterlessCommandFacet({ name: AUDIT_COMMAND_NAME, diff --git a/packages/contracts/src/audit.ts b/packages/contracts/src/audit.ts index 73e0fedfa8..d92673eef7 100644 --- a/packages/contracts/src/audit.ts +++ b/packages/contracts/src/audit.ts @@ -1,5 +1,6 @@ import type { DeviceTarget, PublicPlatform } from '@agent-device/kernel/device'; import type { Rect } from '@agent-device/kernel/snapshot'; +import type { SnapshotProducer } from './snapshot-producers.ts'; export type AuditFindingKind = 'small-target' | 'missing-name' | 'covered-control'; @@ -23,6 +24,10 @@ type AuditFindingBase = { rect?: Rect; /** Control role as the snapshot presents it, such as `button`. */ role: string; + /** Accessible label from the captured node. */ + label?: string; + /** Accessibility identifier or resource id from the captured node. */ + identifier?: string; }; export type AuditFinding = @@ -32,6 +37,7 @@ export type AuditFinding = export type AuditContext = { platform: PublicPlatform; + producer?: SnapshotProducer; /** Absent means a mobile target. */ target?: DeviceTarget; /** Physical pixels per dp, from the Android capture. */ @@ -50,6 +56,7 @@ export type AuditChecks = { export type AuditResult = { findings: AuditFinding[]; + guidance: Partial>; checks: AuditChecks; /** Checks the audit could not run or could only partly run, with the reason. */ limitations: string[]; diff --git a/packages/contracts/src/facades/snapshot.ts b/packages/contracts/src/facades/snapshot.ts index cb1922bc69..ce0fc1b87a 100644 --- a/packages/contracts/src/facades/snapshot.ts +++ b/packages/contracts/src/facades/snapshot.ts @@ -45,4 +45,7 @@ export { isMeaningfulLabel, isMeaningfulSignal, normalizeType, + isNativeIosSnapshotBarNode, + snapshotBarKind, + snapshotNodeKindIncludesAny, } from '../snapshot-text.ts'; diff --git a/packages/contracts/src/snapshot-text.test.ts b/packages/contracts/src/snapshot-text.test.ts index 1370f5e92b..0a64fe7b69 100644 --- a/packages/contracts/src/snapshot-text.test.ts +++ b/packages/contracts/src/snapshot-text.test.ts @@ -1,6 +1,11 @@ import { readFileSync } from 'node:fs'; import { expect, test } from 'vitest'; -import { androidFieldValue, isFillableType } from './snapshot-text.ts'; +import { + androidFieldValue, + isFillableType, + isNativeIosSnapshotBarNode, + snapshotBarKind, +} from './snapshot-text.ts'; const fieldValues = JSON.parse( readFileSync( @@ -25,3 +30,38 @@ test('a web text input is fillable by its ARIA role, a button is not', () => { expect(isFillableType('searchbox', 'web')).toBe(true); expect(isFillableType('button', 'web')).toBe(false); }); + +// The full bar identity lives here, bottom navigation included, so consumers that publish a +// persistent-bar fact read one vocabulary instead of matching platform role strings themselves. +test.each([ + { name: 'an iOS tab bar type', node: { type: 'TabBar' }, kind: 'tab-bar' }, + { + name: 'a UIKit class in role beside a generic type', + node: { type: 'Other', role: 'UINavigationBar' }, + kind: 'navigation-bar', + }, + { + name: 'a toolbar in subrole only', + node: { type: 'Other', subrole: 'AXToolbar' }, + kind: 'toolbar', + }, + { + name: 'an Android bottom navigation view', + node: { type: 'com.google.android.material.bottomnavigation.BottomNavigationView' }, + kind: 'bottom-navigation', + }, +])('snapshotBarKind reads $name', ({ node, kind }) => { + const producer = kind === 'bottom-navigation' ? 'android-uiautomator' : 'apple-runner'; + expect(snapshotBarKind(node, producer)).toBe(kind); + expect(isNativeIosSnapshotBarNode(node, producer)).toBe(kind !== 'bottom-navigation'); + expect(snapshotBarKind(node, 'cdp-dom')).toBeUndefined(); +}); + +test.each([ + { type: 'Button', role: 'UIButton' }, + { type: 'HomeToolbarView' }, + { type: 'Other', role: 'BottomTabBar' }, +])('a non-bar node reads no bar kind for %j', (node) => { + expect(snapshotBarKind(node, 'apple-runner')).toBeUndefined(); + expect(isNativeIosSnapshotBarNode(node, 'apple-runner')).toBe(false); +}); diff --git a/packages/contracts/src/snapshot-text.ts b/packages/contracts/src/snapshot-text.ts index 446575ce0d..e567da3792 100644 --- a/packages/contracts/src/snapshot-text.ts +++ b/packages/contracts/src/snapshot-text.ts @@ -1,5 +1,6 @@ import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import type { SnapshotProducer } from './snapshot-producers.ts'; export function normalizeType(type: string): string { let normalized = type @@ -14,6 +15,75 @@ export function normalizeType(type: string): string { return normalized; } +const APPLE_BAR_KINDS = [ + ['TabBar', 'tab-bar'], + ['Toolbar', 'toolbar'], + ['NavigationBar', 'navigation-bar'], +] as const; + +const ANDROID_BAR_KINDS = [ + ['com.google.android.material.bottomnavigation.BottomNavigationView', 'bottom-navigation'], + ['android.widget.Toolbar', 'toolbar'], + ['androidx.appcompat.widget.Toolbar', 'toolbar'], +] as const; + +type SnapshotBarNode = Pick; +type SnapshotBarKind = (typeof APPLE_BAR_KINDS | typeof ANDROID_BAR_KINDS)[number][1]; + +/** Reads native bar identity only from captures that publish native type or class names. */ +export function snapshotBarKind( + node: SnapshotBarNode, + producer: SnapshotProducer | undefined, +): SnapshotBarKind | undefined { + if ( + producer === 'apple-runner' || + producer === 'simulator-ax-bridge' || + producer === 'macos-helper' + ) { + const names = [node.type, node.role, node.subrole].map((name) => + name + ?.trim() + .replace(/^UIKit\./, '') + .replace(/^(?:XCUIElementType|UI|AX)/, ''), + ); + return APPLE_BAR_KINDS.find(([name]) => names.includes(name))?.[1]; + } + if (producer === 'android-uiautomator') { + const names = [node.type, node.role, node.subrole]; + return ANDROID_BAR_KINDS.find(([name]) => names.includes(name))?.[1]; + } + return undefined; +} + +/** Identifies native iOS bars whose direct items use platform sizing. */ +export function isNativeIosSnapshotBarNode( + node: SnapshotBarNode, + producer: SnapshotProducer | undefined, +): boolean { + return ( + (producer === 'apple-runner' || producer === 'simulator-ax-bridge') && + snapshotBarKind(node, producer) !== undefined + ); +} + +export function snapshotNodeKindIncludesAny( + node: Pick, + fragments: readonly string[], +): boolean { + const normalized = normalizeNodeKind(node); + return fragments.some((fragment) => normalized.includes(fragment)); +} + +// Mutation-lane note: the `?? ''` fallback's exact replacement text is not +// observable through this module's public API — every caller only checks +// substring membership against a fixed, known fragment list, and no +// plausible filler text coincides with any of them, so no test can +// distinguish `''` from another non-matching filler here without reaching +// into this private function directly. +function normalizeNodeKind(node: Pick): string { + return [node.type, node.role, node.subrole].map((value) => normalizeType(value ?? '')).join(' '); +} + export function isFillableType(type: string, platform: Platform | PublicPlatform): boolean { const normalized = normalizeType(type); if (!normalized) return true; diff --git a/packages/daemon/src/__tests__/support/command-surface-fake.ts b/packages/daemon/src/__tests__/support/command-surface-fake.ts index c4d4ec1eca..9fcd5a21d9 100644 --- a/packages/daemon/src/__tests__/support/command-surface-fake.ts +++ b/packages/daemon/src/__tests__/support/command-surface-fake.ts @@ -27,6 +27,7 @@ const fakeCommandSurface: DaemonCommandSurface = { })), auditSnapshot: vi.fn(async () => ({ findings: [], + guidance: {}, checks: { snapshot: 'complete' as const }, limitations: [], })), diff --git a/packages/daemon/src/execution/interaction/internal/__tests__/find-touch-runtime-fixture.ts b/packages/daemon/src/execution/interaction/internal/__tests__/find-touch-runtime-fixture.ts index 7d10afdb83..678e40fa0b 100644 --- a/packages/daemon/src/execution/interaction/internal/__tests__/find-touch-runtime-fixture.ts +++ b/packages/daemon/src/execution/interaction/internal/__tests__/find-touch-runtime-fixture.ts @@ -1,3 +1,4 @@ +import type { SnapshotCaptureProvenance } from '@agent-device/contracts/snapshot'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; import { expect } from 'vitest'; import { legacyDispatchCapture } from '../../../../__tests__/support/legacy-snapshot-capture-fixture.ts'; @@ -60,6 +61,7 @@ export function resetFindTouchRuntimeFixture(): void { export async function runFindClickScenario(options: { positionals: string[]; nodes?: Array>; + snapshotProvenance?: SnapshotCaptureProvenance; flags?: DaemonRequest['flags']; session?: SessionState; createCommandSurface?: CreateDaemonCommandSurface; @@ -77,7 +79,7 @@ export async function runFindClickScenario(options: { if (options.nodes !== undefined) { mockDispatch.mockImplementation(async (_device, command) => { if (command === 'snapshot') { - return { nodes: options.nodes }; + return { nodes: options.nodes, ...options.snapshotProvenance }; } return {}; }); diff --git a/packages/daemon/src/execution/interaction/internal/__tests__/find.test.ts b/packages/daemon/src/execution/interaction/internal/__tests__/find.test.ts index 83beed2136..8b524a2a6c 100644 --- a/packages/daemon/src/execution/interaction/internal/__tests__/find.test.ts +++ b/packages/daemon/src/execution/interaction/internal/__tests__/find.test.ts @@ -509,6 +509,7 @@ test('handleFindCommands focus uses the promoted actionable node center', async test('handleFindCommands focus rejects covered matches before dispatching coordinates', async () => { const { response } = await runFindClickScenario({ positionals: ['Save draft', 'focus'], + snapshotProvenance: { backend: 'xctest', producer: 'apple-runner' }, nodes: [ { index: 0, @@ -521,14 +522,7 @@ test('handleFindCommands focus rejects covered matches before dispatching coordi ref: 'e2', type: 'Button', label: 'Save draft', - hittable: false, - interactionBlocked: 'covered', - coveredBy: { - kind: 'tab-bar', - label: 'Media', - rect: { x: 0, y: 760, width: 390, height: 84 }, - }, - presentationHints: ['covered'], + hittable: true, rect: { x: 16, y: 790, width: 140, height: 44 }, parentIndex: 0, }, @@ -556,6 +550,7 @@ test('handleFindCommands focus rejects covered matches before dispatching coordi kind: 'tab-bar', label: 'Media', rect: { x: 0, y: 760, width: 390, height: 84 }, + persistentBar: true, }); expect(response.error.details?.hint).toContain('Covered by the tab-bar "Media"'); expect(response.error.details?.hint).not.toMatch(/@e\d/); diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 7b915c5b0f..8fae1b0ee8 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -263,12 +263,17 @@ export type SnapshotPresentation = SnapshotOptionFields< * issues refs for, and naming it is diagnosis, not an invitation to target it. */ export type SnapshotCoveringNode = { - /** The cover's presented role, from the same table the text presenter reads (`formatRole`). */ + /** The canonical persistent-bar kind or the cover's presented role. */ kind: string; /** The cover's accessible name, trimmed and clipped to {@link SNAPSHOT_COVERING_LABEL_MAX_LENGTH}. */ label?: string; /** The cover's positive rect as the scan measured it. */ rect?: Rect; + /** + * Set when the cover is persistent bar chrome (tab bar, toolbar, navigation bar, bottom + * navigation), read from the cover's type, role, and subrole. Absent for any other cover. + */ + persistentBar?: true; }; /** Upper bound for a covering-node label carried in diagnostics; longer names are clipped. */ diff --git a/packages/selectors/src/interaction-error.test.ts b/packages/selectors/src/interaction-error.test.ts index da42a6c1f1..cce44dae4c 100644 --- a/packages/selectors/src/interaction-error.test.ts +++ b/packages/selectors/src/interaction-error.test.ts @@ -84,6 +84,19 @@ test('the hint falls back to the bare kind when the label sanitizes to nothing', assert.match(hint, /^Covered by the tab-bar at \(0, 760 390x84\)/); }); +test('the hint for a persistent bar suggests a visible target or inspection', () => { + const bar = { + kind: 'navigation-bar', + rect: { x: 0, y: 0, width: 390, height: 96 }, + persistentBar: true, + } as const; + assert.equal( + coveredInteractionDetails({ ref: 'e2', interactionBlocked: 'covered', coveredBy: bar }).hint, + + 'Covered by the navigation-bar at (0, 0 390x96). Use a different visible target, or inspect with snapshot/screenshot before retrying.', + ); +}); + test('the hint names a cover the capture published without a rect', () => { const hint = String( coveredInteractionDetails({ diff --git a/packages/selectors/src/interaction-error.ts b/packages/selectors/src/interaction-error.ts index 0e8f3950ab..a49f3e8f54 100644 --- a/packages/selectors/src/interaction-error.ts +++ b/packages/selectors/src/interaction-error.ts @@ -88,6 +88,12 @@ function coveredInteractionHint(coveredBy: SnapshotCoveringNode | undefined): st if (!coveredBy) { return 'Use a different visible target, scroll it clear of the overlay, or inspect with snapshot/screenshot before retrying.'; } + if (coveredBy.persistentBar) { + return ( + `Covered by ${describeCoverForHint(coveredBy)}. Use a different visible target, ` + + 'or inspect with snapshot/screenshot before retrying.' + ); + } return ( `Covered by ${describeCoverForHint(coveredBy)}. Dismiss or close that element, ` + 'scroll the target clear of it, or inspect with snapshot/screenshot before retrying.' diff --git a/src/commands/schema/cli-help-workflows.test.ts b/src/commands/schema/cli-help-workflows.test.ts index 613ef7a2b5..99871d1305 100644 --- a/src/commands/schema/cli-help-workflows.test.ts +++ b/src/commands/schema/cli-help-workflows.test.ts @@ -66,6 +66,16 @@ test('usageForCommand resolves manual QA help topic', async () => { assert.match(help, /wait_target_absent: readable capture, no match/); assert.match(help, /wait_target_present: absence timeout/); assert.match(help, /wait_capture_stalled: no readable capture by deadline/); + assert.match(help, /Target covered by other visible UI: refused with target_covered/); + assert.match( + help, + /dismiss or close the cover only when it is dismissible \(sheet, dialog, alert\)/, + ); + assert.match( + help, + /under a persistent bar \(tab bar, toolbar, navigation bar, bottom navigation\) use a different visible target or inspect with snapshot\/screenshot/, + ); + assert.doesNotMatch(help, /Dismiss\/close it or scroll the target clear/); }); test('usageForCommand resolves validate help topic', async () => { diff --git a/src/commands/schema/cli-help-workflows.ts b/src/commands/schema/cli-help-workflows.ts index 81f2fdbd75..ea4bfe63d4 100644 --- a/src/commands/schema/cli-help-workflows.ts +++ b/src/commands/schema/cli-help-workflows.ts @@ -53,7 +53,7 @@ Recovery: Network/typeahead result missing: wait text "Expected result" or wait . A target that should be gone/disappear: use wait absent ; wait exists ... is rejected in favor of the plain selector wait. Keyboard visible over the next target: a tap point under a measured keyboard frame returns tap_keyboard_occludes_target. Press a visible Hide Keyboard key or the app's dismissal control. In apps that support drag dismissal, use gesture pan from inside the scrollable content. Use keyboard enter only to send Return. Missing keyboard nodes never prove absence. A raw coordinate behind the keys taps anyway and reports a warning. - Target covered by other visible UI: refused with target_covered. When the occlusion scan identified the covering element, error.details.coveredBy and the hint name it (role, label, rect) so you know what to dismiss; the cover is diagnosis only -- never target its ref. Dismiss/close it or scroll the target clear, then retry. + Target covered by other visible UI: refused with target_covered. When the occlusion scan identified the covering element, error.details.coveredBy and the hint name it (role, label, rect); the cover is diagnosis only -- never target its ref. Follow the hint's recovery: dismiss or close the cover only when it is dismissible (sheet, dialog, alert); under a persistent bar (tab bar, toolbar, navigation bar, bottom navigation) use a different visible target or inspect with snapshot/screenshot before retrying. Sparse or recovered accessibility snapshot: use screenshot as visual truth, leave the bad screen if needed, then retry snapshot -i. Non-hittable success hint: verify with the settled diff or snapshot; retarget by a better ref/selector if the UI did not change. diff --git a/src/mcp/__tests__/tool-ref-pins.test.ts b/src/mcp/__tests__/tool-ref-pins.test.ts index 9f8f7900eb..23acdfced5 100644 --- a/src/mcp/__tests__/tool-ref-pins.test.ts +++ b/src/mcp/__tests__/tool-ref-pins.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; +import type { AuditSnapshotCommandResult } from '@agent-device/contracts/capture'; import { createToolRefPinStore } from '../tool-ref-pins.ts'; function makeStore() { @@ -479,10 +480,14 @@ test('ref-pin store pins audit finding refs to the audit generation and keeps ot 'audit', { findings: [{ kind: 'missing-name', severity: 'warning', ref: '@e2', role: 'button' }], + guidance: { + 'missing-name': + 'Set an accessibility label (iOS accessibilityLabel, Android contentDescription, React Native accessibilityLabel).', + }, checks, limitations: [], refsGeneration: 9, - }, + } satisfies AuditSnapshotCommandResult, undefined, 'demo', ); @@ -508,7 +513,12 @@ test('ref-pin store leaves pins untouched for an audit without findings', () => ); pins.mergeCommandResult( 'audit', - { findings: [], checks: { snapshot: 'complete' }, limitations: [] }, + { + findings: [], + guidance: {}, + checks: { snapshot: 'complete' }, + limitations: [], + } satisfies AuditSnapshotCommandResult, undefined, 'demo', ); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 15887d1736..1d6f8c1941 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -433,11 +433,11 @@ agent-device audit - `small-target`: an enabled, visible control below 44pt on iOS or 48dp on Android, with a 0.5-unit tolerance for layout rounding. Android pixels convert to dp with the captured display density; without it the check reports a limitation instead. TV and desktop targets skip the check. A link inside a text container is exempt. - `missing-name`: a control with no accessible name. An identifier, a resource id, or a field value is not a name; a label, an Android content description, a placeholder, or a named child is. -- `covered-control`: a control or row the snapshot already marks `[covered]`. +- `covered-control`: a control or row the snapshot already marks `[covered]`. Audit reports covered controls inside scroll containers, including those under persistent bars, sheets, and dialogs. -Controls are recognized by role (button, link, switch, checkbox, radio, slider, menu item, text and search fields). Generic clickable containers on Android are not evaluated. iOS visibility relies on the `hittable` fact. System bars and keyboard keys are skipped. When a control is the only child of a control with the same role, the child drops a finding only if the parent reports the same kind of finding. +Controls are recognized by role (button, link, switch, checkbox, radio, slider, menu item, text and search fields). Generic clickable containers on Android are not evaluated. iOS visibility relies on the `hittable` fact. Size and name checks skip system bars, keyboard keys, and UIKit sheet grabbers. Direct items of recognized native iOS navigation bars, toolbars, and tab bars skip target-size checks and still receive missing-name checks. Custom bar children and items inside nested content containers remain subject to size checks. A system-generated accessible name, such as "add" or "remove" for an icon-only React Native Pressable on iOS 26, prevents a missing-name finding. The audit cannot determine whether the app supplied that name. When a control is the only child of a control with the same role, the child drops a finding only if the parent reports the same kind of finding. -Like `find`, `audit` publishes only the refs it returns. The text prints each as `@eN~s`; press that pinned ref. A plain ref from before the audit stays refused after the frame expired, and an audit with no findings issues no refs. An audit with findings also ends the snapshot's complete frame, so take a new snapshot before you use refs that the audit did not return. `--json` returns `findings`, `checks`, `limitations`, and `refsGeneration` (only with findings). `checks.snapshot` is `unknown` when the capture cannot report truncation, such as Appium page source. +Like `find`, `audit` publishes only the refs it returns. The text prints each as `@eN~s`; press that pinned ref. A plain ref from before the audit stays refused after the frame expired, and an audit with no findings issues no refs. An audit with findings also ends the snapshot's complete frame, so take a new snapshot before you use refs that the audit did not return. Each finding shows its role. Missing-name findings omit labels rejected by the name check and show the captured identifier when available. Other findings include the captured label when available. A control can get its name from a child or placeholder even when it has no label of its own. Guidance follows once per finding kind, with Apple HIG sizes on iOS and Material sizes on Android. `--json` returns `findings` (with `label` and `identifier` when available), `guidance` keyed by finding kind, `checks`, `limitations`, and `refsGeneration` (only with findings). `checks.snapshot` is `unknown` when the capture cannot report truncation, such as Appium page source. ## Wait and alerts diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 3057e0db55..5c69b8274b 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -160,13 +160,17 @@ when it has one, and its `rect`: } ``` -`coveredBy.kind` is the same presented role any other node carries, so a cover can be a generic -container — an alert's container often arrives as `group` — and the label is what makes it nameable. +A persistent bar publishes `persistentBar: true`. Its `coveredBy.kind` is `tab-bar`, `toolbar`, +`navigation-bar`, or `bottom-navigation`. The cover can be identified by its type, +role, or subrole. Other covers use their presented role. For example, an alert's container may +publish as `group` with the alert's accessible name. A `click`/`press`/`fill`/`find ... focus` refused on that node repeats the same description in -`error.details.coveredBy` and names it in `error.hint`. `coveredBy` is diagnosis only: it carries no +`error.details.coveredBy` and names it in `error.hint`. For a persistent bar, use a different visible target or inspect with a snapshot or screenshot +before retrying. Audit reports covered controls inside scroll containers. +`coveredBy` is diagnostic and has no ref, so the cover is not an actionable target. It is absent when nothing nameable covers — an -anonymous container with no accessible name, or a backend policy that reports only the boolean, such +anonymous container with no accessible name that is not a persistent bar, or a backend policy that reports only the boolean, such as an Android replacement surface — and those refusals say only that the target is covered. `role=` selectors (and `find role=...`) match the `kind` vocabulary: a node whose `kind` is `text`