Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
38860ef
feat(audit): name findings and explain platform guidance
thymikee Nov 17, 2026
2bae5b6
fix(audit): exclude UIKit controls from target size findings
thymikee Nov 17, 2026
aee392e
fix(audit): bound bar ownership and type audit fixtures
thymikee Nov 17, 2026
8b1de76
fix(audit): reuse occlusion bar identity for size exemptions
thymikee Nov 17, 2026
7cefe54
fix(audit): exclude scroll rows under bars through a typed persistent…
thymikee Nov 18, 2026
a592890
test(find): expect the persistentBar fact on a scanned tab-bar cover
thymikee Nov 18, 2026
670a93f
fix(occlusion): name an unnamed generic-typed bar cover by its bar kind
thymikee Nov 18, 2026
94484e5
fix(contracts): keep the bar kind type private to its module
thymikee Nov 18, 2026
8d521f1
fix(occlusion): suggest a scroll past a persistent bar only for scrol…
thymikee Nov 18, 2026
5ba9a23
fix(audit): covered-control guidance dismisses only dismissable covers
thymikee Nov 18, 2026
66aadf5
fix(help,contracts): align manual-qa covered recovery with the error …
thymikee Nov 18, 2026
acd8238
fix(contracts): resolve a BottomNavigationBar name to bottom-navigation
thymikee Nov 18, 2026
f070be2
fix(audit): limit target size exemptions to native iOS bars
thymikee Nov 18, 2026
7d61884
fix(occlusion): require viewport evidence for scroll recovery
thymikee Nov 18, 2026
0a7e56e
fix(occlusion): require scroll direction evidence for persistent bars
thymikee Nov 18, 2026
95758cb
docs(audit): explain scroll recovery evidence and limits
thymikee Nov 18, 2026
ea1bbc8
refactor(audit): separate scroll clearance from finding output
thymikee Nov 18, 2026
6d872cc
fix(occlusion): require exact persistent bar identity
thymikee Nov 18, 2026
b584745
docs(audit): clarify grabber checks and rejected labels
thymikee Nov 18, 2026
c70a9dd
fix(occlusion): qualify persistent bar evidence by capture source
thymikee Nov 18, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions packages/capture-kit/src/snapshot-occlusion.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
annotateCoveredSnapshotNodes,
annotateSnapshotNodesCoveredByPolicy,
} from './snapshot-occlusion.ts';
import { buildSnapshotState } from './snapshot-state.ts';

// #1478 P5 codec-extraction regression: `annotateCoveredSnapshotNodes` marks
// a touch candidate `interactionBlocked: 'covered'` when a later, floating
Expand Down Expand Up @@ -258,6 +259,21 @@ test('annotateSnapshotNodesCoveredByPolicy publishes the cover a backend hit tes
assert.deepEqual(result?.coveredBy, { kind: 'button', label: 'Top story', rect: card.rect });
});

test('a labelled ARIA toolbar keeps its hit-test cover without claiming persistence', () => {
const rect = { x: 0, y: 100, width: 300, height: 60 };
const target: RawSnapshotNode = { index: 0, type: 'button', rect, hittable: true };

const [result] = annotateSnapshotNodesCoveredByPolicy([target], () => ({
type: 'toolbar',
role: 'toolbar',
label: 'Editing actions',
rect,
}));

assert.equal(result?.interactionBlocked, 'covered');
assert.deepEqual(result?.coveredBy, { kind: 'toolbar', label: 'Editing actions', rect });
});

test('annotateSnapshotNodesCoveredByPolicy names no cover when the hit element is an anonymous box', () => {
const target: RawSnapshotNode = {
index: 0,
Expand Down Expand Up @@ -707,6 +723,41 @@ test('a full-viewport toolbar container does not cover its application siblings'
assert.equal(annotated[1]?.interactionBlocked, undefined);
});

test.each([
{
type: 'com.google.android.material.bottomnavigation.BottomNavigationView',
cover: { kind: 'bottom-navigation', persistentBar: true },
},
{
type: 'com.example.BottomNavigationView',
cover: { kind: 'bottomnavigationview' },
},
])('Android snapshot assembly preserves qualified cover identity for $type', ({ type, cover }) => {
const rect = { x: 0, y: 0, width: 400, height: 800 };
const state = buildSnapshotState(
{
backend: 'android',
producer: 'android-uiautomator',
nodes: [
{ index: 0, type: 'application', rect },
{
index: 1,
type: 'button',
role: 'button',
parentIndex: 0,
label: 'Room',
rect: { x: 20, y: 100, width: 120, height: 50 },
},
{ index: 2, type, parentIndex: 0, label: 'Actions', rect },
],
},
undefined,
);

assert.equal(state.nodes[1]?.interactionBlocked, 'covered');
assert.deepEqual(state.nodes[1]?.coveredBy, { ...cover, label: 'Actions', rect });
});

test.for([
// #3883: on iOS 27 an app's 2pt top bar pushes the floating-bar container to y=2.
{
Expand Down
72 changes: 42 additions & 30 deletions packages/capture-kit/src/snapshot-occlusion.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ import { areRectsApproximatelyEqual, normalizeRect } from '@agent-device/kernel/
import { containsPoint, rectContains } from '@agent-device/kernel/rect';
import {
isScrollableNodeLike,
normalizeType,
type SnapshotProducer,
snapshotBarKind,
snapshotNodeKindIncludesAny as nodeKindIncludesAny,
isViewportRootNode,
} from '@agent-device/contracts/snapshot';

Expand All @@ -25,7 +27,6 @@ const OVERLAY_KIND_FRAGMENTS = [
'popover',
'menu',
];
const VIEWPORT_CHROME_KIND_FRAGMENTS = ['tabbar', 'toolbar', 'navigationbar'];
const FLOATING_BAR_CONTAINER_ROLE = '_UIFloatingBarContainerView';
/**
* How far below the viewport top a full-viewport chrome container may start, before the 0.5pt rect
Expand Down Expand Up @@ -97,6 +98,7 @@ function collectNeighbourhoodIndex(nodes: readonly RawSnapshotNode[]): Neighbour
}

export type SnapshotOcclusionOptions = {
producer?: SnapshotProducer;
/** Backend-declared overlay roots that use the shared geometric scan (for example Android IME). */
isAdditionalOverlayNode?: (node: RawSnapshotNode) => boolean;
};
Expand Down Expand Up @@ -133,7 +135,10 @@ export function annotateCoveredSnapshotNodes(
if (!isCandidateTouchNode(node)) continue;
const coveringNode = findCoveringNode(scan, position, node, options);
if (!coveringNode) continue;
const coveredBy = describeCoveringNode(coveringNode);
const coveredBy = describeCoveringNode(
coveringNode,
snapshotBarKind(coveringNode, options.producer),
);
coveredPositions.push(coveredBy ? { position, coveredBy } : { position });
}
return annotateCoveredPositions(nodes, coveredPositions);
Expand All @@ -142,6 +147,22 @@ export function annotateCoveredSnapshotNodes(
/** Presented roles that name a box rather than a surface a reader would recognize and dismiss. */
const ANONYMOUS_CONTAINER_KINDS = new Set(['element', 'group', 'other']);

const VIEWPORT_CHROME_KIND_FRAGMENTS = [
...TAB_BAR_KIND_FRAGMENTS,
...TOOLBAR_HOST_KIND_FRAGMENTS,
...NAVIGATION_BAR_KIND_FRAGMENTS,
];

/** Viewport hosts can expose internal classes such as UIKit tab platters. */
function isViewportChromeBarNode(
node: Pick<RawSnapshotNode, 'type' | 'role' | 'subrole'>,
): boolean {
return (
!nodeKindIncludesAny(node, ['bottomnavigation']) &&
nodeKindIncludesAny(node, VIEWPORT_CHROME_KIND_FRAGMENTS)
);
}

/**
* The bounded, non-actionable description of the cover the scan found, or nothing when the cover has
* no name. No ref is ever carried — see {@link SnapshotCoveringNode}.
Expand All @@ -152,22 +173,31 @@ const ANONYMOUS_CONTAINER_KINDS = new Set(['element', 'group', 'other']);
* the surface to dismiss. Strip that label and the same node would say `the group at (...)`, which
* points at a box nobody would recognize; an anonymous container of that shape therefore publishes
* no cover, and the refusal keeps its generic recovery. A named kind (`dialog`, `tab-bar`, `sheet`)
* still describes itself with no label to lean on.
* still describes itself with no label to lean on. Persistent bars always publish their canonical
* bar kind, including when the type is generic and only the role or subrole identifies the bar.
*/
function describeCoveringNode(node: CoveringNodeEvidence): SnapshotCoveringNode | null {
function describeCoveringNode(
node: CoveringNodeEvidence,
barKind?: ReturnType<typeof snapshotBarKind>,
): SnapshotCoveringNode | null {
const label = node.label?.trim();
const kind = formatRole(node.type ?? 'Element');
if (!label && ANONYMOUS_CONTAINER_KINDS.has(kind)) return null;
const typeKind = formatRole(node.type ?? 'Element');
const anonymous = ANONYMOUS_CONTAINER_KINDS.has(typeKind);
if (!label && !barKind && anonymous) return null;
const rect = positiveRect(node.rect);
return {
kind,
kind: barKind ?? typeKind,
...(label ? { label: label.slice(0, SNAPSHOT_COVERING_LABEL_MAX_LENGTH) } : {}),
...(rect ? { rect } : {}),
...(barKind ? { persistentBar: true } : {}),
};
}

/** What a backend hit test observed on top of a node, described like a scanned cover. */
export type CoveringNodeEvidence = Pick<RawSnapshotNode, 'type' | 'label' | 'rect'>;
export type CoveringNodeEvidence = Pick<
RawSnapshotNode,
'type' | 'role' | 'subrole' | 'label' | 'rect'
>;

/**
* Applies backend-owned covered decisions through the same publication annotation contract. A
Expand Down Expand Up @@ -380,7 +410,7 @@ function isSiblingSizedChromeContainer(
node: RawSnapshotNode,
neighbourhood: NeighbourhoodIndex,
): boolean {
if (!nodeKindIncludesAny(node, VIEWPORT_CHROME_KIND_FRAGMENTS)) return false;
if (!isViewportChromeBarNode(node)) return false;
if (typeof node.parentIndex !== 'number') return false;
const rect = positiveRect(node.rect);
if (!rect) return false;
Expand Down Expand Up @@ -539,7 +569,7 @@ function isNavigationBarAround(bar: RawSnapshotNode, rect: Rect): boolean {
/** A bar or a scroll container inside `rect`; a scroll indicator is not a container. */
function isEnclosedHostedRegion(rect: Rect, node: RawSnapshotNode): boolean {
const isHostedRegion =
nodeKindIncludesAny(node, VIEWPORT_CHROME_KIND_FRAGMENTS) ||
isViewportChromeBarNode(node) ||
(isScrollableNodeLike(node) && !nodeKindIncludesAny(node, SCROLL_INDICATOR_KIND_FRAGMENTS));
const nodeRect = positiveRect(node.rect);
return Boolean(isHostedRegion && nodeRect && rectContains(rect, nodeRect));
Expand All @@ -549,7 +579,7 @@ function isFullViewportChromeContainer(
node: RawSnapshotNode,
neighbourhood: NeighbourhoodIndex,
): boolean {
if (!nodeKindIncludesAny(node, VIEWPORT_CHROME_KIND_FRAGMENTS)) return false;
if (!isViewportChromeBarNode(node)) return false;
const rect = positiveRect(node.rect);
if (!rect) return false;

Expand Down Expand Up @@ -665,24 +695,6 @@ function isSemanticTouchNode(node: RawSnapshotNode): boolean {
return nodeKindIncludesAny(node, SEMANTIC_TOUCH_KIND_FRAGMENTS);
}

function nodeKindIncludesAny(
node: Pick<RawSnapshotNode, 'type' | 'role' | 'subrole'>,
fragments: readonly string[],
): boolean {
const normalized = normalizeNodeKind(node);
return fragments.some((fragment) => normalized.includes(fragment));
}

// Mutation-lane note: the `?? ''` fallback's exact replacement text is not
// observable through this module's public API — every caller only checks
// substring membership against a fixed, known fragment list, and no
// plausible filler text coincides with any of them, so no test can
// distinguish `''` from another non-matching filler here without reaching
// into this private function directly.
function normalizeNodeKind(node: Pick<RawSnapshotNode, 'type' | 'role' | 'subrole'>): string {
return [node.type, node.role, node.subrole].map((value) => normalizeType(value ?? '')).join(' ');
}

function areRelatedSnapshotNodes(
left: RawSnapshotNode,
right: RawSnapshotNode,
Expand Down
5 changes: 4 additions & 1 deletion packages/capture-kit/src/snapshot-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,10 @@ export function buildSnapshotState(
snapshotRaw
? scopedNodes
: annotateUnmeasuredSnapshotNodes(
annotateCoveredSnapshotNodes(scopedNodes, overlayOptionsForBackend(data?.backend)),
annotateCoveredSnapshotNodes(scopedNodes, {
...overlayOptionsForBackend(data?.backend),
producer: data?.producer,
}),
),
snapshotRefMinting(data),
);
Expand Down
Loading
Loading