diff --git a/android/snapshot-helper/README.md b/android/snapshot-helper/README.md index 536bdcf693..05f561166d 100644 --- a/android/snapshot-helper/README.md +++ b/android/snapshot-helper/README.md @@ -51,10 +51,19 @@ invocations can pass `0` when immediate capture during ongoing animation is pref acquisition has a separate 500 ms stabilization bound that is used only when no root is available or an active/focused window is temporarily missing its root; complete captures pay no additional wait. +Child reads run in batches of up to eight on a shared worker pool. This lets Android serve +multiple accessibility requests between animation frames on busy apps. XML retains depth-first +child order and the same depth/node limits. Each capture still clears the accessibility cache; +results are not reused across captures. + ## One-Shot Modes Passing `-e mode snapshot|viewport|gesture` selects what a single instrumentation run does; `snapshot` is the default and matches the `Run` section above. +The `viewport` mode reads application and keyboard bounds after the same best-effort idle wait +used by snapshots: a 100 ms quiet interval with a 500 ms maximum wait. This allows short window +transitions, including keyboard appearance, to settle without waiting indefinitely on live content. +Snapshot mode uses the idle wait configured above. ```sh # Read the interactive-window viewport without capturing a snapshot. @@ -96,6 +105,10 @@ closes that connection; the process stays alive for the next connection: the XML body after the header block, as described below. The response protocol literal is always `android-snapshot-helper-v1`, regardless of session or one-shot transport. +When an input method window overlaps the application's vertical swipe path, `viewport` responses +also include `keyboardX`, `keyboardY`, `keyboardWidth`, and `keyboardHeight`. All four fields are +omitted when no such window is reported; their absence does not establish that the keyboard is hidden. + ## Output Contract The APK emits instrumentation status records using diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityReadBatch.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityReadBatch.java new file mode 100644 index 0000000000..1ad9d1c94c --- /dev/null +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityReadBatch.java @@ -0,0 +1,99 @@ +package com.callstack.agentdevice.snapshothelper; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.FutureTask; + +/** + * Reads a bounded group of children concurrently and retains their traversal order. + * + *

Accessibility cache misses are served on the app's UI thread. Queuing siblings together lets + * the app serve multiple reads between animation frames. Workers are reused because Android keeps + * an AccessibilityInteractionClient for each calling thread for the lifetime of the process. + * Closing waits for every read before releasing nodes, including unread results after truncation + * or failure, so no worker outlives the parent node it is reading. + */ +final class AccessibilityReadBatch implements AutoCloseable { + interface Reader { + T read(int index); + } + + interface Releaser { + void release(T value); + } + + static final int MAX_READS = 8; + private static final ExecutorService READERS = + Executors.newFixedThreadPool( + MAX_READS, + task -> { + Thread thread = new Thread(task, "agent-device-accessibility-read"); + thread.setDaemon(true); + return thread; + }); + + private final List> reads = new ArrayList<>(); + private final Releaser release; + + AccessibilityReadBatch(int start, int count, Reader read, Releaser release) { + if (count < 1 || count > MAX_READS) { + throw new IllegalArgumentException("Accessibility read batch must contain 1 to 8 reads"); + } + this.release = release; + for (int offset = 0; offset < count; offset += 1) { + int index = start + offset; + FutureTask task = new FutureTask<>(() -> read.read(index)); + reads.add(task); + READERS.execute(task); + } + } + + T get(int offset) { + return await(reads.get(offset)); + } + + @Override + public void close() { + for (FutureTask read : reads) { + T node; + try { + node = await(read); + } catch (RuntimeException | Error ignored) { + // get() propagates read failures. Other reads must finish before the parent is released. + continue; + } + if (node != null) { + release.release(node); + } + } + } + + private static T await(FutureTask read) { + boolean interrupted = false; + try { + while (true) { + try { + return read.get(); + } catch (InterruptedException error) { + interrupted = true; + } catch (ExecutionException error) { + Throwable cause = error.getCause(); + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + if (cause instanceof Error) { + throw (Error) cause; + } + throw new IllegalStateException("Android accessibility child read failed", cause); + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } +} diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java index 532d47b094..f56e47767f 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeCapture.java @@ -7,13 +7,32 @@ import android.view.accessibility.AccessibilityWindowInfo; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeoutException; /** Captures and serializes the Android accessibility window tree. */ final class AccessibilityTreeCapture { + static final long DEFAULT_WAIT_FOR_IDLE_TIMEOUT_MS = 500; + static final long DEFAULT_WAIT_FOR_IDLE_QUIET_MS = 100; private static final long WINDOW_LIST_SETTLE_TIMEOUT_MS = 1_000; private AccessibilityTreeCapture() {} + static void prepare(UiAutomation automation) throws InterruptedException { + prepare(automation, DEFAULT_WAIT_FOR_IDLE_QUIET_MS, DEFAULT_WAIT_FOR_IDLE_TIMEOUT_MS); + } + + static void prepare(UiAutomation automation, long quietMs, long timeoutMs) + throws InterruptedException { + enableInteractiveWindowRetrieval(automation); + if (timeoutMs > 0) { + try { + automation.waitForIdle(Math.min(quietMs, timeoutMs), timeoutMs); + } catch (TimeoutException ignored) { + // A busy accessibility stream can still expose usable windows and nodes. + } + } + } + static Result capture( UiAutomation automation, int maxDepth, int maxNodes, long stabilizationTimeoutMs) throws InterruptedException, AccessibilityCaptureStabilizer.IncompleteCaptureException { @@ -105,7 +124,7 @@ private static void clearAccessibilityCacheViaServiceInfo(UiAutomation automatio * the window list to name an active window. The platform fills the list after the flag is set, so * a read straight after it can list only the status bar and none of the app's windows. */ - static void enableInteractiveWindowRetrieval(UiAutomation automation) + private static void enableInteractiveWindowRetrieval(UiAutomation automation) throws InterruptedException { if (!turnOnInteractiveWindowRetrieval(automation)) { return; diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java index 93491f8109..334fcaf115 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java @@ -7,7 +7,6 @@ import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction; import android.view.accessibility.AccessibilityWindowInfo; import java.util.List; -import java.util.Locale; /** Serializes accessibility nodes using the helper's host-consumed XML contract. */ final class AccessibilityTreeXml { @@ -91,16 +90,7 @@ static void appendNode( hasAccessibilityAction(node, AccessibilityAction.ACTION_SCROLL_BACKWARD))); } appendAttribute(xml, "password", Boolean.toString(node.isPassword())); - appendAttribute( - xml, - "bounds", - String.format( - Locale.ROOT, - "[%d,%d][%d,%d]", - bounds.left, - bounds.top, - bounds.right, - bounds.bottom)); + appendBoundsAttribute(xml, "bounds", bounds.left, bounds.top, bounds.right, bounds.bottom); int childCount = depth >= maxDepth ? 0 : node.getChildCount(); if (depth >= maxDepth && node.getChildCount() > 0) { @@ -112,20 +102,26 @@ static void appendNode( } xml.append(">"); - for (int index = 0; index < childCount; index += 1) { - if (stats.nodeCount >= maxNodes) { - stats.truncated = true; - break; - } - AccessibilityNodeInfo child = AccessibilityNodeReads.child(node, index); - if (child == null) { - continue; - } - try { - appendNode(xml, child, index, depth + 1, maxDepth, maxNodes, stats, null); - } finally { - child.recycle(); + int index = 0; + while (index < childCount && stats.nodeCount < maxNodes) { + int count = Math.min( + AccessibilityReadBatch.MAX_READS, Math.min(childCount - index, maxNodes - stats.nodeCount)); + int offset = 0; + try (AccessibilityReadBatch children = + new AccessibilityReadBatch<>( + index, count, childIndex -> AccessibilityNodeReads.child(node, childIndex), + AccessibilityNodeInfo::recycle)) { + for (; offset < count && stats.nodeCount < maxNodes; offset += 1) { + AccessibilityNodeInfo child = children.get(offset); + if (child != null) { + appendNode(xml, child, index + offset, depth + 1, maxDepth, maxNodes, stats, null); + } + } } + index += offset; + } + if (index < childCount) { + stats.truncated = true; } xml.append(""); } @@ -206,16 +202,9 @@ private static void appendWindowMetadata(StringBuilder xml, WindowMetadata metad appendAttribute(xml, "window-layer", Integer.toString(metadata.layer)); appendAttribute(xml, "window-active", Boolean.toString(metadata.active)); appendAttribute(xml, "window-focused", Boolean.toString(metadata.focused)); - appendAttribute( - xml, - "window-bounds", - String.format( - Locale.ROOT, - "[%d,%d][%d,%d]", - metadata.bounds.left, - metadata.bounds.top, - metadata.bounds.right, - metadata.bounds.bottom)); + appendBoundsAttribute( + xml, "window-bounds", metadata.bounds.left, metadata.bounds.top, + metadata.bounds.right, metadata.bounds.bottom); if (metadata.regionRect != null) { appendAttribute(xml, "window-region-rect", Boolean.toString(metadata.regionRect)); } @@ -230,6 +219,13 @@ private static void appendAttribute(StringBuilder xml, String name, CharSequence xml.append('"'); } + static void appendBoundsAttribute( + StringBuilder xml, String name, int left, int top, int right, int bottom) { + xml.append(' ').append(name).append("=\"[") + .append(left).append(',').append(top).append("][") + .append(right).append(',').append(bottom).append("]\""); + } + private static boolean hasAccessibilityAction( AccessibilityNodeInfo node, AccessibilityAction action) { List actions = node.getActionList(); diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/GestureViewportReader.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/GestureViewportReader.java index bccd801fa0..0a9e6cefc1 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/GestureViewportReader.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/GestureViewportReader.java @@ -6,7 +6,6 @@ import android.view.accessibility.AccessibilityWindowInfo; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeoutException; /** * Resolves the active application window bounds used to validate planned gestures, plus the input @@ -59,18 +58,13 @@ Rect toRect() { @SuppressWarnings("deprecation") static Reading readReading(UiAutomation automation) throws InterruptedException { - try { - automation.waitForIdle(100, 2_000); - } catch (TimeoutException ignored) { - // Window/root state can still be usable when the app is animating continuously. - } // UiAutomation.getWindows() transfers recyclable AccessibilityWindowInfo instances, and this // read runs repeatedly inside the persistent helper session: copy the bounds the precedence // below needs, then recycle every window before resolving. // UiAutomation.getWindows() answers with an empty list until interactive retrieval is on, which // is the same seam the tree capture already uses. Without it this read sees no windows at all and // the keyboard below is invisible to it. - AccessibilityTreeCapture.enableInteractiveWindowRetrieval(automation); + AccessibilityTreeCapture.prepare(automation); Rect activeBounds = null; Rect fallbackBounds = null; List inputMethodWindows = new ArrayList<>(); diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java index e72bf364f4..ed2dc03f11 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java @@ -24,10 +24,6 @@ public final class SnapshotInstrumentation extends Instrumentation { private static final long GESTURE_UI_AUTOMATION_CONNECT_TIMEOUT_MS = 8_000; private static final int CHUNK_SIZE = 2 * 1024; - // Match the host default: bounded wait for microinteraction reliability without the stock - // uiautomator idle tax. Direct callers can pass 0 when immediate capture is preferred. - private static final long DEFAULT_WAIT_FOR_IDLE_TIMEOUT_MS = 500; - private static final long DEFAULT_WAIT_FOR_IDLE_QUIET_MS = 100; private static final long DEFAULT_TIMEOUT_MS = 8_000; private static final long ROOT_CAPTURE_STABILIZATION_TIMEOUT_MS = 500; private static final int DEFAULT_MAX_DEPTH = 128; @@ -45,9 +41,11 @@ public void onCreate(Bundle arguments) { public void onStart() { super.onStart(); long waitForIdleTimeoutMs = - readLongArgument(arguments, "waitForIdleTimeoutMs", DEFAULT_WAIT_FOR_IDLE_TIMEOUT_MS); + readLongArgument( + arguments, "waitForIdleTimeoutMs", AccessibilityTreeCapture.DEFAULT_WAIT_FOR_IDLE_TIMEOUT_MS); long waitForIdleQuietMs = - readLongArgument(arguments, "waitForIdleQuietMs", DEFAULT_WAIT_FOR_IDLE_QUIET_MS); + readLongArgument( + arguments, "waitForIdleQuietMs", AccessibilityTreeCapture.DEFAULT_WAIT_FOR_IDLE_QUIET_MS); long timeoutMs = readLongArgument(arguments, "timeoutMs", DEFAULT_TIMEOUT_MS); int maxDepth = readIntArgument(arguments, "maxDepth", DEFAULT_MAX_DEPTH); int maxNodes = readIntArgument(arguments, "maxNodes", DEFAULT_MAX_NODES); @@ -357,18 +355,7 @@ private AccessibilityTreeCapture.Result captureXml( InterruptedException, AccessibilityCaptureStabilizer.IncompleteCaptureException { UiAutomation automation = getConnectedUiAutomation(timeoutMs); - AccessibilityTreeCapture.enableInteractiveWindowRetrieval(automation); - if (waitForIdleTimeoutMs > 0) { - try { - // Best-effort settle: wait for the accessibility stream to become idle, but require only - // a short quiet window once it does. Using the full timeout as the quiet window made every - // stable snapshot pay a fixed 500 ms tax. - long quietMs = Math.min(waitForIdleQuietMs, waitForIdleTimeoutMs); - automation.waitForIdle(quietMs, waitForIdleTimeoutMs); - } catch (TimeoutException ignored) { - // Busy or animated apps can still expose a usable root; capture whatever is available. - } - } + AccessibilityTreeCapture.prepare(automation, waitForIdleQuietMs, waitForIdleTimeoutMs); return AccessibilityTreeCapture.capture( automation, maxDepth, maxNodes, ROOT_CAPTURE_STABILIZATION_TIMEOUT_MS); } diff --git a/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/AccessibilityReadBatchTest.java b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/AccessibilityReadBatchTest.java new file mode 100644 index 0000000000..a6e2c7222e --- /dev/null +++ b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/AccessibilityReadBatchTest.java @@ -0,0 +1,147 @@ +package com.callstack.agentdevice.snapshothelper; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public final class AccessibilityReadBatchTest { + private AccessibilityReadBatchTest() {} + + static void run() throws Exception { + queuesConcurrentReadsAndPreservesOrder(); + releasesUnreadResultsAfterFailure(); + waitsForReadersBeforeReleasingTheParent(); + preservesInterruptionDuringCleanup(); + } + + private static void queuesConcurrentReadsAndPreservesOrder() throws Exception { + CountDownLatch started = new CountDownLatch(AccessibilityReadBatch.MAX_READS); + CountDownLatch release = new CountDownLatch(1); + List recycled = new ArrayList<>(); + try (AccessibilityReadBatch batch = + new AccessibilityReadBatch<>( + 10, + AccessibilityReadBatch.MAX_READS, + index -> { + started.countDown(); + await(release); + return index; + }, + recycled::add)) { + try { + if (!started.await(1, TimeUnit.SECONDS)) { + throw new AssertionError("Sibling reads must start before any read completes"); + } + } finally { + release.countDown(); + } + for (int offset = 0; offset < AccessibilityReadBatch.MAX_READS; offset += 1) { + if (batch.get(offset) != 10 + offset) { + throw new AssertionError("Child indexes must retain traversal order"); + } + } + } + if (!recycled.equals(Arrays.asList(10, 11, 12, 13, 14, 15, 16, 17))) { + throw new AssertionError("Each child must be released once: " + recycled); + } + } + + private static void releasesUnreadResultsAfterFailure() { + IllegalStateException failure = new IllegalStateException("window disappeared"); + List recycled = new ArrayList<>(); + try (AccessibilityReadBatch batch = + new AccessibilityReadBatch<>( + 0, 4, + index -> { + if (index == 1) { + throw failure; + } + return index == 2 ? null : index; + }, + recycled::add)) { + if (batch.get(0) != 0) { + throw new AssertionError("Expected the first child"); + } + try { + batch.get(1); + throw new AssertionError("Expected the original accessibility failure"); + } catch (IllegalStateException actual) { + if (actual != failure) { + throw new AssertionError("Read failures must not be wrapped", actual); + } + } + } + if (!recycled.equals(Arrays.asList(0, 3))) { + throw new AssertionError("Unread children must be released after failure: " + recycled); + } + } + + private static void waitsForReadersBeforeReleasingTheParent() throws Exception { + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch closed = new CountDownLatch(1); + List recycled = new ArrayList<>(); + AccessibilityReadBatch batch = + new AccessibilityReadBatch<>(0, 1, index -> { + started.countDown(); + await(release); + return index; + }, recycled::add); + Thread closer = new Thread(() -> { + batch.close(); + closed.countDown(); + }); + try { + if (!started.await(1, TimeUnit.SECONDS)) { + throw new AssertionError("Expected a running child read"); + } + closer.start(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + while (closer.getState() != Thread.State.WAITING + && closed.getCount() != 0 + && System.nanoTime() < deadline) { + Thread.yield(); + } + if (closer.getState() != Thread.State.WAITING || closed.getCount() == 0) { + throw new AssertionError("Cleanup must block on the pending child read before release"); + } + } finally { + release.countDown(); + closer.join(1_000); + } + if (closed.getCount() != 0 || !recycled.equals(Arrays.asList(0))) { + throw new AssertionError("Cleanup must wait for and release the unread child"); + } + } + + private static void preservesInterruptionDuringCleanup() { + List recycled = new ArrayList<>(); + Thread.currentThread().interrupt(); + try (AccessibilityReadBatch batch = + new AccessibilityReadBatch<>(0, 2, index -> index, recycled::add)) { + if (batch.get(1) != 1) { + throw new AssertionError("Expected the completed child"); + } + } finally { + if (!Thread.interrupted()) { + throw new AssertionError("Cleanup must preserve the caller's interrupt status"); + } + } + if (!recycled.equals(Arrays.asList(0, 1))) { + throw new AssertionError("Interruption must not leak child nodes"); + } + } + + private static void await(CountDownLatch latch) { + try { + if (!latch.await(2, TimeUnit.SECONDS)) { + throw new AssertionError("Timed out waiting for the test to release a reader"); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new AssertionError(error); + } + } +} diff --git a/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXmlTest.java b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXmlTest.java new file mode 100644 index 0000000000..1b00898018 --- /dev/null +++ b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXmlTest.java @@ -0,0 +1,21 @@ +package com.callstack.agentdevice.snapshothelper; + +public final class AccessibilityTreeXmlTest { + private AccessibilityTreeXmlTest() {} + + static void run() { + assertBounds("bounds", -40, 0, 1080, 2400, " bounds=\"[-40,0][1080,2400]\""); + assertBounds( + "window-bounds", Integer.MIN_VALUE, -1, 0, Integer.MAX_VALUE, + " window-bounds=\"[-2147483648,-1][0,2147483647]\""); + } + + private static void assertBounds( + String name, int left, int top, int right, int bottom, String expected) { + StringBuilder xml = new StringBuilder(); + AccessibilityTreeXml.appendBoundsAttribute(xml, name, left, top, right, bottom); + if (!expected.equals(xml.toString())) { + throw new AssertionError("Expected " + expected + ", got " + xml); + } + } +} diff --git a/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java index e7913ac8b4..ad86760372 100644 --- a/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java +++ b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java @@ -6,6 +6,8 @@ private SnapshotHelperTestSuite() {} public static void main(String[] args) throws Exception { PointerEventScheduleTest.run(); AccessibilityCaptureStabilizerTest.run(); + AccessibilityReadBatchTest.run(); + AccessibilityTreeXmlTest.run(); BoundedUiAutomationConnectionTest.run(); GestureViewportReaderTest.run(); DisplayExtentTest.run();