fix(apple): refuse captures whose window or panel identity is unknown

apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift

@@ -16,14 +16,14 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro
     $0.owningApplication?.processID == pid && $0.windowLayer == 0 && $0.frame.width > 0
       && $0.frame.height > 0
   }
-  guard
-    let window = candidates.first(where: { Int($0.windowID) == frontWindowID })
-      ?? candidates.max(by: { $0.frame.width * $0.frame.height < $1.frame.width * $1.frame.height })
-  else {
-    throw HelperError.commandFailed(
-      "screenshot could not find a window for the app",
-      details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""]
-    )
+  let window: SCWindow
+  let selection = frontCaptureWindow(
+    candidates, frontWindowID: frontWindowID, windowID: { Int($0.windowID) })
+  switch selection {
+  case .success(let front):
+    window = front
+  case .failure(let refusal):
+    throw screenshotRefusal(refusal, app: app, frontWindowID: frontWindowID)
   }
 
   let filter = SCContentFilter(desktopIndependentWindow: window)
@@ -52,6 +52,45 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro
   try writePNG(capturedImage, to: outPath)
 }
 
+/// Why an app screenshot refuses. The capture is the app's front window or nothing: another
+/// window of the app would show state the user is not looking at. Pinned with the host by
+/// `contracts/fixtures/macos-native-helper-outcomes.json`.
+enum AppWindowScreenshotRefusal: String, CaseIterable, Error {
+  case windowNotFound = "window-not-found"
+  case frontWindowNotShareable = "front-window-not-shareable"
+}
+
+/// The shareable candidate whose WindowServer ID is the app's front window.
+func frontCaptureWindow<Window>(
+  _ candidates: [Window],
+  frontWindowID: Int?,
+  windowID: (Window) -> Int
+) -> Result<Window, AppWindowScreenshotRefusal> {
+  guard let frontWindowID else { return .failure(.windowNotFound) }
+  guard let front = candidates.first(where: { windowID($0) == frontWindowID }) else {
+    return .failure(.frontWindowNotShareable)
+  }
+  return .success(front)
+}
+
+private func screenshotRefusal(
+  _ refusal: AppWindowScreenshotRefusal,
+  app: NSRunningApplication,
+  frontWindowID: Int?
+) -> HelperError {
+  var details = ["reason": refusal.rawValue, "bundleId": app.bundleIdentifier ?? ""]
+  switch refusal {
+  case .windowNotFound:
+    return .commandFailed("screenshot could not find an on-screen window for the app", details: details)
+  case .frontWindowNotShareable:
+    details["frontWindowId"] = frontWindowID.map(String.init) ?? ""
+    return .commandFailed(
+      "screenshot could not capture the app's front window: ScreenCaptureKit does not offer it",
+      details: details
+    )
+  }
+}
+
 @available(macOS 14.0, *)
 private func awaitShareableContent() throws -> SCShareableContent {
   let semaphore = DispatchSemaphore(value: 0)

apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift

@@ -0,0 +1,38 @@
+import XCTest
+
+@testable import AgentDeviceMacOSHelper
+
+private struct Candidate: Equatable {
+  let id: Int
+  let area: Double
+}
+
+/// An app screenshot captures the front window or refuses; it never substitutes another window.
+final class AppWindowScreenshotTests: XCTestCase {
+  func testSelectsTheFrontWindowEvenWhenAnotherIsLarger() {
+    let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 3, area: 100)]
+
+    let selected = frontCaptureWindow(candidates, frontWindowID: 3, windowID: \.id)
+
+    XCTAssertEqual(try selected.get(), Candidate(id: 3, area: 100))
+  }
+
+  func testRefusesWhenTheFrontWindowIsNotShareable() {
+    let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 8, area: 100)]
+
+    let selected = frontCaptureWindow(candidates, frontWindowID: 3, windowID: \.id)
+
+    XCTAssertThrowsError(try selected.get()) { error in
+      XCTAssertEqual(error as? AppWindowScreenshotRefusal, .frontWindowNotShareable)
+    }
+  }
+
+  func testRefusesWhenTheAppHasNoOnScreenWindow() {
+    let selected = frontCaptureWindow(
+      [Candidate(id: 7, area: 10_000)], frontWindowID: nil, windowID: \.id)
+
+    XCTAssertThrowsError(try selected.get()) { error in
+      XCTAssertEqual(error as? AppWindowScreenshotRefusal, .windowNotFound)
+    }
+  }
+}

apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift

@@ -26,6 +26,7 @@ private struct ScrollGestureFixture: Decodable {
 private struct HelperOutcomesFixture: Decodable {
   let refusalReasons: [String]
   let deliveryMechanisms: [String]
+  let screenshotRefusalReasons: [String]
 }
 
 /// `#filePath` is `<repo>/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/<file>`.
@@ -43,6 +44,8 @@ final class BackgroundInteractionTests: XCTestCase {
     )
     XCTAssertEqual(BackgroundRefusal.allCases.map(\.rawValue), fixture.refusalReasons)
     XCTAssertEqual(BackgroundDeliveryMechanism.allCases.map(\.rawValue), fixture.deliveryMechanisms)
+    XCTAssertEqual(
+      AppWindowScreenshotRefusal.allCases.map(\.rawValue), fixture.screenshotRefusalReasons)
   }
 
   /// The native backend scrolls as far as the runner does: its travel agrees with every case of

contracts/fixtures/macos-native-helper-outcomes.json

@@ -1,5 +1,5 @@
 {
-  "description": "Vocabulary the macOS helper reports for app-surface actions. Swift: BackgroundRefusal and BackgroundDeliveryMechanism in apple/macos-helper. TypeScript: the macOS helper client in packages/platform-apple/src/os/macos/helper.ts.",
+  "description": "Vocabulary the macOS helper reports for app-surface actions. Swift: BackgroundRefusal, BackgroundDeliveryMechanism, and AppWindowScreenshotRefusal in apple/macos-helper. TypeScript: the macOS helper client in packages/platform-apple/src/os/macos/helper.ts.",
   "refusalReasons": [
     "background-pointer-gesture",
     "no-accessible-target",
@@ -14,5 +14,6 @@
     "ax-confirm",
     "ax-scroll-bar",
     "key-events"
-  ]
+  ],
+  "screenshotRefusalReasons": ["window-not-found", "front-window-not-shareable"]
 }

packages/platform-apple/src/core/__tests__/display-inventory.test.ts

@@ -266,15 +266,26 @@ describe('buildInventory', () => {
     }
   });
 
-  test('names a capture even when no panel reports primary', () => {
+  test('names the sole lit panel when no panel reports primary', () => {
     const inventory = buildInventory([
-      { ...outerPanel, primary: false },
+      { ...outerPanel, power: 'dark', primary: false },
       { ...innerPanel, power: 'lit', primary: false },
     ]);
     assert.equal(inventory.multiScreen, true);
     assert.equal(inventory.ambiguous, true);
-    // A capture is still named: never the implicit black-producing default.
-    assert.ok(inventory.activeDisplay);
+    assert.equal(inventory.activeDisplay?.name, 'LCD-1');
+  });
+
+  test('names no panel when neither primary nor power identifies one', () => {
+    for (const power of ['lit', 'dark', 'unknown'] as const) {
+      const inventory = buildInventory([
+        { ...outerPanel, power, primary: false },
+        { ...innerPanel, power, primary: false },
+      ]);
+      assert.equal(inventory.multiScreen, true);
+      assert.equal(inventory.ambiguous, true);
+      assert.equal(inventory.activeDisplay, undefined, power);
+    }
   });
 
   test('keeps choosing the lit panel when a third panel is present', () => {
@@ -476,6 +487,58 @@ describe('resolveAppleCaptureTarget', () => {
     assert.equal(target.displayRotation, undefined);
   });
 
+  /** The closed-Duo payload with each panel's `primary` flag and backlight replaced. */
+  function duoReporting(panels: Record<number, { primary: boolean; backlightState: string }>) {
+    return JSON.stringify({
+      result: {
+        displays: IPHONE_DUO_CLOSED.result.displays.map((entry) => ({
+          ...entry,
+          ...panels[entry.displayId],
+        })),
+      },
+    });
+  }
+
+  test('names the primary panel when panel power is ambiguous', async () => {
+    mockReadHostTextFile.mockResolvedValue(
+      duoReporting({
+        1: { primary: true, backlightState: 'activeOn' },
+        3: { primary: false, backlightState: 'activeOn' },
+      }),
+    );
+    assert.equal((await resolveAppleCaptureTarget(IOS_TEST_SIMULATOR)).display?.name, 'LCD');
+  });
+
+  test('names the sole lit panel when no panel reports primary', async () => {
+    mockReadHostTextFile.mockResolvedValue(
+      duoReporting({
+        1: { primary: false, backlightState: 'off' },
+        3: { primary: false, backlightState: 'activeOn' },
+      }),
+    );
+    assert.equal((await resolveAppleCaptureTarget(IOS_TEST_SIMULATOR)).display?.name, 'LCD-1');
+  });
+
+  test('refuses instead of picking a panel by list order when no fact identifies one', async () => {
+    for (const backlightState of ['activeOn', 'off', 'futureState']) {
+      mockReadHostTextFile.mockResolvedValue(
+        duoReporting({
+          1: { primary: false, backlightState },
+          3: { primary: false, backlightState },
+        }),
+      );
+      await assert.rejects(
+        resolveAppleCaptureTarget(IOS_TEST_SIMULATOR),
+        (error: unknown) =>
+          error instanceof AppError &&
+          error.code === 'COMMAND_FAILED' &&
+          error.details?.reason === 'capture-panel-unidentified' &&
+          typeof error.details?.hint === 'string',
+        backlightState,
+      );
+    }
+  });
+
   test('reports nothing when CoreDevice cannot be asked', async () => {
     mockRunXcrun.mockResolvedValueOnce({
       exitCode: 1,

packages/platform-apple/src/core/display-inventory.ts

@@ -1,5 +1,6 @@
 import type { DeviceRotation } from '@agent-device/contracts/device';
 import type { DeviceInfo } from '@agent-device/kernel/device';
+import { AppError } from '@agent-device/kernel/errors';
 import { isCommandTimeoutError } from '@agent-device/host-kit/command';
 import { emitDiagnostic } from '@agent-device/host-kit/diagnostics';
 
@@ -45,6 +46,11 @@ export type AppleDisplayInventory = {
    * single-screen behavior and never route through a display selector.
    */
   multiScreen: boolean;
+  /**
+   * The panel a multi-panel capture names. Absent on a multi-panel device when
+   * neither panel power nor CoreDevice `primary` identifies a panel: list order is
+   * not identity, so the capture refuses instead.
+   */
   activeDisplay?: AppleDeviceDisplay;
   /**
    * True on a multi-panel device whose lit panel could not be decided, so the
@@ -258,7 +264,7 @@ export function buildInventory(displays: AppleDeviceDisplay[]): AppleDisplayInve
   // choice is the highest screen ID, which is exactly how a foldable capture comes
   // back all black and exits 0. When panel power is ambiguous the primary panel is
   // named explicitly and the ambiguity is reported, not hidden.
-  const captureDisplay = decided ? litPanels[0]! : (primaryPanel ?? litPanels[0] ?? panels[0]!);
+  const captureDisplay = decided ? litPanels[0]! : primaryPanel;
   const ambiguous = !decided || primaryPanel === undefined;
   if (ambiguous) {
     emitDiagnostic({
@@ -267,20 +273,27 @@ export function buildInventory(displays: AppleDeviceDisplay[]): AppleDisplayInve
       data: {
         reason: primaryPanel === undefined ? 'no-primary-panel' : 'lit-panel-count',
         litPanelCount: litPanels.length,
-        capturedDisplay: captureDisplay.name,
-        hint: 'Panel power did not identify exactly one lit panel, so the capture names the primary panel without claiming to know which panel the device is showing. Read the device pose from the app under test with UIHinge.status.',
+        ...(captureDisplay
+          ? {
+              capturedDisplay: captureDisplay.name,
+              hint: 'Panel power did not identify exactly one lit panel, so the capture names the primary panel without claiming to know which panel the device is showing. Read the device pose from the app under test with UIHinge.status.',
+            }
+          : { hint: UNIDENTIFIED_PANEL_HINT }),
       },
     });
   }
   return {
     displays: panels,
     multiScreen: true,
-    activeDisplay: captureDisplay,
+    ...(captureDisplay ? { activeDisplay: captureDisplay } : {}),
     ambiguous,
     unresolved: false,
   };
 }
 
+const UNIDENTIFIED_PANEL_HINT =
+  'CoreDevice reported no primary panel and panel power did not identify exactly one lit panel. Run `xcrun devicectl device info displays --device <udid>`, then retry after one panel reports a lit backlightState.';
+
 /**
  * Sanity-checks the panel geometry against a foldable's shape.
  *
@@ -315,8 +328,9 @@ function warnWhenPrimaryPanelIsNotTheSmallest(panels: AppleDeviceDisplay[]): voi
  * `simctl io screenshot` without `--display` silently picks the highest screen
  * ID, which on a closed foldable is the *dark* inner panel: the capture succeeds
  * and comes back black. A multi-panel device therefore always names a display
- * explicitly — even when panel power is ambiguous — and only a genuinely
- * single-panel device (or an unreadable toolchain) keeps the historic argv.
+ * explicitly — even when panel power is ambiguous — or refuses when CoreDevice
+ * identifies no panel. Only a genuinely single-panel device (or an unreadable
+ * toolchain) keeps the historic argv.
  */
 export async function resolveAppleCaptureDisplay(
   device: DeviceInfo,
@@ -336,7 +350,17 @@ export async function resolveAppleCaptureTarget(
   const inventory = await queryAppleDisplayInventory(device, options);
   if (inventory.unresolved) return {};
   if (inventory.multiScreen) {
-    return inventory.activeDisplay ? { display: inventory.activeDisplay } : {};
+    if (inventory.activeDisplay) return { display: inventory.activeDisplay };
+    throw new AppError(
+      'COMMAND_FAILED',
+      `CoreDevice did not identify which panel of ${device.name} to capture`,
+      {
+        deviceId: device.id,
+        reason: 'capture-panel-unidentified',
+        litPanelCount: inventory.displays.filter((display) => display.power === 'lit').length,
+        hint: UNIDENTIFIED_PANEL_HINT,
+      },
+    );
   }
   // Only a built-in panel is the screen simctl captures; an attached display says nothing about it.
   const panel = inventory.displays.find((entry) => entry.integrated);