From 3fdd3aec9a45cf59db418f12fc2e501251760543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 22:09:55 +0200 Subject: [PATCH 1/2] fix(macos): refuse an app screenshot whose front window is not shareable ADR 0031 decides that a native macOS app screenshot captures only the app's front window through ScreenCaptureKit (#3188). When the front window's WindowServer ID was missing from the shareable candidates, the helper captured the app's largest window instead, so the image could show a window the user was not looking at. The helper now selects the front window by ID or refuses with a typed reason: `window-not-found` when the app has no on-screen window, `front-window-not-shareable` when ScreenCaptureKit does not offer the front window. The shared outcomes fixture pins both reasons for Swift and TypeScript, and the host adds a hint for each. Evidence: the audit at bd4fd688e7 (AppWindowScreenshot.swift:21); re-verified on 4c1c786968. Part of #3744. --- .../AppWindowScreenshot.swift | 55 ++++++++++++++++--- .../AppWindowScreenshotTests.swift | 38 +++++++++++++ .../BackgroundInteractionTests.swift | 3 + .../macos-native-helper-outcomes.json | 5 +- .../src/os/macos/helper.test.ts | 49 ++++++++++++++++- .../platform-apple/src/os/macos/helper.ts | 37 ++++++++++++- website/docs/docs/commands.md | 2 +- 7 files changed, 176 insertions(+), 13 deletions(-) create mode 100644 apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift diff --git a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift index da4c6247c7..d2b2522b07 100644 --- a/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift +++ b/apple/macos-helper/Sources/AgentDeviceMacOSHelper/AppWindowScreenshot.swift @@ -16,14 +16,14 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro $0.owningApplication?.processID == pid && $0.windowLayer == 0 && $0.frame.width > 0 && $0.frame.height > 0 } - guard - let window = candidates.first(where: { Int($0.windowID) == frontWindowID }) - ?? candidates.max(by: { $0.frame.width * $0.frame.height < $1.frame.width * $1.frame.height }) - else { - throw HelperError.commandFailed( - "screenshot could not find a window for the app", - details: ["reason": "window-not-found", "bundleId": app.bundleIdentifier ?? ""] - ) + let window: SCWindow + let selection = frontCaptureWindow( + candidates, frontWindowID: frontWindowID, windowID: { Int($0.windowID) }) + switch selection { + case .success(let front): + window = front + case .failure(let refusal): + throw screenshotRefusal(refusal, app: app, frontWindowID: frontWindowID) } let filter = SCContentFilter(desktopIndependentWindow: window) @@ -52,6 +52,45 @@ func captureAppWindowScreenshot(app: NSRunningApplication, outPath: String) thro try writePNG(capturedImage, to: outPath) } +/// Why an app screenshot refuses. The capture is the app's front window or nothing: another +/// window of the app would show state the user is not looking at. Pinned with the host by +/// `contracts/fixtures/macos-native-helper-outcomes.json`. +enum AppWindowScreenshotRefusal: String, CaseIterable, Error { + case windowNotFound = "window-not-found" + case frontWindowNotShareable = "front-window-not-shareable" +} + +/// The shareable candidate whose WindowServer ID is the app's front window. +func frontCaptureWindow( + _ candidates: [Window], + frontWindowID: Int?, + windowID: (Window) -> Int +) -> Result { + guard let frontWindowID else { return .failure(.windowNotFound) } + guard let front = candidates.first(where: { windowID($0) == frontWindowID }) else { + return .failure(.frontWindowNotShareable) + } + return .success(front) +} + +private func screenshotRefusal( + _ refusal: AppWindowScreenshotRefusal, + app: NSRunningApplication, + frontWindowID: Int? +) -> HelperError { + var details = ["reason": refusal.rawValue, "bundleId": app.bundleIdentifier ?? ""] + switch refusal { + case .windowNotFound: + return .commandFailed("screenshot could not find an on-screen window for the app", details: details) + case .frontWindowNotShareable: + details["frontWindowId"] = frontWindowID.map(String.init) ?? "" + return .commandFailed( + "screenshot could not capture the app's front window: ScreenCaptureKit does not offer it", + details: details + ) + } +} + @available(macOS 14.0, *) private func awaitShareableContent() throws -> SCShareableContent { let semaphore = DispatchSemaphore(value: 0) diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift new file mode 100644 index 0000000000..8ab7e3e278 --- /dev/null +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/AppWindowScreenshotTests.swift @@ -0,0 +1,38 @@ +import XCTest + +@testable import AgentDeviceMacOSHelper + +private struct Candidate: Equatable { + let id: Int + let area: Double +} + +/// An app screenshot captures the front window or refuses; it never substitutes another window. +final class AppWindowScreenshotTests: XCTestCase { + func testSelectsTheFrontWindowEvenWhenAnotherIsLarger() { + let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 3, area: 100)] + + let selected = frontCaptureWindow(candidates, frontWindowID: 3, windowID: \.id) + + XCTAssertEqual(try selected.get(), Candidate(id: 3, area: 100)) + } + + func testRefusesWhenTheFrontWindowIsNotShareable() { + let candidates = [Candidate(id: 7, area: 10_000), Candidate(id: 8, area: 100)] + + let selected = frontCaptureWindow(candidates, frontWindowID: 3, windowID: \.id) + + XCTAssertThrowsError(try selected.get()) { error in + XCTAssertEqual(error as? AppWindowScreenshotRefusal, .frontWindowNotShareable) + } + } + + func testRefusesWhenTheAppHasNoOnScreenWindow() { + let selected = frontCaptureWindow( + [Candidate(id: 7, area: 10_000)], frontWindowID: nil, windowID: \.id) + + XCTAssertThrowsError(try selected.get()) { error in + XCTAssertEqual(error as? AppWindowScreenshotRefusal, .windowNotFound) + } + } +} diff --git a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift index 7b61be5fc3..20445946df 100644 --- a/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift +++ b/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/BackgroundInteractionTests.swift @@ -26,6 +26,7 @@ private struct ScrollGestureFixture: Decodable { private struct HelperOutcomesFixture: Decodable { let refusalReasons: [String] let deliveryMechanisms: [String] + let screenshotRefusalReasons: [String] } /// `#filePath` is `/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/`. @@ -43,6 +44,8 @@ final class BackgroundInteractionTests: XCTestCase { ) XCTAssertEqual(BackgroundRefusal.allCases.map(\.rawValue), fixture.refusalReasons) XCTAssertEqual(BackgroundDeliveryMechanism.allCases.map(\.rawValue), fixture.deliveryMechanisms) + XCTAssertEqual( + AppWindowScreenshotRefusal.allCases.map(\.rawValue), fixture.screenshotRefusalReasons) } /// The native backend scrolls as far as the runner does: its travel agrees with every case of diff --git a/contracts/fixtures/macos-native-helper-outcomes.json b/contracts/fixtures/macos-native-helper-outcomes.json index be60f68c9b..c4337931fd 100644 --- a/contracts/fixtures/macos-native-helper-outcomes.json +++ b/contracts/fixtures/macos-native-helper-outcomes.json @@ -1,5 +1,5 @@ { - "description": "Vocabulary the macOS helper reports for app-surface actions. Swift: BackgroundRefusal and BackgroundDeliveryMechanism in apple/macos-helper. TypeScript: the macOS helper client in packages/platform-apple/src/os/macos/helper.ts.", + "description": "Vocabulary the macOS helper reports for app-surface actions. Swift: BackgroundRefusal, BackgroundDeliveryMechanism, and AppWindowScreenshotRefusal in apple/macos-helper. TypeScript: the macOS helper client in packages/platform-apple/src/os/macos/helper.ts.", "refusalReasons": [ "background-pointer-gesture", "no-accessible-target", @@ -14,5 +14,6 @@ "ax-confirm", "ax-scroll-bar", "key-events" - ] + ], + "screenshotRefusalReasons": ["window-not-found", "front-window-not-shareable"] } diff --git a/packages/platform-apple/src/os/macos/helper.test.ts b/packages/platform-apple/src/os/macos/helper.test.ts index 777c0caa92..92857ed84d 100644 --- a/packages/platform-apple/src/os/macos/helper.test.ts +++ b/packages/platform-apple/src/os/macos/helper.test.ts @@ -1,11 +1,13 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { macOsHelperSurface } from '@agent-device/contracts/session'; +import { AppError } from '@agent-device/kernel/errors'; import { createLocalAppleToolProvider, withAppleToolProvider } from '../../core/tool-provider.ts'; import { readFileSync } from 'node:fs'; import { MACOS_DELIVERY_MECHANISMS, MACOS_HELPER_REFUSAL_REASONS, + MACOS_SCREENSHOT_REFUSAL_REASONS, macOsClickScheduleMs, runMacOsPressAction, runMacOsReadTextAction, @@ -16,6 +18,7 @@ import { const desktop = macOsHelperSurface('desktop', 'xctest')!; const menubar = macOsHelperSurface('menubar', 'xctest')!; const frontmostApp = macOsHelperSurface('frontmost-app', 'xctest')!; +const nativeApp = macOsHelperSurface('app', 'native')!; test('macOS helper snapshot passes cancellation to the helper process', async () => { const controller = new AbortController(); @@ -241,6 +244,45 @@ test('macOS helper screenshot argv carries only --out and --surface', async () = assert.deepEqual(receivedArgs, ['screenshot', '--out', '/tmp/out.png', '--surface', 'desktop']); }); +test('an app screenshot that cannot reach the front window refuses with its reason and a hint', async () => { + const provider = createLocalAppleToolProvider({ + macosHelper: { + run: async () => ({ + exitCode: 1, + stdout: JSON.stringify({ + ok: false, + error: { + message: + "screenshot could not capture the app's front window: ScreenCaptureKit does not offer it", + details: { + reason: 'front-window-not-shareable', + bundleId: 'com.apple.TextEdit', + frontWindowId: '4242', + }, + }, + }), + stderr: '', + }), + }, + }); + + const error = await withAppleToolProvider(provider, async () => + runMacOsScreenshotAction('/tmp/out.png', { + surface: nativeApp, + bundleId: 'com.apple.TextEdit', + }).then( + () => undefined, + (error: unknown) => error, + ), + ); + + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'front-window-not-shareable'); + assert.equal(error.details?.frontWindowId, '4242'); + assert.match(String(error.details?.hint), /Retry after the window settles/); +}); + test('helper entry points accept only an owner-routed surface', () => { // Never invoked: each directive fails typecheck once its entry point widens back to an // unbranded or optional surface. @@ -268,7 +310,12 @@ test('the helper vocabulary matches the table the Swift helper is held to', () = ), 'utf8', ), - ) as { refusalReasons: string[]; deliveryMechanisms: string[] }; + ) as { + refusalReasons: string[]; + deliveryMechanisms: string[]; + screenshotRefusalReasons: string[]; + }; assert.deepEqual([...MACOS_HELPER_REFUSAL_REASONS], fixture.refusalReasons); assert.deepEqual([...MACOS_DELIVERY_MECHANISMS], fixture.deliveryMechanisms); + assert.deepEqual([...MACOS_SCREENSHOT_REFUSAL_REASONS], fixture.screenshotRefusalReasons); }); diff --git a/packages/platform-apple/src/os/macos/helper.ts b/packages/platform-apple/src/os/macos/helper.ts index 97cc51f146..8f7de6a1b8 100644 --- a/packages/platform-apple/src/os/macos/helper.ts +++ b/packages/platform-apple/src/os/macos/helper.ts @@ -142,6 +142,21 @@ export const MACOS_HELPER_REFUSAL_REASONS = [ 'no-settable-text-input', 'no-scroll-bar', ] as const; +/** + * Why the helper refused an app-surface screenshot. It captures the app's front window or + * nothing, so each refusal names what kept that window out of reach. + */ +export const MACOS_SCREENSHOT_REFUSAL_REASONS = [ + 'window-not-found', + 'front-window-not-shareable', +] as const; +type MacOsScreenshotRefusalReason = (typeof MACOS_SCREENSHOT_REFUSAL_REASONS)[number]; +const MACOS_SCREENSHOT_REFUSAL_HINTS: Record = { + 'window-not-found': + 'Open or unminimize a window of the app on the current Space, then retry the screenshot.', + 'front-window-not-shareable': + 'The front window may still be opening or closing, or the app excludes it from screen capture. Retry after the window settles.', +}; export const MACOS_DELIVERY_MECHANISMS = [ 'ax-press', 'ax-focus', @@ -581,5 +596,25 @@ export async function runMacOsScreenshotAction( }> { const args = ['screenshot', '--out', outPath]; appendMacOsHelperContextArgs(args, options); - return await runMacOsHelper(args, { signal: options.signal }); + try { + return await runMacOsHelper(args, { signal: options.signal }); + } catch (error) { + throw withScreenshotRefusalHint(error); + } +} + +function withScreenshotRefusalHint(error: unknown): unknown { + if (!(error instanceof AppError)) return error; + const reason = error.details?.reason; + if (!isMacOsScreenshotRefusalReason(reason)) return error; + return new AppError( + error.code, + error.message, + { ...error.details, hint: MACOS_SCREENSHOT_REFUSAL_HINTS[reason] }, + error, + ); +} + +function isMacOsScreenshotRefusalReason(value: unknown): value is MacOsScreenshotRefusalReason { + return (MACOS_SCREENSHOT_REFUSAL_REASONS as readonly unknown[]).includes(value); } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index e0014faa7f..8ce4a66430 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -321,7 +321,7 @@ agent-device snapshot -i --platform apple --target desktop Set `AGENT_DEVICE_MACOS_APP_BACKEND=native` to run macOS app sessions without XCTest. There is no Automation Mode overlay, the app can stay behind other windows, and your pointer stays yours while a drawn pointer shows each action. -- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Chromium-based apps expose their full tree. `screenshot` captures only the app's front window, even when it is covered. +- `click`, `press`, and `fill` use accessibility actions on the element at the target point, which must be inside one of the app's on-screen windows, not its menu bar. `type` inserts at the focused control or falls back to key events. `scroll` moves the scroll bar under the front window's center. Chromium-based apps expose their full tree. `screenshot` captures only the app's front window, even when it is covered. If that window cannot be captured, `screenshot` fails; it never captures another window. - `record`, `prepare`, `back`, `longpress`, `--hold-ms`, `--double-tap`, `--button secondary`, gestures, and clicks on an element with no accessibility action fail with `UNSUPPORTED_OPERATION` (`reason: unsupported-device-backend`). Use the default XCTest backend for those. ```bash From dec7c9ba418bcc23875a3ea26732304028253a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 16 Oct 2026 22:10:01 +0200 Subject: [PATCH 2/2] fix(apple): refuse a foldable capture when CoreDevice names no panel ADR 0025 decides that only CoreDevice facts identify a panel: the sole lit panel, otherwise the `primary` panel, and never array position. When no panel reported `primary` and power did not decide one lit panel, the inventory captured the first lit or first integrated panel in list order. That inventory now names no panel, and capture-target resolution refuses with COMMAND_FAILED, reason `capture-panel-unidentified`, and a hint. The sole-lit-panel path and the primary-panel path are unchanged. The foldable help topic states the refusal. Evidence: the audit at bd4fd688e7 (display-inventory.ts:261); re-verified on 4c1c786968. Part of #3744. --- .../core/__tests__/display-inventory.test.ts | 71 +++++++++++++++++-- .../src/core/display-inventory.ts | 38 ++++++++-- src/commands/system/index.ts | 2 +- 3 files changed, 99 insertions(+), 12 deletions(-) diff --git a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts index a476dafec9..416048de0f 100644 --- a/packages/platform-apple/src/core/__tests__/display-inventory.test.ts +++ b/packages/platform-apple/src/core/__tests__/display-inventory.test.ts @@ -266,15 +266,26 @@ describe('buildInventory', () => { } }); - test('names a capture even when no panel reports primary', () => { + test('names the sole lit panel when no panel reports primary', () => { const inventory = buildInventory([ - { ...outerPanel, primary: false }, + { ...outerPanel, power: 'dark', primary: false }, { ...innerPanel, power: 'lit', primary: false }, ]); assert.equal(inventory.multiScreen, true); assert.equal(inventory.ambiguous, true); - // A capture is still named: never the implicit black-producing default. - assert.ok(inventory.activeDisplay); + assert.equal(inventory.activeDisplay?.name, 'LCD-1'); + }); + + test('names no panel when neither primary nor power identifies one', () => { + for (const power of ['lit', 'dark', 'unknown'] as const) { + const inventory = buildInventory([ + { ...outerPanel, power, primary: false }, + { ...innerPanel, power, primary: false }, + ]); + assert.equal(inventory.multiScreen, true); + assert.equal(inventory.ambiguous, true); + assert.equal(inventory.activeDisplay, undefined, power); + } }); test('keeps choosing the lit panel when a third panel is present', () => { @@ -476,6 +487,58 @@ describe('resolveAppleCaptureTarget', () => { assert.equal(target.displayRotation, undefined); }); + /** The closed-Duo payload with each panel's `primary` flag and backlight replaced. */ + function duoReporting(panels: Record) { + return JSON.stringify({ + result: { + displays: IPHONE_DUO_CLOSED.result.displays.map((entry) => ({ + ...entry, + ...panels[entry.displayId], + })), + }, + }); + } + + test('names the primary panel when panel power is ambiguous', async () => { + mockReadHostTextFile.mockResolvedValue( + duoReporting({ + 1: { primary: true, backlightState: 'activeOn' }, + 3: { primary: false, backlightState: 'activeOn' }, + }), + ); + assert.equal((await resolveAppleCaptureTarget(IOS_TEST_SIMULATOR)).display?.name, 'LCD'); + }); + + test('names the sole lit panel when no panel reports primary', async () => { + mockReadHostTextFile.mockResolvedValue( + duoReporting({ + 1: { primary: false, backlightState: 'off' }, + 3: { primary: false, backlightState: 'activeOn' }, + }), + ); + assert.equal((await resolveAppleCaptureTarget(IOS_TEST_SIMULATOR)).display?.name, 'LCD-1'); + }); + + test('refuses instead of picking a panel by list order when no fact identifies one', async () => { + for (const backlightState of ['activeOn', 'off', 'futureState']) { + mockReadHostTextFile.mockResolvedValue( + duoReporting({ + 1: { primary: false, backlightState }, + 3: { primary: false, backlightState }, + }), + ); + await assert.rejects( + resolveAppleCaptureTarget(IOS_TEST_SIMULATOR), + (error: unknown) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + error.details?.reason === 'capture-panel-unidentified' && + typeof error.details?.hint === 'string', + backlightState, + ); + } + }); + test('reports nothing when CoreDevice cannot be asked', async () => { mockRunXcrun.mockResolvedValueOnce({ exitCode: 1, diff --git a/packages/platform-apple/src/core/display-inventory.ts b/packages/platform-apple/src/core/display-inventory.ts index 6dd4fbee2a..1eee41c9f1 100644 --- a/packages/platform-apple/src/core/display-inventory.ts +++ b/packages/platform-apple/src/core/display-inventory.ts @@ -1,5 +1,6 @@ import type { DeviceRotation } from '@agent-device/contracts/device'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import { AppError } from '@agent-device/kernel/errors'; import { isCommandTimeoutError } from '@agent-device/host-kit/command'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; @@ -45,6 +46,11 @@ export type AppleDisplayInventory = { * single-screen behavior and never route through a display selector. */ multiScreen: boolean; + /** + * The panel a multi-panel capture names. Absent on a multi-panel device when + * neither panel power nor CoreDevice `primary` identifies a panel: list order is + * not identity, so the capture refuses instead. + */ activeDisplay?: AppleDeviceDisplay; /** * True on a multi-panel device whose lit panel could not be decided, so the @@ -258,7 +264,7 @@ export function buildInventory(displays: AppleDeviceDisplay[]): AppleDisplayInve // choice is the highest screen ID, which is exactly how a foldable capture comes // back all black and exits 0. When panel power is ambiguous the primary panel is // named explicitly and the ambiguity is reported, not hidden. - const captureDisplay = decided ? litPanels[0]! : (primaryPanel ?? litPanels[0] ?? panels[0]!); + const captureDisplay = decided ? litPanels[0]! : primaryPanel; const ambiguous = !decided || primaryPanel === undefined; if (ambiguous) { emitDiagnostic({ @@ -267,20 +273,27 @@ export function buildInventory(displays: AppleDeviceDisplay[]): AppleDisplayInve data: { reason: primaryPanel === undefined ? 'no-primary-panel' : 'lit-panel-count', litPanelCount: litPanels.length, - capturedDisplay: captureDisplay.name, - hint: 'Panel power did not identify exactly one lit panel, so the capture names the primary panel without claiming to know which panel the device is showing. Read the device pose from the app under test with UIHinge.status.', + ...(captureDisplay + ? { + capturedDisplay: captureDisplay.name, + hint: 'Panel power did not identify exactly one lit panel, so the capture names the primary panel without claiming to know which panel the device is showing. Read the device pose from the app under test with UIHinge.status.', + } + : { hint: UNIDENTIFIED_PANEL_HINT }), }, }); } return { displays: panels, multiScreen: true, - activeDisplay: captureDisplay, + ...(captureDisplay ? { activeDisplay: captureDisplay } : {}), ambiguous, unresolved: false, }; } +const UNIDENTIFIED_PANEL_HINT = + 'CoreDevice reported no primary panel and panel power did not identify exactly one lit panel. Run `xcrun devicectl device info displays --device `, then retry after one panel reports a lit backlightState.'; + /** * Sanity-checks the panel geometry against a foldable's shape. * @@ -315,8 +328,9 @@ function warnWhenPrimaryPanelIsNotTheSmallest(panels: AppleDeviceDisplay[]): voi * `simctl io screenshot` without `--display` silently picks the highest screen * ID, which on a closed foldable is the *dark* inner panel: the capture succeeds * and comes back black. A multi-panel device therefore always names a display - * explicitly — even when panel power is ambiguous — and only a genuinely - * single-panel device (or an unreadable toolchain) keeps the historic argv. + * explicitly — even when panel power is ambiguous — or refuses when CoreDevice + * identifies no panel. Only a genuinely single-panel device (or an unreadable + * toolchain) keeps the historic argv. */ export async function resolveAppleCaptureDisplay( device: DeviceInfo, @@ -336,7 +350,17 @@ export async function resolveAppleCaptureTarget( const inventory = await queryAppleDisplayInventory(device, options); if (inventory.unresolved) return {}; if (inventory.multiScreen) { - return inventory.activeDisplay ? { display: inventory.activeDisplay } : {}; + if (inventory.activeDisplay) return { display: inventory.activeDisplay }; + throw new AppError( + 'COMMAND_FAILED', + `CoreDevice did not identify which panel of ${device.name} to capture`, + { + deviceId: device.id, + reason: 'capture-panel-unidentified', + litPanelCount: inventory.displays.filter((display) => display.power === 'lit').length, + hint: UNIDENTIFIED_PANEL_HINT, + }, + ); } // Only a built-in panel is the screen simctl captures; an attached display says nothing about it. const panel = inventory.displays.find((entry) => entry.integrated); diff --git a/src/commands/system/index.ts b/src/commands/system/index.ts index 7f17fd3da9..63ec1914d4 100644 --- a/src/commands/system/index.ts +++ b/src/commands/system/index.ts @@ -278,7 +278,7 @@ export const foldableHelpTopic = { A foldable Apple device (iPhone Duo) carries two integrated panels, which Apple calls the outer display and the inner display. Only one is lit at a time, and which one is lit is the device pose. Screens are handled for you: - Each iOS simulator capture resolves the CoreDevice display table, captures the lit panel explicitly, and normalizes density with that panel's own point scale. Do not add a screen flag to the normal loop; there is none, because the lit panel is always the only capturable one: the dark panel yields an all-black PNG. + Each iOS simulator capture resolves the CoreDevice display table, captures the lit panel explicitly, and normalizes density with that panel's own point scale. Do not add a screen flag to the normal loop; there is none, because the lit panel is always the only capturable one: the dark panel yields an all-black PNG. When panel power names no single lit panel, the capture names CoreDevice's primary panel; with no primary panel either, it fails with COMMAND_FAILED and reason capture-panel-unidentified rather than pick a panel by list order. The two panels are different sizes (iPhone Duo: 466x678 points closed on the outer panel, 669x951 open on the inner). A pose change therefore invalidates every ref and coordinate. Re-snapshot after any pose change and never carry coordinates or refs across one. Check which panel is lit before trusting a geometry claim: agent-device screenshot reports its point size, and 466x678 versus 669x951 says which panel you captured.