diff --git a/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift b/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift index ccfe990e8d..0817e70c8d 100644 --- a/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift +++ b/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift @@ -68,7 +68,11 @@ func run() throws { let payload = try Data(contentsOf: eventsURL) let envelope = try JSONDecoder().decode(GestureEnvelope.self, from: payload) - if envelope.events.isEmpty { + // An event whose reference pair is missing or not finite and positive has no proven transform + // into the captured canvas, so its marker is omitted (#4448); the telemetry event and the video + // are preserved either way. With nothing left to render, the raw capture is kept as-is. + let renderableEvents = envelope.events.filter { renderableReference(of: $0) != nil } + if renderableEvents.isEmpty { try FileManager.default.copyItem(at: inputURL, to: outputURL) return } @@ -116,7 +120,7 @@ func run() throws { overlayLayer.frame = parentLayer.frame parentLayer.addSublayer(overlayLayer) - for event in envelope.events { + for event in renderableEvents { switch event.kind { case "tap": addTapLayer(event: event, renderSize: renderSize, to: overlayLayer) @@ -381,65 +385,86 @@ func resolvedFrameDuration(for track: AVAssetTrack, maximumFrameRate: Int32) -> return shortest } -func overlayPoint(event: GestureEvent, x: Double, y: Double, renderSize: CGSize) -> CGPoint { - let scaleX = scaledAxis(renderSize: renderSize.width, referenceSize: event.referenceWidth) - let scaleY = scaledAxis(renderSize: renderSize.height, referenceSize: event.referenceHeight) - let scaledX = x * scaleX - let scaledY = y * scaleY +/// The paired reference an event needs before its coordinates can be projected into the captured +/// canvas: both axes present, finite, and positive. The old scale=1 fallback rendered point-space +/// coordinates as video pixels (#4448); an unproven reference now omits the marker instead. +func renderableReference(of event: GestureEvent) -> (width: Double, height: Double)? { + guard let width = event.referenceWidth, let height = event.referenceHeight, + width.isFinite, height.isFinite, width > 0, height > 0 + else { return nil } + return (width, height) +} + +/// Projects an event coordinate into the render space, or returns nil when the event carries no +/// renderable reference — the caller then omits only this event's marker layers. +func overlayPoint(event: GestureEvent, x: Double, y: Double, renderSize: CGSize) -> CGPoint? { + guard let reference = renderableReference(of: event) else { return nil } + let scaledX = x * (Double(renderSize.width) / reference.width) + let scaledY = y * (Double(renderSize.height) / reference.height) let flippedY = max(0, Double(renderSize.height) - scaledY) return CGPoint(x: scaledX, y: flippedY) } -func scaledAxis(renderSize: CGFloat, referenceSize: Double?) -> Double { - guard let referenceSize, referenceSize > 0 else { return 1.0 } - return Double(renderSize) / referenceSize +/// Adds the event's touch dot, or returns nil when the event has no renderable reference and its +/// marker is omitted. +func addEventTouchDot(event: GestureEvent, renderSize: CGSize, to overlayLayer: CALayer) -> CALayer? { + guard let center = overlayPoint(event: event, x: event.x, y: event.y, renderSize: renderSize) + else { return nil } + let layer = makeTouchDotLayer(center: center, renderSize: renderSize) + overlayLayer.addSublayer(layer) + return layer } func addTapLayer(event: GestureEvent, renderSize: CGSize, to overlayLayer: CALayer) { - let layer = makeTouchDotLayer( - center: overlayPoint(event: event, x: event.x, y: event.y, renderSize: renderSize), - renderSize: renderSize - ) - overlayLayer.addSublayer(layer) - - let opacity = CAKeyframeAnimation(keyPath: "opacity") - opacity.values = [0.0, 0.98, 0.98, 0.0] - opacity.keyTimes = [0.0, 0.08, 0.8, 1.0] - - let scale = CAKeyframeAnimation(keyPath: "transform.scale") - scale.values = [0.84, 1.0, 1.0] - scale.keyTimes = [0.0, 0.22, 1.0] - - let group = makeAnimationGroup( - animations: [opacity, scale], + addPressLayer( + event: event, + renderSize: renderSize, + to: overlayLayer, duration: minimumTapVisibility, - beginTime: AVCoreAnimationBeginTimeAtZero + (event.tMs / 1000.0) + opacityKeyTimes: [0.0, 0.08, 0.8, 1.0], + scaleKeyTimes: [0.0, 0.22, 1.0], + animationKey: "tap" ) - layer.add(group, forKey: "tap") } func addLongPressLayer(event: GestureEvent, renderSize: CGSize, to overlayLayer: CALayer) { - let duration = max(0.75, (event.durationMs ?? 800) / 1000.0) - let layer = makeTouchDotLayer( - center: overlayPoint(event: event, x: event.x, y: event.y, renderSize: renderSize), - renderSize: renderSize + addPressLayer( + event: event, + renderSize: renderSize, + to: overlayLayer, + duration: max(0.75, (event.durationMs ?? 800) / 1000.0), + opacityKeyTimes: [0.0, 0.08, 0.92, 1.0], + scaleKeyTimes: [0.0, 0.15, 1.0], + animationKey: "longpress" ) - overlayLayer.addSublayer(layer) +} + +func addPressLayer( + event: GestureEvent, + renderSize: CGSize, + to overlayLayer: CALayer, + duration: CFTimeInterval, + opacityKeyTimes: [NSNumber], + scaleKeyTimes: [NSNumber], + animationKey: String +) { + guard let layer = addEventTouchDot(event: event, renderSize: renderSize, to: overlayLayer) + else { return } let opacity = CAKeyframeAnimation(keyPath: "opacity") opacity.values = [0.0, 0.98, 0.98, 0.0] - opacity.keyTimes = [0.0, 0.08, 0.92, 1.0] + opacity.keyTimes = opacityKeyTimes let scale = CAKeyframeAnimation(keyPath: "transform.scale") scale.values = [0.84, 1.0, 1.0] - scale.keyTimes = [0.0, 0.15, 1.0] + scale.keyTimes = scaleKeyTimes let group = makeAnimationGroup( animations: [opacity, scale], duration: duration, beginTime: AVCoreAnimationBeginTimeAtZero + (event.tMs / 1000.0) ) - layer.add(group, forKey: "longpress") + layer.add(group, forKey: animationKey) } func addSwipeLayers(event: GestureEvent, renderSize: CGSize, to overlayLayer: CALayer) { @@ -548,9 +573,10 @@ func addTrailLayers( to overlayLayer: CALayer, style: TrailStyle ) { - guard let x2 = event.x2, let y2 = event.y2 else { return } - let startPoint = overlayPoint(event: event, x: event.x, y: event.y, renderSize: renderSize) - let endPoint = overlayPoint(event: event, x: x2, y: y2, renderSize: renderSize) + guard let x2 = event.x2, let y2 = event.y2, + let startPoint = overlayPoint(event: event, x: event.x, y: event.y, renderSize: renderSize), + let endPoint = overlayPoint(event: event, x: x2, y: y2, renderSize: renderSize) + else { return } let duration = max(0.1, (event.durationMs ?? 250) / 1000.0) let visibleDuration = max(minimumSwipeVisibility, duration + style.tail) let beginTime = AVCoreAnimationBeginTimeAtZero + (event.tMs / 1000.0) @@ -659,6 +685,7 @@ func addPinchLayers(event: GestureEvent, renderSize: CGSize, to overlayLayer: CA let startRight = overlayPoint(event: event, x: event.x + Double(startOffset), y: event.y, renderSize: renderSize) let endLeft = overlayPoint(event: event, x: event.x - Double(endOffset), y: event.y, renderSize: renderSize) let endRight = overlayPoint(event: event, x: event.x + Double(endOffset), y: event.y, renderSize: renderSize) + guard let startLeft, let startRight, let endLeft, let endRight else { return } addPinchDot( overlayLayer: overlayLayer, diff --git a/packages/capture-kit/src/recording/__tests__/swift-cache-recording-scripts.test.ts b/packages/capture-kit/src/recording/__tests__/swift-cache-recording-scripts.test.ts index 4e369b67fc..3ee1817c8e 100644 --- a/packages/capture-kit/src/recording/__tests__/swift-cache-recording-scripts.test.ts +++ b/packages/capture-kit/src/recording/__tests__/swift-cache-recording-scripts.test.ts @@ -26,6 +26,7 @@ type TypecheckOutcome = { exitCode: number; stderr: string; source: string }; describe.skipIf(process.platform !== 'darwin')('recording Swift scripts', () => { let outcomes: TypecheckOutcome[] = []; let exports: OverlayExportOutcome[] = []; + let omissions: OmissionOutcome[] = []; beforeAll( async () => { @@ -58,9 +59,17 @@ describe.skipIf(process.platform !== 'darwin')('recording Swift scripts', () => return { source, exitCode: result.exitCode, stderr: result.stderr }; }), ); - exports = await exerciseOverlayExports(swiftCompilerPath, swiftSdkPath, sharedSupport); + const dir = await mkdtempForTest('recording-overlay-export-'); + const tools = await compileRecordingTools( + swiftCompilerPath, + swiftSdkPath, + sharedSupport, + dir, + ); + exports = await exerciseOverlayExports(tools, dir); + omissions = await exerciseOverlayReferenceOmission(tools, dir); }, - SWIFT_TYPECHECK_TIMEOUT_MS * 3 + 30_000, + SWIFT_TYPECHECK_TIMEOUT_MS * 3 + 120_000, ); test('recording inspect Swift script typechecks', () => { @@ -88,6 +97,42 @@ describe.skipIf(process.platform !== 'darwin')('recording Swift scripts', () => assert.ok(outcome.markerDistance! < 3, 'the asymmetric source marker must stay upright'); }); } + + test('overlay omits markers without a finite positive paired reference and keeps the raw video (#4448)', () => { + for (const name of ['missing-reference', 'unpaired-reference', 'non-positive-reference']) { + const outcome = omissions.find((candidate) => candidate.name === name); + assert.ok(outcome, `${name} was never exercised`); + assert.equal(outcome.exitCode, 0, outcome.stderr); + assert.equal( + outcome.byteIdentical, + true, + `${name}: omission must preserve the raw capture byte-for-byte`, + ); + assert.equal( + outcome.outputBluePixels, + 0, + `${name}: no marker may be drawn from an unproven reference`, + ); + assert.deepEqual(outcome.outputSize, outcome.sourceSize); + } + }); + + test('overlay renders only the referenced events when others lack a reference (#4448)', () => { + const outcome = omissions.find((candidate) => candidate.name === 'mixed'); + assert.ok(outcome, 'mixed was never exercised'); + assert.equal(outcome.exitCode, 0, outcome.stderr); + assert.equal( + outcome.byteIdentical, + false, + 'a referenced event still forces the composited export', + ); + assert.deepEqual(outcome.outputSize, outcome.sourceSize); + assert.ok(outcome.outputBluePixels! > 180, 'the referenced marker must draw'); + assert.ok( + outcome.outputBluePixels! <= 1500, + `only the referenced marker may draw, saw ${outcome.outputBluePixels} overlay pixels`, + ); + }); }); type OverlayExportOutcome = { @@ -103,8 +148,14 @@ type OverlayExportOutcome = { markerDistance?: number; }; -async function exerciseOverlayExports(compiler: string, sdk: string, sharedSupport: string) { - const dir = await mkdtempForTest('recording-overlay-export-'); +type RecordingToolBinaries = { overlay: string; fixture: string; inspect: string }; + +async function compileRecordingTools( + compiler: string, + sdk: string, + sharedSupport: string, + dir: string, +): Promise { const overlay = path.join(dir, 'overlay'); const fixture = path.join(dir, 'fixture'); const inspect = path.join(dir, 'inspect'); @@ -119,6 +170,11 @@ async function exerciseOverlayExports(compiler: string, sdk: string, sharedSuppo }); }), ); + return { overlay, fixture, inspect }; +} + +async function exerciseOverlayExports(tools: RecordingToolBinaries, dir: string) { + const { overlay, fixture, inspect } = tools; const results: OverlayExportOutcome[] = []; for (const transform of ['identity', 'rot90-negative-origin', 'rot90-translated']) { const source = path.join(dir, `${transform}.mp4`); @@ -186,6 +242,101 @@ async function exerciseOverlayExports(compiler: string, sdk: string, sharedSuppo return results; } +type OmissionOutcome = { + name: string; + exitCode: number; + stderr: string; + byteIdentical?: boolean; + sourceSize?: [number, number]; + outputSize?: [number, number]; + outputBluePixels?: number; +}; + +// #4448: an event without a finite, positive paired reference has no proven transform into the +// captured canvas. The overlay must omit only that marker — exit successfully, keep the capture's +// geometry, and (when nothing renders) keep the raw video byte-for-byte. +async function exerciseOverlayReferenceOmission( + tools: RecordingToolBinaries, + dir: string, +): Promise { + const { overlay, fixture, inspect } = tools; + const width = 480; + const height = 640; + const cases: Array<{ name: string; events: unknown[] }> = [ + { + name: 'missing-reference', + events: [{ kind: 'tap', tMs: 150, x: width / 2, y: height / 2 }], + }, + { + name: 'unpaired-reference', + events: [{ kind: 'tap', tMs: 150, x: width / 2, y: height / 2, referenceWidth: width }], + }, + { + name: 'non-positive-reference', + events: [ + { + kind: 'tap', + tMs: 150, + x: width / 2, + y: height / 2, + referenceWidth: 0, + referenceHeight: height, + }, + ], + }, + { + name: 'mixed', + events: [ + { + kind: 'tap', + tMs: 150, + x: width / 2, + y: height / 2, + referenceWidth: width, + referenceHeight: height, + }, + { kind: 'tap', tMs: 150, x: 60, y: 80 }, + ], + }, + ]; + + const results: OmissionOutcome[] = []; + for (const { name, events } of cases) { + const source = path.join(dir, `${name}.mp4`); + const output = path.join(dir, `${name}-overlay.mp4`); + const eventsPath = path.join(dir, `${name}.json`); + fs.writeFileSync(eventsPath, JSON.stringify({ events })); + await runCmd(fixture, [source, 'identity'], { timeoutMs: 15_000 }); + const exported = await runCmd( + overlay, + ['--input', source, '--output', output, '--events', eventsPath], + { allowFailure: true, timeoutMs: 15_000 }, + ); + const outcome: OmissionOutcome = { + name, + exitCode: exported.exitCode, + stderr: exported.stderr, + }; + if (exported.exitCode === 0) { + outcome.byteIdentical = fs.readFileSync(output).equals(fs.readFileSync(source)); + const inspectDir = path.join(dir, `${name}-inspect`); + await runCmd( + inspect, + ['--input', output, '--events', eventsPath, '--output-dir', inspectDir], + { + timeoutMs: 15_000, + }, + ); + const manifest = JSON.parse(fs.readFileSync(path.join(inspectDir, 'manifest.json'), 'utf8')); + outcome.sourceSize = [width, height]; + outcome.outputSize = [manifest.renderWidth, manifest.renderHeight]; + outcome.outputBluePixels = inspectExportPixels(manifest.items[0].fullFramePath).bluePixels; + } + results.push(outcome); + } + return results; +} + function inspectExportPixels(framePath: string) { const png = PNG.sync.read(fs.readFileSync(framePath)); let luma = 0; diff --git a/packages/daemon/src/core/__tests__/recording-gestures.test.ts b/packages/daemon/src/core/__tests__/recording-gestures.test.ts index b3301f17c9..5dbe6e6772 100644 --- a/packages/daemon/src/core/__tests__/recording-gestures.test.ts +++ b/packages/daemon/src/core/__tests__/recording-gestures.test.ts @@ -444,6 +444,64 @@ test('edge swipe is classified as a back-swipe telemetry event', () => { assert.equal(event.durationMs, 320); }); +test('a no-snapshot tap keeps its telemetry event without a reference frame (#4448)', () => { + const session = makeSession(); + session.snapshot = undefined; + + recordTouchVisualizationEvent(session, 'press', ['220', '700'], { x: 220, y: 700 }, {}, 1_500); + + const events = session.screenRecording?.handle.inspect().gestureEvents; + assert.equal(events?.length, 1, 'the telemetry event is preserved when no reference exists'); + const event = events?.[0]; + assert.equal(event?.kind, 'tap'); + assert.equal(event?.x, 220); + assert.equal(event?.y, 700); + assert.equal(event?.referenceWidth, undefined); + assert.equal(event?.referenceHeight, undefined); + assert.ok(event && !('referenceWidth' in event), 'no plausible reference is attached'); + assert.ok(event && !('referenceHeight' in event), 'no plausible reference is attached'); +}); + +test('a no-snapshot edge swipe classifies as swipe, never back-swipe, without a reference (#4448)', () => { + const session = makeSession(); + session.snapshot = undefined; + + recordTouchVisualizationEvent( + session, + 'swipe', + ['10', '400', '250', '400', '250'], + { x1: 10, y1: 400, x2: 250, y2: 400, durationMs: 250 }, + {}, + 1_500, + 1_900, + ); + + const event = session.screenRecording?.handle.inspect().gestureEvents[0]; + // Without a reference width there is no edge inset to measure against, so the truthful + // classification is a plain swipe; the renderer then omits the marker for lack of a basis. + assert.equal(event?.kind, 'swipe'); + assert.equal(event?.referenceWidth, undefined); + assert.equal(event?.referenceHeight, undefined); +}); + +test('a non-positive explicit reference is not adopted and the snapshot frame still applies (#4448)', () => { + const session = makeSession(); + + recordTouchVisualizationEvent( + session, + 'press', + ['100', '200'], + { x: 100, y: 200, referenceWidth: 0, referenceHeight: 874 }, + {}, + 1_500, + ); + + const event = session.screenRecording?.handle.inspect().gestureEvents[0]; + assert.equal(event?.kind, 'tap'); + assert.equal(event?.referenceWidth, 402); + assert.equal(event?.referenceHeight, 874); +}); + test('a multi-finger tap records one touch overlay per contact', () => { const session = makeSession();