Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,11 @@ func run() throws {
let payload = try Data(contentsOf: eventsURL)
let envelope = try JSONDecoder().decode(GestureEnvelope.self, from: payload)

if envelope.events.isEmpty {
// An event whose reference pair is missing or not finite and positive has no proven transform
// into the captured canvas, so its marker is omitted (#4448); the telemetry event and the video
// are preserved either way. With nothing left to render, the raw capture is kept as-is.
let renderableEvents = envelope.events.filter { renderableReference(of: $0) != nil }
if renderableEvents.isEmpty {
try FileManager.default.copyItem(at: inputURL, to: outputURL)
return
}
Expand Down Expand Up @@ -116,7 +120,7 @@ func run() throws {
overlayLayer.frame = parentLayer.frame
parentLayer.addSublayer(overlayLayer)

for event in envelope.events {
for event in renderableEvents {
switch event.kind {
case "tap":
addTapLayer(event: event, renderSize: renderSize, to: overlayLayer)
Expand Down Expand Up @@ -381,65 +385,86 @@ func resolvedFrameDuration(for track: AVAssetTrack, maximumFrameRate: Int32) ->
return shortest
}

func overlayPoint(event: GestureEvent, x: Double, y: Double, renderSize: CGSize) -> CGPoint {
let scaleX = scaledAxis(renderSize: renderSize.width, referenceSize: event.referenceWidth)
let scaleY = scaledAxis(renderSize: renderSize.height, referenceSize: event.referenceHeight)
let scaledX = x * scaleX
let scaledY = y * scaleY
/// The paired reference an event needs before its coordinates can be projected into the captured
/// canvas: both axes present, finite, and positive. The old scale=1 fallback rendered point-space
/// coordinates as video pixels (#4448); an unproven reference now omits the marker instead.
func renderableReference(of event: GestureEvent) -> (width: Double, height: Double)? {
guard let width = event.referenceWidth, let height = event.referenceHeight,
width.isFinite, height.isFinite, width > 0, height > 0
else { return nil }
return (width, height)
}

/// Projects an event coordinate into the render space, or returns nil when the event carries no
/// renderable reference — the caller then omits only this event's marker layers.
func overlayPoint(event: GestureEvent, x: Double, y: Double, renderSize: CGSize) -> CGPoint? {
guard let reference = renderableReference(of: event) else { return nil }
let scaledX = x * (Double(renderSize.width) / reference.width)
let scaledY = y * (Double(renderSize.height) / reference.height)
let flippedY = max(0, Double(renderSize.height) - scaledY)
return CGPoint(x: scaledX, y: flippedY)
}

func scaledAxis(renderSize: CGFloat, referenceSize: Double?) -> Double {
guard let referenceSize, referenceSize > 0 else { return 1.0 }
return Double(renderSize) / referenceSize
/// Adds the event's touch dot, or returns nil when the event has no renderable reference and its
/// marker is omitted.
func addEventTouchDot(event: GestureEvent, renderSize: CGSize, to overlayLayer: CALayer) -> CALayer? {
guard let center = overlayPoint(event: event, x: event.x, y: event.y, renderSize: renderSize)
else { return nil }
let layer = makeTouchDotLayer(center: center, renderSize: renderSize)
overlayLayer.addSublayer(layer)
return layer
}

func addTapLayer(event: GestureEvent, renderSize: CGSize, to overlayLayer: CALayer) {
let layer = makeTouchDotLayer(
center: overlayPoint(event: event, x: event.x, y: event.y, renderSize: renderSize),
renderSize: renderSize
)
overlayLayer.addSublayer(layer)

let opacity = CAKeyframeAnimation(keyPath: "opacity")
opacity.values = [0.0, 0.98, 0.98, 0.0]
opacity.keyTimes = [0.0, 0.08, 0.8, 1.0]

let scale = CAKeyframeAnimation(keyPath: "transform.scale")
scale.values = [0.84, 1.0, 1.0]
scale.keyTimes = [0.0, 0.22, 1.0]

let group = makeAnimationGroup(
animations: [opacity, scale],
addPressLayer(
event: event,
renderSize: renderSize,
to: overlayLayer,
duration: minimumTapVisibility,
beginTime: AVCoreAnimationBeginTimeAtZero + (event.tMs / 1000.0)
opacityKeyTimes: [0.0, 0.08, 0.8, 1.0],
scaleKeyTimes: [0.0, 0.22, 1.0],
animationKey: "tap"
)
layer.add(group, forKey: "tap")
}

func addLongPressLayer(event: GestureEvent, renderSize: CGSize, to overlayLayer: CALayer) {
let duration = max(0.75, (event.durationMs ?? 800) / 1000.0)
let layer = makeTouchDotLayer(
center: overlayPoint(event: event, x: event.x, y: event.y, renderSize: renderSize),
renderSize: renderSize
addPressLayer(
event: event,
renderSize: renderSize,
to: overlayLayer,
duration: max(0.75, (event.durationMs ?? 800) / 1000.0),
opacityKeyTimes: [0.0, 0.08, 0.92, 1.0],
scaleKeyTimes: [0.0, 0.15, 1.0],
animationKey: "longpress"
)
overlayLayer.addSublayer(layer)
}

func addPressLayer(
event: GestureEvent,
renderSize: CGSize,
to overlayLayer: CALayer,
duration: CFTimeInterval,
opacityKeyTimes: [NSNumber],
scaleKeyTimes: [NSNumber],
animationKey: String
) {
guard let layer = addEventTouchDot(event: event, renderSize: renderSize, to: overlayLayer)
else { return }

let opacity = CAKeyframeAnimation(keyPath: "opacity")
opacity.values = [0.0, 0.98, 0.98, 0.0]
opacity.keyTimes = [0.0, 0.08, 0.92, 1.0]
opacity.keyTimes = opacityKeyTimes

let scale = CAKeyframeAnimation(keyPath: "transform.scale")
scale.values = [0.84, 1.0, 1.0]
scale.keyTimes = [0.0, 0.15, 1.0]
scale.keyTimes = scaleKeyTimes

let group = makeAnimationGroup(
animations: [opacity, scale],
duration: duration,
beginTime: AVCoreAnimationBeginTimeAtZero + (event.tMs / 1000.0)
)
layer.add(group, forKey: "longpress")
layer.add(group, forKey: animationKey)
}

func addSwipeLayers(event: GestureEvent, renderSize: CGSize, to overlayLayer: CALayer) {
Expand Down Expand Up @@ -548,9 +573,10 @@ func addTrailLayers(
to overlayLayer: CALayer,
style: TrailStyle
) {
guard let x2 = event.x2, let y2 = event.y2 else { return }
let startPoint = overlayPoint(event: event, x: event.x, y: event.y, renderSize: renderSize)
let endPoint = overlayPoint(event: event, x: x2, y: y2, renderSize: renderSize)
guard let x2 = event.x2, let y2 = event.y2,
let startPoint = overlayPoint(event: event, x: event.x, y: event.y, renderSize: renderSize),
let endPoint = overlayPoint(event: event, x: x2, y: y2, renderSize: renderSize)
else { return }
let duration = max(0.1, (event.durationMs ?? 250) / 1000.0)
let visibleDuration = max(minimumSwipeVisibility, duration + style.tail)
let beginTime = AVCoreAnimationBeginTimeAtZero + (event.tMs / 1000.0)
Expand Down Expand Up @@ -659,6 +685,7 @@ func addPinchLayers(event: GestureEvent, renderSize: CGSize, to overlayLayer: CA
let startRight = overlayPoint(event: event, x: event.x + Double(startOffset), y: event.y, renderSize: renderSize)
let endLeft = overlayPoint(event: event, x: event.x - Double(endOffset), y: event.y, renderSize: renderSize)
let endRight = overlayPoint(event: event, x: event.x + Double(endOffset), y: event.y, renderSize: renderSize)
guard let startLeft, let startRight, let endLeft, let endRight else { return }

addPinchDot(
overlayLayer: overlayLayer,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ type TypecheckOutcome = { exitCode: number; stderr: string; source: string };
describe.skipIf(process.platform !== 'darwin')('recording Swift scripts', () => {
let outcomes: TypecheckOutcome[] = [];
let exports: OverlayExportOutcome[] = [];
let omissions: OmissionOutcome[] = [];

beforeAll(
async () => {
Expand Down Expand Up @@ -58,9 +59,17 @@ describe.skipIf(process.platform !== 'darwin')('recording Swift scripts', () =>
return { source, exitCode: result.exitCode, stderr: result.stderr };
}),
);
exports = await exerciseOverlayExports(swiftCompilerPath, swiftSdkPath, sharedSupport);
const dir = await mkdtempForTest('recording-overlay-export-');
const tools = await compileRecordingTools(
swiftCompilerPath,
swiftSdkPath,
sharedSupport,
dir,
);
exports = await exerciseOverlayExports(tools, dir);
omissions = await exerciseOverlayReferenceOmission(tools, dir);
},
SWIFT_TYPECHECK_TIMEOUT_MS * 3 + 30_000,
SWIFT_TYPECHECK_TIMEOUT_MS * 3 + 120_000,
);

test('recording inspect Swift script typechecks', () => {
Expand Down Expand Up @@ -88,6 +97,42 @@ describe.skipIf(process.platform !== 'darwin')('recording Swift scripts', () =>
assert.ok(outcome.markerDistance! < 3, 'the asymmetric source marker must stay upright');
});
}

test('overlay omits markers without a finite positive paired reference and keeps the raw video (#4448)', () => {
for (const name of ['missing-reference', 'unpaired-reference', 'non-positive-reference']) {
const outcome = omissions.find((candidate) => candidate.name === name);
assert.ok(outcome, `${name} was never exercised`);
assert.equal(outcome.exitCode, 0, outcome.stderr);
assert.equal(
outcome.byteIdentical,
true,
`${name}: omission must preserve the raw capture byte-for-byte`,
);
assert.equal(
outcome.outputBluePixels,
0,
`${name}: no marker may be drawn from an unproven reference`,
);
assert.deepEqual(outcome.outputSize, outcome.sourceSize);
}
});

test('overlay renders only the referenced events when others lack a reference (#4448)', () => {
const outcome = omissions.find((candidate) => candidate.name === 'mixed');
assert.ok(outcome, 'mixed was never exercised');
assert.equal(outcome.exitCode, 0, outcome.stderr);
assert.equal(
outcome.byteIdentical,
false,
'a referenced event still forces the composited export',
);
assert.deepEqual(outcome.outputSize, outcome.sourceSize);
assert.ok(outcome.outputBluePixels! > 180, 'the referenced marker must draw');
assert.ok(
outcome.outputBluePixels! <= 1500,
`only the referenced marker may draw, saw ${outcome.outputBluePixels} overlay pixels`,
);
});
});

type OverlayExportOutcome = {
Expand All @@ -103,8 +148,14 @@ type OverlayExportOutcome = {
markerDistance?: number;
};

async function exerciseOverlayExports(compiler: string, sdk: string, sharedSupport: string) {
const dir = await mkdtempForTest('recording-overlay-export-');
type RecordingToolBinaries = { overlay: string; fixture: string; inspect: string };

async function compileRecordingTools(
compiler: string,
sdk: string,
sharedSupport: string,
dir: string,
): Promise<RecordingToolBinaries> {
const overlay = path.join(dir, 'overlay');
const fixture = path.join(dir, 'fixture');
const inspect = path.join(dir, 'inspect');
Expand All @@ -119,6 +170,11 @@ async function exerciseOverlayExports(compiler: string, sdk: string, sharedSuppo
});
}),
);
return { overlay, fixture, inspect };
}

async function exerciseOverlayExports(tools: RecordingToolBinaries, dir: string) {
const { overlay, fixture, inspect } = tools;
const results: OverlayExportOutcome[] = [];
for (const transform of ['identity', 'rot90-negative-origin', 'rot90-translated']) {
const source = path.join(dir, `${transform}.mp4`);
Expand Down Expand Up @@ -186,6 +242,101 @@ async function exerciseOverlayExports(compiler: string, sdk: string, sharedSuppo
return results;
}

type OmissionOutcome = {
name: string;
exitCode: number;
stderr: string;
byteIdentical?: boolean;
sourceSize?: [number, number];
outputSize?: [number, number];
outputBluePixels?: number;
};

// #4448: an event without a finite, positive paired reference has no proven transform into the
// captured canvas. The overlay must omit only that marker — exit successfully, keep the capture's
// geometry, and (when nothing renders) keep the raw video byte-for-byte.
async function exerciseOverlayReferenceOmission(
tools: RecordingToolBinaries,
dir: string,
): Promise<OmissionOutcome[]> {
const { overlay, fixture, inspect } = tools;
const width = 480;
const height = 640;
const cases: Array<{ name: string; events: unknown[] }> = [
{
name: 'missing-reference',
events: [{ kind: 'tap', tMs: 150, x: width / 2, y: height / 2 }],
},
{
name: 'unpaired-reference',
events: [{ kind: 'tap', tMs: 150, x: width / 2, y: height / 2, referenceWidth: width }],
},
{
name: 'non-positive-reference',
events: [
{
kind: 'tap',
tMs: 150,
x: width / 2,
y: height / 2,
referenceWidth: 0,
referenceHeight: height,
},
],
},
{
name: 'mixed',
events: [
{
kind: 'tap',
tMs: 150,
x: width / 2,
y: height / 2,
referenceWidth: width,
referenceHeight: height,
},
{ kind: 'tap', tMs: 150, x: 60, y: 80 },
],
},
];

const results: OmissionOutcome[] = [];
for (const { name, events } of cases) {
const source = path.join(dir, `${name}.mp4`);
const output = path.join(dir, `${name}-overlay.mp4`);
const eventsPath = path.join(dir, `${name}.json`);
fs.writeFileSync(eventsPath, JSON.stringify({ events }));
await runCmd(fixture, [source, 'identity'], { timeoutMs: 15_000 });
const exported = await runCmd(
overlay,
['--input', source, '--output', output, '--events', eventsPath],
{ allowFailure: true, timeoutMs: 15_000 },
);
const outcome: OmissionOutcome = {
name,
exitCode: exported.exitCode,
stderr: exported.stderr,
};
if (exported.exitCode === 0) {
outcome.byteIdentical = fs.readFileSync(output).equals(fs.readFileSync(source));
const inspectDir = path.join(dir, `${name}-inspect`);
await runCmd(
inspect,
['--input', output, '--events', eventsPath, '--output-dir', inspectDir],
{
timeoutMs: 15_000,
},
);
const manifest = JSON.parse(fs.readFileSync(path.join(inspectDir, 'manifest.json'), 'utf8'));
outcome.sourceSize = [width, height];
outcome.outputSize = [manifest.renderWidth, manifest.renderHeight];
outcome.outputBluePixels = inspectExportPixels(manifest.items[0].fullFramePath).bluePixels;
}
results.push(outcome);
}
return results;
}

function inspectExportPixels(framePath: string) {
const png = PNG.sync.read(fs.readFileSync(framePath));
let luma = 0;
Expand Down
Loading
Loading