From f207f612f1fba902d5c1480ea98a90c1469752e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 14 Nov 2026 03:05:19 +0000 Subject: [PATCH 01/12] perf(studio): scale timeline strip frames off the main thread (#7651) --- .../src/player/lib/thumbnailFrameSink.test.ts | 92 +++++++++++++++++++ .../src/player/lib/thumbnailFrameSink.ts | 83 +++++++++++++++++ .../player/lib/thumbnailVideoDecoder.test.ts | 59 ++++++++---- .../src/player/lib/thumbnailVideoDecoder.ts | 12 +-- 4 files changed, 224 insertions(+), 22 deletions(-) create mode 100644 packages/studio/src/player/lib/thumbnailFrameSink.test.ts create mode 100644 packages/studio/src/player/lib/thumbnailFrameSink.ts diff --git a/packages/studio/src/player/lib/thumbnailFrameSink.test.ts b/packages/studio/src/player/lib/thumbnailFrameSink.test.ts new file mode 100644 index 00000000000..bc74cc2d00e --- /dev/null +++ b/packages/studio/src/player/lib/thumbnailFrameSink.test.ts @@ -0,0 +1,92 @@ +// @vitest-environment happy-dom + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { scaledFrames } from "./thumbnailFrameSink"; + +const drawImage = vi.fn(); +const createImageBitmapMock = vi.fn(); +const bitmap = { close: vi.fn() }; + +function sample(over: Partial<{ rotation: number; flipped: boolean }> = {}) { + const frame = { displayWidth: 3456, displayHeight: 2234, close: vi.fn() }; + return { + frame, + rotation: 0, + flipped: false, + toVideoFrame: vi.fn(() => frame), + drawWithFit: vi.fn(), + close: vi.fn(), + ...over, + }; +} + +async function collect(samples: (ReturnType | null)[], fit: "cover" | "contain") { + const sink = { + async *samplesAtTimestamps() { + yield* samples; + }, + }; + const out: ({ canvas: HTMLCanvasElement } | null)[] = []; + const times = (async function* () {})(); + for await (const item of scaledFrames(sink, times, { width: 360, height: 240 }, fit)) + out.push(item as { canvas: HTMLCanvasElement } | null); + return out; +} + +describe("scaledFrames", () => { + beforeEach(() => { + vi.stubGlobal("createImageBitmap", createImageBitmapMock.mockResolvedValue(bitmap)); + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({ + drawImage, + } as unknown as RenderingContext); + }); + afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + vi.clearAllMocks(); + }); + + it("crops and scales a straight frame in createImageBitmap, centred for cover", async () => { + const s = sample(); + const [out] = await collect([s], "cover"); + + expect(s.drawWithFit).not.toHaveBeenCalled(); + const [frame, x, y, w, h, options] = createImageBitmapMock.mock.calls[0]!; + expect(frame).toBe(s.frame); + expect(options).toMatchObject({ resizeWidth: 360, resizeHeight: 240 }); + expect(w / h).toBeCloseTo(360 / 240, 3); + expect(x + w / 2).toBeCloseTo(3456 / 2, 3); + expect(y + h / 2).toBeCloseTo(2234 / 2, 3); + expect(drawImage).toHaveBeenCalledWith(bitmap, 0, 0, 360, 240); + expect([out!.canvas.width, out!.canvas.height]).toEqual([360, 240]); + expect(s.frame.close).toHaveBeenCalled(); + expect(bitmap.close).toHaveBeenCalled(); + expect(s.close).toHaveBeenCalled(); + }); + + it("contain keeps the whole frame", async () => { + await collect([sample()], "contain"); + const [, x, y, w, h] = createImageBitmapMock.mock.calls[0]!; + expect([x, y, w, h]).toEqual([0, 0, 3456, 2234]); + }); + + it("leaves turned and flipped frames to the sample's own draw, which applies the turn", async () => { + const turned = sample({ rotation: 90 }); + const flipped = sample({ flipped: true }); + await collect([turned, flipped], "cover"); + + expect(createImageBitmapMock).not.toHaveBeenCalled(); + expect(turned.drawWithFit).toHaveBeenCalledWith(expect.anything(), { fit: "cover" }); + expect(flipped.drawWithFit).toHaveBeenCalled(); + }); + + it("passes a missing frame through and closes the sample even when the scale fails", async () => { + expect(await collect([null], "cover")).toEqual([null]); + + const s = sample(); + createImageBitmapMock.mockRejectedValueOnce(new Error("decode failed")); + await expect(collect([s], "cover")).rejects.toThrow("decode failed"); + expect(s.frame.close).toHaveBeenCalled(); + expect(s.close).toHaveBeenCalled(); + }); +}); diff --git a/packages/studio/src/player/lib/thumbnailFrameSink.ts b/packages/studio/src/player/lib/thumbnailFrameSink.ts new file mode 100644 index 00000000000..b03f4fab48c --- /dev/null +++ b/packages/studio/src/player/lib/thumbnailFrameSink.ts @@ -0,0 +1,83 @@ +/** The slice of a Mediabunny video sample the strip reads. */ +interface FrameSample { + rotation: number; + flipped?: boolean; + toVideoFrame(): VideoFrame; + drawWithFit( + context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, + options: { fit: "contain" | "cover" }, + ): void; + close(): void; +} + +interface FrameSampleSink { + samplesAtTimestamps(timestamps: AsyncIterable): AsyncIterable; +} + +type FrameCanvas = HTMLCanvasElement | OffscreenCanvas; +type Box = { width: number; height: number }; + +function newCanvas({ width, height }: Box): FrameCanvas { + if (typeof document === "undefined") return new OffscreenCanvas(width, height); + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + return canvas; +} + +/** The part of a source that fills a box of the target's shape, centred. */ +function coverCrop(source: Box, target: Box) { + const scale = Math.max(target.width / source.width, target.height / source.height); + const width = Math.min(source.width, target.width / scale); + const height = Math.min(source.height, target.height / scale); + return { x: (source.width - width) / 2, y: (source.height - height) / 2, width, height }; +} + +async function drawScaled( + sample: FrameSample, + context: CanvasRenderingContext2D, + target: Box, + fit: "contain" | "cover", +): Promise { + const frame = sample.toVideoFrame(); + const source = { width: frame.displayWidth, height: frame.displayHeight }; + const crop = fit === "cover" ? coverCrop(source, target) : { x: 0, y: 0, ...source }; + const bitmap = await createImageBitmap(frame, crop.x, crop.y, crop.width, crop.height, { + resizeWidth: target.width, + resizeHeight: target.height, + resizeQuality: "medium", + }).finally(() => frame.close()); + try { + context.drawImage(bitmap, 0, 0, target.width, target.height); + } finally { + bitmap.close(); + } +} + +/** + * Strip frames cropped and scaled by the browser off the main thread: drawing a 4K frame onto a strip-sized canvas + * holds the main thread for 100 ms or more. Turned or flipped frames keep the sample's own draw, which turns them. + */ +export async function* scaledFrames( + sink: FrameSampleSink, + times: AsyncIterable, + target: Box, + fit: "contain" | "cover", +): AsyncGenerator<{ canvas: FrameCanvas } | null> { + for await (const sample of sink.samplesAtTimestamps(times)) { + if (!sample) { + yield null; + continue; + } + try { + const canvas = newCanvas(target); + const context = canvas.getContext("2d") as CanvasRenderingContext2D | null; + if (!context) throw new Error("Thumbnail canvas has no 2d context"); + if (sample.rotation === 0 && !sample.flipped) await drawScaled(sample, context, target, fit); + else sample.drawWithFit(context, { fit }); + yield { canvas }; + } finally { + sample.close(); + } + } +} diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts index e0dee377dd0..a5bd3358e0c 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.test.ts @@ -11,6 +11,15 @@ const sourceRetryRules: string[] = []; const cancelledReads: ReturnType[] = []; const canvasesAtTimestamps = vi.fn(); const canvasSinks: unknown[] = []; +const drawImage = vi.fn(); +const createImageBitmapMock = vi.fn(); +const fourKSample = () => ({ + rotation: 0, + flipped: false, + toVideoFrame: () => ({ displayWidth: 3456, displayHeight: 2234, close: vi.fn() }), + drawWithFit: vi.fn(), + close: vi.fn(), +}); const getKeyPacket = vi.fn(async (_time: number) => null as { timestamp: number } | null); const input = { getPrimaryVideoTrack: vi.fn(), @@ -37,11 +46,16 @@ vi.mock("mediabunny", () => ({ getPrimaryVideoTrack = input.getPrimaryVideoTrack; dispose = input.dispose; }, - CanvasSink: class { + // Each frame the canvas seam yields becomes one straight (unturned) sample of a 4K source. + VideoSampleSink: class { constructor() { canvasSinks.push(this); } - canvasesAtTimestamps = canvasesAtTimestamps; + async *samplesAtTimestamps(timestamps: AsyncIterable) { + for await (const wrapped of canvasesAtTimestamps(timestamps)) { + yield wrapped === null ? null : fourKSample(); + } + } }, EncodedPacketSink: class { getKeyPacket = getKeyPacket; @@ -89,6 +103,11 @@ beforeEach(() => { HTMLCanvasElement.prototype.toBlob = function toBlob(callback) { callback(new Blob(["frame"], { type: "image/jpeg" })); }; + HTMLCanvasElement.prototype.getContext = (() => ({ drawImage })) as never; + vi.stubGlobal( + "createImageBitmap", + createImageBitmapMock.mockImplementation(async () => ({ close: vi.fn() })), + ); input.getPrimaryVideoTrack.mockResolvedValue({ getDisplayWidth: vi.fn(async () => 1080), getDisplayHeight: vi.fn(async () => 1920), @@ -96,6 +115,28 @@ beforeEach(() => { }); }); +describe("decodeVideoThumbnail frame scaling", () => { + it("never hands a full-size frame to a canvas draw: the browser crops and scales it to the strip's size", async () => { + recordDecodes([]); + await decodeVideoThumbnail( + { source, sourceStart: 0, sourceRangeDuration: 10, frameCount: 2 }, + new AbortController().signal, + ); + + expect(createImageBitmapMock).toHaveBeenCalled(); + for (const [, , , cropWidth, cropHeight, options] of createImageBitmapMock.mock.calls) { + expect(options.resizeWidth).toBeLessThan(cropWidth); + expect(options.resizeHeight).toBeLessThan(cropHeight); + } + // What lands on the canvas is the small bitmap, never a 4K frame. + for (const [image, , , width, height] of drawImage.mock.calls) { + expect(image).not.toHaveProperty("displayWidth"); + expect(width).toBeLessThan(1000); + expect(height).toBeLessThan(1000); + } + }); +}); + describe("videoThumbnailTimestamps", () => { it("uses the midpoint for a poster, and each slice's left edge then the end for a strip", () => { expect(videoThumbnailTimestamps(2, 6, 1)).toEqual([5]); @@ -706,20 +747,6 @@ describe("decodeVideoThumbnail's shared source", () => { expect(dispose).not.toHaveBeenCalled(); }); - it("keeps the reader when an offscreen frame will not encode", async () => { - const offscreen = { width: 1, height: 1, convertToBlob: () => Promise.reject(new Error("no")) }; - canvasesAtTimestamps.mockImplementationOnce(async function* ( - timestamps: AsyncIterable, - ) { - for await (const _time of timestamps) yield { canvas: offscreen }; - }); - await expect(decodeVideoThumbnail(clip(0), signal)).rejects.toThrow("encode failed"); - recordDecodes([]); - await decodeVideoThumbnail(clip(6), signal); - expect(input.getPrimaryVideoTrack).toHaveBeenCalledTimes(1); - expect(dispose).not.toHaveBeenCalled(); - }); - it("keeps the reader of a clip scrolled away from, for the next clip", async () => { vi.useFakeTimers(); try { diff --git a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts index 8a26744f4ac..e1dc928384a 100644 --- a/packages/studio/src/player/lib/thumbnailVideoDecoder.ts +++ b/packages/studio/src/player/lib/thumbnailVideoDecoder.ts @@ -4,6 +4,7 @@ import { type ThumbnailLoadedResult, type ThumbnailValue, } from "./thumbnailScheduler"; +import { scaledFrames } from "./thumbnailFrameSink"; import { MediaFetchError, reportMediaFetchFailure } from "./mediaFetch"; import { acquireThumbnailSource, @@ -395,12 +396,11 @@ async function decodeMissingFrames( } } const target = targetDimensions(info.aspect, budgets); - const sink = new mediabunny.CanvasSink(track, { - width: target.width, - height: target.height, - fit: request.fit ?? "cover", - poolSize: 1, - }); + const frames = new mediabunny.VideoSampleSink(track); + const sink: ThumbnailCanvasSink = { + canvasesAtTimestamps: (timestamps) => + scaledFrames(frames, timestamps, target, request.fit ?? "cover"), + }; const times = decodeTimesAtNearbyKeyframes(); await decodeFrames(sink, times, slots, signal, resources, source.progress); } From 062515bfa9f7002f9c18c6d59b0f6b4834660eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 14 Nov 2026 03:37:05 +0000 Subject: [PATCH 02/12] fix(studio): menus and popovers ease their scale in, not only their opacity (#7583) * fix(studio): menus and popovers ease their scale in, not only their opacity * style(studio): format the floatingMotion test --- .../studio/src/components/ui/Menu.test.tsx | 31 ++++++++++++++++++- packages/studio/src/components/ui/Menu.tsx | 2 +- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/packages/studio/src/components/ui/Menu.test.tsx b/packages/studio/src/components/ui/Menu.test.tsx index 3322657f5ae..f6b2caba2b0 100644 --- a/packages/studio/src/components/ui/Menu.test.tsx +++ b/packages/studio/src/components/ui/Menu.test.tsx @@ -6,7 +6,15 @@ import React, { act, useState } from "react"; import { compile } from "tailwindcss"; import { afterEach, beforeAll, describe, expect, it } from "vitest"; import { loadStylesheet, STYLES_DIR } from "../../styles/styleSources"; -import { ContextMenu, Menu, MenuItem, MenuRadioGroup, MenuRadioItem, MenuSeparator } from "./Menu"; +import { + ContextMenu, + floatingMotion, + Menu, + MenuItem, + MenuRadioGroup, + MenuRadioItem, + MenuSeparator, +} from "./Menu"; import { Popover } from "./Popover"; import { isTypingTarget } from "../../utils/typingTarget"; import { shouldIgnorePlaybackShortcutTarget } from "../../player/lib/playbackShortcuts"; @@ -443,3 +451,24 @@ describe("open motion", () => { expect(compileStudioCss([rowRadius])).toMatch(/calc\(var\(--radius-lg\) \+ 6px\)/); }); }); + +describe("floatingMotion", () => { + it("transitions every property its starting and ending styles set", () => { + const classes = floatingMotion("duration-open").split(/\s+/); + const transitioned = classes.flatMap( + (c) => c.match(/^transition-\[(.+)\]$/)?.[1].split(",") ?? [], + ); + const animated = classes + .flatMap( + (c) => + c + .match( + /^data-\[(?:starting|ending)-style\]:(?:\[([\w-]+):|(opacity|scale|translate|rotate)-)/, + ) + ?.slice(1) ?? [], + ) + .filter(Boolean); + expect(new Set(animated)).toEqual(new Set(["opacity", "scale"])); + for (const property of animated) expect(transitioned).toContain(property); + }); +}); diff --git a/packages/studio/src/components/ui/Menu.tsx b/packages/studio/src/components/ui/Menu.tsx index ebefc3f04b4..5e0046d017c 100644 --- a/packages/studio/src/components/ui/Menu.tsx +++ b/packages/studio/src/components/ui/Menu.tsx @@ -47,7 +47,7 @@ const VIEWPORT_MARGIN = 8; export function floatingMotion(openDuration: "duration-open" | "duration-tooltip") { return cn( "origin-[var(--transform-origin)] outline-hidden", - "transition-[opacity,transform] ease-out-quint", + "transition-[opacity,scale] ease-out-quint", openDuration, "data-[ending-style]:duration-close data-[ending-style]:ease-in", "data-[starting-style]:[opacity:var(--popup-enter-opacity)]", From 0f008f9412864860c0d001c8e076aa54fc480746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Sat, 14 Nov 2026 03:42:12 +0000 Subject: [PATCH 03/12] refactor(cli): keep the warm check idle timeout private to its module (#7702) --- packages/cli/src/warmCheckProtocol.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/warmCheckProtocol.ts b/packages/cli/src/warmCheckProtocol.ts index 3b2860dc40f..9eef23b767c 100644 --- a/packages/cli/src/warmCheckProtocol.ts +++ b/packages/cli/src/warmCheckProtocol.ts @@ -7,7 +7,7 @@ import { fileURLToPath } from "node:url"; import { hyperframesCacheDir } from "@hyperframes/core/cache-dir"; import { hyperframesOwnerArgs, withoutHyperframesOwner } from "@hyperframes/core/browser-marker"; -export const WARM_IDLE_MS = 5 * 60_000; +const WARM_IDLE_MS = 5 * 60_000; export const HANDSHAKE_TIMEOUT_MS = 2_000; export const warmServerArgs = ( From c66a43096768dee19f716b3e06ca04337d9b6f36 Mon Sep 17 00:00:00 2001 From: wonjang <48057918+wonjangcloud9@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:47:50 +0000 Subject: [PATCH 04/12] fix(cli): stop the CLI build looking for the removed gsap skill (#7689) skills/gsap was folded into the animation skill in #1349, but build-copy still listed it, so every CLI build printed "[build-copy] skill not found, skipping: skills/gsap". Drop the entry, and add a test that every bundled skill name has a SKILL.md so the next rename fails CI instead of warning. Co-authored-by: Claude Opus 5.5 --- packages/cli/scripts/build-copy.mjs | 2 +- packages/cli/src/buildCopy.test.ts | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 packages/cli/src/buildCopy.test.ts diff --git a/packages/cli/scripts/build-copy.mjs b/packages/cli/scripts/build-copy.mjs index 2840596d10c..0709e057139 100644 --- a/packages/cli/scripts/build-copy.mjs +++ b/packages/cli/scripts/build-copy.mjs @@ -133,7 +133,7 @@ async function main() { // Skills bundled into the published CLI. Branches don't all carry the same // skills/ tree (it gets restructured), so each entry is existsSync-guarded: // a missing skill dir warns + skips instead of crashing the build. - for (const skill of ["hyperframes", "hyperframes-cli", "gsap", "media-use"]) { + for (const skill of ["hyperframes", "hyperframes-cli", "media-use"]) { const src = join(REPO_ROOT, "skills", skill); if (!existsSync(src)) { console.warn(`[build-copy] skill not found, skipping: skills/${skill}`); diff --git a/packages/cli/src/buildCopy.test.ts b/packages/cli/src/buildCopy.test.ts new file mode 100644 index 00000000000..bb82cc595e3 --- /dev/null +++ b/packages/cli/src/buildCopy.test.ts @@ -0,0 +1,16 @@ +import { existsSync, readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const copySource = readFileSync(new URL("../scripts/build-copy.mjs", import.meta.url), "utf-8"); + +describe("build-copy bundled skills", () => { + it("names only skills that exist in the repo's skills/ tree", () => { + const list = copySource.match(/for \(const skill of \[([^\]]+)\]\)/)?.[1] ?? ""; + const skills = [...list.matchAll(/"([^"]+)"/g)].map((m) => m[1]); + expect(skills.length).toBeGreaterThan(0); + const missing = skills.filter( + (name) => !existsSync(new URL(`../../../skills/${name}/SKILL.md`, import.meta.url)), + ); + expect(missing).toEqual([]); + }); +}); From 08e11cb563725744804d9e61055baab61eefe976 Mon Sep 17 00:00:00 2001 From: wonjang <48057918+wonjangcloud9@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:47:53 +0000 Subject: [PATCH 05/12] fix(check): name a color-graded video by its own selector in layout findings (#7699) A color-graded