From 6cc8167a74d7623a06265cc438bc725facad76d5 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sun, 20 Sep 2026 18:13:31 -0400 Subject: [PATCH 1/4] feat(studio): render geometry-sized video filmstrips --- .../src/hooks/useRenderClipContent.test.ts | 5 ++-- .../studio/src/hooks/useRenderClipContent.ts | 1 - .../player/components/VideoThumbnail.test.tsx | 10 +++---- .../src/player/components/VideoThumbnail.tsx | 28 ++++++++++++++----- .../player/components/thumbnailUtils.test.ts | 9 ++++++ .../src/player/components/thumbnailUtils.ts | 7 +++++ .../src/player/lib/timelineViewportBudgets.ts | 3 +- 7 files changed, 46 insertions(+), 17 deletions(-) diff --git a/packages/studio/src/hooks/useRenderClipContent.test.ts b/packages/studio/src/hooks/useRenderClipContent.test.ts index b7131e046e..a3f896d2a9 100644 --- a/packages/studio/src/hooks/useRenderClipContent.test.ts +++ b/packages/studio/src/hooks/useRenderClipContent.test.ts @@ -208,7 +208,7 @@ describe("useRenderClipContent", () => { } }); - it("forwards the viewport priority and interaction detail to media work", () => { + it("forwards the viewport priority to video media work", () => { usePlayerStore.setState({ thumbnailMode: "adaptive", timelineSessionEpoch: 7 }); const content = renderClipContent( @@ -229,7 +229,6 @@ describe("useRenderClipContent", () => { projectId: string; sessionEpoch: number; priority: string; - rich: boolean; }>(content), ).toBe(true); if (isValidElement(content)) { @@ -237,8 +236,8 @@ describe("useRenderClipContent", () => { projectId: "my-project", sessionEpoch: 7, priority: "interaction", - rich: true, }); + expect(content.props).not.toHaveProperty("rich"); } }); diff --git a/packages/studio/src/hooks/useRenderClipContent.ts b/packages/studio/src/hooks/useRenderClipContent.ts index 3d3a821c13..0fef11ad53 100644 --- a/packages/studio/src/hooks/useRenderClipContent.ts +++ b/packages/studio/src/hooks/useRenderClipContent.ts @@ -220,7 +220,6 @@ export function useRenderClipContent({ projectId: pid, sessionEpoch, priority: context.priority, - rich: context.rich, }); } diff --git a/packages/studio/src/player/components/VideoThumbnail.test.tsx b/packages/studio/src/player/components/VideoThumbnail.test.tsx index 2906ee866b..bb08303a26 100644 --- a/packages/studio/src/player/components/VideoThumbnail.test.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.test.tsx @@ -36,7 +36,8 @@ afterEach(() => { document.body.innerHTML = ""; }); -async function render(rich = false) { +async function render(width = 0) { + Object.defineProperty(host, "clientWidth", { configurable: true, value: width }); root = createRoot(host); await act(async () => { root!.render( @@ -47,7 +48,6 @@ async function render(rich = false) { projectId="p" sessionEpoch={1} priority="visible" - rich={rich} />, ); await Promise.resolve(); @@ -71,16 +71,16 @@ describe("VideoThumbnail", () => { expect(host.querySelector(".animate-pulse")).toBeNull(); }); - it("requests a rich filmstrip only for interaction actors", async () => { + it("requests a geometry-sized filmstrip by default", async () => { vi.mocked(decodeVideoThumbnail).mockResolvedValue({ value: { kind: "filmstrip", urls: ["blob:a", "blob:b"], aspect: 16 / 9 }, weight: 256, }); - await render(true); + await render(500); expect(decodeVideoThumbnail).toHaveBeenCalledWith( - expect.objectContaining({ frameCount: 6 }), + expect.objectContaining({ frameCount: 8 }), expect.any(AbortSignal), ); expect(host.querySelectorAll("img").length).toBeGreaterThan(0); diff --git a/packages/studio/src/player/components/VideoThumbnail.tsx b/packages/studio/src/player/components/VideoThumbnail.tsx index 755daf0e03..8d00e5aab5 100644 --- a/packages/studio/src/player/components/VideoThumbnail.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.tsx @@ -3,7 +3,11 @@ import { useMountEffect } from "../../hooks/useMountEffect"; import { useThumbnailLease } from "../../hooks/useThumbnailLease"; import { createThumbnailKey, type ThumbnailPriority } from "../lib/thumbnailScheduler"; import { decodeVideoThumbnail } from "../lib/thumbnailVideoDecoder"; -import { computeThumbnailStrip, THUMBNAIL_CLIP_HEIGHT } from "./thumbnailUtils"; +import { + computeThumbnailStrip, + quantizeThumbnailFrameCount, + THUMBNAIL_CLIP_HEIGHT, +} from "./thumbnailUtils"; interface VideoThumbnailProps { videoSrc: string; @@ -15,7 +19,6 @@ interface VideoThumbnailProps { projectId?: string; sessionEpoch?: number; priority?: ThumbnailPriority; - rich?: boolean; } /** Sparse, bounded video frames supplied by the shared thumbnail scheduler. */ @@ -29,10 +32,12 @@ export const VideoThumbnail = memo(function VideoThumbnail({ projectId = videoSrc, sessionEpoch = 0, priority = "visible", - rich = false, }: VideoThumbnailProps) { const [containerWidth, setContainerWidth] = useState(0); const observerRef = useRef(null); + const requestFrameCount = quantizeThumbnailFrameCount( + computeThumbnailStrip(containerWidth, 16 / 9).frameCount, + ); const request = useMemo( () => ({ key: createThumbnailKey({ @@ -40,26 +45,35 @@ export const VideoThumbnail = memo(function VideoThumbnail({ source: videoSrc, start: sourceStart, duration: sourceRangeDuration ?? duration, - frames: rich ? 6 : 1, + frames: requestFrameCount, }), projectId, sessionEpoch, kind: "video" as const, priority, - rich, + rich: true, load: (signal: AbortSignal) => decodeVideoThumbnail( { source: videoSrc, sourceStart, sourceRangeDuration: sourceRangeDuration ?? duration, - frameCount: rich ? 6 : 1, + frameCount: requestFrameCount, fit: "cover", }, signal, ), }), - [duration, priority, projectId, rich, sessionEpoch, sourceRangeDuration, sourceStart, videoSrc], + [ + duration, + priority, + projectId, + requestFrameCount, + sessionEpoch, + sourceRangeDuration, + sourceStart, + videoSrc, + ], ); const snapshot = useThumbnailLease(request); const value = snapshot.status === "ready" ? snapshot.value : null; diff --git a/packages/studio/src/player/components/thumbnailUtils.test.ts b/packages/studio/src/player/components/thumbnailUtils.test.ts index 43db158768..40f3320026 100644 --- a/packages/studio/src/player/components/thumbnailUtils.test.ts +++ b/packages/studio/src/player/components/thumbnailUtils.test.ts @@ -3,6 +3,7 @@ import { computeThumbnailStrip, encodePreviewPath, resolveMediaPreviewUrl, + quantizeThumbnailFrameCount, THUMBNAIL_CLIP_HEIGHT, } from "./thumbnailUtils"; @@ -50,6 +51,14 @@ describe("computeThumbnailStrip", () => { }); }); +describe("quantizeThumbnailFrameCount", () => { + it("uses doubling buckets and never exceeds the 4K geometry ceiling", () => { + expect(quantizeThumbnailFrameCount(5)).toBe(8); + expect(quantizeThumbnailFrameCount(32)).toBe(32); + expect(quantizeThumbnailFrameCount(34)).toBe(33); + }); +}); + describe("resolveMediaPreviewUrl", () => { it("reroutes same-origin root media resolved by the preview iframe", () => { expect( diff --git a/packages/studio/src/player/components/thumbnailUtils.ts b/packages/studio/src/player/components/thumbnailUtils.ts index 7a1732d8c1..dfeb9e74a5 100644 --- a/packages/studio/src/player/components/thumbnailUtils.ts +++ b/packages/studio/src/player/components/thumbnailUtils.ts @@ -1,4 +1,5 @@ import { buildProjectApiPath } from "../../utils/projectRouting"; +import { MAX_VISIBLE_THUMBNAIL_FRAMES } from "../lib/timelineViewportBudgets"; /** Rendered height of a timeline-clip thumbnail strip, in CSS px. */ export const THUMBNAIL_CLIP_HEIGHT = 66; @@ -9,6 +10,12 @@ export interface ThumbnailStripLayout { frameCount: number; } +/** Quantize request identities so a pixel-by-pixel resize does not thrash the cache. */ +export function quantizeThumbnailFrameCount(frameCount: number): number { + const safeCount = Math.max(1, Number.isFinite(frameCount) ? Math.ceil(frameCount) : 1); + return Math.min(MAX_VISIBLE_THUMBNAIL_FRAMES, 2 ** Math.ceil(Math.log2(safeCount))); +} + /** * Measure an image without mounting it in React's DOM. The scheduler owns the * abort signal, so an unmounted clip cannot leave Blink retaining a pending diff --git a/packages/studio/src/player/lib/timelineViewportBudgets.ts b/packages/studio/src/player/lib/timelineViewportBudgets.ts index cead6fd8a4..5e862c565d 100644 --- a/packages/studio/src/player/lib/timelineViewportBudgets.ts +++ b/packages/studio/src/player/lib/timelineViewportBudgets.ts @@ -49,6 +49,7 @@ export interface TimelineViewportBudgets { const MEBIBYTE = 1024 * 1024; const DAY_MS = 24 * 60 * 60 * 1000; +export const MAX_VISIBLE_THUMBNAIL_FRAMES = Math.ceil(3840 / (66 * (16 / 9))); // 4K timeline width / tallest 16:9 tile. /** * The sole default budget owner for timeline viewport and media virtualization. @@ -66,7 +67,7 @@ export const TIMELINE_VIEWPORT_BUDGETS: Readonly = Obje posterMaxPhysicalWidth: 240, posterMaxPhysicalHeight: 135, posterDprCap: 1.5, - richPreviewFrameCount: 6, + richPreviewFrameCount: MAX_VISIBLE_THUMBNAIL_FRAMES, concurrentVideoDecodes: 2, concurrentMetadataJobs: 4, concurrentCompositionFetches: 2, From 449979905e5a8d01d3598ca257a0214c0c91055d Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sun, 20 Sep 2026 19:15:01 -0400 Subject: [PATCH 2/4] fix(studio): keep poster tier during timeline scroll --- .../player/components/VideoThumbnail.test.tsx | 14 +-- .../src/player/components/VideoThumbnail.tsx | 116 +++++++++++++----- 2 files changed, 90 insertions(+), 40 deletions(-) diff --git a/packages/studio/src/player/components/VideoThumbnail.test.tsx b/packages/studio/src/player/components/VideoThumbnail.test.tsx index bb08303a26..553b81b801 100644 --- a/packages/studio/src/player/components/VideoThumbnail.test.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.test.tsx @@ -55,7 +55,7 @@ async function render(width = 0) { } describe("VideoThumbnail", () => { - it("renders a scheduler-provided sparse poster", async () => { + it("does not acquire a thumbnail lease before the clip is measured", async () => { vi.mocked(decodeVideoThumbnail).mockResolvedValue({ value: { kind: "image", url: "blob:poster", aspect: 16 / 9 }, weight: 128, @@ -63,12 +63,7 @@ describe("VideoThumbnail", () => { await render(); - expect(decodeVideoThumbnail).toHaveBeenCalledWith( - expect.objectContaining({ frameCount: 1 }), - expect.any(AbortSignal), - ); - expect(host.querySelector('img[src="blob:poster"]')).not.toBeNull(); - expect(host.querySelector(".animate-pulse")).toBeNull(); + expect(decodeVideoThumbnail).not.toHaveBeenCalled(); }); it("requests a geometry-sized filmstrip by default", async () => { @@ -79,6 +74,11 @@ describe("VideoThumbnail", () => { await render(500); + expect(decodeVideoThumbnail).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ frameCount: 1 }), + expect.any(AbortSignal), + ); expect(decodeVideoThumbnail).toHaveBeenCalledWith( expect.objectContaining({ frameCount: 8 }), expect.any(AbortSignal), diff --git a/packages/studio/src/player/components/VideoThumbnail.tsx b/packages/studio/src/player/components/VideoThumbnail.tsx index 8d00e5aab5..eedd3550b2 100644 --- a/packages/studio/src/player/components/VideoThumbnail.tsx +++ b/packages/studio/src/player/components/VideoThumbnail.tsx @@ -1,7 +1,11 @@ import { memo, useCallback, useMemo, useRef, useState } from "react"; import { useMountEffect } from "../../hooks/useMountEffect"; import { useThumbnailLease } from "../../hooks/useThumbnailLease"; -import { createThumbnailKey, type ThumbnailPriority } from "../lib/thumbnailScheduler"; +import { + createThumbnailKey, + type ThumbnailPriority, + type ThumbnailSnapshot, +} from "../lib/thumbnailScheduler"; import { decodeVideoThumbnail } from "../lib/thumbnailVideoDecoder"; import { computeThumbnailStrip, @@ -21,6 +25,66 @@ interface VideoThumbnailProps { priority?: ThumbnailPriority; } +function createVideoThumbnailRequest( + props: Pick< + VideoThumbnailProps, + | "videoSrc" + | "sourceStart" + | "sourceRangeDuration" + | "duration" + | "projectId" + | "sessionEpoch" + | "priority" + >, + frameCount: number, + rich: boolean, +) { + const { + videoSrc, + sourceStart, + sourceRangeDuration, + duration = 5, + projectId = videoSrc, + sessionEpoch = 0, + priority = "visible", + } = props; + return { + key: createThumbnailKey({ + kind: "video", + source: videoSrc, + start: sourceStart, + duration: sourceRangeDuration ?? duration, + frames: frameCount, + }), + projectId, + sessionEpoch, + kind: "video" as const, + priority, + rich, + load: (signal: AbortSignal) => + decodeVideoThumbnail( + { + source: videoSrc, + sourceStart, + sourceRangeDuration: sourceRangeDuration ?? duration, + frameCount, + fit: "cover", + }, + signal, + ), + }; +} + +function selectThumbnailSnapshot( + poster: ThumbnailSnapshot, + rich: ThumbnailSnapshot, +): ThumbnailSnapshot { + if (rich.status === "ready") return rich; + if (poster.status === "ready") return poster; + if (rich.status === "loading" || poster.status === "loading") return { status: "loading" }; + return poster; +} + /** Sparse, bounded video frames supplied by the shared thumbnail scheduler. */ export const VideoThumbnail = memo(function VideoThumbnail({ videoSrc, @@ -38,44 +102,30 @@ export const VideoThumbnail = memo(function VideoThumbnail({ const requestFrameCount = quantizeThumbnailFrameCount( computeThumbnailStrip(containerWidth, 16 / 9).frameCount, ); - const request = useMemo( + const requestProps = useMemo( () => ({ - key: createThumbnailKey({ - kind: "video", - source: videoSrc, - start: sourceStart, - duration: sourceRangeDuration ?? duration, - frames: requestFrameCount, - }), + videoSrc, + sourceStart, + sourceRangeDuration, + duration, projectId, sessionEpoch, - kind: "video" as const, priority, - rich: true, - load: (signal: AbortSignal) => - decodeVideoThumbnail( - { - source: videoSrc, - sourceStart, - sourceRangeDuration: sourceRangeDuration ?? duration, - frameCount: requestFrameCount, - fit: "cover", - }, - signal, - ), }), - [ - duration, - priority, - projectId, - requestFrameCount, - sessionEpoch, - sourceRangeDuration, - sourceStart, - videoSrc, - ], + [duration, priority, projectId, sessionEpoch, sourceRangeDuration, sourceStart, videoSrc], + ); + const posterRequest = useMemo( + () => createVideoThumbnailRequest(requestProps, 1, false), + [requestProps], + ); + const richRequest = useMemo( + () => createVideoThumbnailRequest(requestProps, requestFrameCount, true), + [requestFrameCount, requestProps], ); - const snapshot = useThumbnailLease(request); + const measured = containerWidth > 0; + const posterSnapshot = useThumbnailLease(measured ? posterRequest : null); + const richSnapshot = useThumbnailLease(measured ? richRequest : null); + const snapshot = selectThumbnailSnapshot(posterSnapshot, richSnapshot); const value = snapshot.status === "ready" ? snapshot.value : null; const urls = value?.kind === "filmstrip" ? value.urls : value?.kind === "image" ? [value.url] : []; From f5c25f06d72b2d7a910e10d97d7f720d42bf3908 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Mon, 21 Sep 2026 10:21:46 -0400 Subject: [PATCH 3/4] fix(studio): cap rendered thumbnail tiles --- packages/studio/src/player/components/thumbnailUtils.test.ts | 4 ++++ packages/studio/src/player/components/thumbnailUtils.ts | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/studio/src/player/components/thumbnailUtils.test.ts b/packages/studio/src/player/components/thumbnailUtils.test.ts index 40f3320026..738b105ed7 100644 --- a/packages/studio/src/player/components/thumbnailUtils.test.ts +++ b/packages/studio/src/player/components/thumbnailUtils.test.ts @@ -20,6 +20,10 @@ describe("computeThumbnailStrip", () => { expect(frameCount * frameW).toBeGreaterThanOrEqual(500); }); + it("caps rendered tiles at the shared visible-frame budget", () => { + expect(computeThumbnailStrip(10_000, 1).frameCount).toBe(33); + }); + it("returns one tile when the container width is unknown", () => { expect(computeThumbnailStrip(0, 16 / 9).frameCount).toBe(1); expect(computeThumbnailStrip(-10, 16 / 9).frameCount).toBe(1); diff --git a/packages/studio/src/player/components/thumbnailUtils.ts b/packages/studio/src/player/components/thumbnailUtils.ts index dfeb9e74a5..4315306e9d 100644 --- a/packages/studio/src/player/components/thumbnailUtils.ts +++ b/packages/studio/src/player/components/thumbnailUtils.ts @@ -75,7 +75,10 @@ export function computeThumbnailStrip( ): ThumbnailStripLayout { const safeAspect = Number.isFinite(aspect) && aspect > 0 ? aspect : 16 / 9; const frameW = Math.max(minFrameWidth, Math.round(clipHeight * safeAspect)); - const frameCount = containerWidth > 0 ? Math.max(1, Math.ceil(containerWidth / frameW)) : 1; + const frameCount = + containerWidth > 0 + ? Math.min(MAX_VISIBLE_THUMBNAIL_FRAMES, Math.max(1, Math.ceil(containerWidth / frameW))) + : 1; return { frameW, frameCount }; } From b763371bb6b0985f9d3bb4184c87b43df41a9af9 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Tue, 22 Sep 2026 01:56:46 -0400 Subject: [PATCH 4/4] chore(studio): refresh filmstrip checks