From 8dd8666d4522f897042c826299daf0e5e3f1c93d Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:17:37 -0500 Subject: [PATCH 01/10] feat(infra): serve a video's captions through the playback route and list them for the player --- .../unit/developer-video-captions.test.ts | 89 +++++++++++++++++++ .../unit/developer-video-playback.test.ts | 79 ++++++++++++++++ .../unit/developer-video-player.test.ts | 35 ++++++++ apps/web/lib/developer-video-captions.ts | 36 ++++++++ apps/web/lib/developer-video-playback.ts | 27 ++++++ apps/web/lib/developer-video-player.ts | 7 ++ 6 files changed, 273 insertions(+) diff --git a/apps/web/__tests__/unit/developer-video-captions.test.ts b/apps/web/__tests__/unit/developer-video-captions.test.ts index 0f4efce3c7a..c9b4675a240 100644 --- a/apps/web/__tests__/unit/developer-video-captions.test.ts +++ b/apps/web/__tests__/unit/developer-video-captions.test.ts @@ -79,6 +79,7 @@ vi.mock("@cap/database/helpers", () => ({ vi.mock("@cap/env", () => ({ buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, NODE_ENV: "production", + serverEnv: () => ({ WEB_URL: "https://web.example" }), })); vi.mock("@/lib/infra-access", () => ({ canUseInfra: () => true })); @@ -155,7 +156,10 @@ import { import { captionLanguage, captionLanguageLabel, + captionsFileLanguage, + captionsFileName, developerVideoCaptions, + developerVideoCaptionTracks, findVideoForTranscript, syncVideoCaptions, videoCaptionsKey, @@ -650,6 +654,91 @@ describe("caption files for the playback side", () => { }); }); +describe("caption tracks for the hosted player", () => { + const PLAYBACK = "https://web.example/api/developer/playback/playback-1"; + const PLAYABLE = { ...VIDEO, playbackId: "playback-1" }; + + it("lists nothing for a video without captions", async () => { + expect(await developerVideoCaptionTracks(PLAYABLE)).toEqual([]); + }); + + it("points each language at its file on the playback route", async () => { + seedCompleted("tr-1", "hello there"); + seedCompleted("tr-2", "hallo zusammen", { language: "de" }); + seedCompleted("tr-3", "olá a todos", { language: "pt_BR" }); + + const tracks = await developerVideoCaptionTracks(PLAYABLE); + + expect(tracks).toHaveLength(3); + expect(tracks).toContainEqual({ + language: "en", + label: "English", + url: `${PLAYBACK}/captions-en.vtt`, + }); + expect(tracks).toContainEqual({ + language: "de", + label: "German", + url: `${PLAYBACK}/captions-de.vtt`, + }); + expect(tracks.find((track) => track.language === "pt-br")?.url).toBe( + `${PLAYBACK}/captions-pt-br.vtt`, + ); + }); + + it("never hands out a storage key or a storage URL", async () => { + seedCompleted("tr-1", "hello there"); + + const [track] = await developerVideoCaptionTracks(PLAYABLE); + + expect(track).not.toHaveProperty("key"); + expect(track?.url.startsWith(`${PLAYBACK}/`)).toBe(true); + expect(track?.url).not.toContain("developer/app-1"); + }); + + it("lists nothing for a video with no playback id", async () => { + seedCompleted("tr-1", "hello there"); + + expect( + await developerVideoCaptionTracks({ ...VIDEO, playbackId: null }), + ).toEqual([]); + }); + + it("leaves the video playing without captions when the lookup fails", async () => { + const error = vi + .spyOn(console, "error") + .mockImplementation(() => undefined); + const broken = { ...PLAYABLE }; + Object.defineProperty(broken, "appId", { + get: () => { + throw new Error("database down"); + }, + }); + + expect(await developerVideoCaptionTracks(broken)).toEqual([]); + expect(error).toHaveBeenCalled(); + error.mockRestore(); + }); + + it("names a file by language and reads the language back", () => { + expect(captionsFileName("en")).toBe("captions-en.vtt"); + expect(captionsFileName("PT_br")).toBe("captions-pt-br.vtt"); + expect(captionsFileName("../../x")).toBe("captions-und.vtt"); + expect(captionsFileLanguage("captions-en.vtt")).toBe("en"); + expect(captionsFileLanguage("captions-pt-br.vtt")).toBe("pt-br"); + expect(captionsFileLanguage("captions-und.vtt")).toBe("und"); + for (const file of [ + "captions-EN.vtt", + "captions-.vtt", + "en.vtt", + "captions-en.m3u8", + "master.m3u8", + "video.mp4", + ]) { + expect(captionsFileLanguage(file)).toBeNull(); + } + }); +}); + describe("the caption file of a video", () => { const sync = (language: string | null = "en") => syncVideoCaptions({ appId: "app-1", videoId: "video-1", language }); diff --git a/apps/web/__tests__/unit/developer-video-playback.test.ts b/apps/web/__tests__/unit/developer-video-playback.test.ts index cf1f7f24834..f7b8688b8bd 100644 --- a/apps/web/__tests__/unit/developer-video-playback.test.ts +++ b/apps/web/__tests__/unit/developer-video-playback.test.ts @@ -729,6 +729,85 @@ describe("developer video playback route", () => { expect(response.headers.get("cache-control")).toBe("no-store"); }); + describe("captions", () => { + const CAPTIONS_KEY = "developer/app-1/video-1/captions/en.vtt"; + const vtt = "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHello there\n"; + + beforeEach(() => { + mocks.readText.mockImplementation(async (key: string) => + key === CAPTIONS_KEY ? vtt : null, + ); + }); + + it("serves the WebVTT file from storage to any origin with the playlists' cache", async () => { + const response = await play("captions-en.vtt"); + + expect(response.status).toBe(200); + expect(await response.text()).toBe(vtt); + expect(response.headers.get("content-type")).toBe( + "text/vtt; charset=utf-8", + ); + expect(response.headers.get("cache-control")).toBe( + "public, max-age=60, s-maxage=60", + ); + expect(response.headers.get("access-control-allow-origin")).toBe("*"); + expect(response.headers.get("location")).toBeNull(); + expect(mocks.readText).toHaveBeenCalledWith(CAPTIONS_KEY); + }); + + it("signs no storage URL for captions", async () => { + await play("captions-en.vtt"); + + expect(mocks.sign).not.toHaveBeenCalled(); + }); + + it("reads the file of a language with a region and of an unreported language", async () => { + await play("captions-pt-br.vtt"); + await play("captions-und.vtt"); + + expect(mocks.readText.mock.calls.map(([key]) => key)).toEqual([ + "developer/app-1/video-1/captions/pt-br.vtt", + "developer/app-1/video-1/captions/und.vtt", + ]); + }); + + it("answers 404 without caching when the video has no captions in that language", async () => { + const response = await play("captions-fr.vtt"); + + expect(response.status).toBe(404); + expect(response.headers.get("cache-control")).toBe("no-store"); + }); + + it("answers 404 for captions of a deleted video", async () => { + mocks.store.video = null; + + expect((await play("captions-en.vtt")).status).toBe(404); + expect(mocks.readText).not.toHaveBeenCalled(); + }); + + it("refuses captions of a paused app", async () => { + mocks.standing.mockResolvedValue({ state: "paused", pausesAt: null }); + + expect((await play("captions-en.vtt")).status).toBe(402); + expect(mocks.readText).not.toHaveBeenCalled(); + }); + + it.each([ + "captions-EN.vtt", + "captions-.vtt", + "captions-en", + "captions-e.vtt", + "captions-en-.vtt", + "captions-en/../x.vtt", + "captions-en.vtt.bak", + "en.vtt", + "captions-english-language.vtt", + ])("answers 404 for the captions file name %s", async (file) => { + expect((await play(file)).status).toBe(404); + expect(mocks.readText).not.toHaveBeenCalled(); + }); + }); + it("answers preflight and HEAD requests", async () => { const preflight = OPTIONS(); diff --git a/apps/web/__tests__/unit/developer-video-player.test.ts b/apps/web/__tests__/unit/developer-video-player.test.ts index 3d5c9707468..cda27ea4630 100644 --- a/apps/web/__tests__/unit/developer-video-player.test.ts +++ b/apps/web/__tests__/unit/developer-video-player.test.ts @@ -9,6 +9,7 @@ const mocks = vi.hoisted(() => ({ store: { video: null, appDeleted: false, updates: [] } as DeveloperVideoStore, standing: vi.fn(), redrive: vi.fn(), + captions: vi.fn(), })); vi.mock("@cap/database", async () => { @@ -28,6 +29,9 @@ vi.mock("@/lib/infra-standing", () => ({ vi.mock("@/lib/developer-video-start", () => ({ redriveDeveloperVideoProcessing: mocks.redrive, })); +vi.mock("@/lib/developer-video-captions", () => ({ + developerVideoCaptionTracks: mocks.captions, +})); const { loadDeveloperVideoPlayer } = await import( "@/lib/developer-video-player" @@ -54,6 +58,7 @@ beforeEach(() => { mocks.store.appDeleted = false; mocks.standing.mockResolvedValue({ state: "active", pausesAt: null }); mocks.redrive.mockResolvedValue(false); + mocks.captions.mockResolvedValue([]); }); const MINUTE = 60_000; @@ -73,10 +78,40 @@ describe("developer video player data", () => { height: 1080, source: { hls: `${BASE}/master.m3u8`, mp4: `${BASE}/video.mp4` }, poster: `${BASE}/poster.jpg`, + captions: null, }); expect(mocks.standing).toHaveBeenCalledWith("app-1"); }); + it("lists the captions of a video that has them", async () => { + const english = { + language: "en", + label: "English", + url: `${BASE}/captions-en.vtt`, + }; + mocks.captions.mockResolvedValue([english]); + + const player = await loadDeveloperVideoPlayer(PLAYBACK_ID); + + expect(player).toMatchObject({ kind: "playing", captions: [english] }); + expect(mocks.captions).toHaveBeenCalledWith( + expect.objectContaining({ + id: "video-1", + appId: "app-1", + playbackId: PLAYBACK_ID, + }), + ); + }); + + it("looks up no captions for a video that cannot play yet", async () => { + mocks.store.video = ready({ status: "processing", processing: null }); + + expect(await loadDeveloperVideoPlayer(PLAYBACK_ID)).toMatchObject({ + kind: "processing", + }); + expect(mocks.captions).not.toHaveBeenCalled(); + }); + it("plays the MP4 while the adaptive stream is still encoding", async () => { mocks.store.video = ready({ processing: processingState({ diff --git a/apps/web/lib/developer-video-captions.ts b/apps/web/lib/developer-video-captions.ts index 6118a6b5e9f..7fc72ccf610 100644 --- a/apps/web/lib/developer-video-captions.ts +++ b/apps/web/lib/developer-video-captions.ts @@ -10,6 +10,7 @@ import { } from "@/lib/developer-transcript-storage"; import { buildWebVtt } from "@/lib/developer-transcript-vtt"; import type { TranscriptResultObject } from "@/lib/developer-transcripts"; +import { developerPlaybackBaseUrl } from "@/lib/developer-video-api"; import { developerVideoKeys } from "@/lib/developer-video-plan"; export const CAPTIONS_CONTENT_TYPE = "text/vtt; charset=utf-8"; @@ -49,6 +50,16 @@ export function videoCaptionsKey(video: VideoRef, language: string) { return `developer/${video.appId}/${video.id}/captions/${captionLanguage(language)}.vtt`; } +const CAPTIONS_FILE_PATTERN = /^captions-([a-z]{2,3}(?:-[a-z0-9]{2,8})*)\.vtt$/; + +export function captionsFileName(language: string) { + return `captions-${captionLanguage(language)}.vtt`; +} + +export function captionsFileLanguage(file: string): string | null { + return file.match(CAPTIONS_FILE_PATTERN)?.[1] ?? null; +} + const captionSources = (video: VideoRef) => and( eq(developerTranscripts.appId, video.appId), @@ -81,6 +92,31 @@ export async function developerVideoCaptions( ); } +export type DeveloperVideoCaptionTrack = { + language: string; + label: string; + url: string; +}; + +// Captions are extra to playback, so a failed lookup leaves the video playing +// without them instead of failing the page. +export async function developerVideoCaptionTracks( + video: VideoRef & { playbackId: string | null }, +): Promise { + if (!video.playbackId) return []; + try { + const base = `${developerPlaybackBaseUrl()}/${video.playbackId}`; + return (await developerVideoCaptions(video)).map(({ language, label }) => ({ + language, + label, + url: `${base}/${captionsFileName(language)}`, + })); + } catch (error) { + console.error("[developer-video-captions] Failed to list captions", error); + return []; + } +} + async function newestCaptionSource(video: VideoRef, language: string) { const stored = await db() .selectDistinct({ language: developerTranscripts.language }) diff --git a/apps/web/lib/developer-video-playback.ts b/apps/web/lib/developer-video-playback.ts index b987a9f5327..90708512893 100644 --- a/apps/web/lib/developer-video-playback.ts +++ b/apps/web/lib/developer-video-playback.ts @@ -2,6 +2,11 @@ import { db } from "@cap/database"; import { developerApps, developerVideos } from "@cap/database/schema"; import { and, eq, isNull } from "drizzle-orm"; import { developerVideoOutputs } from "./developer-video-api"; +import { + CAPTIONS_CONTENT_TYPE, + captionsFileLanguage, + videoCaptionsKey, +} from "./developer-video-captions"; import { DEVELOPER_VIDEO_MASTER_PLAYLIST, DEVELOPER_VIDEO_MP4_FILE, @@ -248,6 +253,20 @@ function playlistResponse(playlist: string): Response { }); } +// Captions are read here and sent as they are, like the playlists, rather than +// redirected to a signed URL. A loads in same-origin mode unless its +// video sets crossorigin, so it cannot follow a redirect to storage on another +// origin, and setting crossorigin would change how every media file loads. +function captionsResponse(captions: string): Response { + return new Response(captions, { + headers: { + ...PLAYBACK_CORS_HEADERS, + ...CACHEABLE, + "Content-Type": CAPTIONS_CONTENT_TYPE, + }, + }); +} + export async function findPlayableVideo(playbackId: string) { const [row] = await db() .select({ video: developerVideos }) @@ -282,6 +301,14 @@ export async function developerPlaybackResponse( return paused(); } + const captionsLanguage = captionsFileLanguage(file); + if (captionsLanguage) { + const captions = await readDeveloperVideoText( + videoCaptionsKey(video, captionsLanguage), + ); + return captions === null ? notFound() : captionsResponse(captions); + } + const { processing } = video; const outputs = developerVideoOutputs(video); const keysFor = (output: string) => diff --git a/apps/web/lib/developer-video-player.ts b/apps/web/lib/developer-video-player.ts index 69b8c076ae2..03dd6caebbb 100644 --- a/apps/web/lib/developer-video-player.ts +++ b/apps/web/lib/developer-video-player.ts @@ -1,4 +1,8 @@ import { developerVideoPlaybackUrls } from "./developer-video-api"; +import { + type DeveloperVideoCaptionTrack, + developerVideoCaptionTracks, +} from "./developer-video-captions"; import { findPlayableVideo, PLAYBACK_ID_PATTERN, @@ -22,6 +26,7 @@ export type DeveloperVideoPlayer = DeveloperVideoFrame & kind: "playing"; source: { hls?: string; mp4?: string }; poster: string | null; + captions: DeveloperVideoCaptionTrack[] | null; } | { kind: "processing"; uploading: boolean } | { kind: "failed" } @@ -67,11 +72,13 @@ export async function loadDeveloperVideoPlayer( const { hlsUrl, mp4Url, posterUrl } = developerVideoPlaybackUrls(video); if (hlsUrl || mp4Url) { + const captions = await developerVideoCaptionTracks(video); return { ...frame, kind: "playing", source: { hls: hlsUrl ?? undefined, mp4: mp4Url ?? undefined }, poster: posterUrl, + captions: captions.length > 0 ? captions : null, }; } From c57f7a4e0b043bcbee37846ca4135654008b06e2 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:17:37 -0500 Subject: [PATCH 02/10] feat(infra): captions button and cues in the hosted player --- .../unit/cap-player-captions.test.ts | 259 ++++++++++++++++++ apps/web/components/cap-player/CapPlayer.tsx | 203 +++++++++++++- apps/web/components/cap-player/captions.ts | 70 +++++ .../components/cap-player/player.module.css | 57 ++++ apps/web/components/cap-player/source.ts | 5 + 5 files changed, 590 insertions(+), 4 deletions(-) create mode 100644 apps/web/__tests__/unit/cap-player-captions.test.ts create mode 100644 apps/web/components/cap-player/captions.ts diff --git a/apps/web/__tests__/unit/cap-player-captions.test.ts b/apps/web/__tests__/unit/cap-player-captions.test.ts new file mode 100644 index 00000000000..d91b7fa90d2 --- /dev/null +++ b/apps/web/__tests__/unit/cap-player-captions.test.ts @@ -0,0 +1,259 @@ +// @vitest-environment jsdom + +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + activeCueText, + type CapPlayerCaption, + initialCaptions, + loadCaptionsFile, + toggledCaptions, +} from "@/components/cap-player/captions"; + +vi.mock("@/components/cap-player/source", async (original) => ({ + ...(await original()), + attachSource: () => ({ + stop: () => undefined, + warm: () => undefined, + sharpen: () => undefined, + setLevel: () => undefined, + }), +})); + +const { CapPlayer } = await import("@/components/cap-player/CapPlayer"); + +const english: CapPlayerCaption = { + language: "en", + label: "English", + url: "https://playback.example/p/captions-en.vtt", +}; +const german: CapPlayerCaption = { + language: "de", + label: "German", + url: "https://playback.example/p/captions-de.vtt", +}; +const VTT = "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHello there\n"; + +describe("caption state", () => { + it("starts off unless asked, and on the first language when asked", () => { + expect(initialCaptions([english, german], false)).toBeNull(); + expect(initialCaptions([english, german], true)).toBe("en"); + expect(initialCaptions([], true)).toBeNull(); + }); + + it("turns off when on, and back on in the language picked last", () => { + expect(toggledCaptions("en", "en", [english, german])).toBeNull(); + expect(toggledCaptions(null, "de", [english, german])).toBe("de"); + expect(toggledCaptions(null, null, [english, german])).toBe("en"); + expect(toggledCaptions(null, "fr", [english, german])).toBe("en"); + expect(toggledCaptions(null, null, [])).toBeNull(); + }); + + it("reads the text of every active cue as lines", () => { + const cues = [ + { getCueAsHTML: () => ({ textContent: " Hello &" }) }, + { text: "there" }, + { text: "" }, + ] as unknown as TextTrackCueList; + + expect(activeCueText(cues)).toBe("Hello &\nthere"); + expect(activeCueText(null)).toBe(""); + }); +}); + +describe("loading a caption file", () => { + beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(VTT, { status: 200 })), + ); + URL.createObjectURL = vi.fn(() => "blob:captions"); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("loads each file once and hands it over as a blob URL", async () => { + const cache = new Map>(); + + expect(await loadCaptionsFile(cache, english.url)).toBe("blob:captions"); + expect(await loadCaptionsFile(cache, english.url)).toBe("blob:captions"); + expect(fetch).toHaveBeenCalledTimes(1); + expect(fetch).toHaveBeenCalledWith(english.url, { credentials: "omit" }); + }); + + it("refuses a missing file or one that is not WebVTT, and tries again later", async () => { + const cache = new Map>(); + vi.mocked(fetch).mockResolvedValueOnce( + new Response("Not found", { status: 404 }), + ); + await expect(loadCaptionsFile(cache, english.url)).rejects.toThrow("404"); + expect(cache.size).toBe(0); + + vi.mocked(fetch).mockResolvedValueOnce(new Response("")); + await expect(loadCaptionsFile(cache, english.url)).rejects.toThrow( + "not WebVTT", + ); + + expect(await loadCaptionsFile(cache, english.url)).toBe("blob:captions"); + }); +}); + +describe("the captions button", () => { + let container: HTMLDivElement; + let root: Root; + + const mount = async (props: { + captions?: CapPlayerCaption[]; + showCaptions?: boolean; + controls?: boolean; + }) => { + await act(async () => { + root.render( + createElement(CapPlayer, { + source: { mp4: "https://playback.example/p/video.mp4" }, + title: "Launch demo", + ...props, + }), + ); + }); + }; + const button = () => + container.querySelector( + 'button[aria-label^="Captions"]', + ); + const player = () => container.firstElementChild as HTMLElement; + const press = async (key: string) => { + const event = new KeyboardEvent("keydown", { + key, + bubbles: true, + cancelable: true, + }); + await act(async () => { + player().dispatchEvent(event); + }); + return event; + }; + const click = async (element: Element | null) => { + await act(async () => { + element?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + }; + + beforeEach(() => { + ( + globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean } + ).IS_REACT_ACT_ENVIRONMENT = true; + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(VTT, { status: 200 })), + ); + URL.createObjectURL = vi.fn(() => "blob:captions"); + URL.revokeObjectURL = vi.fn(); + Element.prototype.scrollIntoView = () => undefined; + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); + }); + + afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); + }); + + it("is not there for a video without captions, and c does nothing", async () => { + await mount({}); + + expect(button()).toBeNull(); + expect((await press("c")).defaultPrevented).toBe(false); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("starts off and loads nothing until it is pressed", async () => { + await mount({ captions: [english] }); + + expect(button()?.getAttribute("aria-pressed")).toBe("false"); + expect(fetch).not.toHaveBeenCalled(); + + await click(button()); + + expect(button()?.getAttribute("aria-pressed")).toBe("true"); + expect(fetch).toHaveBeenCalledWith(english.url, { credentials: "omit" }); + expect(container.querySelector("track")?.getAttribute("src")).toBe( + "blob:captions", + ); + + await click(button()); + + expect(button()?.getAttribute("aria-pressed")).toBe("false"); + }); + + it("toggles with the c key", async () => { + await mount({ captions: [english] }); + + const on = await press("c"); + expect(on.defaultPrevented).toBe(true); + expect(button()?.getAttribute("aria-pressed")).toBe("true"); + + await press("C"); + expect(button()?.getAttribute("aria-pressed")).toBe("false"); + }); + + it("starts on when asked", async () => { + await mount({ captions: [english], showCaptions: true }); + + expect(button()?.getAttribute("aria-pressed")).toBe("true"); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("keeps the c key working when the control bar is hidden", async () => { + await mount({ captions: [english], controls: false }); + + await press("c"); + + expect(fetch).toHaveBeenCalledWith(english.url, { credentials: "omit" }); + }); + + it("goes away when the file cannot be loaded", async () => { + vi.mocked(fetch).mockResolvedValue( + new Response("Not found", { status: 404 }), + ); + await mount({ captions: [english], showCaptions: true }); + + expect(button()).toBeNull(); + expect((await press("c")).defaultPrevented).toBe(false); + }); + + it("offers a language menu when there is more than one", async () => { + await mount({ captions: [english, german] }); + + expect(button()?.getAttribute("aria-label")).toBe("Captions, off"); + expect(button()?.getAttribute("aria-haspopup")).toBe("menu"); + + await click(button()); + const items = [...container.querySelectorAll('[role="menuitemradio"]')].map( + (item) => [item.textContent, item.getAttribute("aria-checked")], + ); + expect(items).toEqual([ + ["Off", "true"], + ["English", "false"], + ["German", "false"], + ]); + + await click( + [...container.querySelectorAll('[role="menuitemradio"]')].find( + (item) => item.textContent === "German", + ) ?? null, + ); + expect(button()?.getAttribute("aria-label")).toBe("Captions, German"); + expect(fetch).toHaveBeenCalledWith(german.url, { credentials: "omit" }); + + await press("c"); + expect(button()?.getAttribute("aria-label")).toBe("Captions, off"); + await press("c"); + expect(button()?.getAttribute("aria-label")).toBe("Captions, German"); + }); +}); diff --git a/apps/web/components/cap-player/CapPlayer.tsx b/apps/web/components/cap-player/CapPlayer.tsx index 48930645e0c..943627fa879 100644 --- a/apps/web/components/cap-player/CapPlayer.tsx +++ b/apps/web/components/cap-player/CapPlayer.tsx @@ -9,6 +9,14 @@ import { useRef, useState, } from "react"; +import { + activeCueText, + type CapPlayerCaption, + initialCaptions, + loadCaptionsFile, + releaseCaptionsFiles, + toggledCaptions, +} from "./captions"; import styles from "./player.module.css"; import { type Attachment, @@ -19,6 +27,7 @@ import { resolutionLabel, } from "./source"; +export type { CapPlayerCaption } from "./captions"; export { resolutionLabel } from "./source"; export type CapPlayerControl = @@ -26,6 +35,7 @@ export type CapPlayerControl = | "time" | "speed" | "quality" + | "captions" | "pip" | "fullscreen"; @@ -39,6 +49,8 @@ export interface CapPlayerProps { loop?: boolean; controls?: boolean; startAt?: number; + captions?: readonly CapPlayerCaption[]; + showCaptions?: boolean; focusOnMount?: boolean; accentColor?: string; hideControls?: readonly CapPlayerControl[]; @@ -61,8 +73,14 @@ const MENU_MIN_HEIGHT = 76; const HOVER_INTENT_MS = 100; const PLAYS = '[data-action="toggle"], [data-action="surface"]'; const NO_CONTROLS: readonly CapPlayerControl[] = []; +const NO_CAPTIONS: readonly CapPlayerCaption[] = []; +const CAPTIONS_OFF = -1; + +type MenuName = "speed" | "quality" | "captions"; -type MenuName = "speed" | "quality"; +type NativeViewVideo = HTMLVideoElement & { + webkitDisplayingFullscreen?: boolean; +}; export const clock = (seconds: number) => { const whole = Math.max(0, Math.floor(seconds)); @@ -160,6 +178,16 @@ const PipIcon = () => ( ); +const CaptionsIcon = () => ( + + + + +); + const CheckIcon = () => ( @@ -255,6 +283,8 @@ export const CapPlayer = ({ loop = false, controls = true, startAt = 0, + captions = NO_CAPTIONS, + showCaptions = false, focusOnMount = false, accentColor, hideControls = NO_CONTROLS, @@ -280,8 +310,32 @@ export const CapPlayer = ({ const attachmentRef = useRef(null); const menuDismissedAt = useRef(-1); const levelsRef = useRef([]); - const live = useRef({ onFirstPlay, onEnded, speeds }); - live.current = { onFirstPlay, onEnded, speeds }; + const captionTracks = useRef(new Map()); + const captionFiles = useRef(new Map>()); + + const [unavailable, setUnavailable] = useState([]); + const [captionsOn, setCaptionsOn] = useState(() => + initialCaptions(captions, showCaptions), + ); + const [cue, setCue] = useState(""); + const rememberedCaptions = useRef(captionsOn); + const offered = captions.filter( + (caption) => !unavailable.includes(caption.language), + ); + const toggleCaptions = () => { + if (offered.length === 0) return false; + const next = toggledCaptions( + captionsOn, + rememberedCaptions.current, + offered, + ); + if (next !== null) rememberedCaptions.current = next; + setCaptionsOn(next); + return true; + }; + + const live = useRef({ onFirstPlay, onEnded, speeds, toggleCaptions }); + live.current = { onFirstPlay, onEnded, speeds, toggleCaptions }; const [playing, setPlaying] = useState(false); const [started, setStarted] = useState(autoPlay); @@ -348,6 +402,72 @@ export const CapPlayer = ({ if (showsVolume) videoRef.current?.dispatchEvent(new Event("volumechange")); }, [showsVolume]); + useEffect(() => { + const files = captionFiles.current; + return () => releaseCaptionsFiles(files); + }, []); + + useEffect(() => { + const video = videoRef.current as NativeViewVideo | null; + const caption = captions.find((entry) => entry.language === captionsOn); + for (const [language, element] of captionTracks.current) { + if (language !== caption?.language && element.track) { + element.track.mode = "disabled"; + } + } + setCue(""); + const element = caption && captionTracks.current.get(caption.language); + if (!video || !caption || !element) return; + + let cancelled = false; + let detach: () => void = () => undefined; + // The player draws the cues itself so they match it and clear the + // control bar. Picture in picture and the video's own full screen on + // iPhone cannot show anything the page draws, so there the browser + // shows them instead. + const nativeView = () => + document.pictureInPictureElement === video || + video.webkitDisplayingFullscreen === true; + loadCaptionsFile(captionFiles.current, caption.url).then( + (src) => { + if (cancelled) return; + if (element.getAttribute("src") !== src) element.src = src; + const track = element.track; + if (!track) return; + const paint = () => + setCue(nativeView() ? "" : activeCueText(track.activeCues)); + const place = () => { + track.mode = nativeView() ? "showing" : "hidden"; + paint(); + }; + const viewEvents = [ + "enterpictureinpicture", + "leavepictureinpicture", + "webkitbeginfullscreen", + "webkitendfullscreen", + ]; + track.addEventListener("cuechange", paint); + for (const name of viewEvents) video.addEventListener(name, place); + detach = () => { + track.removeEventListener("cuechange", paint); + for (const name of viewEvents) { + video.removeEventListener(name, place); + } + }; + place(); + }, + () => { + if (cancelled) return; + setUnavailable((languages) => [...languages, caption.language]); + setCaptionsOn(null); + }, + ); + return () => { + cancelled = true; + detach(); + }; + }, [captions, captionsOn]); + useEffect(() => { if (!menu) return; const onPointerDown = (event: PointerEvent) => { @@ -679,6 +799,8 @@ export const CapPlayer = ({ stepSpeed(1); } else if (key === "<") { stepSpeed(-1); + } else if (key === "c") { + if (!live.current.toggleCaptions()) return; } else { return; } @@ -874,6 +996,37 @@ export const CapPlayer = ({ checked: value === speed, })); + const captionItems: MenuItem[] = [ + { value: CAPTIONS_OFF, label: "Off", checked: captionsOn === null }, + ...offered.map((caption, index) => ({ + value: index, + label: caption.label, + checked: caption.language === captionsOn, + })), + ]; + + const pickCaptions = (value: number) => { + const language = offered[value]?.language ?? null; + if (language !== null) rememberedCaptions.current = language; + setCaptionsOn(language); + closeMenu("captions"); + }; + + const captionTrack = (caption: CapPlayerCaption | undefined) => + caption + ? { + srcLang: caption.language, + label: caption.label, + ref: (element: HTMLTrackElement | null) => { + if (element) captionTracks.current.set(caption.language, element); + else captionTracks.current.delete(caption.language); + }, + } + : {}; + + const captionsLabel = + offered.find((caption) => caption.language === captionsOn)?.label ?? "off"; + const canPickQuality = levels.length > 1; const pickedLabel = levels.find((level) => level.index === picked)?.label; const quality = pickedLabel ?? resolution; @@ -906,8 +1059,16 @@ export const CapPlayer = ({ data-action="surface" aria-label={title} > - + + {captions.slice(1).map((caption) => ( + + ))} + {cue ? ( +
+ {cue} +
+ ) : null}