diff --git a/apps/web/__tests__/unit/cap-player-captions.test.ts b/apps/web/__tests__/unit/cap-player-captions.test.ts new file mode 100644 index 0000000000..6713a24d05 --- /dev/null +++ b/apps/web/__tests__/unit/cap-player-captions.test.ts @@ -0,0 +1,301 @@ +// @vitest-environment jsdom + +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + activeCueText, + type CapPlayerCaption, + initialCaptions, + loadCaptionsFile, + toggledCaptions, +} from "@/components/cap-player/captions"; + +vi.mock("@/components/cap-player/source", async (original) => ({ + ...(await original()), + attachSource: () => ({ + stop: () => undefined, + warm: () => undefined, + sharpen: () => undefined, + setLevel: () => undefined, + }), +})); + +const { CapPlayer } = await import("@/components/cap-player/CapPlayer"); + +const english: CapPlayerCaption = { + language: "en", + label: "English", + url: "https://playback.example/p/captions-en.vtt", +}; +const german: CapPlayerCaption = { + language: "de", + label: "German", + url: "https://playback.example/p/captions-de.vtt", +}; +const VTT = "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHello there\n"; + +describe("caption state", () => { + it("starts off unless asked, and on the first language when asked", () => { + expect(initialCaptions([english, german], false)).toBeNull(); + expect(initialCaptions([english, german], true)).toBe("en"); + expect(initialCaptions([], true)).toBeNull(); + }); + + it("turns off when on, and back on in the language picked last", () => { + expect(toggledCaptions("en", "en", [english, german])).toBeNull(); + expect(toggledCaptions(null, "de", [english, german])).toBe("de"); + expect(toggledCaptions(null, null, [english, german])).toBe("en"); + expect(toggledCaptions(null, "fr", [english, german])).toBe("en"); + expect(toggledCaptions(null, null, [])).toBeNull(); + }); + + it("reads the text of every active cue as lines", () => { + const cues = [ + { getCueAsHTML: () => ({ textContent: " Hello &" }) }, + { text: "there" }, + { text: "" }, + ] as unknown as TextTrackCueList; + + expect(activeCueText(cues)).toBe("Hello &\nthere"); + expect(activeCueText(null)).toBe(""); + }); +}); + +describe("loading a caption file", () => { + beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(VTT, { status: 200 })), + ); + URL.createObjectURL = vi.fn(() => "blob:captions"); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("loads each file once and hands it over as a blob URL", async () => { + const cache = new Map>(); + + expect(await loadCaptionsFile(cache, english.url)).toBe("blob:captions"); + expect(await loadCaptionsFile(cache, english.url)).toBe("blob:captions"); + expect(fetch).toHaveBeenCalledTimes(1); + expect(fetch).toHaveBeenCalledWith(english.url, { credentials: "omit" }); + }); + + it("refuses a missing file or one that is not WebVTT, and tries again later", async () => { + const cache = new Map>(); + vi.mocked(fetch).mockResolvedValueOnce( + new Response("Not found", { status: 404 }), + ); + await expect(loadCaptionsFile(cache, english.url)).rejects.toThrow("404"); + expect(cache.size).toBe(0); + + vi.mocked(fetch).mockResolvedValueOnce(new Response("")); + await expect(loadCaptionsFile(cache, english.url)).rejects.toThrow( + "not WebVTT", + ); + + expect(await loadCaptionsFile(cache, english.url)).toBe("blob:captions"); + }); +}); + +describe("the captions button", () => { + let container: HTMLDivElement; + let root: Root; + + const mount = async (props: { + captions?: CapPlayerCaption[]; + showCaptions?: boolean; + controls?: boolean; + }) => { + await act(async () => { + root.render( + createElement(CapPlayer, { + source: { mp4: "https://playback.example/p/video.mp4" }, + title: "Launch demo", + ...props, + }), + ); + }); + }; + const button = () => + container.querySelector( + 'button[aria-label^="Captions"]', + ); + const player = () => container.firstElementChild as HTMLElement; + const press = async (key: string) => { + const event = new KeyboardEvent("keydown", { + key, + bubbles: true, + cancelable: true, + }); + await act(async () => { + player().dispatchEvent(event); + }); + return event; + }; + const click = async (element: Element | null) => { + await act(async () => { + element?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + }; + + beforeEach(() => { + ( + globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean } + ).IS_REACT_ACT_ENVIRONMENT = true; + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(VTT, { status: 200 })), + ); + URL.createObjectURL = vi.fn(() => "blob:captions"); + URL.revokeObjectURL = vi.fn(); + Element.prototype.scrollIntoView = () => undefined; + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); + }); + + afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); + }); + + it("is not there for a video without captions, and c does nothing", async () => { + await mount({}); + + expect(button()).toBeNull(); + expect((await press("c")).defaultPrevented).toBe(false); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("starts off and loads nothing until it is pressed", async () => { + await mount({ captions: [english] }); + + expect(button()?.getAttribute("aria-pressed")).toBe("false"); + expect(fetch).not.toHaveBeenCalled(); + expect(container.querySelectorAll("track[src]")).toHaveLength(0); + + await click(button()); + + expect(button()?.getAttribute("aria-pressed")).toBe("true"); + expect(fetch).toHaveBeenCalledWith(english.url, { credentials: "omit" }); + const track = container.querySelector("track[src]"); + expect(track?.getAttribute("src")).toBe("blob:captions"); + expect(track?.kind).toBe("captions"); + expect(track?.srclang).toBe("en"); + expect(track?.label).toBe("English"); + + await click(button()); + + expect(button()?.getAttribute("aria-pressed")).toBe("false"); + }); + + it("keeps captions on through a refresh that hands over the same list", async () => { + await mount({ captions: [english], showCaptions: true }); + const track = container.querySelector("track[src]"); + + await mount({ captions: [{ ...english }], showCaptions: true }); + + expect(button()?.getAttribute("aria-pressed")).toBe("true"); + expect(fetch).toHaveBeenCalledTimes(1); + expect(container.querySelector("track[src]")).toBe(track); + }); + + it("offers captions that arrive after the player mounted", async () => { + await mount({}); + expect(button()).toBeNull(); + + await mount({ captions: [english] }); + expect(button()?.getAttribute("aria-pressed")).toBe("false"); + + await mount({ captions: [english, german] }); + expect(button()?.getAttribute("aria-label")).toBe("Captions, off"); + }); + + it("turns captions off when a refresh removes the language that was on", async () => { + await mount({ captions: [english, german] }); + await click(button()); + await click( + [...container.querySelectorAll('[role="menuitemradio"]')].find( + (item) => item.textContent === "English", + ) ?? null, + ); + expect(button()?.getAttribute("aria-label")).toBe("Captions, English"); + + await mount({ captions: [german] }); + + expect(button()?.getAttribute("aria-pressed")).toBe("false"); + await press("c"); + expect(button()?.getAttribute("aria-pressed")).toBe("true"); + }); + + it("toggles with the c key", async () => { + await mount({ captions: [english] }); + + const on = await press("c"); + expect(on.defaultPrevented).toBe(true); + expect(button()?.getAttribute("aria-pressed")).toBe("true"); + + await press("C"); + expect(button()?.getAttribute("aria-pressed")).toBe("false"); + }); + + it("starts on when asked", async () => { + await mount({ captions: [english], showCaptions: true }); + + expect(button()?.getAttribute("aria-pressed")).toBe("true"); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("keeps the c key working when the control bar is hidden", async () => { + await mount({ captions: [english], controls: false }); + + await press("c"); + + expect(fetch).toHaveBeenCalledWith(english.url, { credentials: "omit" }); + }); + + it("goes away when the file cannot be loaded", async () => { + vi.mocked(fetch).mockResolvedValue( + new Response("Not found", { status: 404 }), + ); + await mount({ captions: [english], showCaptions: true }); + + expect(button()).toBeNull(); + expect((await press("c")).defaultPrevented).toBe(false); + }); + + it("offers a language menu when there is more than one", async () => { + await mount({ captions: [english, german] }); + + expect(button()?.getAttribute("aria-label")).toBe("Captions, off"); + expect(button()?.getAttribute("aria-haspopup")).toBe("menu"); + + await click(button()); + const items = [...container.querySelectorAll('[role="menuitemradio"]')].map( + (item) => [item.textContent, item.getAttribute("aria-checked")], + ); + expect(items).toEqual([ + ["Off", "true"], + ["English", "false"], + ["German", "false"], + ]); + + await click( + [...container.querySelectorAll('[role="menuitemradio"]')].find( + (item) => item.textContent === "German", + ) ?? null, + ); + expect(button()?.getAttribute("aria-label")).toBe("Captions, German"); + expect(fetch).toHaveBeenCalledWith(german.url, { credentials: "omit" }); + + await press("c"); + expect(button()?.getAttribute("aria-label")).toBe("Captions, off"); + await press("c"); + expect(button()?.getAttribute("aria-label")).toBe("Captions, German"); + }); +}); diff --git a/apps/web/__tests__/unit/developer-transcript-docs.test.ts b/apps/web/__tests__/unit/developer-transcript-docs.test.ts index 326417a6a5..76e5a4b357 100644 --- a/apps/web/__tests__/unit/developer-transcript-docs.test.ts +++ b/apps/web/__tests__/unit/developer-transcript-docs.test.ts @@ -188,12 +188,14 @@ describe("transcribing a stored video", () => { expect(transcriptDocs).toContain("`videoId`, `contentLength`, `name`"); }); - it("says only what ships: no player captions and no captions URL", () => { + it("says only what ships: captions in the hosted player and no captions URL", () => { for (const text of [transcriptDocs, specText]) { - expect(text).not.toMatch(/captions for the hosted player/i); expect(text).not.toMatch(/\.vtt|captions\//i); } - expect(transcriptDocs).not.toMatch(/player/i); + expect(transcriptDocs).toContain( + "The hosted player and the embed then show a captions button, and the `c` key turns captions on and off. Add `captions=1` to `embedUrl` to start with them on.", + ); + expect(specText).toContain("controls=0 and captions=1"); }); }); diff --git a/apps/web/__tests__/unit/developer-video-captions.test.ts b/apps/web/__tests__/unit/developer-video-captions.test.ts index 018baabd02..509b1ec81e 100644 --- a/apps/web/__tests__/unit/developer-video-captions.test.ts +++ b/apps/web/__tests__/unit/developer-video-captions.test.ts @@ -79,6 +79,7 @@ vi.mock("@cap/database/helpers", () => ({ vi.mock("@cap/env", () => ({ buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, NODE_ENV: "production", + serverEnv: () => ({ WEB_URL: "https://web.example" }), })); vi.mock("@/lib/infra-access", () => ({ canUseInfra: () => true })); @@ -155,7 +156,10 @@ import { import { captionLanguage, captionLanguageLabel, + captionsFileLanguage, + captionsFileName, developerVideoCaptions, + developerVideoCaptionTracks, findVideoForTranscript, syncVideoCaptions, videoCaptionsKey, @@ -650,6 +654,91 @@ describe("caption files for the playback side", () => { }); }); +describe("caption tracks for the hosted player", () => { + const PLAYBACK = "https://web.example/api/developer/playback/playback-1"; + const PLAYABLE = { ...VIDEO, playbackId: "playback-1" }; + + it("lists nothing for a video without captions", async () => { + expect(await developerVideoCaptionTracks(PLAYABLE)).toEqual([]); + }); + + it("points each language at its file on the playback route", async () => { + seedCompleted("tr-1", "hello there"); + seedCompleted("tr-2", "hallo zusammen", { language: "de" }); + seedCompleted("tr-3", "olá a todos", { language: "pt_BR" }); + + const tracks = await developerVideoCaptionTracks(PLAYABLE); + + expect(tracks).toHaveLength(3); + expect(tracks).toContainEqual({ + language: "en", + label: "English", + url: `${PLAYBACK}/captions-en.vtt`, + }); + expect(tracks).toContainEqual({ + language: "de", + label: "German", + url: `${PLAYBACK}/captions-de.vtt`, + }); + expect(tracks.find((track) => track.language === "pt-br")?.url).toBe( + `${PLAYBACK}/captions-pt-br.vtt`, + ); + }); + + it("never hands out a storage key or a storage URL", async () => { + seedCompleted("tr-1", "hello there"); + + const [track] = await developerVideoCaptionTracks(PLAYABLE); + + expect(track).not.toHaveProperty("key"); + expect(track?.url.startsWith(`${PLAYBACK}/`)).toBe(true); + expect(track?.url).not.toContain("developer/app-1"); + }); + + it("lists nothing for a video with no playback id", async () => { + seedCompleted("tr-1", "hello there"); + + expect( + await developerVideoCaptionTracks({ ...VIDEO, playbackId: null }), + ).toEqual([]); + }); + + it("leaves the video playing without captions when the lookup fails", async () => { + const error = vi + .spyOn(console, "error") + .mockImplementation(() => undefined); + const broken = { ...PLAYABLE }; + Object.defineProperty(broken, "appId", { + get: () => { + throw new Error("database down"); + }, + }); + + expect(await developerVideoCaptionTracks(broken)).toEqual([]); + expect(error).toHaveBeenCalled(); + error.mockRestore(); + }); + + it("names a file by language and reads the language back", () => { + expect(captionsFileName("en")).toBe("captions-en.vtt"); + expect(captionsFileName("PT_br")).toBe("captions-pt-br.vtt"); + expect(captionsFileName("../../x")).toBe("captions-und.vtt"); + expect(captionsFileLanguage("captions-en.vtt")).toBe("en"); + expect(captionsFileLanguage("captions-pt-br.vtt")).toBe("pt-br"); + expect(captionsFileLanguage("captions-und.vtt")).toBe("und"); + for (const file of [ + "captions-EN.vtt", + "captions-.vtt", + "en.vtt", + "captions-en.m3u8", + "master.m3u8", + "video.mp4", + ]) { + expect(captionsFileLanguage(file)).toBeNull(); + } + }); +}); + describe("the caption file of a video", () => { const sync = (language: string | null = "en") => syncVideoCaptions({ appId: "app-1", videoId: "video-1", language }); diff --git a/apps/web/__tests__/unit/developer-video-page.test.ts b/apps/web/__tests__/unit/developer-video-page.test.ts index 532b7db460..5a98669566 100644 --- a/apps/web/__tests__/unit/developer-video-page.test.ts +++ b/apps/web/__tests__/unit/developer-video-page.test.ts @@ -34,6 +34,13 @@ const playing: DeveloperVideoPlayer = { kind: "playing", source: { hls: `${BASE}/master.m3u8`, mp4: `${BASE}/video.mp4` }, poster: `${BASE}/poster.jpg`, + captions: null, +}; +const captioned: DeveloperVideoPlayer = { + ...playing, + captions: [ + { language: "en", label: "English", url: `${BASE}/captions-en.vtt` }, + ], }; const processing: DeveloperVideoPlayer = { ...frame, @@ -336,6 +343,44 @@ describe("developer video page parameters", () => { expect(embed({ nope: "1" })).toBe(embed({})); }); + it("offers no captions button for a video without captions", () => { + expect(embed({})).not.toContain('aria-label="Captions"'); + expect(embed({ captions: "1" })).toBe(embed({})); + }); + + it("offers a captions button, off by default, when the video has captions", () => { + const html = render( + captioned, + "embed", + developerVideoPlayback({}, "embed"), + ); + + expect(html).toContain('aria-pressed="false"'); + expect(html).toContain('aria-label="Captions"'); + }); + + it("starts with captions on when the embed asks", () => { + const on = (captions: string) => + render(captioned, "embed", developerVideoPlayback({ captions }, "embed")); + + expect(on("1")).toContain('aria-pressed="true"'); + expect(on("0")).toContain('aria-pressed="false"'); + }); + + it("keeps the caption file out of the markup so nothing loads before captions are on", () => { + expect(render(captioned)).not.toContain("captions-en.vtt"); + }); + + it("keeps captions off on the share page whatever the query says", () => { + expect( + render( + captioned, + "share", + developerVideoPlayback({ captions: "1" }, "share"), + ), + ).toContain('aria-pressed="false"'); + }); + it("keeps the embed-only switches off the share page", () => { const query = { autoplay: "1", muted: "1", loop: "1", controls: "0" }; diff --git a/apps/web/__tests__/unit/developer-video-params.test.ts b/apps/web/__tests__/unit/developer-video-params.test.ts index fae3544fdc..596b1111a7 100644 --- a/apps/web/__tests__/unit/developer-video-params.test.ts +++ b/apps/web/__tests__/unit/developer-video-params.test.ts @@ -6,6 +6,7 @@ const DEFAULTS = { muted: false, loop: false, controls: true, + captions: false, startAt: 0, }; @@ -17,7 +18,14 @@ describe("developerVideoPlayback", () => { it("reads every embed parameter", () => { expect( developerVideoPlayback( - { autoplay: "1", muted: "1", loop: "1", controls: "0", t: "90" }, + { + autoplay: "1", + muted: "1", + loop: "1", + controls: "0", + captions: "1", + t: "90", + }, "embed", ), ).toEqual({ @@ -25,6 +33,7 @@ describe("developerVideoPlayback", () => { muted: true, loop: true, controls: false, + captions: true, startAt: 90, }); }); @@ -69,13 +78,41 @@ describe("developerVideoPlayback", () => { (value) => { expect( developerVideoPlayback( - { autoplay: value, muted: value, loop: value, controls: value }, + { + autoplay: value, + muted: value, + loop: value, + controls: value, + captions: value, + }, "embed", ), ).toEqual(DEFAULTS); }, ); + it("keeps captions off unless the embed asks for them", () => { + expect(developerVideoPlayback({}, "embed").captions).toBe(false); + expect(developerVideoPlayback({ captions: "0" }, "embed").captions).toBe( + false, + ); + expect( + developerVideoPlayback({ captions: "false" }, "embed").captions, + ).toBe(false); + expect(developerVideoPlayback({ captions: "1" }, "embed").captions).toBe( + true, + ); + expect(developerVideoPlayback({ captions: "true" }, "embed").captions).toBe( + true, + ); + expect( + developerVideoPlayback({ captions: ["1", "0"] }, "embed").captions, + ).toBe(true); + expect(developerVideoPlayback({ captions: "en" }, "embed").captions).toBe( + false, + ); + }); + it("ignores parameters it does not know", () => { expect( developerVideoPlayback( @@ -97,7 +134,14 @@ describe("developerVideoPlayback", () => { it("takes only the start time on the share page", () => { expect( developerVideoPlayback( - { autoplay: "1", muted: "1", loop: "1", controls: "0", t: "42" }, + { + autoplay: "1", + muted: "1", + loop: "1", + controls: "0", + captions: "1", + t: "42", + }, "share", ), ).toEqual({ ...DEFAULTS, startAt: 42 }); diff --git a/apps/web/__tests__/unit/developer-video-playback.test.ts b/apps/web/__tests__/unit/developer-video-playback.test.ts index cf1f7f2483..c167730be7 100644 --- a/apps/web/__tests__/unit/developer-video-playback.test.ts +++ b/apps/web/__tests__/unit/developer-video-playback.test.ts @@ -729,6 +729,99 @@ describe("developer video playback route", () => { expect(response.headers.get("cache-control")).toBe("no-store"); }); + describe("captions", () => { + const CAPTIONS_KEY = "developer/app-1/video-1/captions/en.vtt"; + const vtt = "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHello there\n"; + + beforeEach(() => { + mocks.readText.mockImplementation(async (key: string) => + key === CAPTIONS_KEY ? vtt : null, + ); + }); + + it("serves the WebVTT file from storage to any origin with the playlists' cache", async () => { + const response = await play("captions-en.vtt"); + + expect(response.status).toBe(200); + expect(await response.text()).toBe(vtt); + expect(response.headers.get("content-type")).toBe( + "text/vtt; charset=utf-8", + ); + expect(response.headers.get("cache-control")).toBe( + "public, max-age=60, s-maxage=60", + ); + expect(response.headers.get("access-control-allow-origin")).toBe("*"); + expect(response.headers.get("location")).toBeNull(); + expect(mocks.readText).toHaveBeenCalledWith(CAPTIONS_KEY); + }); + + it("signs no storage URL for captions", async () => { + await play("captions-en.vtt"); + + expect(mocks.sign).not.toHaveBeenCalled(); + }); + + it("reads the file of a language with a region and of an unreported language", async () => { + await play("captions-pt-br.vtt"); + await play("captions-und.vtt"); + + expect(mocks.readText.mock.calls.map(([key]) => key)).toEqual([ + "developer/app-1/video-1/captions/pt-br.vtt", + "developer/app-1/video-1/captions/und.vtt", + ]); + }); + + it("answers 404 without caching when the video has no captions in that language", async () => { + const response = await play("captions-fr.vtt"); + + expect(response.status).toBe(404); + expect(response.headers.get("cache-control")).toBe("no-store"); + }); + + it("answers 404 for captions of a deleted video", async () => { + mocks.store.video = null; + + expect((await play("captions-en.vtt")).status).toBe(404); + expect(mocks.readText).not.toHaveBeenCalled(); + }); + + it("answers 410 for captions of a video that was taken down", async () => { + mocks.store.video = ready({ + metadata: { + __capTakedown: { at: "2026-11-13T07:00:00.000Z", metadata: null }, + }, + }); + + const response = await play("captions-en.vtt"); + + expect(response.status).toBe(410); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(mocks.readText).not.toHaveBeenCalled(); + }); + + it("refuses captions of a paused app", async () => { + mocks.standing.mockResolvedValue({ state: "paused", pausesAt: null }); + + expect((await play("captions-en.vtt")).status).toBe(402); + expect(mocks.readText).not.toHaveBeenCalled(); + }); + + it.each([ + "captions-EN.vtt", + "captions-.vtt", + "captions-en", + "captions-e.vtt", + "captions-en-.vtt", + "captions-en/../x.vtt", + "captions-en.vtt.bak", + "en.vtt", + "captions-english-language.vtt", + ])("answers 404 for the captions file name %s", async (file) => { + expect((await play(file)).status).toBe(404); + expect(mocks.readText).not.toHaveBeenCalled(); + }); + }); + it("answers preflight and HEAD requests", async () => { const preflight = OPTIONS(); diff --git a/apps/web/__tests__/unit/developer-video-player.test.ts b/apps/web/__tests__/unit/developer-video-player.test.ts index cb8b7bb6df..81d3a2f99d 100644 --- a/apps/web/__tests__/unit/developer-video-player.test.ts +++ b/apps/web/__tests__/unit/developer-video-player.test.ts @@ -9,6 +9,7 @@ const mocks = vi.hoisted(() => ({ store: { video: null, appDeleted: false, updates: [] } as DeveloperVideoStore, standing: vi.fn(), redrive: vi.fn(), + captions: vi.fn(), })); vi.mock("@cap/database", async () => { @@ -28,6 +29,9 @@ vi.mock("@/lib/infra-standing", () => ({ vi.mock("@/lib/developer-video-start", () => ({ redriveDeveloperVideoProcessing: mocks.redrive, })); +vi.mock("@/lib/developer-video-captions", () => ({ + developerVideoCaptionTracks: mocks.captions, +})); const { loadDeveloperVideoPlayer } = await import( "@/lib/developer-video-player" @@ -54,6 +58,7 @@ beforeEach(() => { mocks.store.appDeleted = false; mocks.standing.mockResolvedValue({ state: "active", pausesAt: null }); mocks.redrive.mockResolvedValue(false); + mocks.captions.mockResolvedValue([]); }); const MINUTE = 60_000; @@ -123,10 +128,40 @@ describe("developer video player data", () => { height: 1080, source: { hls: `${BASE}/master.m3u8`, mp4: `${BASE}/video.mp4` }, poster: `${BASE}/poster.jpg`, + captions: null, }); expect(mocks.standing).toHaveBeenCalledWith("app-1"); }); + it("lists the captions of a video that has them", async () => { + const english = { + language: "en", + label: "English", + url: `${BASE}/captions-en.vtt`, + }; + mocks.captions.mockResolvedValue([english]); + + const player = await loadDeveloperVideoPlayer(PLAYBACK_ID); + + expect(player).toMatchObject({ kind: "playing", captions: [english] }); + expect(mocks.captions).toHaveBeenCalledWith( + expect.objectContaining({ + id: "video-1", + appId: "app-1", + playbackId: PLAYBACK_ID, + }), + ); + }); + + it("looks up no captions for a video that cannot play yet", async () => { + mocks.store.video = ready({ status: "processing", processing: null }); + + expect(await loadDeveloperVideoPlayer(PLAYBACK_ID)).toMatchObject({ + kind: "processing", + }); + expect(mocks.captions).not.toHaveBeenCalled(); + }); + it("plays the MP4 while the adaptive stream is still encoding", async () => { mocks.store.video = ready({ processing: processingState({ diff --git a/apps/web/__tests__/unit/infra-video-view.test.ts b/apps/web/__tests__/unit/infra-video-view.test.ts index e1bca97e9d..c0a25b3e13 100644 --- a/apps/web/__tests__/unit/infra-video-view.test.ts +++ b/apps/web/__tests__/unit/infra-video-view.test.ts @@ -50,6 +50,7 @@ const LIST_FIELDS = [ const DETAIL_FIELDS = [ ...LIST_FIELDS, + "captions", "embedCode", "failure", "height", @@ -188,6 +189,7 @@ describe("console video payload", () => { expect(detail.posterUrl).toContain("/playback-id-0123456789ab/"); expect(detail.stillEncoding).toBe(false); expect(detail.failure).toBeNull(); + expect(detail.captions).toEqual([]); const sent = JSON.stringify(detail); expect(sent).not.toContain("developer/app-1/video-1/video"); diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/videos/[videoId]/video-client.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/videos/[videoId]/video-client.tsx index a778cc6782..3017df09dc 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/videos/[videoId]/video-client.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/videos/[videoId]/video-client.tsx @@ -86,6 +86,7 @@ function Player({ video }: { video: InfraVideoDetail }) { source={source} title={video.name} poster={video.posterUrl ?? undefined} + captions={video.captions} aspectRatio={ video.width && video.height ? `${video.width} / ${video.height}` @@ -353,6 +354,14 @@ export function VideoClient({ {video.width}×{video.height} ) : null} + {video.captions.length > 0 && ( + + {video.captions.map((caption) => caption.label).join(", ")} + + )} {createdFormat.format(new Date(video.createdAt))} diff --git a/apps/web/app/(infra)/dashboard/infra/infra-data.ts b/apps/web/app/(infra)/dashboard/infra/infra-data.ts index 8c5faf8e0a..cbc6d41ddb 100644 --- a/apps/web/app/(infra)/dashboard/infra/infra-data.ts +++ b/apps/web/app/(infra)/dashboard/infra/infra-data.ts @@ -23,6 +23,7 @@ import { developerVideoPageUrls, developerVideoPlaybackUrls, } from "@/lib/developer-video-api"; +import { developerVideoCaptionTracks } from "@/lib/developer-video-captions"; import { redriveDeveloperVideoProcessing } from "@/lib/developer-video-start"; import { developerVideoRedrive } from "@/lib/developer-video-state"; import { developerVideoIsTakenDown } from "@/lib/developer-video-takedown"; @@ -202,7 +203,7 @@ export const getDeveloperAppVideo = cache( ) { row = (await findDeveloperAppVideo(appId, videoId)) ?? row; } - return toInfraVideoDetail( + const detail = toInfraVideoDetail( { ...row, legacy: row.playbackId === null && row.processing === null, @@ -211,6 +212,13 @@ export const getDeveloperAppVideo = cache( developerVideoPlaybackUrls(row), row.playbackId ? developerVideoPageUrls(row.playbackId) : null, ); + if (detail.state !== "ready" || (!detail.hlsUrl && !detail.mp4Url)) { + return detail; + } + return { + ...detail, + captions: await developerVideoCaptionTracks({ ...row, appId }), + }; }, ); diff --git a/apps/web/app/dev/_components/DeveloperVideoPage.tsx b/apps/web/app/dev/_components/DeveloperVideoPage.tsx index 79cea4c74c..87aa44f812 100644 --- a/apps/web/app/dev/_components/DeveloperVideoPage.tsx +++ b/apps/web/app/dev/_components/DeveloperVideoPage.tsx @@ -104,6 +104,8 @@ function Stage({ loop={playback.loop} controls={playback.controls} startAt={playback.startAt} + captions={player.captions ?? undefined} + showCaptions={playback.captions} /> ); } diff --git a/apps/web/components/cap-player/CapPlayer.tsx b/apps/web/components/cap-player/CapPlayer.tsx index 616fbf93a9..f7e3dce1a8 100644 --- a/apps/web/components/cap-player/CapPlayer.tsx +++ b/apps/web/components/cap-player/CapPlayer.tsx @@ -9,6 +9,15 @@ import { useRef, useState, } from "react"; +import { + activeCueText, + type CapPlayerCaption, + initialCaptions, + loadCaptionsFile, + releaseCaptionsFiles, + sameCaptionsKey, + toggledCaptions, +} from "./captions"; import styles from "./player.module.css"; import { type Attachment, @@ -19,6 +28,7 @@ import { resolutionLabel, } from "./source"; +export type { CapPlayerCaption } from "./captions"; export { resolutionLabel } from "./source"; export type CapPlayerControl = @@ -26,6 +36,7 @@ export type CapPlayerControl = | "time" | "speed" | "quality" + | "captions" | "pip" | "fullscreen"; @@ -39,6 +50,8 @@ export interface CapPlayerProps { loop?: boolean; controls?: boolean; startAt?: number; + captions?: readonly CapPlayerCaption[]; + showCaptions?: boolean; focusOnMount?: boolean; accentColor?: string; hideControls?: readonly CapPlayerControl[]; @@ -61,8 +74,14 @@ const MENU_MIN_HEIGHT = 76; const HOVER_INTENT_MS = 100; const PLAYS = '[data-action="toggle"], [data-action="surface"]'; const NO_CONTROLS: readonly CapPlayerControl[] = []; +const NO_CAPTIONS: readonly CapPlayerCaption[] = []; +const CAPTIONS_OFF = -1; + +type MenuName = "speed" | "quality" | "captions"; -type MenuName = "speed" | "quality"; +type NativeViewVideo = HTMLVideoElement & { + webkitDisplayingFullscreen?: boolean; +}; export const clock = (seconds: number) => { const whole = Math.max(0, Math.floor(seconds)); @@ -160,6 +179,16 @@ const PipIcon = () => ( ); +const CaptionsIcon = () => ( + + + + +); + const CheckIcon = () => ( @@ -255,6 +284,8 @@ export const CapPlayer = ({ loop = false, controls = true, startAt = 0, + captions: givenCaptions = NO_CAPTIONS, + showCaptions = false, focusOnMount = false, accentColor, hideControls = NO_CONTROLS, @@ -280,8 +311,40 @@ export const CapPlayer = ({ const attachmentRef = useRef(null); const menuDismissedAt = useRef(-1); const levelsRef = useRef([]); - const live = useRef({ onFirstPlay, onEnded, speeds }); - live.current = { onFirstPlay, onEnded, speeds }; + const captionTracks = useRef(new Map()); + const captionFiles = useRef(new Map>()); + // A page that refreshes its data hands over a new list with the same + // captions. Keeping the old list then leaves the captions on screen alone. + const captionsKey = sameCaptionsKey(givenCaptions); + const keptCaptions = useRef({ key: captionsKey, list: givenCaptions }); + if (keptCaptions.current.key !== captionsKey) { + keptCaptions.current = { key: captionsKey, list: givenCaptions }; + } + const captions = keptCaptions.current.list; + + const [unavailable, setUnavailable] = useState([]); + const [captionsOn, setCaptionsOn] = useState(() => + initialCaptions(captions, showCaptions), + ); + const [cue, setCue] = useState(""); + const rememberedCaptions = useRef(captionsOn); + const offered = captions.filter( + (caption) => !unavailable.includes(caption.language), + ); + const toggleCaptions = () => { + if (offered.length === 0) return false; + const next = toggledCaptions( + captionsOn, + rememberedCaptions.current, + offered, + ); + if (next !== null) rememberedCaptions.current = next; + setCaptionsOn(next); + return true; + }; + + const live = useRef({ onFirstPlay, onEnded, speeds, toggleCaptions }); + live.current = { onFirstPlay, onEnded, speeds, toggleCaptions }; const [playing, setPlaying] = useState(false); const [started, setStarted] = useState(autoPlay); @@ -348,6 +411,100 @@ export const CapPlayer = ({ if (showsVolume) videoRef.current?.dispatchEvent(new Event("volumechange")); }, [showsVolume]); + useEffect(() => { + const files = captionFiles.current; + const tracks = captionTracks.current; + return () => { + for (const element of tracks.values()) element.remove(); + tracks.clear(); + releaseCaptionsFiles(files); + }; + }, []); + + useEffect(() => { + const video = videoRef.current as NativeViewVideo | null; + const caption = captions.find((entry) => entry.language === captionsOn); + for (const [language, element] of captionTracks.current) { + if (language !== caption?.language && element.track) { + element.track.mode = "disabled"; + } + } + setCue(""); + if (captionsOn !== null && !caption) { + setCaptionsOn(null); + return; + } + if (!video || !caption) return; + + let cancelled = false; + let detach: () => void = () => undefined; + // The player draws the cues itself so they match it and clear the + // control bar. Picture in picture and the video's own full screen on + // iPhone cannot show anything the page draws, so there the browser + // shows them instead. + const nativeView = () => + document.pictureInPictureElement === video || + video.webkitDisplayingFullscreen === true; + loadCaptionsFile(captionFiles.current, caption.url).then( + (src) => { + if (cancelled) return; + // The track element is made only once its file is here. Chrome + // keeps a track that was ever shown without a source in its error + // state, even after a source is set. + const kept = captionTracks.current.get(caption.language); + const element = + kept?.getAttribute("src") === src + ? kept + : document.createElement("track"); + if (element !== kept) { + kept?.remove(); + element.kind = "captions"; + element.srclang = caption.language; + element.label = caption.label; + element.src = src; + video.append(element); + captionTracks.current.set(caption.language, element); + } + const track = element.track; + if (!track) return; + const paint = () => + setCue(nativeView() ? "" : activeCueText(track.activeCues)); + const place = () => { + track.mode = nativeView() ? "showing" : "hidden"; + paint(); + }; + const viewEvents = [ + "enterpictureinpicture", + "leavepictureinpicture", + "webkitbeginfullscreen", + "webkitendfullscreen", + ]; + // A track that finishes loading while the video is paused has + // active cues but fires no cuechange until the time moves. + track.addEventListener("cuechange", paint); + element.addEventListener("load", paint); + for (const name of viewEvents) video.addEventListener(name, place); + detach = () => { + track.removeEventListener("cuechange", paint); + element.removeEventListener("load", paint); + for (const name of viewEvents) { + video.removeEventListener(name, place); + } + }; + place(); + }, + () => { + if (cancelled) return; + setUnavailable((languages) => [...languages, caption.language]); + setCaptionsOn(null); + }, + ); + return () => { + cancelled = true; + detach(); + }; + }, [captions, captionsOn]); + useEffect(() => { if (!menu) return; const onPointerDown = (event: PointerEvent) => { @@ -679,6 +836,8 @@ export const CapPlayer = ({ stepSpeed(1); } else if (key === "<") { stepSpeed(-1); + } else if (key === "c") { + if (!live.current.toggleCaptions()) return; } else { return; } @@ -874,6 +1033,25 @@ export const CapPlayer = ({ checked: value === speed, })); + const captionItems: MenuItem[] = [ + { value: CAPTIONS_OFF, label: "Off", checked: captionsOn === null }, + ...offered.map((caption, index) => ({ + value: index, + label: caption.label, + checked: caption.language === captionsOn, + })), + ]; + + const pickCaptions = (value: number) => { + const language = offered[value]?.language ?? null; + if (language !== null) rememberedCaptions.current = language; + setCaptionsOn(language); + closeMenu("captions"); + }; + + const captionsLabel = + offered.find((caption) => caption.language === captionsOn)?.label ?? "off"; + const canPickQuality = levels.length > 1; const pickedLabel = levels.find((level) => level.index === picked)?.label; const quality = pickedLabel ?? resolution; @@ -908,6 +1086,11 @@ export const CapPlayer = ({ > + {cue ? ( +
+ {cue} +
+ ) : null}