diff --git a/src/core/export/audio-processor.ts b/src/core/export/audio-processor.ts index d438f778..5da4625b 100644 --- a/src/core/export/audio-processor.ts +++ b/src/core/export/audio-processor.ts @@ -1,98 +1,114 @@ -import { AudioPlayer } from "@canvas/players/audio-player"; -import { PlayerType } from "@canvas/players/player"; -import type { AudioAsset } from "@schemas"; -import { Output, AudioSampleSource, AudioSample } from "mediabunny"; +import { PlayerType, type Player } from "@canvas/players/player"; -export class AudioProcessor { - private audioTracks: { data: ArrayBuffer; start: number; duration: number; volume: number }[] = []; - - async setupAudioTracks(tracks: ReadonlyArray>, output: Output): Promise { - const audioPlayers = this.findAudioPlayers(tracks); - if (!audioPlayers.length) return null; - - this.audioTracks = []; - for (const player of audioPlayers) { - const track = await this.processAudioTrack(player); - if (track) this.audioTracks.push(track); - } - - if (!this.audioTracks.length) return null; +import { buildVolumeAutomation, type VolumeTween } from "./export-timing"; - const audioSource = new AudioSampleSource({ codec: "aac", bitrate: 128000 }); - output.addAudioTrack(audioSource); - return audioSource; - } +const OUTPUT_SAMPLE_RATE = 48_000; +const OUTPUT_CHANNELS = 2; - async processAudioSamples(audioSource: AudioSampleSource): Promise { - if (!this.audioTracks?.length) return; +// Player types whose `src` is (or, for text-to-speech, becomes) an audio-bearing file. +const AUDIO_BEARING = new Set([PlayerType.Audio, PlayerType.Video, PlayerType.TextToSpeech]); - const audioContext = new AudioContext(); - for (const track of this.audioTracks) { - const audioBuffer = await audioContext.decodeAudioData(track.data.slice(0)); - const { numberOfChannels, sampleRate, length: frameCount } = audioBuffer; - const framesToUse = Math.min(frameCount, Math.floor((sampleRate * track.duration) / 1000)); - const interleavedData = new Float32Array(framesToUse * numberOfChannels); +interface AudioBearingClip { + src: string; + start: number; + length: number; + trim: number; + volume: number | VolumeTween[] | undefined; + effect: string | undefined; +} - for (let ch = 0; ch < numberOfChannels; ch += 1) { - const channelData = audioBuffer.getChannelData(ch); - for (let i = 0; i < framesToUse; i += 1) { - interleavedData[i * numberOfChannels + ch] = channelData[i] * track.volume; +/** + * Mixes every audio-bearing clip into one export track + */ +export class AudioProcessor { + /** Decode and mix all audio-bearing clips into one buffer; null when there is no audio. */ + async renderMix(tracks: ReadonlyArray>, totalDuration: number): Promise { + const clips = this.collectClips(tracks); + if (!clips.length || totalDuration <= 0) return null; + + const frames = Math.max(1, Math.ceil(totalDuration * OUTPUT_SAMPLE_RATE)); + const ctx = new OfflineAudioContext(OUTPUT_CHANNELS, frames, OUTPUT_SAMPLE_RATE); + const decodeCache = new Map(); + let scheduled = 0; + + for (const clip of clips) { + const buffer = await this.decode(clip.src, ctx, decodeCache); + if (buffer) { + const source = ctx.createBufferSource(); + source.buffer = buffer; + + const gain = ctx.createGain(); + this.applyVolume(gain.gain, clip); + source.connect(gain).connect(ctx.destination); + + try { + // when = timeline start, offset = trim into the source, duration = clip length + source.start(Math.max(0, clip.start), Math.max(0, clip.trim), Math.max(0, clip.length)); + scheduled += 1; + } catch (error) { + // e.g. trim past the source end — contribute nothing rather than fail the export. + console.warn("Export: skipped an audio clip that could not be scheduled:", error); } } + } + + if (!scheduled) return null; + return ctx.startRendering(); + } - await audioSource.add( - new AudioSample({ - data: interleavedData, - format: "f32", - numberOfChannels, - sampleRate, - timestamp: track.start / 1000 - }) - ); + private applyVolume(param: AudioParam, clip: AudioBearingClip): void { + const points = buildVolumeAutomation(clip.volume, clip.effect, clip.length); + const base = Math.max(0, clip.start); + param.setValueAtTime(points[0].value, base + points[0].time); + for (let i = 1; i < points.length; i += 1) { + param.linearRampToValueAtTime(points[i].value, base + points[i].time); } - this.audioTracks = []; } - private findAudioPlayers(tracks: ReadonlyArray>): AudioPlayer[] { - const players: AudioPlayer[] = []; + private async decode(src: string, ctx: BaseAudioContext, cache: Map): Promise { + if (cache.has(src)) return cache.get(src) ?? null; + let result: AudioBuffer | null = null; + try { + const response = await fetch(src); + // decodeAudioData reads the audio track of audio and video containers; throws if none. + if (response.ok) result = await ctx.decodeAudioData(await response.arrayBuffer()); + } catch (error) { + console.warn("Export: no decodable audio for", src, error); + } + cache.set(src, result); + return result; + } + + private collectClips(tracks: ReadonlyArray>): AudioBearingClip[] { + const clips: AudioBearingClip[] = []; + const seen = new Set(); for (const track of tracks) { for (const clip of track) { - if (this.isAudioPlayer(clip) && !players.includes(clip)) { - players.push(clip); + const resolved = seen.has(clip) ? null : this.asAudioClip(clip); + if (resolved) { + seen.add(clip); + clips.push(resolved); } } } - - return players; - } - - private async processAudioTrack(player: AudioPlayer) { - try { - const asset = player.clipConfiguration?.asset as AudioAsset; - if (!asset?.src) return null; - - const response = await fetch(asset.src); - if (!response.ok) return null; - - return { - data: await response.arrayBuffer(), - start: player.getStart(), - duration: player.getLength(), - volume: player.getVolume() - }; - } catch (error) { - console.warn("Failed to process audio track:", error); - return null; - } + return clips; } - private isAudioPlayer(clip: unknown): clip is AudioPlayer { - if (!clip || typeof clip !== "object") return false; - const c = clip as { playerType?: string }; - if (c.playerType === PlayerType.Audio) return true; - // Fallback for cases where playerType might not exist - const config = (clip as { clipConfiguration?: { asset?: { type?: string } } }).clipConfiguration; - return config?.asset?.type === "audio"; + private asAudioClip(clip: Player): AudioBearingClip | null { + if (!AUDIO_BEARING.has(clip.playerType)) return null; + const asset = clip.clipConfiguration?.asset as + | { src?: unknown; trim?: number; volume?: unknown; volumeEffect?: string; effect?: string } + | undefined; + if (typeof asset?.src !== "string" || !asset.src) return null; + + return { + src: asset.src, + start: clip.getStart(), + length: clip.getLength(), + trim: asset.trim ?? 0, + volume: asset.volume as number | VolumeTween[] | undefined, + effect: asset.volumeEffect ?? asset.effect + }; } } diff --git a/src/core/export/export-coordinator.ts b/src/core/export/export-coordinator.ts index 6de36ffb..b11856ad 100644 --- a/src/core/export/export-coordinator.ts +++ b/src/core/export/export-coordinator.ts @@ -2,13 +2,13 @@ import { Canvas } from "@canvas/shotstack-canvas"; import { ExportCommand } from "@core/commands/export-command"; import { Edit } from "@core/edit-session"; import { sec } from "@core/timing/types"; -import { Output, Mp4OutputFormat, BufferTarget, CanvasSource } from "mediabunny"; +import { Output, Mp4OutputFormat, BufferTarget, CanvasSource, AudioBufferSource } from "mediabunny"; import * as pixi from "pixi.js"; import { AudioProcessor } from "./audio-processor"; import { ExportProgressUI } from "./export-progress-ui"; import { ExportError, BrowserCompatibilityError } from "./export-utils"; -import { VideoFrameProcessor, type VideoPlayerExtended } from "./video-frame-processor"; +import { VideoFrameProcessor } from "./video-frame-processor"; interface ExportConfig { fps: number; @@ -74,14 +74,19 @@ export class ExportCoordinator { const videoSource = new CanvasSource(canvas, { codec: "avc", bitrate: 5_000_000 }); output.addVideoTrack(videoSource); - this.progressUI.update(15, 100, "Audio..."); - const audioSource = await this.audioProcessor.setupAudioTracks(this.exportCommand.getTracks(), output); + this.progressUI.update(15, 100, "Mixing audio..."); + const mixedAudio = await this.audioProcessor.renderMix(this.exportCommand.getTracks(), this.edit.totalDuration); + let audioSource: AudioBufferSource | null = null; + if (mixedAudio) { + audioSource = new AudioBufferSource({ codec: "aac", bitrate: 192_000 }); + output.addAudioTrack(audioSource); + } await output.start(); - if (audioSource) { + if (audioSource && mixedAudio) { this.progressUI.update(20, 100, "Encoding audio..."); - await this.audioProcessor.processAudioSamples(audioSource); + await audioSource.add(mixedAudio); } this.progressUI.update(25, 100, "Exporting..."); @@ -199,25 +204,7 @@ export class ExportCoordinator { private restoreEditState(state: EditState): void { const c = this.edit.getViewportContainer(); this.edit.setExportMode(false); - - for (const clip of this.exportCommand.getClips()) { - if (this.isVideoPlayer(clip)) { - const videoClip = clip as VideoPlayerExtended; - videoClip.skipVideoUpdate = false; - if (videoClip.originalVideoElement && videoClip.texture) { - const texture = pixi.Texture.from(videoClip.originalVideoElement); - videoClip.texture = texture; - if (videoClip.sprite) videoClip.sprite.texture = texture; - delete videoClip.originalVideoElement; - delete videoClip.originalTextureSource; - delete videoClip.lastReplacedTimestamp; - } else if (videoClip.originalTextureSource && videoClip.texture) { - (videoClip.texture as { source: unknown }).source = videoClip.originalTextureSource; - delete videoClip.originalTextureSource; - delete videoClip.lastReplacedTimestamp; - } - } - } + this.videoProcessor.restore(); Object.assign(c.position, state.pos); Object.assign(c.scale, state.scale); @@ -225,13 +212,4 @@ export class ExportCoordinator { this.edit.seek(sec(state.time)); if (state.wasPlaying) this.edit.play(); } - - private isVideoPlayer(clip: unknown): clip is VideoPlayerExtended { - if (!clip || typeof clip !== "object") return false; - const c = clip as Record; - const hasVideoConstructor = c.constructor?.name === "VideoPlayer"; - const texture = c["texture"] as { source?: { resource?: unknown } } | undefined; - const hasVideoTexture = texture?.source?.resource instanceof HTMLVideoElement; - return hasVideoConstructor || hasVideoTexture; - } } diff --git a/src/core/export/export-timing.ts b/src/core/export/export-timing.ts new file mode 100644 index 00000000..ea21b05f --- /dev/null +++ b/src/core/export/export-timing.ts @@ -0,0 +1,65 @@ +/** Pure timing math for export. */ +export function videoSourceTime(timestamp: number, clipStart: number, trim: number): number { + return timestamp - clipStart + trim; +} + +/** A linear volume segment, clip-local seconds. */ +export interface VolumeTween { + from: number; + to: number; + start?: number; + length?: number; +} + +/** A volume automation point, clip-local. `time` seconds from the clip's start. */ +export interface VolumePoint { + time: number; + value: number; +} + +/** + * Resolve a clip's volume into linear automation points, mirroring the player model: + * a volume tween array wins; otherwise a scalar with an optional fade of min(2, length/2)s. + * Apply as setValueAtTime(points[0]) then linearRampToValueAtTime for the rest. + */ +export function buildVolumeAutomation(volume: number | VolumeTween[] | undefined, effect: string | undefined, length: number): VolumePoint[] { + const clamp = (t: number): number => Math.min(length, Math.max(0, t)); + + if (Array.isArray(volume)) { + const points: VolumePoint[] = []; + for (const tween of volume) { + const start = clamp(tween.start ?? 0); + const end = clamp(start + (tween.length ?? length - start)); + points.push({ time: start, value: tween.from }); + points.push({ time: end, value: tween.to }); + } + return points.length ? points : [{ time: 0, value: 1 }]; + } + + const base = typeof volume === "number" ? volume : 1; + const fade = Math.min(2, length / 2); + + if (effect === "fadeIn") { + return [ + { time: 0, value: 0 }, + { time: clamp(fade), value: base }, + { time: length, value: base } + ]; + } + if (effect === "fadeOut") { + return [ + { time: 0, value: base }, + { time: clamp(length - fade), value: base }, + { time: length, value: 0 } + ]; + } + if (effect === "fadeInFadeOut") { + return [ + { time: 0, value: 0 }, + { time: clamp(fade), value: base }, + { time: clamp(length - fade), value: base }, + { time: length, value: 0 } + ]; + } + return [{ time: 0, value: base }]; +} diff --git a/src/core/export/video-frame-processor.ts b/src/core/export/video-frame-processor.ts index 1ad8aef7..8046a0c8 100644 --- a/src/core/export/video-frame-processor.ts +++ b/src/core/export/video-frame-processor.ts @@ -1,14 +1,13 @@ +import { Input, UrlSource, MP4, WEBM, CanvasSink, type InputVideoTrack } from "mediabunny"; import * as pixi from "pixi.js"; -import { SimpleLRUCache } from "./export-utils"; +import { videoSourceTime } from "./export-timing"; +import { BrowserCompatibilityError } from "./export-utils"; export interface VideoPlayerExtended { - texture?: { source?: { resource?: HTMLVideoElement } }; + texture?: pixi.Texture & { source?: { resource?: unknown; update?: () => void } }; sprite?: { texture?: pixi.Texture }; clipConfiguration?: { asset?: { src?: string; trim?: number; type?: string } }; - originalTextureSource?: unknown; - originalVideoElement?: HTMLVideoElement; - lastReplacedTimestamp?: number; skipVideoUpdate?: boolean; getStart?(): number; getEnd?(): number; @@ -18,177 +17,124 @@ export interface VideoPlayerExtended { export function isVideoPlayer(player: unknown): player is VideoPlayerExtended { if (!player || typeof player !== "object") return false; const p = player as Record; - const hasVideoConstructor = p.constructor?.name === "VideoPlayer"; + const name = p.constructor?.name; + if (name === "RichTextPlayer" || name === "RichCaptionPlayer") return false; const texture = p["texture"] as { source?: { resource?: unknown } } | undefined; - const hasVideoTexture = texture?.source?.resource instanceof HTMLVideoElement; - - const isRichText = p.constructor?.name === "RichTextPlayer"; - const isRichCaption = p.constructor?.name === "RichCaptionPlayer"; - if (isRichText || isRichCaption) return false; - - return hasVideoConstructor || hasVideoTexture; -} - -export interface RichTextPlayerExtended { - clipConfiguration?: { asset?: { type?: string } }; - constructor?: { name?: string }; -} - -export function isRichTextPlayer(player: unknown): player is RichTextPlayerExtended { - if (!player || typeof player !== "object") return false; - const p = player as Record; - const hasRichTextConstructor = p.constructor?.name === "RichTextPlayer"; - const config = p["clipConfiguration"] as Record | undefined; - const asset = config?.["asset"] as Record | undefined; - const hasRichTextAsset = asset?.["type"] === "rich-text"; - return hasRichTextConstructor || hasRichTextAsset; -} - -export interface RichCaptionPlayerExtended { - clipConfiguration?: { asset?: { type?: string } }; - constructor?: { name?: string }; + return name === "VideoPlayer" || texture?.source?.resource instanceof HTMLVideoElement; } -export function isRichCaptionPlayer(player: unknown): player is RichCaptionPlayerExtended { - if (!player || typeof player !== "object") return false; - const p = player as Record; - const hasRichCaptionConstructor = p.constructor?.name === "RichCaptionPlayer"; - const config = p["clipConfiguration"] as Record | undefined; - const asset = config?.["asset"] as Record | undefined; - const hasRichCaptionAsset = asset?.["type"] === "rich-caption"; - return hasRichCaptionConstructor || hasRichCaptionAsset; +interface ClipDecoder { + input: Input; + track: InputVideoTrack; + sink: CanvasSink; + trim: number; + start: number; + canvas?: HTMLCanvasElement; + ctx?: CanvasRenderingContext2D; + texture?: pixi.Texture; + originalTexture?: pixi.Texture; } +/** + * Supplies each video clip's source frame during export by decoding with WebCodecs + */ export class VideoFrameProcessor { - private frameCache = new SimpleLRUCache(10); - private textureCache = new SimpleLRUCache(5); - private videoElements = new Map(); - private extractionCanvas: HTMLCanvasElement | null = null; - private extractionContext: CanvasRenderingContext2D | null = null; + private decoders = new Map(); + /** Open a decoder per video clip; throws if the browser cannot decode a source. */ async initialize(clips: ReadonlyArray): Promise { - for (const clip of clips) { - if (isVideoPlayer(clip)) { - const videoClip = clip as VideoPlayerExtended; - const videoElement = videoClip.texture?.source?.resource; - if (videoElement) { - this.videoElements.set(this.getVideoKey(videoClip), { element: videoElement, player: videoClip }); + const players = clips.filter(isVideoPlayer) as VideoPlayerExtended[]; + + await Promise.all( + players.map(async player => { + const src = this.getVideoKey(player); + if (src) { + const input = new Input({ source: new UrlSource(src), formats: [MP4, WEBM] }); + const track = await input.getPrimaryVideoTrack(); + if (track) { + if (!(await track.canDecode())) { + throw new BrowserCompatibilityError(`Cannot decode video source: ${src}`, ["VideoDecoder"]); + } + this.decoders.set(player, { + input, + track, + sink: new CanvasSink(track, { poolSize: 2 }), + trim: player.clipConfiguration?.asset?.trim ?? 0, + start: player.getStart?.() ?? 0, + originalTexture: player.texture + }); + } } - } - } - this.extractionCanvas = document.createElement("canvas"); - this.extractionCanvas.width = 3840; // 4K width - this.extractionCanvas.height = 2160; // 4K height - this.extractionContext = this.extractionCanvas.getContext("2d", { - willReadFrequently: true, - alpha: true - }); - } - - async extractFrame(videoKey: string, timestamp: number): Promise { - const cacheKey = `${videoKey}-${timestamp}`; - const cached = this.frameCache.get(cacheKey); - if (cached) return cached; - - const videoInfo = this.videoElements.get(videoKey); - if (!videoInfo || !this.extractionContext || !this.extractionCanvas) return null; - - try { - const { element: video, player } = videoInfo; - const videoTime = (timestamp - (player.getStart?.() || 0)) / 1000 + (player.clipConfiguration?.asset?.trim || 0); - await this.seekToTime(video, videoTime); - - const width = video.videoWidth || video.width || 1920; - const height = video.videoHeight || video.height || 1080; - - this.extractionContext.clearRect(0, 0, width, height); - this.extractionContext.drawImage(video, 0, 0, width, height); - const imageData = this.extractionContext.getImageData(0, 0, width, height); - this.frameCache.set(cacheKey, imageData); - return imageData; - } catch { - return null; - } + }) + ); } + /** Point a clip's texture at the decoded source frame for the given timeline time. */ async replaceVideoTexture(player: VideoPlayerExtended, timestamp: number): Promise { - const frame = await this.extractFrame(this.getVideoKey(player), timestamp); - if (!frame) return; + const decoder = this.decoders.get(player); + if (!decoder) return; - const textureKey = `${this.getVideoKey(player)}-${timestamp}`; - let texture = this.textureCache.get(textureKey); + const sourceTime = Math.max(0, videoSourceTime(timestamp, decoder.start, decoder.trim)); + const wrapped = await decoder.sink.getCanvas(sourceTime); + if (!wrapped) return; - if (!texture) { + if (!decoder.texture) { const canvas = document.createElement("canvas"); - canvas.width = frame.width; - canvas.height = frame.height; - const ctx = canvas.getContext("2d"); - if (ctx) { - ctx.putImageData(frame, 0, 0); - texture = pixi.Texture.from(canvas); - this.textureCache.set(textureKey, texture); - } + canvas.width = wrapped.canvas.width; + canvas.height = wrapped.canvas.height; + decoder.canvas = canvas; + decoder.ctx = canvas.getContext("2d", { alpha: true }) ?? undefined; + decoder.texture = pixi.Texture.from(canvas); } - if (texture && player.texture) { - if (!player.originalTextureSource) { - // eslint-disable-next-line no-param-reassign - player.originalTextureSource = player.texture.source; - if (player.texture.source?.resource instanceof HTMLVideoElement) { - // eslint-disable-next-line no-param-reassign - player.originalVideoElement = player.texture.source.resource; - } - } - // eslint-disable-next-line no-param-reassign - player.texture = texture; - // eslint-disable-next-line no-param-reassign - if (player.sprite?.texture) player.sprite.texture = texture; + decoder.ctx?.drawImage(wrapped.canvas, 0, 0, decoder.canvas!.width, decoder.canvas!.height); + decoder.texture.source.update(); + + // Export drives the texture directly; restored by restore(). + // eslint-disable-next-line no-param-reassign + player.texture = decoder.texture as VideoPlayerExtended["texture"]; + // eslint-disable-next-line no-param-reassign + if (player.sprite) player.sprite.texture = decoder.texture; + } + + /** Pause the live