fix(export): WebCodecs video decode and full audio mixdown

src/core/export/audio-processor.ts

@@ -1,98 +1,114 @@
-import { AudioPlayer } from "@canvas/players/audio-player";
-import { PlayerType } from "@canvas/players/player";
-import type { AudioAsset } from "@schemas";
-import { Output, AudioSampleSource, AudioSample } from "mediabunny";
+import { PlayerType, type Player } from "@canvas/players/player";
 
-export class AudioProcessor {
-	private audioTracks: { data: ArrayBuffer; start: number; duration: number; volume: number }[] = [];
-
-	async setupAudioTracks(tracks: ReadonlyArray<ReadonlyArray<unknown>>, output: Output): Promise<AudioSampleSource | null> {
-		const audioPlayers = this.findAudioPlayers(tracks);
-		if (!audioPlayers.length) return null;
-
-		this.audioTracks = [];
-		for (const player of audioPlayers) {
-			const track = await this.processAudioTrack(player);
-			if (track) this.audioTracks.push(track);
-		}
-
-		if (!this.audioTracks.length) return null;
+import { buildVolumeAutomation, type VolumeTween } from "./export-timing";
 
-		const audioSource = new AudioSampleSource({ codec: "aac", bitrate: 128000 });
-		output.addAudioTrack(audioSource);
-		return audioSource;
-	}
+const OUTPUT_SAMPLE_RATE = 48_000;
+const OUTPUT_CHANNELS = 2;
 
-	async processAudioSamples(audioSource: AudioSampleSource): Promise<void> {
-		if (!this.audioTracks?.length) return;
+// Player types whose `src` is (or, for text-to-speech, becomes) an audio-bearing file.
+const AUDIO_BEARING = new Set<string>([PlayerType.Audio, PlayerType.Video, PlayerType.TextToSpeech]);
 
-		const audioContext = new AudioContext();
-		for (const track of this.audioTracks) {
-			const audioBuffer = await audioContext.decodeAudioData(track.data.slice(0));
-			const { numberOfChannels, sampleRate, length: frameCount } = audioBuffer;
-			const framesToUse = Math.min(frameCount, Math.floor((sampleRate * track.duration) / 1000));
-			const interleavedData = new Float32Array(framesToUse * numberOfChannels);
+interface AudioBearingClip {
+	src: string;
+	start: number;
+	length: number;
+	trim: number;
+	volume: number | VolumeTween[] | undefined;
+	effect: string | undefined;
+}
 
-			for (let ch = 0; ch < numberOfChannels; ch += 1) {
-				const channelData = audioBuffer.getChannelData(ch);
-				for (let i = 0; i < framesToUse; i += 1) {
-					interleavedData[i * numberOfChannels + ch] = channelData[i] * track.volume;
+/**
+ * Mixes every audio-bearing clip into one export track
+ */
+export class AudioProcessor {
+	/** Decode and mix all audio-bearing clips into one buffer; null when there is no audio. */
+	async renderMix(tracks: ReadonlyArray<ReadonlyArray<Player>>, totalDuration: number): Promise<AudioBuffer | null> {
+		const clips = this.collectClips(tracks);
+		if (!clips.length || totalDuration <= 0) return null;
+
+		const frames = Math.max(1, Math.ceil(totalDuration * OUTPUT_SAMPLE_RATE));
+		const ctx = new OfflineAudioContext(OUTPUT_CHANNELS, frames, OUTPUT_SAMPLE_RATE);
+		const decodeCache = new Map<string, AudioBuffer | null>();
+		let scheduled = 0;
+
+		for (const clip of clips) {
+			const buffer = await this.decode(clip.src, ctx, decodeCache);
+			if (buffer) {
+				const source = ctx.createBufferSource();
+				source.buffer = buffer;
+
+				const gain = ctx.createGain();
+				this.applyVolume(gain.gain, clip);
+				source.connect(gain).connect(ctx.destination);
+
+				try {
+					// when = timeline start, offset = trim into the source, duration = clip length
+					source.start(Math.max(0, clip.start), Math.max(0, clip.trim), Math.max(0, clip.length));
+					scheduled += 1;
+				} catch (error) {
+					// e.g. trim past the source end — contribute nothing rather than fail the export.
+					console.warn("Export: skipped an audio clip that could not be scheduled:", error);
 				}
 			}
+		}
+
+		if (!scheduled) return null;
+		return ctx.startRendering();
+	}
 
-			await audioSource.add(
-				new AudioSample({
-					data: interleavedData,
-					format: "f32",
-					numberOfChannels,
-					sampleRate,
-					timestamp: track.start / 1000
-				})
-			);
+	private applyVolume(param: AudioParam, clip: AudioBearingClip): void {
+		const points = buildVolumeAutomation(clip.volume, clip.effect, clip.length);
+		const base = Math.max(0, clip.start);
+		param.setValueAtTime(points[0].value, base + points[0].time);
+		for (let i = 1; i < points.length; i += 1) {
+			param.linearRampToValueAtTime(points[i].value, base + points[i].time);
 		}
-		this.audioTracks = [];
 	}
 
-	private findAudioPlayers(tracks: ReadonlyArray<ReadonlyArray<unknown>>): AudioPlayer[] {
-		const players: AudioPlayer[] = [];
+	private async decode(src: string, ctx: BaseAudioContext, cache: Map<string, AudioBuffer | null>): Promise<AudioBuffer | null> {
+		if (cache.has(src)) return cache.get(src) ?? null;
 
+		let result: AudioBuffer | null = null;
+		try {
+			const response = await fetch(src);
+			// decodeAudioData reads the audio track of audio and video containers; throws if none.
+			if (response.ok) result = await ctx.decodeAudioData(await response.arrayBuffer());
+		} catch (error) {
+			console.warn("Export: no decodable audio for", src, error);
+		}
+		cache.set(src, result);
+		return result;
+	}
+
+	private collectClips(tracks: ReadonlyArray<ReadonlyArray<Player>>): AudioBearingClip[] {
+		const clips: AudioBearingClip[] = [];
+		const seen = new Set<Player>();
 		for (const track of tracks) {
 			for (const clip of track) {
-				if (this.isAudioPlayer(clip) && !players.includes(clip)) {
-					players.push(clip);
+				const resolved = seen.has(clip) ? null : this.asAudioClip(clip);
+				if (resolved) {
+					seen.add(clip);
+					clips.push(resolved);
 				}
 			}
 		}
-
-		return players;
-	}
-
-	private async processAudioTrack(player: AudioPlayer) {
-		try {
-			const asset = player.clipConfiguration?.asset as AudioAsset;
-			if (!asset?.src) return null;
-
-			const response = await fetch(asset.src);
-			if (!response.ok) return null;
-
-			return {
-				data: await response.arrayBuffer(),
-				start: player.getStart(),
-				duration: player.getLength(),
-				volume: player.getVolume()
-			};
-		} catch (error) {
-			console.warn("Failed to process audio track:", error);
-			return null;
-		}
+		return clips;
 	}
 
-	private isAudioPlayer(clip: unknown): clip is AudioPlayer {
-		if (!clip || typeof clip !== "object") return false;
-		const c = clip as { playerType?: string };
-		if (c.playerType === PlayerType.Audio) return true;
-		// Fallback for cases where playerType might not exist
-		const config = (clip as { clipConfiguration?: { asset?: { type?: string } } }).clipConfiguration;
-		return config?.asset?.type === "audio";
+	private asAudioClip(clip: Player): AudioBearingClip | null {
+		if (!AUDIO_BEARING.has(clip.playerType)) return null;
+		const asset = clip.clipConfiguration?.asset as
+			| { src?: unknown; trim?: number; volume?: unknown; volumeEffect?: string; effect?: string }
+			| undefined;
+		if (typeof asset?.src !== "string" || !asset.src) return null;
+
+		return {
+			src: asset.src,
+			start: clip.getStart(),
+			length: clip.getLength(),
+			trim: asset.trim ?? 0,
+			volume: asset.volume as number | VolumeTween[] | undefined,
+			effect: asset.volumeEffect ?? asset.effect
+		};
 	}
 }

src/core/export/export-coordinator.ts

@@ -2,13 +2,13 @@ import { Canvas } from "@canvas/shotstack-canvas";
 import { ExportCommand } from "@core/commands/export-command";
 import { Edit } from "@core/edit-session";
 import { sec } from "@core/timing/types";
-import { Output, Mp4OutputFormat, BufferTarget, CanvasSource } from "mediabunny";
+import { Output, Mp4OutputFormat, BufferTarget, CanvasSource, AudioBufferSource } from "mediabunny";
 import * as pixi from "pixi.js";
 
 import { AudioProcessor } from "./audio-processor";
 import { ExportProgressUI } from "./export-progress-ui";
 import { ExportError, BrowserCompatibilityError } from "./export-utils";
-import { VideoFrameProcessor, type VideoPlayerExtended } from "./video-frame-processor";
+import { VideoFrameProcessor } from "./video-frame-processor";
 
 interface ExportConfig {
 	fps: number;
@@ -74,14 +74,19 @@ export class ExportCoordinator {
 			const videoSource = new CanvasSource(canvas, { codec: "avc", bitrate: 5_000_000 });
 			output.addVideoTrack(videoSource);
 
-			this.progressUI.update(15, 100, "Audio...");
-			const audioSource = await this.audioProcessor.setupAudioTracks(this.exportCommand.getTracks(), output);
+			this.progressUI.update(15, 100, "Mixing audio...");
+			const mixedAudio = await this.audioProcessor.renderMix(this.exportCommand.getTracks(), this.edit.totalDuration);
+			let audioSource: AudioBufferSource | null = null;
+			if (mixedAudio) {
+				audioSource = new AudioBufferSource({ codec: "aac", bitrate: 192_000 });
+				output.addAudioTrack(audioSource);
+			}
 
 			await output.start();
 
-			if (audioSource) {
+			if (audioSource && mixedAudio) {
 				this.progressUI.update(20, 100, "Encoding audio...");
-				await this.audioProcessor.processAudioSamples(audioSource);
+				await audioSource.add(mixedAudio);
 			}
 
 			this.progressUI.update(25, 100, "Exporting...");
@@ -199,39 +204,12 @@ export class ExportCoordinator {
 	private restoreEditState(state: EditState): void {
 		const c = this.edit.getViewportContainer();
 		this.edit.setExportMode(false);
-
-		for (const clip of this.exportCommand.getClips()) {
-			if (this.isVideoPlayer(clip)) {
-				const videoClip = clip as VideoPlayerExtended;
-				videoClip.skipVideoUpdate = false;
-				if (videoClip.originalVideoElement && videoClip.texture) {
-					const texture = pixi.Texture.from(videoClip.originalVideoElement);
-					videoClip.texture = texture;
-					if (videoClip.sprite) videoClip.sprite.texture = texture;
-					delete videoClip.originalVideoElement;
-					delete videoClip.originalTextureSource;
-					delete videoClip.lastReplacedTimestamp;
-				} else if (videoClip.originalTextureSource && videoClip.texture) {
-					(videoClip.texture as { source: unknown }).source = videoClip.originalTextureSource;
-					delete videoClip.originalTextureSource;
-					delete videoClip.lastReplacedTimestamp;
-				}
-			}
-		}
+		this.videoProcessor.restore();
 
 		Object.assign(c.position, state.pos);
 		Object.assign(c.scale, state.scale);
 		c.visible = state.visible;
 		this.edit.seek(sec(state.time));
 		if (state.wasPlaying) this.edit.play();
 	}
-
-	private isVideoPlayer(clip: unknown): clip is VideoPlayerExtended {
-		if (!clip || typeof clip !== "object") return false;
-		const c = clip as Record<string, unknown>;
-		const hasVideoConstructor = c.constructor?.name === "VideoPlayer";
-		const texture = c["texture"] as { source?: { resource?: unknown } } | undefined;
-		const hasVideoTexture = texture?.source?.resource instanceof HTMLVideoElement;
-		return hasVideoConstructor || hasVideoTexture;
-	}
 }

src/core/export/export-timing.ts

@@ -0,0 +1,65 @@
+/** Pure timing math for export. */
+export function videoSourceTime(timestamp: number, clipStart: number, trim: number): number {
+	return timestamp - clipStart + trim;
+}
+
+/** A linear volume segment, clip-local seconds. */
+export interface VolumeTween {
+	from: number;
+	to: number;
+	start?: number;
+	length?: number;
+}
+
+/** A volume automation point, clip-local. `time` seconds from the clip's start. */
+export interface VolumePoint {
+	time: number;
+	value: number;
+}
+
+/**
+ * Resolve a clip's volume into linear automation points, mirroring the player model:
+ * a volume tween array wins; otherwise a scalar with an optional fade of min(2, length/2)s.
+ * Apply as setValueAtTime(points[0]) then linearRampToValueAtTime for the rest.
+ */
+export function buildVolumeAutomation(volume: number | VolumeTween[] | undefined, effect: string | undefined, length: number): VolumePoint[] {
+	const clamp = (t: number): number => Math.min(length, Math.max(0, t));
+
+	if (Array.isArray(volume)) {
+		const points: VolumePoint[] = [];
+		for (const tween of volume) {
+			const start = clamp(tween.start ?? 0);
+			const end = clamp(start + (tween.length ?? length - start));
+			points.push({ time: start, value: tween.from });
+			points.push({ time: end, value: tween.to });
+		}
+		return points.length ? points : [{ time: 0, value: 1 }];
+	}
+
+	const base = typeof volume === "number" ? volume : 1;
+	const fade = Math.min(2, length / 2);
+
+	if (effect === "fadeIn") {
+		return [
+			{ time: 0, value: 0 },
+			{ time: clamp(fade), value: base },
+			{ time: length, value: base }
+		];
+	}
+	if (effect === "fadeOut") {
+		return [
+			{ time: 0, value: base },
+			{ time: clamp(length - fade), value: base },
+			{ time: length, value: 0 }
+		];
+	}
+	if (effect === "fadeInFadeOut") {
+		return [
+			{ time: 0, value: 0 },
+			{ time: clamp(fade), value: base },
+			{ time: clamp(length - fade), value: base },
+			{ time: length, value: 0 }
+		];
+	}
+	return [{ time: 0, value: base }];
+}