From c0943b73a4dd86533e73f41ab191773de3bd7937 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Tue, 4 Aug 2026 15:32:51 -0700 Subject: [PATCH] feat(engine): let an FX tail decay instead of cutting it at the clip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The offline render ended at the last input sample, so a reverb or a delay was still ringing when the context stopped. Measured on a 1.5 s tone through a default reverb, the render cut at 1.524 s while the tail was still at -29.7 dB — an audible chop, and the one place the render did not match preview. The length does not have to be guessed. Every tail here follows from its own settings: a convolution is exactly as long as its impulse, and `synthesizeReverbImpulse` derives that from room size; a delay's repeats fall by `feedback` every `time`, so the count down to -60 dB is a log. Everything else settles with its input — an all-pass chain has group delay, not a tail, and a 9-second compressor release has no signal to release once the clip stops. `chainTailSeconds` sums them (the chain is serial, so a delay in front of a reverb hands each repeat to the room), reads a lane's maximum rather than the static knob where one is automated, and caps at 5 s — 5 s between repeats at 0.95 feedback is eleven minutes of decay, and the panel can dial exactly that. The mixer's per-track atrim now allows the clip plus its tail; the atrim after apad still holds every track to the composition's length, so a tail can run over what follows but never extends the video. Same fixture after: a smooth decay to -72 dB, last non-zero sample at 3.306 s against the 3.4 s the settings predict. --- packages/core/package-subpaths.json | 6 + packages/core/package.json | 10 ++ packages/core/src/audio/audioFxTail.test.ts | 103 ++++++++++++++++++ packages/core/src/audio/audioFxTail.ts | 96 ++++++++++++++++ packages/core/stubs/audio-fx-runtime-entry.ts | 22 ++-- .../engine/src/services/audioMixer.test.ts | 91 ++++++++++++++++ packages/engine/src/services/audioMixer.ts | 14 ++- .../engine/src/services/audioMixer.types.ts | 6 + 8 files changed, 337 insertions(+), 11 deletions(-) create mode 100644 packages/core/src/audio/audioFxTail.test.ts create mode 100644 packages/core/src/audio/audioFxTail.ts diff --git a/packages/core/package-subpaths.json b/packages/core/package-subpaths.json index 9c321bb685..115fc1cbc9 100644 --- a/packages/core/package-subpaths.json +++ b/packages/core/package-subpaths.json @@ -92,6 +92,12 @@ "types": "./dist/audioFx.d.ts", "environments": ["browser", "bun", "node"] }, + "./audio-fx-tail": { + "source": "./src/audio/audioFxTail.ts", + "runtime": "./dist/audio/audioFxTail.js", + "types": "./dist/audio/audioFxTail.d.ts", + "environments": ["browser", "bun", "node"] + }, "./audio-fx-runtime": { "source": "./src/generated/audio-fx-runtime-inline.ts", "runtime": "./dist/generated/audio-fx-runtime-inline.js", diff --git a/packages/core/package.json b/packages/core/package.json index cdc9a5e5b6..06ac920aa8 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -106,6 +106,12 @@ "import": "./src/audioFx.ts", "types": "./src/audioFx.ts" }, + "./audio-fx-tail": { + "bun": "./src/audio/audioFxTail.ts", + "node": "./dist/audio/audioFxTail.js", + "import": "./src/audio/audioFxTail.ts", + "types": "./src/audio/audioFxTail.ts" + }, "./audio-fx-runtime": { "bun": "./src/generated/audio-fx-runtime-inline.ts", "node": "./dist/generated/audio-fx-runtime-inline.js", @@ -386,6 +392,10 @@ "import": "./dist/audioFx.js", "types": "./dist/audioFx.d.ts" }, + "./audio-fx-tail": { + "import": "./dist/audio/audioFxTail.js", + "types": "./dist/audio/audioFxTail.d.ts" + }, "./audio-fx-runtime": { "import": "./dist/generated/audio-fx-runtime-inline.js", "types": "./dist/generated/audio-fx-runtime-inline.d.ts" diff --git a/packages/core/src/audio/audioFxTail.test.ts b/packages/core/src/audio/audioFxTail.test.ts new file mode 100644 index 0000000000..537f7f774a --- /dev/null +++ b/packages/core/src/audio/audioFxTail.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from "vitest"; +import { chainTailSeconds, MAX_FX_TAIL_SECONDS } from "./audioFxTail.js"; +import { synthesizeReverbImpulse } from "./audioFxGraph.js"; +import type { HfAudioFxChain } from "../audioFx.js"; +import type { HfAutomation } from "../audioAutomation.js"; + +const chain = (nodes: HfAudioFxChain["nodes"]): HfAudioFxChain => ({ version: 1, nodes }); + +describe("chainTailSeconds", () => { + it("is zero for a chain that settles with its input", () => { + expect( + chainTailSeconds( + chain([ + { type: "peaking", id: "n1", params: { frequency: 900, gain: -6, q: 1 } }, + { type: "compressor", id: "n2", params: { threshold: -18, ratio: 4, release: 9000 } }, + // A 9-second release still holds no tail: with no input there is no + // signal to release, so the output is silence either way. + { type: "phaser", id: "n3", params: { delay: 3, decay: 0.4, speed: 0.5 } }, + ]), + ), + ).toBe(0); + }); + + it("matches the reverb impulse it has to make room for", () => { + // The whole point: too short and the render cuts the tail the impulse + // generates. Measured against the generator rather than restating 0.6+2.6. + for (const size of [0.05, 0.4, 0.7, 1]) { + const impulse = synthesizeReverbImpulse(48000, size, 0.5); + const tail = chainTailSeconds( + chain([{ type: "reverb", id: "r", params: { size, damping: 0.5, wet: 0.35, dry: 0.7 } }]), + ); + expect(tail).toBeCloseTo(impulse.length / 48000, 4); + } + }); + + it("counts a delay's repeats down to -60 dB", () => { + // 250 ms at 0.35 feedback: 0.35^7 = 6.4e-4, the first repeat under the floor. + expect( + chainTailSeconds( + chain([{ type: "delay", id: "d", params: { time: 250, feedback: 0.35, mix: 0.5 } }]), + ), + ).toBeCloseTo(1.75, 5); + }); + + it("sums a serial chain instead of taking the longest", () => { + // The delay hands each repeat to the room, so the last repeat still gets a + // full tail — taking the max would cut the end of it. + const both = chainTailSeconds( + chain([ + { type: "delay", id: "d", params: { time: 250, feedback: 0.35, mix: 0.5 } }, + { type: "reverb", id: "r", params: { size: 0.3, damping: 0.5, wet: 0.35, dry: 0.7 } }, + ]), + ); + expect(both).toBeCloseTo(1.75 + (0.6 + 0.3 * 2.6), 5); + }); + + it("caps a tail the panel can dial but nobody wants to render", () => { + // 5 s between repeats at 0.95 feedback decays for eleven minutes. + expect( + chainTailSeconds( + chain([{ type: "delay", id: "d", params: { time: 5000, feedback: 0.95, mix: 1 } }]), + ), + ).toBe(MAX_FX_TAIL_SECONDS); + }); + + it("ignores an effect that is bypassed or mixed out", () => { + const bypassed = chain([ + { + type: "reverb", + id: "r", + enabled: false, + params: { size: 1, damping: 0.5, wet: 1, dry: 0 }, + }, + ]); + expect(chainTailSeconds(bypassed)).toBe(0); + const silent = chain([ + { type: "reverb", id: "r", params: { size: 1, damping: 0.5, wet: 0, dry: 1 } }, + ]); + expect(chainTailSeconds(silent)).toBe(0); + }); + + it("sizes the room for the loudest moment a lane reaches", () => { + // Static wet is 0 — read alone it would say "no tail" and cut the swell the + // lane brings in halfway through the clip. + const automation: HfAutomation = { + version: 1, + lanes: [ + { + target: "fx.r.wet", + points: [ + { t: 0, v: 0 }, + { t: 2, v: 0.8 }, + ], + }, + ], + }; + const withLane = chain([ + { type: "reverb", id: "r", params: { size: 0.5, damping: 0.5, wet: 0, dry: 1 } }, + ]); + expect(chainTailSeconds(withLane)).toBe(0); + expect(chainTailSeconds(withLane, automation)).toBeCloseTo(0.6 + 0.5 * 2.6, 5); + }); +}); diff --git a/packages/core/src/audio/audioFxTail.ts b/packages/core/src/audio/audioFxTail.ts new file mode 100644 index 0000000000..ff7c543ea3 --- /dev/null +++ b/packages/core/src/audio/audioFxTail.ts @@ -0,0 +1,96 @@ +/** + * How long a chain keeps ringing after its input stops. + * + * The render used to end the offline context at the last input sample, so a + * reverb or a delay was cut mid-tail — the one place the render did not match + * preview. The length is not a guess: every tail-producing effect here has a + * decay that follows from its own settings, so the render can ask for exactly + * the room it needs. + */ + +import { fxAutomationTarget, type HfAutomation } from "../audioAutomation.js"; +import { normalizeAudioFxParams, type HfAudioFxChain, type HfAudioFxNode } from "../audioFx.js"; + +/** + * Ceiling on the extension, in seconds. + * + * Delay is unbounded in principle: 5 s between repeats at 0.95 feedback decays + * for eleven minutes, and the panel can dial exactly that. A tail that outruns + * the composition costs render time and mixes into everything after it, so the + * chain gets the room it asks for up to here and is cut beyond it. + */ +export const MAX_FX_TAIL_SECONDS = 5; + +/** + * Where a tail stops counting as audible: -60 dB below the signal that fed it, + * the usual convention for a reverb time. Anything quieter is under the noise + * floor of every codec this renders to. + */ +const TAIL_FLOOR = 0.001; + +/** + * The largest value a knob reaches, over the whole clip. + * + * A lane's `curve` is an exponent, so a segment is monotone between its two + * points and cannot overshoot either — the maximum point value is the maximum + * of the lane, no sampling needed. Room size has to be sized for the loudest + * moment regardless of where in the clip it falls. + */ +function knobMax(node: HfAudioFxNode, key: string, automation?: HfAutomation): number { + // Normalised, so a knob missing from the attribute reads as its default and + // an out-of-range one is clamped the way the graph builder would clamp it. + const fallback = Number(normalizeAudioFxParams(node.type, node.params)[key] ?? 0); + if (!node.id || !automation) return Number.isFinite(fallback) ? fallback : 0; + const lane = automation.lanes.find((l) => l.target === fxAutomationTarget(node.id ?? "", key)); + if (!lane || lane.points.length === 0) return Number.isFinite(fallback) ? fallback : 0; + return lane.points.reduce((max, p) => Math.max(max, p.v), -Infinity); +} + +/** + * Repeats until a feedback loop falls under the floor, times the gap between + * them. `feedback` is capped below 1 by the registry, so this terminates. + */ +function delayTail(time: number, feedback: number): number { + const gap = Math.min(5, time / 1000); + if (gap <= 0) return 0; + const fb = Math.max(0, Math.min(0.999, feedback)); + if (fb <= 0) return gap; + return Math.ceil(Math.log(TAIL_FLOOR) / Math.log(fb)) * gap; +} + +/** One node's tail. Zero when it has none, or when it is mixed out entirely. */ +function nodeTail(node: HfAudioFxNode, automation?: HfAutomation): number { + if (node.enabled === false) return 0; + switch (node.type) { + case "reverb": + // Exactly the generated impulse's length — see synthesizeReverbImpulse, + // which is the same expression. A convolution is as long as its impulse. + return knobMax(node, "wet", automation) > 0 + ? 0.6 + Math.max(0, Math.min(1, knobMax(node, "size", automation))) * 2.6 + : 0; + case "delay": + return knobMax(node, "mix", automation) > 0 + ? delayTail(knobMax(node, "time", automation), knobMax(node, "feedback", automation)) + : 0; + case "chorus": + // A single delay line, no feedback: it rings for one delay (≤100 ms). + return knobMax(node, "mix", automation) > 0 ? knobMax(node, "delay", automation) / 1000 : 0; + default: + // Everything else settles with its input. The phaser is an all-pass chain + // with no recirculation (group delay, not a tail); the dynamics nodes have + // long releases but no signal to release — silence in, silence out; a + // biquad rings for ~Q/f, which is microseconds. + return 0; + } +} + +/** + * The whole chain's tail, in seconds. + * + * Summed, not maxed: the chain is serial, so a delay in front of a reverb hands + * each of its repeats to the room and the last one still gets a full tail. + */ +export function chainTailSeconds(chain: HfAudioFxChain, automation?: HfAutomation): number { + const total = chain.nodes.reduce((sum, node) => sum + nodeTail(node, automation), 0); + return Math.min(MAX_FX_TAIL_SECONDS, total); +} diff --git a/packages/core/stubs/audio-fx-runtime-entry.ts b/packages/core/stubs/audio-fx-runtime-entry.ts index 668171bb25..81f8e5f035 100644 --- a/packages/core/stubs/audio-fx-runtime-entry.ts +++ b/packages/core/stubs/audio-fx-runtime-entry.ts @@ -19,6 +19,7 @@ import { import { scheduleChainAutomation } from "../src/audio/audioFxAutomation.js"; import { parseAutomation, resolveAutomation } from "../src/audioAutomation.js"; import { parseAudioFxChain, type HfAudioFxChain } from "../src/audioFx.js"; +import { chainTailSeconds } from "../src/audio/audioFxTail.js"; declare global { interface Window { @@ -39,11 +40,11 @@ declare global { * Channel count is preserved: folding to mono here collapsed a stereo bed's * width for the render only, while preview kept it stereo. * - * The context is exactly as long as the input. An effect with a tail — reverb, - * delay — is still ringing at that point and is cut there, which is the one place - * the render does not match preview. Extending it would lengthen the clip in the - * mix, so how far a tail may run past a clip's end is a product decision rather - * than something to pick here. + * The context runs past the input by the chain's own tail, so a reverb or a + * delay decays out instead of being cut at the last input sample. The length + * comes from the settings (`chainTailSeconds`), capped, and the returned planes + * are correspondingly longer than what came in — the mixer decides how much of + * that it lets through past the clip's end. */ /** The clip's audio as an AudioBuffer, a plane per channel. */ @@ -80,7 +81,11 @@ async function render( const chain: HfAudioFxChain = parseAudioFxChain(chainJson); const channels = Math.max(1, planes.length); const frames = planes[0]?.length ?? 0; - const ctx = new OfflineAudioContext(channels, frames, sampleRate); + const parsedAutomation = automationJson + ? resolveAutomation(parseAutomation(automationJson), chain) + : null; + const tail = Math.ceil(chainTailSeconds(chain, parsedAutomation ?? undefined) * sampleRate); + const ctx = new OfflineAudioContext(channels, frames + tail, sampleRate); if (chainNeedsWorklets(chain)) await ensureAudioFxWorklets(ctx); @@ -92,9 +97,8 @@ async function render( // The input WAV is the clip's own audio from its first sample, so clip-local // time is offline time — the envelope needs no offset here. Same scheduler as // preview, which is what makes the two agree. - if (automationJson) { - const automation = resolveAutomation(parseAutomation(automationJson), chain); - scheduleChainAutomation(automation, chain, fx.nodes, { + if (parsedAutomation) { + scheduleChainAutomation(parsedAutomation, chain, fx.nodes, { scheduledAt: 0, elapsed: 0, rate: 1, diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts index 096227b9b7..957016d593 100644 --- a/packages/engine/src/services/audioMixer.test.ts +++ b/packages/engine/src/services/audioMixer.test.ts @@ -41,6 +41,21 @@ vi.mock("../utils/runFfmpeg.js", async (importOriginal) => { return { ...actual, runFfmpeg: runFfmpegMock }; }); +// The FX render drives a headless browser; the mix only needs to know the +// processed file exists and how long a tail the chain asked for. +const { applyAudioFxChainMock } = vi.hoisted(() => ({ + applyAudioFxChainMock: vi.fn(async (_src: string, _chain: unknown, outPath: string) => { + const { writeFileSync } = await import("node:fs"); + writeFileSync(outPath, "stub"); + return outPath; + }), +})); + +vi.mock("./audioFxRender.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, applyAudioFxChain: applyAudioFxChainMock }; +}); + vi.mock("../utils/ffprobe.js", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, extractAudioMetadata: extractAudioMetadataMock }; @@ -60,6 +75,7 @@ describe("processCompositionAudio", () => { channels: 2, audioCodec: "aac", }); + applyAudioFxChainMock.mockClear(); capturedFilterScripts.length = 0; for (const dir of tempDirs.splice(0)) { rmSync(dir, { recursive: true, force: true }); @@ -154,6 +170,81 @@ describe("processCompositionAudio", () => { expect(filter).not.toContain("weights="); }); + it("lets an FX tail run past the clip, still bounded by the composition", async () => { + // A reverb is still decaying when the clip's own audio stops. Trimming at + // the clip boundary is what cut every tail short in the render. + const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); + tempDirs.push(baseDir, workDir); + writeFileSync(join(baseDir, "bed.wav"), "stub"); + + const result = await processCompositionAudio( + [ + { + id: "bed", + src: "bed.wav", + start: 0, + end: 2, + mediaStart: 0, + layer: 0, + volume: 1, + type: "audio", + fxChain: JSON.stringify({ + version: 1, + nodes: [ + { type: "reverb", id: "r", params: { size: 0.5, damping: 0.5, wet: 0.4, dry: 0.7 } }, + ], + }), + }, + ], + baseDir, + workDir, + join(baseDir, "out.m4a"), + 8, + ); + + expect(result.success).toBe(true); + expect(applyAudioFxChainMock).toHaveBeenCalledTimes(1); + const filter = capturedFilterScripts[capturedFilterScripts.length - 1]; + // 2 s clip + the 1.9 s tail 0.6 + size * 2.6 generates. + expect(filter).toContain("atrim=0:3.9,"); + // And still cut at the composition's end, so a tail cannot extend the video. + expect(filter).toContain("apad,atrim=0:8"); + }); + + it("cuts at the clip boundary when the chain has no tail", async () => { + const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); + tempDirs.push(baseDir, workDir); + writeFileSync(join(baseDir, "bed.wav"), "stub"); + + await processCompositionAudio( + [ + { + id: "bed", + src: "bed.wav", + start: 0, + end: 2, + mediaStart: 0, + layer: 0, + volume: 1, + type: "audio", + fxChain: JSON.stringify({ + version: 1, + nodes: [{ type: "peaking", id: "n1", params: { frequency: 900, gain: -6, q: 1 } }], + }), + }, + ], + baseDir, + workDir, + join(baseDir, "out.m4a"), + 8, + ); + + const filter = capturedFilterScripts[capturedFilterScripts.length - 1]; + expect(filter).toContain("atrim=0:2,"); + }); + it("compensates amix normalization so multi-track master gain equals track count", async () => { const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts index ac97ff1eea..369562e328 100644 --- a/packages/engine/src/services/audioMixer.ts +++ b/packages/engine/src/services/audioMixer.ts @@ -32,6 +32,7 @@ import { VOLUME_TARGET, type HfAutomationLane, } from "@hyperframes/core/audio-automation"; +import { chainTailSeconds } from "@hyperframes/core/audio-fx-tail"; import { applyAudioFxChain, AudioFxRenderError } from "./audioFxRender.js"; import type { AudioVolumeKeyframe } from "./audioMixer.types.js"; @@ -616,10 +617,14 @@ async function mixAudioTracks( const filterParts: string[] = []; tracks.forEach((track, i) => { const delayMs = Math.round(track.start * 1000); - const trimDuration = track.end - track.start; + // A clip's own audio ends at `end`, but an FX tail is still decaying past + // it. Trimming at the boundary is what cut every reverb short; the final + // atrim below still holds the mix to the composition's length, so a tail + // can run over what follows but never past the end of the video. + const trimDuration = track.end - track.start + (track.tailSeconds ?? 0); const volumeFilter = buildVolumeExpression(track, ignoreAutomation); filterParts.push( - `[${i}:a]atrim=0:${trimDuration},${volumeFilter},adelay=${delayMs}|${delayMs},apad,atrim=0:${formatFilterNumber(totalDuration)}[a${i}]`, + `[${i}:a]atrim=0:${formatFilterNumber(trimDuration)},${volumeFilter},adelay=${delayMs}|${delayMs},apad,atrim=0:${formatFilterNumber(totalDuration)}[a${i}]`, ); }); @@ -896,11 +901,15 @@ export async function processCompositionAudio( ) : null; + let tailSeconds = 0; if (element.fxChain) { // The chain is serialised into the attribute, the same way colour // grading carries its config, so there is no side-car file to find, // resolve or lose. const chain = parseAudioFxChain(element.fxChain); + // The rendered WAV is longer than the input by exactly this much, so + // the mix has to be told to let it through. + tailSeconds = chainTailSeconds(chain, automation ?? undefined); audioSrcPath = await applyAudioFxChain( audioSrcPath, chain, @@ -944,6 +953,7 @@ export async function processCompositionAudio( // Gain is already in the samples when baked, so mix at unity. volume: bakedEnvelope ? 1.0 : (element.volume ?? 1.0), volumeKeyframes: bakedEnvelope ? undefined : (envelopeKeyframes ?? undefined), + ...(tailSeconds > 0 ? { tailSeconds } : {}), }); } catch (err: unknown) { // An FX failure is fatal for the whole mix. Every other failure mode diff --git a/packages/engine/src/services/audioMixer.types.ts b/packages/engine/src/services/audioMixer.types.ts index 1709ecbf95..3a0dbe626d 100644 --- a/packages/engine/src/services/audioMixer.types.ts +++ b/packages/engine/src/services/audioMixer.types.ts @@ -28,6 +28,12 @@ export interface AudioTrack { duration: number; volume: number; volumeKeyframes?: AudioVolumeKeyframe[]; + /** + * Seconds of FX tail past `end` that the mix should let through — a reverb or + * delay still decaying when the clip's own audio stops. Absent means cut at + * the clip boundary, which is what every track without an FX chain wants. + */ + tailSeconds?: number; } export type AudioFailureStage =