diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 7dc7173e36..285b943d6b 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -2743,6 +2743,11 @@ export function initSandboxRuntimeModular(): void { }; window.__hf.leasePausedMedia = leasePausedMedia; window.__hf.releasePausedMedia = releasePausedMedia; + window.__hf.audioMeter = { + start: () => webAudio.startMetering(), + stop: () => webAudio.stopMetering(), + read: () => webAudio.readLevels(), + }; // Same predicate the media cache uses, so the paused side sees exactly the // media the transport drives. Reads attributes only; no cache rebuild. diff --git a/packages/core/src/runtime/levelTap.ts b/packages/core/src/runtime/levelTap.ts new file mode 100644 index 0000000000..0288baea08 --- /dev/null +++ b/packages/core/src/runtime/levelTap.ts @@ -0,0 +1,48 @@ +export interface StereoLevel { + l: number; + r: number; +} + +export interface LevelTap { + /** Peak of the last analyser window per channel, linear 0..1+. */ + read(): StereoLevel; + dispose(): void; +} + +const WINDOW = 2048; + +function peak(analyser: AnalyserNode, buf: Float32Array): number { + analyser.getFloatTimeDomainData(buf); + let max = 0; + for (let i = 0; i < buf.length; i++) max = Math.max(max, Math.abs(buf[i] ?? 0)); + return max; +} + +/** A side branch off `source`; the explicit-stereo gain up-mixes mono to both channels. */ +export function createLevelTap(ctx: BaseAudioContext, source: AudioNode): LevelTap { + const mix = ctx.createGain(); + mix.channelCount = 2; + mix.channelCountMode = "explicit"; + const splitter = ctx.createChannelSplitter(2); + const left = ctx.createAnalyser(); + const right = ctx.createAnalyser(); + left.fftSize = right.fftSize = WINDOW; + source.connect(mix); + mix.connect(splitter); + splitter.connect(left, 0); + splitter.connect(right, 1); + const bufL = new Float32Array(new ArrayBuffer(WINDOW * 4)); + const bufR = new Float32Array(new ArrayBuffer(WINDOW * 4)); + return { + read: () => ({ l: peak(left, bufL), r: peak(right, bufR) }), + dispose: () => { + try { + source.disconnect(mix); + mix.disconnect(); + splitter.disconnect(); + } catch { + // Source already torn down. + } + }, + }; +} diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.ts index db32530e76..604ae57d23 100644 --- a/packages/core/src/runtime/webAudioTransport.test.ts +++ b/packages/core/src/runtime/webAudioTransport.test.ts @@ -31,6 +31,10 @@ function createMockAudioContext(currentTime = 100) { gain: { value: 1 }, connect: vi.fn(), }; + const monitorGain = { + gain: { value: 1 }, + connect: vi.fn(), + }; const ctx = { currentTime, state: "running", @@ -41,7 +45,7 @@ function createMockAudioContext(currentTime = 100) { destination: {}, close: vi.fn(), }; - return { ctx, sourceNode, mediaElementSourceNode, gainNode, masterGain, startFn }; + return { ctx, sourceNode, mediaElementSourceNode, gainNode, masterGain, monitorGain, startFn }; } function setupTransport(currentTime = 100) { @@ -49,6 +53,7 @@ function setupTransport(currentTime = 100) { const mock = createMockAudioContext(currentTime); (transport as unknown as { _ctx: unknown })._ctx = mock.ctx; (transport as unknown as { _masterGain: unknown })._masterGain = mock.masterGain; + (transport as unknown as { _monitorGain: unknown })._monitorGain = mock.monitorGain; const gen = transport.startGeneration(); return { transport, mock, gen }; } @@ -87,12 +92,12 @@ describe("WebAudioTransport author gain vs user volume", () => { it("keeps the user's master volume spec-clamped — it is a fader, not a gain", () => { const transport = new WebAudioTransport(); - const master = { gain: { value: 1 }, connect: vi.fn() }; - (transport as unknown as { _masterGain: unknown })._masterGain = master; + const monitor = { gain: { value: 1 }, connect: vi.fn() }; + (transport as unknown as { _monitorGain: unknown })._monitorGain = monitor; transport.setVolume(99); - expect(master.gain.value).toBe(1); + expect(monitor.gain.value).toBe(1); }); }); @@ -341,11 +346,13 @@ describe("WebAudioTransport", () => { const { transport, mock } = setupTransport(); transport.setVolume(0.4); transport.setMuted(true); - expect(mock.masterGain.gain.value).toBe(0); + expect(mock.monitorGain.gain.value).toBe(0); + expect(mock.masterGain.gain.value).toBe(1); transport.setMuted(false); - expect(mock.masterGain.gain.value).toBe(0.4); + expect(mock.monitorGain.gain.value).toBe(0.4); + expect(mock.masterGain.gain.value).toBe(1); }); it("applies author and user volume once in separate gain layers", async () => { @@ -354,8 +361,11 @@ describe("WebAudioTransport", () => { transport.setVolume(0.5); expect(mock.gainNode.gain.value).toBe(0.8); - expect(mock.masterGain.gain.value).toBe(0.5); - expect(mock.gainNode.gain.value * mock.masterGain.gain.value).toBeCloseTo(0.4); + expect(mock.masterGain.gain.value).toBe(1); + expect(mock.monitorGain.gain.value).toBe(0.5); + expect( + mock.gainNode.gain.value * mock.masterGain.gain.value * mock.monitorGain.gain.value, + ).toBeCloseTo(0.4); }); describe("ownsElement (per-element mute gate)", () => { @@ -738,6 +748,7 @@ describe("WebAudioTransport", () => { getFloatTimeDomainData: ReturnType; }[] = []; const masterGain = { gain: { value: 1 }, connect: vi.fn(), disconnect: vi.fn() }; + const monitorGain = { gain: { value: 1 }, connect: vi.fn(), disconnect: vi.fn() }; const mediaElementSource = { connect: vi.fn(), disconnect: vi.fn() }; const ctx = { currentTime, @@ -784,6 +795,7 @@ describe("WebAudioTransport", () => { analysers.push(node); return node; }), + createChannelSplitter: vi.fn(() => ({ connect: vi.fn(), disconnect: vi.fn() })), // The media-element route needs this as much as the decoded one: without // it `scheduleMediaElementPlayback` throws and its catch returns null, // which reads as "the member did not play" rather than a missing stub. @@ -791,7 +803,7 @@ describe("WebAudioTransport", () => { destination: {}, close: vi.fn(), }; - return { ctx, gainNodes, analysers, masterGain, mediaElementSource }; + return { ctx, gainNodes, analysers, masterGain, monitorGain, mediaElementSource }; } function setupGroupTransport(currentTime = 100) { @@ -799,6 +811,7 @@ describe("WebAudioTransport", () => { const mock = createGroupMockAudioContext(currentTime); (transport as unknown as { _ctx: unknown })._ctx = mock.ctx; (transport as unknown as { _masterGain: unknown })._masterGain = mock.masterGain; + (transport as unknown as { _monitorGain: unknown })._monitorGain = mock.monitorGain; const gen = transport.startGeneration(); return { transport, mock, gen }; } @@ -1085,6 +1098,113 @@ describe("WebAudioTransport", () => { expect(() => transport.setGroupMuted("never-played", true)).not.toThrow(); }); }); + + describe("level metering", () => { + const addGroup = (id: string) => { + document.body.insertAdjacentHTML( + "beforeend", + ``, + ); + }; + const setLevel = (analyser: { getFloatTimeDomainData: unknown }, peak: number) => { + (analyser.getFloatTimeDomainData as ReturnType).mockImplementation( + (buf: Float32Array) => buf.fill(0).fill(-peak, 3, 4), + ); + }; + + it("creates nothing until startMetering", async () => { + addGroup("vo"); + const { transport, mock, gen } = setupGroupTransport(); + await scheduleGrouped(transport, gen, "a", "vo"); + expect(mock.ctx.createAnalyser).not.toHaveBeenCalled(); + expect(transport.readLevels()).toEqual({ master: { l: 0, r: 0 }, groups: {} }); + }); + + it("attaches the master tap once init() lands, even when startMetering ran while the context was still warming up", async () => { + const mock = createGroupMockAudioContext(); + class MockAudioContext { + constructor() { + return mock.ctx as unknown as MockAudioContext; + } + } + vi.stubGlobal("AudioContext", MockAudioContext); + const transport = new WebAudioTransport(); + + transport.startMetering(); + expect(mock.ctx.createAnalyser).not.toHaveBeenCalled(); + expect(transport.readLevels()).toEqual({ master: { l: 0, r: 0 }, groups: {} }); + + const ok = await transport.init(); + + expect(ok).toBe(true); + expect(mock.ctx.createAnalyser).toHaveBeenCalled(); + expect(transport.readLevels()).toEqual({ master: { l: 0, r: 0 }, groups: {} }); + vi.unstubAllGlobals(); + }); + + it("taps master and each group as side branches, once however often it starts", async () => { + addGroup("vo"); + const { transport, mock, gen } = setupGroupTransport(); + await scheduleGrouped(transport, gen, "a", "vo"); + const groupOutput = mock.gainNodes[2]!; + const connectsBefore = groupOutput.connect.mock.calls.length; + + transport.startMetering(); + transport.startMetering(); + + expect(mock.analysers).toHaveLength(4); + expect(groupOutput.connect).toHaveBeenCalledTimes(connectsBefore + 1); + expect(groupOutput.connect).toHaveBeenCalledWith(mock.masterGain); + expect(mock.masterGain.connect).toHaveBeenCalledTimes(1); + expect(mock.monitorGain.connect).not.toHaveBeenCalled(); + }); + + it("keeps program taps off the monitor fader", async () => { + addGroup("vo"); + const { transport, mock, gen } = setupGroupTransport(); + await scheduleGrouped(transport, gen, "a", "vo"); + transport.startMetering(); + transport.setVolume(0.5); + transport.setMuted(true); + const groupOutput = mock.gainNodes.find((n) => + n.connect.mock.calls.some((call) => call[0] === mock.masterGain), + ); + expect(mock.masterGain.gain.value).toBe(1); + expect(groupOutput?.gain.value).toBe(1); + expect(mock.monitorGain.gain.value).toBe(0); + }); + + it("reads the peak per channel and stops cleanly", async () => { + addGroup("vo"); + const { transport, mock, gen } = setupGroupTransport(); + await scheduleGrouped(transport, gen, "a", "vo"); + transport.startMetering(); + const [masterL, , groupL, groupR] = mock.analysers; + setLevel(masterL!, 0.5); + setLevel(groupL!, 0.25); + setLevel(groupR!, 0.125); + + const levels = transport.readLevels(); + expect(levels.master).toEqual({ l: 0.5, r: 0 }); + expect(levels.groups.vo).toEqual({ l: 0.25, r: 0.125 }); + + transport.stopMetering(); + expect(transport.readLevels()).toEqual({ master: { l: 0, r: 0 }, groups: {} }); + expect(mock.gainNodes[2]!.disconnect).toHaveBeenCalled(); + }); + + it("a group built after start is metered, and one removed from the document leaves the read", async () => { + const { transport, mock, gen } = setupGroupTransport(); + transport.startMetering(); + addGroup("late"); + await scheduleGrouped(transport, gen, "a", "late"); + expect(Object.keys(transport.readLevels().groups)).toEqual(["late"]); + + document.querySelector("hf-audio-group")!.remove(); + expect(transport.readLevels().groups).toEqual({}); + expect(mock.analysers).toHaveLength(4); + }); + }); }); describe("decodeAudioElement retry policy (late-asset self-heal)", () => { diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.ts index 2761c98541..19048e8748 100644 --- a/packages/core/src/runtime/webAudioTransport.ts +++ b/packages/core/src/runtime/webAudioTransport.ts @@ -8,6 +8,7 @@ import { import { VOLUME_RANGE } from "../audioAutomation.js"; import { audioGroupOf, readAudioGroupVolume, resolveGroupElement } from "../audioGroups.js"; import { swallow } from "./diagnostics"; +import { createLevelTap, type LevelTap, type StereoLevel } from "./levelTap.js"; import { clampAudioGain } from "../audioGain.js"; import { getDebugSurface } from "./globals.js"; import { readElementPlaybackRate } from "./media.js"; @@ -140,6 +141,8 @@ export class WebAudioTransport { private _mediaElementSources = new WeakMap(); private _activeSources: ScheduledSource[] = []; private _masterGain: GainNode | null = null; + /** Preview volume and mute. Downstream of `_masterGain`; meters tap the program. */ + private _monitorGain: GainNode | null = null; private _masterVolume = 1; private _masterMuted = false; // One shared bus per group id, lazily built the first time a member of that @@ -150,6 +153,10 @@ export class WebAudioTransport { string, { input: GainNode; + /** Where the bus meets master; the level meter taps here. */ + output: GainNode; + /** False once the group's element is gone from the document. */ + isPresent(): boolean; /** Post-FX fader: `data-volume` plus the volume lane. */ fader: GainNode; muteGain: GainNode; @@ -162,6 +169,10 @@ export class WebAudioTransport { dispose(): void; } >(); + // Level taps exist only between `startMetering()` and `stopMetering()`. + private _metering = false; + private _masterTap: LevelTap | null = null; + private _groupTaps = new Map(); // Composition-time reference frame: at AudioContext time `_rateAnchorCtx`, // composition time was `_rateAnchorComp`, and time has been advancing at // `_rate` composition-seconds per wallclock-second since. @@ -175,8 +186,11 @@ export class WebAudioTransport { try { this._ctx = new AudioContext(); this._masterGain = this._ctx.createGain(); - this._masterGain.connect(this._ctx.destination); + this._monitorGain = this._ctx.createGain(); + this._masterGain.connect(this._monitorGain); + this._monitorGain.connect(this._ctx.destination); this.applyMasterGain(); + if (this._metering) this.attachMasterTap(); return true; } catch { return false; @@ -454,6 +468,8 @@ export class WebAudioTransport { this._groups.set(groupId, { input, + output, + isPresent: () => resolveEl() !== null, fader, muteGain, fx, @@ -485,9 +501,44 @@ export class WebAudioTransport { } }, }); + this.attachGroupTap(groupId, output); return input; } + startMetering(): void { + this._metering = true; + this.attachMasterTap(); + for (const [id, group] of this._groups) this.attachGroupTap(id, group.output); + } + + stopMetering(): void { + this._metering = false; + this._masterTap?.dispose(); + this._masterTap = null; + for (const tap of this._groupTaps.values()) tap.dispose(); + this._groupTaps.clear(); + } + + readLevels(): { master: StereoLevel; groups: Record } { + const groups: Record = {}; + for (const [id, tap] of this._groupTaps) { + if (this._groups.get(id)?.isPresent()) groups[id] = tap.read(); + } + return { master: this._masterTap?.read() ?? { l: 0, r: 0 }, groups }; + } + + private attachMasterTap(): void { + if (this._ctx && this._masterGain && !this._masterTap) { + this._masterTap = createLevelTap(this._ctx, this._masterGain); + } + } + + private attachGroupTap(id: string, output: GainNode): void { + if (this._metering && this._ctx && !this._groupTaps.has(id)) { + this._groupTaps.set(id, createLevelTap(this._ctx, output)); + } + } + /** * Group mute, preview side — a separate gain from `input`'s volume fader * so a mute toggle never fights `scheduleVolumeLane`'s ramps on the @@ -760,7 +811,8 @@ export class WebAudioTransport { } private applyMasterGain(): void { - if (this._masterGain) this._masterGain.gain.value = this._masterMuted ? 0 : this._masterVolume; + if (this._monitorGain) + this._monitorGain.gain.value = this._masterMuted ? 0 : this._masterVolume; } isActive(): boolean { @@ -780,6 +832,7 @@ export class WebAudioTransport { destroy(): void { this.stopAll(); + this.stopMetering(); for (const group of this._groups.values()) group.dispose(); this._groups.clear(); this._bufferCache.clear(); @@ -794,6 +847,7 @@ export class WebAudioTransport { } this._ctx = null; this._masterGain = null; + this._monitorGain = null; this._masterVolume = 1; this._masterMuted = false; } diff --git a/packages/core/src/runtime/window.d.ts b/packages/core/src/runtime/window.d.ts index 61ed1662b8..345c220cd7 100644 --- a/packages/core/src/runtime/window.d.ts +++ b/packages/core/src/runtime/window.d.ts @@ -56,6 +56,16 @@ declare global { * runtime's paused-side enforcement leaves it alone. Always release. */ leasePausedMedia?: (el: HTMLMediaElement) => void; releasePausedMedia?: (el: HTMLMediaElement) => void; + /** Read-only level taps for the Studio meters: nothing exists until + * `start()`, and `stop()` removes every tap. Peaks are linear per channel. */ + audioMeter?: { + start(): void; + stop(): void; + read(): { + master: { l: number; r: number }; + groups: Record; + }; + }; /** Declared-compute hold for setup no adapter can see (mesh building, * shader compiles). Runtime and player hold until every promise * registered here, under any key unique to your piece, resolves. */ diff --git a/packages/studio/src/components/TimelineToolbar.tsx b/packages/studio/src/components/TimelineToolbar.tsx index d31ff11c9c..a6d4400007 100644 --- a/packages/studio/src/components/TimelineToolbar.tsx +++ b/packages/studio/src/components/TimelineToolbar.tsx @@ -4,6 +4,7 @@ import { Magnet, MagnifyingGlassMinus, MagnifyingGlassPlus, + SpeakerHigh, Waves, } from "@phosphor-icons/react"; import { @@ -28,6 +29,8 @@ import { Scissors } from "../icons/SystemIcons"; import type { GsapAnimation } from "@hyperframes/core/gsap-parser"; import type { DomEditSelection } from "./editor/domEditingTypes"; import { canSplitElement } from "../utils/timelineElementSplit"; +import { useAudioMetersVisible } from "../utils/audioMeterVisibility"; +import { useProjectHasAudio } from "../utils/audioMeterMath"; import { canAddBeatAt, addBeatAtCompositionTime } from "../utils/beatEditActions"; interface DomEditSessionSlice extends EnableKeyframesSession { @@ -141,6 +144,9 @@ export function TimelineToolbar({ domEditSession, onSplitElement }: TimelineTool const thumbnailMode = usePlayerStore((s) => s.thumbnailMode); const setThumbnailMode = usePlayerStore((s) => s.setThumbnailMode); const thumbnailsVisible = thumbnailMode === "adaptive"; + const audioMetersVisible = useAudioMetersVisible((s) => s.visible); + const setAudioMetersVisible = useAudioMetersVisible((s) => s.setVisible); + const projectHasAudio = useProjectHasAudio(); // Subscribe so the add-beat button reacts to playhead movement and analysis load. const currentTime = usePlayerStore((s) => s.currentTime); const beatAnalysisReady = usePlayerStore((s) => s.beatAnalysis !== null); @@ -249,6 +255,19 @@ export function TimelineToolbar({ domEditSession, onSplitElement }: TimelineTool