From 3fe8cd7c3639a10d2361b0a843cc6c6e7f0e8c66 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sat, 19 Sep 2026 14:45:45 -0400 Subject: [PATCH 1/7] feat(studio): audio level meters at the timeline's right edge --- packages/core/src/runtime/init.ts | 5 + packages/core/src/runtime/levelTap.ts | 48 +++++ .../src/runtime/webAudioTransport.test.ts | 70 +++++++ .../core/src/runtime/webAudioTransport.ts | 48 +++++ packages/core/src/runtime/window.d.ts | 10 + .../studio/src/components/TimelineToolbar.tsx | 19 ++ .../components/nle/AudioMeterStrip.test.tsx | 103 ++++++++++ .../src/components/nle/AudioMeterStrip.tsx | 185 ++++++++++++++++++ .../src/components/nle/TimelinePane.tsx | 52 ++--- .../studio/src/utils/audioMeterMath.test.ts | 44 +++++ packages/studio/src/utils/audioMeterMath.ts | 78 ++++++++ .../studio/src/utils/audioMeterVisibility.ts | 14 ++ .../src/utils/studioUiPreferences.test.ts | 10 + .../studio/src/utils/studioUiPreferences.ts | 5 + 14 files changed, 668 insertions(+), 23 deletions(-) create mode 100644 packages/core/src/runtime/levelTap.ts create mode 100644 packages/studio/src/components/nle/AudioMeterStrip.test.tsx create mode 100644 packages/studio/src/components/nle/AudioMeterStrip.tsx create mode 100644 packages/studio/src/utils/audioMeterMath.test.ts create mode 100644 packages/studio/src/utils/audioMeterMath.ts create mode 100644 packages/studio/src/utils/audioMeterVisibility.ts diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 7dc7173e36..285b943d6b 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -2743,6 +2743,11 @@ export function initSandboxRuntimeModular(): void { }; window.__hf.leasePausedMedia = leasePausedMedia; window.__hf.releasePausedMedia = releasePausedMedia; + window.__hf.audioMeter = { + start: () => webAudio.startMetering(), + stop: () => webAudio.stopMetering(), + read: () => webAudio.readLevels(), + }; // Same predicate the media cache uses, so the paused side sees exactly the // media the transport drives. Reads attributes only; no cache rebuild. diff --git a/packages/core/src/runtime/levelTap.ts b/packages/core/src/runtime/levelTap.ts new file mode 100644 index 0000000000..0288baea08 --- /dev/null +++ b/packages/core/src/runtime/levelTap.ts @@ -0,0 +1,48 @@ +export interface StereoLevel { + l: number; + r: number; +} + +export interface LevelTap { + /** Peak of the last analyser window per channel, linear 0..1+. */ + read(): StereoLevel; + dispose(): void; +} + +const WINDOW = 2048; + +function peak(analyser: AnalyserNode, buf: Float32Array): number { + analyser.getFloatTimeDomainData(buf); + let max = 0; + for (let i = 0; i < buf.length; i++) max = Math.max(max, Math.abs(buf[i] ?? 0)); + return max; +} + +/** A side branch off `source`; the explicit-stereo gain up-mixes mono to both channels. */ +export function createLevelTap(ctx: BaseAudioContext, source: AudioNode): LevelTap { + const mix = ctx.createGain(); + mix.channelCount = 2; + mix.channelCountMode = "explicit"; + const splitter = ctx.createChannelSplitter(2); + const left = ctx.createAnalyser(); + const right = ctx.createAnalyser(); + left.fftSize = right.fftSize = WINDOW; + source.connect(mix); + mix.connect(splitter); + splitter.connect(left, 0); + splitter.connect(right, 1); + const bufL = new Float32Array(new ArrayBuffer(WINDOW * 4)); + const bufR = new Float32Array(new ArrayBuffer(WINDOW * 4)); + return { + read: () => ({ l: peak(left, bufL), r: peak(right, bufR) }), + dispose: () => { + try { + source.disconnect(mix); + mix.disconnect(); + splitter.disconnect(); + } catch { + // Source already torn down. + } + }, + }; +} diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.ts index db32530e76..34343e2b25 100644 --- a/packages/core/src/runtime/webAudioTransport.test.ts +++ b/packages/core/src/runtime/webAudioTransport.test.ts @@ -784,6 +784,7 @@ describe("WebAudioTransport", () => { analysers.push(node); return node; }), + createChannelSplitter: vi.fn(() => ({ connect: vi.fn(), disconnect: vi.fn() })), // The media-element route needs this as much as the decoded one: without // it `scheduleMediaElementPlayback` throws and its catch returns null, // which reads as "the member did not play" rather than a missing stub. @@ -1085,6 +1086,75 @@ describe("WebAudioTransport", () => { expect(() => transport.setGroupMuted("never-played", true)).not.toThrow(); }); }); + + describe("level metering", () => { + const addGroup = (id: string) => { + document.body.insertAdjacentHTML( + "beforeend", + ``, + ); + }; + const setLevel = (analyser: { getFloatTimeDomainData: unknown }, peak: number) => { + (analyser.getFloatTimeDomainData as ReturnType).mockImplementation( + (buf: Float32Array) => buf.fill(0).fill(-peak, 3, 4), + ); + }; + + it("creates nothing until startMetering", async () => { + addGroup("vo"); + const { transport, mock, gen } = setupGroupTransport(); + await scheduleGrouped(transport, gen, "a", "vo"); + expect(mock.ctx.createAnalyser).not.toHaveBeenCalled(); + expect(transport.readLevels()).toEqual({ master: { l: 0, r: 0 }, groups: {} }); + }); + + it("taps master and each group as side branches, once however often it starts", async () => { + addGroup("vo"); + const { transport, mock, gen } = setupGroupTransport(); + await scheduleGrouped(transport, gen, "a", "vo"); + const groupOutput = mock.gainNodes[2]!; + const connectsBefore = groupOutput.connect.mock.calls.length; + + transport.startMetering(); + transport.startMetering(); + + expect(mock.analysers).toHaveLength(4); + expect(groupOutput.connect).toHaveBeenCalledTimes(connectsBefore + 1); + expect(groupOutput.connect).toHaveBeenCalledWith(mock.masterGain); + expect(mock.masterGain.connect).toHaveBeenCalledTimes(1); + }); + + it("reads the peak per channel and stops cleanly", async () => { + addGroup("vo"); + const { transport, mock, gen } = setupGroupTransport(); + await scheduleGrouped(transport, gen, "a", "vo"); + transport.startMetering(); + const [masterL, , groupL, groupR] = mock.analysers; + setLevel(masterL!, 0.5); + setLevel(groupL!, 0.25); + setLevel(groupR!, 0.125); + + const levels = transport.readLevels(); + expect(levels.master).toEqual({ l: 0.5, r: 0 }); + expect(levels.groups.vo).toEqual({ l: 0.25, r: 0.125 }); + + transport.stopMetering(); + expect(transport.readLevels()).toEqual({ master: { l: 0, r: 0 }, groups: {} }); + expect(mock.gainNodes[2]!.disconnect).toHaveBeenCalled(); + }); + + it("a group built after start is metered, and one removed from the document leaves the read", async () => { + const { transport, mock, gen } = setupGroupTransport(); + transport.startMetering(); + addGroup("late"); + await scheduleGrouped(transport, gen, "a", "late"); + expect(Object.keys(transport.readLevels().groups)).toEqual(["late"]); + + document.querySelector("hf-audio-group")!.remove(); + expect(transport.readLevels().groups).toEqual({}); + expect(mock.analysers).toHaveLength(4); + }); + }); }); describe("decodeAudioElement retry policy (late-asset self-heal)", () => { diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.ts index 2761c98541..4b663008de 100644 --- a/packages/core/src/runtime/webAudioTransport.ts +++ b/packages/core/src/runtime/webAudioTransport.ts @@ -8,6 +8,7 @@ import { import { VOLUME_RANGE } from "../audioAutomation.js"; import { audioGroupOf, readAudioGroupVolume, resolveGroupElement } from "../audioGroups.js"; import { swallow } from "./diagnostics"; +import { createLevelTap, type LevelTap, type StereoLevel } from "./levelTap.js"; import { clampAudioGain } from "../audioGain.js"; import { getDebugSurface } from "./globals.js"; import { readElementPlaybackRate } from "./media.js"; @@ -150,6 +151,10 @@ export class WebAudioTransport { string, { input: GainNode; + /** Where the bus meets master; the level meter taps here. */ + output: GainNode; + /** False once the group's element is gone from the document. */ + isPresent(): boolean; /** Post-FX fader: `data-volume` plus the volume lane. */ fader: GainNode; muteGain: GainNode; @@ -162,6 +167,10 @@ export class WebAudioTransport { dispose(): void; } >(); + // Level taps exist only between `startMetering()` and `stopMetering()`. + private _metering = false; + private _masterTap: LevelTap | null = null; + private _groupTaps = new Map(); // Composition-time reference frame: at AudioContext time `_rateAnchorCtx`, // composition time was `_rateAnchorComp`, and time has been advancing at // `_rate` composition-seconds per wallclock-second since. @@ -177,6 +186,7 @@ export class WebAudioTransport { this._masterGain = this._ctx.createGain(); this._masterGain.connect(this._ctx.destination); this.applyMasterGain(); + if (this._metering) this.attachMasterTap(); return true; } catch { return false; @@ -454,6 +464,8 @@ export class WebAudioTransport { this._groups.set(groupId, { input, + output, + isPresent: () => resolveEl() !== null, fader, muteGain, fx, @@ -485,9 +497,44 @@ export class WebAudioTransport { } }, }); + this.attachGroupTap(groupId, output); return input; } + startMetering(): void { + this._metering = true; + this.attachMasterTap(); + for (const [id, group] of this._groups) this.attachGroupTap(id, group.output); + } + + stopMetering(): void { + this._metering = false; + this._masterTap?.dispose(); + this._masterTap = null; + for (const tap of this._groupTaps.values()) tap.dispose(); + this._groupTaps.clear(); + } + + readLevels(): { master: StereoLevel; groups: Record } { + const groups: Record = {}; + for (const [id, tap] of this._groupTaps) { + if (this._groups.get(id)?.isPresent()) groups[id] = tap.read(); + } + return { master: this._masterTap?.read() ?? { l: 0, r: 0 }, groups }; + } + + private attachMasterTap(): void { + if (this._ctx && this._masterGain && !this._masterTap) { + this._masterTap = createLevelTap(this._ctx, this._masterGain); + } + } + + private attachGroupTap(id: string, output: GainNode): void { + if (this._metering && this._ctx && !this._groupTaps.has(id)) { + this._groupTaps.set(id, createLevelTap(this._ctx, output)); + } + } + /** * Group mute, preview side — a separate gain from `input`'s volume fader * so a mute toggle never fights `scheduleVolumeLane`'s ramps on the @@ -780,6 +827,7 @@ export class WebAudioTransport { destroy(): void { this.stopAll(); + this.stopMetering(); for (const group of this._groups.values()) group.dispose(); this._groups.clear(); this._bufferCache.clear(); diff --git a/packages/core/src/runtime/window.d.ts b/packages/core/src/runtime/window.d.ts index 61ed1662b8..345c220cd7 100644 --- a/packages/core/src/runtime/window.d.ts +++ b/packages/core/src/runtime/window.d.ts @@ -56,6 +56,16 @@ declare global { * runtime's paused-side enforcement leaves it alone. Always release. */ leasePausedMedia?: (el: HTMLMediaElement) => void; releasePausedMedia?: (el: HTMLMediaElement) => void; + /** Read-only level taps for the Studio meters: nothing exists until + * `start()`, and `stop()` removes every tap. Peaks are linear per channel. */ + audioMeter?: { + start(): void; + stop(): void; + read(): { + master: { l: number; r: number }; + groups: Record; + }; + }; /** Declared-compute hold for setup no adapter can see (mesh building, * shader compiles). Runtime and player hold until every promise * registered here, under any key unique to your piece, resolves. */ diff --git a/packages/studio/src/components/TimelineToolbar.tsx b/packages/studio/src/components/TimelineToolbar.tsx index d31ff11c9c..a6d4400007 100644 --- a/packages/studio/src/components/TimelineToolbar.tsx +++ b/packages/studio/src/components/TimelineToolbar.tsx @@ -4,6 +4,7 @@ import { Magnet, MagnifyingGlassMinus, MagnifyingGlassPlus, + SpeakerHigh, Waves, } from "@phosphor-icons/react"; import { @@ -28,6 +29,8 @@ import { Scissors } from "../icons/SystemIcons"; import type { GsapAnimation } from "@hyperframes/core/gsap-parser"; import type { DomEditSelection } from "./editor/domEditingTypes"; import { canSplitElement } from "../utils/timelineElementSplit"; +import { useAudioMetersVisible } from "../utils/audioMeterVisibility"; +import { useProjectHasAudio } from "../utils/audioMeterMath"; import { canAddBeatAt, addBeatAtCompositionTime } from "../utils/beatEditActions"; interface DomEditSessionSlice extends EnableKeyframesSession { @@ -141,6 +144,9 @@ export function TimelineToolbar({ domEditSession, onSplitElement }: TimelineTool const thumbnailMode = usePlayerStore((s) => s.thumbnailMode); const setThumbnailMode = usePlayerStore((s) => s.setThumbnailMode); const thumbnailsVisible = thumbnailMode === "adaptive"; + const audioMetersVisible = useAudioMetersVisible((s) => s.visible); + const setAudioMetersVisible = useAudioMetersVisible((s) => s.setVisible); + const projectHasAudio = useProjectHasAudio(); // Subscribe so the add-beat button reacts to playhead movement and analysis load. const currentTime = usePlayerStore((s) => s.currentTime); const beatAnalysisReady = usePlayerStore((s) => s.beatAnalysis !== null); @@ -249,6 +255,19 @@ export function TimelineToolbar({ domEditSession, onSplitElement }: TimelineTool