From 0dd01274557eba3964ebd011ae7e3a835088f951 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 10:48:36 +0000 Subject: [PATCH] Measure tempo, key and energy from a track's sound For the music whose tags say nothing. Plain numpy over ffmpeg-decoded PCM, no new dependency: - Tempo: autocorrelation of a spectral-flux onset curve, scored with its first four multiples and a log-normal prior at 110 BPM. - Key: pitch-class profile against the 24 Krumhansl-Kessler profiles. - Energy: loudness, brightness and drive, in three levels. Gated so talks and noise get nothing: a tempo needs a real beat, and a key without one needs a strong correlation. Fills blanks only, after tags, and is stamped "analysis" in field_provenance. Queued after every song-length audio upload, runnable per track from the Music tab, and library-wide from a new Settings panel, which also gives the existing tag harvest its first button. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DCL4jm6ogZBBhnjEfBBaXU --- backend/app/enrichment/analyze_music.py | 160 +++++++++ backend/app/ingest/audio_analysis.py | 338 ++++++++++++++++++ backend/app/jobs/enrichment.py | 20 ++ backend/app/models/asset.py | 5 + backend/app/models/job.py | 9 + backend/app/routers/assets.py | 32 +- backend/app/routers/enrichment.py | 35 ++ backend/app/services/assets.py | 41 ++- backend/tests/music_synth.py | 136 +++++++ backend/tests/test_audio_analysis.py | 209 +++++++++++ docs/music-profile.md | 36 +- frontend/src/api/assets.ts | 8 + frontend/src/api/enrichment.ts | 8 + frontend/src/components/ActivityIndicator.tsx | 3 + .../src/components/LibraryScanPanel.test.tsx | 76 ++++ frontend/src/components/LibraryScanPanel.tsx | 119 ++++++ frontend/src/components/MusicPanel.test.tsx | 51 ++- frontend/src/components/MusicPanel.tsx | 21 +- frontend/src/views/SettingsView.tsx | 2 + 19 files changed, 1296 insertions(+), 13 deletions(-) create mode 100644 backend/app/enrichment/analyze_music.py create mode 100644 backend/app/ingest/audio_analysis.py create mode 100644 backend/tests/music_synth.py create mode 100644 backend/tests/test_audio_analysis.py create mode 100644 frontend/src/components/LibraryScanPanel.test.tsx create mode 100644 frontend/src/components/LibraryScanPanel.tsx diff --git a/backend/app/enrichment/analyze_music.py b/backend/app/enrichment/analyze_music.py new file mode 100644 index 0000000..ce5f9b3 --- /dev/null +++ b/backend/app/enrichment/analyze_music.py @@ -0,0 +1,160 @@ +"""Filling a track's tempo, key and energy from its sound. + +The measuring is `ingest/audio_analysis.py`; this decides what to measure and what to +keep. The rule is the one the tag harvest follows: fill blanks only. A tag was written +by someone who heard the track, and a person's correction is a correction, so a +measurement never replaces either. That is also what makes the library-wide run safe to +repeat: a second pass finds nothing blank that the first one could fill. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Callable + +from sqlmodel import Session, col, select + +from app.ingest.audio_analysis import ( + AudioAnalysisError, + MIN_SECONDS, + MusicAnalysis, + analyse, + decode, +) +from app.ingest.filetypes import TYPE_AUDIO +from app.models.asset import Asset +from app.services import assets as asset_service +from app.storage import StorageError, build_storage + +logger = logging.getLogger(__name__) + +Progress = Callable[..., None] + +# Longer than any track a video is likely to want under it. Past this an audio file is a +# recording (a podcast, an interview, a lecture) and analysing it as music would at best +# waste the CPU and at worst file a talk under "high energy". +MAX_TRACK_SECONDS = 20 * 60 + +# The fields this can measure. Vocals, genres and moods it does not attempt: see the +# module docstring of `ingest/audio_analysis.py`. +MEASURED_FIELDS = ("music_bpm", "music_key", "music_energy") + + +def analysable(asset: Asset) -> bool: + """An audio file of its own (not a clip), of a length a song could be. + + An unknown duration is let through: the probe failing says nothing about the + content, and decoding will fail cleanly if the file is unreadable. + """ + if asset.asset_type != TYPE_AUDIO or not asset.storage_key: + return False + duration = asset.duration_seconds + return duration is None or MIN_SECONDS <= duration <= MAX_TRACK_SECONDS + + +def missing_fields(asset: Asset) -> list[str]: + return [name for name in MEASURED_FIELDS if getattr(asset, name, None) in (None, "")] + + +def measure(asset: Asset) -> MusicAnalysis: + storage = build_storage() + with storage.materialise(asset.storage_key) as path: + samples = decode(path, asset.duration_seconds) + return analyse(samples) + + +def apply(session: Session, asset: Asset, result: MusicAnalysis) -> list[str]: + """Write what was measured into the fields that are still blank. Returns those.""" + if not result.is_music: + return [] + measured = {"music_bpm": result.bpm, "music_key": result.key, "music_energy": result.energy} + blank = set(missing_fields(asset)) + changes = { + name: value for name, value in measured.items() if value is not None and name in blank + } + if changes: + asset_service.apply_measured_metadata(session, asset, changes) + return list(changes) + + +def describe(result: MusicAnalysis) -> str: + """The activity line: what the track measured as, whether or not it was all kept.""" + if not result.is_music: + return "No beat or key found, so nothing was filled in" + parts = [] + if result.bpm is not None: + parts.append(f"{result.bpm:g} BPM") + if result.key: + parts.append(result.key) + if result.energy: + parts.append(f"{result.energy} energy") + return " · ".join(parts) + + +def run(session: Session, asset: Asset, progress: Progress) -> str: + """Analyse one track. Returns what to show in the activity feed.""" + if not analysable(asset): + raise AudioAnalysisError( + "Only audio files up to 20 minutes long are analysed as music" + ) + + progress("Decoding the audio", 10, "") + result = measure(asset) + progress("Measuring tempo, key and energy", 70, "") + written = apply(session, asset, result) + + detail = describe(result) + if result.is_music and not written: + detail += " (everything was already filled in, so nothing changed)" + return detail + + +@dataclass +class LibraryResult: + scanned: int + filled: int + not_music: int + failed: int + + +def run_library(session: Session, user_id: str, progress: Progress) -> LibraryResult: + """Analyse every audio file in this library that has a measured field still blank.""" + pending = [ + asset + for asset in session.exec( + select(Asset) + .where( + Asset.user_id == user_id, + Asset.asset_type == TYPE_AUDIO, + col(Asset.storage_key).is_not(None), + ) + .order_by(col(Asset.upload_date).desc()) + ).all() + if analysable(asset) and missing_fields(asset) + ] + total = len(pending) + scanned = filled = not_music = failed = 0 + + for index, asset in enumerate(pending): + pct = int(index * 100 / total) if total else 100 + progress("Analysing music", pct, f"{index + 1} of {total} · {asset.name}") + try: + result = measure(asset) + except (AudioAnalysisError, StorageError): + # A file that will not decode, or that went missing underneath its row, is + # this file's problem, not a reason to stop the run. + failed += 1 + continue + except Exception: # noqa: BLE001 - one bad file must not end the run + failed += 1 + logger.warning("Could not analyse %s", asset.id, exc_info=True) + continue + + scanned += 1 + if not result.is_music: + not_music += 1 + elif apply(session, asset, result): + filled += 1 + + return LibraryResult(scanned=scanned, filled=filled, not_music=not_music, failed=failed) diff --git a/backend/app/ingest/audio_analysis.py b/backend/app/ingest/audio_analysis.py new file mode 100644 index 0000000..b01b417 --- /dev/null +++ b/backend/app/ingest/audio_analysis.py @@ -0,0 +1,338 @@ +"""Measuring a track's tempo, key and energy from its sound. + +For the files whose tags say nothing, which is most music downloaded from free +libraries. The tags are still the better source when present (whoever wrote them heard +the track), so this only ever fills what they left blank; see +`enrichment/analyze_music.py`. + +Plain numpy over ffmpeg-decoded PCM, deliberately. librosa would do all three with less +code, but it brings numba, llvmlite, scipy, soundfile and audioread, which is a long list +of compiled wheels to keep matching the pinned Python 3.13 (the reason requirements.txt +is careful about every compiled dependency). These three measurements are well-known +signal processing that fits in one file. + +What it does not attempt, and why: + +- **Vocals.** Telling singing from a lead instrument reliably takes a trained model. A + heuristic that guessed would put songs under narration, the exact failure the + nullable `music_vocals` exists to prevent. +- **Genre and mood.** Same reason, and the words are the person's or the tagger's. + +How good it is: tempo is the dependable one on music with a beat, apart from the +usual octave ambiguity (a 140 BPM track read as 70) which no method escapes. Key is +right more often than not on tonal music and commonly confuses a major key with its +relative minor, which share every note. Energy is a coarse three-way split. Everything +here is a measurement stamped `analysis`, so a person's correction always stands. +""" + +from __future__ import annotations + +import logging +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +import numpy as np + +from app.media_tools import ffmpeg_available + +logger = logging.getLogger(__name__) + +SAMPLE_RATE = 22050 + +# Two minutes is enough to measure a tempo and a key and keeps a long file cheap. Taken +# from past the opening when the file is long enough: intros are often beatless or in +# a different texture, and the body of a track is what a video will play under a scene. +ANALYSIS_SECONDS = 120.0 +SKIP_INTRO_SECONDS = 20.0 +# Below this there is too little to find a beat period in several repetitions. +MIN_SECONDS = 8.0 + +# The onset pass: ~11.6 ms per frame (86 frames a second), fine enough that a beat +# period at 180 BPM spans 29 frames. +_ONSET_FFT = 2048 +_ONSET_HOP = 256 +# The pitch pass needs finer frequency resolution than timing does: 2.7 Hz bins, so +# neighbouring semitones are separate bins down to about C2. +_CHROMA_FFT = 8192 +_CHROMA_HOP = 2048 +_CHROMA_MIN_HZ = 65.0 +_CHROMA_MAX_HZ = 2100.0 +# Frames per FFT batch, so two minutes never needs the whole spectrogram in memory. +_BATCH = 512 + +_MIN_BPM = 50.0 +_MAX_BPM = 210.0 +# Where the tempo search leans when two octaves score alike. Most music people will put +# under a video sits around here, and a log-normal prior is the usual way to break the +# 70/140 tie (the same choice librosa makes). +_PRIOR_BPM = 110.0 +_PRIOR_OCTAVES = 1.0 +# How strongly the onset curve must repeat at the beat period to call it a beat at all. +# Below this the track is beatless (ambient, a drone, speech) and no tempo is better +# than a confident wrong one. A slow pad whose chords change every few seconds reached +# 0.24 in testing; a drum pattern sits above 0.6. +_MIN_BEAT_STRENGTH = 0.3 + +# Krumhansl–Kessler key profiles: how strongly each scale degree is felt in a key. +_MAJOR_PROFILE = np.array([6.35, 2.23, 3.48, 2.33, 4.38, 4.09, 2.52, 5.19, 2.39, 3.66, 2.29, 2.88]) +_MINOR_PROFILE = np.array([6.33, 2.68, 3.52, 5.38, 2.60, 3.53, 2.54, 4.75, 3.98, 2.69, 3.34, 3.17]) +_PITCH_NAMES = ("C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B") +# Correlation with the best key's profile needed to name one. Atonal material, noise and +# speech correlate weakly with every key; tonal music sits well above this. +_MIN_KEY_CORRELATION = 0.6 +# Without a beat, a key alone has to be clearer than that to call the file music at all. +# Pitched speech reached 0.65 in testing; sustained tonal pads sit above 0.85. +_MIN_BEATLESS_KEY_CORRELATION = 0.75 + + +class AudioAnalysisError(Exception): + """The file could not be decoded, or is too short to measure.""" + + +@dataclass(frozen=True) +class MusicAnalysis: + bpm: Optional[float] + key: Optional[str] + energy: Optional[str] + # Kept for the job's detail line and for tests; not stored. + beat_strength: float = 0.0 + key_correlation: float = 0.0 + energy_score: float = 0.0 + + @property + def is_music(self) -> bool: + """A beat, or a key clear enough to stand without one. Otherwise this is + probably speech or noise, and nothing measured here is worth recording.""" + if self.bpm is not None: + return True + return self.key is not None and self.key_correlation >= _MIN_BEATLESS_KEY_CORRELATION + + +# ─── decoding ──────────────────────────────────────────────────────────────── + + +def decode(path: Path, duration_seconds: Optional[float] = None) -> np.ndarray: + """Mono float32 PCM at `SAMPLE_RATE`, up to `ANALYSIS_SECONDS` of it.""" + if not ffmpeg_available(): + raise AudioAnalysisError("ffmpeg is not available on this server") + + start = 0.0 + if duration_seconds and duration_seconds > ANALYSIS_SECONDS + 2 * SKIP_INTRO_SECONDS: + start = SKIP_INTRO_SECONDS + + argv = [ + "ffmpeg", "-nostdin", "-v", "error", + # Before -i, so ffmpeg seeks rather than decoding the intro to discard it. + "-ss", f"{start:.3f}", + "-t", f"{ANALYSIS_SECONDS:.3f}", + "-i", str(path), + "-vn", "-map", "0:a:0", + "-ac", "1", "-ar", str(SAMPLE_RATE), + "-f", "f32le", "-", + ] + try: + completed = subprocess.run(argv, capture_output=True, timeout=300) + except subprocess.TimeoutExpired as exc: + raise AudioAnalysisError("Decoding the audio took too long") from exc + except OSError as exc: + raise AudioAnalysisError(f"Could not run ffmpeg: {exc}") from exc + + if completed.returncode != 0: + stderr = completed.stderr.decode("utf-8", "replace").strip() + if "does not contain any stream" in stderr or "Stream map" in stderr: + raise AudioAnalysisError("This file has no audio track to analyse") + logger.warning("Audio decode failed for %s: %s", path.name, stderr[-400:]) + raise AudioAnalysisError("Could not decode the audio") + + return np.frombuffer(completed.stdout, dtype=" MusicAnalysis: + """Tempo, key and energy of mono PCM. Pure, so it is tested on synthesised audio.""" + samples = np.asarray(samples, dtype=np.float32) + if samples.size < MIN_SECONDS * sample_rate: + raise AudioAnalysisError("Too short to measure a tempo or a key") + + rms = float(np.sqrt(np.mean(np.square(samples, dtype=np.float64)))) + if rms < 1e-4: + # Digital silence (-80 dBFS). Nothing to measure, and dividing by it below would + # turn noise into confident nonsense. + return MusicAnalysis(bpm=None, key=None, energy=None) + + onset, centroid_hz = _onset_envelope(samples, sample_rate) + frame_rate = sample_rate / _ONSET_HOP + + bpm, beat_strength = _tempo(onset, frame_rate) + key, key_correlation = _key(samples, sample_rate) + energy, energy_score = _energy(rms, centroid_hz, bpm, beat_strength) + + return MusicAnalysis( + bpm=bpm, + key=key, + energy=energy, + beat_strength=beat_strength, + key_correlation=key_correlation, + energy_score=energy_score, + ) + + +def _frames(samples: np.ndarray, size: int, hop: int): + """Hann-windowed frames in batches, so memory stays flat however long the input.""" + if samples.size < size: + samples = np.pad(samples, (0, size - samples.size)) + window = np.hanning(size).astype(np.float32) + view = np.lib.stride_tricks.sliding_window_view(samples, size)[::hop] + for start in range(0, view.shape[0], _BATCH): + yield view[start:start + _BATCH] * window + + +def _onset_envelope(samples: np.ndarray, sample_rate: int) -> tuple[np.ndarray, float]: + """Spectral flux: how much new energy arrives in each frame, summed over frequency. + + Log-compressed first, so a quiet hi-hat counts as an onset next to a loud bass and + the curve follows rhythm rather than loudness. Also returns the mean spectral + centroid, which the same magnitudes give for free. + """ + freqs = np.fft.rfftfreq(_ONSET_FFT, 1.0 / sample_rate).astype(np.float32) + flux_parts: list[np.ndarray] = [] + centroid_sum = 0.0 + weight_sum = 0.0 + previous: Optional[np.ndarray] = None + + for batch in _frames(samples, _ONSET_FFT, _ONSET_HOP): + magnitude = np.abs(np.fft.rfft(batch, axis=1)).astype(np.float32) + energy = magnitude.sum(axis=1) + centroid_sum += float((magnitude @ freqs).sum()) + weight_sum += float(energy.sum()) + + compressed = np.log1p(100.0 * magnitude) + stacked = compressed if previous is None else np.vstack([previous[None, :], compressed]) + rise = np.maximum(np.diff(stacked, axis=0), 0.0).sum(axis=1) + if previous is None: + rise = np.concatenate([[0.0], rise]) + flux_parts.append(rise.astype(np.float32)) + previous = compressed[-1] + + flux = np.concatenate(flux_parts) + centroid = centroid_sum / weight_sum if weight_sum > 0 else 0.0 + return flux, centroid + + +def _tempo(onset: np.ndarray, frame_rate: float) -> tuple[Optional[float], float]: + """The beat period at which the onset curve best repeats itself. + + Autocorrelation of the onset curve, scored on a fine BPM grid with the first four + multiples of each candidate period (a real beat repeats at 2x and 3x its period too, + which a single lag misses), then weighted by a prior that settles octave ties. + """ + # Remove the slow swell of loudness, keeping only the pulses on top of it. + smoothing = max(int(frame_rate * 0.4), 1) + trend = np.convolve(onset, np.ones(smoothing) / smoothing, mode="same") + pulses = np.maximum(onset - trend, 0.0) + pulses -= pulses.mean() + norm = float(np.sqrt(np.sum(pulses * pulses))) + if norm == 0: + return None, 0.0 + + size = 1 << int(np.ceil(np.log2(2 * pulses.size))) + spectrum = np.fft.rfft(pulses, size) + acf = np.fft.irfft(spectrum * np.conj(spectrum), size)[: pulses.size] + acf /= acf[0] + + bpms = np.arange(_MIN_BPM, _MAX_BPM + 0.05, 0.1) + periods = 60.0 * frame_rate / bpms + lags = np.arange(acf.size) + + def at(lag: np.ndarray) -> np.ndarray: + return np.interp(lag, lags, acf, right=0.0) + + comb = at(periods) + 0.5 * at(2 * periods) + 0.33 * at(3 * periods) + 0.25 * at(4 * periods) + prior = np.exp(-0.5 * (np.log2(bpms / _PRIOR_BPM) / _PRIOR_OCTAVES) ** 2) + best = int(np.argmax(comb * prior)) + + strength = float(at(np.array([periods[best]]))[0]) + if strength < _MIN_BEAT_STRENGTH: + return None, strength + return round(float(bpms[best]), 1), strength + + +def _chroma(samples: np.ndarray, sample_rate: int) -> np.ndarray: + """How much of each of the twelve pitch classes the track contains, overall. + + Each frame is normalised before it is summed, so a long quiet passage counts as much + as a short loud one: the key is a property of the whole track, not of its peaks. + """ + freqs = np.fft.rfftfreq(_CHROMA_FFT, 1.0 / sample_rate) + band = (freqs >= _CHROMA_MIN_HZ) & (freqs <= _CHROMA_MAX_HZ) + midi = 69.0 + 12.0 * np.log2(freqs[band] / 440.0) + pitch_class = np.mod(np.round(midi), 12).astype(int) + # Bins that sit between two semitones (a detuned partial, a bin straddling both) + # say little about either, so they count for less. + closeness = 1.0 - 2.0 * np.abs(midi - np.round(midi)) + mapping = np.zeros((band.sum(), 12), dtype=np.float32) + mapping[np.arange(band.sum()), pitch_class] = np.clip(closeness, 0.0, 1.0) + + total = np.zeros(12, dtype=np.float64) + for batch in _frames(samples, _CHROMA_FFT, _CHROMA_HOP): + magnitude = np.abs(np.fft.rfft(batch, axis=1))[:, band].astype(np.float32) + frames = magnitude @ mapping + peaks = frames.max(axis=1, keepdims=True) + total += (frames / np.where(peaks > 0, peaks, 1.0)).sum(axis=0) + return total + + +def _key(samples: np.ndarray, sample_rate: int) -> tuple[Optional[str], float]: + """The key whose Krumhansl–Kessler profile best matches the track's pitch classes.""" + chroma = _chroma(samples, sample_rate) + if not np.any(chroma): + return None, 0.0 + + best_name: Optional[str] = None + best_r = -1.0 + for tonic in range(12): + for profile, mode in ((_MAJOR_PROFILE, "major"), (_MINOR_PROFILE, "minor")): + r = float(np.corrcoef(chroma, np.roll(profile, tonic))[0, 1]) + if r > best_r: + best_r = r + best_name = f"{_PITCH_NAMES[tonic]} {mode}" + + if best_r < _MIN_KEY_CORRELATION: + return None, best_r + return best_name, best_r + + +def _energy( + rms: float, centroid_hz: float, bpm: Optional[float], beat_strength: float +) -> tuple[str, float]: + """Low, medium or high, from loudness, brightness and drive. + + A weighted blend rather than any one measure: a loud, slow orchestral swell and a + quiet, fast hi-hat pattern both read as "medium" to a listener, and either measure + alone would put one of them at an extreme. The ranges are where mastered music + actually sits; it is a coarse three-way split and is named as such in the UI. + + Drive comes from the tempo pass rather than from counting onsets. Counted against + the onset curve's own spread, a beatless pad still produced two "onsets" a second of + noise, enough to lift ambient music out of "low". A track with no beat has no drive. + """ + loudness_db = 20.0 * np.log10(max(rms, 1e-9)) + loud = np.clip((loudness_db + 32.0) / 22.0, 0.0, 1.0) # -32 dBFS .. -10 dBFS + bright = np.clip((centroid_hz - 600.0) / 2400.0, 0.0, 1.0) # 600 Hz .. 3 kHz + if bpm is None: + drive = 0.0 + else: + pace = np.clip((bpm - 50.0) / 110.0, 0.0, 1.0) # 50 .. 160 BPM + pulse = np.clip(beat_strength / 0.6, 0.0, 1.0) + drive = 0.5 * pace + 0.5 * pulse + + score = float(0.45 * loud + 0.2 * bright + 0.35 * drive) + if score < 0.38: + return "low", score + if score > 0.62: + return "high", score + return "medium", score diff --git a/backend/app/jobs/enrichment.py b/backend/app/jobs/enrichment.py index a610070..d8b184c 100644 --- a/backend/app/jobs/enrichment.py +++ b/backend/app/jobs/enrichment.py @@ -24,6 +24,8 @@ from app.database import engine from app.embeddings import EmbeddingError, build_embedder from app.enrichment.embed import EmbeddingUnavailable +from app.enrichment.analyze_music import run as run_analyze_music +from app.enrichment.analyze_music import run_library as run_analyze_music_library from app.enrichment.attribute import run as run_attribute from app.enrichment.backfill import run as run_backfill from app.enrichment.harvest_attribution import run as run_harvest_attribution @@ -42,6 +44,7 @@ from app.enrichment.transcribe import TranscriptionError from app.enrichment.transcribe import run as run_transcribe from app.generation.errors import GenerationError +from app.ingest.audio_analysis import AudioAnalysisError from app.generation.run import run as run_generate from app.ingest.ytdlp import UrlImportError from app.providers.base import ProviderError @@ -55,6 +58,8 @@ from app.models.asset import Asset from app.models.job import ( EnrichmentJob, + KIND_ANALYZE_MUSIC, + KIND_ANALYZE_MUSIC_LIBRARY, KIND_AUTOTAG, KIND_ATTRIBUTE, KIND_BACKFILL_EMBEDDINGS, @@ -360,6 +365,16 @@ def _run(job_id: str, q: JobQueue) -> None: if harvested.failed: # Surfaced rather than swallowed, same as the two runs above. detail += f", {harvested.failed} unreadable" + elif job.kind == KIND_ANALYZE_MUSIC: + detail = run_analyze_music(session, asset, progress) + elif job.kind == KIND_ANALYZE_MUSIC_LIBRARY: + analysed = run_analyze_music_library(session, job.user_id, progress) + detail = f"{analysed.filled} filled in of {analysed.scanned} analysed" + if analysed.not_music: + detail += f", {analysed.not_music} with no beat or key" + if analysed.failed: + # Surfaced rather than swallowed, same as the runs above. + detail += f", {analysed.failed} unreadable" else: raise TranscriptionError(f"Unknown enrichment kind: {job.kind}") @@ -420,6 +435,11 @@ def _run(job_id: str, q: JobQueue) -> None: set_fields(session, job, status="error", stage="", error_message=message) logger.info("Enrichment job %s failed: %s", job_id, message) + except AudioAnalysisError as exc: + message = str(exc) + set_fields(session, job, status="error", stage="", error_message=message) + logger.info("Enrichment job %s failed: %s", job_id, message) + except SubvideoExtractionError as exc: message = str(exc) _mark_asset_failed(session, asset, job) diff --git a/backend/app/models/asset.py b/backend/app/models/asset.py index 931f2cf..a9c41a3 100644 --- a/backend/app/models/asset.py +++ b/backend/app/models/asset.py @@ -20,6 +20,11 @@ PROVENANCE_HUMAN = "human" PROVENANCE_AI = "ai" PROVENANCE_EMBEDDED = "embedded" +# Measured from the file's content by GAM itself: a tempo, key or energy computed from +# the audio. Not "ai" (no model guessed it, and nothing was proposed for review) and not +# "embedded" (no one wrote it into the file). Fills blanks only, like "embedded", because +# a tag written by someone who heard the track is the better source. +PROVENANCE_ANALYSIS = "analysis" def new_asset_id() -> str: diff --git a/backend/app/models/job.py b/backend/app/models/job.py index 5f72005..914dfa9 100644 --- a/backend/app/models/job.py +++ b/backend/app/models/job.py @@ -66,6 +66,13 @@ # request, fal's request ids and the assets made all ride in `payload`. Outside # `ENRICHMENT_KINDS`: it enriches nothing. See docs/m8-ai-generation.md. KIND_GENERATE = "generate" +# Measuring an audio file's tempo, key and energy from its sound (docs/music-profile.md). +# Per-asset, queued after every audio upload, and in `ENRICHMENT_KINDS` because it acts on +# one asset; it calls no provider and costs nothing but a second of CPU. +KIND_ANALYZE_MUSIC = "analyze_music" +# The same over every audio file in a library, for the ones uploaded before it existed. +# Library-wide for the reasons `KIND_HARVEST_ATTRIBUTION` is. +KIND_ANALYZE_MUSIC_LIBRARY = "analyze_music_library" # Kinds that run on the generation queue rather than the enrichment one. A generation # is minutes of waiting on fal, and on the shared queue at concurrency 1 a video would @@ -83,6 +90,7 @@ KIND_EXTRACT_TEXT, KIND_GENERATE_ALL, KIND_ATTRIBUTE, + KIND_ANALYZE_MUSIC, } ) @@ -95,6 +103,7 @@ KIND_BACKFILL_EMBEDDINGS, KIND_BULK_ENRICH, KIND_HARVEST_ATTRIBUTION, + KIND_ANALYZE_MUSIC_LIBRARY, KIND_IMPORT_URL, KIND_GENERATE, } diff --git a/backend/app/routers/assets.py b/backend/app/routers/assets.py index 5bb5bb0..319ef58 100644 --- a/backend/app/routers/assets.py +++ b/backend/app/routers/assets.py @@ -16,7 +16,11 @@ from app.jobs import enrichment as enrichment_jobs from app.jobs.registry import KINDS from app.models.asset import Asset -from app.models.job import KIND_EXTRACT_SUBVIDEO, KIND_HARVEST_ATTRIBUTION +from app.models.job import ( + KIND_ANALYZE_MUSIC_LIBRARY, + KIND_EXTRACT_SUBVIDEO, + KIND_HARVEST_ATTRIBUTION, +) from app.models.tag import Tag from app.schemas import DataResponse, ListResponse from app.schemas_assets import AssetRead, AssetUpdate, UploadRejection, UploadResult @@ -575,3 +579,29 @@ def start_attribution_harvest( job = enrichment_jobs.submit_library(session, user.id, KIND_HARVEST_ATTRIBUTION) return DataResponse(data=KINDS["enrichment"].to_activity(job)) + + +@router.post( + "/analyze-music", + response_model=DataResponse[ActivityJobRead], + status_code=status.HTTP_202_ACCEPTED, +) +def start_music_analysis( + user: CurrentUser, session: Session = Depends(get_session) +) -> DataResponse[ActivityJobRead]: + """Measure tempo, key and energy for every track in this library still missing one. + + New uploads are analysed as they land, so this is for the ones that came before. + Safe to repeat for the reason the harvest above is: it only fills blanks. + """ + if enrichment_jobs.active_library_job(session, user.id, KIND_ANALYZE_MUSIC_LIBRARY): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "code": "already_running", + "message": "Your music is already being analysed", + }, + ) + + job = enrichment_jobs.submit_library(session, user.id, KIND_ANALYZE_MUSIC_LIBRARY) + return DataResponse(data=KINDS["enrichment"].to_activity(job)) diff --git a/backend/app/routers/enrichment.py b/backend/app/routers/enrichment.py index 063dcc5..2885cac 100644 --- a/backend/app/routers/enrichment.py +++ b/backend/app/routers/enrichment.py @@ -18,6 +18,7 @@ from app.database import get_session from app.embeddings import build_embedder from app.enrichment import bulk +from app.enrichment.analyze_music import analysable from app.enrichment.describe import describable from app.enrichment.extract_text import extractable from app.enrichment.summarize import summarisable @@ -25,6 +26,7 @@ from app.jobs.registry import KINDS from app.models.asset import Asset from app.models.job import ( + KIND_ANALYZE_MUSIC, KIND_ATTRIBUTE, KIND_AUTOTAG, KIND_BULK_ENRICH, @@ -179,6 +181,39 @@ def start_attribute( return DataResponse(data=KINDS["enrichment"].to_activity(job)) +@router.post( + "/{asset_id}/analyze-music", response_model=DataResponse[ActivityJobRead], status_code=202 +) +def start_analyze_music( + asset_id: str, + user: CurrentUser, + session: Session = Depends(get_session), +) -> DataResponse[ActivityJobRead]: + """Measure one track's tempo, key and energy, filling whichever are still blank. + + No provider check, unlike the endpoints around it: nothing here calls one. + """ + asset = _owned_asset(asset_id, user.id, session) + + if not analysable(asset): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": "not_analysable", + "message": "Only audio files up to 20 minutes long are analysed as music", + }, + ) + + if enrichment_jobs.active_job(session, asset.id, KIND_ANALYZE_MUSIC) is not None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={"code": "already_running", "message": "This track is already being analysed"}, + ) + + job = enrichment_jobs.submit(session, asset, KIND_ANALYZE_MUSIC) + return DataResponse(data=KINDS["enrichment"].to_activity(job)) + + @router.post("/{asset_id}/autotag", response_model=DataResponse[ActivityJobRead], status_code=202) def start_autotag( asset_id: str, diff --git a/backend/app/services/assets.py b/backend/app/services/assets.py index ae2c3f0..f57b190 100644 --- a/backend/app/services/assets.py +++ b/backend/app/services/assets.py @@ -38,6 +38,7 @@ from app.ingest.probe import ProbeResult, probe from app.models.asset import ( PROVENANCE_AI, + PROVENANCE_ANALYSIS, PROVENANCE_EMBEDDED, PROVENANCE_HUMAN, Asset, @@ -180,6 +181,7 @@ def _catalogue( _reindex(session, asset) _describe(session, storage, asset) _chain_transcription(session, asset) + _chain_music_analysis(session, asset) return asset @@ -324,6 +326,33 @@ def _chain_transcription(session: Session, asset: Asset) -> None: logger.warning("Could not queue transcription for asset %s", asset.id, exc_info=True) +def _chain_music_analysis(session: Session, asset: Asset) -> None: + """Measure an audio file's tempo, key and energy the moment it lands. + + Unconditional where transcription is guarded on a key: it calls no provider, costs a + second of CPU, and needs nothing configured. It runs after `_describe` has applied the + file's own tags, and only fills what they left blank. + + Not every audio file is music. `analyze_music.analysable` keeps long recordings out + (a podcast is not going to be a soundtrack), and the analysis itself writes nothing + when it finds neither a beat nor a key, which is what speech looks like to it. + + Imports lazily, for the cycle `_chain_transcription` describes. + """ + from app.enrichment.analyze_music import analysable + from app.jobs import enrichment as enrichment_jobs + from app.models.job import KIND_ANALYZE_MUSIC + + if not analysable(asset): + return + try: + if enrichment_jobs.active_job(session, asset.id, KIND_ANALYZE_MUSIC) is not None: + return + enrichment_jobs.submit(session, asset, KIND_ANALYZE_MUSIC) + except Exception: # noqa: BLE001 - an upload that succeeded must stay succeeded + logger.warning("Could not queue music analysis for asset %s", asset.id, exc_info=True) + + def _reindex(session: Session, asset: Asset) -> None: """Keep the keyword index in step with the row. @@ -547,8 +576,18 @@ def apply_embedded_metadata(session: Session, asset: Asset, changes: dict) -> li return list(changes.keys()) +def apply_measured_metadata(session: Session, asset: Asset, changes: dict) -> list[str]: + """Write fields GAM measured from the file's content (a tempo, a key, an energy). + + The fourth writer. Callers filter to blank fields first, as `apply_embedded_metadata`'s + do: a measurement never overrides a tag, a person or a model. + """ + _apply(session, asset, changes, PROVENANCE_ANALYSIS) + return list(changes.keys()) + + def _apply(session: Session, asset: Asset, changes: dict, provenance_value: str) -> None: - """The shared body of the three writers above: set the fields, stamp who set them. + """The shared body of the writers above: set the fields, stamp who set them. A malformed `field_provenance` is rebuilt rather than raising. It is a record *about* the metadata, and losing that record must never cost the write it describes. diff --git a/backend/tests/music_synth.py b/backend/tests/music_synth.py new file mode 100644 index 0000000..1f64e7c --- /dev/null +++ b/backend/tests/music_synth.py @@ -0,0 +1,136 @@ +"""Synthesised music with a known tempo and key, for the audio-analysis tests. + +Built in code rather than committed as fixtures: what is being tested is whether the +analysis recovers a tempo and a key, and a test that states them in its own source +says what it expects more plainly than a binary file named after them. Seeded, so it +is the same audio on every run. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import numpy as np + +SAMPLE_RATE = 22050 +MAJOR = (0, 2, 4, 5, 7, 9, 11) +MINOR = (0, 2, 3, 5, 7, 8, 10) +PITCH_NAMES = ("C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B") + + +def _hz(midi: float) -> float: + return 440.0 * 2 ** ((midi - 69) / 12) + + +def _tone(freq: float, seconds: float, amp: float, decay: float) -> np.ndarray: + t = np.arange(int(seconds * SAMPLE_RATE)) / SAMPLE_RATE + wave = sum((0.6**h) * np.sin(2 * np.pi * freq * (h + 1) * t) for h in range(5)) + return amp * wave * np.exp(-decay * t) + + +def _kick() -> np.ndarray: + t = np.arange(int(0.25 * SAMPLE_RATE)) / SAMPLE_RATE + sweep = 50 + 100 * np.exp(-30 * t) + return 0.8 * np.sin(2 * np.pi * np.cumsum(sweep) / SAMPLE_RATE) * np.exp(-12 * t) + + +def _hit(rng, seconds: float, amp: float, decay: float) -> np.ndarray: + noise = np.diff(rng.standard_normal(int(seconds * SAMPLE_RATE)), prepend=0.0) + return amp * noise * np.exp(-decay * np.arange(noise.size) / SAMPLE_RATE) + + +def _add(buffer: np.ndarray, sound: np.ndarray, at: float) -> None: + start = max(0, int(at * SAMPLE_RATE)) + end = min(buffer.size, start + sound.size) + if start < end: + buffer[start:end] += sound[: end - start] + + +def _chord(scale, degree: int) -> list[int]: + return [scale[(degree + k) % 7] + 12 * ((degree + k) // 7) for k in (0, 2, 4)] + + +def song(bpm: float, tonic: int, mode: str, *, seconds: float = 30.0, drums: bool = True, + seed: int = 0) -> np.ndarray: + """A beat (kick, snare, hats), chords, bass and a melody, in one key. + + The minor progression uses a major V (i–iv–V–i), the raised leading tone that makes + a listener hear minor rather than its relative major. Without it a minor and a major + progression can share all four chords and the key is genuinely ambiguous. + """ + rng = np.random.default_rng(seed) + buffer = np.zeros(int(seconds * SAMPLE_RATE)) + beat = 60.0 / bpm + scale = MAJOR if mode == "major" else MINOR + progression = (0, 4, 5, 3) if mode == "major" else (0, 3, 4, 0) + root = 48 + tonic + + for index in range(int(seconds / beat)): + at = index * beat + bar, position = divmod(index, 4) + if position == 0: + degree = progression[bar % 4] + chord = _chord(scale, degree) + if mode == "minor" and degree == 4: + chord = [7, 11, 14] + for note in chord: + _add(buffer, _tone(_hz(root + 12 + note), beat * 4, 0.08, 0.4), at) + _add(buffer, _tone(_hz(root - 12 + scale[degree]), beat * 4, 0.15, 0.6), at) + melody = root + 24 + scale[int(rng.integers(0, 7))] + _add(buffer, _tone(_hz(melody), beat, 0.05, 3.0), at + rng.normal(0, 0.008)) + if drums: + if position in (0, 2): + _add(buffer, _kick(), at + rng.normal(0, 0.008)) + if position in (1, 3): + _add(buffer, _hit(rng, 0.15, 0.3, 25), at + rng.normal(0, 0.008)) + _add(buffer, _hit(rng, 0.05, 0.08, 60), at) + _add(buffer, _hit(rng, 0.05, 0.06, 60), at + beat / 2) + + return (buffer / np.max(np.abs(buffer)) * 0.7).astype(np.float32) + + +def pad(tonic: int, mode: str, *, seconds: float = 30.0, seed: int = 0) -> np.ndarray: + """Slow sustained chords with no beat at all: ambient, a drone under narration.""" + rng = np.random.default_rng(seed) + buffer = np.zeros(int(seconds * SAMPLE_RATE)) + scale = MAJOR if mode == "major" else MINOR + at, index = 0.0, 0 + while at < seconds: + length = float(rng.uniform(5, 9)) + envelope_t = np.arange(int(length * SAMPLE_RATE)) / SAMPLE_RATE + envelope = np.minimum(1, np.minimum(envelope_t / 2.0, (length - envelope_t) / 2.0)) + for note in _chord(scale, (0, 5, 3, 4)[index % 4]): + _add(buffer, _tone(_hz(48 + tonic + 12 + note), length, 0.1, 0.0) * envelope, at) + at += length - 1.5 + index += 1 + return (buffer / np.max(np.abs(buffer)) * 0.3).astype(np.float32) + + +def speechlike(*, seconds: float = 30.0, seed: int = 0) -> np.ndarray: + """Irregular voiced syllables with pauses: what a talk looks like to the analysis.""" + rng = np.random.default_rng(seed) + buffer = np.zeros(int(seconds * SAMPLE_RATE)) + at = 0.0 + while at < seconds: + syllable = float(rng.uniform(0.08, 0.3)) + voice = _tone(float(rng.uniform(100, 220)), syllable, 0.2, 4.0) + _add(buffer, voice * (1 + 0.3 * rng.standard_normal(voice.size)), at) + at += syllable + float(rng.uniform(0.02, 0.4)) + if rng.random() < 0.15: + at += float(rng.uniform(0.3, 1.0)) + return (buffer / np.max(np.abs(buffer)) * 0.5).astype(np.float32) + + +def write_mp3(samples: np.ndarray, target: Path) -> Path: + """Encode through ffmpeg, so a test goes through the same decode a real upload does.""" + subprocess.run( + [ + "ffmpeg", "-nostdin", "-v", "error", "-y", + "-f", "f32le", "-ar", str(SAMPLE_RATE), "-ac", "1", "-i", "-", + "-c:a", "libmp3lame", "-b:a", "96k", str(target), + ], + input=samples.astype(" calm.energy_score + assert beat.energy in ("medium", "high") and calm.energy == "low" + + +@pytest.mark.parametrize("seed", [0, 1, 2]) +def test_speech_is_not_music(seed): + """A talk must not be filed under a key and an energy.""" + assert not analyse(synth.speechlike(seed=seed)).is_music + + +def test_noise_and_silence_are_not_music(): + noise = (np.random.default_rng(0).standard_normal(synth.SAMPLE_RATE * 20) * 0.1).astype( + np.float32 + ) + assert not analyse(noise).is_music + silence = np.zeros(synth.SAMPLE_RATE * 20, dtype=np.float32) + assert analyse(silence) == analyse(silence) # no division by zero, no NaN + assert not analyse(silence).is_music + + +def test_too_short_to_measure_is_an_error(): + with pytest.raises(AudioAnalysisError): + analyse(np.zeros(synth.SAMPLE_RATE * 3, dtype=np.float32)) + + +def test_an_encoded_file_decodes_to_the_same_answer(tmp_path): + """Through ffmpeg and MP3 and back, the way an uploaded file is read.""" + path = synth.write_mp3(synth.song(120, 9, "minor"), tmp_path / "song.mp3") + samples = decode(path, duration_seconds=30.0) + assert samples.size == pytest.approx(30 * synth.SAMPLE_RATE, rel=0.02) + result = analyse(samples) + assert result.bpm == pytest.approx(120, abs=1.5) + assert result.key == "A minor" + + +# ─── the job and the API ───────────────────────────────────────────────────── + + +def _upload(client, path): + files = [("files", (path.name, path.read_bytes(), "audio/mpeg"))] + return client.post("/api/assets", files=files).json()["created"][0] + + +def _run_job(job_id, session, monkeypatch): + """Run a queued job against the test database (see test_attribution_api.py).""" + from app.jobs import enrichment as enrichment_jobs + + monkeypatch.setattr(enrichment_jobs.queue(), "engine", session.get_bind()) + enrichment_jobs._run_job(job_id) + session.expire_all() + + +def _job_for(session, asset_id, kind=KIND_ANALYZE_MUSIC): + return session.exec( + select(EnrichmentJob).where(EnrichmentJob.asset_id == asset_id, EnrichmentJob.kind == kind) + ).first() + + +def test_an_audio_upload_is_analysed_without_being_asked(library, session, monkeypatch, tmp_path): + asset = _upload(library, synth.write_mp3(synth.song(92, 7, "major"), tmp_path / "theme.mp3")) + + job = _job_for(session, asset["id"]) + assert job is not None and job.status == "queued" + _run_job(job.id, session, monkeypatch) + + stored = session.get(Asset, asset["id"]) + assert stored.music_bpm == pytest.approx(92, abs=1.5) + assert stored.music_key == "G major" + assert stored.music_energy in ("low", "medium", "high") + provenance = json.loads(stored.field_provenance) + assert provenance["music_bpm"] == PROVENANCE_ANALYSIS + assert session.get(EnrichmentJob, job.id).detail.startswith("92") + + +def test_a_tag_is_kept_over_a_measurement(library, session, monkeypatch): + """The ID3 fixture says 92 BPM and Am, and whoever tagged it heard the track.""" + from pathlib import Path + + fixture = Path(__file__).parent / "fixtures" / "music_audio.mp3" + files = [("files", (fixture.name, fixture.read_bytes(), "audio/mpeg"))] + created = library.post("/api/assets", files=files).json()["created"][0] + stored = session.get(Asset, created["id"]) + # One second of sine: too short to measure, so the job is not even queued. + assert _job_for(session, stored.id) is None + assert stored.music_key == "Am" + assert json.loads(stored.field_provenance)["music_key"] == PROVENANCE_EMBEDDED + + +def test_a_measurement_never_overwrites_a_hand_edit(library, session, monkeypatch, tmp_path): + asset = _upload(library, synth.write_mp3(synth.song(128, 0, "major"), tmp_path / "a.mp3")) + library.patch(f"/api/assets/{asset['id']}", json={"music_key": "C major", "music_bpm": 64}) + + _run_job(_job_for(session, asset["id"]).id, session, monkeypatch) + + stored = session.get(Asset, asset["id"]) + assert stored.music_bpm == 64 and stored.music_key == "C major" + # The blank one still fills. + assert stored.music_energy is not None + + +def test_speech_fills_nothing(library, session, monkeypatch, tmp_path): + asset = _upload(library, synth.write_mp3(synth.speechlike(), tmp_path / "talk.mp3")) + job = _job_for(session, asset["id"]) + _run_job(job.id, session, monkeypatch) + + stored = session.get(Asset, asset["id"]) + assert (stored.music_bpm, stored.music_key, stored.music_energy) == (None, None, None) + assert "No beat or key" in session.get(EnrichmentJob, job.id).detail + + +def test_the_button_analyses_one_track(library, session, tmp_path): + asset = _upload(library, synth.write_mp3(synth.song(100, 0, "major"), tmp_path / "a.mp3")) + # The upload queued one; a second while it is still queued is refused. + assert library.post(f"/api/assets/{asset['id']}/analyze-music").status_code == 409 + + session.delete(_job_for(session, asset["id"])) + session.commit() + response = library.post(f"/api/assets/{asset['id']}/analyze-music") + assert response.status_code == 202 + assert response.json()["data"]["action"] == KIND_ANALYZE_MUSIC + + +def test_only_audio_is_analysed(library): + from pathlib import Path + + video = Path(__file__).parent / "fixtures" / "sample_video.mp4" + files = [("files", (video.name, video.read_bytes(), "video/mp4"))] + asset = library.post("/api/assets", files=files).json()["created"][0] + response = library.post(f"/api/assets/{asset['id']}/analyze-music") + assert response.status_code == 400 + assert response.json()["detail"]["code"] == "not_analysable" + + +def test_the_library_run_fills_what_uploads_before_it_missed( + library, session, monkeypatch, tmp_path +): + song = _upload(library, synth.write_mp3(synth.song(120, 9, "minor"), tmp_path / "s.mp3")) + talk = _upload(library, synth.write_mp3(synth.speechlike(), tmp_path / "t.mp3")) + # As if uploaded before analysis existed: no job ran. + for job in session.exec(select(EnrichmentJob)).all(): + session.delete(job) + session.commit() + + response = library.post("/api/assets/analyze-music") + assert response.status_code == 202 + assert library.post("/api/assets/analyze-music").status_code == 409 + + job = session.exec( + select(EnrichmentJob).where(EnrichmentJob.kind == KIND_ANALYZE_MUSIC_LIBRARY) + ).one() + _run_job(job.id, session, monkeypatch) + + assert session.get(Asset, song["id"]).music_key == "A minor" + assert session.get(Asset, talk["id"]).music_key is None + detail = session.get(EnrichmentJob, job.id).detail + assert detail == "1 filled in of 2 analysed, 1 with no beat or key" diff --git a/docs/music-profile.md b/docs/music-profile.md index 31d9d00..676dfa9 100644 --- a/docs/music-profile.md +++ b/docs/music-profile.md @@ -1,6 +1,6 @@ # Music profile -**Status:** built. Step 1 of making the library useful as a soundtrack source for Gecko +**Status:** built, with analysis of the sound. Step 1 of making the library useful as a soundtrack source for Gecko Video Creator. GVC searches GAM for music, and until this an audio asset carried nothing that said what it sounds like, so a search could only match a track's title. @@ -20,8 +20,8 @@ On `Asset`, all nullable. Parsing and the field list live in `backend/app/music. ## Where values come from -**The file's own tags,** on upload and in the existing "re-read file metadata" job -(`harvest_attribution`), audio only. Read from the ffprobe tag block `probe.py` already +**The file's own tags,** on upload and in the "Read file tags" library run +(`harvest_attribution`, Settings → Fill in metadata), audio only. Read from the ffprobe tag block `probe.py` already collects, with nothing new to install: | Field | ID3 (MP3) | Vorbis (FLAC/Ogg) | MP4 | @@ -39,6 +39,31 @@ correction. Instruments and energy have no common tag, so they are typed by hand **URL import** carries a site's genre (SoundCloud and Bandcamp report one) onto audio imports. YouTube's `categories` are left out: they say "Music" for every song. +**Measured from the sound** (`backend/app/ingest/audio_analysis.py`), for files whose +tags say nothing. Tempo, key and energy only. Queued after every audio upload of a +song's length (8 s to 20 min), runnable per track from the Music tab, and across the +library from Settings → Fill in metadata. It fills blanks only, after the tags have been +read, and is stamped `analysis` in `field_provenance`, a fourth value beside human, ai and +embedded: no model guessed it and nobody wrote it into the file. + +| Measure | How | How good | +|---|---|---| +| Tempo | Autocorrelation of a log spectral-flux onset curve, scored over 50–210 BPM with its first four multiples, weighted by a log-normal prior at 110 BPM | Exact on synthetic beats from 92 to 150 BPM. Very slow or fast tracks can read an octave off (72 as 144, 170 as 85), the ambiguity every method has | +| Key | Pitch-class profile (8192-point FFT, 65 Hz–2.1 kHz, each frame normalised) correlated with the 24 Krumhansl–Kessler profiles | Exact on synthetic progressions with a harmonic-minor V; a progression that shares all its chords with its relative major is reported as either, which is what it is | +| Energy | Blend of loudness (−32…−10 dBFS), spectral centroid (0.6–3 kHz) and drive (tempo and beat strength; none without a beat) | Coarse by design: three levels | + +Two gates keep it from describing things that are not music. A tempo needs a beat +strength of 0.3 (a slow pad whose chords change every few seconds reached 0.24). Without +a beat, a key needs a correlation of 0.75 to count as music (pitched speech reached +0.65). When neither is found, nothing is written, which is what a talk or a podcast gets. + +Plain numpy over ffmpeg-decoded PCM, with no new dependency: librosa would bring numba, +llvmlite, scipy and soundfile, a lot of compiled wheels to keep matching the pinned +Python. It reads two minutes from past a 20-second intro and takes about a second. + +Not measured: vocals (telling singing from a lead instrument takes a trained model, and +a guess would put songs under narration), genre and mood. + **By hand,** in the asset's Music tab (audio assets only), or through `PATCH /api/assets/{id}`, where a list replaces the stored one and `[]` clears it. @@ -55,7 +80,6 @@ from a song can be its quiet intro. ## Not done yet -- GVC reading the profile: `search_assets` should return it to the assistant (step 2). -- Analysing the audio itself (BPM, energy, vocals, audio–text embeddings), for files - whose tags say nothing. +- Vocals detection, and audio–text embeddings (CLAP) so a track can be found by how it + sounds rather than by the words on it. Both need a trained model, and so torch. - Range filters (`bpm_min`/`bpm_max`) on `/api/assets`. diff --git a/frontend/src/api/assets.ts b/frontend/src/api/assets.ts index bd18644..fba4cfd 100644 --- a/frontend/src/api/assets.ts +++ b/frontend/src/api/assets.ts @@ -262,4 +262,12 @@ export const assetsApi = { .post>('/assets/harvest-attribution') .then((r) => r.data.data) }, + + /** Measure tempo, key and energy for every track still missing one. New uploads are + * analysed as they land; this is for the ones that came before. */ + analyzeMusicLibrary(): Promise { + return client + .post>('/assets/analyze-music') + .then((r) => r.data.data) + }, } diff --git a/frontend/src/api/enrichment.ts b/frontend/src/api/enrichment.ts index b571b3a..5872385 100644 --- a/frontend/src/api/enrichment.ts +++ b/frontend/src/api/enrichment.ts @@ -62,6 +62,14 @@ export interface DocumentPage { } export const enrichmentApi = { + /** Measure one track's tempo, key and energy from its sound. Fills blanks only, and + * calls no provider, so it needs nothing configured. */ + analyzeMusic(assetId: string): Promise { + return client + .post>(`/assets/${assetId}/analyze-music`) + .then((r) => r.data.data) + }, + summarize(assetId: string): Promise { return client .post>(`/assets/${assetId}/summarize`) diff --git a/frontend/src/components/ActivityIndicator.tsx b/frontend/src/components/ActivityIndicator.tsx index 7324432..c2f679a 100644 --- a/frontend/src/components/ActivityIndicator.tsx +++ b/frontend/src/components/ActivityIndicator.tsx @@ -21,6 +21,9 @@ const LABELS: Record = { extract_text: 'Reading text', import_url: 'Importing', generate: 'Generating', + harvest_attribution: 'Reading file metadata', + analyze_music: 'Analysing music', + analyze_music_library: 'Analysing your music', } function subject(job: ActivityJob): string { diff --git a/frontend/src/components/LibraryScanPanel.test.tsx b/frontend/src/components/LibraryScanPanel.test.tsx new file mode 100644 index 0000000..206e854 --- /dev/null +++ b/frontend/src/components/LibraryScanPanel.test.tsx @@ -0,0 +1,76 @@ +import { render, screen } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import LibraryScanPanel from '@/components/LibraryScanPanel' +import { assetsApi } from '@/api/assets' +import type { ActivityJob } from '@/api/transcripts' +import { useActivityStore } from '@/stores/activity' + +function job(overrides: Partial = {}): ActivityJob { + return { + id: 'j1', + kind: 'enrichment', + action: 'analyze_music_library', + status: 'processing', + stalled: false, + stage: 'Analysing music', + progress: 30, + detail: '', + asset_id: null, + asset_name: '', + model: '', + result_asset_id: null, + result_asset_ids: [], + error_message: null, + created_at: '2026-10-26T00:00:00Z', + updated_at: '2026-10-26T00:00:00Z', + ...overrides, + } +} + +const refresh = vi.fn().mockResolvedValue(undefined) + +beforeEach(() => { + refresh.mockClear() + useActivityStore.setState({ jobs: [], refresh }) +}) + +afterEach(() => vi.restoreAllMocks()) + +describe('LibraryScanPanel', () => { + it('starts a library-wide music analysis', async () => { + const start = vi.spyOn(assetsApi, 'analyzeMusicLibrary').mockResolvedValue(job()) + render() + + await userEvent.click(screen.getByRole('button', { name: 'Analyse' })) + + expect(start).toHaveBeenCalled() + expect(refresh).toHaveBeenCalled() + }) + + it('gives the tag harvest a button too', async () => { + const start = vi + .spyOn(assetsApi, 'harvestAttribution') + .mockResolvedValue(job({ action: 'harvest_attribution' })) + render() + + await userEvent.click(screen.getByRole('button', { name: 'Read tags' })) + expect(start).toHaveBeenCalled() + }) + + it('cannot be started twice while it runs', () => { + useActivityStore.setState({ jobs: [job()] }) + render() + + expect(screen.getByRole('button', { name: /Running/ })).toBeDisabled() + expect(screen.getByRole('button', { name: 'Read tags' })).toBeEnabled() + }) + + it('says what the last run did', () => { + useActivityStore.setState({ + jobs: [job({ status: 'done', detail: '12 filled in of 14 analysed' })], + }) + render() + expect(screen.getByText('Last run: 12 filled in of 14 analysed')).toBeInTheDocument() + }) +}) diff --git a/frontend/src/components/LibraryScanPanel.tsx b/frontend/src/components/LibraryScanPanel.tsx new file mode 100644 index 0000000..25aa05d --- /dev/null +++ b/frontend/src/components/LibraryScanPanel.tsx @@ -0,0 +1,119 @@ +import { useState } from 'react' +import { AudioWaveform, FileSearch, Loader2 } from 'lucide-react' +import type { LucideIcon } from 'lucide-react' +import { assetsApi } from '@/api/assets' +import { apiErrorMessage } from '@/api/client' +import type { ActivityJob } from '@/api/transcripts' +import { isActive, useActivityStore } from '@/stores/activity' + +/** + * The two library-wide passes that fill metadata in without asking anything of anyone. + * + * Both already happen to every new upload. These buttons are for the files that were in + * the library before each existed, which on a long-running instance is most of it. Both + * only ever fill blanks, so pressing either twice, or after hand corrections, is safe; + * the copy says so, because "will this overwrite what I fixed?" is the question that + * stops people pressing a library-wide button. + */ + +interface ScanSpec { + action: string + icon: LucideIcon + title: string + body: string + button: string + start: () => Promise +} + +const SCANS: ScanSpec[] = [ + { + action: 'harvest_attribution', + icon: FileSearch, + title: 'Read file tags', + body: + 'Fills credits from EXIF, ID3 and document properties, and genre, tempo, key and ' + + 'mood from music tags.', + button: 'Read tags', + start: () => assetsApi.harvestAttribution(), + }, + { + action: 'analyze_music_library', + icon: AudioWaveform, + title: 'Analyse music', + body: + 'Measures tempo, key and energy from the sound of every track still missing one. ' + + 'Files with no beat or key, such as talks, are left alone.', + button: 'Analyse', + start: () => assetsApi.analyzeMusicLibrary(), + }, +] + +function Scan({ spec }: { spec: ScanSpec }) { + const job = useActivityStore((s) => s.jobs.find((j) => j.action === spec.action)) + const refreshActivity = useActivityStore((s) => s.refresh) + const [error, setError] = useState(null) + const running = job ? isActive(job) : false + const Icon = spec.icon + + const start = async () => { + setError(null) + try { + await spec.start() + await refreshActivity() + } catch (err) { + setError(apiErrorMessage(err, 'Could not start')) + } + } + + return ( +
+
+

+ + {spec.title} +

+

{spec.body}

+ {job && !running && job.detail && ( +

+ Last run: {job.detail} +

+ )} + {error &&

{error}

} +
+ +
+ ) +} + +export default function LibraryScanPanel() { + return ( +
+
+

+ Fill in metadata +

+

+ New uploads get both of these automatically. Run them once for files that were + here before. Each fills only empty fields, so nothing you have typed is changed. +

+
+ {SCANS.map((spec) => ( + + ))} +
+ ) +} diff --git a/frontend/src/components/MusicPanel.test.tsx b/frontend/src/components/MusicPanel.test.tsx index 3b0c0ed..5699ac3 100644 --- a/frontend/src/components/MusicPanel.test.tsx +++ b/frontend/src/components/MusicPanel.test.tsx @@ -1,7 +1,10 @@ import { render, screen } from '@testing-library/react' import userEvent from '@testing-library/user-event' +import { MemoryRouter } from 'react-router-dom' +import type React from 'react' import { beforeEach, describe, expect, it, vi } from 'vitest' import MusicPanel from '@/components/MusicPanel' +import { enrichmentApi } from '@/api/enrichment' import { splitTerms } from '@/utils/terms' import { useLibraryStore } from '@/stores/library' import type { Asset } from '@/api/assets' @@ -40,6 +43,11 @@ function asset(overrides: Partial = {}): Asset { const update = vi.fn().mockResolvedValue(undefined) +// The measure button renders a Link (to Settings, on a provider error), so needs a router. +function renderPanel(panel: React.ReactElement) { + return render({panel}) +} + beforeEach(() => { update.mockClear() useLibraryStore.setState({ update }) @@ -59,7 +67,7 @@ describe('splitTerms', () => { describe('MusicPanel', () => { it('shows what the file tags already said', () => { - render( + renderPanel( { it('sends only the fields that changed', async () => { // A BPM read from the file must keep its "embedded" stamp until someone edits it. const user = userEvent.setup() - render() + renderPanel() await user.type(screen.getByLabelText('Moods'), 'Tense, dark') await user.selectOptions(screen.getByLabelText('Energy'), 'low') @@ -93,7 +101,9 @@ describe('MusicPanel', () => { it('clears a field back to not known', async () => { const user = userEvent.setup() - render() + renderPanel( + + ) await user.selectOptions(screen.getByLabelText('Vocals'), '') await user.clear(screen.getByLabelText('Genres')) @@ -102,9 +112,42 @@ describe('MusicPanel', () => { expect(update).toHaveBeenCalledWith('a1', { music_genres: [], music_vocals: null }) }) + it('measures the track from its sound on request', async () => { + const start = vi.spyOn(enrichmentApi, 'analyzeMusic').mockResolvedValue({ + id: 'j1', + kind: 'enrichment', + action: 'analyze_music', + status: 'queued', + stalled: false, + stage: 'Queued', + progress: 0, + detail: '', + asset_id: 'a1', + asset_name: 'Caravan theme', + model: '', + result_asset_id: null, + result_asset_ids: [], + error_message: null, + created_at: '2026-10-26T00:00:00Z', + updated_at: '2026-10-26T00:00:00Z', + }) + const user = userEvent.setup() + renderPanel() + + await user.click( + screen.getByRole('button', { name: /Measure tempo, key and energy/ }) + ) + expect(start).toHaveBeenCalledWith('a1') + }) + + it('offers no measuring on a clip, which has no sound of its own', () => { + renderPanel() + expect(screen.queryByRole('button', { name: /Measure/ })).not.toBeInTheDocument() + }) + it('will not save a tempo that is not one', async () => { const user = userEvent.setup() - render() + renderPanel() await user.type(screen.getByLabelText('Tempo (BPM)'), '2019') diff --git a/frontend/src/components/MusicPanel.tsx b/frontend/src/components/MusicPanel.tsx index bc0e7e7..784cdbe 100644 --- a/frontend/src/components/MusicPanel.tsx +++ b/frontend/src/components/MusicPanel.tsx @@ -1,8 +1,10 @@ import { useEffect, useMemo, useState } from 'react' import type { KeyboardEvent } from 'react' -import { Check } from 'lucide-react' +import { AudioWaveform, Check } from 'lucide-react' import { MUSIC_ENERGY_LEVELS } from '@/api/assets' import type { Asset, AssetUpdate, MusicEnergy } from '@/api/assets' +import { enrichmentApi } from '@/api/enrichment' +import EnrichmentButton from '@/components/EnrichmentButton' import { useLibraryStore } from '@/stores/library' import { splitTerms } from '@/utils/terms' import { useSavedFlash } from '@/utils/useSavedFlash' @@ -141,6 +143,23 @@ export default function MusicPanel({ asset }: Props) { Anything the file's own tags said is already filled in.

+ {/* Measures from the sound and fills only what is blank, so it can never undo a + correction. Tempo, key and energy only: vocals, genre and mood need a person. */} + {!asset.parent_asset_id && ( + + )} +

+ Measuring fills only empty fields. Clear one first to measure it again. +

+ {LIST_FIELDS.map((field) => (