From 39bbe4c7bc6fb47d175b0dc66d93026226ca4de5 Mon Sep 17 00:00:00 2001 From: Crowvic <168686597+MattModeCode@users.noreply.github.com> Date: Wed, 9 Sep 2026 10:05:19 -0400 Subject: [PATCH] feat(tuning): search the delivery knobs against real recordings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `double-chin tune`, which searches exaggeration, cfg_weight, temperature and rate for the values that make synthesized takes least distinguishable from the user's own recordings, and wires the result into the app as the profile Studio opens with. - src/double_chin/delivery.py: validated DeliveryProfile, the neutral and tuned profiles, and atomic persistence to ~/.double-chin/delivery.json with a logged fallback so a corrupt settings file cannot stop the app from opening. - src/double_chin/tuning/: coordinate-descent search scoring each candidate as a set of takes through the existing indistinguishability gate, with a measured noise floor, a held-out check, a resumable JSONL ledger and a synthesis budget. What the search ranks on is a parameter, because the gate's discrimination component is saturated in this configuration. - GET /api/delivery serves the opening profile and the neutral baseline, so the frontend stops hardcoding knob values; "Match my voice" restores the tuned profile and "Reset to neutral" goes to the engine baseline. Result on the owner's voice (docs/tuning.md): a tie for expression, adherence and variation — the tuning-script winner did not survive the held-out check — so the tuned profile is the neutral one. Speaking rate is the one clear finding and it is negative: any value other than 1.00x drops speaker similarity from ~0.66 to ~0.40. Claude-Session: https://claude.ai/code/session_01FdWprv4TKfVzapYFJnRffz --- README.md | 15 + docs/design.md | 48 +++ docs/tuning.md | 157 ++++++++++ src/double_chin/cli.py | 119 +++++++- src/double_chin/delivery.py | 183 ++++++++++++ src/double_chin/studio/app.py | 17 ++ src/double_chin/studio/static/api.js | 1 + src/double_chin/studio/static/app.js | 59 +++- src/double_chin/studio/static/index.html | 3 +- src/double_chin/studio/static/style.css | 7 +- src/double_chin/tuning/__init__.py | 31 ++ src/double_chin/tuning/evalset.py | 68 +++++ src/double_chin/tuning/runner.py | 223 ++++++++++++++ src/double_chin/tuning/scripts.py | 32 ++ src/double_chin/tuning/sweep.py | 364 +++++++++++++++++++++++ tests/test_delivery.py | 140 +++++++++ tests/test_e2e.py | 50 ++++ tests/test_studio.py | 90 ++++++ tests/test_tuning.py | 236 +++++++++++++++ 19 files changed, 1830 insertions(+), 13 deletions(-) create mode 100644 docs/tuning.md create mode 100644 src/double_chin/delivery.py create mode 100644 src/double_chin/tuning/__init__.py create mode 100644 src/double_chin/tuning/evalset.py create mode 100644 src/double_chin/tuning/runner.py create mode 100644 src/double_chin/tuning/scripts.py create mode 100644 src/double_chin/tuning/sweep.py create mode 100644 tests/test_delivery.py create mode 100644 tests/test_tuning.py diff --git a/README.md b/README.md index 379aab2..21b7d9e 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,21 @@ double-chin train yourvoice path/to/recordings/ --manifest path/to/recordings/ma double-chin say "Hello, world." --voice yourvoice -o out.wav --verify ``` +Then tune how that voice is *delivered*: + +```bash +double-chin tune yourvoice --real-dir path/to/recordings/ --write-defaults +``` + +`tune` searches the four delivery controls — expression, reference adherence, +variation and speaking rate — synthesizing several takes per candidate and +scoring each set against your own recordings with the indistinguishability +gate. It prints the winning values and the run-to-run noise floor, so a tie +never gets reported as a win, and `--write-defaults` saves the winner as the +values the app opens with. In the window, **Match my voice** puts the sliders +back on that profile; **Reset to neutral** returns to the engine's own +baseline. Method and results: [docs/tuning.md](docs/tuning.md). + Every enrolled voice lives only in `~/.double-chin/voices/`; deleting that folder revokes it entirely. Don't clone a voice you don't have the right to clone — every output carries an inaudible [Perth watermark](https://github.com/resemble-ai/chatterbox#watermarking), but it's a provenance signal, not a control. ## Licences diff --git a/docs/design.md b/docs/design.md index 4b31180..01c84d2 100644 --- a/docs/design.md +++ b/docs/design.md @@ -257,3 +257,51 @@ Load-bearing decisions, each verifiable in `tests/test_studio.py`: **Gate integration:** Studio will eventually embed a gate-running UI; for now, `double-chin gate VOICE` runs the full suite from the CLI. Known limits, honestly: single-process, single-user by design; no job cancellation (kill the server); no ASR/content check on outputs (inherited from §8 — the upgrade path stands); SSE drops don't kill a job (state is polled/replayed from `/api/jobs/{id}`) but the UI tells you to check History rather than pretending nothing happened. Every claim in this section survived a dedicated second-round red team; its findings and the code fixes are in [red-team.md](red-team.md). + +## 11. Delivery tuning + +The engine exposes four delivery controls — `exaggeration` (Expression), +`cfg_weight` (Reference adherence), `temperature` (Variation) and `rate` +(Speaking rate). Until now their defaults were Chatterbox's own, which are +not the values that best match any particular voice. `double-chin tune` +searches them. + +Method, and the reasoning behind each choice: + +- **Score with the existing gate, not a new metric.** Each candidate is + synthesized several times and the resulting *set* of clips is scored against + a set of the owner's real recordings with `indistinguishability_gate`. +- **Rank on the components that respond.** With eight real clips against three + clone clips, the gate's `discrimination` component sits at 0.0 for every + candidate — the classifier separates the sets outright, and a component that + is constant cannot order anything, it only makes the composite unreadable. + The search therefore ranks on speaker similarity, naturalness and prosody + combined with the gate's own weights, and records the full composite for + every candidate anyway. The objective is a parameter of the search + (`SweepContext.objective`), not a hardcoded assumption. +- **Repeats, and a noise floor.** Generation is stochastic, so the baseline is + re-measured several times with different seeds before the search starts. A + winner that does not beat the baseline by more than that spread is reported + as a tie, not a win. +- **Plain scripts.** The eval scripts carry no prosody markup: `*emphasis*` + adjusts exaggeration and cfg_weight per chunk and clamps at the knob's range, + which would distort exactly the candidates near the ends of the grid. +- **A held-out check.** The winner is re-scored on a second script and a + disjoint set of real clips, which is what catches a profile fitted to one + passage. +- **A resumable ledger.** Every measurement is appended to + `/tuning//ledger.jsonl`, keyed by profile, script and repeat, so + an interrupted run continues rather than restarting. + +The winner is stored as `delivery.TUNED_PROFILE`, which is what Studio opens +with and what "Match my voice" restores; `GET /api/delivery` serves it (or the +user's saved `~/.double-chin/delivery.json` over it) alongside the neutral +baseline, so the numbers live in one place instead of being duplicated in the +frontend. + +**Outcome on the owner's voice:** a tie for three of the four knobs — no +setting of expression, adherence or variation beat the defaults on held-out +text, so the tuned profile is the baseline. Speaking rate is the one clear +result, and it is negative: any value other than 1.00x drops speaker +similarity from ~0.66 to ~0.40, because the time-stretch is applied to +finished audio. Run, results and honest limits: [tuning.md](tuning.md). diff --git a/docs/tuning.md b/docs/tuning.md new file mode 100644 index 0000000..e3177d8 --- /dev/null +++ b/docs/tuning.md @@ -0,0 +1,157 @@ +# Delivery tuning — method and results + +`double-chin tune` searches the four delivery controls for the values that +make synthesized takes least distinguishable from your own recordings. This +page records the run against the owner's voice, and what it found. + +**Headline: three of the four knobs came back a tie, and the fourth has a +large, one-sided result.** Expression, reference adherence and variation could +not be shown to beat the engine's own defaults on held-out text — every +difference sat inside run-to-run noise. Speaking rate is different: any value +other than 1.00× measurably damages the voice. + +## Setup + +| | | +|---|---| +| Voice | `owner` (LoRA fine-tuned, 20 s reference) | +| Machine | M-series Mac, MPS, 2026-09-09 | +| Real clips | 8 recordings for the search, 6 disjoint ones for the held-out check | +| Scripts | two plain-prose passages, ~10 s of audio each | +| Takes per candidate | 3, deterministic seeds derived from the candidate key | +| Candidates measured | 24 on the tuning script, plus 2 held-out checks | +| Cost | ~5.5 min per candidate (3 syntheses + one gate run) | + +Recordings 001–010 are excluded from both sets: `reference.wav` is built from +the start of the corpus and caps at 20 s, so scoring against them would compare +the clone to its own conditioning audio. + +The eval scripts carry no prosody markup. `*emphasis*` raises exaggeration and +lowers cfg_weight for the chunk it spans and clamps at the knob's range, which +would distort exactly the candidates near the ends of the grid. Marks layer on +top of whatever profile wins; they are not part of what is being measured. + +## What is being maximized + +Each candidate's three takes are scored as a *set* against the real clips with +`verification.gate.indistinguishability_gate`. The gate's own composite turns +out to be unusable as a ranking signal here: its `discrimination` component is +**0.000 for every candidate**, because with eight real clips against three +clone clips the classifier separates the two sets outright. A weighted +geometric mean with a zero in it collapses — every candidate scores ~0.026 — +and a component that never varies cannot order anything. + +So the search ranks on the three components that do respond, using the gate's +own weights: + +``` +objective = weighted geometric mean of + speaker_similarity (0.30), naturalness (0.25), prosody (0.20) +``` + +The full gate result is still recorded for every candidate. The objective is a +parameter of the search (`SweepContext.objective`), not a hidden assumption. + +## Noise floor + +The baseline profile was measured three times with different seeds before the +search started: + +| repeat | objective | speaker | prosody | +|---|---|---|---| +| v0 | 0.7777 | 0.637 | 0.766 | +| v1 | 0.7698 | 0.632 | 0.747 | +| v2 | 0.7943 | 0.673 | 0.763 | + +**mean 0.781, sd 0.012.** A candidate has to beat 0.781 by more than 0.012 to +be worth anything. + +## Results + +Best candidates on the tuning script (all at rate 1.00×): + +| exaggeration | cfg_weight | temperature | objective | speaker | prosody | +|---|---|---|---|---|---| +| 0.50 | 0.80 | 0.45 | **0.8018** | 0.668 | 0.800 | +| 0.50 | 0.20 | 0.80 | 0.7991 | 0.664 | 0.798 | +| 0.50 | 0.80 | 0.50 | 0.7942 | 0.657 | 0.792 | +| 0.50 | 0.80 | 0.80 | 0.7881 | 0.656 | 0.771 | +| 0.50 | 0.50 | 0.80 | 0.7806 (baseline) | 0.647 | 0.759 | +| 0.50 | 0.80 | 0.65 | 0.7625 | 0.621 | 0.740 | + +Every rate-1.00× candidate measured, best to worst, spans 0.7625 to 0.8018 — a +range of 0.039, or about three standard deviations of the noise floor, across +the entire grid. Individual differences are one to one and a half sd. + +The search's winner (0.50 / 0.80 / 0.45 / 1.00×) beat the baseline by +0.021 on +the tuning script, which clears the noise floor. It then **failed the held-out +check**: + +| profile | tuning script | held-out script + held-out clips | +|---|---|---| +| winner 0.50 / 0.80 / 0.45 | 0.8018 | 0.8104 | +| baseline 0.50 / 0.50 / 0.80 | 0.7806 | **0.8213** | + +On unseen text and unseen real clips the baseline scores *higher* than the +winner. The +0.021 was fitted to the tuning passage. Note also that the +baseline moved 0.781 → 0.821 between conditions: the variance between +script/clip-set conditions (~0.04) is larger than any parameter effect measured +within one (~0.02). This measurement cannot resolve a winner among those three +knobs, and reporting one anyway would be reporting noise. + +### Speaking rate is the exception + +| rate | objective | speaker similarity | +|---|---|---| +| 1.00× | 0.7942 | 0.657 | +| 0.95× | 0.6556 | 0.421 | +| 1.05× | 0.6182 | 0.403 | +| 1.10× | 0.6483 | 0.389 | + +Every departure from 1.00× costs 0.13–0.18 objective — roughly ten standard +deviations — and speaker similarity falls from ~0.66 to ~0.40. That is the +expected consequence of how the knob works: rate is a pitch-preserving +time-stretch applied to *finished* audio (`engine.synthesize`), so it resamples +away formant and timing detail the speaker embedding relies on. It is a real +control for pacing, and it is the wrong tool for sounding more like yourself. + +## Conclusion + +`delivery.TUNED_PROFILE` is set to **0.50 / 0.50 / 0.80 / 1.00×** — identical to +the neutral baseline, because nothing beat it out of sample. Studio opens on +that profile, "Match my voice" restores it, and "Reset to neutral" goes to the +same place today. The two buttons diverge as soon as a run on a different +corpus writes a different profile with `tune --write-defaults`. + +The plausible reading of this result: the LoRA fine-tune is already doing the +voice matching, and the delivery knobs have little headroom left on top of it. +That is a finding about this voice, not a defect in the search. + +## Limits + +- 24 candidates, one coordinate pass plus refinement — a fuller search might + find a real effect, though the noise floor sets a hard limit on how small an + effect this method can resolve at all. +- Three takes per candidate. More repeats would shrink the noise floor; the + cost is linear. +- ~10 s clips. Longer clips give the prosody metric more to work with and cost + proportionally more. +- The gate composite is unreadable in this configuration (~0.026 for + everything) because of the collapsed `discrimination` component. Compare + candidates on the objective, and use `double-chin gate` — which uses a + balanced set — for an absolute verdict on a voice. +- The real clips were part of the fine-tune's training corpus, so the absolute + similarity numbers are optimistic. This affects every candidate equally, so + it does not bias the ranking. + +## Re-running + +```bash +double-chin tune owner --real-dir Recording-scripts-audio --takes 3 --budget 120 +``` + +Add `--write-defaults` to save the winner to `~/.double-chin/delivery.json`, +which is what Studio then opens with. Measurements are appended to +`~/.double-chin/tuning//ledger.jsonl` and reused on the next run, so an +interrupted sweep continues where it stopped and re-running costs only the +candidates it has not seen. diff --git a/src/double_chin/cli.py b/src/double_chin/cli.py index 0b307fe..85dc109 100644 --- a/src/double_chin/cli.py +++ b/src/double_chin/cli.py @@ -22,7 +22,7 @@ # ahead of these, so `double-chin verify` (arbitrary wav files, no storage) and # argparse-level exits (--help, unknown/missing command) never trigger it. _STORAGE_COMMANDS = frozenset( - {"enroll", "say", "train", "voices", "doctor", "studio", "app", "gate"} + {"enroll", "say", "train", "voices", "doctor", "studio", "app", "gate", "tune"} ) # Default script synthesized for `double-chin gate VOICE` when no clone clips are @@ -142,6 +142,47 @@ def build_parser() -> argparse.ArgumentParser: "--json", action="store_true", help="Emit the raw result dict as JSON." ) + tune_parser = subparsers.add_parser( + "tune", + help=( + "Search the four delivery knobs for the values that score best " + "against your own recordings, and print the winner." + ), + ) + tune_parser.add_argument("name", help="Name of an already-enrolled voice.") + tune_parser.add_argument( + "--real-dir", type=Path, required=True, + help="Directory of genuine recordings to score against.", + ) + tune_parser.add_argument( + "--takes", type=int, default=3, + help="Clips synthesized per candidate (default: 3).", + ) + tune_parser.add_argument( + "--budget", type=int, default=240, + help=( + "Stop after this many syntheses and report the best so far. " + "Counted across resumes, since the ledger persists." + ), + ) + tune_parser.add_argument( + "--passes", type=int, default=2, help="Coordinate-descent passes (default: 2)." + ) + tune_parser.add_argument( + "--work-dir", type=Path, default=None, + help="Where takes and the resumable ledger live (default: /tuning).", + ) + tune_parser.add_argument( + "--write-defaults", action="store_true", + help="Save the winner as the defaults Studio opens with.", + ) + tune_parser.add_argument( + "--device", default=None, help="Force a device (mps/cuda/cpu)." + ) + tune_parser.add_argument( + "--json", action="store_true", help="Emit the full result as JSON." + ) + subparsers.add_parser("doctor", help="Report environment diagnostics.") studio_parser = subparsers.add_parser( @@ -355,6 +396,81 @@ def _cmd_gate(args: argparse.Namespace) -> int: return 0 if result["passed"] else 1 +def _cmd_tune(args: argparse.Namespace) -> int: + import json + + from double_chin.config import double_chin_home + from double_chin.delivery import NEUTRAL_PROFILE, save_defaults + from double_chin.tuning import run_sweep, score + from double_chin.tuning.runner import ( + build_context, + delivery_objective, + format_report, + prepare_sets, + ) + from double_chin.tuning.scripts import TUNING_SCRIPTS + from double_chin.voices import get_voice + + voice = get_voice(args.name) + work_dir = args.work_dir or (double_chin_home() / "tuning" / voice.name) + + print(f"double-chin: preparing real clips from {args.real_dir}", file=sys.stderr) + tuning_clips, holdout_clips = prepare_sets(args.real_dir, work_dir) + + def on_event(kind: str, data: dict) -> None: + if kind == "scored": + values = " ".join(f"{k}={v:g}" for k, v in data["profile"].items()) + print( + f" obj {delivery_objective(data):.4f} gate {data['score']:.3f} " + f"{values}", + file=sys.stderr, + ) + + ctx = build_context( + voice=voice, + real_clips=tuning_clips, + scripts=TUNING_SCRIPTS, + work_dir=work_dir, + takes=args.takes, + budget=args.budget, + device=args.device, + on_event=on_event, + ) + result = run_sweep(ctx, start=NEUTRAL_PROFILE, script_id="tuning", passes=args.passes) + + # Re-score winner and baseline on the held-out script and the held-out + # real clips: the only check that the winner is not overfit to one passage. + ctx.real_clips = holdout_clips + ctx.budget = None + holdout = { + "best": delivery_objective(score(ctx, result.best, "holdout")), + "baseline": delivery_objective(score(ctx, result.baseline, "holdout")), + } + + if args.json: + print(json.dumps( + { + "best": result.best.as_dict(), + "best_score": result.best_score, + "baseline": result.baseline.as_dict(), + "baseline_score": result.baseline_score, + "noise_floor": result.noise_floor, + "beats_noise": result.beats_noise, + "holdout": holdout, + "synthesis_count": result.synthesis_count, + }, + indent=2, + )) + else: + print(format_report(result, holdout)) + + if args.write_defaults: + path = save_defaults(result.best) + print(f"\nSaved as the opening defaults: {path}") + + return 0 + + def _which_in(directories: list[str], name: str) -> str | None: for directory in directories: candidate = Path(directory) / name @@ -453,6 +569,7 @@ def _cmd_app(args: argparse.Namespace) -> int: "voices": _cmd_voices, "verify": _cmd_verify, "gate": _cmd_gate, + "tune": _cmd_tune, "doctor": _cmd_doctor, "studio": _cmd_studio, "app": _cmd_app, diff --git a/src/double_chin/delivery.py b/src/double_chin/delivery.py new file mode 100644 index 0000000..e9a9ac2 --- /dev/null +++ b/src/double_chin/delivery.py @@ -0,0 +1,183 @@ +"""Delivery profiles: the four knobs, their tuned values, and where they persist. + +The engine exposes four delivery controls (exaggeration, cfg_weight, +temperature, rate) as bare defaults. This module gives them a name, a +validated shape, and two named profiles: + +* `NEUTRAL_PROFILE` — Chatterbox's own baseline, what "Reset" returns to. +* `TUNED_PROFILE` — the outcome of the sweep in `docs/tuning.md`, which is + what Studio opens with and what "Match my voice" restores. On the owner's + own corpus that sweep came back a tie, so the two profiles currently hold + the same values; `tune --write-defaults` is how a different corpus moves + the opening profile without touching this constant. + +The user can save their own defaults over the tuned profile +(`~/.double-chin/delivery.json`); a missing, corrupt, or out-of-range file +falls back to `TUNED_PROFILE` with a logged reason rather than failing, since +a bad settings file must never stop the app from opening. +""" + +from __future__ import annotations + +import json +import logging +import os +import tempfile +from dataclasses import asdict, dataclass +from pathlib import Path + +from double_chin.config import double_chin_home, ensure_double_chin_home +from double_chin.engine import ( + DEFAULT_CFG_WEIGHT, + DEFAULT_EXAGGERATION, + DEFAULT_RATE, + DEFAULT_TEMPERATURE, + MAX_RATE, + MIN_RATE, +) + +_log = logging.getLogger(__name__) + +DEFAULTS_FILENAME = "delivery.json" + +# (minimum, maximum, inclusive-minimum) per field, mirroring the bounds the +# Studio API already enforces in `studio/app.py` — temperature is the one +# exclusive lower bound, since 0 is not a usable sampling temperature. +_BOUNDS = { + "exaggeration": (0.0, 1.0, True), + "cfg_weight": (0.0, 1.0, True), + "temperature": (0.0, 2.0, False), + "rate": (MIN_RATE, MAX_RATE, True), +} + + +@dataclass(frozen=True) +class DeliveryProfile: + """One complete set of delivery-knob values.""" + + exaggeration: float + cfg_weight: float + temperature: float + rate: float + + def as_dict(self) -> dict[str, float]: + return asdict(self) + + def validate(self) -> None: + """Raise ValueError if any field is non-numeric or out of range.""" + for name, (low, high, inclusive) in _BOUNDS.items(): + value = getattr(self, name) + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be a number, got {value!r}") + too_low = value < low if inclusive else value <= low + if too_low or value > high: + edge = ">=" if inclusive else ">" + raise ValueError( + f"{name} must be {edge} {low} and <= {high}, got {value}" + ) + + @classmethod + def from_mapping(cls, data) -> "DeliveryProfile": + """Build a validated profile from a mapping, ignoring unknown keys. + + Raises: + ValueError: if a field is missing, non-numeric, or out of range. + """ + if not isinstance(data, dict): + raise ValueError(f"expected an object of delivery values, got {type(data)}") + missing = [name for name in _BOUNDS if name not in data] + if missing: + raise ValueError(f"missing delivery values: {', '.join(missing)}") + + profile = cls(**{name: data[name] for name in _BOUNDS}) + profile.validate() + return profile + + +# Chatterbox's own baseline. "Reset to defaults" returns here, so the user +# always has one keystroke back to untuned behaviour. +NEUTRAL_PROFILE = DeliveryProfile( + exaggeration=DEFAULT_EXAGGERATION, + cfg_weight=DEFAULT_CFG_WEIGHT, + temperature=DEFAULT_TEMPERATURE, + rate=DEFAULT_RATE, +) + +# Outcome of the sweep against the owner's real recordings (docs/tuning.md): +# no setting of expression, adherence or variation beat the neutral baseline +# on held-out text — every difference sat inside run-to-run noise — so the +# tuned profile *is* the baseline for those three. `rate` is the one knob with +# a large, unambiguous result: any value other than 1.0 wrecks speaker +# similarity (0.67 -> 0.39 at 1.05x), because the time-stretch is applied to +# finished audio. This constant is what Studio opens with and what "Match my +# voice" restores; re-run `double-chin tune --write-defaults` on your own +# recordings to override it per machine. +TUNED_PROFILE = DeliveryProfile( + exaggeration=0.50, + cfg_weight=0.50, + temperature=0.80, + rate=1.00, +) + + +def defaults_path() -> Path: + """Where the user's saved delivery defaults live.""" + return double_chin_home() / DEFAULTS_FILENAME + + +def load_defaults() -> DeliveryProfile: + """Return the profile Studio should open with. + + Falls back to `TUNED_PROFILE` — logging why — when no saved file exists, + or when the one on disk is unreadable, malformed, or out of range. + """ + path = defaults_path() + if not path.is_file(): + return TUNED_PROFILE + + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + _log.warning("ignoring unreadable %s: %s", DEFAULTS_FILENAME, exc) + return TUNED_PROFILE + + try: + return DeliveryProfile.from_mapping(data) + except ValueError as exc: + _log.warning("ignoring invalid %s: %s", DEFAULTS_FILENAME, exc) + return TUNED_PROFILE + + +def save_defaults(profile: DeliveryProfile) -> Path: + """Persist `profile` as the opening defaults, atomically. + + The write goes to a temporary file in the same directory and is renamed + into place, so an interrupted save can never leave a half-written file + where the app expects settings. + + Raises: + ValueError: if the profile is out of range (nothing is written). + OSError: if the write or rename fails (the previous file survives). + """ + profile.validate() + home = ensure_double_chin_home() + target = defaults_path() + + handle = tempfile.NamedTemporaryFile( + "w", dir=home, prefix=".delivery-", suffix=".tmp", delete=False + ) + tmp_path = Path(handle.name) + try: + with handle: + json.dump(profile.as_dict(), handle, indent=2) + handle.write("\n") + os.replace(tmp_path, target) + except OSError: + tmp_path.unlink(missing_ok=True) + raise + return target + + +def clear_defaults() -> None: + """Delete any saved defaults, so the app falls back to `TUNED_PROFILE`.""" + defaults_path().unlink(missing_ok=True) diff --git a/src/double_chin/studio/app.py b/src/double_chin/studio/app.py index 7937a95..519e49c 100644 --- a/src/double_chin/studio/app.py +++ b/src/double_chin/studio/app.py @@ -242,6 +242,23 @@ async def stream(): headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, ) + @app.get("/api/delivery") + def get_delivery(): + """The delivery values the page opens with, and the neutral baseline. + + `defaults` is what the sliders hydrate from and what "Match my voice" + restores — the tuned profile, or whatever `tune --write-defaults` + saved over it. `neutral` is Chatterbox's own baseline, what "Reset" + returns to. Both live server-side so the frontend never hardcodes a + number. + """ + from double_chin.delivery import NEUTRAL_PROFILE, load_defaults + + return { + "defaults": load_defaults().as_dict(), + "neutral": NEUTRAL_PROFILE.as_dict(), + } + @app.get("/api/history") def get_history(): records = history.read_history() diff --git a/src/double_chin/studio/static/api.js b/src/double_chin/studio/static/api.js index 4a05e36..7ae3e2a 100644 --- a/src/double_chin/studio/static/api.js +++ b/src/double_chin/studio/static/api.js @@ -26,6 +26,7 @@ export const api = { headers: { "Content-Type": "application/json" }, body: JSON.stringify(payload), }), + delivery: () => request("/api/delivery"), listHistory: () => request("/api/history"), doctor: () => request("/api/doctor"), jobEvents: (jobId) => new EventSource(`/api/jobs/${jobId}/events`), diff --git a/src/double_chin/studio/static/app.js b/src/double_chin/studio/static/app.js index a879086..a4466c0 100644 --- a/src/double_chin/studio/static/app.js +++ b/src/double_chin/studio/static/app.js @@ -18,9 +18,21 @@ import { verdictTone, } from "./format.js"; -const DELIVERY_DEFAULTS = Object.freeze({ +/* Slider id -> the delivery field it carries. The values are deliberately not + * hardcoded here: /api/delivery is the single source of truth for what the + * page opens with and for the two presets. PLACEHOLDER_PROFILE only mirrors + * the `value=` attributes in the HTML, for the moment before that fetch + * lands (or if it fails outright). */ +const KNOB_FIELDS = Object.freeze({ + exaggeration: "exaggeration", + cfg: "cfg_weight", + temperature: "temperature", + rate: "rate", +}); + +const PLACEHOLDER_PROFILE = Object.freeze({ exaggeration: 0.5, - cfg: 0.5, + cfg_weight: 0.5, temperature: 0.8, rate: 1, }); @@ -53,6 +65,7 @@ function storeVoice(name) { } const state = { + presets: { defaults: PLACEHOLDER_PROFILE, neutral: PLACEHOLDER_PROFILE }, voice: null, jobRunning: false, eventSource: null, @@ -64,9 +77,13 @@ const state = { async function init() { wireControls(); - syncDelivery(); updateScriptStats(); - await Promise.allSettled([refreshVoice(), refreshEnvironment(), refreshHistory()]); + await Promise.allSettled([ + refreshDelivery(), + refreshVoice(), + refreshEnvironment(), + refreshHistory(), + ]); } /* ---------- voice ---------- */ @@ -214,14 +231,15 @@ function knobValue(id) { /** Keeps the range fills, the numeric outputs, and the collapsed summary in step. */ function syncDelivery() { + const opening = state.presets.defaults; const changed = []; - for (const [id, fallback] of Object.entries(DELIVERY_DEFAULTS)) { + for (const [id, field] of Object.entries(KNOB_FIELDS)) { const input = $(id); setRangeFill(input); const value = knobValue(id); const suffix = id === "rate" ? "×" : ""; $(`${id}-val`).textContent = `${value.toFixed(2)}${suffix}`; - if (Math.abs(value - fallback) > 1e-9) { + if (Math.abs(value - opening[field]) > 1e-9) { changed.push(`${KNOB_LABELS[id]} ${value.toFixed(2)}${suffix}`); } } @@ -230,14 +248,34 @@ function syncDelivery() { $("delivery-summary").textContent = changed.length === 0 ? "Default" : changed.join(" · "); } -function resetDelivery() { - for (const [id, fallback] of Object.entries(DELIVERY_DEFAULTS)) { - $(id).value = String(fallback); +/** Moves every slider onto `profile`, which is keyed by API field name. */ +function applyProfile(profile) { + for (const [id, field] of Object.entries(KNOB_FIELDS)) { + $(id).value = String(profile[field]); } + syncDelivery(); +} + +/** Hydrates the sliders from the server rather than the HTML placeholders. */ +async function refreshDelivery() { + try { + state.presets = await api.delivery(); + } catch (_) { /* keep the placeholders the page shipped with */ } + applyProfile(state.presets.defaults); +} + +/** Back to Chatterbox's own baseline — the untuned starting point. */ +function resetDelivery() { + applyProfile(state.presets.neutral); $("seed").value = ""; syncDelivery(); } +/** Back to the tuned profile the app opens with. */ +function matchMyVoice() { + applyProfile(state.presets.defaults); +} + /* ---------- generate ---------- */ async function generate() { @@ -503,10 +541,11 @@ function wireControls() { $("generate").addEventListener("click", generate); $("script").addEventListener("input", updateScriptStats); - for (const id of Object.keys(DELIVERY_DEFAULTS)) { + for (const id of Object.keys(KNOB_FIELDS)) { $(id).addEventListener("input", syncDelivery); } $("seed").addEventListener("input", syncDelivery); + $("match-voice").addEventListener("click", matchMyVoice); $("reset-delivery").addEventListener("click", resetDelivery); $("script-file").addEventListener("change", async () => { diff --git a/src/double_chin/studio/static/index.html b/src/double_chin/studio/static/index.html index bebca2d..40cbb58 100644 --- a/src/double_chin/studio/static/index.html +++ b/src/double_chin/studio/static/index.html @@ -90,7 +90,8 @@

Script

Set a number to get the same take back every time.

- + +
diff --git a/src/double_chin/studio/static/style.css b/src/double_chin/studio/static/style.css index 21809d3..3b0f9e0 100644 --- a/src/double_chin/studio/static/style.css +++ b/src/double_chin/studio/static/style.css @@ -148,7 +148,12 @@ body{ border-top: .5px solid var(--separator); } -.disc-foot{ display: flex; justify-content: flex-end; padding-top: var(--gap-2); } +/* Two actions now: "Match my voice" restores the tuned profile, "Reset to + neutral" goes back to Chatterbox's own baseline. */ +.disc-foot{ + display: flex; justify-content: flex-end; gap: var(--gap-4); + padding-top: var(--gap-2); +} /* ---------- knobs ---------- */ .knob{ padding: var(--gap-3) 0 var(--gap-2); } diff --git a/src/double_chin/tuning/__init__.py b/src/double_chin/tuning/__init__.py new file mode 100644 index 0000000..ac1cbac --- /dev/null +++ b/src/double_chin/tuning/__init__.py @@ -0,0 +1,31 @@ +"""Delivery-parameter tuning: search the four knobs against real recordings. + +`evalset` prepares the genuine-voice clips the gate scores against; `sweep` +runs the coordinate-descent search that picks the values Studio opens with. +""" + +from double_chin.tuning.evalset import prepare_real_clips +from double_chin.tuning.sweep import ( + AXIS_GRID, + BudgetExhausted, + Ledger, + SweepContext, + SweepResult, + coordinate_pass, + noise_floor, + run_sweep, + score, +) + +__all__ = [ + "AXIS_GRID", + "BudgetExhausted", + "Ledger", + "SweepContext", + "SweepResult", + "coordinate_pass", + "noise_floor", + "prepare_real_clips", + "run_sweep", + "score", +] diff --git a/src/double_chin/tuning/evalset.py b/src/double_chin/tuning/evalset.py new file mode 100644 index 0000000..cff16ac --- /dev/null +++ b/src/double_chin/tuning/evalset.py @@ -0,0 +1,68 @@ +"""Prepare the genuine-voice clips a sweep scores its candidates against. + +`voices.enroll` concatenates recordings into a single 20 s reference clip; +the indistinguishability gate instead wants a *set* of separate clips, so it +can measure how separable the real and cloned sets are. This module does the +same per-file conditioning as enrolment — mono, resampled, peak-normalized — +but writes each source out as its own wav. +""" + +from __future__ import annotations + +from pathlib import Path + +from double_chin.config import SAMPLE_RATE, ensure_dir + +# Matches voices._PEAK_TARGET: the reference clips the model conditions on are +# normalized to this peak, so the real set the gate compares against should be +# conditioned identically or the level difference leaks into the metrics. +_PEAK_TARGET = 0.9 + + +def prepare_real_clips(sources, out_dir: Path, prefix: str = "real") -> list[Path]: + """Write each clip in `sources` to `out_dir` as a normalized mono wav. + + Args: + sources: paths to genuine recordings (any format enrolment accepts, + including .m4a via the ffmpeg fallback). + out_dir: directory to write into; created if missing. + prefix: filename stem prefix, so several sets can share a directory. + + Returns: + The written wav paths, in the order the sources were given. + + Raises: + ValueError: if `sources` is empty, a clip is missing, or a clip is + silent (silence would make every downstream metric meaningless). + """ + import torchaudio + + from double_chin.voices import _load_waveform + + source_paths = [Path(s) for s in sources] + if not source_paths: + raise ValueError("no source clips were given for the real set.") + + out_dir = ensure_dir(Path(out_dir)) + written: list[Path] = [] + + for index, source in enumerate(source_paths, start=1): + if not source.is_file(): + raise ValueError(f"source clip does not exist: {source}") + + waveform, source_sr = _load_waveform(source) + if waveform.shape[0] > 1: + waveform = waveform.mean(dim=0, keepdim=True) + if source_sr != SAMPLE_RATE: + waveform = torchaudio.functional.resample(waveform, source_sr, SAMPLE_RATE) + + peak = waveform.abs().max() + if peak <= 0: + raise ValueError(f"source clip is silent: {source}") + waveform = waveform * (_PEAK_TARGET / peak) + + target = out_dir / f"{prefix}-{index:03d}.wav" + torchaudio.save(str(target), waveform, SAMPLE_RATE) + written.append(target) + + return written diff --git a/src/double_chin/tuning/runner.py b/src/double_chin/tuning/runner.py new file mode 100644 index 0000000..be270b0 --- /dev/null +++ b/src/double_chin/tuning/runner.py @@ -0,0 +1,223 @@ +"""Wire the real engine and gate into a sweep, and report what it found. + +`sweep.py` knows nothing about Chatterbox or Resemblyzer — it takes a +synthesize callable and a gate callable. This module supplies the real ones, +picks which genuine recordings to score against, and renders the result as a +scorecard a person can read. +""" + +from __future__ import annotations + +from pathlib import Path + +from double_chin.delivery import DeliveryProfile +from double_chin.tuning.evalset import prepare_real_clips +from double_chin.tuning.sweep import Ledger, SweepContext, SweepResult + +_AUDIO_EXTENSIONS = {".wav", ".flac", ".mp3", ".m4a"} + +# Components the delivery knobs can actually move, with the gate's own weights. +# `discrimination` is deliberately absent: with several real clips against three +# clone clips the classifier separates the sets outright, so that component sits +# at 0.0 for every candidate. Left in, it multiplies every score by the same +# near-zero constant — it cannot order candidates, it just makes the number +# unreadable. The full gate composite is still recorded and reported. +_OBJECTIVE_WEIGHTS = { + "speaker_similarity": 0.30, + "naturalness": 0.25, + "prosody": 0.20, +} +OBJECTIVE_NAME = "speaker+naturalness+prosody (gate weights, discrimination excluded)" + + +def delivery_objective(entry: dict) -> float: + """Rank candidates on the gate components that respond to delivery. + + Weighted geometric mean, matching how the gate itself combines components: + one weak component drags the result down rather than being averaged away. + """ + import math + + components = entry.get("components", {}) + total = weight_sum = 0.0 + for name, weight in _OBJECTIVE_WEIGHTS.items(): + if name not in components: + continue + total += weight * math.log(max(components[name], 1e-6)) + weight_sum += weight + return math.exp(total / weight_sum) if weight_sum else 0.0 + + +# The first few recordings are what `reference.wav` is built from (it caps at +# 20 s), so scoring against them would compare the clone to its own +# conditioning audio. Start well past them. +DEFAULT_SKIP = 10 +DEFAULT_TUNING_CLIPS = 8 +DEFAULT_HOLDOUT_CLIPS = 6 + + +def list_recordings(real_dir: Path) -> list[Path]: + """Every audio file in `real_dir`, sorted by name. + + Raises: + ValueError: if the directory is missing or holds no audio. + """ + real_dir = Path(real_dir) + if not real_dir.is_dir(): + raise ValueError(f"recordings directory not found: {real_dir}") + found = sorted( + path + for path in real_dir.iterdir() + if path.is_file() and path.suffix.lower() in _AUDIO_EXTENSIONS + ) + if not found: + raise ValueError(f"no audio files in {real_dir}") + return found + + +def select_real_clips( + recordings: list[Path], + tuning_count: int = DEFAULT_TUNING_CLIPS, + holdout_count: int = DEFAULT_HOLDOUT_CLIPS, + skip: int = DEFAULT_SKIP, +) -> tuple[list[Path], list[Path]]: + """Split recordings into disjoint tuning and holdout sets. + + Both sets are spread evenly across the corpus rather than taken as blocks, + so neither is dominated by one recording session, and they never overlap. + + Raises: + ValueError: if there are not enough recordings for both sets. + """ + pool = recordings[skip:] + needed = tuning_count + holdout_count + if len(pool) < needed: + raise ValueError( + f"need {needed} recordings after skipping {skip}, found {len(pool)}" + ) + + stride = len(pool) // needed + picked = [pool[index * stride] for index in range(needed)] + return picked[:tuning_count], picked[tuning_count:] + + +def build_context( + voice, + real_clips: list[Path], + scripts: dict[str, str], + work_dir: Path, + takes: int, + budget: int | None = None, + device: str | None = None, + on_event=None, +) -> SweepContext: + """A sweep context backed by the real engine and the real gate.""" + from double_chin.engine import DoubleChinEngine + from double_chin.verification import indistinguishability_gate + + engine = DoubleChinEngine(device=device) + lora_path = voice.lora_path + + def synth(profile: DeliveryProfile, script: str, out_path: Path, seed: int): + return engine.synthesize( + script=script, + reference_wav=voice.reference_wav, + out_path=out_path, + seed=seed, + lora_path=lora_path, + **profile.as_dict(), + ) + + def gate(real, clone): + return indistinguishability_gate(real, clone) + + work_dir = Path(work_dir) + return SweepContext( + synth=synth, + gate=gate, + real_clips=real_clips, + scripts=scripts, + work_dir=work_dir / "takes", + ledger=Ledger(work_dir / "ledger.jsonl"), + takes=takes, + budget=budget, + on_event=on_event, + objective=delivery_objective, + objective_name=OBJECTIVE_NAME, + ) + + +def prepare_sets(real_dir: Path, work_dir: Path, **selection) -> tuple[list, list]: + """Normalize the chosen recordings into wav sets the gate can read.""" + tuning_sources, holdout_sources = select_real_clips( + list_recordings(real_dir), **selection + ) + work_dir = Path(work_dir) + return ( + prepare_real_clips(tuning_sources, work_dir / "real", prefix="tuning"), + prepare_real_clips(holdout_sources, work_dir / "real", prefix="holdout"), + ) + + +def _knob_line(name: str, tuned: float, baseline: float) -> str: + suffix = "x" if name == "rate" else "" + delta = tuned - baseline + move = "unchanged" if abs(delta) < 1e-9 else f"{delta:+.2f}" + return f" {name:<13} {tuned:.2f}{suffix:<2} (was {baseline:.2f}{suffix}, {move})" + + +def format_report(result: SweepResult, holdout: dict | None = None) -> str: + """Render a sweep result as a readable scorecard. + + Deliberately states the noise floor and refuses to call a win a win when + the margin sits inside it. + """ + lines = [ + f"Delivery sweep on script '{result.script_id}'", + f" ranked on: {result.objective_name}", + f" {result.synthesis_count} clips synthesized" + + (" (budget exhausted)" if result.budget_exhausted else ""), + "", + "Winning profile:", + ] + baseline_values = result.baseline.as_dict() + for name, value in result.best.as_dict().items(): + lines.append(_knob_line(name, value, baseline_values[name])) + + floor = result.noise_floor + lines += [ + "", + f" score {result.best_score:.3f} (baseline {result.baseline_score:.3f}, " + f"{result.margin:+.3f})", + f" noise floor sd {floor.get('sd', 0.0):.3f} over {floor.get('repeats', 0)} " + "repeats of the baseline", + f" full gate composite at the winner: {result.best_gate_score:.3f}", + ] + if result.beats_noise: + lines.append(" the gain clears the noise floor") + else: + lines.append( + " the gain does NOT clear the noise floor — treat this as a tie " + "and keep the baseline" + ) + + if result.best_components: + lines.append("") + lines.append("Components at the winner:") + for name, value in result.best_components.items(): + lines.append(f" {name:<18} {value:.3f}") + + if holdout is not None: + lines += [ + "", + f"Held-out script: winner {holdout['best']:.3f} vs baseline " + f"{holdout['baseline']:.3f} ({holdout['best'] - holdout['baseline']:+.3f})", + ( + " holds up on unseen text" + if holdout["best"] > holdout["baseline"] + else " does NOT hold up on unseen text — likely overfit to the " + "tuning script" + ), + ] + + return "\n".join(lines) diff --git a/src/double_chin/tuning/scripts.py b/src/double_chin/tuning/scripts.py new file mode 100644 index 0000000..9df51ea --- /dev/null +++ b/src/double_chin/tuning/scripts.py @@ -0,0 +1,32 @@ +"""The fixed scripts a sweep synthesizes. + +Two of them, and they are deliberately plain prose: no `[pause:N]`, no +`*emphasis*`. Emphasis markup raises exaggeration and lowers cfg_weight for +the spanned chunk (see `engine.synthesize`), and those adjustments clamp at +the knob's range — so a marked-up script would distort exactly the candidates +sitting near the ends of the grid. Delivery marks layer on top of whatever +profile wins; they are not part of what is being measured. + +Both are kept to a single chunk (under `chunk.DEFAULT_MAX_CHARS`, about ten +seconds of audio). Length is pure cost here: every candidate is synthesized +several times over, and a longer passage buys no extra signal for metrics +that compare sets of clips. + +`TUNING` drives the search. `HOLDOUT` is never searched on — it is only used +to re-score the winner, which is what catches a profile that has quietly +overfit to one passage. +""" + +from __future__ import annotations + +TUNING = ( + "I keep a running list of things I meant to fix, and it gets longer every " + "week. Most of them are small. The rest I have been putting off since March." +) + +HOLDOUT = ( + "The part nobody tells you about building your own tools is how much of it " + "is deciding what to leave out. I spent an afternoon on one, then deleted it." +) + +TUNING_SCRIPTS = {"tuning": TUNING, "holdout": HOLDOUT} diff --git a/src/double_chin/tuning/sweep.py b/src/double_chin/tuning/sweep.py new file mode 100644 index 0000000..4007034 --- /dev/null +++ b/src/double_chin/tuning/sweep.py @@ -0,0 +1,364 @@ +"""Coordinate-descent search over the four delivery knobs. + +The question this answers is narrow: of the settings the Studio sliders can +express, which one makes synthesized takes least distinguishable from the +owner's real recordings? Scoring is the existing +`verification.gate.indistinguishability_gate` — no new metric is invented here. +What the search *ranks* on is separate and pluggable (`SweepContext.objective`), +because a gate component can be saturated in this setting and then contributes +no ordering; the full gate result is recorded for every candidate either way. + +Three properties matter more than the search itself: + +* **Repeats.** Generation is stochastic, so one take per candidate measures + luck. Each candidate is scored from `takes` clips at once, which is also how + the gate wants its input (it measures set separability, not clip pairs). +* **A noise floor.** Before searching, the starting profile is re-measured + several times with different seeds. A candidate only counts as better if it + beats the baseline by more than that spread. +* **Resumability.** Every measurement is appended to a JSONL ledger keyed by + (profile, script, variant), so an interrupted two-hour run continues instead + of restarting, and a re-run costs nothing. +""" + +from __future__ import annotations + +import json +import statistics +import zlib +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable + +from double_chin.delivery import DeliveryProfile + +# Coarse grid per axis: four values spanning the useful part of each range, +# with the neutral default among them so the baseline measurement is reused +# rather than repeated. Values sit on the studio's 0.05 slider step so a winner +# can be dialled in by hand, and stay inside the API's bounds. Four rather than +# more is a cost decision: a candidate costs three syntheses plus a gate run +# (~5 min on this machine), and `refine` walks the winner in 0.05 steps +# afterwards, which is where the last bit of resolution comes from. +AXIS_GRID: dict[str, tuple[float, ...]] = { + "cfg_weight": (0.35, 0.50, 0.65, 0.80), + "exaggeration": (0.25, 0.40, 0.50, 0.65), + "temperature": (0.50, 0.65, 0.80, 0.95), + "rate": (0.95, 1.00, 1.05, 1.10), +} + +# Adherence first: it moves similarity most, so the later axes are searched +# from a sensible place rather than from an obviously wrong one. +AXIS_ORDER = ("cfg_weight", "exaggeration", "temperature", "rate") + +SLIDER_STEP = 0.05 +DEFAULT_TAKES = 3 +DEFAULT_NOISE_REPEATS = 3 + + +class BudgetExhausted(RuntimeError): + """Raised when a sweep has used its allowance of syntheses.""" + + +class Ledger: + """Append-only JSONL record of every measurement, keyed for resume.""" + + def __init__(self, path: Path) -> None: + self.path = Path(path) + self._entries: dict[str, dict] = {} + if self.path.is_file(): + self._load() + + def _load(self) -> None: + for line in self.path.read_text().splitlines(): + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue # a half-written final line from a killed run + if "key" in entry: + self._entries[entry["key"]] = entry + + @staticmethod + def key(profile: DeliveryProfile, script_id: str, variant: int) -> str: + values = profile.as_dict() + knobs = "-".join(f"{name[0]}{values[name]:g}" for name in sorted(values)) + return f"{knobs}|{script_id}|v{variant}" + + def get(self, key: str) -> dict | None: + return self._entries.get(key) + + def record(self, entry: dict) -> None: + self._entries[entry["key"]] = entry + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a") as handle: + handle.write(json.dumps(entry) + "\n") + + @property + def entries(self) -> list[dict]: + return list(self._entries.values()) + + @property + def synthesis_count(self) -> int: + """How many clips have been synthesized across every recorded run.""" + return sum(entry.get("takes", 0) for entry in self._entries.values()) + + +@dataclass +class SweepContext: + """Everything a sweep needs: how to synthesize, how to score, and where.""" + + synth: Callable[[DeliveryProfile, str, Path, int], Any] + gate: Callable[[list[Path], list[Path]], dict] + real_clips: list[Path] + scripts: dict[str, str] + work_dir: Path + ledger: Ledger + takes: int = DEFAULT_TAKES + budget: int | None = None + on_event: Callable[[str, dict], None] | None = None + # What the search maximizes, given a ledger entry. Defaults to the gate's + # own composite; supply one when a component of that composite is + # saturated and therefore carries no ranking signal (see runner.py). + objective: Callable[[dict], float] | None = None + objective_name: str = "gate composite" + + def value(self, entry: dict) -> float: + """The number the search compares candidates on.""" + return self.objective(entry) if self.objective is not None else entry["score"] + + def emit(self, kind: str, data: dict) -> None: + if self.on_event is not None: + self.on_event(kind, data) + + +@dataclass +class SweepResult: + """The outcome of a sweep, honest about whether the win is real.""" + + best: DeliveryProfile + best_score: float + best_components: dict + baseline: DeliveryProfile + baseline_score: float + noise_floor: dict + script_id: str + synthesis_count: int + budget_exhausted: bool = False + history: list[dict] = field(default_factory=list) + objective_name: str = "gate composite" + # The full gate composite at the winner, kept alongside the objective so a + # report can state both rather than only the number that was optimized. + best_gate_score: float = 0.0 + + @property + def margin(self) -> float: + return self.best_score - self.baseline_score + + @property + def beats_noise(self) -> bool: + """True only if the gain exceeds the baseline's own run-to-run spread.""" + return self.margin > self.noise_floor.get("sd", 0.0) + + +def _seed_for(key: str, take: int) -> int: + """A stable per-take seed, so a resumed or repeated run reproduces.""" + return zlib.crc32(f"{key}#{take}".encode()) % (2**31) + + +def _take_dir(ctx: SweepContext, key: str) -> Path: + return Path(ctx.work_dir) / key.replace("|", "_") + + +def score( + ctx: SweepContext, + profile: DeliveryProfile, + script_id: str, + variant: int = 0, +) -> dict: + """Score one candidate: synthesize `ctx.takes` clips and run the gate. + + Cached by (profile, script, variant) through the ledger, so re-asking for + a measurement already on disk costs nothing. + + Raises: + BudgetExhausted: if the sweep has already used its synthesis budget. + ValueError: if `script_id` is not in `ctx.scripts`. + """ + if script_id not in ctx.scripts: + raise ValueError(f"unknown script '{script_id}'") + + key = Ledger.key(profile, script_id, variant) + cached = ctx.ledger.get(key) + if cached is not None: + return cached + + if ctx.budget is not None and ctx.ledger.synthesis_count >= ctx.budget: + raise BudgetExhausted( + f"synthesis budget of {ctx.budget} reached before scoring {key}" + ) + + ctx.emit("candidate", {"key": key, "profile": profile.as_dict()}) + out_dir = _take_dir(ctx, key) + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "profile.json").write_text( + json.dumps({"profile": profile.as_dict(), "variant": variant}, indent=2) + ) + + clone_clips: list[Path] = [] + for take in range(ctx.takes): + out_path = out_dir / f"take-{take + 1}.wav" + ctx.synth(profile, ctx.scripts[script_id], out_path, _seed_for(key, take)) + clone_clips.append(out_path) + + gate_result = ctx.gate(list(ctx.real_clips), clone_clips) + entry = { + "key": key, + "profile": profile.as_dict(), + "script_id": script_id, + "variant": variant, + "takes": ctx.takes, + "score": gate_result["score"], + "verdict": gate_result.get("verdict"), + "components": { + name: data["component"] + for name, data in gate_result.get("components", {}).items() + }, + "recorded": datetime.now(timezone.utc).isoformat(), + } + ctx.ledger.record(entry) + ctx.emit("scored", entry) + return entry + + +def noise_floor( + ctx: SweepContext, + profile: DeliveryProfile, + script_id: str, + repeats: int = DEFAULT_NOISE_REPEATS, +) -> dict: + """Measure the same profile `repeats` times to size run-to-run variation.""" + scores = [ + ctx.value(score(ctx, profile, script_id, variant=variant)) + for variant in range(repeats) + ] + return { + "scores": scores, + "mean": statistics.fmean(scores), + "sd": statistics.stdev(scores) if len(scores) > 1 else 0.0, + "repeats": repeats, + } + + +def _with(profile: DeliveryProfile, name: str, value: float) -> DeliveryProfile: + return DeliveryProfile.from_mapping(profile.as_dict() | {name: value}) + + +def coordinate_pass( + ctx: SweepContext, + start: DeliveryProfile, + script_id: str, + axes: tuple[str, ...] = AXIS_ORDER, +) -> tuple[DeliveryProfile, list[dict]]: + """Walk each axis in turn, keeping the best value found before moving on.""" + best = start + best_score = ctx.value(score(ctx, best, script_id)) + history: list[dict] = [] + + for axis in axes: + for value in AXIS_GRID[axis]: + candidate = _with(best, axis, value) + entry = score(ctx, candidate, script_id) + history.append(entry) + if ctx.value(entry) > best_score: + best, best_score = candidate, ctx.value(entry) + + return best, history + + +def refine( + ctx: SweepContext, + start: DeliveryProfile, + script_id: str, + axes: tuple[str, ...] = AXIS_ORDER, +) -> tuple[DeliveryProfile, list[dict]]: + """Try one slider step either side of the winner on each axis.""" + best = start + best_score = ctx.value(score(ctx, best, script_id)) + history: list[dict] = [] + + for axis in axes: + current = getattr(best, axis) + for value in (current - SLIDER_STEP, current + SLIDER_STEP): + value = round(value, 2) + try: + candidate = _with(best, axis, value) + except ValueError: + continue # stepped outside the knob's range + entry = score(ctx, candidate, script_id) + history.append(entry) + if ctx.value(entry) > best_score: + best, best_score = candidate, ctx.value(entry) + + return best, history + + +def run_sweep( + ctx: SweepContext, + start: DeliveryProfile, + script_id: str, + passes: int = 2, + noise_repeats: int = DEFAULT_NOISE_REPEATS, +) -> SweepResult: + """Run the full search: noise floor, coordinate passes, then refinement. + + Stops early and returns the best profile found so far if the synthesis + budget runs out, so a capped run still produces a usable answer. + + Raises: + ValueError: if `script_id` is not one of `ctx.scripts`. + """ + if script_id not in ctx.scripts: + raise ValueError(f"unknown script '{script_id}'") + + exhausted = False + history: list[dict] = [] + best = start + best_score = 0.0 + floor: dict = {"scores": [], "mean": 0.0, "sd": 0.0, "repeats": 0} + + try: + floor = noise_floor(ctx, start, script_id, repeats=noise_repeats) + + for _ in range(passes): + previous = best + best, pass_history = coordinate_pass(ctx, best, script_id) + history.extend(pass_history) + if best == previous: + break # a whole pass moved nothing; further passes cannot either + + best, refine_history = refine(ctx, best, script_id) + history.extend(refine_history) + except BudgetExhausted: + exhausted = True + + best_entry = ctx.ledger.get(Ledger.key(best, script_id, 0)) + if best_entry is not None: + best_score = ctx.value(best_entry) + + return SweepResult( + best=best, + best_score=best_score, + best_components=(best_entry or {}).get("components", {}), + baseline=start, + baseline_score=floor["mean"], + noise_floor=floor, + script_id=script_id, + synthesis_count=ctx.ledger.synthesis_count, + budget_exhausted=exhausted, + history=history, + objective_name=ctx.objective_name, + best_gate_score=(best_entry or {}).get("score", 0.0), + ) diff --git a/tests/test_delivery.py b/tests/test_delivery.py new file mode 100644 index 0000000..96b5822 --- /dev/null +++ b/tests/test_delivery.py @@ -0,0 +1,140 @@ +"""Tests for delivery profiles and their persisted defaults. + +All of these run offline: the module only touches JSON under DOUBLECHIN_HOME. +""" + +from __future__ import annotations + +import json + +import pytest + +from double_chin import delivery + + +@pytest.fixture +def home(tmp_path, monkeypatch): + monkeypatch.setenv("DOUBLECHIN_HOME", str(tmp_path)) + return tmp_path + + +def test_neutral_profile_matches_engine_defaults(): + from double_chin.engine import ( + DEFAULT_CFG_WEIGHT, + DEFAULT_EXAGGERATION, + DEFAULT_RATE, + DEFAULT_TEMPERATURE, + ) + + assert delivery.NEUTRAL_PROFILE.exaggeration == DEFAULT_EXAGGERATION + assert delivery.NEUTRAL_PROFILE.cfg_weight == DEFAULT_CFG_WEIGHT + assert delivery.NEUTRAL_PROFILE.temperature == DEFAULT_TEMPERATURE + assert delivery.NEUTRAL_PROFILE.rate == DEFAULT_RATE + + +def test_tuned_profile_is_expressible_on_the_sliders(): + # The studio sliders move in 0.05 steps; a tuned value the user cannot + # dial back in by hand would make the preset button unreproducible. + for value in delivery.TUNED_PROFILE.as_dict().values(): + steps = value / 0.05 + assert abs(steps - round(steps)) < 1e-6 + + +def test_from_mapping_round_trip(): + data = {"exaggeration": 0.4, "cfg_weight": 0.65, "temperature": 0.75, "rate": 1.05} + profile = delivery.DeliveryProfile.from_mapping(data) + assert profile.as_dict() == data + + +@pytest.mark.parametrize( + "field,value", + [ + ("exaggeration", -0.1), + ("exaggeration", 1.5), + ("cfg_weight", 2.0), + ("temperature", 0.0), + ("temperature", 2.5), + ("rate", 0.1), + ("rate", 3.0), + ], +) +def test_from_mapping_rejects_out_of_range(field, value): + data = delivery.NEUTRAL_PROFILE.as_dict() | {field: value} + with pytest.raises(ValueError, match=field): + delivery.DeliveryProfile.from_mapping(data) + + +def test_from_mapping_rejects_missing_and_non_numeric(): + partial = {"exaggeration": 0.5} + with pytest.raises(ValueError): + delivery.DeliveryProfile.from_mapping(partial) + + bad = delivery.NEUTRAL_PROFILE.as_dict() | {"rate": "fast"} + with pytest.raises(ValueError): + delivery.DeliveryProfile.from_mapping(bad) + + +def test_from_mapping_ignores_unknown_keys(): + data = delivery.NEUTRAL_PROFILE.as_dict() | {"seed": 7, "voice": "owner"} + assert delivery.DeliveryProfile.from_mapping(data) == delivery.NEUTRAL_PROFILE + + +def test_load_defaults_without_a_file_returns_tuned(home): + assert not delivery.defaults_path().exists() + assert delivery.load_defaults() == delivery.TUNED_PROFILE + + +def test_save_then_load_round_trip(home): + profile = delivery.DeliveryProfile(0.35, 0.7, 0.6, 0.95) + delivery.save_defaults(profile) + + assert delivery.load_defaults() == profile + stored = json.loads(delivery.defaults_path().read_text()) + assert stored["exaggeration"] == 0.35 + + +def test_load_defaults_falls_back_on_corrupt_json(home, caplog): + delivery.defaults_path().parent.mkdir(parents=True, exist_ok=True) + delivery.defaults_path().write_text("{not json at all") + + assert delivery.load_defaults() == delivery.TUNED_PROFILE + assert "delivery.json" in caplog.text + + +def test_load_defaults_falls_back_on_out_of_range_file(home, caplog): + delivery.save_defaults(delivery.NEUTRAL_PROFILE) + data = json.loads(delivery.defaults_path().read_text()) + data["temperature"] = 9.9 + delivery.defaults_path().write_text(json.dumps(data)) + + assert delivery.load_defaults() == delivery.TUNED_PROFILE + assert "temperature" in caplog.text + + +def test_clear_defaults_is_idempotent(home): + delivery.save_defaults(delivery.DeliveryProfile(0.4, 0.4, 0.7, 1.0)) + delivery.clear_defaults() + delivery.clear_defaults() + + assert not delivery.defaults_path().exists() + assert delivery.load_defaults() == delivery.TUNED_PROFILE + + +def test_save_defaults_rejects_an_invalid_profile(home): + with pytest.raises(ValueError): + delivery.save_defaults(delivery.DeliveryProfile(0.5, 0.5, 5.0, 1.0)) + assert not delivery.defaults_path().exists() + + +def test_save_defaults_does_not_clobber_on_write_failure(home, monkeypatch): + good = delivery.DeliveryProfile(0.4, 0.6, 0.7, 1.0) + delivery.save_defaults(good) + + def boom(*args, **kwargs): + raise OSError("disk full") + + monkeypatch.setattr(delivery.os, "replace", boom) + with pytest.raises(OSError): + delivery.save_defaults(delivery.DeliveryProfile(0.9, 0.9, 1.5, 1.5)) + + assert delivery.load_defaults() == good diff --git a/tests/test_e2e.py b/tests/test_e2e.py index d491e4b..a3b3296 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -8,6 +8,7 @@ import os import re +import shutil from pathlib import Path import pytest @@ -53,3 +54,52 @@ def test_say_produces_audio_with_real_model(tmp_path, monkeypatch, capsys): score = float(match.group(1)) assert score > 0.75 + + +@pytest.mark.skipif( + os.environ.get("DOUBLECHIN_E2E") != "1", + reason="set DOUBLECHIN_E2E=1 to run real-model end-to-end tests", +) +def test_tune_scores_candidates_and_writes_a_resumable_ledger(tmp_path, monkeypatch): + """A budget-capped sweep on the stand-in voice, end to end. + + Deliberately tiny: the point is that the real engine, the real gate and + the ledger fit together, not that the winner is meaningful with this + little audio. + """ + import json + + from double_chin.cli import main + from double_chin.voices import enroll + + monkeypatch.setenv("DOUBLECHIN_HOME", str(tmp_path / "home")) + + real_dir = tmp_path / "real" + real_dir.mkdir() + for index in range(16): + shutil.copy(REFERENCE_WAV, real_dir / f"{index:03d}.wav") + enroll("standin", [real_dir / "000.wav", real_dir / "001.wav"]) + + work_dir = tmp_path / "work" + exit_code = main([ + "tune", "standin", + "--real-dir", str(real_dir), + "--takes", "1", + "--budget", "2", + "--passes", "1", + "--work-dir", str(work_dir), + "--write-defaults", + ]) + assert exit_code == 0 + + entries = [ + json.loads(line) + for line in (work_dir / "ledger.jsonl").read_text().splitlines() + if line.strip() + ] + assert entries, "the sweep recorded nothing" + assert all(0.0 <= entry["score"] <= 1.0 for entry in entries) + + from double_chin.delivery import DeliveryProfile, load_defaults + + assert isinstance(load_defaults(), DeliveryProfile) diff --git a/tests/test_studio.py b/tests/test_studio.py index be5bb50..dac7952 100644 --- a/tests/test_studio.py +++ b/tests/test_studio.py @@ -418,3 +418,93 @@ def test_jobs_pruned_to_cap(tmp_path, monkeypatch): _wait_done(client, job_id) assert len(manager._jobs) <= _MAX_RETAINED_JOBS + + +def test_delivery_serves_the_opening_profile_and_the_neutral_baseline(studio): + client, _engine, _ = studio + from double_chin.delivery import NEUTRAL_PROFILE, TUNED_PROFILE + + payload = client.get("/api/delivery").json() + + assert payload["defaults"] == TUNED_PROFILE.as_dict() # no saved file yet + assert payload["neutral"] == NEUTRAL_PROFILE.as_dict() + assert set(payload["defaults"]) == {"exaggeration", "cfg_weight", "temperature", "rate"} + + +def test_delivery_defaults_reflect_a_saved_profile(studio): + client, _engine, _ = studio + from double_chin.delivery import TUNED_PROFILE, DeliveryProfile, save_defaults + + saved = DeliveryProfile(0.3, 0.75, 0.6, 0.95) + save_defaults(saved) + + payload = client.get("/api/delivery").json() + assert payload["defaults"] == saved.as_dict() + assert payload["defaults"] != TUNED_PROFILE.as_dict() + assert payload["neutral"] != saved.as_dict() # reset still goes somewhere else + + +def test_delivery_defaults_survive_a_corrupt_file(studio): + client, _engine, _ = studio + from double_chin.delivery import TUNED_PROFILE, defaults_path + + defaults_path().parent.mkdir(parents=True, exist_ok=True) + defaults_path().write_text("}{") + + payload = client.get("/api/delivery").json() + assert payload["defaults"] == TUNED_PROFILE.as_dict() + + +def test_delivery_values_are_accepted_by_the_jobs_endpoint(studio, tmp_path): + client, engine, _ = studio + _enroll_test_voice(tmp_path) + + payload = client.get("/api/delivery").json() + for name in ("defaults", "neutral"): + response = client.post( + "/api/jobs", json={"voice": "testvoice", "text": "A line.", **payload[name]} + ) + assert response.status_code == 202, f"{name} rejected: {response.text}" + _wait_done(client, response.json()["job_id"]) + + assert engine.calls[0]["params"]["cfg_weight"] == payload["defaults"]["cfg_weight"] + + +def test_every_element_id_the_frontend_touches_exists_in_the_page(): + """`$("some-id")` in app.js must name a real element in index.html. + + The studio frontend has no test runner of its own, so a renamed or + forgotten id would otherwise only surface as a dead button in the browser. + """ + import re + + static = Path(__file__).resolve().parents[1] / "src" / "double_chin" / "studio" / "static" + script = (static / "app.js").read_text() + markup = (static / "index.html").read_text() + + referenced = set(re.findall(r'\$\("([a-z0-9-]+)"\)', script)) + # Some elements are built by app.js itself (the empty-state link), so ids + # it assigns count as declared too. + declared = set(re.findall(r'id="([a-z0-9-]+)"', markup)) + declared |= set(re.findall(r'id:\s*"([a-z0-9-]+)"', script)) + + missing = sorted(referenced - declared) + assert not missing, f"app.js references ids that index.html does not define: {missing}" + + +def test_delivery_knob_ids_have_a_numeric_readout_each(): + """Every slider id also needs its `-val` output, which syncDelivery writes.""" + import re + + static = Path(__file__).resolve().parents[1] / "src" / "double_chin" / "studio" / "static" + script = (static / "app.js").read_text() + markup = (static / "index.html").read_text() + + block = re.search(r"const KNOB_FIELDS = Object\.freeze\(\{(.+?)\}\)", script, re.S) + knob_ids = re.findall(r"^\s*([a-z_]+):", block.group(1), re.M) + assert knob_ids, "KNOB_FIELDS is empty; the delivery panel would never sync" + + declared = set(re.findall(r'id="([a-z0-9-]+)"', markup)) + for knob_id in knob_ids: + assert knob_id in declared, f"no slider with id {knob_id}" + assert f"{knob_id}-val" in declared, f"no readout with id {knob_id}-val" diff --git a/tests/test_tuning.py b/tests/test_tuning.py new file mode 100644 index 0000000..b5dc769 --- /dev/null +++ b/tests/test_tuning.py @@ -0,0 +1,236 @@ +"""Offline tests for the delivery-parameter sweep. + +The search is exercised against a stub synthesizer and a stub gate whose +optimum is known, so the coordinate descent, the ledger, the noise floor and +the budget cap are all tested without loading a model. +""" + +from __future__ import annotations + +import json + +import pytest + +from double_chin.delivery import NEUTRAL_PROFILE, DeliveryProfile +from double_chin.tuning import sweep as sweeplib + +# Every value is a point on AXIS_GRID, so the search can actually reach it. +TARGET = DeliveryProfile(exaggeration=0.40, cfg_weight=0.65, temperature=0.65, rate=1.05) + + +class StubRig: + """A synthesizer/gate pair whose score peaks at TARGET. + + Score falls off with distance from TARGET, so the coordinate descent has a + single obvious optimum on the grid. `variant` perturbs the score slightly, + standing in for the real engine's run-to-run randomness. + """ + + def __init__(self) -> None: + self.syntheses = 0 + self.gate_calls = 0 + + def synth(self, profile, script, out_path, seed): + self.syntheses += 1 + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(b"fake wav") + + def gate(self, real_clips, clone_clips): + self.gate_calls += 1 + profile, variant = self._decode(clone_clips) + distance = sum( + abs(getattr(profile, name) - getattr(TARGET, name)) + for name in ("exaggeration", "cfg_weight", "temperature", "rate") + ) + score = max(0.0, 0.9 - distance) + 0.001 * variant + return { + "passed": score >= 0.7, + "score": score, + "verdict": "near-indistinguishable", + "components": { + "speaker_similarity": {"component": score, "weight": 0.3}, + "discrimination": {"component": score, "weight": 0.25}, + "naturalness": {"component": score, "weight": 0.25}, + "prosody": {"component": score, "weight": 0.2}, + }, + } + + @staticmethod + def _decode(clone_clips): + """Recover the profile under test from the take directory.""" + meta = json.loads((clone_clips[0].parent / "profile.json").read_text()) + return DeliveryProfile.from_mapping(meta["profile"]), meta["variant"] + + +@pytest.fixture +def ctx(tmp_path): + rig = StubRig() + context = sweeplib.SweepContext( + synth=rig.synth, + gate=rig.gate, + real_clips=[tmp_path / "real.wav"], + scripts={"a": "a short line of script", "b": "a second, held-out line"}, + work_dir=tmp_path / "work", + ledger=sweeplib.Ledger(tmp_path / "ledger.jsonl"), + takes=2, + ) + context.rig = rig + return context + + +def test_score_writes_takes_and_records_the_result(ctx): + result = sweeplib.score(ctx, TARGET, "a") + + assert result["score"] == pytest.approx(0.9) + assert ctx.rig.syntheses == 2 + # The ledger keeps components flattened to one number each. + assert result["components"]["prosody"] == pytest.approx(0.9) + assert ctx.ledger.synthesis_count == 2 + + +def test_score_is_cached_by_profile_script_and_variant(ctx): + sweeplib.score(ctx, TARGET, "a") + sweeplib.score(ctx, TARGET, "a") + assert ctx.rig.syntheses == 2 # second call served from the ledger + + sweeplib.score(ctx, TARGET, "b") + assert ctx.rig.syntheses == 4 # a different script is a different measurement + + sweeplib.score(ctx, TARGET, "a", variant=1) + assert ctx.rig.syntheses == 6 # a different variant is a fresh sample + + +def test_ledger_resumes_from_disk(ctx, tmp_path): + sweeplib.score(ctx, TARGET, "a") + + resumed = sweeplib.SweepContext( + synth=ctx.synth, + gate=ctx.gate, + real_clips=ctx.real_clips, + scripts=ctx.scripts, + work_dir=ctx.work_dir, + ledger=sweeplib.Ledger(tmp_path / "ledger.jsonl"), + takes=2, + ) + before = ctx.rig.syntheses + result = sweeplib.score(resumed, TARGET, "a") + + assert ctx.rig.syntheses == before # nothing re-synthesized + assert result["score"] == pytest.approx(0.9) + + +def test_noise_floor_samples_distinct_variants(ctx): + floor = sweeplib.noise_floor(ctx, NEUTRAL_PROFILE, "a", repeats=3) + + assert len(floor["scores"]) == 3 + assert floor["sd"] > 0 # the variants really did differ + assert floor["mean"] == pytest.approx(sum(floor["scores"]) / 3) + + +def test_coordinate_pass_moves_each_axis_to_its_best_grid_value(ctx): + best, _ = sweeplib.coordinate_pass(ctx, NEUTRAL_PROFILE, "a") + + assert best.exaggeration == pytest.approx(0.40) + assert best.cfg_weight == pytest.approx(0.65) + assert best.temperature == pytest.approx(0.65) + assert best.rate == pytest.approx(1.05) + + +def test_run_sweep_finds_the_optimum_and_reports_the_noise_floor(ctx): + result = sweeplib.run_sweep(ctx, start=NEUTRAL_PROFILE, script_id="a", passes=2) + + assert result.best == TARGET + assert result.best_score > result.baseline_score + assert result.noise_floor["sd"] >= 0 + assert result.beats_noise is True + assert result.synthesis_count == ctx.ledger.synthesis_count + + +def test_run_sweep_stops_at_the_budget_and_returns_the_best_so_far(ctx): + ctx.budget = 6 + result = sweeplib.run_sweep(ctx, start=NEUTRAL_PROFILE, script_id="a", passes=2) + + assert result.budget_exhausted is True + assert ctx.rig.syntheses <= 8 # the in-flight candidate may finish + assert isinstance(result.best, DeliveryProfile) + + +def test_run_sweep_refuses_an_unknown_script(ctx): + with pytest.raises(ValueError, match="unknown script"): + sweeplib.run_sweep(ctx, start=NEUTRAL_PROFILE, script_id="nope") + + +def test_grid_values_sit_on_the_slider_step(): + for name, values in sweeplib.AXIS_GRID.items(): + for value in values: + steps = value / 0.05 + assert abs(steps - round(steps)) < 1e-6, f"{name}={value} is off-grid" + + +def test_grid_values_are_inside_the_profile_bounds(): + for name, values in sweeplib.AXIS_GRID.items(): + for value in values: + profile = DeliveryProfile.from_mapping( + NEUTRAL_PROFILE.as_dict() | {name: value} + ) + assert getattr(profile, name) == value + + +def test_prepare_real_clips_normalizes_into_wavs(tmp_path): + import torch + import torchaudio + + from double_chin.config import SAMPLE_RATE + from double_chin.tuning import evalset + + source = tmp_path / "source.wav" + tone = torch.sin(torch.linspace(0, 400, 16000)).reshape(1, -1) * 0.05 + torchaudio.save(str(source), torch.cat([tone, tone], dim=0), 8000) + + out = evalset.prepare_real_clips([source], tmp_path / "real") + + assert len(out) == 1 + wav, sr = torchaudio.load(str(out[0])) + assert sr == SAMPLE_RATE + assert wav.shape[0] == 1 # collapsed to mono + assert wav.abs().max() == pytest.approx(0.9, abs=0.01) # peak-normalized + + +def test_prepare_real_clips_rejects_an_empty_selection(tmp_path): + from double_chin.tuning import evalset + + with pytest.raises(ValueError, match="no source clips"): + evalset.prepare_real_clips([], tmp_path / "real") + + +def test_search_follows_a_supplied_objective_not_the_raw_gate_score(ctx): + """A saturated gate component must not decide the ranking. + + Here the raw score peaks at TARGET, but the objective is built to peak at + the neutral profile instead; the search must follow the objective. + """ + ctx.objective = lambda entry: 1.0 - abs(entry["profile"]["cfg_weight"] - 0.35) + ctx.objective_name = "cfg-only stub" + + best, _ = sweeplib.coordinate_pass(ctx, NEUTRAL_PROFILE, "a", axes=("cfg_weight",)) + + assert best.cfg_weight == pytest.approx(0.35) + + +def test_delivery_objective_ignores_the_saturated_discrimination_component(): + from double_chin.tuning.runner import delivery_objective + + collapsed = { + "components": { + "speaker_similarity": 0.64, + "discrimination": 0.0, + "naturalness": 1.0, + "prosody": 0.77, + } + } + better_prosody = { + "components": dict(collapsed["components"], prosody=0.85) + } + + assert delivery_objective(collapsed) > 0.5 # readable, not collapsed to ~0 + assert delivery_objective(better_prosody) > delivery_objective(collapsed)