diff --git a/.gitignore b/.gitignore index a6decda..6c13a4b 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ swift/Packaging/Tacet.app/ # Stray build output. The packaging script regenerates this. build/ + +# Eval audio, server log and per-clip results (eval/tacet_eval.py regenerates them). +eval/.cache/ diff --git a/README.md b/README.md index f501bc4..79cb6df 100644 --- a/README.md +++ b/README.md @@ -369,6 +369,127 @@ never the content — of each transcript; `status.json` carries the live microphone, Accessibility and hotkey state. A flake an hour old otherwise leaves no evidence anywhere. +## Measuring accuracy + +`eval/tacet_eval.py` measures the transcription leg on its own: how accurate +and how fast whisper-server is with tacet's model and flags. + +It has two kinds of input. The shipped corpus is synthetic: 44 sentences +spoken by macOS `say`. It is a smoke test, good for "did this flag break +something" and for server latency. It is too small and too clean to settle an +accuracy question: one word moves a voice's WER by about a quarter of a point, +and a text-to-speech voice has no microphone, no room and no hesitation. For +accuracy, record your own voice ([below](#recording-your-own-voice)) and score +that. + +```bash +uv run python eval/tacet_eval.py # human-readable, worst clips listed +uv run python eval/tacet_eval.py --json # one JSON object on stdout +``` + +It speaks the 44 utterances in `eval/corpus.toml` with macOS `say` in four +voices (US, UK, Australian, Irish). Most of them are terminal and coding-agent +dictation: Postgres, tmux, kubectl, flags, file paths read out loud. It +converts each one with `afconvert` to the 16 kHz mono 16-bit WAV tacet records, +and caches the audio in `eval/.cache/` (gitignored). Then it starts its own +whisper-server on a free loopback port, with `--model` and the flags +`install-server.sh` renders. It POSTs each clip to `/inference` exactly as +`WhisperClient.swift` does, one at a time after one untimed warm-up, and stops +the server at the end. The live services on 8910 and 8911 are never touched. +The first run takes about two minutes. Later runs take about one. + +Pass-throughs for experiments: `--prompt TEXT`, `--beam-size N`, and +`--server-arg=FLAG` for any other whisper-server flag, repeatable and +shell-split (write it with `=`, e.g. `--server-arg="--entropy-thold 2.8"`). +`--model`, `--voices` and `--corpus` swap the other inputs. Every clip's +reference, transcript, errors and latency go to `eval/.cache/last-run.json` for +diffing two runs. + +**What it reports.** Word error rate is total word errors over total reference +words, overall and per voice. It is not clamped, so a model that hallucinates +can score above 1. Latency is the p50 and p95 wall time of the speech requests, +by nearest rank. The silence count is how many of the 7 silence and +near-silence clips came back with a letter or digit. That is the same test +tacet applies before it pastes anything. The report also says how many of those +clips are loud enough to pass tacet's RMS gate (150), since only those would +ever reach a user. + +**Normalization**, applied to both reference and transcript before scoring: + +- Case and punctuation never count. Apostrophes are deleted. +- Symbols read as the words people say for them: `/etc/hosts` scores the + same as "slash etc slash hosts", and `--dry-run` the same as "dash dash dry + run". The eval measures recognition, not formatting. +- Hyphenated compounds close up (`stand-up` = `standup`). +- Whisper splitting a word never counts: "CH migrate" = `ch-migrate`, "TKT + 1687" = `TKT-1687`. Gluing two words together ("Ontomain") still does. +- Spelled-out numbers become digits ("twenty five" = 25), and `7:15` = `7.15` = + "seven fifteen". + +The full rules are in the `normalize()` docstring, and `tests/test_eval.py` +pins them. + +**Baseline, 2026-11-05** (Apple M4 Max, whisper.cpp 1.8.4, +ggml-large-v3-turbo, production flags, empty `--prompt`): + +| | | +|---|---| +| WER | 3.53%, 58 of 1,644 words (Samantha 3.16%, Daniel 2.68%, Karen 4.62%, Moira 3.65%) | +| latency | p50 242 ms, p95 264 ms | +| silence | 2 of 7 clips returned text ("you" on digital silence). Both are below the RMS gate, so tacet never sends them | + +Most of the errors are jargon whisper has never seen. It writes "cube +control" for kubectl, "Maine" for main, "launch deep list" for launchd plist, +"tacit" for tacet, and "post-gaz" for Postgres. That is what `--prompt` is for. +Synthetic voices are cleaner than a real microphone in a real room, so read +these numbers as a fixed yardstick for comparing settings, not as the accuracy +you will get. Some misses are the voice's, not whisper's: the corpus tells +`say` to pronounce kubectl "cube control", as many people do, and whisper +writes exactly that. + +### Recording your own voice + +`eval/record.py` records a script line by line through `eval/recorder.swift`, +which captures the way tacet does (AVAudioEngine's default input, a 4096-frame +tap, AVAudioConverter to 16 kHz mono 16-bit, the same 44-byte header). + +```bash +uv run python eval/record.py --script my-script.toml # resumes where it stopped +uv run python eval/tacet_eval.py --corpus my-script.toml --recordings --split tune +``` + +Press Enter and start talking, as with the hotkey, then Enter again to stop. +Each take can be kept, redone or played back, and the recorder warns about a +take quieter than tacet's gate, one that cut off, or a terminal without +microphone access. Run it in a plain terminal window on the Mac whose +microphone you mean to test; macOS asks once for access. + +Takes go to `~/.local/share/tacet-eval/recordings//.wav`, outside +the repository because they are your voice. A session is one microphone on one +Mac, named after both, and is reported in place of a voice. Beside each take, +`.json` records what was read, where, and how loud it was. A take whose +script line has changed since is refused until you `--redo` it. + +The script uses the corpus format with three optional fields and one table: + +```toml +[[utterance]] +id = "r12" +split = "tune" # or "holdout": tune on one, score the other once +kind = "short" # any label; the report breaks WER down by it +text = "Edit ~/.claude/settings.json" # what should be pasted +spoken = "Edit tilde slash dot claude slash settings dot json" # shown as a reading hint + +[[quiet]] # a take with no speech: any text back is a false paste +id = "q1" +split = "tune" +do = "Take one slow breath in and out. No words." +``` + +Fix the split before anyone sees a score, and keep jargon in the holdout that +never appears in a tune line. That way a `--prompt` built while tuning shows +what it does to words it was never given. + ## What's been tested, and what hasn't One person, two Macs, one microphone. CI runs the pytest suite on Linux and @@ -454,7 +575,8 @@ swift/ Packaging/build-app.sh assembles and signs Tacet.app Tests/ SwiftPM suite (48) config.example.toml shape of ~/.config/tacet/config.toml -tests/ pytest suite — drives the installers as subprocesses +eval/ accuracy + latency harness and its corpus +tests/ pytest suite — the installers as subprocesses, the eval scoring .github/workflows/ci.yml pytest + shellcheck + the signed bundle build docs/ design specs ``` diff --git a/eval/corpus.toml b/eval/corpus.toml new file mode 100644 index 0000000..89ad70d --- /dev/null +++ b/eval/corpus.toml @@ -0,0 +1,269 @@ +# tacet's accuracy corpus, read by eval/tacet_eval.py. +# +# Every [[utterance]] is spoken once per voice by macOS `say`, converted to the +# 16 kHz mono 16-bit WAV tacet records, and sent to a private whisper-server. +# +# id stable name; the cache key and the row in the per-clip report +# text the reference: what a user who said it wants pasted +# spoken optional; what `say` is given when it would mangle `text` (tmux, +# kubectl, file paths). It stands in for how a person says it. It is +# never scored against. +# +# References are written the way the text should come out, symbols included +# (/etc/hosts, --dry-run). The scorer's normalization reads symbols as the words +# a person speaks for them, so "/etc/hosts" and "slash etc slash hosts" score +# the same: the eval measures recognition, not formatting. +# +# Generated audio lives in eval/.cache/, which is gitignored. Editing a line +# here regenerates that line's audio on the next run. + +# Distinct English accents that ship with macOS: US, UK, Australian, Irish. +voices = ["Samantha", "Daniel", "Karen", "Moira"] + +# --- Terminal and coding-agent dictation ------------------------------------ + +[[utterance]] +id = "pg-migration" +text = "Run the Postgres migration before you restart the API server." + +[[utterance]] +id = "k8s-pods" +text = "Use kubectl to list the pods in the staging namespace." +spoken = "Use cube control to list the pods in the staging namespace." + +[[utterance]] +id = "tmux-split" +text = "Split the tmux window and run the tests in the right pane." +spoken = "Split the tee mux window and run the tests in the right pane." + +[[utterance]] +id = "clickhouse-order-by" +text = "The ClickHouse table needs a different ORDER BY key." +spoken = "The ClickHouse table needs a different order by key." + +[[utterance]] +id = "terraform-plan" +text = "Run terraform plan and show me what it wants to destroy." + +[[utterance]] +id = "open-pr" +text = "Open a PR against main and ask for a review." +spoken = "Open a P R against main and ask for a review." + +[[utterance]] +id = "schema-column" +text = "Add a nullable column to the users schema." + +[[utterance]] +id = "git-force-flag" +text = "Push the branch with --force-with-lease." +spoken = "Push the branch with dash dash force with lease." + +[[utterance]] +id = "etc-hosts" +text = "Add the new host to /etc/hosts." +spoken = "Add the new host to slash etc slash hosts." + +[[utterance]] +id = "readme-path" +text = "Update the README.md in the docs folder." +spoken = "Update the read me dot M D in the docs folder." + +[[utterance]] +id = "swift-file" +text = "The bug is in WhisperClient.swift, in the multipart body." +spoken = "The bug is in WhisperClient dot swift, in the multipart body." + +[[utterance]] +id = "clone-repo" +text = "Clone the repo and check out the release branch." + +[[utterance]] +id = "rebase-main" +text = "Rebase onto main and fix the merge conflicts." + +[[utterance]] +id = "cli-dry-run" +text = "Run the installer with --dry-run first." +spoken = "Run the installer with dash dash dry run first." + +[[utterance]] +id = "docker-compose" +text = "Restart the Docker container and tail the logs." + +[[utterance]] +id = "env-var" +text = "Set the DATABASE_URL environment variable." +spoken = "Set the database underscore URL environment variable." + +[[utterance]] +id = "json-response" +text = "The endpoint returns JSON with a text field." + +[[utterance]] +id = "yaml-config" +text = "The workflow YAML is missing a timeout." + +[[utterance]] +id = "grep-todo" +text = "Grep the source tree for every TODO comment." +spoken = "Grep the source tree for every to do comment." + +[[utterance]] +id = "port-number" +text = "The server listens on port 8080 on localhost." +spoken = "The server listens on port eighty eighty on localhost." + +[[utterance]] +id = "unit-tests" +text = "Write unit tests for the parser and run pytest." + +[[utterance]] +id = "launchd-plist" +text = "The launchd plist never loaded after the reboot." +spoken = "The launch D plist never loaded after the reboot." + +[[utterance]] +id = "ssh-mosh" +text = "Connect over SSH, then switch to mosh on the laptop." + +[[utterance]] +id = "clickhouse-mv" +text = "Drop the materialized view and rebuild it from the raw table." + +[[utterance]] +id = "terraform-state" +text = "Do not touch the Terraform state file by hand." + +[[utterance]] +id = "kubernetes-deploy" +text = "Scale the Kubernetes deployment to three replicas." + +[[utterance]] +id = "postgres-index" +text = "Add an index on the created at column in Postgres." + +[[utterance]] +id = "claude-code" +text = "Ask Claude Code to refactor the function into two smaller ones." + +[[utterance]] +id = "home-config" +text = "The config lives in ~/.config/tacet/config.toml." +spoken = "The config lives in tilde slash dot config slash tacet slash config dot tommel." + +[[utterance]] +id = "type-error" +text = "The build fails with a type error in the client module." + +[[utterance]] +id = "npm-install" +text = "Delete the lock file and run npm install again." +spoken = "Delete the lock file and run N P M install again." + +[[utterance]] +id = "http-status" +text = "The health check returns a 503 until the model loads." +spoken = "The health check returns a five oh three until the model loads." + +# --- Ordinary sentences ------------------------------------------------------- + +[[utterance]] +id = "lunch" +text = "Let's grab lunch after the standup tomorrow." + +[[utterance]] +id = "running-late" +text = "I'm running about ten minutes late, start without me." + +[[utterance]] +id = "thanks-review" +text = "Thanks for the quick review, I'll fix those comments tonight." + +[[utterance]] +id = "weather" +text = "It looks like rain this afternoon, so bring an umbrella." + +[[utterance]] +id = "meeting-move" +text = "Can we move the meeting to Thursday at three?" + +[[utterance]] +id = "groceries" +text = "We need milk, eggs, and a loaf of bread." + +[[utterance]] +id = "invoice" +text = "Please send the invoice by the end of the month." + +[[utterance]] +id = "question" +text = "Did you get a chance to look at the draft I sent?" + +[[utterance]] +id = "dog-walk" +text = "I'll walk the dog when I get home." + +[[utterance]] +id = "sounds-good" +text = "Sounds good, let me know if anything changes." + +[[utterance]] +id = "flight" +text = "My flight lands at seven fifteen on Sunday evening." + +[[utterance]] +id = "book" +text = "I finished the book you lent me and loved the ending." + +# --- Silence and near-silence ------------------------------------------------- +# +# Synthesised in Python from a fixed seed, so every run sends the same bytes. +# `dbfs` is the RMS level relative to full scale. tacet drops anything below +# RMS 150 (about -47 dBFS) before whisper sees it, so the clips above that line +# are the ones whose hallucinations would reach a user. +# +# kind = "zero" digital silence +# kind = "white" white noise +# kind = "brown" integrated (room-tone-like) noise +# kind = "hum" a mains-hum sine + +[[silence]] +id = "zero-1s" +kind = "zero" +seconds = 1.0 + +[[silence]] +id = "zero-3s" +kind = "zero" +seconds = 3.0 + +[[silence]] +id = "white-60db" +kind = "white" +seconds = 2.0 +dbfs = -60 + +[[silence]] +id = "brown-50db" +kind = "brown" +seconds = 3.0 +dbfs = -50 + +[[silence]] +id = "white-40db" +kind = "white" +seconds = 2.0 +dbfs = -40 + +[[silence]] +id = "brown-35db" +kind = "brown" +seconds = 3.0 +dbfs = -35 + +[[silence]] +id = "hum-40db" +kind = "hum" +seconds = 2.0 +dbfs = -40 diff --git a/eval/record.py b/eval/record.py new file mode 100644 index 0000000..88109ea --- /dev/null +++ b/eval/record.py @@ -0,0 +1,276 @@ +"""Record your own voice for tacet's eval, one take per corpus line. + + uv run python eval/record.py --script PATH [--session NAME] [--out DIR] + [--split tune|holdout] [--redo ID ...] + +Shows each line of the script and records it through eval/recorder.swift, +which captures the way tacet does (AVAudioEngine's default input, converted to +16 kHz mono 16-bit). Each take is saved as //.wav with an +.json note of what was read, on which Mac and which input. Lines already +recorded are skipped, so a second run picks up where the first stopped. + +Press Enter and start talking, the way you would with the hotkey; press Enter +again when you are done. Then keep the take, redo it, or play it back. +[[quiet]] lines ask for a take with no speech in it: a breath, typing, the room. + +Run it in a plain terminal window (not inside tmux, not over SSH) on the Mac +whose microphone you mean to test. macOS asks once for microphone access for +that terminal app. Score the takes afterwards with: + + uv run python eval/tacet_eval.py --corpus PATH --recordings --split tune +""" + +from __future__ import annotations + +import argparse +import datetime +import json +import platform +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import tacet_eval as te # noqa: E402 + +RECORDER_SOURCE = te.EVAL_DIR / "recorder.swift" +RECORDER_BINARY = te.CACHE_DIR / "tacet-recorder" +SHORT_TAKE_S = 0.6 + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + corpus = te.select_split(te.load_corpus(args.script), args.split) + items = [*corpus.utterances, *corpus.quiet] + with Microphone(build_recorder()) as mic: + folder = args.out / ( + args.session or session_name(platform.node(), mic.device) + ) + todo = pending(items, folder, args.redo) + print(f"{len(todo)} of {len(items)} takes to record into {folder}") + print(f"input: {mic.device}\n") + for n, item in enumerate(todo, 1): + print(f"[{n}/{len(todo)}] {item.id}") + if not record_item(mic, item, folder): + break + except te.EvalError as e: + print(f"record: {e}", file=sys.stderr) + return 1 + except KeyboardInterrupt: + print("\nstopped; takes kept so far are saved", file=sys.stderr) + return 130 + return 0 + + +@dataclass(frozen=True) +class Take: + """What the recorder reported for one take.""" + + frames: int + rms: float + peak: int + + @property + def seconds(self) -> float: + return self.frames / te.SAMPLE_RATE + + +class Microphone: + """The tacet-recorder process: one long-lived capture engine, as in tacet.""" + + def __init__(self, binary: Path): + self.proc = subprocess.Popen( + [str(binary)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + bufsize=1, + ) + self.device = "" + + def __enter__(self) -> Microphone: + reply = self._read() + if reply == "denied": + raise te.EvalError( + "microphone access is denied for this terminal app: allow it in " + "System Settings > Privacy & Security > Microphone, then rerun" + ) + if not reply.startswith("ready "): + raise te.EvalError(f"recorder did not start: {reply!r}") + self.device = reply.removeprefix("ready ") + return self + + def __exit__(self, *exc: object) -> None: + if self.proc.poll() is None: + self._send("quit") + self.proc.wait(timeout=10) + + def start(self) -> None: + self._send("start") + reply = self._read() + if reply != "started": + raise te.EvalError(f"could not start capture: {reply}") + + def stop(self, path: Path) -> Take: + self._send(f"stop {path}") + reply = self._read() + if not reply.startswith("saved "): + raise te.EvalError(f"could not save the take: {reply}") + frames, rms, peak = reply.split()[1:] + return Take(int(frames), float(rms), int(peak)) + + def _send(self, line: str) -> None: + assert self.proc.stdin is not None + self.proc.stdin.write(line + "\n") + self.proc.stdin.flush() + + def _read(self) -> str: + assert self.proc.stdout is not None + line = self.proc.stdout.readline() + if not line: + raise te.EvalError(f"recorder exited with {self.proc.wait()}") + return line.strip() + + +def parse_args(argv: list[str] | None) -> argparse.Namespace: + p = argparse.ArgumentParser(description="Record takes for tacet's eval.") + p.add_argument("--script", type=Path, required=True, help="the corpus TOML to read") + p.add_argument("--session", help="name for this mic and Mac (default host-input)") + p.add_argument("--out", type=Path, default=te.RECORDINGS_DIR) + p.add_argument("--split", choices=te.SPLITS, help="record one split only") + p.add_argument( + "--redo", nargs="+", default=[], metavar="ID", help="re-record these" + ) + return p.parse_args(argv) + + +def build_recorder() -> Path: + """Compile recorder.swift into the cache when it is missing or older.""" + stale = ( + not RECORDER_BINARY.exists() + or RECORDER_BINARY.stat().st_mtime < RECORDER_SOURCE.stat().st_mtime + ) + if stale: + print("building the recorder (once, about 15 s)...", file=sys.stderr) + RECORDER_BINARY.parent.mkdir(parents=True, exist_ok=True) + built = subprocess.run( + ["swiftc", "-O", str(RECORDER_SOURCE), "-o", str(RECORDER_BINARY)], + capture_output=True, + text=True, + ) + if built.returncode != 0: + raise te.EvalError(f"swiftc failed: {built.stderr.strip()[-800:]}") + return RECORDER_BINARY + + +def session_name(host: str, device: str) -> str: + """'Dans-Mac-Studio.local' + 'MacBook Pro Microphone' -> 'dans-mac-studio-macbook-pro-microphone'.""" + return f"{_slug(host.split('.')[0])}-{_slug(device)}" + + +def pending(items: list, folder: Path, redo: list[str]) -> list: + """Lines with no take yet, plus any asked for again, in script order.""" + unknown = sorted(set(redo) - {item.id for item in items}) + if unknown: + raise te.EvalError(f"--redo names ids not in the script: {', '.join(unknown)}") + return [i for i in items if i.id in redo or not (folder / f"{i.id}.wav").exists()] + + +def record_item(mic: Microphone, item: te.Utterance | te.Quiet, folder: Path) -> bool: + """Record one line until it is kept. False when the speaker quits.""" + _show(item) + folder.mkdir(parents=True, exist_ok=True) + draft = folder / f"{item.id}.part.wav" + while True: + if input(" Enter to record, q to quit: ").strip().lower() == "q": + return False + mic.start() + input(" ● recording... Enter to stop ") + take = mic.stop(draft) + print( + f" {take.seconds:.1f} s, level {take.rms:.0f} (tacet drops audio under 150)" + ) + for warning in check_take(item, take): + print(f" ! {warning}") + choice = _choose(draft) + if choice == "keep": + _keep(item, draft, take, mic.device) + return True + if choice == "quit": + draft.unlink(missing_ok=True) + return False + + +def check_take(item: te.Utterance | te.Quiet, take: Take) -> list[str]: + """What a speaker should know before keeping a take.""" + if take.frames == 0: + return ["no audio arrived: is the input device connected?"] + if take.peak == 0: + return [ + "every sample is zero: the input is muted, or this terminal has no " + "microphone access (System Settings > Privacy & Security > Microphone)" + ] + if isinstance(item, te.Quiet): + return [] + warnings = [] + if take.rms < te.SILENCE_RMS_GATE: + warnings.append( + "quieter than tacet's gate, so tacet would drop it: move closer" + ) + if take.seconds < SHORT_TAKE_S: + warnings.append("very short: did it cut off?") + return warnings + + +def _show(item: te.Utterance | te.Quiet) -> None: + if isinstance(item, te.Quiet): + print(f" (no speech) {item.do}") + return + print(f"\n {item.text}\n") + if item.spoken != item.text: + print(f" read it as: {item.spoken}") + + +def _choose(draft: Path) -> str: + while True: + answer = input(" Enter to keep, r to redo, p to play it back, q to quit: ") + answer = answer.strip().lower() + if answer == "": + return "keep" + if answer in ("r", "q"): + return {"r": "redo", "q": "quit"}[answer] + if answer == "p": + subprocess.run(["afplay", str(draft)], check=False) + + +def _keep(item: te.Utterance | te.Quiet, draft: Path, take: Take, device: str) -> None: + wav = draft.with_name(f"{item.id}.wav") + draft.replace(wav) + note = { + "id": item.id, + "script_text": te.script_text(item), + "split": item.split, + "kind": getattr(item, "kind", None), + "session": wav.parent.name, + "host": platform.node(), + "device": device, + "seconds": round(take.seconds, 2), + "rms": round(take.rms, 1), + "peak": take.peak, + "recorded_at": datetime.datetime.now() + .astimezone() + .isoformat(timespec="seconds"), + } + wav.with_suffix(".json").write_text(json.dumps(note, indent=2) + "\n") + + +def _slug(name: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eval/recorder.swift b/eval/recorder.swift new file mode 100644 index 0000000..5e02120 --- /dev/null +++ b/eval/recorder.swift @@ -0,0 +1,153 @@ +// tacet-recorder: microphone capture for eval/record.py, done the way tacet's +// own Recorder (swift/Sources/tacet/Recorder.swift) does it, so a recorded eval +// clip is the audio tacet would have sent: AVAudioEngine's default input, a tap +// with a 4096-frame buffer, AVAudioConverter to 16 kHz mono 16-bit, and the +// canonical 44-byte WAV header. +// +// Built by record.py with `swiftc -O`. It speaks a line protocol: +// stdin: start | stop PATH | quit +// stdout: ready DEVICE-NAME (at launch) | denied | started +// | saved FRAMES RMS PEAK | error MESSAGE +// `tacet-recorder --selftest PATH` writes one second of a 440 Hz tone through +// the same WAV writer and exits, without touching the microphone. + +import AVFoundation +import Foundation + +struct Failure: Error { + let message: String + init(_ message: String) { self.message = message } +} + +final class Capture { + private let engine = AVAudioEngine() + private let queue = DispatchQueue(label: "tacet.eval.recorder") + private var samples = [Int16]() + private var peak: Int32 = 0 + private var started = false + + func start() throws { + queue.sync { + samples.removeAll(keepingCapacity: true) + peak = 0 + } + let input = engine.inputNode + let inFormat = input.inputFormat(forBus: 0) + guard inFormat.sampleRate > 0, inFormat.channelCount > 0 else { + throw Failure("default input reports no usable stream (\(inFormat.channelCount) ch, \(inFormat.sampleRate) Hz)") + } + guard let target = AVAudioFormat(commonFormat: .pcmFormatInt16, + sampleRate: 16000, channels: 1, interleaved: true), + let converter = AVAudioConverter(from: inFormat, to: target) else { + throw Failure("no converter from \(inFormat) to 16 kHz mono 16-bit") + } + input.installTap(onBus: 0, bufferSize: 4096, format: inFormat) { [weak self] buffer, _ in + self?.process(buffer, converter: converter, target: target) + } + engine.prepare() + try engine.start() + started = true + } + + func stop() -> (samples: [Int16], peak: Int32) { + guard started else { return ([], 0) } + started = false + engine.inputNode.removeTap(onBus: 0) + engine.stop() + return queue.sync { (samples, peak) } + } + + private func process(_ buffer: AVAudioPCMBuffer, converter: AVAudioConverter, + target: AVAudioFormat) { + let ratio = target.sampleRate / buffer.format.sampleRate + let capacity = AVAudioFrameCount(Double(buffer.frameLength) * ratio) + 1024 + guard let out = AVAudioPCMBuffer(pcmFormat: target, frameCapacity: capacity) else { return } + var provided = false + var error: NSError? + converter.convert(to: out, error: &error) { _, status in + if provided { status.pointee = .noDataNow; return nil } + provided = true + status.pointee = .haveData + return buffer + } + guard out.frameLength > 0, let channel = out.int16ChannelData else { return } + queue.sync { + for i in 0.. Data { + let dataBytes = samples.count * 2 + var out = Data(capacity: 44 + dataBytes) + func ascii(_ s: String) { out.append(Data(s.utf8)) } + func le16(_ v: Int) { out.append(UInt8(v & 0xFF)); out.append(UInt8((v >> 8) & 0xFF)) } + func le32(_ v: Int) { le16(v & 0xFFFF); le16((v >> 16) & 0xFFFF) } + ascii("RIFF"); le32(36 + dataBytes); ascii("WAVE") + ascii("fmt "); le32(16); le16(1); le16(1); le32(16000); le32(32000); le16(2); le16(16) + ascii("data"); le32(dataBytes) + samples.withUnsafeBytes { out.append(contentsOf: $0) } // little endian on arm64 and x86 + return out +} + +func save(_ samples: [Int16], peak: Int32, to path: String) throws -> String { + try wavData(samples).write(to: URL(fileURLWithPath: path), options: .atomic) + let sumSquares = samples.reduce(0.0) { $0 + Double($1) * Double($1) } + let rms = samples.isEmpty ? 0 : (sumSquares / Double(samples.count)).squareRoot() + return "saved \(samples.count) \(String(format: "%.1f", rms)) \(peak)" +} + +func microphoneAllowed() -> Bool { + switch AVCaptureDevice.authorizationStatus(for: .audio) { + case .authorized: return true + case .notDetermined: + let done = DispatchSemaphore(value: 0) + var granted = false + AVCaptureDevice.requestAccess(for: .audio) { granted = $0; done.signal() } + done.wait() + return granted + default: return false + } +} + +func say(_ line: String) { + print(line) + fflush(stdout) +} + +func serve() -> Int32 { + guard microphoneAllowed() else { say("denied"); return 3 } + say("ready \(AVCaptureDevice.default(for: .audio)?.localizedName ?? "unknown input")") + let capture = Capture() + while let line = readLine() { + let parts = line.split(separator: " ", maxSplits: 1).map(String.init) + do { + switch parts.first { + case "start": try capture.start(); say("started") + case "stop" where parts.count == 2: + let (samples, peak) = capture.stop() + say(try save(samples, peak: peak, to: parts[1])) + case "quit": _ = capture.stop(); return 0 + default: say("error unknown command: \(line)") + } + } catch let failure as Failure { + say("error \(failure.message)") + } catch { + say("error \(error)") + } + } + _ = capture.stop() + return 0 +} + +let arguments = CommandLine.arguments +if arguments.count == 3, arguments[1] == "--selftest" { + let tone = (0..<16000).map { Int16(8000 * sin(2 * Double.pi * 440 * Double($0) / 16000)) } + say((try? save(tone, peak: 8000, to: arguments[2])) ?? "error could not write \(arguments[2])") + exit(0) +} +exit(serve()) diff --git a/eval/tacet_eval.py b/eval/tacet_eval.py new file mode 100644 index 0000000..83fabd5 --- /dev/null +++ b/eval/tacet_eval.py @@ -0,0 +1,1001 @@ +"""Accuracy and latency eval for tacet's whisper-server setup. + + uv run python eval/tacet_eval.py [--model PATH] [--json] + [--prompt TEXT] [--beam-size N] [--server-arg=FLAG ...] + [--voices A,B] [--corpus PATH] [--details PATH] + [--recordings [DIR]] [--split tune|holdout] + +Speaks every utterance in eval/corpus.toml with macOS `say` in several +voices, converts it with `afconvert` to the 16 kHz mono 16-bit WAV tacet's +Recorder sends, starts a PRIVATE whisper-server on a free loopback port with +the flags install-server.sh renders, and POSTs each clip to /inference the way +WhisperClient.swift does. It reports word error rate, request latency, and how +many silence clips came back with text, then stops the server. The live tacet +services (ports 8910 and 8911) are never contacted. + +With --recordings it sends recorded takes instead (eval/record.py makes +them): DIR//.wav for every utterance and [[quiet]] take in the +corpus, one session per microphone and Mac, reported in place of the voices. +--split scores only the corpus lines marked `split = "tune"` or "holdout". + +Standard library only. Generated audio is cached in eval/.cache/ (gitignored). +Progress goes to stderr; with --json, stdout carries exactly one JSON object. + +--server-arg takes a whisper-server flag with its value, shell-split, and can +be repeated. Write it with `=` so argparse does not read the flag as its own: + --server-arg=--no-fallback --server-arg="--entropy-thold 2.8" +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import datetime +import hashlib +import http.client +import json +import math +import os +import platform +import random +import re +import shlex +import shutil +import socket +import subprocess +import sys +import tempfile +import time +import tomllib +import unicodedata +import uuid +import wave +from dataclasses import dataclass, field +from pathlib import Path + +EVAL_DIR = Path(__file__).resolve().parent +DEFAULT_CORPUS = EVAL_DIR / "corpus.toml" +CACHE_DIR = EVAL_DIR / ".cache" +DEFAULT_MODEL = Path.home() / ".local/share/whisper-cpp/ggml-large-v3-turbo.bin" +# Recorded takes live outside git: they are someone's voice. +RECORDINGS_DIR = Path.home() / ".local/share/tacet-eval/recordings" +SPLITS = ("tune", "holdout") + +# tacet's own services. The harness refuses to run a server on either. +LIVE_PORTS = frozenset({8910, 8911}) + +# The flags install-server.sh renders into the whisper plist, minus --model and +# --port, which the harness supplies. --prompt is added only when non-empty, +# exactly as the installer does. +PRODUCTION_FLAGS = ( + "--host", + "127.0.0.1", + "--language", + "en", + "--no-timestamps", + "--suppress-nst", +) + +# The form fields WhisperClient.makeMultipart sends after the file part. +REQUEST_FIELDS = (("response_format", "json"), ("temperature", "0.0")) + +SAMPLE_RATE = 16000 +# tacet's server drops audio whose RMS is below this before whisper sees it +# (audio.silence_rms_threshold, Config.swift). +SILENCE_RMS_GATE = 150.0 +REQUEST_TIMEOUT_S = 60.0 + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + corpus = select_split(load_corpus(args.corpus), args.split) + clips, voices = gather_clips(corpus, args) + settings = run_settings(args, voices) + with WhisperServer(settings, args.model, args.load_timeout) as server: + results = transcribe_all(server, clips) + except EvalError as e: + print(f"tacet_eval: {e}", file=sys.stderr) + return 1 + report = build_report(args.model, settings, results) + _write_details(args.details, report, results) + if args.json: + print(json.dumps(report, indent=2)) + else: + print(format_summary(report, results)) + return 0 + + +class EvalError(RuntimeError): + """A failure that ends the run with a message rather than a traceback.""" + + +@dataclass(frozen=True) +class Utterance: + id: str + text: str + spoken: str + split: str | None = None + kind: str | None = None + + +@dataclass(frozen=True) +class Quiet: + """A recorded take with no speech in it: any text back is a false paste.""" + + id: str + do: str + split: str | None = None + + +@dataclass(frozen=True) +class Silence: + id: str + kind: str + seconds: float + dbfs: float | None + + +@dataclass(frozen=True) +class Corpus: + voices: list[str] + utterances: list[Utterance] + silences: list[Silence] + quiet: list[Quiet] = field(default_factory=list) + + +@dataclass(frozen=True) +class Clip: + """One WAV to send: an utterance in one voice or recording session, or a + clip with no speech (generated silence has no voice; a quiet take does).""" + + id: str + voice: str | None + reference: str + path: Path + silent: bool = False + kind: str | None = None + + @property + def is_silence(self) -> bool: + return self.silent or self.voice is None + + +@dataclass(frozen=True) +class ClipResult: + clip: Clip + text: str + ms: float + rms: float + + @property + def reference_words(self) -> list[str]: + return normalize(self.clip.reference) + + @property + def errors(self) -> int: + reference = self.reference_words + return word_errors(reference, join_split_words(reference, normalize(self.text))) + + +@dataclass(frozen=True) +class Settings: + """What was measured: the server command line and the request fields.""" + + binary: str + prompt: str + beam_size: int | None + server_args: list[str] + voices: list[str] + corpus: str + split: str | None = None + recordings: str | None = None + flags: list[str] = field(init=False) + + def __post_init__(self) -> None: + flags = list(PRODUCTION_FLAGS) + if self.prompt: + flags += ["--prompt", self.prompt] + if self.beam_size is not None: + flags += ["--beam-size", str(self.beam_size)] + object.__setattr__(self, "flags", flags + self.server_args) + + def as_dict(self) -> dict: + return { + "whisper_server": self.binary, + "flags": self.flags, + "prompt": self.prompt, + "beam_size": self.beam_size, + "server_args": self.server_args, + "request": dict(REQUEST_FIELDS), + "voices": self.voices, + "corpus": self.corpus, + "split": self.split, + "recordings": self.recordings, + "host": platform.node(), + "date": datetime.date.today().isoformat(), + } + + +class WhisperServer: + """A private whisper-server on a free loopback port, stopped on exit.""" + + def __init__(self, settings: Settings, model: Path, load_timeout: float): + self.settings = settings + self.model = model + self.load_timeout = load_timeout + self.port = 0 + self.proc: subprocess.Popen | None = None + self.log_path = CACHE_DIR / "whisper-server.log" + + def __enter__(self) -> WhisperServer: + if not self.model.is_file(): + raise EvalError(f"model not found: {self.model}") + if not os.access(self.settings.binary, os.X_OK): + raise EvalError(f"whisper-server not found: {self.settings.binary}") + self.port = _free_port() + command = [ + self.settings.binary, + "--model", + str(self.model), + "--port", + str(self.port), + *self.settings.flags, + ] + _log(f"starting whisper-server on 127.0.0.1:{self.port}") + with open(self.log_path, "wb") as log: + self.proc = subprocess.Popen( + command, + cwd=CACHE_DIR, + stdin=subprocess.DEVNULL, + stdout=log, + stderr=subprocess.STDOUT, + ) + return self + + def __exit__(self, *exc: object) -> None: + if self.proc is None or self.proc.poll() is not None: + return + self.proc.terminate() + try: + self.proc.wait(timeout=10) + except subprocess.TimeoutExpired: + self.proc.kill() + self.proc.wait() + + def wait_until_ready(self, probe: bytes) -> None: + """POST a speech clip until it succeeds: that is load plus warm-up.""" + deadline = time.monotonic() + self.load_timeout + while time.monotonic() < deadline: + self._check_alive() + try: + self.transcribe(probe) + return + except (ConnectionError, socket.timeout, _NotReady): + time.sleep(0.5) + raise EvalError(f"whisper-server did not load in {self.load_timeout:.0f}s") + + def transcribe(self, wav: bytes) -> tuple[str, float]: + """Send one WAV; return the text and the request's wall time in ms.""" + boundary = f"TacetBoundary{str(uuid.uuid4()).upper()}" + body = multipart_body(boundary, wav) + conn = http.client.HTTPConnection( + "127.0.0.1", self.port, timeout=REQUEST_TIMEOUT_S + ) + headers = {"Content-Type": f"multipart/form-data; boundary={boundary}"} + start = time.perf_counter() + try: + conn.request("POST", "/inference", body=body, headers=headers) + response = conn.getresponse() + payload = response.read() + finally: + conn.close() + ms = (time.perf_counter() - start) * 1000 + if response.status == 503: + raise _NotReady() + return _response_text(response.status, payload), ms + + def _check_alive(self) -> None: + assert self.proc is not None + if self.proc.poll() is not None: + tail = self.log_path.read_text(errors="replace").splitlines()[-15:] + raise EvalError( + f"whisper-server exited with {self.proc.returncode}:\n" + + "\n".join(tail) + ) + + +def parse_args(argv: list[str] | None) -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Measure tacet's whisper-server accuracy and latency.", + epilog="Pass whisper-server flags as --server-arg=FLAG (with '=').", + ) + p.add_argument("--model", type=Path, default=DEFAULT_MODEL) + p.add_argument("--json", action="store_true", help="print one JSON object") + p.add_argument("--prompt", default="", help="whisper-server --prompt") + p.add_argument("--beam-size", type=int, help="whisper-server --beam-size") + p.add_argument( + "--server-arg", + action="append", + default=[], + dest="server_args", + help="extra whisper-server flag, shell-split; repeatable", + ) + p.add_argument( + "--voices", + type=_voice_list, + help="comma-separated say voices, or recording sessions with --recordings", + ) + p.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS) + p.add_argument( + "--recordings", + type=Path, + nargs="?", + const=RECORDINGS_DIR, + help=f"send recorded takes from DIR (default {RECORDINGS_DIR})", + ) + p.add_argument("--split", choices=SPLITS, help="score one split of the corpus") + p.add_argument("--whisper-server", dest="binary", default=_default_binary()) + p.add_argument("--details", type=Path, default=CACHE_DIR / "last-run.json") + p.add_argument("--load-timeout", type=float, default=300.0) + args = p.parse_args(argv) + args.server_args = [token for arg in args.server_args for token in shlex.split(arg)] + return args + + +def load_corpus(path: Path) -> Corpus: + data = tomllib.loads(path.read_text()) + utterances = [ + Utterance( + u["id"], + u["text"], + u.get("spoken", u["text"]), + u.get("split"), + u.get("kind"), + ) + for u in data.get("utterance", []) + ] + silences = [ + Silence(s["id"], s["kind"], float(s["seconds"]), s.get("dbfs")) + for s in data.get("silence", []) + ] + quiet = [Quiet(q["id"], q["do"], q.get("split")) for q in data.get("quiet", [])] + corpus = Corpus(list(data.get("voices", [])), utterances, silences, quiet) + _check_corpus(corpus) + return corpus + + +def select_split(corpus: Corpus, split: str | None) -> Corpus: + """Keep only the lines of one split. Generated silence has no split and is + kept either way; it never reaches a tuning decision on its own.""" + if split is None: + return corpus + utterances = [u for u in corpus.utterances if u.split == split] + if not utterances: + raise EvalError(f"no utterances marked split = {split!r} in the corpus") + quiet = [q for q in corpus.quiet if q.split == split] + return Corpus(corpus.voices, utterances, corpus.silences, quiet) + + +def gather_clips( + corpus: Corpus, args: argparse.Namespace +) -> tuple[list[Clip], list[str]]: + """The clips to send and the voices (or recording sessions) they come in.""" + if args.recordings is None: + voices = args.voices or corpus.voices + return prepare_audio(corpus, voices), voices + sessions = recorded_sessions(args.recordings, args.voices) + return load_recordings(corpus, args.recordings, sessions), sessions + + +def recorded_sessions(root: Path, wanted: list[str] | None) -> list[str]: + """Session directories under `root` that hold takes, or the ones asked for.""" + if not root.is_dir(): + raise EvalError(f"no recordings directory: {root} (record with eval/record.py)") + found = sorted( + d.name for d in root.iterdir() if d.is_dir() and any(d.glob("*.wav")) + ) + missing = [w for w in wanted or [] if w not in found] + if missing: + raise EvalError( + f"no takes for {', '.join(missing)} in {root} (found: {', '.join(found) or 'none'})" + ) + if not (wanted or found): + raise EvalError(f"no recording sessions in {root}") + return list(wanted or found) + + +def load_recordings(corpus: Corpus, root: Path, sessions: list[str]) -> list[Clip]: + """Every utterance and quiet take, once per session. A missing take, or one + recorded for different text, stops the run: a partial set is a different + yardstick.""" + clips, problems = [], [] + for session in sessions: + for item in [*corpus.utterances, *corpus.quiet]: + path = root / session / f"{item.id}.wav" + problem = _take_problem(path, script_text(item)) + if problem: + problems.append(f"{session}/{item.id}: {problem}") + elif isinstance(item, Quiet): + clips.append(Clip(item.id, session, "", path, silent=True)) + else: + clips.append(Clip(item.id, session, item.text, path, kind=item.kind)) + if problems: + shown = "\n ".join(problems[:20]) + more = f"\n ...and {len(problems) - 20} more" if len(problems) > 20 else "" + raise EvalError( + f"recorded takes need attention (eval/record.py):\n {shown}{more}" + ) + return clips + + +def prepare_audio(corpus: Corpus, voices: list[str]) -> list[Clip]: + """Generate any WAV missing from the cache; return every clip to send.""" + if not voices: + raise EvalError("no voices: set `voices` in the corpus or pass --voices") + _check_tools(voices) + CACHE_DIR.mkdir(parents=True, exist_ok=True) + jobs = [(u, v) for v in voices for u in corpus.utterances] + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + paths = list(pool.map(lambda job: _speech_wav(*job), jobs)) + clips = [ + Clip(u.id, v, u.text, path, kind=u.kind) for (u, v), path in zip(jobs, paths) + ] + clips += [Clip(s.id, None, "", _silence_wav(s)) for s in corpus.silences] + return clips + + +def script_text(item: Utterance | Quiet) -> str: + """What the speaker was shown for a take: the line, or a quiet take's instruction.""" + return item.do if isinstance(item, Quiet) else item.text + + +def run_settings(args: argparse.Namespace, voices: list[str]) -> Settings: + try: + corpus = str(args.corpus.resolve().relative_to(EVAL_DIR.parent)) + except ValueError: + corpus = str(args.corpus) + return Settings( + binary=os.path.realpath(args.binary), + prompt=args.prompt, + beam_size=args.beam_size, + server_args=args.server_args, + voices=voices, + corpus=corpus, + split=args.split, + recordings=str(args.recordings) if args.recordings else None, + ) + + +def transcribe_all(server: WhisperServer, clips: list[Clip]) -> list[ClipResult]: + """Send every clip one at a time, as tacet does, after an untimed warm-up.""" + first_speech = next(c for c in clips if not c.is_silence) + server.wait_until_ready(first_speech.path.read_bytes()) + _log(f"server ready; transcribing {len(clips)} clips") + results = [] + for n, clip in enumerate(clips, 1): + wav = clip.path.read_bytes() + try: + text, ms = server.transcribe(wav) + except (OSError, _NotReady) as e: + raise EvalError( + f"request for {clip.id} ({clip.voice}) failed: {e!r}" + ) from e + results.append(ClipResult(clip, text, ms, wav_rms(wav))) + if n % 20 == 0 or n == len(clips): + _log(f" {n}/{len(clips)}") + return results + + +def build_report(model: Path, settings: Settings, results: list[ClipResult]) -> dict: + speech = [r for r in results if not r.clip.is_silence] + silence = [r for r in results if r.clip.is_silence] + latencies = [r.ms for r in speech] + kinds = sorted({r.clip.kind for r in speech if r.clip.kind}) + return { + "model": str(model), + "settings": settings.as_dict(), + "utterances": len({r.clip.id for r in speech}), + "wer": round(corpus_wer(speech), 4), + "per_voice": { + voice: _voice_score([r for r in speech if r.clip.voice == voice]) + for voice in settings.voices + }, + "per_kind": { + kind: _voice_score([r for r in speech if r.clip.kind == kind]) + for kind in kinds + }, + "latency_ms": { + "p50": round(percentile(latencies, 50), 1), + "p95": round(percentile(latencies, 95), 1), + }, + "silence": { + "clips": len(silence), + "false_text": sum(has_text(r.text) for r in silence), + "false_text_past_gate": sum( + has_text(r.text) and r.rms >= SILENCE_RMS_GATE for r in silence + ), + }, + } + + +def format_summary(report: dict, results: list[ClipResult], worst: int = 12) -> str: + lat, sil = report["latency_ms"], report["silence"] + lines = [ + f"model {report['model']}", + f"flags {shlex.join(report['settings']['flags'])}", + f"WER {report['wer']:.2%} over {report['utterances']} utterances", + ] + for voice, score in report["per_voice"].items(): + lines.append( + f" {voice:<10} {score['wer']:.2%} ({score['errors']}/{score['words']} words)" + ) + for kind, score in report["per_kind"].items(): + lines.append( + f" [{kind}]{'':<{8 - len(kind)}} {score['wer']:.2%} ({score['errors']}/{score['words']} words)" + ) + lines.append(f"latency p50 {lat['p50']} ms, p95 {lat['p95']} ms") + lines.append( + f"silence {sil['false_text']}/{sil['clips']} clips came back with text " + f"({sil['false_text_past_gate']} loud enough to pass tacet's RMS gate)" + ) + speech = [r for r in results if not r.clip.is_silence] + misses = sorted((r for r in speech if r.errors), key=lambda r: -r.errors) + if misses: + lines.append(f"\nworst clips (of {len(misses)} with errors):") + for r in misses[:worst]: + lines.append(f" [{r.errors}] {r.clip.id} ({r.clip.voice})") + lines.append(f" ref: {r.clip.reference}") + lines.append(f" hyp: {r.text.strip()}") + return "\n".join(lines) + + +def normalize(text: str) -> list[str]: + """Reduce a transcript to comparable words. Applied to both sides. + + 1. Unicode NFKC; curly apostrophes become straight ones. + 2. Symbols become the words people say for them, so a path or flag scores + the same written or spoken: "/" slash, "." inside a token dot (between + digits, point), a leading "-" or "--" dash, "_" underscore, "~" tilde, + "@" at, "%" percent, "&" and, "+" plus, "=" equals. + 3. Hyphenated compounds are closed up ("stand-up" -> "standup", "T-Mux" + -> "tmux"), except inside a flag ("--dry-run" -> "dash dash dry run") + and between number words ("twenty-five" -> "25"). + 4. Lowercase. Apostrophes are deleted ("don't" -> "dont"); every other + punctuation mark becomes a space, so case and punctuation never count. + 5. Numbers: thousands separators are dropped ("1,000" -> "1000"), a clock + time reads as two numbers ("7:15" and "7.15" -> "7 15"), and spelled-out + cardinals become digits ("twenty five" -> "25", "three thousand and" + -> "3000 and"). Runs that are not one number stay split ("seven + fifteen" -> "7 15"). Ordinals and "oh" are left alone. + """ + text = unicodedata.normalize("NFKC", text).replace("’", "'").replace("‘", "'") + text = re.sub(r"(? int: + """Levenshtein distance over words: substitutions + deletions + insertions.""" + previous = list(range(len(hypothesis) + 1)) + for i, ref_word in enumerate(reference, 1): + current = [i] + for j, hyp_word in enumerate(hypothesis, 1): + current.append( + min( + previous[j] + 1, + current[j - 1] + 1, + previous[j - 1] + (ref_word != hyp_word), + ) + ) + previous = current + return previous[-1] + + +def join_split_words(reference: list[str], hypothesis: list[str]) -> list[str]: + """Whisper splitting a word never counts: "CH migrate" scores the same as + ch-migrate ("chmigrate"), and "TKT 1687" the same as TKT-1687. Two or three + adjacent hypothesis words are joined when the joined form is a reference + word. Gluing two words together ("Ontomain" for "onto main") still counts.""" + return _join_runs(hypothesis, set(reference)) + + +def corpus_wer(results: list[ClipResult]) -> float: + """Micro-averaged WER: total word errors over total reference words.""" + words = sum(len(r.reference_words) for r in results) + return sum(r.errors for r in results) / words if words else 0.0 + + +def percentile(values: list[float], pct: float) -> float: + """Nearest-rank percentile: the smallest value with pct% at or below it.""" + if not values: + return 0.0 + ordered = sorted(values) + rank = max(1, math.ceil(pct / 100 * len(ordered))) + return ordered[rank - 1] + + +def has_text(transcript: str) -> bool: + """tacet pastes a transcript only if it holds a letter or digit.""" + return any(ch.isalnum() for ch in transcript) + + +def multipart_body(boundary: str, wav: bytes) -> bytes: + """The body WhisperClient.makeMultipart builds, byte for byte.""" + parts = [ + f"--{boundary}\r\n".encode(), + b'Content-Disposition: form-data; name="file"; filename="audio.wav"\r\n', + b"Content-Type: audio/wav\r\n\r\n", + wav, + b"\r\n", + ] + for name, value in REQUEST_FIELDS: + parts.append(f"--{boundary}\r\n".encode()) + parts.append(f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode()) + parts.append(f"{value}\r\n".encode()) + parts.append(f"--{boundary}--\r\n".encode()) + return b"".join(parts) + + +def wav_rms(wav: bytes) -> float: + """RMS of the 16-bit samples after a canonical 44-byte header, as tacet measures.""" + data = wav[44:] + pcm = memoryview(data[: len(data) - len(data) % 2]).cast("h") + return math.sqrt(sum(s * s for s in pcm) / len(pcm)) if len(pcm) else 0.0 + + +class _NotReady(Exception): + """whisper-server answered 503: still loading the model.""" + + +class _NumberRun: + """Accumulates consecutive spelled-out number words into one integer.""" + + def __init__(self) -> None: + self.total = 0 + self.current = 0 + self.last: str | None = None + + def take(self, word: str) -> bool: + """Absorb `word` if it extends this number; False if it cannot.""" + kind, value = _number_word(word) + if kind is None or self.last not in _CAN_FOLLOW[kind]: + return False + if kind == "hundred": + if self.current >= 100: + return False + self.current = (self.current or 1) * 100 + elif kind == "scale": + self.total += (self.current or 1) * value + self.current = 0 + else: + self.current += value + self.last = kind + return True + + def flush(self) -> list[str]: + if self.last is None: + return [] + value = self.total + self.current + self.total, self.current, self.last = 0, 0, None + return [str(value)] + + +_SYMBOL_WORDS = { + "/": "slash", + "_": "underscore", + "~": "tilde", + "@": "at", + "%": "percent", + "&": "and", + "+": "plus", + "=": "equals", +} + +_UNITS = { + w: i + for i, w in enumerate( + "zero one two three four five six seven eight nine ten eleven twelve " + "thirteen fourteen fifteen sixteen seventeen eighteen nineteen".split() + ) +} +_TENS = { + w: 10 * i + for i, w in enumerate( + "twenty thirty forty fifty sixty seventy eighty ninety".split(), 2 + ) +} +_SCALES = {"thousand": 1000, "million": 1_000_000, "billion": 1_000_000_000} + +# Which word kinds may follow which, inside one number. `None` is "nothing yet". +_CAN_FOLLOW = { + "zero": {None}, + "unit": {None, "tens", "hundred", "scale"}, + "teen": {None, "hundred", "scale"}, + "tens": {None, "hundred", "scale"}, + "hundred": {None, "unit", "teen", "tens"}, + "scale": {None, "unit", "teen", "tens", "hundred"}, +} + +_SILENCE_KINDS = ("zero", "white", "brown", "hum") + + +def _check_corpus(corpus: Corpus) -> None: + items = [*corpus.utterances, *corpus.silences, *corpus.quiet] + ids = [x.id for x in items] + duplicates = sorted({i for i in ids if ids.count(i) > 1}) + if duplicates: + raise EvalError(f"duplicate corpus ids: {', '.join(duplicates)}") + unknown = sorted({s.kind for s in corpus.silences} - set(_SILENCE_KINDS)) + if unknown: + raise EvalError(f"unknown silence kinds: {', '.join(unknown)}") + splits = {x.split for x in [*corpus.utterances, *corpus.quiet]} - {None, *SPLITS} + if splits: + raise EvalError(f"unknown splits: {', '.join(sorted(splits))}") + + +def _take_problem(path: Path, script_text: str) -> str | None: + """Why a recorded take cannot be scored, or None. The note record.py writes + beside each take says what was read; if the script line changed since, the + take no longer matches its reference.""" + if not path.is_file(): + return "not recorded" + with wave.open(str(path), "rb") as w: + shape = (w.getnchannels(), w.getsampwidth(), w.getframerate()) + if shape != (1, 2, SAMPLE_RATE) or path.read_bytes()[36:40] != b"data": + return f"not a canonical 16 kHz mono 16-bit WAV ({shape})" + note = path.with_suffix(".json") + if ( + note.is_file() + and json.loads(note.read_text()).get("script_text") != script_text + ): + return "recorded for different text; redo it" + return None + + +def _join_runs(words: list[str], targets: set[str]) -> list[str]: + out, i = [], 0 + while i < len(words): + for n in (3, 2): + joined = "".join(words[i : i + n]) + if i + n <= len(words) and joined in targets: + out.append(joined) + i += n + break + else: + out.append(words[i]) + i += 1 + return out + + +def _voice_list(value: str) -> list[str]: + return [v.strip() for v in value.split(",") if v.strip()] + + +def _default_binary() -> str: + return shutil.which("whisper-server") or "/opt/homebrew/bin/whisper-server" + + +def _check_tools(voices: list[str]) -> None: + for tool in ("say", "afconvert"): + if shutil.which(tool) is None: + raise EvalError(f"`{tool}` not found; the eval needs macOS") + listing = subprocess.run( + ["say", "-v", "?"], capture_output=True, text=True, check=True + ) + installed = { + m.group(1) for m in re.finditer(r"^(.+?)\s{2,}\w\w_\w+", listing.stdout, re.M) + } + missing = [v for v in voices if v not in installed] + if missing: + raise EvalError(f"say voices not installed: {', '.join(missing)}") + + +def _speech_wav(utterance: Utterance, voice: str) -> Path: + key = _digest(voice, utterance.spoken) + dest = CACHE_DIR / "audio" / _slug(voice) / f"{utterance.id}-{key}.wav" + if dest.exists(): + return dest + dest.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=CACHE_DIR) as tmp: + script, aiff, converted = ( + Path(tmp, "text.txt"), + Path(tmp, "say.aiff"), + Path(tmp, "c.wav"), + ) + script.write_text(utterance.spoken) + _run(["say", "-v", voice, "-o", str(aiff), "-f", str(script)]) + _run( + [ + "afconvert", + "-f", + "WAVE", + "-d", + f"LEI16@{SAMPLE_RATE}", + "-c", + "1", + str(aiff), + str(converted), + ] + ) + _canonical_wav(converted, dest) + return dest + + +def _canonical_wav(source: Path, dest: Path) -> None: + """Re-wrap afconvert's output (which carries a FLLR pad chunk) in the plain + 44-byte header tacet's Recorder writes.""" + with wave.open(str(source), "rb") as w: + shape = (w.getnchannels(), w.getsampwidth(), w.getframerate()) + if shape != (1, 2, SAMPLE_RATE): + raise EvalError(f"afconvert produced {shape}, not 16 kHz mono 16-bit") + frames = w.readframes(w.getnframes()) + _write_wav(dest, frames) + + +def _silence_wav(clip: Silence) -> Path: + key = _digest(clip.kind, str(clip.seconds), str(clip.dbfs)) + dest = CACHE_DIR / "audio" / "silence" / f"{clip.id}-{key}.wav" + if not dest.exists(): + dest.parent.mkdir(parents=True, exist_ok=True) + samples = _silence_samples(clip) + _write_wav( + dest, b"".join(s.to_bytes(2, "little", signed=True) for s in samples) + ) + return dest + + +def _silence_samples(clip: Silence) -> list[int]: + """Deterministic samples scaled to the clip's RMS level in dBFS.""" + n = int(clip.seconds * SAMPLE_RATE) + if clip.kind == "zero": + return [0] * n + rng = random.Random(clip.id) + if clip.kind == "white": + raw = [rng.gauss(0.0, 1.0) for _ in range(n)] + elif clip.kind == "brown": + raw, level = [], 0.0 + for _ in range(n): + level = 0.995 * level + rng.gauss(0.0, 1.0) + raw.append(level) + else: + hz = 2 * math.pi / SAMPLE_RATE + raw = [math.sin(60 * hz * t) + 0.3 * math.sin(120 * hz * t) for t in range(n)] + rms = math.sqrt(sum(x * x for x in raw) / n) + gain = 32768 * 10 ** ((clip.dbfs if clip.dbfs is not None else -60) / 20) / rms + return [max(-32768, min(32767, round(x * gain))) for x in raw] + + +def _write_wav(dest: Path, frames: bytes) -> None: + partial = dest.with_name(dest.name + ".part") + with wave.open(str(partial), "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(SAMPLE_RATE) + w.writeframes(frames) + partial.replace(dest) + + +def _free_port() -> int: + """A free loopback port from the kernel, never one of tacet's own.""" + for _ in range(20): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + if port not in LIVE_PORTS: + return port + raise EvalError("could not find a free loopback port") + + +def _response_text(status: int, payload: bytes) -> str: + if not 200 <= status < 300: + raise EvalError(f"whisper-server returned HTTP {status}: {payload[:200]!r}") + try: + text = json.loads(payload)["text"] + except (ValueError, KeyError, TypeError) as e: + raise EvalError( + f"malformed response from whisper-server: {payload[:200]!r}" + ) from e + if not isinstance(text, str): + raise EvalError(f"malformed response from whisper-server: {payload[:200]!r}") + return text + + +def _voice_score(results: list[ClipResult]) -> dict: + return { + "wer": round(corpus_wer(results), 4), + "errors": sum(r.errors for r in results), + "words": sum(len(r.reference_words) for r in results), + } + + +def _write_details(path: Path, report: dict, results: list[ClipResult]) -> None: + """Every clip's reference, hypothesis, errors and latency, for diffing runs.""" + clips = [ + { + "id": r.clip.id, + "voice": r.clip.voice, + "reference": r.clip.reference, + "hypothesis": r.text, + "errors": r.errors, + "words": len(r.reference_words), + "ms": round(r.ms, 1), + "rms": round(r.rms, 1), + } + for r in results + ] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({**report, "clips": clips}, indent=2) + "\n") + _log(f"per-clip detail written to {path}") + + +def _speak_dots(text: str) -> str: + """Read "." the way it is spoken. Runs before lowercasing, because a period + followed by a capital is a sentence break whisper glued shut.""" + text = re.sub(r"(?<=\d)\.(?=\d)", " point ", text) + text = re.sub(r"\.(?=[A-Z])", " ", text) + return re.sub(r"(? str: + return " dash " * len(match.group(1)) + match.group(2).replace("-", " ") + + +def _close_compound(match: re.Match) -> str: + parts = match.group(0).split("-") + if all(_number_word(part)[0] for part in parts): + return " ".join(parts) + return "".join(parts) + + +def _words_to_numbers(words: list[str]) -> list[str]: + out: list[str] = [] + run = _NumberRun() + for word in words: + if run.take(word): + continue + out += run.flush() + if not run.take(word): + out.append(word) + return out + run.flush() + + +def _number_word(word: str) -> tuple[str | None, int]: + if word in _UNITS: + value = _UNITS[word] + return ("zero" if value == 0 else "unit" if value < 10 else "teen"), value + if word in _TENS: + return "tens", _TENS[word] + if word == "hundred": + return "hundred", 100 + if word in _SCALES: + return "scale", _SCALES[word] + return None, 0 + + +def _digest(*parts: str) -> str: + return hashlib.sha256("\0".join(parts).encode()).hexdigest()[:12] + + +def _slug(name: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") + + +def _run(command: list[str]) -> None: + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode != 0: + raise EvalError(f"{command[0]} failed: {result.stderr.strip()}") + + +def _log(message: str) -> None: + print(message, file=sys.stderr, flush=True) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_eval.py b/tests/test_eval.py new file mode 100644 index 0000000..e5f5618 --- /dev/null +++ b/tests/test_eval.py @@ -0,0 +1,403 @@ +"""The eval harness's scoring: normalization, WER, percentiles, and its inputs. + +None of this needs macOS or a whisper-server. The harness is loaded from its +file because eval/ is a directory of scripts, not a package (and `eval` would +shadow the builtin if it were one). +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +_spec = importlib.util.spec_from_file_location( + "tacet_eval", ROOT / "eval" / "tacet_eval.py" +) +te = importlib.util.module_from_spec(_spec) +sys.modules["tacet_eval"] = te +_spec.loader.exec_module(te) + + +def result(reference: str, hypothesis: str, voice: str = "Samantha"): + clip = te.Clip("x", voice, reference, Path("/nonexistent.wav")) + return te.ClipResult(clip, hypothesis, ms=1.0, rms=1000.0) + + +class TestNormalizeCaseAndPunctuation: + def test_case_and_punctuation_never_count(self): + assert te.normalize("Hello, World!") == te.normalize("hello world") + + def test_apostrophes_are_deleted_not_spaced(self): + assert te.normalize("Don't") == ["dont"] + assert te.normalize("I’ll") == ["ill"] + + def test_empty_and_punctuation_only(self): + assert te.normalize("") == [] + assert te.normalize(" . ") == [] + + def test_sentence_glued_by_whisper_splits(self): + assert te.normalize("Done.Next line") == ["done", "next", "line"] + + def test_hyphenated_compound_closes_up(self): + assert te.normalize("stand-up") == te.normalize("standup") + assert te.normalize("T-Mux") == ["tmux"] + + +class TestNormalizeSymbols: + def test_written_path_scores_as_spoken_path(self): + assert te.normalize("/etc/hosts") == te.normalize("slash etc slash hosts") + + def test_dotted_filename(self): + assert te.normalize("README.md") == ["readme", "dot", "md"] + + def test_dotfile_in_a_home_path(self): + assert te.normalize("~/.config") == ["tilde", "slash", "dot", "config"] + + def test_long_flag_keeps_its_words(self): + spoken = "dash dash force with lease" + assert te.normalize("--force-with-lease") == te.normalize(spoken) + + def test_short_flag(self): + assert te.normalize("-v") == ["dash", "v"] + + def test_free_standing_dash_is_punctuation(self): + assert te.normalize("a - b") == ["a", "b"] + + def test_underscore(self): + assert te.normalize("DATABASE_URL") == ["database", "underscore", "url"] + + +class TestNormalizeNumbers: + @pytest.mark.parametrize( + "spoken, digits", + [ + ("ten", "10"), + ("twenty five", "25"), + ("twenty-five", "25"), + ("one hundred five", "105"), + ("three thousand five hundred twenty one", "3521"), + ], + ) + def test_spelled_cardinals_become_digits(self, spoken, digits): + assert te.normalize(spoken) == [digits] + + def test_thousands_separator_dropped(self): + assert te.normalize("8,080") == ["8080"] + + def test_runs_that_are_not_one_number_stay_split(self): + assert te.normalize("seven fifteen") == ["7", "15"] + assert te.normalize("nineteen eighty four") == ["19", "84"] + assert te.normalize("one two three") == ["1", "2", "3"] + + def test_clock_time_either_separator(self): + assert ( + te.normalize("7:15") + == te.normalize("7.15") + == te.normalize("seven fifteen") + ) + + def test_decimal_reads_as_point(self): + assert te.normalize("3.5") == te.normalize("three point five") + + def test_numbers_inside_words_are_left_alone(self): + assert te.normalize("ten minutes late") == ["10", "minutes", "late"] + assert te.normalize("tenant") == ["tenant"] + + +class TestWordErrors: + def test_identical(self): + assert te.word_errors(["a", "b"], ["a", "b"]) == 0 + + def test_substitution(self): + assert te.word_errors(["a", "b", "c"], ["a", "x", "c"]) == 1 + + def test_deletion(self): + assert te.word_errors(["a", "b", "c"], ["a", "c"]) == 1 + + def test_insertion(self): + assert te.word_errors(["a", "c"], ["a", "b", "c"]) == 1 + + def test_empty_hypothesis_deletes_everything(self): + assert te.word_errors(["a", "b", "c"], []) == 3 + + def test_empty_reference_counts_insertions(self): + assert te.word_errors([], ["you"]) == 1 + + +class TestCorpusWer: + def test_micro_average_weights_by_reference_words(self): + # One error in a 1-word clip and none in a 3-word clip: 1/4, not the + # 0.5 a per-clip average would give. + results = [result("hello", "goodbye"), result("one two three", "1 2 3")] + assert te.corpus_wer(results) == pytest.approx(0.25) + + def test_normalization_applies_to_both_sides(self): + assert ( + te.corpus_wer( + [result("Add it to /etc/hosts.", "add it to slash etc slash hosts")] + ) + == 0.0 + ) + + def test_insertions_can_push_past_one(self): + # Not clamped: WER above 1 is a real signal (a hallucinating model). + assert te.corpus_wer([result("yes", "yes yes yes")]) == pytest.approx(2.0) + + def test_no_reference_words(self): + assert te.corpus_wer([]) == 0.0 + + +class TestPercentile: + def test_nearest_rank(self): + values = list(range(1, 101)) + assert te.percentile(values, 50) == 50 + assert te.percentile(values, 95) == 95 + + def test_small_sample(self): + assert te.percentile([30.0, 10.0, 20.0], 50) == 20.0 + assert te.percentile([30.0, 10.0, 20.0], 95) == 30.0 + + def test_empty(self): + assert te.percentile([], 50) == 0.0 + + +class TestSilenceText: + @pytest.mark.parametrize("text", ["", " ", ".", " . "]) + def test_punctuation_is_not_text(self, text): + assert not te.has_text(text) + + @pytest.mark.parametrize("text", ["you", " Thank you.", "[BLANK_AUDIO]"]) + def test_any_letter_is_text(self, text): + assert te.has_text(text) + + +class TestCorpus: + def test_shipped_corpus_meets_the_floor(self): + corpus = te.load_corpus(te.DEFAULT_CORPUS) + assert len({u.id for u in corpus.utterances}) >= 40 + assert len(corpus.silences) >= 5 + assert len(corpus.voices) >= 3 + + def test_shipped_corpus_has_loud_silence(self): + # Clips under tacet's RMS gate never reach whisper in production, so a + # silence set made only of those would measure nothing that matters. + corpus = te.load_corpus(te.DEFAULT_CORPUS) + loud = [s for s in corpus.silences if s.dbfs is not None and s.dbfs > -47] + assert loud + + def test_spoken_defaults_to_text(self, tmp_path): + path = tmp_path / "c.toml" + path.write_text('voices = ["A"]\n[[utterance]]\nid = "u"\ntext = "hi"\n') + assert te.load_corpus(path).utterances[0].spoken == "hi" + + def test_duplicate_ids_are_refused(self, tmp_path): + path = tmp_path / "c.toml" + path.write_text( + '[[utterance]]\nid = "u"\ntext = "a"\n[[utterance]]\nid = "u"\ntext = "b"\n' + ) + with pytest.raises(te.EvalError, match="duplicate"): + te.load_corpus(path) + + def test_unknown_silence_kind_is_refused(self, tmp_path): + path = tmp_path / "c.toml" + path.write_text('[[silence]]\nid = "s"\nkind = "pink"\nseconds = 1.0\n') + with pytest.raises(te.EvalError, match="pink"): + te.load_corpus(path) + + def test_silence_samples_hit_their_level(self): + clip = te.Silence("s", "white", 0.5, -40) + samples = te._silence_samples(clip) + rms = (sum(s * s for s in samples) / len(samples)) ** 0.5 + assert rms == pytest.approx(327.7, rel=0.01) + assert te._silence_samples(clip) == samples # seeded: same bytes every run + + +class TestArguments: + def test_server_args_are_shell_split_and_repeatable(self): + args = te.parse_args( + ["--server-arg=--no-fallback", "--server-arg=--entropy-thold 2.8"] + ) + assert args.server_args == ["--no-fallback", "--entropy-thold", "2.8"] + + def test_production_flags_by_default(self): + settings = te.run_settings(te.parse_args([]), ["Samantha"]) + assert settings.flags == list(te.PRODUCTION_FLAGS) + assert "--prompt" not in settings.flags + + def test_prompt_and_beam_size_pass_through(self): + args = te.parse_args( + ["--prompt", "tmux, kubectl", "--beam-size", "5", "--server-arg=-nf"] + ) + flags = te.run_settings(args, ["Samantha"]).flags + assert flags[-5:] == ["--prompt", "tmux, kubectl", "--beam-size", "5", "-nf"] + + def test_voices_override(self): + assert te.parse_args(["--voices", "Daniel, Karen"]).voices == [ + "Daniel", + "Karen", + ] + + +class TestRequest: + def test_multipart_matches_whisper_client(self): + body = te.multipart_body("B", b"RIFF") + assert body == ( + b"--B\r\n" + b'Content-Disposition: form-data; name="file"; filename="audio.wav"\r\n' + b"Content-Type: audio/wav\r\n\r\n" + b"RIFF\r\n" + b'--B\r\nContent-Disposition: form-data; name="response_format"\r\n\r\njson\r\n' + b'--B\r\nContent-Disposition: form-data; name="temperature"\r\n\r\n0.0\r\n' + b"--B--\r\n" + ) + + def test_free_port_is_never_a_live_tacet_port(self): + assert te._free_port() not in te.LIVE_PORTS + + +SCRIPT = """ +voices = ["A"] +[[utterance]] +id = "u1" +split = "tune" +kind = "short" +text = "merge it" +[[utterance]] +id = "u2" +split = "holdout" +kind = "ids" +text = "Close TKT-1618." +[[quiet]] +id = "q1" +split = "tune" +do = "Say nothing." +""" + + +def write_take( + folder: Path, clip_id: str, samples: list[int], note: dict | None = None +): + folder.mkdir(parents=True, exist_ok=True) + frames = b"".join(s.to_bytes(2, "little", signed=True) for s in samples) + te._write_wav(folder / f"{clip_id}.wav", frames) + if note is not None: + (folder / f"{clip_id}.json").write_text(te.json.dumps(note)) + + +class TestSpacingAlignment: + @pytest.mark.parametrize( + "reference, hypothesis", + [ + ("Bump ch-migrate.", "Bump CH migrate."), + ("Move TKT-1687 to Backlog.", "Move TKT 1687 to backlog."), + ("the standup", "the stand up"), + ("Try large-v3-turbo.", "Try large V3 turbo."), + ], + ) + def test_spacing_inside_a_word_never_counts(self, reference, hypothesis): + assert result(reference, hypothesis).errors == 0 + + def test_a_wrong_word_still_counts(self): + # Nothing joins to "chmigrate", so it is a substitution plus an insertion. + assert result("Bump ch-migrate.", "Bump CH migrant.").errors == 2 + + def test_gluing_two_words_still_counts(self): + assert result("Rebase onto main.", "Rebase Ontomain.").errors == 2 + + +class TestSplitsAndQuiet: + def corpus(self, tmp_path): + path = tmp_path / "script.toml" + path.write_text(SCRIPT) + return te.load_corpus(path) + + def test_split_kind_and_quiet_are_read(self, tmp_path): + corpus = self.corpus(tmp_path) + assert [(u.id, u.split, u.kind) for u in corpus.utterances] == [ + ("u1", "tune", "short"), + ("u2", "holdout", "ids"), + ] + assert corpus.quiet == [te.Quiet("q1", "Say nothing.", "tune")] + + def test_select_split_keeps_one_side(self, tmp_path): + tune = te.select_split(self.corpus(tmp_path), "tune") + assert [u.id for u in tune.utterances] == ["u1"] + assert [q.id for q in tune.quiet] == ["q1"] + + def test_a_split_nobody_marked_is_an_error(self, tmp_path): + path = tmp_path / "c.toml" + path.write_text('[[utterance]]\nid = "u"\ntext = "hi"\n') + with pytest.raises(te.EvalError, match="no utterances"): + te.select_split(te.load_corpus(path), "holdout") + + def test_unknown_split_is_refused(self, tmp_path): + path = tmp_path / "c.toml" + path.write_text('[[utterance]]\nid = "u"\ntext = "hi"\nsplit = "dev"\n') + with pytest.raises(te.EvalError, match="dev"): + te.load_corpus(path) + + +class TestRecordings: + def setup_corpus(self, tmp_path): + path = tmp_path / "script.toml" + path.write_text(SCRIPT) + return te.load_corpus(path), tmp_path / "recordings" + + def record_all(self, root: Path, session: str): + write_take(root / session, "u1", [900, -900] * 800, {"script_text": "merge it"}) + write_take( + root / session, "u2", [900, -900] * 800, {"script_text": "Close TKT-1618."} + ) + write_take(root / session, "q1", [3, -3] * 800, {"script_text": "Say nothing."}) + + def test_every_take_becomes_a_clip_per_session(self, tmp_path): + corpus, root = self.setup_corpus(tmp_path) + self.record_all(root, "studio-mic") + clips = te.load_recordings(corpus, root, ["studio-mic"]) + assert [(c.id, c.voice, c.is_silence, c.kind) for c in clips] == [ + ("u1", "studio-mic", False, "short"), + ("u2", "studio-mic", False, "ids"), + ("q1", "studio-mic", True, None), + ] + + def test_missing_takes_stop_the_run(self, tmp_path): + corpus, root = self.setup_corpus(tmp_path) + write_take(root / "s", "u1", [900] * 1600) + with pytest.raises(te.EvalError, match="s/u2: not recorded"): + te.load_recordings(corpus, root, ["s"]) + + def test_a_take_recorded_for_other_text_is_stale(self, tmp_path): + corpus, root = self.setup_corpus(tmp_path) + self.record_all(root, "s") + write_take(root / "s", "u1", [900] * 1600, {"script_text": "merge that"}) + with pytest.raises(te.EvalError, match="different text"): + te.load_recordings(corpus, root, ["s"]) + + def test_sessions_are_discovered_or_checked(self, tmp_path): + _, root = self.setup_corpus(tmp_path) + self.record_all(root, "laptop") + self.record_all(root, "studio") + assert te.recorded_sessions(root, None) == ["laptop", "studio"] + with pytest.raises(te.EvalError, match="no takes for desk"): + te.recorded_sessions(root, ["desk"]) + + def test_report_scores_sessions_kinds_and_quiet_takes(self, tmp_path): + corpus, root = self.setup_corpus(tmp_path) + self.record_all(root, "s") + clips = te.load_recordings(corpus, root, ["s"]) + hyps = {"u1": "merge it", "u2": "Close TKT 1618.", "q1": "you"} + results = [te.ClipResult(c, hyps[c.id], 100.0, 160.0) for c in clips] + settings = te.Settings( + "ws", "", None, [], ["s"], "script.toml", "tune", str(root) + ) + report = te.build_report(Path("m.bin"), settings, results) + assert report["per_voice"]["s"]["errors"] == 0 + assert set(report["per_kind"]) == {"short", "ids"} + assert report["silence"] == { + "clips": 1, + "false_text": 1, + "false_text_past_gate": 1, + } + assert report["settings"]["split"] == "tune" diff --git a/tests/test_record.py b/tests/test_record.py new file mode 100644 index 0000000..83e9903 --- /dev/null +++ b/tests/test_record.py @@ -0,0 +1,59 @@ +"""eval/record.py's decisions: what to record next and what to warn about. + +The capture itself needs macOS and a microphone, so it is exercised by hand; +eval/recorder.swift --selftest checks its WAV writer without one. +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +_spec = importlib.util.spec_from_file_location("record", ROOT / "eval" / "record.py") +rec = importlib.util.module_from_spec(_spec) +sys.modules["record"] = rec +_spec.loader.exec_module(rec) +te = rec.te + +LINE = te.Utterance("r1", "merge it", "merge it", "tune", "short") +QUIET = te.Quiet("q1", "Say nothing.", "tune") + + +def test_session_name_is_host_and_input(): + name = rec.session_name("Dans-Mac-Studio.local", "MacBook Pro Microphone") + assert name == "dans-mac-studio-macbook-pro-microphone" + + +def test_pending_skips_recorded_takes_unless_redone(tmp_path): + (tmp_path / "r1.wav").write_bytes(b"") + assert rec.pending([LINE, QUIET], tmp_path, []) == [QUIET] + assert rec.pending([LINE, QUIET], tmp_path, ["r1"]) == [LINE, QUIET] + + +def test_redo_of_an_unknown_id_is_an_error(tmp_path): + with pytest.raises(te.EvalError, match="r9"): + rec.pending([LINE], tmp_path, ["r9"]) + + +@pytest.mark.parametrize( + "take, expected", + [ + (rec.Take(0, 0.0, 0), "no audio arrived"), + (rec.Take(16000, 0.0, 0), "every sample is zero"), + (rec.Take(16000, 80.0, 400), "quieter than tacet's gate"), + (rec.Take(4000, 900.0, 9000), "very short"), + ], +) +def test_bad_speech_takes_are_flagged(take, expected): + assert any(expected in w for w in rec.check_take(LINE, take)) + + +def test_a_good_take_and_a_quiet_take_pass(): + assert rec.check_take(LINE, rec.Take(40000, 1200.0, 14000)) == [] + assert rec.check_take(QUIET, rec.Take(32000, 40.0, 300)) == [] + + +def test_take_seconds(): + assert rec.Take(24000, 1.0, 1).seconds == 1.5