From 9cd984172c812bb1a00dd618b7c289f2e935ea99 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Thu, 5 Nov 2026 15:20:54 -0500 Subject: [PATCH 1/3] [eval] Add an accuracy and latency eval harness Nothing measured how well whisper-server transcribes tacet's kind of dictation, so no change to the prompt, decoding or model could be judged. eval/tacet_eval.py is that yardstick. It speaks 44 jargon-heavy utterances in four `say` voices, converts each with afconvert to the WAV tacet records, and starts a private whisper-server on a free loopback port with install-server.sh's flags. It then POSTs every clip the way WhisperClient does. It reports WER after a documented normalization, p50/p95 latency, and how many of 7 silence clips came back with text. Normalization reads symbols as their spoken words. A path or flag said aloud therefore scores the same as its written form, and the number reflects recognition rather than formatting. Baseline on an M4 Max: WER 4.32%, p50 239 ms, p95 253 ms, silence 2/7. Both false texts are under tacet's RMS gate, so a user never sees them. Solves: no measurement of tacet's transcription quality Tests: tests/test_eval.py (54) plus the existing suite, 166 passed --- .gitignore | 3 + README.md | 69 +++- eval/corpus.toml | 269 +++++++++++++++ eval/tacet_eval.py | 821 +++++++++++++++++++++++++++++++++++++++++++++ tests/test_eval.py | 257 ++++++++++++++ 5 files changed, 1418 insertions(+), 1 deletion(-) create mode 100644 eval/corpus.toml create mode 100644 eval/tacet_eval.py create mode 100644 tests/test_eval.py diff --git a/.gitignore b/.gitignore index a6decda..6c13a4b 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ swift/Packaging/Tacet.app/ # Stray build output. The packaging script regenerates this. build/ + +# Eval audio, server log and per-clip results (eval/tacet_eval.py regenerates them). +eval/.cache/ diff --git a/README.md b/README.md index f501bc4..5eda545 100644 --- a/README.md +++ b/README.md @@ -369,6 +369,72 @@ never the content — of each transcript; `status.json` carries the live microphone, Accessibility and hotkey state. A flake an hour old otherwise leaves no evidence anywhere. +## Measuring accuracy + +`eval/tacet_eval.py` measures the transcription leg on its own: how accurate +and how fast whisper-server is with tacet's model and flags. + +```bash +uv run python eval/tacet_eval.py # human-readable, worst clips listed +uv run python eval/tacet_eval.py --json # one JSON object on stdout +``` + +It speaks the 44 utterances in `eval/corpus.toml` with macOS `say` in four +voices (US, UK, Australian, Irish). Most of them are terminal and coding-agent +dictation: Postgres, tmux, kubectl, flags, file paths read out loud. It +converts each one with `afconvert` to the 16 kHz mono 16-bit WAV tacet records, +and caches the audio in `eval/.cache/` (gitignored). Then it starts its own +whisper-server on a free loopback port, with `--model` and the flags +`install-server.sh` renders. It POSTs each clip to `/inference` exactly as +`WhisperClient.swift` does, one at a time after one untimed warm-up, and stops +the server at the end. The live services on 8910 and 8911 are never touched. +The first run takes about two minutes. Later runs take about one. + +Pass-throughs for experiments: `--prompt TEXT`, `--beam-size N`, and +`--server-arg=FLAG` for any other whisper-server flag, repeatable and +shell-split (write it with `=`, e.g. `--server-arg="--entropy-thold 2.8"`). +`--model`, `--voices` and `--corpus` swap the other inputs. Every clip's +reference, transcript, errors and latency go to `eval/.cache/last-run.json` for +diffing two runs. + +**What it reports.** Word error rate is total word errors over total reference +words, overall and per voice. It is not clamped, so a model that hallucinates +can score above 1. Latency is the p50 and p95 wall time of the speech requests, +by nearest rank. The silence count is how many of the 7 silence and +near-silence clips came back with a letter or digit. That is the same test +tacet applies before it pastes anything. The report also says how many of those +clips are loud enough to pass tacet's RMS gate (150), since only those would +ever reach a user. + +**Normalization**, applied to both reference and transcript before scoring: + +- Case and punctuation never count. Apostrophes are deleted. +- Symbols read as the words people say for them: `/etc/hosts` scores the + same as "slash etc slash hosts", and `--dry-run` the same as "dash dash dry + run". The eval measures recognition, not formatting. +- Hyphenated compounds close up (`stand-up` = `standup`). +- Spelled-out numbers become digits ("twenty five" = 25), and `7:15` = `7.15` = + "seven fifteen". + +The full rules are in the `normalize()` docstring, and `tests/test_eval.py` +pins them. + +**Baseline, 2026-11-05** (Apple M4 Max, whisper.cpp 1.8.4, +ggml-large-v3-turbo, production flags, empty `--prompt`): + +| | | +|---|---| +| WER | 4.32% (Samantha 3.41%, Daniel 4.38%, Karen 4.38%, Moira 5.11%) | +| latency | p50 239 ms, p95 253 ms | +| silence | 2 of 7 clips returned text ("you" on digital silence). Both are below the RMS gate, so tacet never sends them | + +Most of the errors are jargon whisper has never seen. It writes "cube +control" for kubectl, "Maine" for main, "launch deep list" for launchd plist, +"tacit" for tacet, and "post-gaz" for Postgres. That is what `--prompt` is for. +Synthetic voices are cleaner than a real microphone in a real room, so read +these numbers as a fixed yardstick for comparing settings, not as the accuracy +you will get. + ## What's been tested, and what hasn't One person, two Macs, one microphone. CI runs the pytest suite on Linux and @@ -454,7 +520,8 @@ swift/ Packaging/build-app.sh assembles and signs Tacet.app Tests/ SwiftPM suite (48) config.example.toml shape of ~/.config/tacet/config.toml -tests/ pytest suite — drives the installers as subprocesses +eval/ accuracy + latency harness and its corpus +tests/ pytest suite — the installers as subprocesses, the eval scoring .github/workflows/ci.yml pytest + shellcheck + the signed bundle build docs/ design specs ``` diff --git a/eval/corpus.toml b/eval/corpus.toml new file mode 100644 index 0000000..89ad70d --- /dev/null +++ b/eval/corpus.toml @@ -0,0 +1,269 @@ +# tacet's accuracy corpus, read by eval/tacet_eval.py. +# +# Every [[utterance]] is spoken once per voice by macOS `say`, converted to the +# 16 kHz mono 16-bit WAV tacet records, and sent to a private whisper-server. +# +# id stable name; the cache key and the row in the per-clip report +# text the reference: what a user who said it wants pasted +# spoken optional; what `say` is given when it would mangle `text` (tmux, +# kubectl, file paths). It stands in for how a person says it. It is +# never scored against. +# +# References are written the way the text should come out, symbols included +# (/etc/hosts, --dry-run). The scorer's normalization reads symbols as the words +# a person speaks for them, so "/etc/hosts" and "slash etc slash hosts" score +# the same: the eval measures recognition, not formatting. +# +# Generated audio lives in eval/.cache/, which is gitignored. Editing a line +# here regenerates that line's audio on the next run. + +# Distinct English accents that ship with macOS: US, UK, Australian, Irish. +voices = ["Samantha", "Daniel", "Karen", "Moira"] + +# --- Terminal and coding-agent dictation ------------------------------------ + +[[utterance]] +id = "pg-migration" +text = "Run the Postgres migration before you restart the API server." + +[[utterance]] +id = "k8s-pods" +text = "Use kubectl to list the pods in the staging namespace." +spoken = "Use cube control to list the pods in the staging namespace." + +[[utterance]] +id = "tmux-split" +text = "Split the tmux window and run the tests in the right pane." +spoken = "Split the tee mux window and run the tests in the right pane." + +[[utterance]] +id = "clickhouse-order-by" +text = "The ClickHouse table needs a different ORDER BY key." +spoken = "The ClickHouse table needs a different order by key." + +[[utterance]] +id = "terraform-plan" +text = "Run terraform plan and show me what it wants to destroy." + +[[utterance]] +id = "open-pr" +text = "Open a PR against main and ask for a review." +spoken = "Open a P R against main and ask for a review." + +[[utterance]] +id = "schema-column" +text = "Add a nullable column to the users schema." + +[[utterance]] +id = "git-force-flag" +text = "Push the branch with --force-with-lease." +spoken = "Push the branch with dash dash force with lease." + +[[utterance]] +id = "etc-hosts" +text = "Add the new host to /etc/hosts." +spoken = "Add the new host to slash etc slash hosts." + +[[utterance]] +id = "readme-path" +text = "Update the README.md in the docs folder." +spoken = "Update the read me dot M D in the docs folder." + +[[utterance]] +id = "swift-file" +text = "The bug is in WhisperClient.swift, in the multipart body." +spoken = "The bug is in WhisperClient dot swift, in the multipart body." + +[[utterance]] +id = "clone-repo" +text = "Clone the repo and check out the release branch." + +[[utterance]] +id = "rebase-main" +text = "Rebase onto main and fix the merge conflicts." + +[[utterance]] +id = "cli-dry-run" +text = "Run the installer with --dry-run first." +spoken = "Run the installer with dash dash dry run first." + +[[utterance]] +id = "docker-compose" +text = "Restart the Docker container and tail the logs." + +[[utterance]] +id = "env-var" +text = "Set the DATABASE_URL environment variable." +spoken = "Set the database underscore URL environment variable." + +[[utterance]] +id = "json-response" +text = "The endpoint returns JSON with a text field." + +[[utterance]] +id = "yaml-config" +text = "The workflow YAML is missing a timeout." + +[[utterance]] +id = "grep-todo" +text = "Grep the source tree for every TODO comment." +spoken = "Grep the source tree for every to do comment." + +[[utterance]] +id = "port-number" +text = "The server listens on port 8080 on localhost." +spoken = "The server listens on port eighty eighty on localhost." + +[[utterance]] +id = "unit-tests" +text = "Write unit tests for the parser and run pytest." + +[[utterance]] +id = "launchd-plist" +text = "The launchd plist never loaded after the reboot." +spoken = "The launch D plist never loaded after the reboot." + +[[utterance]] +id = "ssh-mosh" +text = "Connect over SSH, then switch to mosh on the laptop." + +[[utterance]] +id = "clickhouse-mv" +text = "Drop the materialized view and rebuild it from the raw table." + +[[utterance]] +id = "terraform-state" +text = "Do not touch the Terraform state file by hand." + +[[utterance]] +id = "kubernetes-deploy" +text = "Scale the Kubernetes deployment to three replicas." + +[[utterance]] +id = "postgres-index" +text = "Add an index on the created at column in Postgres." + +[[utterance]] +id = "claude-code" +text = "Ask Claude Code to refactor the function into two smaller ones." + +[[utterance]] +id = "home-config" +text = "The config lives in ~/.config/tacet/config.toml." +spoken = "The config lives in tilde slash dot config slash tacet slash config dot tommel." + +[[utterance]] +id = "type-error" +text = "The build fails with a type error in the client module." + +[[utterance]] +id = "npm-install" +text = "Delete the lock file and run npm install again." +spoken = "Delete the lock file and run N P M install again." + +[[utterance]] +id = "http-status" +text = "The health check returns a 503 until the model loads." +spoken = "The health check returns a five oh three until the model loads." + +# --- Ordinary sentences ------------------------------------------------------- + +[[utterance]] +id = "lunch" +text = "Let's grab lunch after the standup tomorrow." + +[[utterance]] +id = "running-late" +text = "I'm running about ten minutes late, start without me." + +[[utterance]] +id = "thanks-review" +text = "Thanks for the quick review, I'll fix those comments tonight." + +[[utterance]] +id = "weather" +text = "It looks like rain this afternoon, so bring an umbrella." + +[[utterance]] +id = "meeting-move" +text = "Can we move the meeting to Thursday at three?" + +[[utterance]] +id = "groceries" +text = "We need milk, eggs, and a loaf of bread." + +[[utterance]] +id = "invoice" +text = "Please send the invoice by the end of the month." + +[[utterance]] +id = "question" +text = "Did you get a chance to look at the draft I sent?" + +[[utterance]] +id = "dog-walk" +text = "I'll walk the dog when I get home." + +[[utterance]] +id = "sounds-good" +text = "Sounds good, let me know if anything changes." + +[[utterance]] +id = "flight" +text = "My flight lands at seven fifteen on Sunday evening." + +[[utterance]] +id = "book" +text = "I finished the book you lent me and loved the ending." + +# --- Silence and near-silence ------------------------------------------------- +# +# Synthesised in Python from a fixed seed, so every run sends the same bytes. +# `dbfs` is the RMS level relative to full scale. tacet drops anything below +# RMS 150 (about -47 dBFS) before whisper sees it, so the clips above that line +# are the ones whose hallucinations would reach a user. +# +# kind = "zero" digital silence +# kind = "white" white noise +# kind = "brown" integrated (room-tone-like) noise +# kind = "hum" a mains-hum sine + +[[silence]] +id = "zero-1s" +kind = "zero" +seconds = 1.0 + +[[silence]] +id = "zero-3s" +kind = "zero" +seconds = 3.0 + +[[silence]] +id = "white-60db" +kind = "white" +seconds = 2.0 +dbfs = -60 + +[[silence]] +id = "brown-50db" +kind = "brown" +seconds = 3.0 +dbfs = -50 + +[[silence]] +id = "white-40db" +kind = "white" +seconds = 2.0 +dbfs = -40 + +[[silence]] +id = "brown-35db" +kind = "brown" +seconds = 3.0 +dbfs = -35 + +[[silence]] +id = "hum-40db" +kind = "hum" +seconds = 2.0 +dbfs = -40 diff --git a/eval/tacet_eval.py b/eval/tacet_eval.py new file mode 100644 index 0000000..2dac47f --- /dev/null +++ b/eval/tacet_eval.py @@ -0,0 +1,821 @@ +"""Accuracy and latency eval for tacet's whisper-server setup. + + uv run python eval/tacet_eval.py [--model PATH] [--json] + [--prompt TEXT] [--beam-size N] [--server-arg=FLAG ...] + [--voices A,B] [--corpus PATH] [--details PATH] + +Speaks every utterance in eval/corpus.toml with macOS `say` in several +voices, converts it with `afconvert` to the 16 kHz mono 16-bit WAV tacet's +Recorder sends, starts a PRIVATE whisper-server on a free loopback port with +the flags install-server.sh renders, and POSTs each clip to /inference the way +WhisperClient.swift does. It reports word error rate, request latency, and how +many silence clips came back with text, then stops the server. The live tacet +services (ports 8910 and 8911) are never contacted. + +Standard library only. Generated audio is cached in eval/.cache/ (gitignored). +Progress goes to stderr; with --json, stdout carries exactly one JSON object. + +--server-arg takes a whisper-server flag with its value, shell-split, and can +be repeated. Write it with `=` so argparse does not read the flag as its own: + --server-arg=--no-fallback --server-arg="--entropy-thold 2.8" +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import datetime +import hashlib +import http.client +import json +import math +import os +import platform +import random +import re +import shlex +import shutil +import socket +import subprocess +import sys +import tempfile +import time +import tomllib +import unicodedata +import uuid +import wave +from dataclasses import dataclass, field +from pathlib import Path + +EVAL_DIR = Path(__file__).resolve().parent +DEFAULT_CORPUS = EVAL_DIR / "corpus.toml" +CACHE_DIR = EVAL_DIR / ".cache" +DEFAULT_MODEL = Path.home() / ".local/share/whisper-cpp/ggml-large-v3-turbo.bin" + +# tacet's own services. The harness refuses to run a server on either. +LIVE_PORTS = frozenset({8910, 8911}) + +# The flags install-server.sh renders into the whisper plist, minus --model and +# --port, which the harness supplies. --prompt is added only when non-empty, +# exactly as the installer does. +PRODUCTION_FLAGS = ( + "--host", + "127.0.0.1", + "--language", + "en", + "--no-timestamps", + "--suppress-nst", +) + +# The form fields WhisperClient.makeMultipart sends after the file part. +REQUEST_FIELDS = (("response_format", "json"), ("temperature", "0.0")) + +SAMPLE_RATE = 16000 +# tacet's server drops audio whose RMS is below this before whisper sees it +# (audio.silence_rms_threshold, Config.swift). +SILENCE_RMS_GATE = 150.0 +REQUEST_TIMEOUT_S = 60.0 + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + corpus = load_corpus(args.corpus) + voices = args.voices or corpus.voices + clips = prepare_audio(corpus, voices) + settings = run_settings(args, voices) + with WhisperServer(settings, args.model, args.load_timeout) as server: + results = transcribe_all(server, clips) + except EvalError as e: + print(f"tacet_eval: {e}", file=sys.stderr) + return 1 + report = build_report(args.model, settings, results) + _write_details(args.details, report, results) + if args.json: + print(json.dumps(report, indent=2)) + else: + print(format_summary(report, results)) + return 0 + + +class EvalError(RuntimeError): + """A failure that ends the run with a message rather than a traceback.""" + + +@dataclass(frozen=True) +class Utterance: + id: str + text: str + spoken: str + + +@dataclass(frozen=True) +class Silence: + id: str + kind: str + seconds: float + dbfs: float | None + + +@dataclass(frozen=True) +class Corpus: + voices: list[str] + utterances: list[Utterance] + silences: list[Silence] + + +@dataclass(frozen=True) +class Clip: + """One WAV to send: a spoken utterance in one voice, or a silence clip.""" + + id: str + voice: str | None + reference: str + path: Path + + @property + def is_silence(self) -> bool: + return self.voice is None + + +@dataclass(frozen=True) +class ClipResult: + clip: Clip + text: str + ms: float + rms: float + + @property + def reference_words(self) -> list[str]: + return normalize(self.clip.reference) + + @property + def errors(self) -> int: + return word_errors(self.reference_words, normalize(self.text)) + + +@dataclass(frozen=True) +class Settings: + """What was measured: the server command line and the request fields.""" + + binary: str + prompt: str + beam_size: int | None + server_args: list[str] + voices: list[str] + corpus: str + flags: list[str] = field(init=False) + + def __post_init__(self) -> None: + flags = list(PRODUCTION_FLAGS) + if self.prompt: + flags += ["--prompt", self.prompt] + if self.beam_size is not None: + flags += ["--beam-size", str(self.beam_size)] + object.__setattr__(self, "flags", flags + self.server_args) + + def as_dict(self) -> dict: + return { + "whisper_server": self.binary, + "flags": self.flags, + "prompt": self.prompt, + "beam_size": self.beam_size, + "server_args": self.server_args, + "request": dict(REQUEST_FIELDS), + "voices": self.voices, + "corpus": self.corpus, + "host": platform.node(), + "date": datetime.date.today().isoformat(), + } + + +class WhisperServer: + """A private whisper-server on a free loopback port, stopped on exit.""" + + def __init__(self, settings: Settings, model: Path, load_timeout: float): + self.settings = settings + self.model = model + self.load_timeout = load_timeout + self.port = 0 + self.proc: subprocess.Popen | None = None + self.log_path = CACHE_DIR / "whisper-server.log" + + def __enter__(self) -> WhisperServer: + if not self.model.is_file(): + raise EvalError(f"model not found: {self.model}") + if not os.access(self.settings.binary, os.X_OK): + raise EvalError(f"whisper-server not found: {self.settings.binary}") + self.port = _free_port() + command = [ + self.settings.binary, + "--model", + str(self.model), + "--port", + str(self.port), + *self.settings.flags, + ] + _log(f"starting whisper-server on 127.0.0.1:{self.port}") + with open(self.log_path, "wb") as log: + self.proc = subprocess.Popen( + command, + cwd=CACHE_DIR, + stdin=subprocess.DEVNULL, + stdout=log, + stderr=subprocess.STDOUT, + ) + return self + + def __exit__(self, *exc: object) -> None: + if self.proc is None or self.proc.poll() is not None: + return + self.proc.terminate() + try: + self.proc.wait(timeout=10) + except subprocess.TimeoutExpired: + self.proc.kill() + self.proc.wait() + + def wait_until_ready(self, probe: bytes) -> None: + """POST a speech clip until it succeeds: that is load plus warm-up.""" + deadline = time.monotonic() + self.load_timeout + while time.monotonic() < deadline: + self._check_alive() + try: + self.transcribe(probe) + return + except (ConnectionError, socket.timeout, _NotReady): + time.sleep(0.5) + raise EvalError(f"whisper-server did not load in {self.load_timeout:.0f}s") + + def transcribe(self, wav: bytes) -> tuple[str, float]: + """Send one WAV; return the text and the request's wall time in ms.""" + boundary = f"TacetBoundary{str(uuid.uuid4()).upper()}" + body = multipart_body(boundary, wav) + conn = http.client.HTTPConnection( + "127.0.0.1", self.port, timeout=REQUEST_TIMEOUT_S + ) + headers = {"Content-Type": f"multipart/form-data; boundary={boundary}"} + start = time.perf_counter() + try: + conn.request("POST", "/inference", body=body, headers=headers) + response = conn.getresponse() + payload = response.read() + finally: + conn.close() + ms = (time.perf_counter() - start) * 1000 + if response.status == 503: + raise _NotReady() + return _response_text(response.status, payload), ms + + def _check_alive(self) -> None: + assert self.proc is not None + if self.proc.poll() is not None: + tail = self.log_path.read_text(errors="replace").splitlines()[-15:] + raise EvalError( + f"whisper-server exited with {self.proc.returncode}:\n" + + "\n".join(tail) + ) + + +def parse_args(argv: list[str] | None) -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Measure tacet's whisper-server accuracy and latency.", + epilog="Pass whisper-server flags as --server-arg=FLAG (with '=').", + ) + p.add_argument("--model", type=Path, default=DEFAULT_MODEL) + p.add_argument("--json", action="store_true", help="print one JSON object") + p.add_argument("--prompt", default="", help="whisper-server --prompt") + p.add_argument("--beam-size", type=int, help="whisper-server --beam-size") + p.add_argument( + "--server-arg", + action="append", + default=[], + dest="server_args", + help="extra whisper-server flag, shell-split; repeatable", + ) + p.add_argument("--voices", type=_voice_list, help="comma-separated say voices") + p.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS) + p.add_argument("--whisper-server", dest="binary", default=_default_binary()) + p.add_argument("--details", type=Path, default=CACHE_DIR / "last-run.json") + p.add_argument("--load-timeout", type=float, default=300.0) + args = p.parse_args(argv) + args.server_args = [token for arg in args.server_args for token in shlex.split(arg)] + return args + + +def load_corpus(path: Path) -> Corpus: + data = tomllib.loads(path.read_text()) + utterances = [ + Utterance(u["id"], u["text"], u.get("spoken", u["text"])) + for u in data.get("utterance", []) + ] + silences = [ + Silence(s["id"], s["kind"], float(s["seconds"]), s.get("dbfs")) + for s in data.get("silence", []) + ] + ids = [x.id for x in utterances + silences] + duplicates = sorted({i for i in ids if ids.count(i) > 1}) + if duplicates: + raise EvalError(f"duplicate corpus ids: {', '.join(duplicates)}") + unknown = sorted({s.kind for s in silences} - set(_SILENCE_KINDS)) + if unknown: + raise EvalError(f"unknown silence kinds: {', '.join(unknown)}") + return Corpus(list(data.get("voices", [])), utterances, silences) + + +def prepare_audio(corpus: Corpus, voices: list[str]) -> list[Clip]: + """Generate any WAV missing from the cache; return every clip to send.""" + if not voices: + raise EvalError("no voices: set `voices` in the corpus or pass --voices") + _check_tools(voices) + CACHE_DIR.mkdir(parents=True, exist_ok=True) + jobs = [(u, v) for v in voices for u in corpus.utterances] + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + paths = list(pool.map(lambda job: _speech_wav(*job), jobs)) + clips = [Clip(u.id, v, u.text, path) for (u, v), path in zip(jobs, paths)] + clips += [Clip(s.id, None, "", _silence_wav(s)) for s in corpus.silences] + return clips + + +def run_settings(args: argparse.Namespace, voices: list[str]) -> Settings: + try: + corpus = str(args.corpus.resolve().relative_to(EVAL_DIR.parent)) + except ValueError: + corpus = str(args.corpus) + return Settings( + binary=os.path.realpath(args.binary), + prompt=args.prompt, + beam_size=args.beam_size, + server_args=args.server_args, + voices=voices, + corpus=corpus, + ) + + +def transcribe_all(server: WhisperServer, clips: list[Clip]) -> list[ClipResult]: + """Send every clip one at a time, as tacet does, after an untimed warm-up.""" + first_speech = next(c for c in clips if not c.is_silence) + server.wait_until_ready(first_speech.path.read_bytes()) + _log(f"server ready; transcribing {len(clips)} clips") + results = [] + for n, clip in enumerate(clips, 1): + wav = clip.path.read_bytes() + try: + text, ms = server.transcribe(wav) + except (OSError, _NotReady) as e: + raise EvalError( + f"request for {clip.id} ({clip.voice}) failed: {e!r}" + ) from e + results.append(ClipResult(clip, text, ms, wav_rms(wav))) + if n % 20 == 0 or n == len(clips): + _log(f" {n}/{len(clips)}") + return results + + +def build_report(model: Path, settings: Settings, results: list[ClipResult]) -> dict: + speech = [r for r in results if not r.clip.is_silence] + silence = [r for r in results if r.clip.is_silence] + latencies = [r.ms for r in speech] + return { + "model": str(model), + "settings": settings.as_dict(), + "utterances": len({r.clip.id for r in speech}), + "wer": round(corpus_wer(speech), 4), + "per_voice": { + voice: _voice_score([r for r in speech if r.clip.voice == voice]) + for voice in settings.voices + }, + "latency_ms": { + "p50": round(percentile(latencies, 50), 1), + "p95": round(percentile(latencies, 95), 1), + }, + "silence": { + "clips": len(silence), + "false_text": sum(has_text(r.text) for r in silence), + "false_text_past_gate": sum( + has_text(r.text) and r.rms >= SILENCE_RMS_GATE for r in silence + ), + }, + } + + +def format_summary(report: dict, results: list[ClipResult], worst: int = 12) -> str: + lat, sil = report["latency_ms"], report["silence"] + lines = [ + f"model {report['model']}", + f"flags {shlex.join(report['settings']['flags'])}", + f"WER {report['wer']:.2%} over {report['utterances']} utterances", + ] + for voice, score in report["per_voice"].items(): + lines.append( + f" {voice:<10} {score['wer']:.2%} ({score['errors']}/{score['words']} words)" + ) + lines.append(f"latency p50 {lat['p50']} ms, p95 {lat['p95']} ms") + lines.append( + f"silence {sil['false_text']}/{sil['clips']} clips came back with text " + f"({sil['false_text_past_gate']} loud enough to pass tacet's RMS gate)" + ) + speech = [r for r in results if not r.clip.is_silence] + misses = sorted((r for r in speech if r.errors), key=lambda r: -r.errors) + if misses: + lines.append(f"\nworst clips (of {len(misses)} with errors):") + for r in misses[:worst]: + lines.append(f" [{r.errors}] {r.clip.id} ({r.clip.voice})") + lines.append(f" ref: {r.clip.reference}") + lines.append(f" hyp: {r.text.strip()}") + return "\n".join(lines) + + +def normalize(text: str) -> list[str]: + """Reduce a transcript to comparable words. Applied to both sides. + + 1. Unicode NFKC; curly apostrophes become straight ones. + 2. Symbols become the words people say for them, so a path or flag scores + the same written or spoken: "/" slash, "." inside a token dot (between + digits, point), a leading "-" or "--" dash, "_" underscore, "~" tilde, + "@" at, "%" percent, "&" and, "+" plus, "=" equals. + 3. Hyphenated compounds are closed up ("stand-up" -> "standup", "T-Mux" + -> "tmux"), except inside a flag ("--dry-run" -> "dash dash dry run") + and between number words ("twenty-five" -> "25"). + 4. Lowercase. Apostrophes are deleted ("don't" -> "dont"); every other + punctuation mark becomes a space, so case and punctuation never count. + 5. Numbers: thousands separators are dropped ("1,000" -> "1000"), a clock + time reads as two numbers ("7:15" and "7.15" -> "7 15"), and spelled-out + cardinals become digits ("twenty five" -> "25", "three thousand and" + -> "3000 and"). Runs that are not one number stay split ("seven + fifteen" -> "7 15"). Ordinals and "oh" are left alone. + """ + text = unicodedata.normalize("NFKC", text).replace("’", "'").replace("‘", "'") + text = re.sub(r"(? int: + """Levenshtein distance over words: substitutions + deletions + insertions.""" + previous = list(range(len(hypothesis) + 1)) + for i, ref_word in enumerate(reference, 1): + current = [i] + for j, hyp_word in enumerate(hypothesis, 1): + current.append( + min( + previous[j] + 1, + current[j - 1] + 1, + previous[j - 1] + (ref_word != hyp_word), + ) + ) + previous = current + return previous[-1] + + +def corpus_wer(results: list[ClipResult]) -> float: + """Micro-averaged WER: total word errors over total reference words.""" + words = sum(len(r.reference_words) for r in results) + return sum(r.errors for r in results) / words if words else 0.0 + + +def percentile(values: list[float], pct: float) -> float: + """Nearest-rank percentile: the smallest value with pct% at or below it.""" + if not values: + return 0.0 + ordered = sorted(values) + rank = max(1, math.ceil(pct / 100 * len(ordered))) + return ordered[rank - 1] + + +def has_text(transcript: str) -> bool: + """tacet pastes a transcript only if it holds a letter or digit.""" + return any(ch.isalnum() for ch in transcript) + + +def multipart_body(boundary: str, wav: bytes) -> bytes: + """The body WhisperClient.makeMultipart builds, byte for byte.""" + parts = [ + f"--{boundary}\r\n".encode(), + b'Content-Disposition: form-data; name="file"; filename="audio.wav"\r\n', + b"Content-Type: audio/wav\r\n\r\n", + wav, + b"\r\n", + ] + for name, value in REQUEST_FIELDS: + parts.append(f"--{boundary}\r\n".encode()) + parts.append(f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode()) + parts.append(f"{value}\r\n".encode()) + parts.append(f"--{boundary}--\r\n".encode()) + return b"".join(parts) + + +def wav_rms(wav: bytes) -> float: + """RMS of the 16-bit samples after a canonical 44-byte header, as tacet measures.""" + data = wav[44:] + pcm = memoryview(data[: len(data) - len(data) % 2]).cast("h") + return math.sqrt(sum(s * s for s in pcm) / len(pcm)) if len(pcm) else 0.0 + + +class _NotReady(Exception): + """whisper-server answered 503: still loading the model.""" + + +class _NumberRun: + """Accumulates consecutive spelled-out number words into one integer.""" + + def __init__(self) -> None: + self.total = 0 + self.current = 0 + self.last: str | None = None + + def take(self, word: str) -> bool: + """Absorb `word` if it extends this number; False if it cannot.""" + kind, value = _number_word(word) + if kind is None or self.last not in _CAN_FOLLOW[kind]: + return False + if kind == "hundred": + if self.current >= 100: + return False + self.current = (self.current or 1) * 100 + elif kind == "scale": + self.total += (self.current or 1) * value + self.current = 0 + else: + self.current += value + self.last = kind + return True + + def flush(self) -> list[str]: + if self.last is None: + return [] + value = self.total + self.current + self.total, self.current, self.last = 0, 0, None + return [str(value)] + + +_SYMBOL_WORDS = { + "/": "slash", + "_": "underscore", + "~": "tilde", + "@": "at", + "%": "percent", + "&": "and", + "+": "plus", + "=": "equals", +} + +_UNITS = { + w: i + for i, w in enumerate( + "zero one two three four five six seven eight nine ten eleven twelve " + "thirteen fourteen fifteen sixteen seventeen eighteen nineteen".split() + ) +} +_TENS = { + w: 10 * i + for i, w in enumerate( + "twenty thirty forty fifty sixty seventy eighty ninety".split(), 2 + ) +} +_SCALES = {"thousand": 1000, "million": 1_000_000, "billion": 1_000_000_000} + +# Which word kinds may follow which, inside one number. `None` is "nothing yet". +_CAN_FOLLOW = { + "zero": {None}, + "unit": {None, "tens", "hundred", "scale"}, + "teen": {None, "hundred", "scale"}, + "tens": {None, "hundred", "scale"}, + "hundred": {None, "unit", "teen", "tens"}, + "scale": {None, "unit", "teen", "tens", "hundred"}, +} + +_SILENCE_KINDS = ("zero", "white", "brown", "hum") + + +def _voice_list(value: str) -> list[str]: + return [v.strip() for v in value.split(",") if v.strip()] + + +def _default_binary() -> str: + return shutil.which("whisper-server") or "/opt/homebrew/bin/whisper-server" + + +def _check_tools(voices: list[str]) -> None: + for tool in ("say", "afconvert"): + if shutil.which(tool) is None: + raise EvalError(f"`{tool}` not found; the eval needs macOS") + listing = subprocess.run( + ["say", "-v", "?"], capture_output=True, text=True, check=True + ) + installed = { + m.group(1) for m in re.finditer(r"^(.+?)\s{2,}\w\w_\w+", listing.stdout, re.M) + } + missing = [v for v in voices if v not in installed] + if missing: + raise EvalError(f"say voices not installed: {', '.join(missing)}") + + +def _speech_wav(utterance: Utterance, voice: str) -> Path: + key = _digest(voice, utterance.spoken) + dest = CACHE_DIR / "audio" / _slug(voice) / f"{utterance.id}-{key}.wav" + if dest.exists(): + return dest + dest.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=CACHE_DIR) as tmp: + script, aiff, converted = ( + Path(tmp, "text.txt"), + Path(tmp, "say.aiff"), + Path(tmp, "c.wav"), + ) + script.write_text(utterance.spoken) + _run(["say", "-v", voice, "-o", str(aiff), "-f", str(script)]) + _run( + [ + "afconvert", + "-f", + "WAVE", + "-d", + f"LEI16@{SAMPLE_RATE}", + "-c", + "1", + str(aiff), + str(converted), + ] + ) + _canonical_wav(converted, dest) + return dest + + +def _canonical_wav(source: Path, dest: Path) -> None: + """Re-wrap afconvert's output (which carries a FLLR pad chunk) in the plain + 44-byte header tacet's Recorder writes.""" + with wave.open(str(source), "rb") as w: + shape = (w.getnchannels(), w.getsampwidth(), w.getframerate()) + if shape != (1, 2, SAMPLE_RATE): + raise EvalError(f"afconvert produced {shape}, not 16 kHz mono 16-bit") + frames = w.readframes(w.getnframes()) + _write_wav(dest, frames) + + +def _silence_wav(clip: Silence) -> Path: + key = _digest(clip.kind, str(clip.seconds), str(clip.dbfs)) + dest = CACHE_DIR / "audio" / "silence" / f"{clip.id}-{key}.wav" + if not dest.exists(): + dest.parent.mkdir(parents=True, exist_ok=True) + samples = _silence_samples(clip) + _write_wav( + dest, b"".join(s.to_bytes(2, "little", signed=True) for s in samples) + ) + return dest + + +def _silence_samples(clip: Silence) -> list[int]: + """Deterministic samples scaled to the clip's RMS level in dBFS.""" + n = int(clip.seconds * SAMPLE_RATE) + if clip.kind == "zero": + return [0] * n + rng = random.Random(clip.id) + if clip.kind == "white": + raw = [rng.gauss(0.0, 1.0) for _ in range(n)] + elif clip.kind == "brown": + raw, level = [], 0.0 + for _ in range(n): + level = 0.995 * level + rng.gauss(0.0, 1.0) + raw.append(level) + else: + hz = 2 * math.pi / SAMPLE_RATE + raw = [math.sin(60 * hz * t) + 0.3 * math.sin(120 * hz * t) for t in range(n)] + rms = math.sqrt(sum(x * x for x in raw) / n) + gain = 32768 * 10 ** ((clip.dbfs if clip.dbfs is not None else -60) / 20) / rms + return [max(-32768, min(32767, round(x * gain))) for x in raw] + + +def _write_wav(dest: Path, frames: bytes) -> None: + partial = dest.with_name(dest.name + ".part") + with wave.open(str(partial), "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(SAMPLE_RATE) + w.writeframes(frames) + partial.replace(dest) + + +def _free_port() -> int: + """A free loopback port from the kernel, never one of tacet's own.""" + for _ in range(20): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + if port not in LIVE_PORTS: + return port + raise EvalError("could not find a free loopback port") + + +def _response_text(status: int, payload: bytes) -> str: + if not 200 <= status < 300: + raise EvalError(f"whisper-server returned HTTP {status}: {payload[:200]!r}") + try: + text = json.loads(payload)["text"] + except (ValueError, KeyError, TypeError) as e: + raise EvalError( + f"malformed response from whisper-server: {payload[:200]!r}" + ) from e + if not isinstance(text, str): + raise EvalError(f"malformed response from whisper-server: {payload[:200]!r}") + return text + + +def _voice_score(results: list[ClipResult]) -> dict: + return { + "wer": round(corpus_wer(results), 4), + "errors": sum(r.errors for r in results), + "words": sum(len(r.reference_words) for r in results), + } + + +def _write_details(path: Path, report: dict, results: list[ClipResult]) -> None: + """Every clip's reference, hypothesis, errors and latency, for diffing runs.""" + clips = [ + { + "id": r.clip.id, + "voice": r.clip.voice, + "reference": r.clip.reference, + "hypothesis": r.text, + "errors": r.errors, + "words": len(r.reference_words), + "ms": round(r.ms, 1), + "rms": round(r.rms, 1), + } + for r in results + ] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({**report, "clips": clips}, indent=2) + "\n") + _log(f"per-clip detail written to {path}") + + +def _speak_dots(text: str) -> str: + """Read "." the way it is spoken. Runs before lowercasing, because a period + followed by a capital is a sentence break whisper glued shut.""" + text = re.sub(r"(?<=\d)\.(?=\d)", " point ", text) + text = re.sub(r"\.(?=[A-Z])", " ", text) + return re.sub(r"(? str: + return " dash " * len(match.group(1)) + match.group(2).replace("-", " ") + + +def _close_compound(match: re.Match) -> str: + parts = match.group(0).split("-") + if all(_number_word(part)[0] for part in parts): + return " ".join(parts) + return "".join(parts) + + +def _words_to_numbers(words: list[str]) -> list[str]: + out: list[str] = [] + run = _NumberRun() + for word in words: + if run.take(word): + continue + out += run.flush() + if not run.take(word): + out.append(word) + return out + run.flush() + + +def _number_word(word: str) -> tuple[str | None, int]: + if word in _UNITS: + value = _UNITS[word] + return ("zero" if value == 0 else "unit" if value < 10 else "teen"), value + if word in _TENS: + return "tens", _TENS[word] + if word == "hundred": + return "hundred", 100 + if word in _SCALES: + return "scale", _SCALES[word] + return None, 0 + + +def _digest(*parts: str) -> str: + return hashlib.sha256("\0".join(parts).encode()).hexdigest()[:12] + + +def _slug(name: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") + + +def _run(command: list[str]) -> None: + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode != 0: + raise EvalError(f"{command[0]} failed: {result.stderr.strip()}") + + +def _log(message: str) -> None: + print(message, file=sys.stderr, flush=True) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_eval.py b/tests/test_eval.py new file mode 100644 index 0000000..26d6966 --- /dev/null +++ b/tests/test_eval.py @@ -0,0 +1,257 @@ +"""The eval harness's scoring: normalization, WER, percentiles, and its inputs. + +None of this needs macOS or a whisper-server. The harness is loaded from its +file because eval/ is a directory of scripts, not a package (and `eval` would +shadow the builtin if it were one). +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +_spec = importlib.util.spec_from_file_location( + "tacet_eval", ROOT / "eval" / "tacet_eval.py" +) +te = importlib.util.module_from_spec(_spec) +sys.modules["tacet_eval"] = te +_spec.loader.exec_module(te) + + +def result(reference: str, hypothesis: str, voice: str = "Samantha"): + clip = te.Clip("x", voice, reference, Path("/nonexistent.wav")) + return te.ClipResult(clip, hypothesis, ms=1.0, rms=1000.0) + + +class TestNormalizeCaseAndPunctuation: + def test_case_and_punctuation_never_count(self): + assert te.normalize("Hello, World!") == te.normalize("hello world") + + def test_apostrophes_are_deleted_not_spaced(self): + assert te.normalize("Don't") == ["dont"] + assert te.normalize("I’ll") == ["ill"] + + def test_empty_and_punctuation_only(self): + assert te.normalize("") == [] + assert te.normalize(" . ") == [] + + def test_sentence_glued_by_whisper_splits(self): + assert te.normalize("Done.Next line") == ["done", "next", "line"] + + def test_hyphenated_compound_closes_up(self): + assert te.normalize("stand-up") == te.normalize("standup") + assert te.normalize("T-Mux") == ["tmux"] + + +class TestNormalizeSymbols: + def test_written_path_scores_as_spoken_path(self): + assert te.normalize("/etc/hosts") == te.normalize("slash etc slash hosts") + + def test_dotted_filename(self): + assert te.normalize("README.md") == ["readme", "dot", "md"] + + def test_dotfile_in_a_home_path(self): + assert te.normalize("~/.config") == ["tilde", "slash", "dot", "config"] + + def test_long_flag_keeps_its_words(self): + spoken = "dash dash force with lease" + assert te.normalize("--force-with-lease") == te.normalize(spoken) + + def test_short_flag(self): + assert te.normalize("-v") == ["dash", "v"] + + def test_free_standing_dash_is_punctuation(self): + assert te.normalize("a - b") == ["a", "b"] + + def test_underscore(self): + assert te.normalize("DATABASE_URL") == ["database", "underscore", "url"] + + +class TestNormalizeNumbers: + @pytest.mark.parametrize( + "spoken, digits", + [ + ("ten", "10"), + ("twenty five", "25"), + ("twenty-five", "25"), + ("one hundred five", "105"), + ("three thousand five hundred twenty one", "3521"), + ], + ) + def test_spelled_cardinals_become_digits(self, spoken, digits): + assert te.normalize(spoken) == [digits] + + def test_thousands_separator_dropped(self): + assert te.normalize("8,080") == ["8080"] + + def test_runs_that_are_not_one_number_stay_split(self): + assert te.normalize("seven fifteen") == ["7", "15"] + assert te.normalize("nineteen eighty four") == ["19", "84"] + assert te.normalize("one two three") == ["1", "2", "3"] + + def test_clock_time_either_separator(self): + assert ( + te.normalize("7:15") + == te.normalize("7.15") + == te.normalize("seven fifteen") + ) + + def test_decimal_reads_as_point(self): + assert te.normalize("3.5") == te.normalize("three point five") + + def test_numbers_inside_words_are_left_alone(self): + assert te.normalize("ten minutes late") == ["10", "minutes", "late"] + assert te.normalize("tenant") == ["tenant"] + + +class TestWordErrors: + def test_identical(self): + assert te.word_errors(["a", "b"], ["a", "b"]) == 0 + + def test_substitution(self): + assert te.word_errors(["a", "b", "c"], ["a", "x", "c"]) == 1 + + def test_deletion(self): + assert te.word_errors(["a", "b", "c"], ["a", "c"]) == 1 + + def test_insertion(self): + assert te.word_errors(["a", "c"], ["a", "b", "c"]) == 1 + + def test_empty_hypothesis_deletes_everything(self): + assert te.word_errors(["a", "b", "c"], []) == 3 + + def test_empty_reference_counts_insertions(self): + assert te.word_errors([], ["you"]) == 1 + + +class TestCorpusWer: + def test_micro_average_weights_by_reference_words(self): + # One error in a 1-word clip and none in a 3-word clip: 1/4, not the + # 0.5 a per-clip average would give. + results = [result("hello", "goodbye"), result("one two three", "1 2 3")] + assert te.corpus_wer(results) == pytest.approx(0.25) + + def test_normalization_applies_to_both_sides(self): + assert ( + te.corpus_wer( + [result("Add it to /etc/hosts.", "add it to slash etc slash hosts")] + ) + == 0.0 + ) + + def test_insertions_can_push_past_one(self): + # Not clamped: WER above 1 is a real signal (a hallucinating model). + assert te.corpus_wer([result("yes", "yes yes yes")]) == pytest.approx(2.0) + + def test_no_reference_words(self): + assert te.corpus_wer([]) == 0.0 + + +class TestPercentile: + def test_nearest_rank(self): + values = list(range(1, 101)) + assert te.percentile(values, 50) == 50 + assert te.percentile(values, 95) == 95 + + def test_small_sample(self): + assert te.percentile([30.0, 10.0, 20.0], 50) == 20.0 + assert te.percentile([30.0, 10.0, 20.0], 95) == 30.0 + + def test_empty(self): + assert te.percentile([], 50) == 0.0 + + +class TestSilenceText: + @pytest.mark.parametrize("text", ["", " ", ".", " . "]) + def test_punctuation_is_not_text(self, text): + assert not te.has_text(text) + + @pytest.mark.parametrize("text", ["you", " Thank you.", "[BLANK_AUDIO]"]) + def test_any_letter_is_text(self, text): + assert te.has_text(text) + + +class TestCorpus: + def test_shipped_corpus_meets_the_floor(self): + corpus = te.load_corpus(te.DEFAULT_CORPUS) + assert len({u.id for u in corpus.utterances}) >= 40 + assert len(corpus.silences) >= 5 + assert len(corpus.voices) >= 3 + + def test_shipped_corpus_has_loud_silence(self): + # Clips under tacet's RMS gate never reach whisper in production, so a + # silence set made only of those would measure nothing that matters. + corpus = te.load_corpus(te.DEFAULT_CORPUS) + loud = [s for s in corpus.silences if s.dbfs is not None and s.dbfs > -47] + assert loud + + def test_spoken_defaults_to_text(self, tmp_path): + path = tmp_path / "c.toml" + path.write_text('voices = ["A"]\n[[utterance]]\nid = "u"\ntext = "hi"\n') + assert te.load_corpus(path).utterances[0].spoken == "hi" + + def test_duplicate_ids_are_refused(self, tmp_path): + path = tmp_path / "c.toml" + path.write_text( + '[[utterance]]\nid = "u"\ntext = "a"\n[[utterance]]\nid = "u"\ntext = "b"\n' + ) + with pytest.raises(te.EvalError, match="duplicate"): + te.load_corpus(path) + + def test_unknown_silence_kind_is_refused(self, tmp_path): + path = tmp_path / "c.toml" + path.write_text('[[silence]]\nid = "s"\nkind = "pink"\nseconds = 1.0\n') + with pytest.raises(te.EvalError, match="pink"): + te.load_corpus(path) + + def test_silence_samples_hit_their_level(self): + clip = te.Silence("s", "white", 0.5, -40) + samples = te._silence_samples(clip) + rms = (sum(s * s for s in samples) / len(samples)) ** 0.5 + assert rms == pytest.approx(327.7, rel=0.01) + assert te._silence_samples(clip) == samples # seeded: same bytes every run + + +class TestArguments: + def test_server_args_are_shell_split_and_repeatable(self): + args = te.parse_args( + ["--server-arg=--no-fallback", "--server-arg=--entropy-thold 2.8"] + ) + assert args.server_args == ["--no-fallback", "--entropy-thold", "2.8"] + + def test_production_flags_by_default(self): + settings = te.run_settings(te.parse_args([]), ["Samantha"]) + assert settings.flags == list(te.PRODUCTION_FLAGS) + assert "--prompt" not in settings.flags + + def test_prompt_and_beam_size_pass_through(self): + args = te.parse_args( + ["--prompt", "tmux, kubectl", "--beam-size", "5", "--server-arg=-nf"] + ) + flags = te.run_settings(args, ["Samantha"]).flags + assert flags[-5:] == ["--prompt", "tmux, kubectl", "--beam-size", "5", "-nf"] + + def test_voices_override(self): + assert te.parse_args(["--voices", "Daniel, Karen"]).voices == [ + "Daniel", + "Karen", + ] + + +class TestRequest: + def test_multipart_matches_whisper_client(self): + body = te.multipart_body("B", b"RIFF") + assert body == ( + b"--B\r\n" + b'Content-Disposition: form-data; name="file"; filename="audio.wav"\r\n' + b"Content-Type: audio/wav\r\n\r\n" + b"RIFF\r\n" + b'--B\r\nContent-Disposition: form-data; name="response_format"\r\n\r\njson\r\n' + b'--B\r\nContent-Disposition: form-data; name="temperature"\r\n\r\n0.0\r\n' + b"--B--\r\n" + ) + + def test_free_port_is_never_a_live_tacet_port(self): + assert te._free_port() not in te.LIVE_PORTS From 350b9d12b9dc2538096d0da533988464b004bb56 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Thu, 5 Nov 2026 16:26:12 -0500 Subject: [PATCH 2/3] [eval] Don't count whisper splitting a word Whisper often writes a compound or identifier in pieces: "CH migrate" for ch-migrate, "DRY 1687" for DRY-1687, "large V3 turbo". Pasted into an agent prompt those read fine, yet each scored two or three errors, so the WER measured spacing as much as recognition. Two or three adjacent hypothesis words are now joined when the joined form is a reference word. Gluing two real words together ("Ontomain") still counts, and WER still divides by the reference's own word count. The synthetic baseline drops from 3.83% to 3.53% on the same transcripts (two fresh runs agree exactly). The 4.32% first recorded for this corpus did not reproduce under the old rules either. Solves: WER counting split compounds and ids as misrecognition Tests: tests/test_eval.py spacing cases; full suite passes --- README.md | 6 ++++-- eval/tacet_eval.py | 26 +++++++++++++++++++++++++- tests/test_eval.py | 21 +++++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5eda545..370f51d 100644 --- a/README.md +++ b/README.md @@ -413,6 +413,8 @@ ever reach a user. same as "slash etc slash hosts", and `--dry-run` the same as "dash dash dry run". The eval measures recognition, not formatting. - Hyphenated compounds close up (`stand-up` = `standup`). +- Whisper splitting a word never counts: "CH migrate" = `ch-migrate`, "TKT + 1687" = `TKT-1687`. Gluing two words together ("Ontomain") still does. - Spelled-out numbers become digits ("twenty five" = 25), and `7:15` = `7.15` = "seven fifteen". @@ -424,8 +426,8 @@ ggml-large-v3-turbo, production flags, empty `--prompt`): | | | |---|---| -| WER | 4.32% (Samantha 3.41%, Daniel 4.38%, Karen 4.38%, Moira 5.11%) | -| latency | p50 239 ms, p95 253 ms | +| WER | 3.53%, 58 of 1,644 words (Samantha 3.16%, Daniel 2.68%, Karen 4.62%, Moira 3.65%) | +| latency | p50 242 ms, p95 264 ms | | silence | 2 of 7 clips returned text ("you" on digital silence). Both are below the RMS gate, so tacet never sends them | Most of the errors are jargon whisper has never seen. It writes "cube diff --git a/eval/tacet_eval.py b/eval/tacet_eval.py index 2dac47f..8a3734f 100644 --- a/eval/tacet_eval.py +++ b/eval/tacet_eval.py @@ -151,7 +151,8 @@ def reference_words(self) -> list[str]: @property def errors(self) -> int: - return word_errors(self.reference_words, normalize(self.text)) + reference = self.reference_words + return word_errors(reference, join_split_words(reference, normalize(self.text))) @dataclass(frozen=True) @@ -474,6 +475,14 @@ def word_errors(reference: list[str], hypothesis: list[str]) -> int: return previous[-1] +def join_split_words(reference: list[str], hypothesis: list[str]) -> list[str]: + """Whisper splitting a word never counts: "CH migrate" scores the same as + ch-migrate ("chmigrate"), and "TKT 1687" the same as TKT-1687. Two or three + adjacent hypothesis words are joined when the joined form is a reference + word. Gluing two words together ("Ontomain" for "onto main") still counts.""" + return _join_runs(hypothesis, set(reference)) + + def corpus_wer(results: list[ClipResult]) -> float: """Micro-averaged WER: total word errors over total reference words.""" words = sum(len(r.reference_words) for r in results) @@ -594,6 +603,21 @@ def flush(self) -> list[str]: _SILENCE_KINDS = ("zero", "white", "brown", "hum") +def _join_runs(words: list[str], targets: set[str]) -> list[str]: + out, i = [], 0 + while i < len(words): + for n in (3, 2): + joined = "".join(words[i : i + n]) + if i + n <= len(words) and joined in targets: + out.append(joined) + i += n + break + else: + out.append(words[i]) + i += 1 + return out + + def _voice_list(value: str) -> list[str]: return [v.strip() for v in value.split(",") if v.strip()] diff --git a/tests/test_eval.py b/tests/test_eval.py index 26d6966..a2ae39f 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -255,3 +255,24 @@ def test_multipart_matches_whisper_client(self): def test_free_port_is_never_a_live_tacet_port(self): assert te._free_port() not in te.LIVE_PORTS + + +class TestSpacingAlignment: + @pytest.mark.parametrize( + "reference, hypothesis", + [ + ("Bump ch-migrate.", "Bump CH migrate."), + ("Move TKT-1687 to Backlog.", "Move TKT 1687 to backlog."), + ("the standup", "the stand up"), + ("Try large-v3-turbo.", "Try large V3 turbo."), + ], + ) + def test_spacing_inside_a_word_never_counts(self, reference, hypothesis): + assert result(reference, hypothesis).errors == 0 + + def test_a_wrong_word_still_counts(self): + # Nothing joins to "chmigrate", so it is a substitution plus an insertion. + assert result("Bump ch-migrate.", "Bump CH migrant.").errors == 2 + + def test_gluing_two_words_still_counts(self): + assert result("Rebase onto main.", "Rebase Ontomain.").errors == 2 From 38c551caa0059bbc3e8b7beea2a0fa3b003e18e5 Mon Sep 17 00:00:00 2001 From: Daniel Young Date: Thu, 5 Nov 2026 16:27:00 -0500 Subject: [PATCH 3/3] [eval] Record and score your own voice The synthetic corpus is 44 sentences of text-to-speech: about 410 words per voice, with no microphone, no room and no hesitation, and some misses are say's own pronunciation (it reads "p95" as "page 95"). It can catch a flag that breaks something; it can't settle whether a prompt, a decoder setting or a model is better for a real speaker. eval/record.py records a script line by line through eval/recorder.swift, which copies Recorder.swift's capture path (AVAudioEngine default input, 4096-frame tap, AVAudioConverter to 16 kHz mono 16-bit, canonical 44-byte header). Press Enter and talk as with the hotkey; keep, redo or play back each take. Takes go outside the repo, to ~/.local/share/tacet-eval/recordings//, each with a note of what was read, on which Mac and input, and how loud. tacet_eval.py --recordings scores them, one session per microphone and Mac in place of a voice. Corpus lines can carry a split (tune on one, score the other once) and a kind, reported separately, and [[quiet]] takes record real non-speech in place of digital silence. A missing take, or one recorded for text that has since changed, stops the run. Solves: no way to measure tacet on a real voice and microphone Tests: tests/test_eval.py recordings, splits, quiet takes; tests/test_record.py; recorder.swift --selftest --- README.md | 55 ++++++++- eval/record.py | 276 +++++++++++++++++++++++++++++++++++++++++++ eval/recorder.swift | 153 ++++++++++++++++++++++++ eval/tacet_eval.py | 188 ++++++++++++++++++++++++++--- tests/test_eval.py | 125 ++++++++++++++++++++ tests/test_record.py | 59 +++++++++ 6 files changed, 839 insertions(+), 17 deletions(-) create mode 100644 eval/record.py create mode 100644 eval/recorder.swift create mode 100644 tests/test_record.py diff --git a/README.md b/README.md index 370f51d..79cb6df 100644 --- a/README.md +++ b/README.md @@ -374,6 +374,14 @@ no evidence anywhere. `eval/tacet_eval.py` measures the transcription leg on its own: how accurate and how fast whisper-server is with tacet's model and flags. +It has two kinds of input. The shipped corpus is synthetic: 44 sentences +spoken by macOS `say`. It is a smoke test, good for "did this flag break +something" and for server latency. It is too small and too clean to settle an +accuracy question: one word moves a voice's WER by about a quarter of a point, +and a text-to-speech voice has no microphone, no room and no hesitation. For +accuracy, record your own voice ([below](#recording-your-own-voice)) and score +that. + ```bash uv run python eval/tacet_eval.py # human-readable, worst clips listed uv run python eval/tacet_eval.py --json # one JSON object on stdout @@ -435,7 +443,52 @@ control" for kubectl, "Maine" for main, "launch deep list" for launchd plist, "tacit" for tacet, and "post-gaz" for Postgres. That is what `--prompt` is for. Synthetic voices are cleaner than a real microphone in a real room, so read these numbers as a fixed yardstick for comparing settings, not as the accuracy -you will get. +you will get. Some misses are the voice's, not whisper's: the corpus tells +`say` to pronounce kubectl "cube control", as many people do, and whisper +writes exactly that. + +### Recording your own voice + +`eval/record.py` records a script line by line through `eval/recorder.swift`, +which captures the way tacet does (AVAudioEngine's default input, a 4096-frame +tap, AVAudioConverter to 16 kHz mono 16-bit, the same 44-byte header). + +```bash +uv run python eval/record.py --script my-script.toml # resumes where it stopped +uv run python eval/tacet_eval.py --corpus my-script.toml --recordings --split tune +``` + +Press Enter and start talking, as with the hotkey, then Enter again to stop. +Each take can be kept, redone or played back, and the recorder warns about a +take quieter than tacet's gate, one that cut off, or a terminal without +microphone access. Run it in a plain terminal window on the Mac whose +microphone you mean to test; macOS asks once for access. + +Takes go to `~/.local/share/tacet-eval/recordings//.wav`, outside +the repository because they are your voice. A session is one microphone on one +Mac, named after both, and is reported in place of a voice. Beside each take, +`.json` records what was read, where, and how loud it was. A take whose +script line has changed since is refused until you `--redo` it. + +The script uses the corpus format with three optional fields and one table: + +```toml +[[utterance]] +id = "r12" +split = "tune" # or "holdout": tune on one, score the other once +kind = "short" # any label; the report breaks WER down by it +text = "Edit ~/.claude/settings.json" # what should be pasted +spoken = "Edit tilde slash dot claude slash settings dot json" # shown as a reading hint + +[[quiet]] # a take with no speech: any text back is a false paste +id = "q1" +split = "tune" +do = "Take one slow breath in and out. No words." +``` + +Fix the split before anyone sees a score, and keep jargon in the holdout that +never appears in a tune line. That way a `--prompt` built while tuning shows +what it does to words it was never given. ## What's been tested, and what hasn't diff --git a/eval/record.py b/eval/record.py new file mode 100644 index 0000000..88109ea --- /dev/null +++ b/eval/record.py @@ -0,0 +1,276 @@ +"""Record your own voice for tacet's eval, one take per corpus line. + + uv run python eval/record.py --script PATH [--session NAME] [--out DIR] + [--split tune|holdout] [--redo ID ...] + +Shows each line of the script and records it through eval/recorder.swift, +which captures the way tacet does (AVAudioEngine's default input, converted to +16 kHz mono 16-bit). Each take is saved as //.wav with an +.json note of what was read, on which Mac and which input. Lines already +recorded are skipped, so a second run picks up where the first stopped. + +Press Enter and start talking, the way you would with the hotkey; press Enter +again when you are done. Then keep the take, redo it, or play it back. +[[quiet]] lines ask for a take with no speech in it: a breath, typing, the room. + +Run it in a plain terminal window (not inside tmux, not over SSH) on the Mac +whose microphone you mean to test. macOS asks once for microphone access for +that terminal app. Score the takes afterwards with: + + uv run python eval/tacet_eval.py --corpus PATH --recordings --split tune +""" + +from __future__ import annotations + +import argparse +import datetime +import json +import platform +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import tacet_eval as te # noqa: E402 + +RECORDER_SOURCE = te.EVAL_DIR / "recorder.swift" +RECORDER_BINARY = te.CACHE_DIR / "tacet-recorder" +SHORT_TAKE_S = 0.6 + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + corpus = te.select_split(te.load_corpus(args.script), args.split) + items = [*corpus.utterances, *corpus.quiet] + with Microphone(build_recorder()) as mic: + folder = args.out / ( + args.session or session_name(platform.node(), mic.device) + ) + todo = pending(items, folder, args.redo) + print(f"{len(todo)} of {len(items)} takes to record into {folder}") + print(f"input: {mic.device}\n") + for n, item in enumerate(todo, 1): + print(f"[{n}/{len(todo)}] {item.id}") + if not record_item(mic, item, folder): + break + except te.EvalError as e: + print(f"record: {e}", file=sys.stderr) + return 1 + except KeyboardInterrupt: + print("\nstopped; takes kept so far are saved", file=sys.stderr) + return 130 + return 0 + + +@dataclass(frozen=True) +class Take: + """What the recorder reported for one take.""" + + frames: int + rms: float + peak: int + + @property + def seconds(self) -> float: + return self.frames / te.SAMPLE_RATE + + +class Microphone: + """The tacet-recorder process: one long-lived capture engine, as in tacet.""" + + def __init__(self, binary: Path): + self.proc = subprocess.Popen( + [str(binary)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + bufsize=1, + ) + self.device = "" + + def __enter__(self) -> Microphone: + reply = self._read() + if reply == "denied": + raise te.EvalError( + "microphone access is denied for this terminal app: allow it in " + "System Settings > Privacy & Security > Microphone, then rerun" + ) + if not reply.startswith("ready "): + raise te.EvalError(f"recorder did not start: {reply!r}") + self.device = reply.removeprefix("ready ") + return self + + def __exit__(self, *exc: object) -> None: + if self.proc.poll() is None: + self._send("quit") + self.proc.wait(timeout=10) + + def start(self) -> None: + self._send("start") + reply = self._read() + if reply != "started": + raise te.EvalError(f"could not start capture: {reply}") + + def stop(self, path: Path) -> Take: + self._send(f"stop {path}") + reply = self._read() + if not reply.startswith("saved "): + raise te.EvalError(f"could not save the take: {reply}") + frames, rms, peak = reply.split()[1:] + return Take(int(frames), float(rms), int(peak)) + + def _send(self, line: str) -> None: + assert self.proc.stdin is not None + self.proc.stdin.write(line + "\n") + self.proc.stdin.flush() + + def _read(self) -> str: + assert self.proc.stdout is not None + line = self.proc.stdout.readline() + if not line: + raise te.EvalError(f"recorder exited with {self.proc.wait()}") + return line.strip() + + +def parse_args(argv: list[str] | None) -> argparse.Namespace: + p = argparse.ArgumentParser(description="Record takes for tacet's eval.") + p.add_argument("--script", type=Path, required=True, help="the corpus TOML to read") + p.add_argument("--session", help="name for this mic and Mac (default host-input)") + p.add_argument("--out", type=Path, default=te.RECORDINGS_DIR) + p.add_argument("--split", choices=te.SPLITS, help="record one split only") + p.add_argument( + "--redo", nargs="+", default=[], metavar="ID", help="re-record these" + ) + return p.parse_args(argv) + + +def build_recorder() -> Path: + """Compile recorder.swift into the cache when it is missing or older.""" + stale = ( + not RECORDER_BINARY.exists() + or RECORDER_BINARY.stat().st_mtime < RECORDER_SOURCE.stat().st_mtime + ) + if stale: + print("building the recorder (once, about 15 s)...", file=sys.stderr) + RECORDER_BINARY.parent.mkdir(parents=True, exist_ok=True) + built = subprocess.run( + ["swiftc", "-O", str(RECORDER_SOURCE), "-o", str(RECORDER_BINARY)], + capture_output=True, + text=True, + ) + if built.returncode != 0: + raise te.EvalError(f"swiftc failed: {built.stderr.strip()[-800:]}") + return RECORDER_BINARY + + +def session_name(host: str, device: str) -> str: + """'Dans-Mac-Studio.local' + 'MacBook Pro Microphone' -> 'dans-mac-studio-macbook-pro-microphone'.""" + return f"{_slug(host.split('.')[0])}-{_slug(device)}" + + +def pending(items: list, folder: Path, redo: list[str]) -> list: + """Lines with no take yet, plus any asked for again, in script order.""" + unknown = sorted(set(redo) - {item.id for item in items}) + if unknown: + raise te.EvalError(f"--redo names ids not in the script: {', '.join(unknown)}") + return [i for i in items if i.id in redo or not (folder / f"{i.id}.wav").exists()] + + +def record_item(mic: Microphone, item: te.Utterance | te.Quiet, folder: Path) -> bool: + """Record one line until it is kept. False when the speaker quits.""" + _show(item) + folder.mkdir(parents=True, exist_ok=True) + draft = folder / f"{item.id}.part.wav" + while True: + if input(" Enter to record, q to quit: ").strip().lower() == "q": + return False + mic.start() + input(" ● recording... Enter to stop ") + take = mic.stop(draft) + print( + f" {take.seconds:.1f} s, level {take.rms:.0f} (tacet drops audio under 150)" + ) + for warning in check_take(item, take): + print(f" ! {warning}") + choice = _choose(draft) + if choice == "keep": + _keep(item, draft, take, mic.device) + return True + if choice == "quit": + draft.unlink(missing_ok=True) + return False + + +def check_take(item: te.Utterance | te.Quiet, take: Take) -> list[str]: + """What a speaker should know before keeping a take.""" + if take.frames == 0: + return ["no audio arrived: is the input device connected?"] + if take.peak == 0: + return [ + "every sample is zero: the input is muted, or this terminal has no " + "microphone access (System Settings > Privacy & Security > Microphone)" + ] + if isinstance(item, te.Quiet): + return [] + warnings = [] + if take.rms < te.SILENCE_RMS_GATE: + warnings.append( + "quieter than tacet's gate, so tacet would drop it: move closer" + ) + if take.seconds < SHORT_TAKE_S: + warnings.append("very short: did it cut off?") + return warnings + + +def _show(item: te.Utterance | te.Quiet) -> None: + if isinstance(item, te.Quiet): + print(f" (no speech) {item.do}") + return + print(f"\n {item.text}\n") + if item.spoken != item.text: + print(f" read it as: {item.spoken}") + + +def _choose(draft: Path) -> str: + while True: + answer = input(" Enter to keep, r to redo, p to play it back, q to quit: ") + answer = answer.strip().lower() + if answer == "": + return "keep" + if answer in ("r", "q"): + return {"r": "redo", "q": "quit"}[answer] + if answer == "p": + subprocess.run(["afplay", str(draft)], check=False) + + +def _keep(item: te.Utterance | te.Quiet, draft: Path, take: Take, device: str) -> None: + wav = draft.with_name(f"{item.id}.wav") + draft.replace(wav) + note = { + "id": item.id, + "script_text": te.script_text(item), + "split": item.split, + "kind": getattr(item, "kind", None), + "session": wav.parent.name, + "host": platform.node(), + "device": device, + "seconds": round(take.seconds, 2), + "rms": round(take.rms, 1), + "peak": take.peak, + "recorded_at": datetime.datetime.now() + .astimezone() + .isoformat(timespec="seconds"), + } + wav.with_suffix(".json").write_text(json.dumps(note, indent=2) + "\n") + + +def _slug(name: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eval/recorder.swift b/eval/recorder.swift new file mode 100644 index 0000000..5e02120 --- /dev/null +++ b/eval/recorder.swift @@ -0,0 +1,153 @@ +// tacet-recorder: microphone capture for eval/record.py, done the way tacet's +// own Recorder (swift/Sources/tacet/Recorder.swift) does it, so a recorded eval +// clip is the audio tacet would have sent: AVAudioEngine's default input, a tap +// with a 4096-frame buffer, AVAudioConverter to 16 kHz mono 16-bit, and the +// canonical 44-byte WAV header. +// +// Built by record.py with `swiftc -O`. It speaks a line protocol: +// stdin: start | stop PATH | quit +// stdout: ready DEVICE-NAME (at launch) | denied | started +// | saved FRAMES RMS PEAK | error MESSAGE +// `tacet-recorder --selftest PATH` writes one second of a 440 Hz tone through +// the same WAV writer and exits, without touching the microphone. + +import AVFoundation +import Foundation + +struct Failure: Error { + let message: String + init(_ message: String) { self.message = message } +} + +final class Capture { + private let engine = AVAudioEngine() + private let queue = DispatchQueue(label: "tacet.eval.recorder") + private var samples = [Int16]() + private var peak: Int32 = 0 + private var started = false + + func start() throws { + queue.sync { + samples.removeAll(keepingCapacity: true) + peak = 0 + } + let input = engine.inputNode + let inFormat = input.inputFormat(forBus: 0) + guard inFormat.sampleRate > 0, inFormat.channelCount > 0 else { + throw Failure("default input reports no usable stream (\(inFormat.channelCount) ch, \(inFormat.sampleRate) Hz)") + } + guard let target = AVAudioFormat(commonFormat: .pcmFormatInt16, + sampleRate: 16000, channels: 1, interleaved: true), + let converter = AVAudioConverter(from: inFormat, to: target) else { + throw Failure("no converter from \(inFormat) to 16 kHz mono 16-bit") + } + input.installTap(onBus: 0, bufferSize: 4096, format: inFormat) { [weak self] buffer, _ in + self?.process(buffer, converter: converter, target: target) + } + engine.prepare() + try engine.start() + started = true + } + + func stop() -> (samples: [Int16], peak: Int32) { + guard started else { return ([], 0) } + started = false + engine.inputNode.removeTap(onBus: 0) + engine.stop() + return queue.sync { (samples, peak) } + } + + private func process(_ buffer: AVAudioPCMBuffer, converter: AVAudioConverter, + target: AVAudioFormat) { + let ratio = target.sampleRate / buffer.format.sampleRate + let capacity = AVAudioFrameCount(Double(buffer.frameLength) * ratio) + 1024 + guard let out = AVAudioPCMBuffer(pcmFormat: target, frameCapacity: capacity) else { return } + var provided = false + var error: NSError? + converter.convert(to: out, error: &error) { _, status in + if provided { status.pointee = .noDataNow; return nil } + provided = true + status.pointee = .haveData + return buffer + } + guard out.frameLength > 0, let channel = out.int16ChannelData else { return } + queue.sync { + for i in 0.. Data { + let dataBytes = samples.count * 2 + var out = Data(capacity: 44 + dataBytes) + func ascii(_ s: String) { out.append(Data(s.utf8)) } + func le16(_ v: Int) { out.append(UInt8(v & 0xFF)); out.append(UInt8((v >> 8) & 0xFF)) } + func le32(_ v: Int) { le16(v & 0xFFFF); le16((v >> 16) & 0xFFFF) } + ascii("RIFF"); le32(36 + dataBytes); ascii("WAVE") + ascii("fmt "); le32(16); le16(1); le16(1); le32(16000); le32(32000); le16(2); le16(16) + ascii("data"); le32(dataBytes) + samples.withUnsafeBytes { out.append(contentsOf: $0) } // little endian on arm64 and x86 + return out +} + +func save(_ samples: [Int16], peak: Int32, to path: String) throws -> String { + try wavData(samples).write(to: URL(fileURLWithPath: path), options: .atomic) + let sumSquares = samples.reduce(0.0) { $0 + Double($1) * Double($1) } + let rms = samples.isEmpty ? 0 : (sumSquares / Double(samples.count)).squareRoot() + return "saved \(samples.count) \(String(format: "%.1f", rms)) \(peak)" +} + +func microphoneAllowed() -> Bool { + switch AVCaptureDevice.authorizationStatus(for: .audio) { + case .authorized: return true + case .notDetermined: + let done = DispatchSemaphore(value: 0) + var granted = false + AVCaptureDevice.requestAccess(for: .audio) { granted = $0; done.signal() } + done.wait() + return granted + default: return false + } +} + +func say(_ line: String) { + print(line) + fflush(stdout) +} + +func serve() -> Int32 { + guard microphoneAllowed() else { say("denied"); return 3 } + say("ready \(AVCaptureDevice.default(for: .audio)?.localizedName ?? "unknown input")") + let capture = Capture() + while let line = readLine() { + let parts = line.split(separator: " ", maxSplits: 1).map(String.init) + do { + switch parts.first { + case "start": try capture.start(); say("started") + case "stop" where parts.count == 2: + let (samples, peak) = capture.stop() + say(try save(samples, peak: peak, to: parts[1])) + case "quit": _ = capture.stop(); return 0 + default: say("error unknown command: \(line)") + } + } catch let failure as Failure { + say("error \(failure.message)") + } catch { + say("error \(error)") + } + } + _ = capture.stop() + return 0 +} + +let arguments = CommandLine.arguments +if arguments.count == 3, arguments[1] == "--selftest" { + let tone = (0..<16000).map { Int16(8000 * sin(2 * Double.pi * 440 * Double($0) / 16000)) } + say((try? save(tone, peak: 8000, to: arguments[2])) ?? "error could not write \(arguments[2])") + exit(0) +} +exit(serve()) diff --git a/eval/tacet_eval.py b/eval/tacet_eval.py index 8a3734f..83fabd5 100644 --- a/eval/tacet_eval.py +++ b/eval/tacet_eval.py @@ -3,6 +3,7 @@ uv run python eval/tacet_eval.py [--model PATH] [--json] [--prompt TEXT] [--beam-size N] [--server-arg=FLAG ...] [--voices A,B] [--corpus PATH] [--details PATH] + [--recordings [DIR]] [--split tune|holdout] Speaks every utterance in eval/corpus.toml with macOS `say` in several voices, converts it with `afconvert` to the 16 kHz mono 16-bit WAV tacet's @@ -12,6 +13,11 @@ many silence clips came back with text, then stops the server. The live tacet services (ports 8910 and 8911) are never contacted. +With --recordings it sends recorded takes instead (eval/record.py makes +them): DIR//.wav for every utterance and [[quiet]] take in the +corpus, one session per microphone and Mac, reported in place of the voices. +--split scores only the corpus lines marked `split = "tune"` or "holdout". + Standard library only. Generated audio is cached in eval/.cache/ (gitignored). Progress goes to stderr; with --json, stdout carries exactly one JSON object. @@ -51,6 +57,9 @@ DEFAULT_CORPUS = EVAL_DIR / "corpus.toml" CACHE_DIR = EVAL_DIR / ".cache" DEFAULT_MODEL = Path.home() / ".local/share/whisper-cpp/ggml-large-v3-turbo.bin" +# Recorded takes live outside git: they are someone's voice. +RECORDINGS_DIR = Path.home() / ".local/share/tacet-eval/recordings" +SPLITS = ("tune", "holdout") # tacet's own services. The harness refuses to run a server on either. LIVE_PORTS = frozenset({8910, 8911}) @@ -80,9 +89,8 @@ def main(argv: list[str] | None = None) -> int: args = parse_args(argv) try: - corpus = load_corpus(args.corpus) - voices = args.voices or corpus.voices - clips = prepare_audio(corpus, voices) + corpus = select_split(load_corpus(args.corpus), args.split) + clips, voices = gather_clips(corpus, args) settings = run_settings(args, voices) with WhisperServer(settings, args.model, args.load_timeout) as server: results = transcribe_all(server, clips) @@ -107,6 +115,17 @@ class Utterance: id: str text: str spoken: str + split: str | None = None + kind: str | None = None + + +@dataclass(frozen=True) +class Quiet: + """A recorded take with no speech in it: any text back is a false paste.""" + + id: str + do: str + split: str | None = None @dataclass(frozen=True) @@ -122,20 +141,24 @@ class Corpus: voices: list[str] utterances: list[Utterance] silences: list[Silence] + quiet: list[Quiet] = field(default_factory=list) @dataclass(frozen=True) class Clip: - """One WAV to send: a spoken utterance in one voice, or a silence clip.""" + """One WAV to send: an utterance in one voice or recording session, or a + clip with no speech (generated silence has no voice; a quiet take does).""" id: str voice: str | None reference: str path: Path + silent: bool = False + kind: str | None = None @property def is_silence(self) -> bool: - return self.voice is None + return self.silent or self.voice is None @dataclass(frozen=True) @@ -165,6 +188,8 @@ class Settings: server_args: list[str] voices: list[str] corpus: str + split: str | None = None + recordings: str | None = None flags: list[str] = field(init=False) def __post_init__(self) -> None: @@ -185,6 +210,8 @@ def as_dict(self) -> dict: "request": dict(REQUEST_FIELDS), "voices": self.voices, "corpus": self.corpus, + "split": self.split, + "recordings": self.recordings, "host": platform.node(), "date": datetime.date.today().isoformat(), } @@ -294,8 +321,20 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: dest="server_args", help="extra whisper-server flag, shell-split; repeatable", ) - p.add_argument("--voices", type=_voice_list, help="comma-separated say voices") + p.add_argument( + "--voices", + type=_voice_list, + help="comma-separated say voices, or recording sessions with --recordings", + ) p.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS) + p.add_argument( + "--recordings", + type=Path, + nargs="?", + const=RECORDINGS_DIR, + help=f"send recorded takes from DIR (default {RECORDINGS_DIR})", + ) + p.add_argument("--split", choices=SPLITS, help="score one split of the corpus") p.add_argument("--whisper-server", dest="binary", default=_default_binary()) p.add_argument("--details", type=Path, default=CACHE_DIR / "last-run.json") p.add_argument("--load-timeout", type=float, default=300.0) @@ -307,21 +346,87 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: def load_corpus(path: Path) -> Corpus: data = tomllib.loads(path.read_text()) utterances = [ - Utterance(u["id"], u["text"], u.get("spoken", u["text"])) + Utterance( + u["id"], + u["text"], + u.get("spoken", u["text"]), + u.get("split"), + u.get("kind"), + ) for u in data.get("utterance", []) ] silences = [ Silence(s["id"], s["kind"], float(s["seconds"]), s.get("dbfs")) for s in data.get("silence", []) ] - ids = [x.id for x in utterances + silences] - duplicates = sorted({i for i in ids if ids.count(i) > 1}) - if duplicates: - raise EvalError(f"duplicate corpus ids: {', '.join(duplicates)}") - unknown = sorted({s.kind for s in silences} - set(_SILENCE_KINDS)) - if unknown: - raise EvalError(f"unknown silence kinds: {', '.join(unknown)}") - return Corpus(list(data.get("voices", [])), utterances, silences) + quiet = [Quiet(q["id"], q["do"], q.get("split")) for q in data.get("quiet", [])] + corpus = Corpus(list(data.get("voices", [])), utterances, silences, quiet) + _check_corpus(corpus) + return corpus + + +def select_split(corpus: Corpus, split: str | None) -> Corpus: + """Keep only the lines of one split. Generated silence has no split and is + kept either way; it never reaches a tuning decision on its own.""" + if split is None: + return corpus + utterances = [u for u in corpus.utterances if u.split == split] + if not utterances: + raise EvalError(f"no utterances marked split = {split!r} in the corpus") + quiet = [q for q in corpus.quiet if q.split == split] + return Corpus(corpus.voices, utterances, corpus.silences, quiet) + + +def gather_clips( + corpus: Corpus, args: argparse.Namespace +) -> tuple[list[Clip], list[str]]: + """The clips to send and the voices (or recording sessions) they come in.""" + if args.recordings is None: + voices = args.voices or corpus.voices + return prepare_audio(corpus, voices), voices + sessions = recorded_sessions(args.recordings, args.voices) + return load_recordings(corpus, args.recordings, sessions), sessions + + +def recorded_sessions(root: Path, wanted: list[str] | None) -> list[str]: + """Session directories under `root` that hold takes, or the ones asked for.""" + if not root.is_dir(): + raise EvalError(f"no recordings directory: {root} (record with eval/record.py)") + found = sorted( + d.name for d in root.iterdir() if d.is_dir() and any(d.glob("*.wav")) + ) + missing = [w for w in wanted or [] if w not in found] + if missing: + raise EvalError( + f"no takes for {', '.join(missing)} in {root} (found: {', '.join(found) or 'none'})" + ) + if not (wanted or found): + raise EvalError(f"no recording sessions in {root}") + return list(wanted or found) + + +def load_recordings(corpus: Corpus, root: Path, sessions: list[str]) -> list[Clip]: + """Every utterance and quiet take, once per session. A missing take, or one + recorded for different text, stops the run: a partial set is a different + yardstick.""" + clips, problems = [], [] + for session in sessions: + for item in [*corpus.utterances, *corpus.quiet]: + path = root / session / f"{item.id}.wav" + problem = _take_problem(path, script_text(item)) + if problem: + problems.append(f"{session}/{item.id}: {problem}") + elif isinstance(item, Quiet): + clips.append(Clip(item.id, session, "", path, silent=True)) + else: + clips.append(Clip(item.id, session, item.text, path, kind=item.kind)) + if problems: + shown = "\n ".join(problems[:20]) + more = f"\n ...and {len(problems) - 20} more" if len(problems) > 20 else "" + raise EvalError( + f"recorded takes need attention (eval/record.py):\n {shown}{more}" + ) + return clips def prepare_audio(corpus: Corpus, voices: list[str]) -> list[Clip]: @@ -333,11 +438,18 @@ def prepare_audio(corpus: Corpus, voices: list[str]) -> list[Clip]: jobs = [(u, v) for v in voices for u in corpus.utterances] with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: paths = list(pool.map(lambda job: _speech_wav(*job), jobs)) - clips = [Clip(u.id, v, u.text, path) for (u, v), path in zip(jobs, paths)] + clips = [ + Clip(u.id, v, u.text, path, kind=u.kind) for (u, v), path in zip(jobs, paths) + ] clips += [Clip(s.id, None, "", _silence_wav(s)) for s in corpus.silences] return clips +def script_text(item: Utterance | Quiet) -> str: + """What the speaker was shown for a take: the line, or a quiet take's instruction.""" + return item.do if isinstance(item, Quiet) else item.text + + def run_settings(args: argparse.Namespace, voices: list[str]) -> Settings: try: corpus = str(args.corpus.resolve().relative_to(EVAL_DIR.parent)) @@ -350,6 +462,8 @@ def run_settings(args: argparse.Namespace, voices: list[str]) -> Settings: server_args=args.server_args, voices=voices, corpus=corpus, + split=args.split, + recordings=str(args.recordings) if args.recordings else None, ) @@ -377,6 +491,7 @@ def build_report(model: Path, settings: Settings, results: list[ClipResult]) -> speech = [r for r in results if not r.clip.is_silence] silence = [r for r in results if r.clip.is_silence] latencies = [r.ms for r in speech] + kinds = sorted({r.clip.kind for r in speech if r.clip.kind}) return { "model": str(model), "settings": settings.as_dict(), @@ -386,6 +501,10 @@ def build_report(model: Path, settings: Settings, results: list[ClipResult]) -> voice: _voice_score([r for r in speech if r.clip.voice == voice]) for voice in settings.voices }, + "per_kind": { + kind: _voice_score([r for r in speech if r.clip.kind == kind]) + for kind in kinds + }, "latency_ms": { "p50": round(percentile(latencies, 50), 1), "p95": round(percentile(latencies, 95), 1), @@ -411,6 +530,10 @@ def format_summary(report: dict, results: list[ClipResult], worst: int = 12) -> lines.append( f" {voice:<10} {score['wer']:.2%} ({score['errors']}/{score['words']} words)" ) + for kind, score in report["per_kind"].items(): + lines.append( + f" [{kind}]{'':<{8 - len(kind)}} {score['wer']:.2%} ({score['errors']}/{score['words']} words)" + ) lines.append(f"latency p50 {lat['p50']} ms, p95 {lat['p95']} ms") lines.append( f"silence {sil['false_text']}/{sil['clips']} clips came back with text " @@ -603,6 +726,39 @@ def flush(self) -> list[str]: _SILENCE_KINDS = ("zero", "white", "brown", "hum") +def _check_corpus(corpus: Corpus) -> None: + items = [*corpus.utterances, *corpus.silences, *corpus.quiet] + ids = [x.id for x in items] + duplicates = sorted({i for i in ids if ids.count(i) > 1}) + if duplicates: + raise EvalError(f"duplicate corpus ids: {', '.join(duplicates)}") + unknown = sorted({s.kind for s in corpus.silences} - set(_SILENCE_KINDS)) + if unknown: + raise EvalError(f"unknown silence kinds: {', '.join(unknown)}") + splits = {x.split for x in [*corpus.utterances, *corpus.quiet]} - {None, *SPLITS} + if splits: + raise EvalError(f"unknown splits: {', '.join(sorted(splits))}") + + +def _take_problem(path: Path, script_text: str) -> str | None: + """Why a recorded take cannot be scored, or None. The note record.py writes + beside each take says what was read; if the script line changed since, the + take no longer matches its reference.""" + if not path.is_file(): + return "not recorded" + with wave.open(str(path), "rb") as w: + shape = (w.getnchannels(), w.getsampwidth(), w.getframerate()) + if shape != (1, 2, SAMPLE_RATE) or path.read_bytes()[36:40] != b"data": + return f"not a canonical 16 kHz mono 16-bit WAV ({shape})" + note = path.with_suffix(".json") + if ( + note.is_file() + and json.loads(note.read_text()).get("script_text") != script_text + ): + return "recorded for different text; redo it" + return None + + def _join_runs(words: list[str], targets: set[str]) -> list[str]: out, i = [], 0 while i < len(words): diff --git a/tests/test_eval.py b/tests/test_eval.py index a2ae39f..e5f5618 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -257,6 +257,35 @@ def test_free_port_is_never_a_live_tacet_port(self): assert te._free_port() not in te.LIVE_PORTS +SCRIPT = """ +voices = ["A"] +[[utterance]] +id = "u1" +split = "tune" +kind = "short" +text = "merge it" +[[utterance]] +id = "u2" +split = "holdout" +kind = "ids" +text = "Close TKT-1618." +[[quiet]] +id = "q1" +split = "tune" +do = "Say nothing." +""" + + +def write_take( + folder: Path, clip_id: str, samples: list[int], note: dict | None = None +): + folder.mkdir(parents=True, exist_ok=True) + frames = b"".join(s.to_bytes(2, "little", signed=True) for s in samples) + te._write_wav(folder / f"{clip_id}.wav", frames) + if note is not None: + (folder / f"{clip_id}.json").write_text(te.json.dumps(note)) + + class TestSpacingAlignment: @pytest.mark.parametrize( "reference, hypothesis", @@ -276,3 +305,99 @@ def test_a_wrong_word_still_counts(self): def test_gluing_two_words_still_counts(self): assert result("Rebase onto main.", "Rebase Ontomain.").errors == 2 + + +class TestSplitsAndQuiet: + def corpus(self, tmp_path): + path = tmp_path / "script.toml" + path.write_text(SCRIPT) + return te.load_corpus(path) + + def test_split_kind_and_quiet_are_read(self, tmp_path): + corpus = self.corpus(tmp_path) + assert [(u.id, u.split, u.kind) for u in corpus.utterances] == [ + ("u1", "tune", "short"), + ("u2", "holdout", "ids"), + ] + assert corpus.quiet == [te.Quiet("q1", "Say nothing.", "tune")] + + def test_select_split_keeps_one_side(self, tmp_path): + tune = te.select_split(self.corpus(tmp_path), "tune") + assert [u.id for u in tune.utterances] == ["u1"] + assert [q.id for q in tune.quiet] == ["q1"] + + def test_a_split_nobody_marked_is_an_error(self, tmp_path): + path = tmp_path / "c.toml" + path.write_text('[[utterance]]\nid = "u"\ntext = "hi"\n') + with pytest.raises(te.EvalError, match="no utterances"): + te.select_split(te.load_corpus(path), "holdout") + + def test_unknown_split_is_refused(self, tmp_path): + path = tmp_path / "c.toml" + path.write_text('[[utterance]]\nid = "u"\ntext = "hi"\nsplit = "dev"\n') + with pytest.raises(te.EvalError, match="dev"): + te.load_corpus(path) + + +class TestRecordings: + def setup_corpus(self, tmp_path): + path = tmp_path / "script.toml" + path.write_text(SCRIPT) + return te.load_corpus(path), tmp_path / "recordings" + + def record_all(self, root: Path, session: str): + write_take(root / session, "u1", [900, -900] * 800, {"script_text": "merge it"}) + write_take( + root / session, "u2", [900, -900] * 800, {"script_text": "Close TKT-1618."} + ) + write_take(root / session, "q1", [3, -3] * 800, {"script_text": "Say nothing."}) + + def test_every_take_becomes_a_clip_per_session(self, tmp_path): + corpus, root = self.setup_corpus(tmp_path) + self.record_all(root, "studio-mic") + clips = te.load_recordings(corpus, root, ["studio-mic"]) + assert [(c.id, c.voice, c.is_silence, c.kind) for c in clips] == [ + ("u1", "studio-mic", False, "short"), + ("u2", "studio-mic", False, "ids"), + ("q1", "studio-mic", True, None), + ] + + def test_missing_takes_stop_the_run(self, tmp_path): + corpus, root = self.setup_corpus(tmp_path) + write_take(root / "s", "u1", [900] * 1600) + with pytest.raises(te.EvalError, match="s/u2: not recorded"): + te.load_recordings(corpus, root, ["s"]) + + def test_a_take_recorded_for_other_text_is_stale(self, tmp_path): + corpus, root = self.setup_corpus(tmp_path) + self.record_all(root, "s") + write_take(root / "s", "u1", [900] * 1600, {"script_text": "merge that"}) + with pytest.raises(te.EvalError, match="different text"): + te.load_recordings(corpus, root, ["s"]) + + def test_sessions_are_discovered_or_checked(self, tmp_path): + _, root = self.setup_corpus(tmp_path) + self.record_all(root, "laptop") + self.record_all(root, "studio") + assert te.recorded_sessions(root, None) == ["laptop", "studio"] + with pytest.raises(te.EvalError, match="no takes for desk"): + te.recorded_sessions(root, ["desk"]) + + def test_report_scores_sessions_kinds_and_quiet_takes(self, tmp_path): + corpus, root = self.setup_corpus(tmp_path) + self.record_all(root, "s") + clips = te.load_recordings(corpus, root, ["s"]) + hyps = {"u1": "merge it", "u2": "Close TKT 1618.", "q1": "you"} + results = [te.ClipResult(c, hyps[c.id], 100.0, 160.0) for c in clips] + settings = te.Settings( + "ws", "", None, [], ["s"], "script.toml", "tune", str(root) + ) + report = te.build_report(Path("m.bin"), settings, results) + assert report["per_voice"]["s"]["errors"] == 0 + assert set(report["per_kind"]) == {"short", "ids"} + assert report["silence"] == { + "clips": 1, + "false_text": 1, + "false_text_past_gate": 1, + } + assert report["settings"]["split"] == "tune" diff --git a/tests/test_record.py b/tests/test_record.py new file mode 100644 index 0000000..83e9903 --- /dev/null +++ b/tests/test_record.py @@ -0,0 +1,59 @@ +"""eval/record.py's decisions: what to record next and what to warn about. + +The capture itself needs macOS and a microphone, so it is exercised by hand; +eval/recorder.swift --selftest checks its WAV writer without one. +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +_spec = importlib.util.spec_from_file_location("record", ROOT / "eval" / "record.py") +rec = importlib.util.module_from_spec(_spec) +sys.modules["record"] = rec +_spec.loader.exec_module(rec) +te = rec.te + +LINE = te.Utterance("r1", "merge it", "merge it", "tune", "short") +QUIET = te.Quiet("q1", "Say nothing.", "tune") + + +def test_session_name_is_host_and_input(): + name = rec.session_name("Dans-Mac-Studio.local", "MacBook Pro Microphone") + assert name == "dans-mac-studio-macbook-pro-microphone" + + +def test_pending_skips_recorded_takes_unless_redone(tmp_path): + (tmp_path / "r1.wav").write_bytes(b"") + assert rec.pending([LINE, QUIET], tmp_path, []) == [QUIET] + assert rec.pending([LINE, QUIET], tmp_path, ["r1"]) == [LINE, QUIET] + + +def test_redo_of_an_unknown_id_is_an_error(tmp_path): + with pytest.raises(te.EvalError, match="r9"): + rec.pending([LINE], tmp_path, ["r9"]) + + +@pytest.mark.parametrize( + "take, expected", + [ + (rec.Take(0, 0.0, 0), "no audio arrived"), + (rec.Take(16000, 0.0, 0), "every sample is zero"), + (rec.Take(16000, 80.0, 400), "quieter than tacet's gate"), + (rec.Take(4000, 900.0, 9000), "very short"), + ], +) +def test_bad_speech_takes_are_flagged(take, expected): + assert any(expected in w for w in rec.check_take(LINE, take)) + + +def test_a_good_take_and_a_quiet_take_pass(): + assert rec.check_take(LINE, rec.Take(40000, 1200.0, 14000)) == [] + assert rec.check_take(QUIET, rec.Take(32000, 40.0, 300)) == [] + + +def test_take_seconds(): + assert rec.Take(24000, 1.0, 1).seconds == 1.5