From 8bc61ae5a3153a90a9554d26ef68b6ef11001495 Mon Sep 17 00:00:00 2001 From: Gemini Cloud AI Date: Mon, 1 Jun 2026 15:14:48 +0200 Subject: [PATCH 01/10] fix(cpu): enable robust CPU support and address PR feedback --- .../fun_asr_nano/serve_realtime_ws.py | 560 ++++++++++++++++++ 1 file changed, 560 insertions(+) create mode 100644 examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py diff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py new file mode 100644 index 000000000..9f3451a5d --- /dev/null +++ b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py @@ -0,0 +1,560 @@ +#!/usr/bin/env python3 +"""Fun-ASR-Nano Streaming WebSocket Server. + +Features: +- Streaming VAD segmentation (fsmn-vad) +- Per-segment ASR decoding (Fun-ASR-Nano via vLLM or AutoModel) +- Speaker diarization (eres2netv2 + ClusterBackend) +- Hotword customization +- Hallucination detection & prevention +""" + +import asyncio +import json +import logging +import os +import time +import argparse +import numpy as np +import torch +import warnings +import regex +import websockets + +warnings.filterwarnings('ignore') +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s') +logger = logging.getLogger(__name__) + + +def detect_and_fix_hallucination(text, max_ngram_length=12, max_occurrences=3): + """Detect repeated patterns (hallucination) and truncate to keep one occurrence.""" + if not text or len(text) < max_ngram_length * 2: + return text, False + + cleaned = regex.sub(r'\p{P}+', '', text) + + word_pattern = rf'(?= 0: + end_pos = text.find(repeated, pos + len(repeated)) + if end_pos >= 0: + return text[:end_pos + len(repeated)], True + return text[:len(text)//2], True + + for length in range(1, max_ngram_length): + pattern = rf'(?= 0: + end_pos = text.find(repeated, pos + len(repeated)) + if end_pos >= 0: + return text[:end_pos + len(repeated)], True + return text[:len(text)//2], True + + return text, False + + +def _clean_asr_text(text): + """Remove timestamp tags and artifacts from vLLM output.""" + import re + text = re.sub(r'<[^>]*>', '', text) + text = re.sub(r'\[.*?\]', '', text) + text = re.sub(r'[Ｏ\[\]&＆|｜]', '', text) + text = re.sub(r'/sil|endofbreak|FFFF', '', text) + text = re.sub(r'\s+', ' ', text) + return text.strip() + + +from funasr.models.fsmn_vad_streaming.dynamic_vad import DynamicStreamingVAD + + +class HybridSpeakerTracker: + """Speaker diarization: streaming ClusterBackend + final re-clustering.""" + + def __init__(self, spk_model, device, threshold=0.6): + self.spk_model = spk_model + self.device = device + self.threshold = threshold + self.speaker_centers = [] + from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spk + from funasr.models.campplus.cluster_backend import ClusterBackend + self.sv_chunk = sv_chunk + self.postprocess = postprocess + self.distribute_spk = distribute_spk + self.cluster_backend = ClusterBackend(merge_thr=0.78).to(device) + self.all_chunks = [] + self.all_embeddings = [] + self.display_map = {} + self.next_display_id = 0 + + @torch.no_grad() + def assign_streaming(self, audio_samples, seg_start_s, seg_end_s, sentence): + """Assign speaker ID during streaming using ClusterBackend.""" + vad_seg = [[seg_start_s, seg_end_s, audio_samples]] + chunks = self.sv_chunk(vad_seg) + if not chunks: + sentence["spk"] = self.next_display_id + self.next_display_id += 1 + return + + self.all_chunks.extend(chunks) + speech_list = [ch[2] for ch in chunks] + spk_res = self.spk_model.generate(input=speech_list, cache={}, is_final=True) + embs = torch.cat([r["spk_embedding"] for r in spk_res], dim=0) + self.all_embeddings.append(embs) + + all_embs = torch.cat(self.all_embeddings, dim=0) + labels = self.cluster_backend(all_embs.cpu(), oracle_num=None) + if not isinstance(labels, np.ndarray): + labels = np.array(labels) + + all_sorted = sorted(self.all_chunks, key=lambda x: x[0]) + sv_output = self.postprocess(all_sorted, None, labels, all_embs.cpu()) + temp = [{"start": int(seg_start_s*1000), "end": int(seg_end_s*1000), "text": sentence["text"]}] + self.distribute_spk(temp, sv_output) + raw_spk = temp[0].get("spk", 0) + + if raw_spk not in self.display_map: + self.display_map[raw_spk] = self.next_display_id + self.next_display_id += 1 + sentence["spk"] = self.display_map[raw_spk] + + @torch.no_grad() + def finalize(self, sentences, min_split_s=3.0): + """Final re-clustering for accurate speaker assignment.""" + if not self.all_embeddings or not sentences: + return sentences + + all_embs = torch.cat(self.all_embeddings, dim=0) + labels = self.cluster_backend(all_embs.cpu(), oracle_num=None) + if not isinstance(labels, np.ndarray): + labels = np.array(labels) + + all_sorted = sorted(self.all_chunks, key=lambda x: x[0]) + sv_output = self.postprocess(all_sorted, None, labels, all_embs.cpu()) + + for s in sentences: + s.pop("spk", None) + self.distribute_spk(sentences, sv_output) + + id_map = {} + next_id = 0 + for s in sentences: + raw = s.get("spk", 0) + if raw not in id_map: + id_map[raw] = next_id + next_id += 1 + s["spk"] = id_map[raw] + + final_sentences = [] + for s in sentences: + sub = self._try_split(s, sv_output, id_map, min_split_s) + final_sentences.extend(sub) + + return final_sentences + + def _try_split(self, sentence, sv_output, id_map, min_split_s): + """Split a sentence if multiple speakers detected within its time range.""" + sent_start = sentence["start"] / 1000.0 + sent_end = sentence["end"] / 1000.0 + text = sentence["text"] + + overlapping = [] + for sv_start, sv_end, sv_spk in sv_output: + o_start = max(sent_start, sv_start) + o_end = min(sent_end, sv_end) + if o_end > o_start: + mapped_spk = id_map.get(int(sv_spk), int(sv_spk)) + overlapping.append([o_start, o_end, mapped_spk]) + + if len(overlapping) <= 1: + return [sentence] + + filtered = [overlapping[0]] + for i in range(1, len(overlapping)): + cur = overlapping[i] + prev = filtered[-1] + if cur[2] == prev[2]: + filtered[-1] = [prev[0], cur[1], prev[2]] + elif (cur[1] - cur[0]) < min_split_s: + filtered[-1] = [prev[0], cur[1], prev[2]] + else: + filtered.append(cur) + + merged = [filtered[0]] + for i in range(1, len(filtered)): + if (merged[-1][1] - merged[-1][0]) < min_split_s: + merged[-1] = [merged[-1][0], filtered[i][1], filtered[i][2]] + else: + merged.append(filtered[i]) + if len(merged) > 1 and (merged[-1][1] - merged[-1][0]) < min_split_s: + merged[-2] = [merged[-2][0], merged[-1][1], merged[-2][2]] + merged.pop() + + if len(merged) <= 1: + return [sentence] + + total_dur = sum(m[1] - m[0] for m in merged) + sub_sentences = [] + char_pos = 0 + for i, (m_start, m_end, m_spk) in enumerate(merged): + if i == len(merged) - 1: + sub_text = text[char_pos:] + else: + n_chars = max(1, int(len(text) * (m_end - m_start) / total_dur)) + sub_text = text[char_pos:char_pos + n_chars] + char_pos += n_chars + if sub_text.strip(): + sub_sentences.append({"text": sub_text.strip(), "start": int(m_start*1000), "end": int(m_end*1000), "spk": m_spk}) + + return sub_sentences if sub_sentences else [sentence] + + def reset(self): + self.speaker_centers = [] + self.all_chunks = [] + self.all_embeddings = [] + self.display_map = {} + self.next_display_id = 0 + + +class RealtimeASRSession: + """Manages a single streaming ASR session.""" + + def __init__(self, vllm_engine, asr_kwargs, vad, spk_tracker=None, sample_rate=16000, chunk_ms=960): + self.vllm_engine = vllm_engine + self.asr_kwargs = asr_kwargs + self.vad = vad + self.sample_rate = sample_rate + self.chunk_samples = int(sample_rate * chunk_ms / 1000) + self.first_chunk_samples = int(sample_rate * 480 / 1000) + self.first_decode_done = False + + self.audio_buffer = np.array([], dtype=np.float32) + self.vad_fed_samples = 0 + self.prev_text = "" + self.last_partial_text = "" + self.last_decode_samples = 0 + self.locked_sentences = [] + self.prev_seg_text = "" + self.spk_tracker = spk_tracker + self.use_context = True + self.is_active = False + + def add_audio(self, pcm_bytes): + if len(pcm_bytes) % 2 != 0: + pcm_bytes = pcm_bytes[:len(pcm_bytes) - (len(pcm_bytes) % 2)] + audio_int16 = np.frombuffer(pcm_bytes, dtype=np.int16) + audio_float = audio_int16.astype(np.float32) / 32768.0 + self.audio_buffer = np.concatenate([self.audio_buffer, audio_float]) + + new_audio = self.audio_buffer[self.vad_fed_samples:] + if len(new_audio) > 0: + new_confirmed = self.vad.feed(torch.from_numpy(new_audio).float(), is_final=False) + self.vad_fed_samples = len(self.audio_buffer) + + for seg in new_confirmed: + seg_text = self._decode_segment(seg) + self.prev_text = "" + if not seg_text.strip(): + continue + self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])}) + if self.spk_tracker: + s0 = int(seg[0] * self.sample_rate / 1000) + s1 = min(int(seg[1] * self.sample_rate / 1000), len(self.audio_buffer)) + self.spk_tracker.assign_streaming(self.audio_buffer[s0:s1], seg[0]/1000, seg[1]/1000, self.locked_sentences[-1]) + logger.info(f"Locked: [{seg[0]}-{seg[1]}ms] \"{seg_text[:40]}\"") + + def should_decode(self): + threshold = self.first_chunk_samples if not self.first_decode_done else self.chunk_samples + return (len(self.audio_buffer) - self.last_decode_samples) >= threshold + + @torch.no_grad() + def decode(self, is_final=False): + if len(self.audio_buffer) < self.chunk_samples: + return self._build_response(is_final) + + if is_final: + remaining = self.audio_buffer[self.vad_fed_samples:] + if len(remaining) > 0: + new_confirmed = self.vad.feed(torch.from_numpy(remaining).float(), is_final=True) + self.vad_fed_samples = len(self.audio_buffer) + for seg in new_confirmed: + seg_text = self._decode_segment(seg) + if not seg_text.strip(): + continue + self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])}) + + if self.vad.current_speech_start is not None: + end_ms = int(len(self.audio_buffer) * 1000 / self.sample_rate) + seg = [self.vad.current_speech_start, end_ms] + seg_text = self._decode_segment(seg) + if seg_text.strip(): + self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])}) + self.vad.current_speech_start = None + + if self.spk_tracker and self.locked_sentences: + self.locked_sentences = self.spk_tracker.finalize(self.locked_sentences) + return self._build_response(is_final) + + if self.vad.current_speech_start is not None: + seg_start_sample = int(self.vad.current_speech_start * self.sample_rate / 1000) + seg_audio = self.audio_buffer[seg_start_sample:] + else: + self.last_decode_samples = len(self.audio_buffer) + self.last_partial_text = "" + return self._build_response(is_final) + + if len(seg_audio) < self.chunk_samples // 2: + return self._build_response(is_final) + + audio_tensor = torch.from_numpy(seg_audio).float() + try: + if hasattr(self.vllm_engine, 'generate') and not hasattr(self.vllm_engine, '_engine'): + # Standard AutoModel: returns list of dicts with 'text' + results = self.vllm_engine.generate( + input=audio_tensor, + hotwords=self.asr_kwargs.get("hotwords"), + language=self.asr_kwargs.get("language"), + ) + # AutoModel results format is typically: [{'text': '...'}] + if isinstance(results, list) and len(results) > 0: + text = results[0].get('text', '') + else: + text = str(results) if results is not None else "" + else: + # AutoModelVLLM + results = self.vllm_engine.generate( + inputs=[audio_tensor], + hotwords=self.asr_kwargs.get("hotwords"), + language=self.asr_kwargs.get("language"), + max_new_tokens=200, + ) + text = results[0]["text"] if (results and len(results) > 0) else "" + + text = _clean_asr_text(text) + except Exception as e: + logger.error(f"ASR error: {e}", exc_info=True) + return self._build_response(is_final) + + text, hallucinated = detect_and_fix_hallucination(text) + if hallucinated: + self.prev_text = "" + + self.last_decode_samples = len(self.audio_buffer) + self.last_partial_text = text + if text.strip() and not self.first_decode_done: + self.first_decode_done = True + + if hasattr(self.vllm_engine, '_engine'): + tokenizer = self.vllm_engine._engine.tokenizer + else: + tokenizer = self.vllm_engine.kwargs.get("tokenizer") + + if tokenizer is not None: + encoded = tokenizer.encode(text) + if len(encoded) > 5: + try: + self.prev_text = tokenizer.decode(encoded[:-5], skip_special_tokens=True) + except TypeError: + self.prev_text = tokenizer.decode(encoded[:-5]) + else: + self.prev_text = "" + else: + self.prev_text = "" + + return self._build_response(is_final) + + @torch.no_grad() + def _decode_segment(self, seg): + """Decode a completed VAD segment via vLLM or AutoModel.""" + start_sample = int(seg[0] * self.sample_rate / 1000) + end_sample = min(int(seg[1] * self.sample_rate / 1000), len(self.audio_buffer)) + seg_audio = self.audio_buffer[start_sample:end_sample] + if len(seg_audio) < 1600: + return "" + audio_tensor = torch.from_numpy(seg_audio).float() + try: + if hasattr(self.vllm_engine, 'generate') and not hasattr(self.vllm_engine, '_engine'): + # Standard AutoModel + results = self.vllm_engine.generate( + input=audio_tensor, + hotwords=self.asr_kwargs.get("hotwords"), + language=self.asr_kwargs.get("language"), + ) + text = results[0].get('text', '') if (isinstance(results, list) and len(results) > 0) else str(results) + else: + # AutoModelVLLM + results = self.vllm_engine.generate( + inputs=[audio_tensor], + hotwords=self.asr_kwargs.get("hotwords"), + language=self.asr_kwargs.get("language"), + max_new_tokens=512, + ) + text = results[0]["text"] if (results and len(results) > 0) else "" + + text = _clean_asr_text(text) + self.prev_seg_text = text + return text + except Exception as e: + logger.error(f"Segment decode error: {e}") + return "" + + def _build_response(self, is_final): + duration_ms = int(len(self.audio_buffer) * 1000 / self.sample_rate) + sentences = list(self.locked_sentences) + partial = self.last_partial_text + partial_start = self.vad.current_speech_start or duration_ms + + if is_final: + return {"sentences": sentences, "partial": "", "partial_start_ms": 0, + "duration_ms": duration_ms, "is_final": True} + return {"sentences": sentences, "partial": partial, + "partial_start_ms": partial_start, + "duration_ms": duration_ms, "is_final": False} + + def reset(self): + self.audio_buffer = np.array([], dtype=np.float32) + self.vad_fed_samples = 0 + self.first_decode_done = False + self.vad.reset() + self.prev_text = "" + self.last_partial_text = "" + self.last_decode_samples = 0 + self.locked_sentences = [] + if self.spk_tracker: + self.spk_tracker.reset() + + +_vllm_engine = None +_asr_kwargs = None +_vad_model = None +_spk_model = None + + +def load_models(args): + global _vllm_engine, _asr_kwargs, _vad_model, _spk_model + if _vllm_engine is None: + from funasr import AutoModel + from funasr.auto.auto_model_vllm import AutoModelVLLM + + if "cuda" not in args.device: + logger.info("Using standard AutoModel for non-CUDA device (vLLM bypassed)") + _vllm_engine = AutoModel(model=args.model, hub=args.hub, device=args.device, disable_update=True) + else: + logger.info(f"Loading ASR (vLLM): {args.model}") + _vllm_engine = AutoModelVLLM( + model=args.model, hub=args.hub, device=args.device, + dtype=getattr(args, 'dtype', 'bf16'), + tensor_parallel_size=getattr(args, 'tensor_parallel_size', 1), + gpu_memory_utilization=getattr(args, 'gpu_memory_utilization', 0.8), + max_model_len=getattr(args, 'max_model_len', 2048), + ) + + _asr_kwargs = {} + hw_file = getattr(args, 'hotword_file', '热词列表') + if hw_file and os.path.isfile(hw_file): + with open(hw_file, "r", encoding="utf-8") as hf: + hotwords = [line.strip() for line in hf if line.strip()] + _asr_kwargs["hotwords"] = hotwords + logger.info(f"Loaded {len(hotwords)} hotwords from '{hw_file}'") + + if getattr(args, 'language', None): + _asr_kwargs["language"] = args.language + logger.info(f"Language: {args.language}") + + logger.info("Loading VAD: fsmn-vad (streaming)") + _vad_model = AutoModel(model="fsmn-vad", device=args.device, disable_update=True) + + logger.info("Loading SPK: eres2netv2") + _spk_model = AutoModel(model="iic/speech_eres2netv2_sv_zh-cn_16k-common", device=args.device, disable_update=True) + + logger.info("All models ready!") + return _vllm_engine, _asr_kwargs, _vad_model, _spk_model + + +async def handle_client(websocket, args): + vllm_engine, asr_kwargs, vad_model, spk_model = load_models(args) + vad = DynamicStreamingVAD(vad_model) + spk_tracker = HybridSpeakerTracker(spk_model, args.device) + session = RealtimeASRSession(vllm_engine, asr_kwargs, vad, spk_tracker=spk_tracker) + logger.info(f"Client connected: {websocket.remote_address}") + + decode_interval = args.decode_interval + last_decode_time = 0 + + try: + async for message in websocket: + if isinstance(message, str): + cmd = message.strip() + if cmd.upper() == "START": + session.reset() + session.is_active = True + await websocket.send(json.dumps({"event": "started"})) + logger.info("Session started") + elif cmd.upper().startswith("HOTWORDS:"): + hw_str = cmd[9:] + hotwords = [w.strip() for w in hw_str.split(",") if w.strip()] + session.asr_kwargs = dict(session.asr_kwargs) + session.asr_kwargs["hotwords"] = hotwords + await websocket.send(json.dumps({"event": "hotwords_set", "hotwords": hotwords})) + logger.info(f"Hotwords set: {len(hotwords)} words") + elif cmd.upper().startswith("LANGUAGE:"): + lang = cmd[9:].strip() + session.asr_kwargs = dict(session.asr_kwargs) + session.asr_kwargs["language"] = lang if lang else None + await websocket.send(json.dumps({"event": "language_set", "language": lang})) + logger.info(f"Language set: {lang}") + elif cmd.upper() == "STOP": + if session.is_active and len(session.audio_buffer) > 0: + result = session.decode(is_final=True) + await websocket.send(json.dumps(result)) + logger.info(f"Final: {len(result['sentences'])} sentences") + session.is_active = False + await websocket.send(json.dumps({"event": "stopped"})) + elif isinstance(message, bytes) and session.is_active: + session.add_audio(message) + now = time.time() + if now - last_decode_time >= decode_interval and session.should_decode(): + result = session.decode(is_final=False) + await websocket.send(json.dumps(result)) + last_decode_time = now + + except websockets.exceptions.ConnectionClosed: + logger.info("Client disconnected") + except Exception as e: + logger.error(f"Error: {e}", exc_info=True) + + +async def main(args): + load_models(args) + logger.info(f"Server on ws://0.0.0.0:{args.port}") + async with websockets.serve( + lambda ws: handle_client(ws, args), "0.0.0.0", args.port, + max_size=10*1024*1024, + ): + await asyncio.Future() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Fun-ASR-Nano Streaming WebSocket Server") + parser.add_argument("--port", type=int, default=10095) + parser.add_argument("--model", type=str, default="FunAudioLLM/Fun-ASR-Nano-2512") + parser.add_argument("--hub", type=str, default="ms", choices=["ms", "hf"]) + parser.add_argument("--device", type=str, default="cuda:0") + parser.add_argument("--use-context", action="store_true", default=True) + parser.add_argument("--no-context", dest="use_context", action="store_false") + parser.add_argument("--decode-interval", type=float, default=0.48) + parser.add_argument("--hotword-file", type=str, default="热词列表") + parser.add_argument("--language", type=str, default=None, help="Language hint (e.g. 中文, English, 日本語)") + parser.add_argument("--dtype", type=str, default="bf16", choices=["bf16", "fp16", "fp32"]) + parser.add_argument("--tensor-parallel-size", type=int, default=1) + parser.add_argument("--gpu-memory-utilization", type=float, default=0.8) + parser.add_argument("--max-model-len", type=int, default=2048) + args = parser.parse_args() + asyncio.run(main(args)) From 23c50bb0c45c938ad4276f33724a8992806cc03b Mon Sep 17 00:00:00 2001 From: Gemini Cloud AI Date: Mon, 1 Jun 2026 15:18:52 +0200 Subject: [PATCH 02/10] fix(cpu): sync with main and ensure CPU support bypasses vLLM correctly From c225f6f6c20b13d8800da2a37d17f6b657e36220 Mon Sep 17 00:00:00 2001 From: Gemini Cloud AI Date: Mon, 1 Jun 2026 15:22:31 +0200 Subject: [PATCH 03/10] fix(cpu): enable robust CPU support and address PR feedback --- .../fun_asr_nano/serve_realtime_ws.py | 81 ++++++++++++------- 1 file changed, 52 insertions(+), 29 deletions(-) diff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py index 9f3451a5d..b6a2341f3 100644 --- a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py +++ b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py @@ -41,7 +41,7 @@ def detect_and_fix_hallucination(text, max_ngram_length=12, max_occurrences=3): if pos >= 0: end_pos = text.find(repeated, pos + len(repeated)) if end_pos >= 0: - return text[:end_pos + len(repeated)], True + return text[:end_pos], True return text[:len(text)//2], True for length in range(1, max_ngram_length): @@ -54,7 +54,7 @@ def detect_and_fix_hallucination(text, max_ngram_length=12, max_occurrences=3): if pos >= 0: end_pos = text.find(repeated, pos + len(repeated)) if end_pos >= 0: - return text[:end_pos + len(repeated)], True + return text[:end_pos], True return text[:len(text)//2], True return text, False @@ -235,7 +235,9 @@ def __init__(self, vllm_engine, asr_kwargs, vad, spk_tracker=None, sample_rate=1 self.first_chunk_samples = int(sample_rate * 480 / 1000) self.first_decode_done = False - self.audio_buffer = np.array([], dtype=np.float32) + self._audio_buffer = np.array([], dtype=np.float32) + self.audio_chunks = [] + self._audio_buffer_dirty = False self.vad_fed_samples = 0 self.prev_text = "" self.last_partial_text = "" @@ -246,29 +248,46 @@ def __init__(self, vllm_engine, asr_kwargs, vad, spk_tracker=None, sample_rate=1 self.use_context = True self.is_active = False + @property + def audio_buffer(self): + if getattr(self, '_audio_buffer_dirty', False): + if self.audio_chunks: + self._audio_buffer = np.concatenate([self._audio_buffer, *self.audio_chunks]) + self.audio_chunks = [] + self._audio_buffer_dirty = False + return self._audio_buffer + + @audio_buffer.setter + def audio_buffer(self, value): + self._audio_buffer = value + self.audio_chunks = [] + self._audio_buffer_dirty = False + def add_audio(self, pcm_bytes): if len(pcm_bytes) % 2 != 0: pcm_bytes = pcm_bytes[:len(pcm_bytes) - (len(pcm_bytes) % 2)] + if not pcm_bytes: + return audio_int16 = np.frombuffer(pcm_bytes, dtype=np.int16) audio_float = audio_int16.astype(np.float32) / 32768.0 - self.audio_buffer = np.concatenate([self.audio_buffer, audio_float]) + + self.audio_chunks.append(audio_float) + self._audio_buffer_dirty = True - new_audio = self.audio_buffer[self.vad_fed_samples:] - if len(new_audio) > 0: - new_confirmed = self.vad.feed(torch.from_numpy(new_audio).float(), is_final=False) - self.vad_fed_samples = len(self.audio_buffer) + new_confirmed = self.vad.feed(torch.from_numpy(audio_float).float(), is_final=False) + self.vad_fed_samples += len(audio_float) - for seg in new_confirmed: - seg_text = self._decode_segment(seg) - self.prev_text = "" - if not seg_text.strip(): - continue - self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])}) - if self.spk_tracker: - s0 = int(seg[0] * self.sample_rate / 1000) - s1 = min(int(seg[1] * self.sample_rate / 1000), len(self.audio_buffer)) - self.spk_tracker.assign_streaming(self.audio_buffer[s0:s1], seg[0]/1000, seg[1]/1000, self.locked_sentences[-1]) - logger.info(f"Locked: [{seg[0]}-{seg[1]}ms] \"{seg_text[:40]}\"") + for seg in new_confirmed: + seg_text = self._decode_segment(seg) + self.prev_text = "" + if not seg_text.strip(): + continue + self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])}) + if self.spk_tracker: + s0 = int(seg[0] * self.sample_rate / 1000) + s1 = min(int(seg[1] * self.sample_rate / 1000), len(self.audio_buffer)) + self.spk_tracker.assign_streaming(self.audio_buffer[s0:s1], seg[0]/1000, seg[1]/1000, self.locked_sentences[-1]) + logger.info(f"Locked: [{seg[0]}-{seg[1]}ms] \"{seg_text[:40]}\"") def should_decode(self): threshold = self.first_chunk_samples if not self.first_decode_done else self.chunk_samples @@ -319,7 +338,7 @@ def decode(self, is_final=False): # Standard AutoModel: returns list of dicts with 'text' results = self.vllm_engine.generate( input=audio_tensor, - hotwords=self.asr_kwargs.get("hotwords"), + hotword=self.asr_kwargs.get("hotwords"), language=self.asr_kwargs.get("language"), ) # AutoModel results format is typically: [{'text': '...'}] @@ -354,16 +373,20 @@ def decode(self, is_final=False): if hasattr(self.vllm_engine, '_engine'): tokenizer = self.vllm_engine._engine.tokenizer else: - tokenizer = self.vllm_engine.kwargs.get("tokenizer") + tokenizer = getattr(self.vllm_engine, 'kwargs', {}).get("tokenizer") if tokenizer is not None: - encoded = tokenizer.encode(text) - if len(encoded) > 5: - try: - self.prev_text = tokenizer.decode(encoded[:-5], skip_special_tokens=True) - except TypeError: - self.prev_text = tokenizer.decode(encoded[:-5]) - else: + try: + encoded = tokenizer.encode(text) + if len(encoded) > 5: + try: + self.prev_text = tokenizer.decode(encoded[:-5], skip_special_tokens=True) + except TypeError: + self.prev_text = tokenizer.decode(encoded[:-5]) + else: + self.prev_text = "" + except Exception as e: + logger.warning(f"Failed to encode/decode text with tokenizer: {e}") self.prev_text = "" else: self.prev_text = "" @@ -384,7 +407,7 @@ def _decode_segment(self, seg): # Standard AutoModel results = self.vllm_engine.generate( input=audio_tensor, - hotwords=self.asr_kwargs.get("hotwords"), + hotword=self.asr_kwargs.get("hotwords"), language=self.asr_kwargs.get("language"), ) text = results[0].get('text', '') if (isinstance(results, list) and len(results) > 0) else str(results) From 924cb29028aa1888a5fb82d14e8ba4684c196a2e Mon Sep 17 00:00:00 2001 From: Gemini Cloud AI Date: Mon, 1 Jun 2026 15:31:44 +0200 Subject: [PATCH 04/10] docs(GEMINI): add project documentation and current status quo --- GEMINI.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 GEMINI.md diff --git a/GEMINI.md b/GEMINI.md new file mode 100644 index 000000000..f1a985d89 --- /dev/null +++ b/GEMINI.md @@ -0,0 +1,49 @@ +# FunASR Project Updates & Strategic Assessment (June 2026) + +This document tracks the evolution of FunASR and provides a "warts and all" assessment of its current stability for integration into other projects. + +## 🚨 Repository Alert Status (Updated June 1, 2026) +**Current Status:** **IMPROVED STABILITY** on `main` branch (Version 1.3.9). + +### 1. Recent Major Updates (June 2026) +- **CPU Support (Production Path):** Realtime WebSocket server now includes a robust bypass for vLLM when running on non-CUDA devices (CPU), utilizing standard `AutoModel` (PyTorch) with improved argument handling. +- **Production Infrastructure:** Massive expansion of `openai_api` examples, including Gradio, Kubernetes manifests, and full API specification for easier enterprise integration. +- **Improved Performance:** Refactored streaming buffer management to resolve O(N^2) complexity issues. +- **Model Support:** Enhanced native support for Whisper and GLM models. + +### 2. The Transformers v5 "Deadlock" (Still Present) +- **The Problem:** Modern stacks (Transformers 5.x) are incompatible with `qwen-asr` (v0.0.6) and other sub-models. It triggers an `AttributeError: 'Qwen3ASRConfig' object has no attribute 'thinker_config'`. +- **The Workaround:** Users **must** pin their environment: `pip install "transformers==4.57.6" "huggingface-hub<1.0"`. + +--- + +## 🧪 Experimental Results (Polish ASR) + +### 1. Model Quality Tier List (Polish) + +| Tier | Model | Parameters | Verdict | +| :--- | :--- | :--- | :--- | +| **Godzilla** | **`Qwen/Qwen2-Audio-7B-Instruct`** | **7 Billion** | **Peak Intelligence.** Best accuracy for Polish. Extremely heavy on CPU. | +| **Industrial** | **`FunAudioLLM/Fun-ASR-MLT-Nano-2512`** | **800 Million** | **The Workhorse.** Optimized for 31 languages. Balanced quality/speed. | +| **Mobile** | `iic/SenseVoiceSmall` | 234 Million | **Fast Default.** Great for low-latency tasks. CPU viable. | + +--- + +## ⚖️ Strategic Comparison: FunASR vs. WhisperX + +| Metric | WhisperX (The Scribe) | FunASR (The Engine) | +| :--- | :--- | :--- | +| **CPU Performance** | 🥇 **Champion.** Optimized C++. | 🥉 **Laggard.** Unoptimized PyTorch. | +| **Polish Accuracy** | 🥇 **Stable.** | 🥈 **Variable.** Needs "Godzilla" models to compete. | +| **Orchestration** | 🥈 **Rigid.** | 🥇 **Powerful.** VAD + SPK + Emotion in one pipeline. | + +**Verdict on Migration:** +Use FunASR for **complex pipelines** (Diarization, Emotion, Hotwords) or **high-end GPUs**. Stick with WhisperX for simple, fast, CPU-bound Polish transcription. + +--- + +## 🛠️ Recommended Components & Best Practices +1. **`funasr/fsmn-vad`**: Efficient segmentation. +2. **`funasr/campplus`**: Speaker identity. +3. **Configuration**: Use `hub="hf"`, `trust_remote_code=True`. +4. **vLLM Integration**: For production NVIDIA GPUs, use `AutoModelVLLM`. For CPU/General use, `AutoModel` is now supported via device detection bypass. From 63fb27d3bba3f87d6fc1a626ee837d90c4d96806 Mon Sep 17 00:00:00 2001 From: Gemini Cloud AI Date: Mon, 1 Jun 2026 15:34:29 +0200 Subject: [PATCH 05/10] docs: remove GEMINI.md from PR branch --- GEMINI.md | 49 ------------------------------------------------- 1 file changed, 49 deletions(-) delete mode 100644 GEMINI.md diff --git a/GEMINI.md b/GEMINI.md deleted file mode 100644 index f1a985d89..000000000 --- a/GEMINI.md +++ /dev/null @@ -1,49 +0,0 @@ -# FunASR Project Updates & Strategic Assessment (June 2026) - -This document tracks the evolution of FunASR and provides a "warts and all" assessment of its current stability for integration into other projects. - -## 🚨 Repository Alert Status (Updated June 1, 2026) -**Current Status:** **IMPROVED STABILITY** on `main` branch (Version 1.3.9). - -### 1. Recent Major Updates (June 2026) -- **CPU Support (Production Path):** Realtime WebSocket server now includes a robust bypass for vLLM when running on non-CUDA devices (CPU), utilizing standard `AutoModel` (PyTorch) with improved argument handling. -- **Production Infrastructure:** Massive expansion of `openai_api` examples, including Gradio, Kubernetes manifests, and full API specification for easier enterprise integration. -- **Improved Performance:** Refactored streaming buffer management to resolve O(N^2) complexity issues. -- **Model Support:** Enhanced native support for Whisper and GLM models. - -### 2. The Transformers v5 "Deadlock" (Still Present) -- **The Problem:** Modern stacks (Transformers 5.x) are incompatible with `qwen-asr` (v0.0.6) and other sub-models. It triggers an `AttributeError: 'Qwen3ASRConfig' object has no attribute 'thinker_config'`. -- **The Workaround:** Users **must** pin their environment: `pip install "transformers==4.57.6" "huggingface-hub<1.0"`. - ---- - -## 🧪 Experimental Results (Polish ASR) - -### 1. Model Quality Tier List (Polish) - -| Tier | Model | Parameters | Verdict | -| :--- | :--- | :--- | :--- | -| **Godzilla** | **`Qwen/Qwen2-Audio-7B-Instruct`** | **7 Billion** | **Peak Intelligence.** Best accuracy for Polish. Extremely heavy on CPU. | -| **Industrial** | **`FunAudioLLM/Fun-ASR-MLT-Nano-2512`** | **800 Million** | **The Workhorse.** Optimized for 31 languages. Balanced quality/speed. | -| **Mobile** | `iic/SenseVoiceSmall` | 234 Million | **Fast Default.** Great for low-latency tasks. CPU viable. | - ---- - -## ⚖️ Strategic Comparison: FunASR vs. WhisperX - -| Metric | WhisperX (The Scribe) | FunASR (The Engine) | -| :--- | :--- | :--- | -| **CPU Performance** | 🥇 **Champion.** Optimized C++. | 🥉 **Laggard.** Unoptimized PyTorch. | -| **Polish Accuracy** | 🥇 **Stable.** | 🥈 **Variable.** Needs "Godzilla" models to compete. | -| **Orchestration** | 🥈 **Rigid.** | 🥇 **Powerful.** VAD + SPK + Emotion in one pipeline. | - -**Verdict on Migration:** -Use FunASR for **complex pipelines** (Diarization, Emotion, Hotwords) or **high-end GPUs**. Stick with WhisperX for simple, fast, CPU-bound Polish transcription. - ---- - -## 🛠️ Recommended Components & Best Practices -1. **`funasr/fsmn-vad`**: Efficient segmentation. -2. **`funasr/campplus`**: Speaker identity. -3. **Configuration**: Use `hub="hf"`, `trust_remote_code=True`. -4. **vLLM Integration**: For production NVIDIA GPUs, use `AutoModelVLLM`. For CPU/General use, `AutoModel` is now supported via device detection bypass. From 7b186407e0d706a78f3737bf3b584775e6fd4154 Mon Sep 17 00:00:00 2001 From: Gemini AI Agent Date: Mon, 1 Jun 2026 17:53:56 +0200 Subject: [PATCH 06/10] fix(cpu): add API signature fix for AutoModel CPU fallback in demo_vllm.py --- .../fun_asr_nano/demo_vllm.py | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.py diff --git a/examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.py b/examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.py new file mode 100644 index 000000000..dc4f8c902 --- /dev/null +++ b/examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Demo: Fun-ASR-Nano with vLLM inference backend. + +Usage: + # Single GPU (greedy decoding) + python demo_vllm.py + + # Multi-GPU tensor parallel + python demo_vllm.py --tensor-parallel-size 2 + + # Batch inference from wav.scp + python demo_vllm.py --input wav.scp --tensor-parallel-size 4 --batch-size 32 + + # With hotwords and language + python demo_vllm.py --input audio.wav --language 中文 --hotwords 开放时间 周一 +""" + +import argparse +import os +import time + +import torch + + +def main(): + parser = argparse.ArgumentParser(description="Fun-ASR-Nano vLLM Inference Demo") + parser.add_argument( + "--model-dir", + type=str, + default="FunAudioLLM/Fun-ASR-Nano-2512", + help="Model name (from hub) or local directory path", + ) + parser.add_argument("--input", type=str, default=None, help="Audio file, wav.scp, or jsonl") + parser.add_argument("--hub", type=str, default="ms", choices=["ms", "hf"]) + parser.add_argument("--device", type=str, default="cuda:0", help="Device for audio encoder") + parser.add_argument("--dtype", type=str, default="bf16", choices=["bf16", "fp16", "fp32"]) + parser.add_argument( + "--tensor-parallel-size", type=int, default=1, help="Number of GPUs for vLLM" + ) + parser.add_argument("--gpu-memory-utilization", type=float, default=0.8) + parser.add_argument("--max-model-len", type=int, default=2048) + parser.add_argument("--max-new-tokens", type=int, default=512) + parser.add_argument("--language", type=str, default="中文", help="Language hint") + parser.add_argument("--hotwords", type=str, nargs="*", default=[], help="Hotwords list") + parser.add_argument("--no-itn", action="store_true", help="Disable inverse text normalization") + parser.add_argument("--batch-size", type=int, default=16, help="Batch size for inference") + parser.add_argument("--output", type=str, default=None, help="Output file for results") + args = parser.parse_args() + + from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM + + print(f"=" * 60) + print(f"Fun-ASR-Nano vLLM Inference") + print(f"=" * 60) + print(f" Model: {args.model_dir}") + print(f" Tensor Parallel: {args.tensor_parallel_size} GPU(s)") + print(f" Dtype: {args.dtype}") + print(f" Language: {args.language}") + print(f" Hotwords: {args.hotwords or '(none)'}") + print() + + t_load = time.perf_counter() + # Godzilla/CPU Patch: Bypass vLLM on CPU + if args.device == "cpu": + from funasr import AutoModel + print("CPU detected: Falling back to standard AutoModel (bypassing vLLM)") + engine = AutoModel( + model=args.model_dir, + hub=args.hub, + device="cpu", + trust_remote_code=True, + disable_update=True + ) + else: + from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM + engine = FunASRNanoVLLM.from_pretrained( + model=args.model_dir, + hub=args.hub, + device=args.device, + dtype=args.dtype, + tensor_parallel_size=args.tensor_parallel_size, + gpu_memory_utilization=args.gpu_memory_utilization, + max_model_len=args.max_model_len, + ) + print(f"Model loaded in {time.perf_counter() - t_load:.1f}s\n") + + # Determine input files + if args.input is None: + # Use default example audio + example_dir = os.path.join(engine.model_dir, "example") + if os.path.isdir(example_dir): + wav_files = [ + os.path.join(example_dir, f) + for f in sorted(os.listdir(example_dir)) + if f.endswith((".wav", ".mp3", ".flac")) + ] + else: + print("No --input specified and no example/ directory found.") + print("Usage: python demo_vllm.py --input ") + return + if not wav_files: + print("No audio files found in example/ directory.") + return + audio_files = wav_files + print(f"Using example audio: {audio_files}") + elif args.input.endswith(".scp"): + audio_files = [] + with open(args.input, "r") as f: + for line in f: + parts = line.strip().split(maxsplit=1) + if len(parts) == 2: + audio_files.append(parts[1]) + elif len(parts) == 1: + audio_files.append(parts[0]) + print(f"Loaded {len(audio_files)} files from {args.input}") + elif args.input.endswith(".jsonl"): + import json + + audio_files = [] + with open(args.input, "r") as f: + for line in f: + item = json.loads(line.strip()) + audio_files.append(item["source"]) + print(f"Loaded {len(audio_files)} files from {args.input}") + else: + audio_files = [args.input] + + # Run inference in batches + all_results = [] + total_audio_time = 0 + total_infer_time = 0 + + print(f"\nProcessing {len(audio_files)} audio file(s)...") + for i in range(0, len(audio_files), args.batch_size): + batch = audio_files[i : i + args.batch_size] + t0 = time.perf_counter() + + # CPU Fallback: Handle API signature mismatch + if args.device == "cpu": + results = engine.generate( + input=batch, + language=args.language, + ) + else: + results = engine.generate( + inputs=batch, + hotwords=args.hotwords if args.hotwords else None, + language=args.language, + itn=not args.no_itn, + max_new_tokens=args.max_new_tokens, + ) + + t1 = time.perf_counter() + batch_time = t1 - t0 + total_infer_time += batch_time + all_results.extend(results) + + batch_num = i // args.batch_size + 1 + total_batches = (len(audio_files) + args.batch_size - 1) // args.batch_size + print(f" Batch {batch_num}/{total_batches}: {len(batch)} files in {batch_time:.2f}s") + + # Print results + print(f"\n{'=' * 60}") + print(f"Results: {len(all_results)} samples, total inference time: {total_infer_time:.2f}s") + print(f"{'=' * 60}") + for r in all_results: + print(f"\n[{r['key']}]") + print(f" Text: {r['text']}") + if "timestamps" in r and r["timestamps"]: + ts_preview = r["timestamps"][:5] + ts_str = " | ".join( + [f"{t['token']}({t['start_time']:.2f}-{t['end_time']:.2f}s)" for t in ts_preview] + ) + if len(r["timestamps"]) > 5: + ts_str += f" ... ({len(r['timestamps'])} total)" + print(f" Timestamps: {ts_str}") + + # Save results to file + if args.output: + import json + + with open(args.output, "w", encoding="utf-8") as f: + for r in all_results: + # Remove non-serializable fields + out = {k: v for k, v in r.items() if k != "timestamps"} + if "timestamps" in r: + out["timestamps"] = r["timestamps"] + f.write(json.dumps(out, ensure_ascii=False) + "\n") + print(f"\nResults saved to {args.output}") + + +if __name__ == "__main__": + main() From e90f41fc4ef4f2eb17ec89664e21b0a8bdc60be0 Mon Sep 17 00:00:00 2001 From: Gemini AI Agent Date: Mon, 1 Jun 2026 17:56:00 +0200 Subject: [PATCH 07/10] fix(vad): update VAD import for FunASR 1.3.9 refactor --- .../fun_asr_nano/serve_realtime_ws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py index b6a2341f3..f0bd25aaa 100644 --- a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py +++ b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py @@ -71,7 +71,7 @@ def _clean_asr_text(text): return text.strip() -from funasr.models.fsmn_vad_streaming.dynamic_vad import DynamicStreamingVAD +from funasr.models.fsmn_vad_streaming.model import FsmnVADStreaming as DynamicStreamingVAD class HybridSpeakerTracker: From 5ff8f7931edfa13005455c0cf2536257211f7bd9 Mon Sep 17 00:00:00 2001 From: Gemini AI Agent Date: Mon, 1 Jun 2026 17:56:55 +0200 Subject: [PATCH 08/10] fix(cpu): make vLLM import conditional to prevent CPU crashes --- .../fun_asr_nano/serve_realtime_ws.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py index f0bd25aaa..16fd80bd9 100644 --- a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py +++ b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py @@ -464,12 +464,15 @@ def load_models(args): global _vllm_engine, _asr_kwargs, _vad_model, _spk_model if _vllm_engine is None: from funasr import AutoModel - from funasr.auto.auto_model_vllm import AutoModelVLLM - - if "cuda" not in args.device: - logger.info("Using standard AutoModel for non-CUDA device (vLLM bypassed)") - _vllm_engine = AutoModel(model=args.model, hub=args.hub, device=args.device, disable_update=True) + + if args.device == "cpu": + logger.info(f"CPU detected: Loading ASR (AutoModel) instead of vLLM") + _vllm_engine = AutoModel( + model=args.model, hub=args.hub, device=args.device, + trust_remote_code=True, disable_update=True + ) else: + from funasr.auto.auto_model_vllm import AutoModelVLLM logger.info(f"Loading ASR (vLLM): {args.model}") _vllm_engine = AutoModelVLLM( model=args.model, hub=args.hub, device=args.device, From 69df5d72682a2512c68da9dd173dc1955d4acf5b Mon Sep 17 00:00:00 2001 From: Gemini AI Agent Date: Mon, 1 Jun 2026 18:06:13 +0200 Subject: [PATCH 09/10] chore: sync missing client_python.py from main --- .../fun_asr_nano/client_python.py | 233 ++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 examples/industrial_data_pretraining/fun_asr_nano/client_python.py diff --git a/examples/industrial_data_pretraining/fun_asr_nano/client_python.py b/examples/industrial_data_pretraining/fun_asr_nano/client_python.py new file mode 100644 index 000000000..570fd86cb --- /dev/null +++ b/examples/industrial_data_pretraining/fun_asr_nano/client_python.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Fun-ASR-Nano Python WebSocket Client. + +Supports real-time microphone recording and audio file streaming. + +Usage: + # Microphone mode + python client_python.py --server ws://localhost:10095 --mic + + # File mode + python client_python.py --server ws://localhost:10095 --file audio.wav + + # With hotwords + python client_python.py --server ws://localhost:10095 --file audio.wav --hotwords "张三,李四,北京" + + # Disable speaker diarization display + python client_python.py --server ws://localhost:10095 --mic --no-spk +""" + +import asyncio +import argparse +import json +import sys +import numpy as np + +try: + import websockets +except ImportError: + print("Please install websockets: pip install websockets") + sys.exit(1) + + +SAMPLE_RATE = 16000 +CHUNK_DURATION_MS = 100 +CHUNK_SAMPLES = int(SAMPLE_RATE * CHUNK_DURATION_MS / 1000) + +SPK_COLORS = [ + "\033[36m", "\033[35m", "\033[33m", "\033[32m", + "\033[34m", "\033[91m", "\033[96m", "\033[95m", +] +RESET = "\033[0m" +GRAY = "\033[90m" +GREEN = "\033[92m" + + +def format_time(ms): + s = ms / 1000 + return f"{s:.1f}s" + + +def print_result(data, show_spk=True): + """Print ASR result to terminal.""" + sentences = data.get("sentences", []) + partial = data.get("partial", "") + partial_start = data.get("partial_start_ms", 0) + is_final = data.get("is_final", False) + + sys.stdout.write("\033[2J\033[H") + + print(f"{GREEN}Fun-ASR-Nano Streaming ASR{RESET}") + print(f"{GRAY}{'─' * 60}{RESET}") + + for s in sentences: + start = s.get("start", s.get("start_ms", 0)) + end = s.get("end", s.get("end_ms", 0)) + spk = s.get("spk", -1) + text = s["text"] + + time_str = f"{GRAY}[{format_time(start)}-{format_time(end)}]{RESET}" + spk_str = "" + if show_spk and spk >= 0: + color = SPK_COLORS[spk % len(SPK_COLORS)] + spk_str = f" {color}SPK{spk}{RESET}" + + print(f" {time_str}{spk_str} {text}") + + if partial: + print(f" {GRAY}[{format_time(partial_start)}-...] {partial}{RESET}") + + if is_final: + print(f"\n{GRAY}{'─' * 60}{RESET}") + print(f"{GREEN}Done.{RESET} {len(sentences)} sentences") + else: + print(f"\n{GRAY}Recording... Press Ctrl+C to stop{RESET}") + + sys.stdout.flush() + + +async def run_mic(args): + """Stream from microphone.""" + try: + import sounddevice as sd + except ImportError: + print("Please install sounddevice: pip install sounddevice") + sys.exit(1) + + print(f"Connecting to {args.server}...") + async with websockets.connect(args.server, ping_interval=None) as ws: + await ws.send("START") + resp = await ws.recv() + event = json.loads(resp) + if event.get("event") != "started": + print(f"Unexpected response: {resp}") + return + + if args.hotwords: + await ws.send(f"HOTWORDS:{args.hotwords}") + await ws.recv() + + print("Recording... Press Ctrl+C to stop\n") + + audio_queue = asyncio.Queue() + + def audio_callback(indata, frames, time_info, status): + audio_queue.put_nowait(indata.copy()) + + stream = sd.InputStream( + samplerate=SAMPLE_RATE, channels=1, dtype='int16', + blocksize=CHUNK_SAMPLES, callback=audio_callback, + ) + + async def send_audio(): + with stream: + while True: + chunk = await audio_queue.get() + await ws.send(chunk.tobytes()) + + async def recv_results(): + async for msg in ws: + data = json.loads(msg) + if "sentences" in data: + print_result(data, show_spk=args.spk) + if data.get("is_final") or data.get("event") == "stopped": + break + + send_task = asyncio.create_task(send_audio()) + recv_task = asyncio.create_task(recv_results()) + + try: + await asyncio.gather(send_task, recv_task) + except (KeyboardInterrupt, asyncio.CancelledError): + pass + finally: + send_task.cancel() + if ws.open: + await ws.send("STOP") + async for msg in ws: + data = json.loads(msg) + if "sentences" in data: + print_result(data, show_spk=args.spk) + if data.get("is_final") or data.get("event") == "stopped": + break + + +async def run_file(args): + """Stream an audio file.""" + try: + import soundfile as sf + except ImportError: + print("Please install soundfile: pip install soundfile") + sys.exit(1) + + audio, sr = sf.read(args.file) + if sr != SAMPLE_RATE: + try: + import librosa + audio = librosa.resample(audio, orig_sr=sr, target_sr=SAMPLE_RATE) + except ImportError: + print(f"Audio is {sr}Hz, need 16kHz. Install librosa: pip install librosa") + sys.exit(1) + if audio.ndim > 1: + audio = audio[:, 0] + audio = audio.astype(np.float32) + + duration = len(audio) / SAMPLE_RATE + print(f"File: {args.file} ({duration:.1f}s)") + print(f"Connecting to {args.server}...") + + async with websockets.connect(args.server, ping_interval=None) as ws: + await ws.send("START") + await ws.recv() + + if args.hotwords: + await ws.send(f"HOTWORDS:{args.hotwords}") + await ws.recv() + + int16 = (audio * 32768).clip(-32768, 32767).astype(np.int16) + + chunk_size = CHUNK_SAMPLES + total_chunks = (len(int16) + chunk_size - 1) // chunk_size + + async def send_audio(): + for i in range(0, len(int16), chunk_size): + chunk = int16[i:i+chunk_size] + await ws.send(chunk.tobytes()) + await asyncio.sleep(CHUNK_DURATION_MS / 1000 * 0.5) + await ws.send("STOP") + + async def recv_results(): + async for msg in ws: + data = json.loads(msg) + if "sentences" in data: + print_result(data, show_spk=args.spk) + if data.get("is_final") or data.get("event") == "stopped": + break + + await asyncio.gather(send_audio(), recv_results()) + + +def main(): + parser = argparse.ArgumentParser(description="Fun-ASR-Nano Python Client") + parser.add_argument("--server", type=str, default="ws://localhost:10095") + parser.add_argument("--mic", action="store_true", help="Use microphone input") + parser.add_argument("--file", type=str, help="Audio file to transcribe") + parser.add_argument("--hotwords", type=str, default="", help="Hotwords (comma-separated)") + parser.add_argument("--spk", action="store_true", default=True, help="Show speaker IDs") + parser.add_argument("--no-spk", dest="spk", action="store_false") + args = parser.parse_args() + + if not args.mic and not args.file: + parser.error("Specify --mic or --file") + + try: + if args.mic: + asyncio.run(run_mic(args)) + else: + asyncio.run(run_file(args)) + except KeyboardInterrupt: + print(f"\n{RESET}Interrupted.") + + +if __name__ == "__main__": + main() From f693a320b58f39e3aef7a5db42555f34d804ff55 Mon Sep 17 00:00:00 2001 From: Gemini AI Agent Date: Mon, 1 Jun 2026 18:07:41 +0200 Subject: [PATCH 10/10] fix(vad): handle null encoder_conf in FsmnVADStreaming --- funasr/models/fsmn_vad_streaming/model.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.py index 3acd481e1..eb9dee2c5 100644 --- a/funasr/models/fsmn_vad_streaming/model.py +++ b/funasr/models/fsmn_vad_streaming/model.py @@ -375,6 +375,9 @@ def __init__( self.vad_opts = VADXOptions(**kwargs) encoder_class = tables.encoder_classes.get(encoder) + # Fix: Ensure encoder_conf is a dictionary + if encoder_conf is None: + encoder_conf = {} encoder = encoder_class(**encoder_conf) self.encoder = encoder self.encoder_conf = encoder_conf