fix(paraformer): skip blank lines and read hotword files as UTF-8

funasr/models/contextual_paraformer/model.py

@@ -7,7 +7,6 @@
 import re
 import time
 import torch
-import codecs
 import logging
 import tempfile
 import requests
@@ -596,9 +595,11 @@ def seg_tokenize(txt, seg_dict):
             logging.info("Attempting to parse hotwords from local txt...")
             hotword_list = []
             hotword_str_list = []
-            with codecs.open(hotword_list_or_file, "r") as fin:
+            with open(hotword_list_or_file, "r", encoding="utf-8") as fin:
                 for line in fin.readlines():
                     hw = line.strip()
+                    if not hw:
+                        continue
                     hw_list = hw.split()
                     if seg_dict is not None:
                         hw_list = seg_tokenize(hw_list, seg_dict)
@@ -623,9 +624,11 @@ def seg_tokenize(txt, seg_dict):
             hotword_list_or_file = text_file_path
             hotword_list = []
             hotword_str_list = []
-            with codecs.open(hotword_list_or_file, "r") as fin:
+            with open(hotword_list_or_file, "r", encoding="utf-8") as fin:
                 for line in fin.readlines():
                     hw = line.strip()
+                    if not hw:
+                        continue
                     hw_list = hw.split()
                     if seg_dict is not None:
                         hw_list = seg_tokenize(hw_list, seg_dict)

funasr/models/seaco_paraformer/model.py

@@ -8,7 +8,6 @@
 import time
 import copy
 import torch
-import codecs
 import logging
 import tempfile
 import requests
@@ -629,9 +628,11 @@ def seg_tokenize(txt, seg_dict):
             logging.info("Attempting to parse hotwords from local txt...")
             hotword_list = []
             hotword_str_list = []
-            with codecs.open(hotword_list_or_file, "r") as fin:
+            with open(hotword_list_or_file, "r", encoding="utf-8") as fin:
                 for line in fin.readlines():
                     hw = line.strip()
+                    if not hw:
+                        continue
                     hw_list = hw.split()
                     if seg_dict is not None:
                         hw_list = seg_tokenize(hw_list, seg_dict)
@@ -656,9 +657,11 @@ def seg_tokenize(txt, seg_dict):
             hotword_list_or_file = text_file_path
             hotword_list = []
             hotword_str_list = []
-            with codecs.open(hotword_list_or_file, "r") as fin:
+            with open(hotword_list_or_file, "r", encoding="utf-8") as fin:
                 for line in fin.readlines():
                     hw = line.strip()
+                    if not hw:
+                        continue
                     hw_list = hw.split()
                     if seg_dict is not None:
                         hw_list = seg_tokenize(hw_list, seg_dict)

tests/test_paraformer_hotword_file.py

@@ -0,0 +1,84 @@
+"""Hotword ``.txt`` parsing for SeACo-Paraformer and Contextual Paraformer.
+
+``generate_hotwords_list`` turns every line of a hotword file into a list of
+token ids. A blank line (for example an extra newline at the end of the file)
+used to become an empty hotword, and the zero length then made
+``pack_padded_sequence`` raise during decoding. The file was also read with the
+platform's locale encoding, so a UTF-8 Chinese hotword file decoded into
+``<unk>`` tokens (or raised) on Windows.
+"""
+
+import builtins
+import os
+import types
+
+import pytest
+import torch
+
+from funasr.models.contextual_paraformer.model import ContextualParaformer
+from funasr.models.seaco_paraformer.model import SeacoParaformer
+
+SOS = 1
+VOCAB = {"<unk>": 0, "<s>": SOS, "阿": 2, "里": 3, "巴": 4, "云": 5}
+
+
+class _Tokenizer:
+    def tokens2ids(self, tokens):
+        return [VOCAB.get(t, VOCAB["<unk>"]) for t in tokens]
+
+
+def _generate(model_cls, tmp_path, content):
+    # Released Paraformer models ship a seg_dict next to am.mvn; it splits
+    # Chinese words into characters.
+    (tmp_path / "seg_dict").write_text("阿 阿\n里 里\n巴 巴\n云 云\n", encoding="utf-8")
+    hotword_file = tmp_path / "hotwords.txt"
+    hotword_file.write_text(content, encoding="utf-8")
+    frontend = types.SimpleNamespace(cmvn_file=os.path.join(str(tmp_path), "am.mvn"))
+    return model_cls.generate_hotwords_list(
+        types.SimpleNamespace(sos=SOS),
+        str(hotword_file),
+        tokenizer=_Tokenizer(),
+        frontend=frontend,
+    )
+
+
+@pytest.fixture(autouse=True)
+def _fresh_seg_dict_cache():
+    from funasr.models.seaco_paraformer import model as seaco_model
+
+    seaco_model.load_seg_dict.cache_clear()
+    yield
+    seaco_model.load_seg_dict.cache_clear()
+
+
+@pytest.mark.parametrize("model_cls", [SeacoParaformer, ContextualParaformer])
+def test_blank_lines_in_hotword_file_are_skipped(model_cls, tmp_path):
+    hotword_list = _generate(model_cls, tmp_path, "阿里巴巴\n\n   \n阿里云\n\n")
+
+    assert hotword_list == [[2, 3, 4, 4], [2, 3, 5], [SOS]]
+    # Every hotword must be packable by the bias encoder.
+    lengths = torch.tensor([len(hw) for hw in hotword_list])
+    torch.nn.utils.rnn.pack_padded_sequence(
+        torch.zeros(len(hotword_list), int(lengths.max()), 4),
+        lengths,
+        batch_first=True,
+        enforce_sorted=False,
+    )
+
+
+@pytest.mark.parametrize("model_cls", [SeacoParaformer, ContextualParaformer])
+def test_hotword_file_is_read_as_utf8_regardless_of_locale(model_cls, tmp_path, monkeypatch):
+    real_open = builtins.open
+
+    def cp1252_default_open(file, mode="r", *args, **kwargs):
+        # Simulate a Windows locale: text mode without an explicit encoding.
+        if "b" not in mode and kwargs.get("encoding") is None and len(args) < 2:
+            kwargs["encoding"] = "cp1252"
+            kwargs.setdefault("errors", "replace")
+        return real_open(file, mode, *args, **kwargs)
+
+    monkeypatch.setattr(builtins, "open", cp1252_default_open)
+
+    hotword_list = _generate(model_cls, tmp_path, "阿里巴巴\n阿里云\n")
+
+    assert hotword_list == [[2, 3, 4, 4], [2, 3, 5], [SOS]]