diff --git a/funasr/models/contextual_paraformer/model.py b/funasr/models/contextual_paraformer/model.py index 4c4df739b9..a8f23e9260 100644 --- a/funasr/models/contextual_paraformer/model.py +++ b/funasr/models/contextual_paraformer/model.py @@ -7,7 +7,6 @@ import re import time import torch -import codecs import logging import tempfile import requests @@ -596,9 +595,11 @@ def seg_tokenize(txt, seg_dict): logging.info("Attempting to parse hotwords from local txt...") hotword_list = [] hotword_str_list = [] - with codecs.open(hotword_list_or_file, "r") as fin: + with open(hotword_list_or_file, "r", encoding="utf-8") as fin: for line in fin.readlines(): hw = line.strip() + if not hw: + continue hw_list = hw.split() if seg_dict is not None: hw_list = seg_tokenize(hw_list, seg_dict) @@ -623,9 +624,11 @@ def seg_tokenize(txt, seg_dict): hotword_list_or_file = text_file_path hotword_list = [] hotword_str_list = [] - with codecs.open(hotword_list_or_file, "r") as fin: + with open(hotword_list_or_file, "r", encoding="utf-8") as fin: for line in fin.readlines(): hw = line.strip() + if not hw: + continue hw_list = hw.split() if seg_dict is not None: hw_list = seg_tokenize(hw_list, seg_dict) diff --git a/funasr/models/seaco_paraformer/model.py b/funasr/models/seaco_paraformer/model.py index efa4745a35..6a57875308 100644 --- a/funasr/models/seaco_paraformer/model.py +++ b/funasr/models/seaco_paraformer/model.py @@ -8,7 +8,6 @@ import time import copy import torch -import codecs import logging import tempfile import requests @@ -629,9 +628,11 @@ def seg_tokenize(txt, seg_dict): logging.info("Attempting to parse hotwords from local txt...") hotword_list = [] hotword_str_list = [] - with codecs.open(hotword_list_or_file, "r") as fin: + with open(hotword_list_or_file, "r", encoding="utf-8") as fin: for line in fin.readlines(): hw = line.strip() + if not hw: + continue hw_list = hw.split() if seg_dict is not None: hw_list = seg_tokenize(hw_list, seg_dict) @@ -656,9 +657,11 @@ def seg_tokenize(txt, seg_dict): hotword_list_or_file = text_file_path hotword_list = [] hotword_str_list = [] - with codecs.open(hotword_list_or_file, "r") as fin: + with open(hotword_list_or_file, "r", encoding="utf-8") as fin: for line in fin.readlines(): hw = line.strip() + if not hw: + continue hw_list = hw.split() if seg_dict is not None: hw_list = seg_tokenize(hw_list, seg_dict) diff --git a/tests/test_paraformer_hotword_file.py b/tests/test_paraformer_hotword_file.py new file mode 100644 index 0000000000..c12ff45575 --- /dev/null +++ b/tests/test_paraformer_hotword_file.py @@ -0,0 +1,84 @@ +"""Hotword ``.txt`` parsing for SeACo-Paraformer and Contextual Paraformer. + +``generate_hotwords_list`` turns every line of a hotword file into a list of +token ids. A blank line (for example an extra newline at the end of the file) +used to become an empty hotword, and the zero length then made +``pack_padded_sequence`` raise during decoding. The file was also read with the +platform's locale encoding, so a UTF-8 Chinese hotword file decoded into +```` tokens (or raised) on Windows. +""" + +import builtins +import os +import types + +import pytest +import torch + +from funasr.models.contextual_paraformer.model import ContextualParaformer +from funasr.models.seaco_paraformer.model import SeacoParaformer + +SOS = 1 +VOCAB = {"": 0, "": SOS, "阿": 2, "里": 3, "巴": 4, "云": 5} + + +class _Tokenizer: + def tokens2ids(self, tokens): + return [VOCAB.get(t, VOCAB[""]) for t in tokens] + + +def _generate(model_cls, tmp_path, content): + # Released Paraformer models ship a seg_dict next to am.mvn; it splits + # Chinese words into characters. + (tmp_path / "seg_dict").write_text("阿 阿\n里 里\n巴 巴\n云 云\n", encoding="utf-8") + hotword_file = tmp_path / "hotwords.txt" + hotword_file.write_text(content, encoding="utf-8") + frontend = types.SimpleNamespace(cmvn_file=os.path.join(str(tmp_path), "am.mvn")) + return model_cls.generate_hotwords_list( + types.SimpleNamespace(sos=SOS), + str(hotword_file), + tokenizer=_Tokenizer(), + frontend=frontend, + ) + + +@pytest.fixture(autouse=True) +def _fresh_seg_dict_cache(): + from funasr.models.seaco_paraformer import model as seaco_model + + seaco_model.load_seg_dict.cache_clear() + yield + seaco_model.load_seg_dict.cache_clear() + + +@pytest.mark.parametrize("model_cls", [SeacoParaformer, ContextualParaformer]) +def test_blank_lines_in_hotword_file_are_skipped(model_cls, tmp_path): + hotword_list = _generate(model_cls, tmp_path, "阿里巴巴\n\n \n阿里云\n\n") + + assert hotword_list == [[2, 3, 4, 4], [2, 3, 5], [SOS]] + # Every hotword must be packable by the bias encoder. + lengths = torch.tensor([len(hw) for hw in hotword_list]) + torch.nn.utils.rnn.pack_padded_sequence( + torch.zeros(len(hotword_list), int(lengths.max()), 4), + lengths, + batch_first=True, + enforce_sorted=False, + ) + + +@pytest.mark.parametrize("model_cls", [SeacoParaformer, ContextualParaformer]) +def test_hotword_file_is_read_as_utf8_regardless_of_locale(model_cls, tmp_path, monkeypatch): + real_open = builtins.open + + def cp1252_default_open(file, mode="r", *args, **kwargs): + # Simulate a Windows locale: text mode without an explicit encoding. + if "b" not in mode and kwargs.get("encoding") is None and len(args) < 2: + kwargs["encoding"] = "cp1252" + kwargs.setdefault("errors", "replace") + return real_open(file, mode, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", cp1252_default_open) + + hotword_list = _generate(model_cls, tmp_path, "阿里巴巴\n阿里云\n") + + assert hotword_list == [[2, 3, 4, 4], [2, 3, 5], [SOS]]