From ddc54ade8f3d0abf8265ff216f6765000cf9807f Mon Sep 17 00:00:00 2001 From: jaya-TN Date: Wed, 29 Jul 2026 19:36:05 +0530 Subject: [PATCH 1/3] te ITN: add Telugu inverse text normalization for cardinals Add the Telugu (te) cardinal semiotic class for ITN: taggers, verbalizers, data files, graph_utils and utils. Uses the Indian numbering system (hundred/thousand/lakh/crore) with a fixed-width place-value graph and native Telugu digit output; English digits are passed through. The number lexicon is data-driven via TSV files. Adds pytest cases and a Sparrowhawk test, and registers te in the Jenkinsfile and pynini_export. Signed-off-by: jaya-TN --- Jenkinsfile | 11 + .../inverse_normalize.py | 6 + .../run_evaluate.py | 1 + .../inverse_text_normalization/te/__init__.py | 17 ++ .../te/data/__init__.py | 13 ++ .../te/data/numbers/__init__.py | 13 ++ .../te/data/numbers/case_suffix.tsv | 5 + .../te/data/numbers/crore.tsv | 3 + .../te/data/numbers/digit.tsv | 9 + .../te/data/numbers/digit_people.tsv | 9 + .../te/data/numbers/hundred.tsv | 5 + .../te/data/numbers/lakh.tsv | 4 + .../te/data/numbers/special_numbers.tsv | 2 + .../te/data/numbers/teens_and_ties.tsv | 18 ++ .../te/data/numbers/thousand.tsv | 3 + .../te/data/numbers/ties_prefix.tsv | 8 + .../te/data/numbers/zero.tsv | 1 + .../te/graph_utils.py | 196 ++++++++++++++++++ .../te/taggers/__init__.py | 13 ++ .../te/taggers/cardinal.py | 142 +++++++++++++ .../te/taggers/tokenize_and_classify.py | 75 +++++++ .../te/taggers/word.py | 30 +++ .../inverse_text_normalization/te/utils.py | 63 ++++++ .../te/verbalizers/__init__.py | 13 ++ .../te/verbalizers/cardinal.py | 55 +++++ .../te/verbalizers/verbalize.py | 32 +++ .../te/verbalizers/verbalize_final.py | 49 +++++ .../te/verbalizers/word.py | 38 ++++ tests/nemo_text_processing/te/__init__.py | 13 ++ .../test_cases_cardinal.txt | 51 +++++ .../nemo_text_processing/te/test_cardinal.py | 35 ++++ ..._sparrowhawk_inverse_text_normalization.sh | 25 +++ .../pynini_export.py | 10 +- 33 files changed, 967 insertions(+), 1 deletion(-) create mode 100644 nemo_text_processing/inverse_text_normalization/te/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/te/data/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/te/data/numbers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/te/data/numbers/case_suffix.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/te/data/numbers/crore.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/te/data/numbers/digit.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/te/data/numbers/digit_people.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/te/data/numbers/lakh.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/te/data/numbers/special_numbers.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/te/data/numbers/teens_and_ties.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/te/data/numbers/thousand.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/te/data/numbers/ties_prefix.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/te/data/numbers/zero.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/te/graph_utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/te/taggers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/te/taggers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/te/taggers/tokenize_and_classify.py create mode 100644 nemo_text_processing/inverse_text_normalization/te/taggers/word.py create mode 100644 nemo_text_processing/inverse_text_normalization/te/utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/te/verbalizers/__init__.py create mode 100644 nemo_text_processing/inverse_text_normalization/te/verbalizers/cardinal.py create mode 100644 nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize.py create mode 100644 nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize_final.py create mode 100644 nemo_text_processing/inverse_text_normalization/te/verbalizers/word.py create mode 100644 tests/nemo_text_processing/te/__init__.py create mode 100644 tests/nemo_text_processing/te/data_inverse_text_normalization/test_cases_cardinal.txt create mode 100644 tests/nemo_text_processing/te/test_cardinal.py create mode 100644 tests/nemo_text_processing/te/test_sparrowhawk_inverse_text_normalization.sh diff --git a/Jenkinsfile b/Jenkinsfile index 38e05bb00..4f5ee5c23 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,6 +27,7 @@ pipeline { HE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-24-25-0' HY_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-0' MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' + TE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-22-26-0' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-26-5' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' @@ -297,6 +298,11 @@ pipeline { sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=mr --text="शून्य " --cache_dir ${MR_TN_CACHE}' } } + stage('L0: TE ITN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=te --text="సున్నా " --cache_dir ${TE_TN_CACHE}' + } + } stage('L0: HY TN grammars') { steps { sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hy --text="6" --cache_dir ${HY_TN_CACHE}' @@ -463,6 +469,11 @@ pipeline { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/mr/ -m "not pleasefixme" --cpu --tn_cache_dir ${MR_TN_CACHE}' } } + stage('L1: Run all TE ITN tests (restore grammars from cache)') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/te/ -m "not pleasefixme" --cpu --tn_cache_dir ${TE_TN_CACHE}' + } + } stage('L1: Run all HY TN/ITN tests (restore grammars from cache)') { steps { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/hy/ -m "not pleasefixme" --cpu --tn_cache_dir ${HY_TN_CACHE}' diff --git a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py index 9a6fcc64c..4ce15346c 100644 --- a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py +++ b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py @@ -121,6 +121,11 @@ def __init__( from nemo_text_processing.inverse_text_normalization.hi.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) + elif lang == 'te': # Telugu + from nemo_text_processing.inverse_text_normalization.te.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize_final import ( + VerbalizeFinalFst, + ) elif lang == 'hi_en': # Hindi-English code-switch from nemo_text_processing.inverse_text_normalization.hi_en.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.inverse_text_normalization.hi_en.verbalizers.verbalize_final import ( @@ -207,6 +212,7 @@ def parse_args(): 'zh', 'he', 'hi', + 'te', 'hy', 'mr', 'ja', diff --git a/nemo_text_processing/inverse_text_normalization/run_evaluate.py b/nemo_text_processing/inverse_text_normalization/run_evaluate.py index cf9b29fce..3a989f96c 100644 --- a/nemo_text_processing/inverse_text_normalization/run_evaluate.py +++ b/nemo_text_processing/inverse_text_normalization/run_evaluate.py @@ -44,6 +44,7 @@ def parse_args(): "fr", "hi", "hi_en", + "te", "hy", "ko", "mr", diff --git a/nemo_text_processing/inverse_text_normalization/te/__init__.py b/nemo_text_processing/inverse_text_normalization/te/__init__.py new file mode 100644 index 000000000..d3f64b4cb --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.te.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/inverse_text_normalization/te/data/__init__.py b/nemo_text_processing/inverse_text_normalization/te/data/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/te/data/numbers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/case_suffix.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/case_suffix.tsv new file mode 100644 index 000000000..98724c4f5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/case_suffix.tsv @@ -0,0 +1,5 @@ +కి +కు +లో +తో +కోసం \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/crore.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/crore.tsv new file mode 100644 index 000000000..af423dc36 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/crore.tsv @@ -0,0 +1,3 @@ + కోట్ల + కోట్లు +౦౦౦౦౦౦౧ కోటి diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit.tsv new file mode 100644 index 000000000..6606ff8b3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +౧ ఒకటి +౨ రెండు +౩ మూడు +౪ నాలుగు +౫ ఐదు +౬ ఆరు +౭ ఏడు +౮ ఎనిమిది +౯ తొమ్మిది \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit_people.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit_people.tsv new file mode 100644 index 000000000..6a3bb21b9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit_people.tsv @@ -0,0 +1,9 @@ +౨ ఇద్దరు +౩ ముగ్గురు +౪ నలుగురు +౫ ఐదుగురు +౬ ఆరుగురు +౭ ఏడుగురు +౮ ఎనిమిదుగురు +౯ తొమ్మిదుగురు +౧౦ పది మంది diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred.tsv new file mode 100644 index 000000000..c07663ad7 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred.tsv @@ -0,0 +1,5 @@ + వందల + వందలు +౧ నూట +౧ వంద +౧ నూరు diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/lakh.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/lakh.tsv new file mode 100644 index 000000000..8c2c8cc85 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/lakh.tsv @@ -0,0 +1,4 @@ + లక్షల + లక్షలు +౦౧ లక్ష +౦౧ లక్షా diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/special_numbers.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/special_numbers.tsv new file mode 100644 index 000000000..87013e433 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/special_numbers.tsv @@ -0,0 +1,2 @@ +౧౯ పంతొమ్మిది +౧ ఒక diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/teens_and_ties.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/teens_and_ties.tsv new file mode 100644 index 000000000..5c0ca90ed --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/teens_and_ties.tsv @@ -0,0 +1,18 @@ +౧౦ పది +౧౧ పదకొండు +౧౨ పన్నెండు +౧౩ పదమూడు +౧౪ పద్నాలుగు +౧౫ పదిహేను +౧౬ పదహారు +౧౭ పదిహేడు +౧౮ పద్దెనిమిది +౧౯ పందొమ్మిది +౨౦ ఇరవై +౩౦ ముప్పై +౪౦ నలభై +౫౦ యాభై +౬౦ అరవై +౭౦ డెబ్బై +౮౦ ఎనభై +౯౦ తొంభై \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousand.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousand.tsv new file mode 100644 index 000000000..717da5c6b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousand.tsv @@ -0,0 +1,3 @@ + వేల + వేలు +౦౧ వెయ్యి diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/ties_prefix.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/ties_prefix.tsv new file mode 100644 index 000000000..78949df54 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/ties_prefix.tsv @@ -0,0 +1,8 @@ +౨ ఇరవై +౩ ముప్పై +౪ నలభై +౫ యాభై +౬ అరవై +౭ డెబ్బై +౮ ఎనభై +౯ తొంభై diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/zero.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/zero.tsv new file mode 100644 index 000000000..5f95b48d6 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/zero.tsv @@ -0,0 +1 @@ +౦ సున్నా \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/graph_utils.py b/nemo_text_processing/inverse_text_normalization/te/graph_utils.py new file mode 100644 index 000000000..15d6616c1 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/graph_utils.py @@ -0,0 +1,196 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.export import export +from pynini.lib import pynutil, utf8 + +from nemo_text_processing.inverse_text_normalization.te.utils import load_labels + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_TE_DIGIT = pynini.union("౦", "౧", "౨", "౩", "౪", "౫", "౬", "౭", "౮", "౯").optimize() + +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = u"\u00a0" +NEMO_ZWNJ = u"\u200c" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_CHAR, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) +) + + +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("ఋణాత్మక", "మైనస్").optimize() + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +def convert_space(fst) -> 'pynini.FstLike': + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [written_capitalized, spoken.capitalize()], + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], + ] + ) + + spoken_no_space = spoken.replace(" ", "") + + if len(spoken) == (2 * len(spoken_no_space) - 1): + logging.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [[written, spoken_no_space], [written_capitalized, spoken_no_space.upper()]] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/te/taggers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/te/taggers/cardinal.py new file mode 100644 index 000000000..d9bf0864a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/cardinal.py @@ -0,0 +1,142 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import ( + INPUT_LOWER_CASED, + MINUS, + NEMO_CHAR, + NEMO_NOT_SPACE, + NEMO_SPACE, + NEMO_TE_DIGIT, + GraphFst, + delete_space, +) +from nemo_text_processing.inverse_text_normalization.te.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying Telugu cardinals, e.g. + ఇరవై మూడు -> cardinal { integer: "౨౩" } + మైనస్ ఇరవై మూడు -> cardinal { integer: "౨౩" negative: "-" } + + + Args: + input_case: accepting either "lower_cased" or "cased" input. + """ + + def __init__(self, input_case: str = INPUT_LOWER_CASED): + super().__init__(name="cardinal", kind="classify") + self.input_case = input_case + + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert() + graph_teens_and_ties = pynini.string_file(get_abs_path("data/numbers/teens_and_ties.tsv")).invert() + graph_ties_prefix = pynini.string_file(get_abs_path("data/numbers/ties_prefix.tsv")).invert() + + graph_people = pynini.string_file(get_abs_path("data/numbers/digit_people.tsv")).invert() + graph_special = pynini.string_file(get_abs_path("data/numbers/special_numbers.tsv")).invert() + two_te_digits = NEMO_TE_DIGIT + NEMO_TE_DIGIT + graph_digit |= (graph_people | graph_special) @ NEMO_TE_DIGIT + + self.graph_zero = graph_zero + self.graph_digit = graph_digit + + graph_two_digit = ( + graph_teens_and_ties + | (graph_ties_prefix + delete_space + graph_digit) + | (pynutil.insert("౦") + graph_digit) + | ((graph_people | graph_special) @ two_te_digits) + ) + self.graph_two_digit = graph_two_digit + two_digit_or_zeros = graph_two_digit | pynutil.insert("౦౦") + + graph_case_suffix = pynini.string_file(get_abs_path("data/numbers/case_suffix.tsv")) + optional_case = pynini.closure(graph_case_suffix, 0, 1) + + optional_one = pynini.closure(pynutil.delete("ఒక") + delete_space, 0, 1) + + graph_hundred = pynini.string_file(get_abs_path("data/numbers/hundred.tsv")).invert() + delete_hundred = graph_hundred @ pynini.accep("") + one_hundred_value = graph_hundred @ pynini.closure(NEMO_TE_DIGIT, 1) + + graph_thousand = pynini.string_file(get_abs_path("data/numbers/thousand.tsv")).invert() + delete_thousand = graph_thousand @ pynini.accep("") + one_thousand_value = graph_thousand @ pynini.closure(NEMO_TE_DIGIT, 1) + + graph_lakh = pynini.string_file(get_abs_path("data/numbers/lakh.tsv")).invert() + delete_lakh = graph_lakh @ pynini.accep("") + one_lakh_value = graph_lakh @ pynini.closure(NEMO_TE_DIGIT, 1) + + graph_crore = pynini.string_file(get_abs_path("data/numbers/crore.tsv")).invert() + delete_crore = graph_crore @ pynini.accep("") + one_crore_value = graph_crore @ pynini.closure(NEMO_TE_DIGIT, 1) + + hundred_prefix = ( + (graph_digit + delete_space + delete_hundred) | (optional_one + one_hundred_value) | pynutil.insert("౦") + ) + graph_hundreds = hundred_prefix + delete_space + two_digit_or_zeros + self.graph_hundreds = graph_hundreds + + thousand_block = ( + (graph_two_digit + delete_space + delete_thousand) + | (optional_one + one_thousand_value) + | pynutil.insert("౦౦") + ) + + lakh_block = ( + (graph_two_digit + delete_space + delete_lakh) | (optional_one + one_lakh_value) | pynutil.insert("౦౦") + ) + + graph_below_crore = lakh_block + delete_space + thousand_block + delete_space + graph_hundreds + + at_least_one_non_zero = pynini.closure(NEMO_TE_DIGIT) + (NEMO_TE_DIGIT - "౦") + pynini.closure(NEMO_TE_DIGIT) + + crore_block = ( + ((graph_below_crore @ at_least_one_non_zero) + delete_space + delete_crore) + | (optional_one + one_crore_value) + | pynutil.insert("౦౦౦౦౦౦౦") + ) + + graph_full = crore_block + delete_space + graph_below_crore + strip_leading_zeros = ( + pynutil.delete(pynini.closure("౦")) + (NEMO_TE_DIGIT - "౦") + pynini.closure(NEMO_TE_DIGIT) + ) + + no_trailing_space = pynini.closure(NEMO_CHAR) + NEMO_NOT_SPACE + graph_number = (no_trailing_space @ graph_full) @ strip_leading_zeros + + graph_leading_zeros = graph_zero + pynini.closure(delete_space + (graph_zero | graph_digit), 1) + + graph = (graph_number | graph_zero) + optional_case + graph |= graph_leading_zeros + + self.graph_no_exception = graph + + self.graph = graph.optimize() + + optional_minus_graph = pynini.closure( + pynutil.insert("negative: ") + pynini.cross(MINUS, "\"-\"") + NEMO_SPACE, + 0, + 1, + ) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/te/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..a85323680 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/tokenize_and_classify.py @@ -0,0 +1,75 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.inverse_text_normalization.te.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.te.taggers.word import WordFst + + +class ClassifyFst(GraphFst): + """ + Final classification grammar for Telugu ITN cardinal processing. + """ + + def __init__( + self, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + input_case: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "te_itn.far") + + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info("Creating Telugu ClassifyFst grammar.") + + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + word_graph = WordFst().fst + + classify = pynutil.add_weight(cardinal_graph, 1.1) | pynutil.add_weight(word_graph, 100) + + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + + graph = token + pynini.closure(delete_extra_space + token) + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() + + if far_file: + generator_main( + far_file, + {"tokenize_and_classify": self.fst}, + ) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/word.py b/nemo_text_processing/inverse_text_normalization/te/taggers/word.py new file mode 100644 index 000000000..95045582a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/word.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain Telugu tokens. + """ + + def __init__(self): + super().__init__(name="word", kind="classify") + word = pynutil.insert('name: "') + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert('"') + self.fst = word.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/te/utils.py b/nemo_text_processing/inverse_text_normalization/te/utils.py new file mode 100644 index 000000000..5830b50d0 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/utils.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import logging +import os +import pynini + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + abs_path = os.path.dirname(os.path.abspath(__file__)) + os.sep + rel_path + + if not os.path.exists(abs_path): + logging.warning(f'{abs_path} does not exist') + return abs_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + label_tsv = open(abs_path, encoding="utf-8") + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels + + +from pynini.lib import pynutil + + +def apply_fst(text, fst): + """Given a string input, returns the output string + produced by traversing the path with lowest weight. + If no valid path accepts input string, returns an + error. + """ + try: + print(pynini.shortestpath(text @ fst).string()) + except pynini.FstOpError: + print(f"Error: No valid output with given input: '{text}'") diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/cardinal.py new file mode 100644 index 000000000..cf6c3a903 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/cardinal.py @@ -0,0 +1,55 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal + e.g. + cardinal { integer: "౨౩" } -> ౨౩ + cardinal { integer: "౨౩" negative: "-" } -> -౨౩ + """ + + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + + optional_sign = pynini.closure( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + NEMO_NOT_QUOTE + + pynutil.delete("\"") + + delete_space, + 0, + 1, + ) + + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + self.numbers = graph + graph = optional_sign + graph + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize.py new file mode 100644 index 000000000..cff457b72 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize.py @@ -0,0 +1,32 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.te.verbalizers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.te.verbalizers.word import WordFst + + +class VerbalizeFst(GraphFst): + """ + Composes the Telugu cardinal verbalizer grammar. + """ + + def __init__(self): + super().__init__(name="verbalize", kind="verbalize") + + cardinal_graph = CardinalFst().fst + word_graph = WordFst().fst + self.fst = (cardinal_graph | word_graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize_final.py new file mode 100644 index 000000000..c097ebee7 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize_final.py @@ -0,0 +1,49 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import GraphFst, delete_extra_space, delete_space +from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize import VerbalizeFst + + +class VerbalizeFinalFst(GraphFst): + """ + Verbalizes full Telugu ITN token output. + + Example: + tokens { cardinal { integer: "౨౩" } } -> ౨౩ + tokens { name: "పరుగులు" } -> పరుగులు + """ + + def __init__(self): + super().__init__(name="verbalize_final", kind="verbalize") + + verbalize = VerbalizeFst().fst + + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + verbalize + + delete_space + + pynutil.delete("}") + ) + + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/word.py new file mode 100644 index 000000000..28603646c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/word.py @@ -0,0 +1,38 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import ( + NEMO_CHAR, + NEMO_SIGMA, + GraphFst, + delete_space, +) + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing plain Telugu tokens. + """ + + def __init__(self): + super().__init__(name="word", kind="verbalize") + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete('"') + chars + pynutil.delete('"') + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/tests/nemo_text_processing/te/__init__.py b/tests/nemo_text_processing/te/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/tests/nemo_text_processing/te/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/te/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/te/data_inverse_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..a9e447128 --- /dev/null +++ b/tests/nemo_text_processing/te/data_inverse_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,51 @@ +నాలుగు నాలుగులు~౪ నాలుగులు +ఆరు మంది ఆటగాళ్లు బయటకు~౬ మంది ఆటగాళ్లు బయటకు +నాలుగు ఓవర్లలో పదిహేడు పరుగులు~౪ ఓవర్లలో ౧౭ పరుగులు +ఐదు చాక్లెట్లు తొమ్మిది టాఫీలు~౫ చాక్లెట్లు ౯ టాఫీలు +పది వేల తొంభై తొమ్మిది~౧౦౦౯౯ +లక్షా ఒకటి~౧౦౦౦౦౧ +ఐదు నాలుగులు~౫ నాలుగులు +నాలుగు మంది మృతి పద్దెనిమిది మందికి గాయాలు~౪ మంది మృతి ౧౮ మందికి గాయాలు +ఐదు వేల నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౫౧౦౨౨౩౪౫౫౬౭ +అధ్యాయం పది శ్లోకం రెండు~అధ్యాయం ౧౦ శ్లోకం ౨ +నూట ఒకటి~౧౦౧ +నూట ఇరవై మూడు~౧౨౩ +వంద~౧౦౦ +నూట ఐదు~౧౦౫ +రెండు వందలు~౨౦౦ +రెండు వందల మూడు~౨౦౩ +ఐదు వందల రెండు~౫౦౨ +నూట పదకొండు~౧౧౧ +నూట పంతొమ్మిది~౧౧౯ +నూట ముప్పై~౧౩౦ +నూట యాభై~౧౫౦ +మూడు వందల పన్నెండు~౩౧౨ +ఐదు వందల ఇరవై ఐదు~౫౨౫ +నాలుగు వందల ఇరవై ఐదు~౪౨౫ +తొమ్మిది వందల పదిహేను~౯౧౫ +వెయ్యి ఒకటి~౧౦౦౧ +ఐదు వేల ఐదు వందల యాభై ఒకటి అరటిపండ్లు~౫౫౫౧ అరటిపండ్లు +ఎనభై ఐదు~౮౫ +తొంభై ఒకటి~౯౧ +వెయ్యి మూడు వందల ఇరవై మూడు~౧౩౨౩ +వెయ్యి మూడు వందల నలభై ఐదు~౧౩౪౫ +పదమూడు వేల నాలుగు వందల యాభై ఆరు~౧౩౪౫౬ +పన్నెండు వేల మూడు వందల నలభై ఆరు~౧౨౩౪౬ +లక్ష ఇరవై మూడు వేల నాలుగు వందల యాభై ఆరు~౧౨౩౪౫౬ +ఎనిమిది లక్షల డెబ్బై రెండు వేల తొమ్మిది వందల ఎనభై ఏడు~౮౭౨౯౮౭ +తొమ్మిది లక్షల ఎనభై ఏడు వేల ఆరు వందల తొమ్మిది~౯౮౭౬౦౯ +తొంభై ఎనిమిది లక్షల డెబ్బై ఆరు వేల ఏడు వందల ఎనభై తొమ్మిది~౯౮౭౬౭౮౯ +ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౨౩౪౫౫౬౭ +కోటి ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౧౨౩౪౫౫౬౭ +కోటి ఇరవై ఒక లక్షల ఇరవై ఒక వేల రెండు వందల పన్నెండు~౧౨౧౨౧౨౧౨ +నూట పన్నెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౧౧౨౨౩౪౫౫౬౭ +నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౧౦౨౨౩౪౫౫౬౭ +వెయ్యి నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౧౧౦౨౨౩౪౫౫౬౭ +ఐదు వేల నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౫౧౦౨౨౩౪౫౫౬౭ +సున్నా ఐదు~౦౫ +సున్నా ఒకటి~౦౧ +సున్నా ఏడు మూడు~౦౭౩ +సున్నా సున్నా సున్నా ఒకటి~౦౦౦౧ +సున్నా సున్నా సున్నా~౦౦౦ +మైనస్ ఇరవై మూడు~-౨౩ +ఋణాత్మక ఐదు~-౫ diff --git a/tests/nemo_text_processing/te/test_cardinal.py b/tests/nemo_text_processing/te/test_cardinal.py new file mode 100644 index 000000000..85a10a177 --- /dev/null +++ b/tests/nemo_text_processing/te/test_cardinal.py @@ -0,0 +1,35 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.te.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize_final import VerbalizeFinalFst + +from ..utils import parse_test_case_file + + +class TestCardinal: + tagger = ClassifyFst(overwrite_cache=True).fst + verbalizer = VerbalizeFinalFst().fst + + @parameterized.expand(parse_test_case_file('te/data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + tagged = pynini.shortestpath(test_input @ self.tagger).string() + pred = pynini.shortestpath(tagged @ self.verbalizer).string() + assert pred == expected diff --git a/tests/nemo_text_processing/te/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/te/test_sparrowhawk_inverse_text_normalization.sh new file mode 100644 index 000000000..a270ae4f5 --- /dev/null +++ b/tests/nemo_text_processing/te/test_sparrowhawk_inverse_text_normalization.sh @@ -0,0 +1,25 @@ +#! /bin/sh + +PROJECT_DIR=/workspace/tests + +runtest () { + input=$1 + cd /workspace/sparrowhawk/documentation/grammars + + while read testcase; do + IFS='~' read spoken written <<< $testcase + denorm_pred=$(echo $spoken | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + + written="$(echo -e "${written}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + assertEquals "$spoken" "$written" "$denorm_pred" + done < "$input" +} + +testITNCardinal() { + input=$PROJECT_DIR/te/data_inverse_text_normalization/test_cases_cardinal.txt + runtest $input +} + +. $PROJECT_DIR/../shunit2/shunit2 diff --git a/tools/text_processing_deployment/pynini_export.py b/tools/text_processing_deployment/pynini_export.py index 73a4fc138..f183fdf0c 100644 --- a/tools/text_processing_deployment/pynini_export.py +++ b/tools/text_processing_deployment/pynini_export.py @@ -106,6 +106,7 @@ def parse_args(): 'hi_en', 'hy', 'mr', + 'te', 'ja', 'rw', 'ko', @@ -140,7 +141,7 @@ def parse_args(): if __name__ == '__main__': args = parse_args() - if args.language in ['ru', 'es_en', 'hi_en', 'mr'] and args.grammars == 'tn_grammars': + if args.language in ['ru', 'es_en', 'hi_en', 'mr', 'te'] and args.grammars == 'tn_grammars': raise ValueError('Only ITN grammars could be deployed in Sparrowhawk for the selected languages.') TNPostProcessingFst = None ITNPostProcessingFst = None @@ -305,6 +306,13 @@ def parse_args(): from nemo_text_processing.inverse_text_normalization.mr.verbalizers.verbalize import ( VerbalizeFst as ITNVerbalizeFst, ) + elif args.language == 'te': + from nemo_text_processing.inverse_text_normalization.te.taggers.tokenize_and_classify import ( + ClassifyFst as ITNClassifyFst, + ) + from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize import ( + VerbalizeFst as ITNVerbalizeFst, + ) elif args.language == 'he': from nemo_text_processing.inverse_text_normalization.he.taggers.tokenize_and_classify import ( ClassifyFst as ITNClassifyFst, From df866462cc0f2beca2b0cbd7569f8d8de0efb5c7 Mon Sep 17 00:00:00 2001 From: jaya-TN Date: Thu, 30 Jul 2026 09:02:40 +0530 Subject: [PATCH 2/3] style: fix import order (isort) Signed-off-by: jaya-TN --- .../inverse_text_normalization/te/__init__.py | 2 +- .../te/data/__init__.py | 2 +- .../te/data/numbers/__init__.py | 2 +- .../te/data/numbers/crore.tsv | 6 +-- .../te/data/numbers/digit.tsv | 18 ++++---- .../te/data/numbers/digit_people.tsv | 18 ++++---- .../te/data/numbers/hundred.tsv | 10 ++--- .../te/data/numbers/lakh.tsv | 8 ++-- .../te/data/numbers/special_numbers.tsv | 4 +- .../te/data/numbers/teens_and_ties.tsv | 36 ++++++++-------- .../te/data/numbers/thousand.tsv | 6 +-- .../te/data/numbers/ties_prefix.tsv | 16 +++---- .../te/data/numbers/zero.tsv | 2 +- .../te/graph_utils.py | 3 +- .../te/taggers/__init__.py | 2 +- .../te/taggers/cardinal.py | 42 +++++++------------ .../te/taggers/tokenize_and_classify.py | 3 +- .../te/taggers/word.py | 3 +- .../inverse_text_normalization/te/utils.py | 2 +- .../te/verbalizers/__init__.py | 2 +- .../te/verbalizers/cardinal.py | 2 +- .../te/verbalizers/verbalize.py | 3 +- .../te/verbalizers/verbalize_final.py | 3 +- .../te/verbalizers/word.py | 3 +- tests/nemo_text_processing/te/__init__.py | 2 +- .../nemo_text_processing/te/test_cardinal.py | 14 +++---- .../pynini_export.py | 2 +- 27 files changed, 96 insertions(+), 120 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/te/__init__.py b/nemo_text_processing/inverse_text_normalization/te/__init__.py index d3f64b4cb..848a62938 100644 --- a/nemo_text_processing/inverse_text_normalization/te/__init__.py +++ b/nemo_text_processing/inverse_text_normalization/te/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/data/__init__.py b/nemo_text_processing/inverse_text_normalization/te/data/__init__.py index 4fc25d0d3..963711f1c 100644 --- a/nemo_text_processing/inverse_text_normalization/te/data/__init__.py +++ b/nemo_text_processing/inverse_text_normalization/te/data/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/te/data/numbers/__init__.py index 4fc25d0d3..963711f1c 100644 --- a/nemo_text_processing/inverse_text_normalization/te/data/numbers/__init__.py +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/crore.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/crore.tsv index af423dc36..d23d22cb9 100644 --- a/nemo_text_processing/inverse_text_normalization/te/data/numbers/crore.tsv +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/crore.tsv @@ -1,3 +1,3 @@ - కోట్ల - కోట్లు -౦౦౦౦౦౦౧ కోటి +కోట్ల +కోట్లు +కోటి ౦౦౦౦౦౦౧ diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit.tsv index 6606ff8b3..14faf522a 100644 --- a/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit.tsv +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit.tsv @@ -1,9 +1,9 @@ -౧ ఒకటి -౨ రెండు -౩ మూడు -౪ నాలుగు -౫ ఐదు -౬ ఆరు -౭ ఏడు -౮ ఎనిమిది -౯ తొమ్మిది \ No newline at end of file +ఒకటి ౧ +రెండు ౨ +మూడు ౩ +నాలుగు ౪ +ఐదు ౫ +ఆరు ౬ +ఏడు ౭ +ఎనిమిది ౮ +తొమ్మిది ౯ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit_people.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit_people.tsv index 6a3bb21b9..1d5a5e17e 100644 --- a/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit_people.tsv +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit_people.tsv @@ -1,9 +1,9 @@ -౨ ఇద్దరు -౩ ముగ్గురు -౪ నలుగురు -౫ ఐదుగురు -౬ ఆరుగురు -౭ ఏడుగురు -౮ ఎనిమిదుగురు -౯ తొమ్మిదుగురు -౧౦ పది మంది +ఇద్దరు ౨ +ముగ్గురు ౩ +నలుగురు ౪ +ఐదుగురు ౫ +ఆరుగురు ౬ +ఏడుగురు ౭ +ఎనిమిదుగురు ౮ +తొమ్మిదుగురు ౯ +పది మంది ౧౦ diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred.tsv index c07663ad7..5e01db331 100644 --- a/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred.tsv +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred.tsv @@ -1,5 +1,5 @@ - వందల - వందలు -౧ నూట -౧ వంద -౧ నూరు +వందల +వందలు +నూట ౧ +వంద ౧ +నూరు ౧ diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/lakh.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/lakh.tsv index 8c2c8cc85..32276c8e4 100644 --- a/nemo_text_processing/inverse_text_normalization/te/data/numbers/lakh.tsv +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/lakh.tsv @@ -1,4 +1,4 @@ - లక్షల - లక్షలు -౦౧ లక్ష -౦౧ లక్షా +లక్షల +లక్షలు +లక్ష ౦౧ +లక్షా ౦౧ diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/special_numbers.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/special_numbers.tsv index 87013e433..49da8c121 100644 --- a/nemo_text_processing/inverse_text_normalization/te/data/numbers/special_numbers.tsv +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/special_numbers.tsv @@ -1,2 +1,2 @@ -౧౯ పంతొమ్మిది -౧ ఒక +పంతొమ్మిది ౧౯ +ఒక ౧ diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/teens_and_ties.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/teens_and_ties.tsv index 5c0ca90ed..1588f202b 100644 --- a/nemo_text_processing/inverse_text_normalization/te/data/numbers/teens_and_ties.tsv +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/teens_and_ties.tsv @@ -1,18 +1,18 @@ -౧౦ పది -౧౧ పదకొండు -౧౨ పన్నెండు -౧౩ పదమూడు -౧౪ పద్నాలుగు -౧౫ పదిహేను -౧౬ పదహారు -౧౭ పదిహేడు -౧౮ పద్దెనిమిది -౧౯ పందొమ్మిది -౨౦ ఇరవై -౩౦ ముప్పై -౪౦ నలభై -౫౦ యాభై -౬౦ అరవై -౭౦ డెబ్బై -౮౦ ఎనభై -౯౦ తొంభై \ No newline at end of file +పది ౧౦ +పదకొండు ౧౧ +పన్నెండు ౧౨ +పదమూడు ౧౩ +పద్నాలుగు ౧౪ +పదిహేను ౧౫ +పదహారు ౧౬ +పదిహేడు ౧౭ +పద్దెనిమిది ౧౮ +పందొమ్మిది ౧౯ +ఇరవై ౨౦ +ముప్పై ౩౦ +నలభై ౪౦ +యాభై ౫౦ +అరవై ౬౦ +డెబ్బై ౭౦ +ఎనభై ౮౦ +తొంభై ౯౦ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousand.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousand.tsv index 717da5c6b..a8b17f293 100644 --- a/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousand.tsv +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousand.tsv @@ -1,3 +1,3 @@ - వేల - వేలు -౦౧ వెయ్యి +వేల +వేలు +వెయ్యి ౦౧ diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/ties_prefix.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/ties_prefix.tsv index 78949df54..290439a24 100644 --- a/nemo_text_processing/inverse_text_normalization/te/data/numbers/ties_prefix.tsv +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/ties_prefix.tsv @@ -1,8 +1,8 @@ -౨ ఇరవై -౩ ముప్పై -౪ నలభై -౫ యాభై -౬ అరవై -౭ డెబ్బై -౮ ఎనభై -౯ తొంభై +ఇరవై ౨ +ముప్పై ౩ +నలభై ౪ +యాభై ౫ +అరవై ౬ +డెబ్బై ౭ +ఎనభై ౮ +తొంభై ౯ diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/zero.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/zero.tsv index 5f95b48d6..0a4ee4c8b 100644 --- a/nemo_text_processing/inverse_text_normalization/te/data/numbers/zero.tsv +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/zero.tsv @@ -1 +1 @@ -౦ సున్నా \ No newline at end of file +సున్నా ౦ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/graph_utils.py b/nemo_text_processing/inverse_text_normalization/te/graph_utils.py index 15d6616c1..f5eef1639 100644 --- a/nemo_text_processing/inverse_text_normalization/te/graph_utils.py +++ b/nemo_text_processing/inverse_text_normalization/te/graph_utils.py @@ -1,5 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# Copyright 2015 and onwards Google, Inc +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/te/taggers/__init__.py index 4fc25d0d3..963711f1c 100644 --- a/nemo_text_processing/inverse_text_normalization/te/taggers/__init__.py +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/te/taggers/cardinal.py index d9bf0864a..d99082fc3 100644 --- a/nemo_text_processing/inverse_text_normalization/te/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/cardinal.py @@ -1,5 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# Copyright 2026 and onwards Google, Inc. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -42,48 +41,39 @@ class CardinalFst(GraphFst): def __init__(self, input_case: str = INPUT_LOWER_CASED): super().__init__(name="cardinal", kind="classify") - self.input_case = input_case - graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() - graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert() - graph_teens_and_ties = pynini.string_file(get_abs_path("data/numbers/teens_and_ties.tsv")).invert() - graph_ties_prefix = pynini.string_file(get_abs_path("data/numbers/ties_prefix.tsv")).invert() + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_teens_and_ties = pynini.string_file(get_abs_path("data/numbers/teens_and_ties.tsv")) + graph_ties_prefix = pynini.string_file(get_abs_path("data/numbers/ties_prefix.tsv")) + graph_people = pynini.string_file(get_abs_path("data/numbers/digit_people.tsv")) + graph_special = pynini.string_file(get_abs_path("data/numbers/special_numbers.tsv")) + graph_case_suffix = pynini.string_file(get_abs_path("data/numbers/case_suffix.tsv")) + graph_hundred = pynini.string_file(get_abs_path("data/numbers/hundred.tsv")) + graph_thousand = pynini.string_file(get_abs_path("data/numbers/thousand.tsv")) + graph_lakh = pynini.string_file(get_abs_path("data/numbers/lakh.tsv")) + graph_crore = pynini.string_file(get_abs_path("data/numbers/crore.tsv")) - graph_people = pynini.string_file(get_abs_path("data/numbers/digit_people.tsv")).invert() - graph_special = pynini.string_file(get_abs_path("data/numbers/special_numbers.tsv")).invert() two_te_digits = NEMO_TE_DIGIT + NEMO_TE_DIGIT graph_digit |= (graph_people | graph_special) @ NEMO_TE_DIGIT - self.graph_zero = graph_zero - self.graph_digit = graph_digit - graph_two_digit = ( graph_teens_and_ties | (graph_ties_prefix + delete_space + graph_digit) | (pynutil.insert("౦") + graph_digit) | ((graph_people | graph_special) @ two_te_digits) ) - self.graph_two_digit = graph_two_digit two_digit_or_zeros = graph_two_digit | pynutil.insert("౦౦") - graph_case_suffix = pynini.string_file(get_abs_path("data/numbers/case_suffix.tsv")) optional_case = pynini.closure(graph_case_suffix, 0, 1) - optional_one = pynini.closure(pynutil.delete("ఒక") + delete_space, 0, 1) - graph_hundred = pynini.string_file(get_abs_path("data/numbers/hundred.tsv")).invert() delete_hundred = graph_hundred @ pynini.accep("") one_hundred_value = graph_hundred @ pynini.closure(NEMO_TE_DIGIT, 1) - - graph_thousand = pynini.string_file(get_abs_path("data/numbers/thousand.tsv")).invert() delete_thousand = graph_thousand @ pynini.accep("") one_thousand_value = graph_thousand @ pynini.closure(NEMO_TE_DIGIT, 1) - - graph_lakh = pynini.string_file(get_abs_path("data/numbers/lakh.tsv")).invert() delete_lakh = graph_lakh @ pynini.accep("") one_lakh_value = graph_lakh @ pynini.closure(NEMO_TE_DIGIT, 1) - - graph_crore = pynini.string_file(get_abs_path("data/numbers/crore.tsv")).invert() delete_crore = graph_crore @ pynini.accep("") one_crore_value = graph_crore @ pynini.closure(NEMO_TE_DIGIT, 1) @@ -91,7 +81,6 @@ def __init__(self, input_case: str = INPUT_LOWER_CASED): (graph_digit + delete_space + delete_hundred) | (optional_one + one_hundred_value) | pynutil.insert("౦") ) graph_hundreds = hundred_prefix + delete_space + two_digit_or_zeros - self.graph_hundreds = graph_hundreds thousand_block = ( (graph_two_digit + delete_space + delete_thousand) @@ -125,10 +114,7 @@ def __init__(self, input_case: str = INPUT_LOWER_CASED): graph = (graph_number | graph_zero) + optional_case graph |= graph_leading_zeros - - self.graph_no_exception = graph - - self.graph = graph.optimize() + graph = graph.optimize() optional_minus_graph = pynini.closure( pynutil.insert("negative: ") + pynini.cross(MINUS, "\"-\"") + NEMO_SPACE, @@ -136,7 +122,7 @@ def __init__(self, input_case: str = INPUT_LOWER_CASED): 1, ) - final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + graph + pynutil.insert("\"") final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/te/taggers/tokenize_and_classify.py index a85323680..5b9538d82 100644 --- a/nemo_text_processing/inverse_text_normalization/te/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/tokenize_and_classify.py @@ -1,5 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# Copyright 2026 and onwards Google, Inc. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/word.py b/nemo_text_processing/inverse_text_normalization/te/taggers/word.py index 95045582a..4e78aed4d 100644 --- a/nemo_text_processing/inverse_text_normalization/te/taggers/word.py +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/word.py @@ -1,5 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# Copyright 2026 and onwards Google, Inc. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/utils.py b/nemo_text_processing/inverse_text_normalization/te/utils.py index 5830b50d0..18843a931 100644 --- a/nemo_text_processing/inverse_text_normalization/te/utils.py +++ b/nemo_text_processing/inverse_text_normalization/te/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/__init__.py index 4fc25d0d3..963711f1c 100644 --- a/nemo_text_processing/inverse_text_normalization/te/verbalizers/__init__.py +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/cardinal.py index cf6c3a903..02c0189cf 100644 --- a/nemo_text_processing/inverse_text_normalization/te/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/cardinal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize.py index cff457b72..f41d28a79 100644 --- a/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize.py +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize.py @@ -1,5 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# Copyright 2026 and onwards Google, Inc. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize_final.py index c097ebee7..9bf7ddb05 100644 --- a/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize_final.py +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize_final.py @@ -1,5 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# Copyright 2026 and onwards Google, Inc. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/word.py index 28603646c..b89bf6de6 100644 --- a/nemo_text_processing/inverse_text_normalization/te/verbalizers/word.py +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/word.py @@ -1,5 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. -# Copyright 2026 and onwards Google, Inc. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/nemo_text_processing/te/__init__.py b/tests/nemo_text_processing/te/__init__.py index 4fc25d0d3..963711f1c 100644 --- a/tests/nemo_text_processing/te/__init__.py +++ b/tests/nemo_text_processing/te/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/nemo_text_processing/te/test_cardinal.py b/tests/nemo_text_processing/te/test_cardinal.py index 85a10a177..520c442c6 100644 --- a/tests/nemo_text_processing/te/test_cardinal.py +++ b/tests/nemo_text_processing/te/test_cardinal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,24 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pynini import pytest from parameterized import parameterized -from nemo_text_processing.inverse_text_normalization.te.taggers.tokenize_and_classify import ClassifyFst -from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize_final import VerbalizeFinalFst +from nemo_text_processing.inverse_text_normalization.inverse_normalize import InverseNormalizer -from ..utils import parse_test_case_file +from ..utils import CACHE_DIR, parse_test_case_file class TestCardinal: - tagger = ClassifyFst(overwrite_cache=True).fst - verbalizer = VerbalizeFinalFst().fst + inverse_normalizer = InverseNormalizer(lang='te', cache_dir=CACHE_DIR, overwrite_cache=False) @parameterized.expand(parse_test_case_file('te/data_inverse_text_normalization/test_cases_cardinal.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit def test_denorm(self, test_input, expected): - tagged = pynini.shortestpath(test_input @ self.tagger).string() - pred = pynini.shortestpath(tagged @ self.verbalizer).string() + pred = self.inverse_normalizer.inverse_normalize(test_input, verbose=False) assert pred == expected diff --git a/tools/text_processing_deployment/pynini_export.py b/tools/text_processing_deployment/pynini_export.py index f183fdf0c..2a540b1d7 100644 --- a/tools/text_processing_deployment/pynini_export.py +++ b/tools/text_processing_deployment/pynini_export.py @@ -141,7 +141,7 @@ def parse_args(): if __name__ == '__main__': args = parse_args() - if args.language in ['ru', 'es_en', 'hi_en', 'mr', 'te'] and args.grammars == 'tn_grammars': + if args.language in ['ru', 'es_en', 'hi_en', 'mr'] and args.grammars == 'tn_grammars': raise ValueError('Only ITN grammars could be deployed in Sparrowhawk for the selected languages.') TNPostProcessingFst = None ITNPostProcessingFst = None From 514d54937ced492cb6627f9fb4112ff2ac792ed4 Mon Sep 17 00:00:00 2001 From: jaya-TN Date: Thu, 30 Jul 2026 09:07:02 +0530 Subject: [PATCH 3/3] DCO Remediation Commit for jaya-TN I, jaya-TN , hereby add my Signed-off-by to this commit: df866462cc0f2beca2b0cbd7569f8d8de0efb5c7 I, jaya-TN , hereby add my Signed-off-by to this commit: ddc54ade8f3d0abf8265ff216f6765000cf9807f Signed-off-by: jaya-TN