From d0c3aab5d85d5853dfd3cb3be344f5880988927e Mon Sep 17 00:00:00 2001 From: Shreyas Pawar Date: Tue, 28 Jul 2026 04:24:08 +0000 Subject: [PATCH 1/3] feat: implement dynamic compositional Roman FST with support upto 3999 Signed-off-by: Shreyas Pawar --- .../hi/data/roman/roman_components.tsv | 30 ++++++ .../hi/data/roman/roman_to_spoken.tsv | 100 ------------------ .../text_normalization/hi/taggers/roman.py | 70 ++++++------ .../hi/taggers/tokenize_and_classify.py | 2 +- .../hi/verbalizers/roman.py | 6 +- .../test_cases_roman.txt | 10 +- 6 files changed, 78 insertions(+), 140 deletions(-) create mode 100644 nemo_text_processing/text_normalization/hi/data/roman/roman_components.tsv delete mode 100644 nemo_text_processing/text_normalization/hi/data/roman/roman_to_spoken.tsv diff --git a/nemo_text_processing/text_normalization/hi/data/roman/roman_components.tsv b/nemo_text_processing/text_normalization/hi/data/roman/roman_components.tsv new file mode 100644 index 000000000..b6cbb191f --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/roman/roman_components.tsv @@ -0,0 +1,30 @@ +I 1 +II 2 +III 3 +IV 4 +V 5 +VI 6 +VII 7 +VIII 8 +IX 9 +X 1 +XX 2 +XXX 3 +XL 4 +L 5 +LX 6 +LXX 7 +LXXX 8 +XC 9 +C 1 +CC 2 +CCC 3 +CD 4 +D 5 +DC 6 +DCC 7 +DCCC 8 +CM 9 +M 1 +MM 2 +MMM 3 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/roman/roman_to_spoken.tsv b/nemo_text_processing/text_normalization/hi/data/roman/roman_to_spoken.tsv deleted file mode 100644 index 69b760196..000000000 --- a/nemo_text_processing/text_normalization/hi/data/roman/roman_to_spoken.tsv +++ /dev/null @@ -1,100 +0,0 @@ -I एक -II दो -III तीन -IV चार -V पाँच -VI छह -VII सात -VIII आठ -IX नौ -X दस -XI ग्यारह -XII बारह -XIII तेरह -XIV चौदह -XV पंद्रह -XVI सोलह -XVII सत्रह -XVIII अठारह -XIX उन्नीस -XX बीस -XXI इक्कीस -XXII बाईस -XXIII तेईस -XXIV चौबीस -XXV पच्चीस -XXVI छब्बीस -XXVII सत्ताईस -XXVIII अट्ठाईस -XXIX उनतीस -XXX तीस -XXXI इकतीस -XXXII बत्तीस -XXXIII तैंतीस -XXXIV चौंतीस -XXXV पैंतीस -XXXVI छत्तीस -XXXVII सैंतीस -XXXVIII अड़तीस -XXXIX उनचालीस -XL चालीस -XLI इकतालीस -XLII बयालीस -XLIII तैंतालीस -XLIV चौंतालीस -XLV पैंतालीस -XLVI छियालीस -XLVII सैंतालीस -XLVIII अड़तालीस -XLIX उनचास -L पचास -LI इक्यावन -LII बावन -LIII तिरपन -LIV चौवन -LV पचपन -LVI छप्पन -LVII सत्तावन -LVIII अट्ठावन -LIX उनसठ -LX साठ -LXI इकसठ -LXII बासठ -LXIII तिरसठ -LXIV चौंसठ -LXV पैंसठ -LXVI छियासठ -LXVII सड़सठ -LXVIII अड़सठ -LXIX उनहत्तर -LXX सत्तर -LXXI इकहत्तर -LXXII बहत्तर -LXXIII तिहत्तर -LXXIV चौहत्तर -LXXV पचहत्तर -LXXVI छिहत्तर -LXXVII सतहत्तर -LXXVIII अठहत्तर -LXXIX उनासी -LXXX अस्सी -LXXXI इक्यासी -LXXXII बयासी -LXXXIII तिरासी -LXXXIV चौरासी -LXXXV पचासी -LXXXVI छियासी -LXXXVII सत्तासी -LXXXVIII अट्ठासी -LXXXIX नवासी -XC नब्बे -XCI इक्यानवे -XCII बानवे -XCIII तिरानवे -XCIV चौरानवे -XCV पचानवे -XCVI छियानवे -XCVII सत्तानवे -XCVIII अट्ठानवे -XCIX निन्यानवे -C एक सौ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/taggers/roman.py b/nemo_text_processing/text_normalization/hi/taggers/roman.py index ea8d259fe..a1cf8506b 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/roman.py +++ b/nemo_text_processing/text_normalization/hi/taggers/roman.py @@ -32,11 +32,28 @@ class RomanFst(GraphFst): for False multiple transduction are generated (used for audio-based normalization) """ - def __init__(self, deterministic: bool = True): + def __init__(self, cardinal: GraphFst, deterministic: bool = True): super().__init__(name="roman", kind="classify", deterministic=deterministic) - roman_graph = pynini.string_file(get_abs_path("data/roman/roman_to_spoken.tsv")).optimize() - roman_numeral_only = pynini.project(roman_graph, "input").optimize() + components = load_labels(get_abs_path("data/roman/roman_components.tsv")) + ones = pynini.string_map(components[0:9]).optimize() + tens = pynini.string_map(components[9:18]).optimize() + hundreds = pynini.string_map(components[18:27]).optimize() + thousands = pynini.string_map(components[27:30]).optimize() + + zero = pynutil.insert("0") + opt_hundreds = hundreds | zero + opt_tens = tens | zero + opt_ones = ones | zero + + roman_to_arabic = pynini.union( + thousands + opt_hundreds + opt_tens + opt_ones, + hundreds + opt_tens + opt_ones, + tens + opt_ones, + ones + ).optimize() + + roman_to_spoken_fst = pynini.compose(roman_to_arabic, cardinal.graph_without_leading_zeros).optimize() devanagari_chars = pynini.project( pynini.string_file(get_abs_path("data/serial/chars.tsv")), "input" @@ -58,14 +75,14 @@ def __init__(self, deterministic: bool = True): + pynutil.delete(separator) + insert_space + pynutil.insert('integer: "') - + roman_numeral_only + + roman_to_spoken_fst + pynutil.insert('"') ).optimize() numeral_before_key = ( pynutil.insert("preserve_order: true ") + pynutil.insert('integer: "') - + roman_numeral_only + + roman_to_spoken_fst + pynutil.insert('"') + pynutil.delete(separator) + insert_space @@ -74,45 +91,30 @@ def __init__(self, deterministic: bool = True): + pynutil.insert('"') ).optimize() - roman_rows = load_labels(get_abs_path("data/roman/roman_to_spoken.tsv")) - numerals_by_len_desc = sorted((n for n, _ in roman_rows), key=len, reverse=True) - exception_rows = load_labels(get_abs_path("data/roman/roman_ordinal_exceptions.tsv")) - exception_fused_set = {fused for fused, _ in exception_rows} - - suffix_rows_raw = load_labels(get_abs_path("data/ordinal/suffixes.tsv")) + load_labels( - get_abs_path("data/ordinal/suffixes_map.tsv") - ) - + exception_graphs = [] for fused, spoken_word in exception_rows: - matched_numeral = next(c for c in numerals_by_len_desc if fused.startswith(c)) exception_graphs.append( - pynutil.insert('integer: "' + matched_numeral + '"') + pynutil.insert('integer: "-"') + insert_space + pynutil.insert('default_ordinal: "' + spoken_word + '"') + pynutil.delete(fused) ) glued_ordinal_exceptions_graph = pynini.union(*exception_graphs).optimize() - regular_row_graphs = [] - for numeral, spoken in roman_rows: - for row in suffix_rows_raw: - - suffix_input = row[0] - suffix_output = row[1] if len(row) > 1 else row[0] - - fused = numeral + suffix_input - if fused in exception_fused_set: - continue - spoken_ordinal = spoken + suffix_output - regular_row_graphs.append( - pynutil.insert('integer: "' + numeral + '"') - + insert_space - + pynutil.insert('default_ordinal: "' + spoken_ordinal + '"') - + pynutil.delete(fused) - ) - glued_ordinal_regular_graph = pynini.union(*regular_row_graphs).optimize() + suffixes_fst = pynini.union( + pynini.string_file(get_abs_path("data/ordinal/suffixes.tsv")), + pynini.string_file(get_abs_path("data/ordinal/suffixes_map.tsv")) + ).optimize() + + glued_ordinal_regular_graph = ( + pynutil.insert('integer: "-"') + + insert_space + + pynutil.insert('default_ordinal: "') + + (roman_to_spoken_fst + suffixes_fst) + + pynutil.insert('"') + ).optimize() roman_glued_ordinal_fields = pynini.union( pynutil.add_weight(glued_ordinal_exceptions_graph, -0.1), diff --git a/nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py index 0e8d7fc73..d7af12f1d 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py @@ -116,7 +116,7 @@ def __init__( word = WordFst(punctuation=punctuation, deterministic=deterministic) word_graph = word.fst - roman = RomanFst(deterministic=deterministic) + roman = RomanFst(cardinal=cardinal, deterministic=deterministic) roman_graph = roman.fst telephone = TelephoneFst() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/roman.py b/nemo_text_processing/text_normalization/hi/verbalizers/roman.py index c28084a77..eaed3b8d0 100644 --- a/nemo_text_processing/text_normalization/hi/verbalizers/roman.py +++ b/nemo_text_processing/text_normalization/hi/verbalizers/roman.py @@ -40,13 +40,13 @@ class RomanFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="roman", kind="verbalize", deterministic=deterministic) - roman_to_spoken = pynini.string_file(get_abs_path("data/roman/roman_to_spoken.tsv")).optimize() - key_cardinal = ( pynutil.delete('key_cardinal: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') ).optimize() - integer = (pynutil.delete('integer: "') + roman_to_spoken + pynutil.delete('"')).optimize() + integer = ( + pynutil.delete('integer: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + ).optimize() default_ordinal = ( pynutil.delete('default_ordinal: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt index 00f697a89..2f2d0d193 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt @@ -19,5 +19,11 @@ XIIवीं कक्षा की परीक्षा~बारहवीं अध्याय IV के प्रश्न~अध्याय चार के प्रश्न IVथी कक्षा के विद्यार्थी~चौथी कक्षा के विद्यार्थी XC विद्यार्थी~नब्बे विद्यार्थी -LIII वा गणतंत्र दिन~तिरपन वा गणतंत्र दिन -भाग-XCIX~भाग निन्यानवे \ No newline at end of file +LIII वा गणतंत्र दिन~तिरेपन वा गणतंत्र दिन +भाग-XCIX~भाग निन्यानबे +भाग-CDXLIV~भाग चार सौ चौवालीस +रॉकेट-MMM~रॉकेट तीन हज़ार +अध्याय MCMXCIX~अध्याय एक हज़ार नौ सौ निन्यानबे +MMXXIVवीं वर्षगांठ~दो हज़ार चौबीसवीं वर्षगांठ +MCMXCIX वा स्थापना दिवस~एक हज़ार नौ सौ निन्यानबे वा स्थापना दिवस +MMMCMXCIX सैनिक~तीन हज़ार नौ सौ निन्यानबे सैनिक \ No newline at end of file From ed48f309beb6b631475c76be0e0d9caf940cc417 Mon Sep 17 00:00:00 2001 From: Shreyas Pawar Date: Tue, 28 Jul 2026 12:47:10 +0000 Subject: [PATCH 2/3] Jenkinsfile date update Signed-off-by: Shreyas Pawar --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a6abeb361..28c46a9ef 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,7 +28,7 @@ pipeline { MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-23-26-0' - HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-10-26-0' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/28-07-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { From 195a529c5d5b7bbdd41cf0b039b45c77aa350f4c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:51:18 +0000 Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/hi/taggers/roman.py | 15 ++++++--------- .../text_normalization/hi/verbalizers/roman.py | 4 +--- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/nemo_text_processing/text_normalization/hi/taggers/roman.py b/nemo_text_processing/text_normalization/hi/taggers/roman.py index a1cf8506b..566c28efc 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/roman.py +++ b/nemo_text_processing/text_normalization/hi/taggers/roman.py @@ -47,10 +47,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): opt_ones = ones | zero roman_to_arabic = pynini.union( - thousands + opt_hundreds + opt_tens + opt_ones, - hundreds + opt_tens + opt_ones, - tens + opt_ones, - ones + thousands + opt_hundreds + opt_tens + opt_ones, hundreds + opt_tens + opt_ones, tens + opt_ones, ones ).optimize() roman_to_spoken_fst = pynini.compose(roman_to_arabic, cardinal.graph_without_leading_zeros).optimize() @@ -75,14 +72,14 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): + pynutil.delete(separator) + insert_space + pynutil.insert('integer: "') - + roman_to_spoken_fst + + roman_to_spoken_fst + pynutil.insert('"') ).optimize() numeral_before_key = ( pynutil.insert("preserve_order: true ") + pynutil.insert('integer: "') - + roman_to_spoken_fst + + roman_to_spoken_fst + pynutil.insert('"') + pynutil.delete(separator) + insert_space @@ -92,11 +89,11 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): ).optimize() exception_rows = load_labels(get_abs_path("data/roman/roman_ordinal_exceptions.tsv")) - + exception_graphs = [] for fused, spoken_word in exception_rows: exception_graphs.append( - pynutil.insert('integer: "-"') + pynutil.insert('integer: "-"') + insert_space + pynutil.insert('default_ordinal: "' + spoken_word + '"') + pynutil.delete(fused) @@ -105,7 +102,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): suffixes_fst = pynini.union( pynini.string_file(get_abs_path("data/ordinal/suffixes.tsv")), - pynini.string_file(get_abs_path("data/ordinal/suffixes_map.tsv")) + pynini.string_file(get_abs_path("data/ordinal/suffixes_map.tsv")), ).optimize() glued_ordinal_regular_graph = ( diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/roman.py b/nemo_text_processing/text_normalization/hi/verbalizers/roman.py index eaed3b8d0..adb6e61c5 100644 --- a/nemo_text_processing/text_normalization/hi/verbalizers/roman.py +++ b/nemo_text_processing/text_normalization/hi/verbalizers/roman.py @@ -44,9 +44,7 @@ def __init__(self, deterministic: bool = True): pynutil.delete('key_cardinal: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') ).optimize() - integer = ( - pynutil.delete('integer: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') - ).optimize() + integer = (pynutil.delete('integer: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"')).optimize() default_ordinal = ( pynutil.delete('default_ordinal: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"')