Hi TN Roman: Expanding the numeral support for roman class till 3999 with compositional approach

Jenkinsfile

@@ -28,7 +28,7 @@ pipeline {
     MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1'
     JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1'
     KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-23-26-0'
-    HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-10-26-0'
+    HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/28-07-26-0'
     DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
   }
   stages {

nemo_text_processing/text_normalization/hi/data/roman/roman_components.tsv

@@ -0,0 +1,30 @@
+I	1
+II	2
+III	3
+IV	4
+V	5
+VI	6
+VII	7
+VIII	8
+IX	9
+X	1
+XX	2
+XXX	3
+XL	4
+L	5
+LX	6
+LXX	7
+LXXX	8
+XC	9
+C	1
+CC	2
+CCC	3
+CD	4
+D	5
+DC	6
+DCC	7
+DCCC	8
+CM	9
+M	1
+MM	2
+MMM	3

nemo_text_processing/text_normalization/hi/taggers/roman.py

@@ -32,11 +32,25 @@ class RomanFst(GraphFst):
             for False multiple transduction are generated (used for audio-based normalization)
     """
 
-    def __init__(self, deterministic: bool = True):
+    def __init__(self, cardinal: GraphFst, deterministic: bool = True):
         super().__init__(name="roman", kind="classify", deterministic=deterministic)
 
-        roman_graph = pynini.string_file(get_abs_path("data/roman/roman_to_spoken.tsv")).optimize()
-        roman_numeral_only = pynini.project(roman_graph, "input").optimize()
+        components = load_labels(get_abs_path("data/roman/roman_components.tsv"))
+        ones = pynini.string_map(components[0:9]).optimize()
+        tens = pynini.string_map(components[9:18]).optimize()
+        hundreds = pynini.string_map(components[18:27]).optimize()
+        thousands = pynini.string_map(components[27:30]).optimize()
+
+        zero = pynutil.insert("0")
+        opt_hundreds = hundreds | zero
+        opt_tens = tens | zero
+        opt_ones = ones | zero
+
+        roman_to_arabic = pynini.union(
+            thousands + opt_hundreds + opt_tens + opt_ones, hundreds + opt_tens + opt_ones, tens + opt_ones, ones
+        ).optimize()
+
+        roman_to_spoken_fst = pynini.compose(roman_to_arabic, cardinal.graph_without_leading_zeros).optimize()
 
         devanagari_chars = pynini.project(
             pynini.string_file(get_abs_path("data/serial/chars.tsv")), "input"
@@ -58,14 +72,14 @@ def __init__(self, deterministic: bool = True):
             + pynutil.delete(separator)
             + insert_space
             + pynutil.insert('integer: "')
-            + roman_numeral_only
+            + roman_to_spoken_fst
             + pynutil.insert('"')
         ).optimize()
 
         numeral_before_key = (
             pynutil.insert("preserve_order: true ")
             + pynutil.insert('integer: "')
-            + roman_numeral_only
+            + roman_to_spoken_fst
             + pynutil.insert('"')
             + pynutil.delete(separator)
             + insert_space
@@ -74,45 +88,30 @@ def __init__(self, deterministic: bool = True):
             + pynutil.insert('"')
         ).optimize()
 
-        roman_rows = load_labels(get_abs_path("data/roman/roman_to_spoken.tsv"))
-        numerals_by_len_desc = sorted((n for n, _ in roman_rows), key=len, reverse=True)
-
         exception_rows = load_labels(get_abs_path("data/roman/roman_ordinal_exceptions.tsv"))
-        exception_fused_set = {fused for fused, _ in exception_rows}
-
-        suffix_rows_raw = load_labels(get_abs_path("data/ordinal/suffixes.tsv")) + load_labels(
-            get_abs_path("data/ordinal/suffixes_map.tsv")
-        )
 
         exception_graphs = []
         for fused, spoken_word in exception_rows:
-            matched_numeral = next(c for c in numerals_by_len_desc if fused.startswith(c))
             exception_graphs.append(
-                pynutil.insert('integer: "' + matched_numeral + '"')
+                pynutil.insert('integer: "-"')
                 + insert_space
                 + pynutil.insert('default_ordinal: "' + spoken_word + '"')
                 + pynutil.delete(fused)
             )
         glued_ordinal_exceptions_graph = pynini.union(*exception_graphs).optimize()
 
-        regular_row_graphs = []
-        for numeral, spoken in roman_rows:
-            for row in suffix_rows_raw:
-
-                suffix_input = row[0]
-                suffix_output = row[1] if len(row) > 1 else row[0]
-
-                fused = numeral + suffix_input
-                if fused in exception_fused_set:
-                    continue
-                spoken_ordinal = spoken + suffix_output
-                regular_row_graphs.append(
-                    pynutil.insert('integer: "' + numeral + '"')
-                    + insert_space
-                    + pynutil.insert('default_ordinal: "' + spoken_ordinal + '"')
-                    + pynutil.delete(fused)
-                )
-        glued_ordinal_regular_graph = pynini.union(*regular_row_graphs).optimize()
+        suffixes_fst = pynini.union(
+            pynini.string_file(get_abs_path("data/ordinal/suffixes.tsv")),
+            pynini.string_file(get_abs_path("data/ordinal/suffixes_map.tsv")),
+        ).optimize()
+
+        glued_ordinal_regular_graph = (
+            pynutil.insert('integer: "-"')
+            + insert_space
+            + pynutil.insert('default_ordinal: "')
+            + (roman_to_spoken_fst + suffixes_fst)
+            + pynutil.insert('"')
+        ).optimize()
 
         roman_glued_ordinal_fields = pynini.union(
             pynutil.add_weight(glued_ordinal_exceptions_graph, -0.1),

nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py

@@ -116,7 +116,7 @@ def __init__(
             word = WordFst(punctuation=punctuation, deterministic=deterministic)
             word_graph = word.fst
 
-            roman = RomanFst(deterministic=deterministic)
+            roman = RomanFst(cardinal=cardinal, deterministic=deterministic)
             roman_graph = roman.fst
 
             telephone = TelephoneFst()

nemo_text_processing/text_normalization/hi/verbalizers/roman.py

@@ -40,13 +40,11 @@ class RomanFst(GraphFst):
     def __init__(self, deterministic: bool = True):
         super().__init__(name="roman", kind="verbalize", deterministic=deterministic)
 
-        roman_to_spoken = pynini.string_file(get_abs_path("data/roman/roman_to_spoken.tsv")).optimize()
-
         key_cardinal = (
             pynutil.delete('key_cardinal: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"')
         ).optimize()
 
-        integer = (pynutil.delete('integer: "') + roman_to_spoken + pynutil.delete('"')).optimize()
+        integer = (pynutil.delete('integer: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"')).optimize()
 
         default_ordinal = (
             pynutil.delete('default_ordinal: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"')

tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt

@@ -19,5 +19,11 @@ XIIवीं कक्षा की परीक्षा~बारहवीं
 अध्याय IV के प्रश्न~अध्याय चार के प्रश्न
 IVथी कक्षा के विद्यार्थी~चौथी कक्षा के विद्यार्थी
 XC विद्यार्थी~नब्बे विद्यार्थी
-LIII वा गणतंत्र दिन~तिरपन वा गणतंत्र दिन
-भाग-XCIX~भाग निन्यानवे
+LIII वा गणतंत्र दिन~तिरेपन वा गणतंत्र दिन
+भाग-XCIX~भाग निन्यानबे
+भाग-CDXLIV~भाग चार सौ चौवालीस
+रॉकेट-MMM~रॉकेट तीन हज़ार
+अध्याय MCMXCIX~अध्याय एक हज़ार नौ सौ निन्यानबे
+MMXXIVवीं वर्षगांठ~दो हज़ार चौबीसवीं वर्षगांठ
+MCMXCIX वा स्थापना दिवस~एक हज़ार नौ सौ निन्यानबे वा स्थापना दिवस
+MMMCMXCIX सैनिक~तीन हज़ार नौ सौ निन्यानबे सैनिक