Ar TN/TN

nemo_text_processing/inverse_text_normalization/ar/taggers/date.py

@@ -0,0 +1,114 @@
+# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+from pynini.lib import pynutil
+
+from nemo_text_processing.text_normalization.ar.graph_utils import (
+    NEMO_DIGIT,
+    NEMO_NOT_QUOTE,
+    NEMO_SIGMA,
+    GraphFst,
+    convert_space,
+)
+from nemo_text_processing.text_normalization.ar.utils import get_abs_path, load_labels
+
+
+class DateFst(GraphFst):
+    """
+    Finite state transducer for classifying spoken Arabic date into a digit form,
+    by inverting the text-normalization date tagger and verbalizer, e.g.
+        الأول من شهر نوفمبر لعام ألفين وعشرة -> tokens { name: "01/11/2010" }
+        الأول من شهر نوفمبر -> tokens { name: "1 نوفمبر" }
+        الأول من شهر ربيع الثاني لعام ... هجري -> tokens { name: "01/04/1445 هـ" }
+
+    Args:
+        itn_cardinal_tagger: ITN cardinal tagger
+        tn_date_tagger: TN date tagger
+        tn_date_verbalizer: TN date verbalizer
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(
+        self,
+        itn_cardinal_tagger: GraphFst,
+        tn_date_tagger: GraphFst,
+        tn_date_verbalizer: GraphFst,
+        deterministic: bool = True,
+    ):
+        super().__init__(name="date", kind="classify", deterministic=deterministic)
+
+        add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT)
+        optional_delete_space = pynini.closure(NEMO_SIGMA | pynutil.delete(" ", weight=0.0001))
+        ordinals = pynini.string_file(get_abs_path("data/ordinal/ordinals_date.tsv")).invert().optimize()
+        hijri_suffixes = pynini.string_file(get_abs_path("data/months/hijri_suffixes_inverse.tsv")).optimize()
+        months_names = [x[0] for x in load_labels(get_abs_path("data/months/months_name.tsv"))]
+        months_names = pynini.union(*months_names)
+        month_to_number = tn_date_tagger.number_to_month.invert().optimize()
+        month_to_number_hijri = tn_date_tagger.month_hijri.invert().optimize()
+        tagger = tn_date_verbalizer.graph.invert().optimize()
+
+        delete_day_marker = (
+            pynutil.delete("day: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"")
+        ) @ ordinals
+
+        month_as_number = pynutil.delete("month: \"") + itn_cardinal_tagger.graph + pynutil.delete("\"")
+        month_as_string = pynutil.delete("month: \"") + months_names + pynutil.delete("\"")
+        month_name_to_number = (
+            pynutil.delete("month: \"") + (month_to_number | month_to_number_hijri) + pynutil.delete("\"")
+        )
+
+        convert_year = (tn_date_tagger.year @ optional_delete_space).invert().optimize()
+        convert_year |= (
+            (tn_date_tagger.year @ optional_delete_space).invert().optimize() + pynini.accep(" ") + hijri_suffixes
+        )
+        delete_year_marker = (
+            pynutil.delete("year: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"")
+        ) @ convert_year
+
+        # day month as string (year)
+        verbalizer = pynutil.add_weight(
+            (
+                pynini.closure(delete_day_marker + pynini.accep(" "), 0, 1)
+                + month_as_string
+                + pynini.closure(pynini.accep(" ") + delete_year_marker, 0, 1)
+            ),
+            weight=0.0001,
+        )
+
+        # day month as number (year); trailing "/" only when a year is present
+        verbalizer |= (
+            delete_day_marker @ add_leading_zero_to_double_digit
+            + pynutil.insert("/")
+            + pynutil.delete(" ")
+            + month_as_number @ add_leading_zero_to_double_digit
+            + pynini.closure(pynutil.insert("/") + pynutil.delete(" ") + delete_year_marker, 0, 1)
+        )
+        # 02/03/2022 هـ
+        verbalizer |= (
+            delete_day_marker @ add_leading_zero_to_double_digit
+            + pynutil.insert("/")
+            + pynutil.delete(" ")
+            + month_name_to_number @ add_leading_zero_to_double_digit
+            + pynini.closure(pynutil.insert("/") + pynutil.delete(" ") + delete_year_marker, 0, 1)
+        )
+
+        # year
+        verbalizer |= delete_year_marker
+
+        final_graph = tagger @ verbalizer
+
+        graph = pynutil.insert("name: \"") + convert_space(final_graph) + pynutil.insert("\"")
+        self.fst = graph.optimize()

nemo_text_processing/inverse_text_normalization/ar/taggers/fraction.py

@@ -45,9 +45,7 @@ def __init__(self, tn_cardinal: GraphFst):
 
         # create unions for special cases
         denominator_singular = pynini.union("نصف", "ثلث", "ربع", "خمس", "سدس", "سبع", "ثمن", "تسع", "عشر")
-        denominator_dual = pynini.union(
-            "نصفين", "ثلثين", "ربعين", "خمسين", "سدسين", "سبعين", "ثمنين", "تسعين", "عشرين"
-        )
+        denominator_dual = pynini.union("نصفي", "ثلثي", "ربعي", "خمسي", "سدسي", "سبعي", "ثمني", "تسعي", "عشري")
         denominator_plural = pynini.union("أخماس", "أرباع", "أثلاث", "أسداس", "أسباع", "أثمان", "أتساع", "أعشار")
         numerator_three_to_ten = pynini.union("خمسة", "سبعة", "عشرة", "ثلاثة", "أربعة", "ستة", "ثمانية", "تسعة")
 

nemo_text_processing/inverse_text_normalization/ar/taggers/time.py

@@ -0,0 +1,42 @@
+# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import pynini
+from pynini.lib import pynutil
+
+from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_SIGMA, GraphFst
+
+
+class TimeFst(GraphFst):
+    """
+    Finite state transducer for classifying spoken Arabic time by inverting the
+    text-normalization time verbalizer, e.g.
+        الثالثة وخمس عشرة دقيقة -> time { hours: "3" minutes: "15" }
+        الخامسة وعشر دقائق وثانيتان -> time { hours: "5" minutes: "10" seconds: "2" }
+        التاسعة صباحًا -> time { hours: "9" suffix: "صباحًا" }
+
+    Args:
+        tn_time_verbalizer: TN time verbalizer, whose .graph maps the tagged fields
+            (hours/minutes/seconds/suffix/zone) to their spoken Arabic form.
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
+    """
+
+    def __init__(self, tn_time_verbalizer: GraphFst, deterministic: bool = True):
+        super().__init__(name="time", kind="classify", deterministic=deterministic)
+        # allow the spoken form to be matched with flexible spacing
+        optional_delete_space = pynini.closure(NEMO_SIGMA | pynutil.delete(" ", weight=0.0001))
+        graph = (tn_time_verbalizer.graph @ optional_delete_space).invert().optimize()
+        self.fst = self.add_tokens(graph).optimize()

nemo_text_processing/inverse_text_normalization/ar/taggers/tokenize_and_classify.py

@@ -18,19 +18,24 @@
 from pynini.lib import pynutil
 
 from nemo_text_processing.inverse_text_normalization.ar.taggers.cardinal import CardinalFst
+from nemo_text_processing.inverse_text_normalization.ar.taggers.date import DateFst
 from nemo_text_processing.inverse_text_normalization.ar.taggers.decimal import DecimalFst
 from nemo_text_processing.inverse_text_normalization.ar.taggers.fraction import FractionFst
 from nemo_text_processing.inverse_text_normalization.ar.taggers.measure import MeasureFst
 from nemo_text_processing.inverse_text_normalization.ar.taggers.money import MoneyFst
 from nemo_text_processing.inverse_text_normalization.ar.taggers.punctuation import PunctuationFst
+from nemo_text_processing.inverse_text_normalization.ar.taggers.time import TimeFst
 from nemo_text_processing.inverse_text_normalization.ar.taggers.word import WordFst
 from nemo_text_processing.text_normalization.ar.graph_utils import (
     GraphFst,
     delete_extra_space,
     delete_space,
     generator_main,
 )
+from nemo_text_processing.text_normalization.ar.taggers.date import DateFst as TNDateTagger
 from nemo_text_processing.text_normalization.ar.taggers.tokenize_and_classify import ClassifyFst as TNClassifyFst
+from nemo_text_processing.text_normalization.ar.verbalizers.date import DateFst as TNDateVerbalizer
+from nemo_text_processing.text_normalization.ar.verbalizers.time import TimeFst as TNTimeVerbalizer
 from nemo_text_processing.text_normalization.en.graph_utils import INPUT_LOWER_CASED
 from nemo_text_processing.utils.logging import logger
 
@@ -85,6 +90,17 @@ def __init__(
                 deterministic=True,
             )
             measure_graph = measure.fst
+            tn_time_verbalizer = TNTimeVerbalizer(cardinal_tagger=tn_classify.cardinal, deterministic=True)
+            time = TimeFst(tn_time_verbalizer=tn_time_verbalizer)
+            time_graph = time.fst
+            tn_date_tagger = TNDateTagger(cardinal=tn_classify.cardinal, deterministic=True)
+            tn_date_verbalizer = TNDateVerbalizer(deterministic=True)
+            date = DateFst(
+                itn_cardinal_tagger=cardinal,
+                tn_date_tagger=tn_date_tagger,
+                tn_date_verbalizer=tn_date_verbalizer,
+            )
+            date_graph = date.fst
             word_graph = WordFst().fst
             punct_graph = PunctuationFst().fst
 
@@ -94,6 +110,8 @@ def __init__(
                 | pynutil.add_weight(fraction_graph, 1.1)
                 | pynutil.add_weight(money_graph, 1.1)
                 | pynutil.add_weight(measure_graph, 1.1)
+                | pynutil.add_weight(time_graph, 1.1)
+                | pynutil.add_weight(date_graph, 1.1)
                 | pynutil.add_weight(word_graph, 100)
             )
 

nemo_text_processing/inverse_text_normalization/ar/verbalizers/time.py

@@ -0,0 +1,51 @@
+# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+from pynini.lib import pynutil
+
+from nemo_text_processing.text_normalization.ar.graph_utils import NEMO_DIGIT, NEMO_NOT_QUOTE, GraphFst, delete_space
+
+
+class TimeFst(GraphFst):
+    """
+    Finite state transducer for verbalizing a time token into digits, e.g.
+        time { hours: "8" minutes: "30" } -> 08:30
+        time { hours: "8" minutes: "30" seconds: "10" } -> 08:30:10
+        time { hours: "9" suffix: "صباحًا" } -> 9 صباحًا
+        time { hours: "8" } -> 8
+    """
+
+    def __init__(self, deterministic: bool = True):
+        super().__init__(name="time", kind="verbalize", deterministic=deterministic)
+
+        add_leading_zero_to_double_digit = (NEMO_DIGIT + NEMO_DIGIT) | (pynutil.insert("0") + NEMO_DIGIT)
+        hour = pynutil.delete("hours: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"")
+        minute = pynutil.delete("minutes: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"")
+        second = pynutil.delete("seconds: \"") + pynini.closure(NEMO_DIGIT, 1) + pynutil.delete("\"")
+        zone = pynutil.delete("zone: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"")
+        optional_zone = pynini.closure(pynini.accep(" ") + zone, 0, 1)
+        suffix = pynutil.delete("suffix: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"")
+        optional_suffix = pynini.closure(pynini.accep(" ") + suffix, 0, 1)
+
+        graph_minutes_seconds = (
+            delete_space
+            + pynutil.insert(":")
+            + (minute @ add_leading_zero_to_double_digit)
+            + pynini.closure(delete_space + pynutil.insert(":") + (second @ add_leading_zero_to_double_digit), 0, 1)
+        )
+        graph_h = hour
+        graph_hms = hour @ add_leading_zero_to_double_digit + graph_minutes_seconds
+        final_graph = (graph_hms | graph_h) + optional_suffix + optional_zone
+        self.fst = self.delete_tokens(final_graph).optimize()

nemo_text_processing/inverse_text_normalization/ar/verbalizers/verbalize.py

@@ -18,6 +18,7 @@
 from nemo_text_processing.inverse_text_normalization.ar.verbalizers.fraction import FractionFst
 from nemo_text_processing.inverse_text_normalization.ar.verbalizers.measure import MeasureFst
 from nemo_text_processing.inverse_text_normalization.ar.verbalizers.money import MoneyFst
+from nemo_text_processing.inverse_text_normalization.ar.verbalizers.time import TimeFst
 from nemo_text_processing.text_normalization.ar.graph_utils import GraphFst
 
 
@@ -40,5 +41,7 @@ def __init__(self):
         money_graph = money.fst
         measure = MeasureFst(decimal=decimal, cardinal=cardinal, deterministic=True)
         measure_graph = measure.fst
-        graph = cardinal_graph | decimal_graph | fraction_graph | money_graph | measure_graph
+        time = TimeFst()
+        time_graph = time.fst
+        graph = cardinal_graph | decimal_graph | fraction_graph | money_graph | measure_graph | time_graph
         self.fst = graph

nemo_text_processing/text_normalization/ar/data/months/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/ar/data/months/abbr_to_name.tsv

@@ -0,0 +1,12 @@
+جانوري	يناير
+فابيوري	فبراير
+مارش	مارس
+أبريل	ابريل
+ماي	مايو
+جون	يونيو
+جولاي	يوليو
+أوقست	أغسطس
+سبتمبر	سبتمبر
+أوكتوبر	أكتوبر
+نوفمبر	نوفمبر
+ديسمبر	ديسمبر

nemo_text_processing/text_normalization/ar/data/months/hijri_months.tsv

@@ -0,0 +1,24 @@
+1	محرم
+2	صفر
+3	ربيع الأول
+4	ربيع الثاني
+5	جماد الأول
+6	جماد الثاني
+7	رجب
+8	شعبان
+9	رمضان
+10	شوال
+11	ذي القعدة
+12	ذي الحجة
+01	محرم
+02	صفر
+03	ربيع الأول
+04	ربيع الثاني
+05	جماد الأول
+06	جماد الثاني
+07	رجب
+08	شعبان
+09	رمضان
+10	شوال
+11	ذي القعدة
+12	ذي الحجة

nemo_text_processing/text_normalization/ar/data/months/hijri_suffixes.tsv

@@ -0,0 +1,4 @@
+هـ	هجري
+هجري	هجري
+هجريا	هجري
+ه	هجري

nemo_text_processing/text_normalization/ar/data/months/hijri_suffixes_inverse.tsv

@@ -0,0 +1,5 @@
+هـ	هـ
+هجري	هـ
+هجريا	هـ
+ه	هـ
+هجري	هـ

nemo_text_processing/text_normalization/ar/data/months/months_name.tsv

@@ -0,0 +1,33 @@
+محرم
+صفر
+ربيع الأول
+ربيع الثاني
+جماد الأول
+جماد الثاني
+رجب
+شعبان
+رمضان
+شوال
+ذي القعدة
+ذي الحجة
+جانوري
+يناير
+فابيوري
+فبراير
+مارش
+مارس
+أبريل
+ابريل
+ماي
+مايو
+جون
+يونيو
+جولاي
+يوليو
+أوقست
+أغسطس
+سبتمبر
+أوكتوبر
+أكتوبر
+نوفمبر
+ديسمبر