feat: add Hindi-English code-mixed ITN with raw-field tagging

README.md

@@ -44,6 +44,7 @@ ITN converts spoken expressions into written forms, for example `十二` → `12
 | French | `fr` | Yes | Yes | `french` |
 | German | `de` | Yes | Yes | `german` |
 | Hindi | `hi` | Yes | Yes | `hindi` |
+| Hindi–English (code-mixed) | `hi_en` | No | Yes | `hindi_english` |
 | Japanese | `ja` | Yes | Yes | `japanese` |
 | Kinyarwanda | `rw` | Yes | Yes | `kinyarwanda` |
 | Korean | `ko` | Yes | Yes | `korean` |

itn/hindi_english/data/measurements.tsv

@@ -0,0 +1,2 @@
+°C	degrees celsius
+°F	degrees fahrenheit

itn/hindi_english/inverse_normalizer.py

@@ -0,0 +1,93 @@
+# Copyright (c) 2026 Zhendong Peng ([email protected])
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pynini.lib.pynutil import add_weight, delete
+
+from itn.english.inverse_normalizer import TOKEN_ORDERS as EN_TOKEN_ORDERS
+from itn.hindi.inverse_normalizer import TOKEN_ORDERS as HI_TOKEN_ORDERS
+from itn.hindi.rules.cardinal import Cardinal
+from itn.hindi.rules.date import Date
+from itn.hindi.rules.decimal import Decimal
+from itn.hindi.rules.fraction import Fraction
+from itn.hindi.rules.measure import Measure
+from itn.hindi.rules.money import Money
+from itn.hindi.rules.ordinal import Ordinal
+from itn.hindi.rules.punctuation import Punctuation
+from itn.hindi.rules.telephone import Telephone
+from itn.hindi.rules.time import Time
+from itn.hindi.rules.whitelist import Whitelist
+from itn.hindi.rules.word import Word
+from itn.hindi_english.rules import english
+from tn.processor import Processor, RuleSpec
+
+# Hindi and English tokens share names (cardinal, date, money, ...); a token is verbalized by whichever
+# language's verbalizer accepts its raw field text. The Hindi rules emit fields in written order, so only
+# the English canonical orders are needed (Hindi money keeps its own order for its single "value" field).
+TOKEN_ORDERS = dict(HI_TOKEN_ORDERS)
+TOKEN_ORDERS.update(EN_TOKEN_ORDERS)
+
+# Classifier weight of a punctuation mark glued to a token.
+PUNCT_WEIGHT = 1.1
+
+
+class InverseNormalizer(Processor):
+
+    def __init__(self, cache_dir=None, overwrite_cache=False, cased=False):
+        """``cased=True`` also accepts English spoken words with a capitalized first letter ("July twenty")."""
+        super().__init__(name="hi_en_inverse_normalizer", ordertype="itn", token_orders=TOKEN_ORDERS)
+        self.cased = cased
+        self.build_fst("hi_en_itn", cache_dir, overwrite_cache, {"cased": cased})
+
+    def build_tagger_and_verbalizer(self):
+        cased = self.cased
+        cardinal = Cardinal()
+        decimal = Decimal(cardinal)
+        en_cardinal = english.Cardinal(cased=cased)
+        en_ordinal = english.Ordinal(cardinal=en_cardinal, cased=cased)
+        en_decimal = english.Decimal(cardinal=en_cardinal, cased=cased)
+        punctuation = Punctuation()
+        word = Word()
+        rules = (
+            RuleSpec(Whitelist(), 1.01),
+            RuleSpec(english.Whitelist(cased=cased), 1.01),
+            RuleSpec(Time(cardinal), 1.1),
+            RuleSpec(english.Time(cardinal=en_cardinal, cased=cased), 1.1),
+            RuleSpec(Date(cardinal), 1.1),
+            RuleSpec(english.Date(cardinal=en_cardinal, ordinal=en_ordinal, cased=cased), 1.09),
+            RuleSpec(decimal, 1.1),
+            RuleSpec(en_decimal, 1.1),
+            RuleSpec(Fraction(cardinal), 1.1),
+            RuleSpec(Measure(cardinal, decimal), 1.1),
+            RuleSpec(english.Measure(cardinal=en_cardinal, decimal=en_decimal, cased=cased), 1.1),
+            RuleSpec(cardinal, 1.1),
+            RuleSpec(en_cardinal, 1.1),
+            RuleSpec(Ordinal(cardinal), 1.1),
+            RuleSpec(en_ordinal, 1.09),
+            RuleSpec(Money(cardinal, decimal), 1.1),
+            RuleSpec(english.Money(cardinal=en_cardinal, decimal=en_decimal, cased=cased), 1.08),
+            RuleSpec(Telephone(), 1.1),
+            RuleSpec(english.Telephone(cardinal=en_cardinal, cased=cased), 1.1),
+            RuleSpec(english.Electronic(cased=cased), 1.1),
+            RuleSpec(word, 100),
+            RuleSpec(punctuation),
+        )
+        classify = self.tagger_union(rules)
+        punct = add_weight(punctuation.tagger, PUNCT_WEIGHT)
+        token = punct.star + classify + punct.star
+        tagger = token + (delete(self.SPACE.plus) + token).star
+        self.tagger = self.DELETE_SPACE + tagger + self.DELETE_SPACE
+        self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus))
+
+        verbalize = self.verbalizer_union(rules)
+        self.verbalizer = verbalize + (self.INSERT_SPACE + verbalize).star

itn/hindi_english/rules/english.py

@@ -0,0 +1,175 @@
+# Copyright (c) 2026 Zhendong Peng ([email protected])
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Code-mixed glue around the English ITN rules.
+
+The English rules are reused unchanged except for three code-mixed details
+(Indian lakh/crore numbering, a comma before the year of a full date as in the
+Hindi date verbalizer, and plural "degrees" temperature units), plus optional
+cased input. Cased input is handled field by field: a field also accepts its
+spoken form with a capitalized first letter (the raw text, e.g. ``July``, is
+kept in the tag), and the verbalizer lower-cases that letter before converting
+and re-capitalizes a written form that starts with a lower-case letter
+(``July`` -> ``July``, ``Twenty`` -> ``20``).
+"""
+
+import string
+
+import pynini
+from pynini import closure, union
+from pynini.lib import utf8
+from pynini.lib.pynutil import delete, insert
+
+from itn.english.rules.cardinal import Cardinal as EnCardinal
+from itn.english.rules.date import Date as EnDate
+from itn.english.rules.decimal import Decimal as EnDecimal
+from itn.english.rules.electronic import Electronic as EnElectronic
+from itn.english.rules.measure import Measure as EnMeasure
+from itn.english.rules.money import Money as EnMoney
+from itn.english.rules.ordinal import Ordinal as EnOrdinal
+from itn.english.rules.telephone import Telephone as EnTelephone
+from itn.english.rules.time import Time as EnTime
+from itn.english.rules.whitelist import Whitelist as EnWhitelist
+from itn.utils import get_abs_path
+
+_SIGMA = utf8.VALID_UTF8_CHAR.star
+_LOWER = union(*string.ascii_lowercase)
+_TO_LOWER = union(*[pynini.cross(u, lo) for u, lo in zip(string.ascii_uppercase, string.ascii_lowercase)])
+# "July" -> "july"; only strings with a capitalized first letter.
+LOWER_FIRST = (_TO_LOWER + _SIGMA).optimize()
+# Written form re-capitalized only when it starts with a lower-case letter ("july" -> "July", "20" -> "20").
+RECAPITALIZE = (pynini.invert(_TO_LOWER) + _SIGMA | pynini.difference(utf8.VALID_UTF8_CHAR, _LOWER) + _SIGMA
+                | "").optimize()
+
+# Indian numbering magnitudes, highest first (each takes a 1-99 multiplier).
+INDIAN_MAGNITUDES = ("shankh", "padma", "nil", "kharab", "arab", "crore", "lakh")
+
+
+def data_file(rel_path):
+    """Loads hi_en ``written<TAB>spoken`` data as spoken -> written."""
+
+    return pynini.string_file(get_abs_path("hindi_english/data/" + rel_path)).invert().optimize()
+
+
+class CasedFields:
+    """Mixin: with ``cased=True`` every tagged field also accepts a capitalized first letter."""
+
+    def __init__(self, *args, cased=False, **kwargs):
+        self.cased = cased
+        super().__init__(*args, **kwargs)
+
+    def tag_field(self, name, graph):
+        if self.cased:
+            graph = graph | (LOWER_FIRST @ graph)
+        return super().tag_field(name, graph)
+
+    def verbalize_field(self, name, graph=None, leading_space=False):
+        if self.cased and graph is not None:
+            graph = graph | (LOWER_FIRST @ graph @ RECAPITALIZE)
+        return super().verbalize_field(name, graph, leading_space)
+
+
+class Cardinal(CasedFields, EnCardinal):
+    """English cardinal plus Indian numbering: ``twenty one crore ninety eight lakh ...`` -> ``219836593``."""
+
+    def build_tagger(self):
+        super().build_tagger()
+        indian = self.indian_graph()
+        self.graph = (self.graph | indian).optimize()
+        self.graph_no_exception = self.graph
+        self.integer = (self.integer | indian).optimize()
+        optional_minus = closure(self.tag_field("negative", self.negative) + delete(" ") + insert(" "), 0, 1)
+        self.tagger = self.add_tokens(optional_minus + self.tag_field("integer", self.integer))
+
+    def indian_graph(self):
+        digit = pynini.string_file(get_abs_path("english/data/numbers/digit.tsv"))
+        two_digit = self.graph_two_digit
+        two_padded = two_digit | insert("0") + digit
+        ds = delete(" ")
+        hundreds = digit + ds + delete("hundred") + (ds + two_digit | ds + insert("0") + digit | insert("00"))
+        up_to_999_padded = hundreds | insert("0") + two_digit | insert("00") + digit
+
+        # Parsed over the input with one leading space, so every present component is " <number> <magnitude>".
+        graph = pynini.accep("")
+        for magnitude in INDIAN_MAGNITUDES + ("thousand", ):
+            words = pynini.union(magnitude, magnitude + "s") if magnitude != "thousand" else pynini.accep(magnitude)
+            graph += ds + two_padded + ds + delete(words) | insert("00")
+        graph += ds + up_to_999_padded | insert("000")
+        graph = (insert(" ") + _SIGMA) @ graph
+        has_magnitude = _SIGMA + pynini.union(*INDIAN_MAGNITUDES) + _SIGMA
+        graph = has_magnitude @ graph
+        graph = graph @ (delete(closure("0")) + pynini.difference(self.DIGIT, "0") + closure(self.DIGIT))
+        delete_and = self.build_rule(delete("and "), " ", self.ALPHA)
+        return (delete_and @ graph).optimize()
+
+
+class Ordinal(CasedFields, EnOrdinal):
+    pass
+
+
+class Decimal(CasedFields, EnDecimal):
+    pass
+
+
+class Date(CasedFields, EnDate):
+    """English date written like the Hindi date: a comma separates day and year (``july 25, 2012``)."""
+
+    def build_verbalizer(self):
+        month = self.verbalize_field("month", self.month)
+        day = self.verbalize_field("day", self.day | self.day_dmy)
+        period = self.verbalize_field("day", self.day_quarter | self.day_half)
+        year = self.verbalize_field(
+            "year", self.year | self.year_decade | self.year_bc | self.year_century | self.year_millennium)
+        delete_po = delete("preserve_order:") + self.DELETE_SPACE + delete('"true"')
+        space = self.DELETE_SPACE + insert(" ")
+        comma_year = closure(self.DELETE_SPACE + insert(", ") + year, 0, 1)
+
+        graph = pynini.union(
+            month + space + day + comma_year,
+            month + closure(space + year, 0, 1),
+            day + space + month + comma_year,
+            period + space + year,
+            year,
+        )
+        self.verbalizer = self.delete_tokens(graph + self.DELETE_SPACE + delete_po)
+
+
+class Measure(CasedFields, EnMeasure):
+    """English measure that also accepts plural temperature units (``five degrees celsius`` -> ``5 °C``)."""
+
+    def build_tagger(self):
+        super().build_tagger()
+        self.units = (self.units | data_file("measurements.tsv")).optimize()
+        value = self.tag_field("value", self.value)
+        units = self.tag_field("units", self.units)
+        self.tagger = self.add_tokens(value + delete(" ") + insert(" ") + units)
+
+
+class Money(CasedFields, EnMoney):
+    pass
+
+
+class Time(CasedFields, EnTime):
+    pass
+
+
+class Telephone(CasedFields, EnTelephone):
+    pass
+
+
+class Electronic(CasedFields, EnElectronic):
+    pass
+
+
+class Whitelist(CasedFields, EnWhitelist):
+    pass

itn/hindi_english/test/data/test_cases_address.txt

@@ -0,0 +1,41 @@
+दिल्ली एक एक शून्य शून्य शून्य एक~दिल्ली ११०००१
+मुंबई चार शून्य शून्य शून्य शून्य एक~मुंबई ४००००१
+चेन्नई छह शून्य शून्य शून्य शून्य एक~चेन्नई ६००००१
+कोलकाता सात शून्य शून्य शून्य शून्य एक~कोलकाता ७००००१
+बेंगलुरु पाँच छह शून्य शून्य शून्य एक~बेंगलुरु ५६०००१
+सात शून्य शून्य ओक स्ट्रीट~७०० ओक स्ट्रीट
+एक एक जंगल रोड~११ जंगल रोड
+तीन शून्य एक पार्क एवेन्यू~३०१ पार्क एवेन्यू
+गली नंबर एक सात जीएकगढ़~गली नंबर १७ जीएकगढ़
+अदनान अपार्टमेंट फ्लैट नंबर पाँच पाँच~अदनान अपार्टमेंट फ्लैट नंबर ५५
+प्लॉट नंबर आठ बालाजी मार्केट~प्लॉट नंबर ८ बालाजी मार्केट
+बूथ सात शून्य, सेक्टर आठ, चंडीगढ़~बूथ ७०, सेक्टर ८, चंडीगढ़
+दो दो दो एक सदर्न स्ट्रीट~२२२१ सदर्न स्ट्रीट
+छह दो पाँच स्कूल स्ट्रीट~६२५ स्कूल स्ट्रीट
+पाँच शून्य छह स्टेट रोड~५०६ स्टेट रोड
+छह छह हाइफ़न चार, पार्कहर्स्ट रोड~६६-४, पार्कहर्स्ट रोड
+एक चार बटा तीन, मथुरा रोड~१४/३, मथुरा रोड
+अमरावती छह पाँच पाँच नौ तीन शून्य~अमरावती ६५५९३०
+अमरावती चार छह आठ दो पाँच दो~अमरावती ४६८२५२
+शिमला, हिमाचल प्रदेश पाँच नौ तीन नौ आठ आठ~शिमला, हिमाचल प्रदेश ५९३९८८
+रांची, झारखंड सात तीन छह पाँच पाँच सात~रांची, झारखंड ७३६५५७
+कोहिमा, नागालैंड चार चार आठ तीन सात सात~कोहिमा, नागालैंड ४४८३७७
+मुंबई, महाराष्ट्र आठ तीन नौ चार आठ आठ~मुंबई, महाराष्ट्र ८३९४८८
+मुंबई, महाराष्ट्र दो नौ शून्य नौ तीन सात~मुंबई, महाराष्ट्र २९०९३७
+गांधीनगर, गुजरात आठ शून्य आठ तीन सात चार~गांधीनगर, गुजरात ८०८३७४
+रायपुर, छत्तीसगढ़ एक एक शून्य छह तीन पाँच~रायपुर, छत्तीसगढ़ ११०६३५
+भोपाल, मध्य प्रदेश सात पाँच एक दो दो पाँच~भोपाल, मध्य प्रदेश ७५१२२५
+अगरतला, त्रिपुरा नौ एक पाँच तीन शून्य पाँच~अगरतला, त्रिपुरा ९१५३०५
+लखनऊ, उत्तर प्रदेश आठ शून्य दो चार आठ एक~लखनऊ, उत्तर प्रदेश ८०२४८१
+श्रीनगर, जम्मू और कश्मीर नौ छह चार पाँच दो तीन~श्रीनगर, जम्मू और कश्मीर ९६४५२३
+seven hundred oak street~700 oak street
+eleven hundred park avenue~1100 park avenue
+three hundred one main street~301 main street
+two hundred and fifty four san tomas avenue~254 san tomas avenue
+fourteen hundred twenty eight elm street~1428 elm street
+twelve hundred and eleven east avenue~1211 east avenue
+one thousand one hundred laguna court~1100 laguna court
+two hundred and fifty four arques avenue~254 arques avenue
+मुंबई one one zero zero zero one~मुंबई ११०००१
+चेन्नई six zero zero zero zero one~चेन्नई ६००००१
+दिल्ली one one zero zero zero one~दिल्ली ११०००१

itn/hindi_english/test/data/test_cases_cardinal.txt

@@ -0,0 +1,61 @@
+शून्य~०
+एक~१
+दो~२
+तीन~३
+दस~१०
+ग्यारह~११
+बारह~१२
+तेरह~१३
+चौदह~१४
+पन्द्रह~१५
+सोलह~१६
+बीस~२०
+तेईस~२३
+पचास~५०
+सत्तर~७०
+नब्बे~९०
+सौ~१००
+दो सौ~२००
+एक सौ दस~११०
+तीन सौ पचास~३५०
+हज़ार~१०००
+एक हज़ार~१०००
+दो हज़ार~२०००
+दस हज़ार~१००००
+एक लाख~१०००००
+दो लाख~२०००००
+एक करोड़~१०००००००
+पाँच करोड़~५०००००००
+साढ़े तीन सौ~३५०
+सवा दो सौ~२२५
+डेढ़ सौ~१५०
+ढाई सौ~२५०
+साढ़े तीन हज़ार~३५००
+सवा दो हज़ार~२२५०
+ढाई हज़ार~२५००
+पौने चार सौ~३७५
+पौने दो सौ~१७५
+zero~zero
+sixty~60
+nineteen~19
+two hundred and fifty four~254
+one hundred forty seven thousand four hundred fifty one~147451
+one million one hundred fifty six thousand one hundred seventy three~1156173
+one billion five hundred ninety three million seventy two thousand nine hundred sixty one~1593072961
+minus twenty five thousand thirty seven~-25037
+minus sixty~-60
+forty six thousand six hundred sixty four~46664
+two million three~2000003
+one thousand thirteen~1013
+one thousand one~1001
+one thousand one hundred~1100
+one thousand twenty six~1026
+one thousand one hundred twenty six~1126
+eleven hundred~1100
+twenty one hundred~2100
+twenty one hundred and eleven~2111
+eleven hundred twenty one~1121
+twenty one crore ninety eight lakh thirty six thousand five hundred and ninety three~219836593
+दस and twenty~१० and 20
+सौ and one hundred~१०० and 100
+two hundred और तीन सौ~200 और ३००