latest

README.md

@@ -0,0 +1,10 @@
+---
+title: Phonemize in Hebrew
+emoji: 🐢
+colorFrom: red
+colorTo: green
+sdk: gradio
+sdk_version: "4.44.0"
+app_file: app.py
+pinned: false
+---

app.py

@@ -0,0 +1,43 @@
+"""
+uv sync
+uv pip install "gradio>=5.15.0"
+uv run gradio examples/editor.py
+"""
+
+from mishkal import phonemize, normalize
+import gradio as gr
+
+default_text = """
+כָּל עֶרֶב יָאִיר (הַשֵּׁם הַמָּלֵא וּמְקוֹם הָעֲבוֹדָה שֶׁלּוֹ שְׁמוּרִים בַּמַּעֲרֶכֶת) רָץ 20 קִילוֹמֶטֶר. הוּא מְסַפֵּר לִי שֶׁזֶּה מְנַקֶּה לוֹ אֶת הָרֹאשׁ אַחֲרֵי הָעֲבוֹדָה, "שָׁעָה וָחֵצִי בְּלִי עֲבוֹדָה, אִשָּׁה וִילָדִים" כְּמוֹ שֶׁהוּא מַגְדִּיר זֹאת. אֲבָל אַחֲרֵי הַמִּקְלַחַת הוּא מַתְחִיל בְּמָה שֶׁנִּתָּן לְכַנּוֹת הָעֲבוֹדָה הַשְּׁנִיָּה שֶׁלּוֹ: לִמְצֹא לוֹ קוֹלֵגוֹת חֲדָשׁוֹת לָעֲבוֹדָה, כִּי יָאִיר הוּא כַּנִּרְאֶה הַמֶּלֶךְ שֶׁל "חָבֵר מֵבִיא חָבֵר" בְּיִשְׂרָאֵל.
+"""
+
+theme = gr.themes.Soft(font=[gr.themes.GoogleFont("Roboto")])
+
+
+def on_submit_debug(text: str) -> str:
+    phonemes = phonemize(text, preserve_punctuation=True)
+    normalized_text = normalize(text)
+    return phonemes + "\n\nNormalized:\n" + normalized_text
+
+
+def on_submit(text: str) -> str:
+    return phonemize(text, preserve_punctuation=False)
+
+
+with gr.Blocks(theme=theme) as demo:
+    text_input = gr.Textbox(
+        value=default_text, label="Text", rtl=True, elem_classes=["input"]
+    )
+    checkbox = gr.Checkbox(value=False, label="Enable Debug Mode")
+    phonemes_output = gr.Textbox(label="Phonemes")
+    submit_button = gr.Button("Create")
+
+    submit_button.click(
+        fn=lambda text, debug: on_submit_debug(text) if debug else on_submit(text),
+        inputs=[text_input, checkbox],
+        outputs=[phonemes_output],
+    )
+
+
+if __name__ == "__main__":
+    demo.launch()

mishkal/__init__.py

@@ -0,0 +1,24 @@
+"""
+High level phonemize functions
+"""
+
+from .phonemize import Phonemizer
+from .utils import normalize  # noqa: F401
+from typing import Callable
+
+phonemizer = Phonemizer()
+
+
+def phonemize(
+    text: str,
+    preserve_punctuation=True,
+    preserve_stress=True,
+    fallback: Callable[[str], str] = None,
+) -> str:
+    phonemes = phonemizer.phonemize(
+        text,
+        preserve_punctuation=preserve_punctuation,
+        preserve_stress=preserve_stress,
+        fallback=fallback,
+    )
+    return phonemes

mishkal/data/kamatz_katan.json

@@ -0,0 +1,3 @@
+{
+    "כל": "kol"
+}

mishkal/data/silver_top_307.json

@@ -0,0 +1,398 @@
+{
+    "מִשְׁפַּחַה": "miʃpaˈxa",
+    "הַמִזְבֵּחַ": "hamizˈbeax",
+    "וַאַנַחְנו": "vehaˈnaxnu",
+    "בְּעִיקַר": "beiˈkar",
+    "בְּרֹאשׁוֹ": "beroˈʃo",
+    "לִקְרוֹת": "likˈrot",
+    "בְּמַהַלַךְ": "bemahalax",
+    "בְּפִי": "befi",
+    "הַסְפִינַה": "hasfina",
+    "נִמְצְאו": "nimtseu",
+    "גְדוֹלוֹת": "gdolot",
+    "וַיֵלֵךְ": "vajelex",
+    "בִּשְׁאַר": "biʃˈar",
+    "הַגְדוֹלִים": "hagdolim",
+    "הַכְּפַר": "hakfar",
+    "לְבַקֵשׁ": "levakeʃ",
+    "כַּלכַּךְ": "kolkax",
+    "הַעֵלְיוֹן": "haeljon",
+    "תְשׁובַה": "tsuva",
+    "לְמַחֹרַת": "lemaxorat",
+    "שְׁמַע": "ʃma",
+    "וַיִקְרַא": "vajikra",
+    "פְּרִי": "pri",
+    "בְּצַד": "betsad",
+    "נִזְכַּר": "nizkar",
+    "בַּזְמַן": "bizman",
+    "לִבְנוֹת": "livnot",
+    "שִׁבְעַת": "ʃivˈat",
+    "מִלְבַד": "milvad",
+    "מִדִבְרֵי": "midivrej",
+    "לְאַן": "lean",
+    "לְמַקוֹם": "lemakom",
+    "הַשְׁאַר": "haʃˈar",
+    "לְפַנֵיךַ": "lefanexa",
+    "הִתְחִילַה": "hitxila",
+    "תִרְאֵה": "tirˈe",
+    "הַשְׁנִיַה": "haʃnija",
+    "הֵבַנְתִי": "hevanti",
+    "בִּשְׁנֵי": "biʃnej",
+    "לְהַנִיחַ": "leanix",
+    "סַבְתַא": "savta",
+    "שְׁאֵלַה": "ʃeela",
+    "תִשְׁעַה": "tiʃah",
+    "ולְאַחַר": "uleaxar",
+    "הַטְרוֹיִים": "atrojim",
+    "שְׁמַה": "ʃma",
+    "בִּשְׁעַת": "biʃnat",
+    "לְעִתִים": "leitim",
+    "יַשְׁבַה": "jaʃva",
+    "בִּדְבַרִים": "bidvarim",
+    "ושְׁאַר": "uʃar",
+    "קַרְקַע": "karka",
+    "נִכְנְסַה": "nixnesa",
+    "הַרְחֵק": "harxek",
+    "בַּעְלַה": "baˈla",
+    "קַרְאו": "karˈu",
+    "שֵׁתִהְיֵה": "ʃetihje",
+    "הַשְׁנִיַיה": "haʃnija",
+    "וַיֹאמְרו": "vajomru",
+    "הַקְטַנִים": "haktanim",
+    "שְׁתַיִם": "ʃtaim",
+    "בְּאַחַת": "beaxat",
+    "מַרְאֵה": "marˈeh",
+    "בְּדַבַר": "bedavar",
+    "לְשַׁם": "leʃam",
+    "בְּמַה": "bema",
+    "בְּשֵׁל": "beʃel",
+    "נַתְנַה": "natna",
+    "טֻמְאַת": "tumˈat",
+    "מְבַקֵשׁ": "mevakeʃ",
+    "עַנְתַה": "anta",
+    "לְעַצְמִי": "leatsmi",
+    "הַלַכְתִי": "halaxti",
+    "הַמִשְׁפַּחַה": "hamiʃpaxa",
+    "הַבְּהֵמַה": "habehema",
+    "אַרְגַן": "argan",
+    "בְּשֵׁקֵט": "beʃeket",
+    "דְמֵי": "dmej",
+    "לְאוֹרֵךְ": "leorex",
+    "רְאובֵן": "reuven",
+    "נַתְנו": "natnu",
+    "הַרְחוֹב": "harexov",
+    "שֵׁצַרִיךְ": "ʃetsarix",
+    "לִחְיוֹת": "lixjot",
+    "שְׁטַר": "ʃtar",
+    "הִרְגִישׁ": "hirgiʃ",
+    "עַלְתַה": "alta",
+    "יַדְעַה": "jadˈa",
+    "בִּפְנִים": "bifnim",
+    "לְפֵתַע": "lefeta",
+    "הַכְּלִי": "hakli",
+    "שֵׁלַךְ": "ʃelax",
+    "שִׁמְעוֹן": "ʃimˈon",
+    "ובְנֵי": "uvnej",
+    "בְּעַד": "bead",
+    "בִּמְיוחַד": "bimjuxad",
+    "הוֹלְכִים": "holxim",
+    "לְיִשְׂרַאֵל": "lejisraˈel",
+    "מְקַבֵּל": "mekabel",
+    "שְׁאֵינַן": "ʃeejnan",
+    "דְבַר": "dvar",
+    "הַשְׁלִישִׁי": "haʃliʃi",
+    "הַבְּרִית": "habrit",
+    "לֵךְ": "lex",
+    "חַזְרַה": "xazra",
+    "דְבַרַיו": "dvarav",
+    "הִרְגִישַׁה": "hirgiʃa",
+    "לִקְרַאת": "likrat",
+    "צְרִיכִים": "tsrixim",
+    "טְפַחִים": "tfaxim",
+    "הִרְגַשְׁתִי": "hirgaʃti",
+    "נַפְלַה": "nafla",
+    "בְּיַחַד": "bejaxad",
+    "אַמַרְתַ": "amart",
+    "שֵׁאַנַחְנו": "ʃeanaxnu",
+    "הַמִשְׁנֵה": "amiʃna",
+    "הַקַרְקַע": "akarka",
+    "לְמַשַׁל": "lemaʃal",
+    "לִקְרוֹא": "likro",
+    "יַדְעו": "jadu",
+    "כְּלַפֵּי": "klapej",
+    "לְאַט": "leat",
+    "שְׁאִם": "ʃeim",
+    "לִשְׁמוֹעַ": "liʃmoa",
+    "מִמְךַ": "mimex",
+    "לִבְנֵי": "livnej",
+    "בְּגַדִים": "bgadim",
+    "דַוְוקַא": "davka",
+    "לִקְנוֹת": "liknot",
+    "אַרְצַה": "artsa",
+    "לְבֵין": "lvejn",
+    "בִּשְׁנַת": "biʃnat",
+    "יַשְׁבו": "jaʃvu",
+    "לְהַבִין": "lehavin",
+    "לְסַפֵּר": "lesaper",
+    "לְצַד": "letsad",
+    "בְּעַצְמוֹ": "beatsmo",
+    "בְּנִי": "bni",
+    "לְבַדוֹ": "levado",
+    "יְכוֹלִים": "jexolim",
+    "לְכַאן": "lekan",
+    "הַרִצְפַּה": "haritspa",
+    "ה��נְנִי": "hineni",
+    "שְׁנִיַה": "ʃnija",
+    "הַלְכַה": "alxa",
+    "בְּעֵצֵם": "beetsem",
+    "מִשְׁפַּט": "miʃpat",
+    "פְּנִימַה": "pnima",
+    "יְהודִי": "jeudi",
+    "מְלַאכַה": "melaxa",
+    "מְבַרֵךְ": "mevarex",
+    "עַמְדַה": "amda",
+    "לְאַחוֹר": "leaxor",
+    "בְּיִשְׂרַאֵל": "bejisraˈel",
+    "מִבְּלִי": "mibli",
+    "לִמְצוֹא": "limtso",
+    "מְדַבֵּר": "medaber",
+    "לְמִי": "lemi",
+    "שִׂמְחַה": "simxa",
+    "סַמוךְ": "samux",
+    "בְּסוֹף": "besof",
+    "תְרומַה": "truma",
+    "בְּאֵחַד": "beexad",
+    "מִצְוַת": "mitsvat",
+    "קַרְבַּן": "korban",
+    "לְקַבֵּל": "lekabel",
+    "קְטַנִים": "ktanim",
+    "לְרֵגַע": "lerega",
+    "הַמִשְׁפַּט": "hamiʃpat",
+    "צְרִיכִין": "tsrixin",
+    "יַכוֹלְתִי": "jaxolti",
+    "בְּעֵינַיו": "beejnav",
+    "כְּלוֹמַר": "klomar",
+    "בְּאֵמְצַע": "beemtsa",
+    "סְבִיב": "sviv",
+    "בִּרְשׁות": "birʃut",
+    "כַּנִרְאֵה": "kanire",
+    "גְדוֹלִים": "gdolim",
+    "הַשׁולְחַן": "haʃulxan",
+    "שַׁאַלְתִי": "ʃaˈhalti",
+    "לְבַסוֹף": "levasof",
+    "כְּדֵרֵךְ": "kederex",
+    "מְנַת": "menat",
+    "אֵצְלוֹ": "etslo",
+    "לְבַד": "levad",
+    "לְהַגִיעַ": "leagia",
+    "לְהַגִיד": "leagid",
+    "בְּרַכַה": "brexa",
+    "דַעְתוֹ": "dahato",
+    "מַסְפִּיק": "masˈpik",
+    "בְּשַׁעַת": "beʃat",
+    "עַמְדו": "amdu",
+    "יוֹדְעִים": "jodˈim",
+    "בְּעֵינֵי": "beeˈnej",
+    "וַיְדַבֵּר": "vajedaber",
+    "נִשְׁמַע": "niʃma",
+    "בְּלִבּוֹ": "belibo",
+    "בְּמֵשֵׁךְ": "bemeʃex",
+    "לְהוֹצִיא": "lehotsi",
+    "הַקְטַנַה": "haktana",
+    "לְכַךְ": "lexax",
+    "לְזֵה": "leze",
+    "בְּעֵת": "beˈet",
+    "לִשְׁתוֹת": "liʃtot",
+    "נִשְׁבַּע": "niʃba",
+    "לְפַחוֹת": "lefaxot",
+    "שְׁלוֹשִׁים": "ʃloʃim",
+    "לְעֵבֵר": "leˈever",
+    "מִצְוַה": "mitsva",
+    "בְּרֹאשׁ": "beroʃ",
+    "אֵתְכֵם": "etxem",
+    "בְּשֵׁם": "beʃem",
+    "סְתַם": "stam",
+    "בְּכַךְ": "bexax",
+    "מִלְחַמַה": "milxama",
+    "יַצְאו": "jatsˈu",
+    "נַפְשׁוֹ": "nafʃo",
+    "כְּלֵי": "klej",
+    "נַפְשִׁי": "nafʃi",
+    "מִצְווֹת": "mitsvot",
+    "יַלְדַה": "jalda",
+    "בְּאֵרֵץ": "beerets",
+    "בְּעוֹד": "beod",
+    "שֵׁיִהְיו": "ʃejihju",
+    "הַזְקֵנַה": "hazkena",
+    "תְחִלַה": "txila",
+    "מִצְרַיִם": "mitsraim",
+    "פִּתְאֹם": "pitˈom",
+    "אַרְבַּעִים": "arbaˈim",
+    "כְּשֵׁהוא": "kʃehu",
+    "פְּלוֹנִי": "ploni",
+    "בְּדֵרֵךְ": "bederex",
+    "הַלְכו": "halxu",
+    "הַמֵמְשַׁלַה": "hamemʃala",
+    "שַׁלוֹם": "ʃaˈlom",
+    "לְמַטַה": "lemata",
+    "כְּכַל": "kexol",
+    "שַׁלוֹם": "ʃaˈlom",
+    "בְּאוֹפֵן": "beofen",
+    "לְעַצְמוֹ": "leatsmo",
+    "שְׁמוֹ": "ʃmo",
+    "לְגַמְרֵי": "legamerej",
+    "שֵׁכְּבַר": "ʃekvar",
+    "עַצְמִי": "atsmi",
+    "שְׁלֹמֹה": "ʃlomo",
+    "בַּדֵרֵךְ": "baderex",
+    "הַגְדוֹלַה": "hagdola",
+    "בְּמַקוֹם": "bemakom",
+    "נִדְמֵה": "nidme",
+    "בְּאוֹתַה": "beota",
+    "רַגְלַיו": "raglav",
+    "יְהודַה": "jehuda",
+    "מְקוֹם": "mekom",
+    "כְּנֵגֵד": "keneged",
+    "בְּסֵדֵר": "beseder",
+    "חַשַׁבְתִי": "xaʃavti",
+    "יְלַדִים": "jeladim",
+    "רְשׁות": "reʃut",
+    "ובְכַל": "vebekol",
+    "בְּהֵמַה": "behema",
+    "יְכוֹלַה": "jexola",
+    "שִׁבְעַה": "ʃivˈa",
+    "לְהַבִיא": "lehavi",
+    "כְּאִלו": "keilu",
+    "שְׁנֵיהֵם": "ʃnejhem",
+    "בִּלְתִי": "bilti",
+    "כְּלִי": "kli",
+    "נִשְׁאַר": "niʃˈar",
+    "מִסְפַּר": "mispar",
+    "וכְבַר": "vekvar",
+    "בְּשַׁעַה": "beʃaˈa",
+    "צְרִיכַה": "tsrixa",
+    "הַמְדִינַה": "amedina",
+    "יְמֵי": "jemej",
+    "שֵׁבֵּאַרְנו": "ʃebaˈarnu",
+    "שֵׁהַיְתַה": "ʃeajta",
+    "דִבְרֵי": "divrej",
+    "בִּזְמַן": "bizman",
+    "לְשֵׁם": "leʃam",
+    "לְמַעְלַה": "lemala",
+    "שְׁנַיִם": "ʃnajim",
+    "בְּזֵה": "beze",
+    "עֵשְׂרֵה": "esre",
+    "טֻמְאַה": "tumˈa",
+    "שַׁמַעְתִי": "ʃamati",
+    "בִּמְקוֹם": "bimkom",
+    "בִּגְלַל": "biglal",
+    "שְׁאַר": "ʃˈar",
+    "פִּתְאוֹם": "pitom",
+    "יִהְיו": "jihju",
+    "הִתְחִיל": "hitxil",
+    "בַּלַיְלַה": "balajla",
+    "פְּעַמִים": "peamim",
+    "קְטַנַה": "ktana",
+    "בַּרוךְ": "barux",
+    "בִּכְלַל": "bixlal",
+    "יַדַעְתִי": "jaˈdati",
+    "עַצְמַה": "atsman",
+    "הַיְלַדִים": "hajeladim",
+    "בְּיַד": "bejad",
+    "בְּלֹא": "beˈlo",
+    "לְמַעַן": "lemaan",
+    "הַכְּנֵסֵת": "hakneset",
+    "לְלֹא": "lelo",
+    "בִּשְׁבִיל": "biʃvil",
+    "שְׁהִיא": "ʃehi",
+    "לְפַנַיו": "lefanav",
+    "שְׁהֵן": "ʃehen",
+    "כְּפִי": "kfi",
+    "לְדַבֵּר": "ledaber",
+    "בְּיַדוֹ": "bejado",
+    "הַבַּיְתַה": "habajta",
+    "בְּדִיוק": "bedijuk",
+    "אוֹמְרִים": "omrim",
+    "בְּנוֹ": "bno",
+    "בְּקוֹל": "bekol",
+    "לְבֵית": "levejt",
+    "בְּאוֹתוֹ": "beoto",
+    "בִּפְנֵי": "bifnej",
+    "אַרְבַּעַה": "arbaha",
+    "הַדְבַרִים": "advarim",
+    "שֵׁלְךַ": "ʃelxa",
+    "לְתוֹךְ": "letox",
+    "שְׁלוֹשַׁה": "ʃloˈʃa",
+    "כְּלַל": "klal",
+    "לְיַד": "lejad",
+    "לְפִיכַּךְ": "lefikax",
+    "שֵׁיִהְיֵה": "ʃejihje",
+    "נִכְנַס": "nixnas",
+    "הַזְמַן": "azman",
+    "אַרְבַּע": "arba",
+    "בִּלְבַד": "bilvad",
+    "נִמְצַא": "nimtsa",
+    "תִהְיֵה": "tihje",
+    "בְּיוֹתֵר": "bejoter",
+    "מֵלֵךְ": "melex",
+    "עֵשְׂרִים": "esrim",
+    "הִלְכּוֹת": "hilxot",
+    "גְדוֹלַה": "gdola",
+    "כְּגוֹן": "kegon",
+    "וַיְהִי": "vajehi",
+    "לְעוֹלַם": "leolam",
+    "אוֹתְךַ": "otxa",
+    "פְּנֵי": "pnej",
+    "כִּמְעַט": "kimat",
+    "הַלַךְ": "alax",
+    "הַלַיְלַה": "halajla",
+    "יְדֵי": "jedej",
+    "בְּבֵית": "bevejt",
+    "אַמְרו": "amru",
+    "מִתוֹךְ": "mitox",
+    "קְצַת": "ktsat",
+    "מְעַט": "meat",
+    "לְפִי": "leˈfi",
+    "נִרְאֵה": "nirhe",
+    "בְּיוֹם": "bejom",
+    "כְּאִילו": "keilu",
+    "לִרְאוֹת": "lirhot",
+    "כְּלום": "klum",
+    "מְאוֹד": "meod",
+    "הַיְיתַה": "haˈjta",
+    "שְׁתֵי": "ʃtej",
+    "אַנַחְנו": "anaxnu",
+    "לְכַל": "lekol",
+    "לַךְ": "lax",
+    "אַמַרְתִי": "amarti",
+    "עַכְשַׁיו": "axʃav",
+    "בְּלִי": "bli",
+    "הַרְבֵּה": "arbe",
+    "עַצְמוֹ": "atsmo",
+    "דְבַרִים": "dvarim",
+    "דֵרֵךְ": "derex",
+    "לְאַחַר": "leaxar",
+    "שְׁהוא": "ʃehu",
+    "בְּתוֹךְ": "betox",
+    "שְׁאֵין": "ʃeejn",
+    "מְאֹד": "meod",
+    "בְּנֵי": "bnej",
+    "זְמַן": "zman",
+    "שְׁנֵי": "ʃnej",
+    "לִהְיוֹת": "lihjot",
+    "יִהְיֵה": "jihje",
+    "הַמֵלֵךְ": "hamelex",
+    "צַרִיךְ": "tsarix",
+    "מִפְּנֵי": "mipnej",
+    "יִשְׂרַאֵל": "jisraˈel",
+    "לִפְנֵי": "lifnej",
+    "בְּכַל": "bexol",
+    "יְהוַה": "jeave",
+    "אַמְרַה": "amra",
+    "כְּבַר": "kvar",
+    "לְךַ": "lexa",
+    "כְּדֵי": "kedej",
+    "אַךְ": "ax",
+    "כְּמוֹ": "kmo",
+    "כַּךְ": "kax"
+}

mishkal/data/symbols.json

@@ -0,0 +1,4 @@
+{
+    "₪": "ʃeˈkel",
+    "$": "doˈlar"
+}

mishkal/expander/__init__.py

@@ -0,0 +1,32 @@
+"""
+Expand dates and numbers into words with niqqud
+This happens before phonemization
+"""
+
+from .numbers import num_to_word
+from .dates import date_to_word
+from .time_to_word import time_to_word
+from .dictionary import Dictionary
+from mishkal.log import log
+
+
+class Expander:
+    def __init__(self):
+        self.dictionary = Dictionary()
+
+    def expand_text(self, text: str):
+        text = self.dictionary.expand_text(text)
+
+        words = []
+        for source_word in text.split():
+            try:
+                word = date_to_word(source_word)
+                if word == source_word:
+                    word = time_to_word(word)
+                if word == source_word:
+                    word = num_to_word(word)
+                words.append(word)
+            except Exception as e:
+                log.error(f"Failed to expand {word} with error: {e}")
+                words.append(source_word)
+        return " ".join(words)

mishkal/expander/dates.py

@@ -0,0 +1,60 @@
+from datetime import datetime
+from .numbers import num_to_word
+
+# Mapping of month names in Hebrew with diacritics (Gregorian months)
+MONTHS = {
+    1: "יָנוּאָר",
+    2: "פֶבְרוּאָר",
+    3: "מֵרְץ",
+    4: "אֵפְרִיל",
+    5: "מַאי",
+    6: "יוּנִי",
+    7: "יוּלִי",
+    8: "אוֹגֻסְט",
+    9: "סֶפְּטֶמְבֶּר",
+    10: "אוֹקְטוֹבֶּר",
+    11: "נוֹבֶמְבֶּר",
+    12: "דֶּצֶמְבֶּר",
+}
+
+# Mapping of day names in Hebrew with diacritics
+DAYS = {
+    0: "יוֹם רִאשׁוֹן",
+    1: "יוֹם שֵׁנִי",
+    2: "יוֹם שְׁלִישִׁי",
+    3: "יוֹם רֵבִיעִי",
+    4: "יוֹם חֲמִישִׁי",
+    5: "יוֹם שִׁישִׁי",
+    6: "יוֹם שַׁבָּת",
+}
+
+
+def date_to_word(word: str, include_day_name=False) -> str:
+    """
+    Converts a given date string in formats (YYYY-MM-DD, YYYY.MM.DD, YYYY/MM/DD) to Hebrew date format with diacritics.
+    Returns the original word if it's not a valid date.
+    """
+    separators = ["-", ".", "/"]
+    orders = [("%Y", "%m", "%d"), ("%d", "%m", "%Y")]
+    date_formats = [sep.join(order) for order in orders for sep in separators]
+
+    for date_format in date_formats:
+        try:
+            # Try parsing the word with each date format
+            date_obj = datetime.strptime(word, date_format)
+
+            # Get the Hebrew day name with diacritics
+            day_name = DAYS[date_obj.weekday()]
+
+            # Convert month to Hebrew name with diacritics
+            month_name = MONTHS[date_obj.month]
+            day = num_to_word(str(date_obj.day))
+            year = num_to_word(str(date_obj.year))
+
+            text = f"{day} בֵּ{month_name} {year}"
+            if include_day_name:
+                text = f"{day_name}, {text}"
+            return text
+        except ValueError:
+            continue
+    return word

mishkal/expander/dictionary.py

@@ -0,0 +1,78 @@
+"""
+Dictionaries are tab separated key value words
+"""
+
+from pathlib import Path
+import json
+import re
+from mishkal.utils import remove_niqqud
+from mishkal.utils import normalize
+import unicodedata
+
+files = Path(__file__).parent.joinpath("../data").glob("*.json")
+# Sort in reverse order to prioritize the most recent and best
+order = {"bronze": 1, "silver": 2, "gold": 3}
+files = sorted(
+    files, key=lambda f: order.get(next((x for x in order if x in f.stem), ""), 0)
+)
+
+
+class Dictionary:
+    def __init__(self):
+        self.dict = {}
+        self.load_dictionaries()
+
+    def load_dictionaries(self):
+        for file in files:
+            with open(file, "r", encoding="utf-8") as f:
+                dictionary: dict = json.load(f)
+                normalized_dictionary = {}
+
+                # normalize niqqud keys
+                for k, v in dictionary.items():
+                    k = normalize(k)
+                    # Ensure not empty
+                    if k and v:
+                        normalized_dictionary[k] = v
+                self.dict.update(normalized_dictionary)
+
+    def replace_hebrew_only_callback(self, match: re.Match[str]) -> str:
+        source: str = match.group(0)
+        # decomposite
+        source = unicodedata.normalize("NFD", source)
+        raw_lookup = self.dict.get(source)
+
+        without_niqqud_lookup = self.dict.get(remove_niqqud(source))
+        with_niqqud_lookup = self.dict.get(normalize(source))
+        # Compare without niqqud ONLY if source has no niqqud
+        if raw_lookup:
+            return raw_lookup
+        if without_niqqud_lookup:
+            return without_niqqud_lookup
+        elif with_niqqud_lookup:
+            return with_niqqud_lookup
+        return source
+
+    def replace_non_whitespace_callback(self, match: re.Match[str]) -> str:
+        raw_source: str = match.group(0)
+        if raw_source.isnumeric():
+            return raw_source
+
+        raw_lookup = self.dict.get(raw_source)
+
+        # Compare without niqqud ONLY if source has no niqqud
+        if raw_lookup:
+            return raw_lookup
+        # search by only ', space, regular niqqud, alphabet
+        raw_source = re.sub(
+            r"[\u05B0-\u05EB ']+", self.replace_hebrew_only_callback, raw_source
+        )
+        return raw_source
+
+    def expand_text(self, text: str) -> str:
+        """
+        TODO: if key doesn't have diacritics expand even diacritized words
+        """
+        text = re.sub(r"\S+", self.replace_non_whitespace_callback, text)
+
+        return text

mishkal/expander/number_names.py

@@ -0,0 +1,191 @@
+"""
+See https://github.com/savoirfairelinux/num2words/blob/master/num2words/lang_HE.py
+"""
+
+ZERO = {"אפס": "אֶפֶס"}
+
+
+ONES = {
+    "אחת": "אַחַת",
+    "אחד": "אֶחָד",
+    "ראשונה": "רִאשׁוֹנָה",
+    "ראשון": "רִאשׁוֹן",
+    "ראשונות": "רִאשׁוֹנוֹת",
+    "ראשונים": "רִאשׁוֹנִים",
+    "שתיים": "שְׁתַּיִם",
+    "שניים": "שְׁנַיִם",
+    "שתי": "שְׁתֵּי",
+    "שני": "שְׁנֵי",
+    "שנייה": "שְׁנִיָּה",
+    "שניות": "שְׁנִיּוֹת",
+    "שלוש": "שָׁלוֹשׁ",
+    "שלושה": "שְׁלוֹשָׁה",
+    "שלושת": "שְׁלוֹשֶׁת",
+    "שלישית": "שְׁלִישִׁית",
+    "שלישי": "שְׁלִישִׁי",
+    "שלישיות": "שְׁלִישִׁיּוֹת",
+    "שלישיים": "שְׁלִישִׁיִּים",
+    "ארבע": "אַרְבַּע",
+    "ארבעה": "אַרְבַּעָה",
+    "ארבעת": "אַרְבַּעַת",
+    "רביעית": "רֵבִיעִית",
+    "רביעי": "רֵבִיעִי",
+    "רביעיות": "רֵבִיעִיוֹת",
+    "רביעיים": "רֵבִיעִיִּים",
+    "חמש": "חָמֵשׁ",
+    "חמישה": "חֲמִשָּׁה",
+    "חמשת": "חֲמֵשֶׁת",
+    "חמישית": "חֲמִישִּׁית",
+    "חמישי": "חֲמִישִּׁי",
+    "חמישיות": "חֲמִישִּׁיוֹת",
+    "חמישיים": "חֲמִישִּׁיִּים",
+    "שש": "שֵׁשׁ",
+    "שישה": "שִׁשָּׁה",
+    "ששת": "שֵׁשֶׁת",
+    "שישית": "שִׁשִּׁית",
+    "שישי": "שִׁשִּׁי",
+    "שישיות": "שִׁשִּׁיוֹת",
+    "שישיים": "שִׁשִּׁיִּים",
+    "שבע": "שֶׁבַע",
+    "שבעה": "שִׁבְעָה",
+    "שבעת": "שִׁבְעַת",
+    "שביעית": "שְׁבִיעִית",
+    "שביעי": "שְׁבִיעִי",
+    "שביעיות": "שְׁבִיעִיוֹת",
+    "שביעיים": "שְׁבִיעִיִּים",
+    "שמונה": "שְׁמוֹנֶה",
+    "שמונת": "שְׁמוֹנַת",
+    "שמינית": "שְׁמִינִית",
+    "שמיני": "שְׁמִינִי",
+    "שמיניות": "שְׁמִינִיוֹת",
+    "שמיניים": "שְׁמִינִיִּים",
+    "תשע": "תֵּשַׁע",
+    "תשעה": "תִּשְׁעָה",
+    "תשעת": "תִּשְׁעַת",
+    "תשיעית": "תְּשִׁיעִית",
+    "תשיעי": "תְּשִׁיעִי",
+    "תשיעיות": "תְּשִׁיעִיּוֹת",
+    "תשיעיים": "תְּשִׁיעִיִּים",
+}
+
+
+TENS = {
+    "עשר": "עֶשֶׂר",
+    "עשרה": "עֲשָׁרָה",
+    "עשרת": "עֲשֶׁרֶת",
+    "עשירית": "עֲשִׁירִית",
+    "עשירי": "עֲשִׁירִי",
+    "עשיריות": "עֲשִׁירִיוֹת",
+    "עשיריים": "עֲשִׁירִיִּים",
+    "שתים עשרה": "שְׁתֵּים עֶשְׂרֵה",
+    "שנים עשר": "שְׁנֵים עָשָׂר",
+}
+
+
+TWENTIES = {
+    "עשרים": "עֶשְׂרִים",
+    "שלושים": "שְׁלוֹשִׁים",
+    "ארבעים": "אַרְבָּעִים",
+    "חמישים": "חֲמִשִּׁים",
+    "שישים": "שִׁשִּׁים",
+    "שבעים": "שִׁבְעִים",
+    "שמונים": "שְׁמוֹנִים",
+    "תשעים": "תִּשְׁעִים",
+}
+
+
+HUNDREDS = {
+    "מאה": "מֵאָה",
+    "מאת": "מֵאַת",
+    "מאתיים": "מָאתַיִם",
+    "מאות": "מֵאוֹת",
+}
+
+THOUSANDS = {
+    "אלף": "אֶלֶף",
+    "אלפיים": "אַלְפַּיִם",
+    "אלפים": "אֲלָפִים",
+    "אלפי": "אַלְפִּי",
+}
+
+
+LARGE = {
+    "מיליון": "מִילְיוֹן",
+    "מיליוני": "מִילְיוֹנִי",
+    "מיליארד": "מִילְיַארְד",
+    "מיליארדי": "מִילְיַארְדִּי",
+    "טריליון": "טְרִילְיוֹן",
+    "טריליוני": "טְרִילְיוֹנִי",
+    "קוודריליון": "קוֹוַדְרִילְיוֹן",
+    "קוודריליוני": "קוֹוַדְרִילְיוֹנִי",
+    "קווינטיליון": "קוִוִּנְטִילְיוֹן",
+    "קווינטיליוני": "קוִוִּנְטִילְיוֹנִי",
+    "סקסטיליון": "סְקֶסְטִילְיוֹן",
+    "סקסטיליוני": "סְקֶסְטִילְיוֹנִי",
+    "ספטיליון": "סְפֶּטִילְיוֹן",
+    "ספטיליוני": "סְפֶּטִילְיוֹנִי",
+    "אוקטיליון": "אוֹקְטִילְיוֹן",
+    "אוקטיליוני": "אוֹקְטִילְיוֹנִי",
+    "נוניליון": "נוּנִילְיוֹן",
+    "נוניליוני": "נוּנִילְיוֹנִי",
+    "דסיליון": "דֶּסִילְיוֹן",
+    "דסיליוני": "דֶּסִילְיוֹנִי",
+    "אונדסיליון": "אוּנְדְסִילְיוֹן",
+    "אונדסיליוני": "אוּנְדְסִילְיוֹנִי",
+    "דואודסיליון": "דוּאודְסִילְיוֹן",
+    "דואודסיליוני": "דוּאודְסִילְיוֹנִי",
+    "טרדסיליון": "טֶרְדְסִילְיוֹן",
+    "טרדסיליוני": "טֶרְדְסִילְיוֹנִי",
+    "קווטואורדסיליון": "קוּוטְוָאורְדְסִילְיוֹן",
+    "קווטואורדסיליוני": "קוּוטְוָאורְדְסִילְיוֹנִי",
+    "קווינדסיליון": "קוִוִּנְדְסִילְיוֹן",
+    "קווינדסיליוני": "קוִוִּנְדְסִילְיוֹנִי",
+    "סקסדסיליון": "סֶקְסְדְסִילְיוֹן",
+    "סקסדסיליוני": "סֶקְסְדְסִילְיוֹנִי",
+    "ספטנדסיליון": "סְפֶּטַנְדְסִילְיוֹן",
+    "ספטנדסיליוני": "סְפֶּטַנְדְסִילְיוֹנִי",
+    "אוקטודסיליון": "אוֹקְטוֹדְסִילְיוֹן",
+    "אוקטודסיליוני": "אוֹקְטוֹדְסִילְיוֹנִי",
+    "נובמדסיליון": "נוֹבְמַדְסִילְיוֹן",
+    "נובמדסיליוני": "נוֹבְמַדְסִילְיוֹנִי",
+    "ויגינטיליון": "וִיגִּינְטִילְיוֹן",
+    "ויגינטיליוני": "וִיגִּינְטִילְיוֹנִי",
+}
+
+
+LETTERS = {
+    "ו": "וֵ",
+    "ה": "הַ",
+}
+
+
+CURRENCY = {
+    "שקל": "שֵׁקֶל",
+    "שקלים": "שְׁקָלִים",
+    "אגורה": "אֲגוֹרָה",
+    "אגורות": "אֲגוֹרוֹת",
+    "אירו": "אֵירוֹ",
+    "סנט": "סֵנְט",
+    "סנטים": "סֵנְטִים",
+    "דולר": "דוֹלָר",
+    "דולרים": "דוֹלָרִים",
+}
+
+
+POINTS = {
+    "מינוס": "מִינּוּס",
+    "נקודה": "נְקֻדָּה",
+}
+
+NUMBER_NAMES = {
+    **CURRENCY,
+    **HUNDREDS,
+    **LARGE,
+    **LETTERS,
+    **ONES,
+    **POINTS,
+    **TENS,
+    **THOUSANDS,
+    **TWENTIES,
+    **ZERO,
+}

mishkal/expander/numbers.py

@@ -0,0 +1,28 @@
+import num2words
+from .number_names import NUMBER_NAMES
+import re
+
+
+def add_diacritics(words: str):
+    new_words = []
+    for word in words.split():
+        if NUMBER_NAMES.get(word):
+            new_words.append(NUMBER_NAMES[word])
+        elif NUMBER_NAMES.get(word[1:]):
+            # With Vav or Bet
+            new_words.append(NUMBER_NAMES[word[0]] + NUMBER_NAMES[word[1:]])
+        else:
+            new_words.append(word)
+    return " ".join(new_words)
+
+
+def num_to_word(maybe_number: str) -> str:
+    def replace_number(match):
+        num = match.group()
+        words = num2words.num2words(num, lang="he", ordinal=False)
+        return add_diacritics(words)
+
+    # Replace all whole numbers in the string
+    result = re.sub(r"\d+", replace_number, maybe_number)
+
+    return result

mishkal/expander/time_to_word.py

@@ -0,0 +1,104 @@
+"""
+Convert time to words
+TODO: fix zeros eg. 22:00
+"""
+
+import re
+
+PATTERNS = [
+    r"(\d{1,2})([apm]{2})",  # AM/PM format
+    r"(\d{1,2}):(\d{2})",  # HH:MM format
+]
+
+
+def extract_time(match):
+    """
+    Extract hour and minute from a string in HH:MM or AM/PM format
+    and return as integers.
+    """
+    time_str = match.group(0).lower().strip()
+
+    # Check for HH:MM format
+    match = re.match(r"(\d{1,2}):(\d{2})", time_str)
+    if match:
+        h = int(match.group(1))
+        m = int(match.group(2))
+        return f"{convert_to_word(h, m)}"
+
+    # Check for AM/PM format
+    match = re.match(r"(\d{1,2})([apm]{2})", time_str)
+    if match:
+        h = int(match.group(1))
+        period = match.group(2)
+
+        # Normalize to 24-hour format
+        if period == "am" and h == 12:
+            h = 0
+        elif period == "pm" and h != 12:
+            h += 12
+        return f"{convert_to_word(h, 0)}"  # Defaulting to 0 minutes when only hour is provided
+
+    return match.group(0)  # Return original text if the format is not recognized
+
+
+def convert_to_word(h, m):
+    hours = [
+        "אֶפֶס",
+        "אַחַת",
+        "שְׁנַיִם",  # Will be replaced with "שֵׁנִי" when needed
+        "שָׁלוֹשׁ",
+        "אַרְבַּע",
+        "חָמֵשׁ",
+        "שֵׁשׁ",
+        "שֶׁבַע",
+        "שְׁמוֹנֵה",
+        "תֵּשַׁע",
+        "עֵשֵׂר",
+        "אַחַת עֶשְׂרֵה",
+        "שְׁתֵּים עֶשְׂרֵה",
+    ]
+
+    tens = ["", "עֵשֵׂר", "עֶשְׂרִים", "שְׁלוֹשִׁים", "אַרְבָּעִים", "חֲמִשִּׁים"]
+
+    ten_to_twenty = [
+        "עֵשֵׂר",
+        "אַחַת עֶשְׂרֵה",
+        "שְׁתֵּים עֶשְׂרֵה",
+        "שְׁלוֹשׁ עֶשְׂרֵה",
+        "אַרְבַּע עֶשְׂרֵה",
+        "חֲמֵשׁ עֶשְׂרֵה",
+        "שֵׁשׁ עֶשְׂרֵה",
+        "שְׁבַע עֶשְׂרֵה",
+        "שְׁמוֹנֶה עֶשְׂרֵה",
+        "תְּשַׁע עֶשְׂרֵה",
+    ]
+
+    vocab = {"minutes": "דַּקּוֹת", "and": "וֵ", "shtey": "שְׁתֵּי"}
+
+    # Convert 0 hours to 12 (midnight)
+    if h == 0:
+        h = 12
+
+    elif h > 12:
+        h -= 12
+
+    if m == 0:
+        return f"{hours[h]}"
+
+    elif 1 <= m <= 9:
+        minute_word = (
+            vocab["shtey"] if m == 2 else hours[m]
+        )  # Replace "שניים" with "שני"
+        return f"{hours[h]} {vocab['and']}{minute_word} {vocab['minutes']}"
+
+    elif 10 <= m <= 19:
+        return f"{hours[h]} {vocab['and']}{ten_to_twenty[m - 10]} {vocab['minutes']}"
+
+    else:
+        tens_part = f"{vocab['and']}{tens[m // 10]}"
+        units_part = f"{vocab['and']}{hours[m % 10]}" if m % 10 != 0 else ""
+        return f"{hours[h]} {tens_part} {units_part} {vocab['minutes']}".strip()
+
+
+def time_to_word(text: str):
+    return re.sub("|".join(PATTERNS), extract_time, text)

mishkal/log.py

@@ -0,0 +1,35 @@
+import logging
+import os
+import colorlog
+
+
+def _create_logger():
+    """
+    Create a logger with colorized output
+    Usage: LOG_LEVEL=DEBUG python <script.py>
+    """
+
+    handler = colorlog.StreamHandler()
+    fmt = "%(log_color)s%(levelname)-8s%(reset)s [%(filename)s:%(lineno)d] %(message)s"
+    handler.setFormatter(
+        colorlog.ColoredFormatter(
+            fmt=fmt,
+            log_colors={
+                "DEBUG": "blue",
+                "INFO": "green",
+                "WARNING": "yellow",
+                "ERROR": "red",
+                "CRITICAL": "red",
+            },
+        )
+    )
+    # Get log level from LOG_LEVEL environment variable
+    log_level = os.getenv("LOG_LEVEL", "WARNING").upper()
+    logger = colorlog.getLogger(__package__)
+    logger.setLevel(level=getattr(logging, log_level, logging.WARNING))
+    # Setup logging to stdout
+    logger.addHandler(handler)
+    return logger
+
+
+log = _create_logger()

mishkal/phonemize.py

@@ -0,0 +1,261 @@
+"""
+The actual letters phonemization happens here.
+Phonemes generated based on rules.
+
+Early rules:
+1. Niqqud malle vowels
+2. Dagesh (custom beged kefet)
+3. Final letter without niqqud
+4. Final Het gnuva
+5. Geresh (Gimel, Ttadik, Zain)
+6. Shva nax and na
+Revised rules:
+1. Consonants
+2. Niqqud
+
+Reference:
+- https://hebrew-academy.org.il/2020/08/11/איך-הוגים-את-השווא-הנע
+- https://en.wikipedia.org/wiki/Unicode_and_HTML_for_the_Hebrew_alphabet#Compact_table
+- https://en.wikipedia.org/wiki/Help:IPA/Hebrew
+"""
+
+from .vocab import LETTERS_NAMES_PHONEMES, Letter, Token
+from mishkal import vocab, utils
+from .expander import Expander
+from mishkal.utils import normalize
+import re
+from typing import Callable
+
+
+# Vav vowel
+vavs = {
+    "doubles_identical": {"וּוּ": "vu", "וֹוֹ": "vo"},
+    "doubles": {
+        "ווּ": "vu",
+        "ווֹ": "vo",
+    },
+    "start": {
+        "וַ": "va",
+        "וְ": "ve",
+        "וֵ": "ve",
+        "וִ": "vi",
+        "וֹ": "vo",
+        "וּ": "u",
+        "וֻ": "vu",
+    },
+    "middle": {
+        "וַ": "va",
+        "וְ": "v",
+        "וֵ": "ve",
+        "וִ": "vi",
+        "וֹ": "o",
+        "וּ": "u",
+        "וֻ": "u",
+    },
+}
+
+
+class Phonemizer:
+    def __init__(self):
+        self.expander = Expander()
+
+    def phonemize(
+        self,
+        text: str,
+        preserve_punctuation=True,
+        preserve_stress=True,
+        fallback: Callable[[str], str] = None,
+    ) -> str:
+        # TODO: is that enough? what if there's punctuation around? other chars?
+        he_pattern = r"[\u05b0-\u05ea]+"
+        fallback_pattern = r"[a-zA-Z]+"
+
+        def fallback_replace_callback(match: re.Match):
+            word = match.group(0)
+            if self.expander.dictionary.dict.get(word):
+                # skip
+                # TODO: better API
+                return word
+            phonemes = fallback(word).strip()
+            # TODO: check that it has only IPA?!
+            for c in phonemes:
+                vocab.SET_OUTPUT_CHARACTERS.add(c)
+            return phonemes
+
+        if fallback is not None:
+            text = re.sub(fallback_pattern, fallback_replace_callback, text)
+        text = self.expander.expand_text(text)
+        tokens: list[Token] = []
+        self.fallback = fallback
+
+        def heb_replace_callback(match: re.Match):
+            word = match.group(0)
+            word = normalize(word)
+            word = "".join(
+                i for i in word if i in vocab.SET_LETTERS or i in vocab.SET_NIQQUD
+            )
+            letters = utils.extract_letters(word)
+            hebrew_tokens = self.phonemize_hebrew(letters)
+            tokens.extend(hebrew_tokens)
+            return "".join(i.phonemes for i in hebrew_tokens)
+
+        text = re.sub(he_pattern, heb_replace_callback, text)
+
+        if not preserve_punctuation:
+            text = "".join(i for i in text if i not in vocab.PUNCTUATION or i == " ")
+        if not preserve_stress:
+            text = "".join(
+                i for i in text if i not in [vocab.STRESS, vocab.SECONDARY_STRESS]
+            )
+        text = "".join(i for i in text if i in vocab.SET_OUTPUT_CHARACTERS)
+        return text
+
+    def phonemize_hebrew(self, letters: list[Letter]) -> list[Token]:
+        tokens: list[Token] = []
+        i = 0
+        while i < len(letters):
+            cur = letters[i]
+            prev = letters[i - 1] if i > 0 else None
+            next = letters[i + 1] if i < len(letters) - 1 else None
+
+            # early rules
+
+            # Single letter name
+            if not next and not prev and cur and not cur.symbols:
+                token = Token(
+                    cur.as_str(), LETTERS_NAMES_PHONEMES.get(cur.letter_str, "")
+                )
+                tokens.append(token)
+                i += 1
+                continue
+
+            if cur.letter_str == "ו":
+                # special doubles
+                if next and cur.as_str() == next.as_str():
+                    phonemes = vavs["doubles_identical"].get(
+                        cur.as_str() + next.as_str(), "vo"
+                    )
+                    tokens.append(Token(cur.as_str() + next.as_str(), phonemes))
+                    i += 2
+                    continue
+                # doubles with one has no symbols
+                if next and (
+                    cur == "ו" and next == "ו" and (not cur.symbols or not next.symbols)
+                ):
+                    phonemes = vavs["doubles"].get(cur.as_str() + next.as_str())
+                    if not phonemes:
+                        # take the one with the symbols
+                        letter = cur.as_str() if cur.symbols else next.as_str()
+                        phonemes = vavs["middle"].get(letter, "v")
+                    tokens.append(Token(cur.as_str() + next.as_str(), phonemes))
+                    i += 2
+                    continue
+                # start
+                if not prev:
+                    phonemes = vavs["start"].get(cur.as_str(), "v")
+                    tokens.append(Token(cur.as_str(), phonemes))
+                    i += 1
+                    continue
+                # middle
+                phonemes = vavs["middle"].get(cur.as_str(), "v")
+                tokens.append(Token(cur.as_str(), phonemes))
+                i += 1
+                continue
+            # Yod vowel
+            if cur == "י" and prev and not cur.symbols:  # Yod without niqqud
+                # Not possible to say ii
+                if tokens[-1].phonemes.endswith("i"):
+                    token = Token(prev.as_str() + cur.as_str(), "")
+                    tokens.append(token)
+                    i += 1
+                    continue
+                if not prev.symbols:
+                    phoneme = vocab.VOWEL_I
+                    token = Token(prev.as_str() + cur.as_str(), phoneme)
+                    tokens.append(token)
+                    i += 1
+                    continue
+                elif "\u05b4" in prev.symbols:  # Hirik
+                    phoneme = ""
+                    token = Token(cur.as_str(), phoneme)
+                    tokens.append(token)
+                    i += 1
+                    continue
+
+            # Some final letters can be silent
+            if not next and cur.letter_str in "אהע" and not cur.symbols:
+                phoneme = ""
+                token = Token(cur.as_str(), phoneme)
+                tokens.append(token)
+                i += 1
+                continue
+            # Het gnuva
+            if not next and cur == "ח" and "\u05b7" in cur.symbols:  # Patah
+                phoneme = vocab.HET_GNUVA
+                token = Token(cur.as_str(), phoneme)
+                tokens.append(token)
+                i += 1
+                continue
+
+            # Geresh
+            if "'" in cur.symbols and cur.letter_str in ["ג", "ז", "צ"]:
+                phoneme = (
+                    vocab.GIMEL_OR_ZAIN_WITH_DAGESH
+                    if cur.letter_str in ["ג", "ז"]
+                    else vocab.TSADIK_WITH_DAGESH
+                )
+                phoneme += "".join(
+                    [vocab.NIQQUD_PHONEMES.get(niqqud, "") for niqqud in cur.symbols]
+                )
+                token = Token(cur.as_str() + (next.as_str() if next else ""), phoneme)
+                tokens.append(token)
+                i += 1
+                continue
+
+            # Shva nax and na
+            if "\u05b0" in cur.symbols:
+                phoneme = vocab.LETTERS_PHONEMES.get(cur.letter_str, "")
+                # First
+                if not prev:
+                    if cur.letter_str == "ו":
+                        phoneme += vocab.VOWEL_E
+                    elif cur.letter_str in "למנרי":
+                        phoneme += vocab.VOWEL_E
+                    elif next and next.letter_str in "אהע":  # Groni
+                        phoneme += vocab.VOWEL_E
+                # Middle
+                else:
+                    # After vav with dagesh nax
+                    if prev and prev.letter_str == "ו" and "\u05bc" in prev.symbols:
+                        phoneme += ""
+                    # Double final shva(s) nax
+                    elif i == len(letters) - 1 and prev and "\u05b0" in prev.symbols:
+                        phoneme += ""
+                    elif i == len(letters) - 1 and next and "\u05b0" in next.symbols:
+                        phoneme += ""
+                    # Double shva same letter
+                    elif next and next.letter_str == cur.letter_str:
+                        phoneme += vocab.VOWEL_E
+                    # Double shva
+                    elif next and "\u05b0" in next.symbols:
+                        phoneme += ""
+                    # Previous nax
+                    elif tokens:
+                        if "\u05b0" in prev.symbols and not tokens[
+                            -1
+                        ].phonemes.endswith(vocab.VOWEL_E):
+                            phoneme += vocab.VOWEL_E
+                token = Token(cur.letter_str, phoneme)
+                tokens.append(token)
+                i += 1
+                continue
+
+            # revised rules
+            phoneme = vocab.LETTERS_PHONEMES.get(cur.letter_str, "")
+            phoneme += "".join(
+                [vocab.NIQQUD_PHONEMES.get(niqqud, "") for niqqud in cur.symbols]
+            )
+            token = Token(cur.letter_str, phoneme)
+            tokens.append(token)
+            i += 1
+        return tokens

mishkal/utils.py

@@ -0,0 +1,102 @@
+import re
+from mishkal.phonemize import Letter
+from mishkal import vocab
+import unicodedata
+
+NORMALIZE_PATTERNS = {
+    # Alphabet followed by 1/2 symbols then dagesh. make dagesh first
+    "([\u05d0-\u05ea])([\u05b0-\u05c7]{1,2})(\u05bc)": r"\1\3\2",
+    r"([^בכךפףו])(\u05bc)": r"\1",
+}
+
+
+def remove_niqqud(text: str):
+    return re.sub(vocab.HE_NIQQUD_PATTERN, "", text)
+
+
+def has_niqqud(text: str):
+    return re.search(vocab.HE_NIQQUD_PATTERN, text) is not None
+
+
+def normalize(text: str) -> str:
+    """
+    Normalize unicode (decomposite)
+    Deduplicate niqqud (eg. only Patah instead of Kamatz)
+    Keep only Hebrew characters / punctuation / IPA
+    """
+    # Decompose text
+
+    text = unicodedata.normalize("NFD", text)
+    for k, v in NORMALIZE_PATTERNS.items():
+        text = re.sub(k, v, text)
+    # Normalize niqqud, remove duplicate phonetics 'sounds' (eg. only Patah)
+    for k, v in vocab.NIQQUD_NORMALIZE.items():
+        text = text.replace(k, v)
+
+    # Keep only lexicon characters
+    text = "".join(
+        [
+            c
+            for c in text
+            if c in vocab.SET_INPUT_CHARACTERS or c in vocab.SET_OUTPUT_CHARACTERS
+        ]
+    )
+    return text
+
+
+def extract_letters(word: str) -> list[Letter]:
+    """
+    Extract letters from word
+    We assume that:
+        - Dates expanded to words
+        - Numbers expanded to word
+        - Symbols expanded already
+        - Known words converted to phonemes
+        - Rashey Tavot (acronyms) expanded already
+        - English words converted to phonemes already
+        - Text normalized using unicodedata.normalize('NFD')
+
+    This function extract *ONLY* hebrew letters and niqqud from LEXICON
+    Other characters ignored!
+    """
+    # Normalize niqqud
+    for niqqud, normalized in vocab.NIQQUD_NORMALIZE.items():
+        word = word.replace(niqqud, normalized)
+    # Remove non-lexicon characters
+    word = "".join([c for c in word if c in vocab.SET_INPUT_CHARACTERS])
+    letters = []
+    i = 0
+    while i < len(word):
+        char = word[i]
+        if char in vocab.SET_LETTERS or char == "'":
+            symbols = []
+            i += 1  # Move to potential niqqud
+            # Collect symbols attached to this letter
+            while i < len(word) and (
+                word[i] in vocab.SET_LETTER_SYMBOLS or word[i] == "'"
+            ):
+                symbols.append(word[i])
+                i += 1  # Move to the next character
+
+            if char in "בכפ" and "\u05bc" in symbols:
+                char += "\u05bc"  # Add dagesh to the letter itself
+            if (
+                "\u05bc" in symbols and char not in "ו"
+            ):  # we'll keep dagesh symbol only for vav
+                symbols.remove("\u05bc")  # remove dagesh
+            # Shin
+            if "\u05c1" in symbols:
+                char += "\u05c1"
+                symbols.remove("\u05c1")
+            # Sin
+            if "\u05c2" in symbols:
+                char += "\u05c2"
+                symbols.remove("\u05c2")
+            letters.append(Letter(char, set(symbols)))
+        else:
+            i += 1  # Skip non-letter symbols
+    return letters
+
+
+def get_unicode_names(text: str):
+    return [unicodedata.name(c, "?") for c in text]

mishkal/vocab.py

@@ -0,0 +1,194 @@
+"""
+ASCII IPA transcription of Hebrew consonants and vowels.
+"""
+
+import unicodedata
+
+
+class Letter:
+    def __init__(self, letter_str: str, symbols: list[str] = []):
+        self.letter_str = letter_str
+        self.symbols: list[str] = symbols
+
+    def __repr__(self):
+        return f"{self.letter_str + ' ' + ', '.join(unicodedata.name(s) for s in self.symbols)}"
+
+    def __eq__(self, other: str):
+        return (
+            self.letter_str == other or self.letter_str + "".join(self.symbols) == other
+        )
+
+    def as_str(self):
+        return self.letter_str + "".join(self.symbols)
+
+
+class Token:
+    def __init__(self, token: str, phonemes: str = ""):
+        self.token = token
+        self.phonemes: str = phonemes
+
+    def __repr__(self):
+        return f"{self.token} {self.phonemes}"
+
+
+# https://en.wikipedia.org/wiki/Unicode_and_HTML_for_the_Hebrew_alphabet#Compact_table
+HE_CHARS_PATTERN = (
+    r"\b[\u05B0-\u05EA\u05F3\u0027]+\b"  # Chars including niqqud, geresh and en_geresh
+)
+HE_NIQQUD_PATTERN = r"[\u05B0-\u05C7]"
+
+PUNCTUATION = r"""- .,"':!?()"""
+
+# Special
+GIMEL_OR_ZAIN_WITH_DAGESH = "dʒ"
+TSADIK_WITH_DAGESH = "tʃ"
+SHIN_WITH_POINT = "ʃ"
+SIN_WITH_POINT = "s"
+STRESS = "\u02c8"
+SECONDARY_STRESS = "\u02cc"
+HET_GNUVA = "ax"
+
+LETTERS_NAMES_PHONEMES = {
+    "א": "alef",  # Alef, glottal stop
+    "ב": "bet",  # Bet
+    "ג": "gimel",  # Gimel
+    "ד": "dalet",  # Dalet
+    "ה": "hej",  # He
+    "ו": "vav",  # Vav
+    "ז": "zajin",  # Zayin
+    "ח": "xet",  # Het
+    "ט": "tet",  # Tet
+    "י": "jud",  # Yod
+    "ך": "xaf sofit",  # Haf sofit
+    "כ": "xaf",  # Haf
+    "ל": "lamed",  # Lamed
+    "ם": "mem sofit",  # Mem Sofit
+    "מ": "mem",  # Mem
+    "ן": "nun sofit",  # Nun Sofit
+    "נ": "nun",  # Nun
+    "ס": "samex",  # Samekh
+    "ע": "ajin",  # Ayin, glottal stop
+    "פ": "fey",  # Fey
+    "ף": "fey sofit",  # Fey Sofit
+    "ץ": "tsadik sofit",  # Tsadik sofit
+    "צ": "tsadik",  # Tsadik
+    "ק": "kuf",  # Kuf
+    "ר": "rejiʃ",  # Resh
+    "ש": "ʃin",  # Shin
+    "ת": "taf",  # Taf
+}
+
+# Consonants
+LETTERS_PHONEMES = {
+    "א": "",  # Alef
+    "ב": "v",  # Bet
+    "ג": "g",  # Gimel
+    "ד": "d",  # Dalet
+    "ה": "h",  # He
+    "ו": "v",  # Vav
+    "ז": "z",  # Zayin
+    "ח": "x",  # Het
+    "ט": "t",  # Tet
+    "י": "j",  # Yod
+    "ך": "x",  # Haf sofit
+    "כ": "x",  # Haf
+    "ל": "l",  # Lamed
+    "ם": "m",  # Mem Sofit
+    "מ": "m",  # Mem
+    "ן": "n",  # Nun Sofit
+    "נ": "n",  # Nun
+    "ס": "s",  # Samekh
+    "ע": "",  # Ayin, only voweled
+    "פ": "f",  # Fey
+    "ף": "f",  # Fey Sofit
+    "ץ": "ts",  # Tsadik sofit
+    "צ": "ts",  # Tsadik
+    "ק": "k",  # Kuf
+    "ר": "r",  # Resh
+    "ש": "ʃ",  # Shin
+    "ת": "t",  # Taf
+    # Beged Kefet
+    "בּ": "b",
+    "כּ": "k",
+    "פּ": "p",
+    "שׁ": "ʃ",
+    "שׂ": "s",
+}
+
+# Vowels
+VOWEL_A = "a"
+VOWEL_E = "e"
+VOWEL_I = "i"
+VOWEL_O = "o"
+VOWEL_U = "u"
+
+NIQQUD_PHONEMES = {
+    "\u05b4": "i",  # Hiriq
+    "\u05b5": "e",  # Tsere
+    "\u05b7": "a",  # Patah
+    "\u05b9": "o",  # Holam
+    "\u05ba": "o",  # Holam haser for vav
+    "\u05bb": "u",  # Qubuts
+}
+
+SET_LETTER_SYMBOLS = {
+    "\u05b0",  # Shva
+    "\u05b4",  # Hiriq
+    "\u05b5",  # Tsere
+    "\u05b7",  # Patah
+    "\u05b9",  # Holam
+    "\u05ba",  # Holam haser for vav
+    "\u05bb",  # Qubuts
+    "\u05bc",  # Dagesh
+    "\u05c1",  # Shin dot
+    "\u05c2",  # Sin dot
+    "'",  # Geresh
+}
+
+"""
+We're left with the following niqqud (10):  
+Shva, Hiriq, Tsere, Patah, Holam, Qubuts, Dagesh, 
+Holam haser for vav, Shin dot, Sin dot
+"""
+NIQQUD_NORMALIZE = {
+    "\u05b1": "\u05b5",  # Hataf Segol -> Tsere
+    "\u05b2": "\u05b7",  # Hataf Patah -> Patah
+    "\u05b3": "\u05b9",  # Hataf Qamats -> Holam
+    "\u05b6": "\u05b5",  # Segol -> Tsere
+    # Kamatz -> Patah
+    "\u05b8": "\u05b7",  # Qamats -> Patah
+    "\u05c7": "\u05b9",  # Qamats Qatan -> Holam
+    "\u05f3": "'",  # Hebrew geresh to regular geresh
+}
+
+
+SET_OUTPUT_CHARACTERS = set(
+    [*GIMEL_OR_ZAIN_WITH_DAGESH, TSADIK_WITH_DAGESH, SHIN_WITH_POINT, SIN_WITH_POINT]
+    + [STRESS, SECONDARY_STRESS]
+    + list(LETTERS_PHONEMES.values())
+    + list(NIQQUD_PHONEMES.values())
+    + [VOWEL_A, VOWEL_E, VOWEL_I, VOWEL_O, VOWEL_U]
+    + list(PUNCTUATION)
+)
+
+SET_NIQQUD = {
+    # Shva, Hiriq, Tsere, Patah, Holam, Holam haser for vav, Qubuts, Dagesh, Shin dot, Sin dot
+    "\u05b0",
+    "\u05b4",
+    "\u05b5",
+    "\u05b7",
+    "\u05b9",
+    "\u05ba",
+    "\u05bb",
+    "\u05bc",
+    "\u05c1",
+    "\u05c2",
+}
+SET_LETTERS = set(LETTERS_PHONEMES.keys())
+SET_PUNCTUATION = set(PUNCTUATION)
+
+
+# Set for fast lookup
+SET_INPUT_CHARACTERS = set(
+    list(LETTERS_PHONEMES.keys()) + list(SET_NIQQUD) + list(PUNCTUATION) + ["'"]
+)

requirements.txt

@@ -0,0 +1,3 @@
+gradio>=5.15.0
+num2words
+colorlog