Upload folder using huggingface_hub

modules/app.py

@@ -0,0 +1,117 @@
+'''
+        Created By Lewis Kamau Kimaru 
+        Sema fastapi backend
+        August 2023
+'''
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import HTMLResponse
+import gradio as gr
+import ctranslate2
+import sentencepiece as spm
+import fasttext
+import uvicorn
+from pyngrok import ngrok
+import nest_asyncio
+import os
+
+app = FastAPI()
+
+# Set your ngrok authtoken
+#ngrok.set_auth_token("2UAhCqf5zP0cCgJzeadNANkbIqx_7ZJvhkDSNWccqMX2hyxXP")
+#ngrok.set_auth_token("2S6xeFEoSVFWr2egtDRcqgeUtSx_2juefHFkEW6nGbpRHS37W")
+#ngrok.set_auth_token("2UAmdjHdAFV9x84TdyEknIfNhYk_4Ye8n4YK7ZhfCMob3yPBh")
+#ngrok.set_auth_token("2UAqm26HuWiWvQjzK58xYufSGpy_6tStKSyLLyR9f7pcezh6R")
+ngrok.set_auth_token("2UGQqzZoI3bx7SSk8H4wuFC3iaC_2WniWyNAsW5fd2rFyKVq1")
+
+
+fasttext.FastText.eprint = lambda x: None
+
+# Load the model and tokenizer ..... only once!
+beam_size = 1  # change to a smaller value for faster inference
+device = "cpu"  # or "cuda"
+
+# Language Prediction model
+print("\nimporting Language Prediction model")
+lang_model_file = "Sema/lid218e.bin"
+lang_model_full_path = os.path.join(os.path.dirname(__file__), lang_model_file)
+lang_model = fasttext.load_model(lang_model_full_path)
+
+
+# Load the source SentencePiece model
+print("\nimporting SentencePiece model")
+sp_model_file = "Sema/spm.model"
+sp_model_full_path = os.path.join(os.path.dirname(__file__), sp_model_file)
+sp = spm.SentencePieceProcessor()
+sp.load(sp_model_full_path)
+
+# Import The Translator model
+print("\nimporting Translator model")
+ct_model_file = "Sema/sematrans-3.3B"
+ct_model_full_path = os.path.join(os.path.dirname(__file__), ct_model_file)
+translator = ctranslate2.Translator(ct_model_full_path, device)
+
+print('\nDone importing models\n')
+
+    
+def translate_text(userinput: str, target_lang: str):
+    source_sents = [userinput]
+    source_sents = [sent.strip() for sent in source_sents]
+    target_prefix = [[target_lang]] * len(source_sents)
+
+    # Predict the source language
+    predictions = lang_model.predict(source_sents[0], k=1)
+    source_lang = predictions[0][0].replace('__label__', '')
+
+    # Subword the source sentences
+    source_sents_subworded = sp.encode(source_sents, out_type=str)
+    source_sents_subworded = [[source_lang] + sent + ["</s>"] for sent in source_sents_subworded]
+
+    # Translate the source sentences
+    translations = translator.translate_batch(
+        source_sents_subworded,
+        batch_type="tokens",
+        max_batch_size=2024,
+        beam_size=beam_size,
+        target_prefix=target_prefix,
+    )
+    translations = [translation[0]['tokens'] for translation in translations]
+
+    # Desubword the target sentences
+    translations_desubword = sp.decode(translations)
+    translations_desubword = [sent[len(target_lang):] for sent in translations_desubword]
+
+    # Return the source language and the translated text
+    return source_lang, translations_desubword
+
+
+@app.get("/")
+def read_root():
+    return {"message": "Welcome to the Sema Translation API! \nThis API was created by Lewsi Kamau Kimaru"}
+
+
+@app.post("/translate/")
+async def translate_endpoint(request: Request):
+    data = await request.json()
+    userinput = data.get("userinput")
+    target_lang = data.get("target_lang")
+    print(f"\n Target Language; {target_lang}, User Input: {userinput}\n")
+
+    if not userinput or not target_lang:
+        raise HTTPException(status_code=422, detail="Both 'userinput' and 'target_lang' are required.")
+
+    source_lang, translated_text = translate_text(userinput, target_lang)
+    print(f"\nsource_language: {source_lang}, Translated Text: {translated_text}\n\n")
+    return {
+        "source_language": source_lang,
+        "translated_text": translated_text[0],
+    }
+
+ngrok_tunnel = ngrok.connect(7860)
+public_url = ngrok_tunnel.public_url
+print('\nPublic URL✅:', public_url)
+nest_asyncio.apply()
+
+print("\nAPI starting .......\n")
+#uvicorn.run(app, port=7860)

modules/flores200_codes.py

@@ -0,0 +1,211 @@
+codes_as_string = '''Acehnese (Arabic script)	ace_Arab
+Acehnese (Latin script)	ace_Latn
+Mesopotamian Arabic	acm_Arab
+Ta’izzi-Adeni Arabic	acq_Arab
+Tunisian Arabic	aeb_Arab
+Afrikaans	afr_Latn
+South Levantine Arabic	ajp_Arab
+Akan	aka_Latn
+Amharic	amh_Ethi
+North Levantine Arabic	apc_Arab
+Modern Standard Arabic	arb_Arab
+Modern Standard Arabic (Romanized)	arb_Latn
+Najdi Arabic	ars_Arab
+Moroccan Arabic	ary_Arab
+Egyptian Arabic	arz_Arab
+Assamese	asm_Beng
+Asturian	ast_Latn
+Awadhi	awa_Deva
+Central Aymara	ayr_Latn
+South Azerbaijani	azb_Arab
+North Azerbaijani	azj_Latn
+Bashkir	bak_Cyrl
+Bambara	bam_Latn
+Balinese	ban_Latn
+Belarusian	bel_Cyrl
+Bemba	bem_Latn
+Bengali	ben_Beng
+Bhojpuri	bho_Deva
+Banjar (Arabic script)	bjn_Arab
+Banjar (Latin script)	bjn_Latn
+Standard Tibetan	bod_Tibt
+Bosnian	bos_Latn
+Buginese	bug_Latn
+Bulgarian	bul_Cyrl
+Catalan	cat_Latn
+Cebuano	ceb_Latn
+Czech	ces_Latn
+Chokwe	cjk_Latn
+Central Kurdish	ckb_Arab
+Crimean Tatar	crh_Latn
+Welsh	cym_Latn
+Danish	dan_Latn
+German	deu_Latn
+Southwestern Dinka	dik_Latn
+Dyula	dyu_Latn
+Dzongkha	dzo_Tibt
+Greek	ell_Grek
+English	eng_Latn
+Esperanto	epo_Latn
+Estonian	est_Latn
+Basque	eus_Latn
+Ewe	ewe_Latn
+Faroese	fao_Latn
+Fijian	fij_Latn
+Finnish	fin_Latn
+Fon	fon_Latn
+French	fra_Latn
+Friulian	fur_Latn
+Nigerian Fulfulde	fuv_Latn
+Scottish Gaelic	gla_Latn
+Irish	gle_Latn
+Galician	glg_Latn
+Guarani	grn_Latn
+Gujarati	guj_Gujr
+Haitian Creole	hat_Latn
+Hausa	hau_Latn
+Hebrew	heb_Hebr
+Hindi	hin_Deva
+Chhattisgarhi	hne_Deva
+Croatian	hrv_Latn
+Hungarian	hun_Latn
+Armenian	hye_Armn
+Igbo	ibo_Latn
+Ilocano	ilo_Latn
+Indonesian	ind_Latn
+Icelandic	isl_Latn
+Italian	ita_Latn
+Javanese	jav_Latn
+Japanese	jpn_Jpan
+Kabyle	kab_Latn
+Jingpho	kac_Latn
+Kamba	kam_Latn
+Kannada	kan_Knda
+Kashmiri (Arabic script)	kas_Arab
+Kashmiri (Devanagari script)	kas_Deva
+Georgian	kat_Geor
+Central Kanuri (Arabic script)	knc_Arab
+Central Kanuri (Latin script)	knc_Latn
+Kazakh	kaz_Cyrl
+Kabiyè	kbp_Latn
+Kabuverdianu	kea_Latn
+Khmer	khm_Khmr
+Kikuyu	kik_Latn
+Kinyarwanda	kin_Latn
+Kyrgyz	kir_Cyrl
+Kimbundu	kmb_Latn
+Northern Kurdish	kmr_Latn
+Kikongo	kon_Latn
+Korean	kor_Hang
+Lao	lao_Laoo
+Ligurian	lij_Latn
+Limburgish	lim_Latn
+Lingala	lin_Latn
+Lithuanian	lit_Latn
+Lombard	lmo_Latn
+Latgalian	ltg_Latn
+Luxembourgish	ltz_Latn
+Luba-Kasai	lua_Latn
+Ganda	lug_Latn
+Luo	luo_Latn
+Mizo	lus_Latn
+Standard Latvian	lvs_Latn
+Magahi	mag_Deva
+Maithili	mai_Deva
+Malayalam	mal_Mlym
+Marathi	mar_Deva
+Minangkabau (Arabic script)	min_Arab
+Minangkabau (Latin script)	min_Latn
+Macedonian	mkd_Cyrl
+Plateau Malagasy	plt_Latn
+Maltese	mlt_Latn
+Meitei (Bengali script)	mni_Beng
+Halh Mongolian	khk_Cyrl
+Mossi	mos_Latn
+Maori	mri_Latn
+Burmese	mya_Mymr
+Dutch	nld_Latn
+Norwegian Nynorsk	nno_Latn
+Norwegian Bokmål	nob_Latn
+Nepali	npi_Deva
+Northern Sotho	nso_Latn
+Nuer	nus_Latn
+Nyanja	nya_Latn
+Occitan	oci_Latn
+West Central Oromo	gaz_Latn
+Odia	ory_Orya
+Pangasinan	pag_Latn
+Eastern Panjabi	pan_Guru
+Papiamento	pap_Latn
+Western Persian	pes_Arab
+Polish	pol_Latn
+Portuguese	por_Latn
+Dari	prs_Arab
+Southern Pashto	pbt_Arab
+Ayacucho Quechua	quy_Latn
+Romanian	ron_Latn
+Rundi	run_Latn
+Russian	rus_Cyrl
+Sango	sag_Latn
+Sanskrit	san_Deva
+Santali	sat_Olck
+Sicilian	scn_Latn
+Shan	shn_Mymr
+Sinhala	sin_Sinh
+Slovak	slk_Latn
+Slovenian	slv_Latn
+Samoan	smo_Latn
+Shona	sna_Latn
+Sindhi	snd_Arab
+Somali	som_Latn
+Southern Sotho	sot_Latn
+Spanish	spa_Latn
+Tosk Albanian	als_Latn
+Sardinian	srd_Latn
+Serbian	srp_Cyrl
+Swati	ssw_Latn
+Sundanese	sun_Latn
+Swedish	swe_Latn
+Swahili	swh_Latn
+Silesian	szl_Latn
+Tamil	tam_Taml
+Tatar	tat_Cyrl
+Telugu	tel_Telu
+Tajik	tgk_Cyrl
+Tagalog	tgl_Latn
+Thai	tha_Thai
+Tigrinya	tir_Ethi
+Tamasheq (Latin script)	taq_Latn
+Tamasheq (Tifinagh script)	taq_Tfng
+Tok Pisin	tpi_Latn
+Tswana	tsn_Latn
+Tsonga	tso_Latn
+Turkmen	tuk_Latn
+Tumbuka	tum_Latn
+Turkish	tur_Latn
+Twi	twi_Latn
+Central Atlas Tamazight	tzm_Tfng
+Uyghur	uig_Arab
+Ukrainian	ukr_Cyrl
+Umbundu	umb_Latn
+Urdu	urd_Arab
+Northern Uzbek	uzn_Latn
+Venetian	vec_Latn
+Vietnamese	vie_Latn
+Waray	war_Latn
+Wolof	wol_Latn
+Xhosa	xho_Latn
+Eastern Yiddish	ydd_Hebr
+Yoruba	yor_Latn
+Yue Chinese	yue_Hant
+Chinese (Simplified)	zho_Hans
+Chinese (Traditional)	zho_Hant
+Standard Malay	zsm_Latn
+Zulu	zul_Latn'''
+
+codes_as_string = codes_as_string.split('\n')
+
+flores_codes = {}
+for code in codes_as_string:
+    lang, lang_code = code.split('\t')
+    flores_codes[lang] = lang_code