Duplicate from Sunbird/luganda2english-stt

Co-authored-by: Ali <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Luganda2English Speech Translation
+emoji: 🦓
+colorFrom: yellow
+colorTo: purple
+sdk: gradio
+sdk_version: 3.35.2
+app_file: app.py
+pinned: false
+duplicated_from: Sunbird/luganda2english-stt
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,45 @@
+import gradio as gr
+import torch
+import librosa
+import json
+from transformers import pipeline
+from stitched_model import CombinedModel
+
+
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+model = CombinedModel("ak3ra/wav2vec2-sunbird-speech-lug", "Sunbird/sunbird-mul-en-mbart-merged", device="cpu")
+
+
+
+def transcribe(audio_file_mic=None, audio_file_upload=None):
+    if audio_file_mic:
+        audio_file = audio_file_mic
+    elif audio_file_upload:
+        audio_file = audio_file_upload
+    else:
+        return "Please upload an audio file or record one"
+
+    # Make sure audio is 16kHz
+    speech, sample_rate = librosa.load(audio_file)
+    if sample_rate != 16000:
+        speech = librosa.resample(speech, orig_sr=sample_rate, target_sr=16000)
+    speech = torch.tensor([speech])
+
+    with torch.no_grad():
+        transcription, translation = model({"audio":speech})
+    
+    return transcription, translation[0]
+
+description = '''Luganda to English Speech Translation'''
+
+iface = gr.Interface(fn=transcribe,
+                     inputs=[
+                         gr.Audio(source="microphone", type="filepath", label="Record Audio"),
+                         gr.Audio(source="upload", type="filepath", label="Upload Audio")],
+                     outputs=[gr.Textbox(label="Transcription"),
+                              gr.Textbox(label="Translation")
+                     ],
+                     description=description
+                     )
+iface.launch()

requirements.txt

@@ -0,0 +1,4 @@
+transformers[torch]
+librosa
+sentencepiece
+jiwer

stitched_model.py

@@ -0,0 +1,30 @@
+import torch
+from torch import nn
+from transformers import Wav2Vec2Processor, Wav2Vec2ForCTC, AutoTokenizer, AutoModelForSeq2SeqLM
+
+class CombinedModel(nn.Module):
+    def __init__(self, stt_model_name, nmt_model_name,device = "cuda"):
+        super(CombinedModel, self).__init__()
+
+        self.stt_processor = Wav2Vec2Processor.from_pretrained(stt_model_name)
+        self.stt_model = Wav2Vec2ForCTC.from_pretrained(stt_model_name)
+        self.nmt_tokenizer = AutoTokenizer.from_pretrained(nmt_model_name)
+        self.nmt_model = AutoModelForSeq2SeqLM.from_pretrained(nmt_model_name)
+        self.device = device
+
+    def forward(self, batch, *args, **kwargs):
+        # Use stt_model to transcribe the audio to text
+        device = self.device
+        audio = torch.tensor(batch["audio"][0]).to(self.device)
+        input_features = self.stt_processor(audio,sampling_rate=16000, return_tensors="pt",max_length=110000, padding=True, truncation=True)
+        stt_output = self.stt_model(input_features.input_values.to(device), attention_mask= input_features.attention_mask.to(device) )
+        transcription = self.stt_processor.decode(torch.squeeze(stt_output.logits.argmax(axis=-1)).to(device))
+        input_nmt_tokens = self.nmt_tokenizer(transcription, return_tensors="pt", padding=True, truncation=True)
+        output_nmt_output = self.nmt_model.generate(input_ids = input_nmt_tokens.input_ids.to(device), attention_mask= input_nmt_tokens.attention_mask.to(device))
+        decoded_nmt_output = self.nmt_tokenizer.batch_decode(output_nmt_output, skip_special_tokens=True)
+
+
+        return transcription, decoded_nmt_output
+
+# Usage
+#model = CombinedModel("ak3ra/wav2vec2-sunbird-speech-lug", "Sunbird/sunbird-mul-en-mbart-merged", device="cpu")