Update app.py

app.py

@@ -1,50 +1,46 @@
 import gradio as gr
-import torchaudio
 import os
+import torch
 import soundfile as sf
-from speechbrain.pretrained import EncoderClassifier, Tacotron2, HIFIGAN, EncoderDecoderASR
+import torchaudio
+from scipy.io.wavfile import write
+from transformers import VitsProcessor, VitsForConditionalGeneration
+from speechbrain.pretrained import EncoderClassifier, EncoderDecoderASR
 
-# Ensure output directory exists
+# Ensure the output directory exists
 os.makedirs("output_audio", exist_ok=True)
 
-# Load models
-encoder = EncoderClassifier.from_hparams(source="speechbrain/spkrec-ecapa-voxceleb", savedir="models/encoder")
-tacotron2 = Tacotron2.from_hparams(source="speechbrain/tts-tacotron2-ljspeech", savedir="models/tacotron2")
-hifigan = HIFIGAN.from_hparams(source="speechbrain/tts-hifigan-ljspeech", savedir="models/hifigan")
+# Load the Facebook MMS TTS model and processor
+tts_processor = VitsProcessor.from_pretrained("facebook/mms-tts-eng")
+tts_model = VitsForConditionalGeneration.from_pretrained("facebook/mms-tts-eng")
+
+# SpeechBrain ASR Model for Speech to Text
 asr_model = EncoderDecoderASR.from_hparams(source="speechbrain/asr-conformer-transformerlm-librispeech", savedir="models/asr")
 
 def speech_to_text(input_audio):
     sig, sr = torchaudio.load(input_audio)
-    # Ensure the sample rate is 16000, expected by the model
     if sr != 16000:
         sig = torchaudio.transforms.Resample(orig_freq=sr, new_freq=16000)(sig)
     transcription = asr_model.transcribe_file(input_audio)
     return transcription
 
-def speech_to_speech(input_audio, target_text):
-    # Load and encode speaker from input audio
-    signal, fs = torchaudio.load(input_audio)
-    if fs != 16000:
-        signal = torchaudio.transforms.Resample(orig_freq=fs, new_freq=16000)(signal)
-    embedding = encoder.encode_batch(signal)
-
-    # Synthesize speech from text
-    mel_output, mel_length, alignment = tacotron2.encode_text(target_text, embedding)
-    waveform = hifigan.decode_batch(mel_output)
-
-    # Save output audio
-    output_path = "output_audio/synthesized_speech.wav"
-    sf.write(output_path, waveform.squeeze().cpu().numpy(), 22050)
-    return output_path
-
 def text_to_speech(text):
-    mel_output, mel_length, alignment = tacotron2.encode_text(text)
-    waveform = hifigan.decode_batch(mel_output)
+    inputs = tts_processor(text, return_tensors="pt")
+    with torch.no_grad():
+        generated = tts_model.generate(**inputs)
+        waveform = generated.audio.squeeze().cpu().numpy()
 
     output_path = "output_audio/text_to_speech.wav"
-    sf.write(output_path, waveform.squeeze().cpu().numpy(), 22050)
+    sf.write(output_path, waveform, 22050)
     return output_path
 
+def speech_to_speech(input_audio, target_text):
+    # Speech to Text
+    transcription = speech_to_text(input_audio)
+    
+    # Text to Speech with Facebook MMS TTS
+    return text_to_speech(target_text)
+
 iface = gr.Interface(
     fn={
         "Speech to Text": speech_to_text,
@@ -62,7 +58,7 @@ iface = gr.Interface(
         "Speech to Speech": gr.outputs.Audio(type="file", label="Synthesized Speech")
     },
     title="Speech Processing App",
-    description="Upload an audio file or enter text to perform various speech processing tasks.",
+    description="This app uses SpeechBrain for speech to text and Facebook's MMS for text to speech.",
     layout="vertical"
 )