app.py

import gradio as gr
import librosa
import numpy as np
import torch
import string
import googletrans

from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan


checkpoint = "microsoft/speecht5_tts"
processor = SpeechT5Processor.from_pretrained(checkpoint)
model = SpeechT5ForTextToSpeech.from_pretrained("Edmon02/speecht5_finetuned_voxpopuli_hy")
vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")


speaker_embeddings = {
    "BDL": "cmu_us_bdl_arctic-wav-arctic_a0009.npy",
}

import pandas as pd
import inflect
import re
from googletrans import Translator

translator = Translator()

def convert_number_to_words(number: float) -> str:
    p = inflect.engine()
    words = p.number_to_words(number)
    words = translator.translate(words, dest='hy').text
    return words

def process_text(text: str) -> str:
    # Convert numbers to words
    words = []
    text = str(text) if str(text) else ''
    for word in text.split():
        # Check if the word is a number
        if re.search(r'\d', word):
            words.append(convert_number_to_words(int(''.join(filter(str.isdigit, word)))))
        else:
            words.append(word)

    # Join the words back into a sentence
    processed_text = ' '.join(words)
    return processed_text

# Read CSV file into a pandas DataFrame
df = pd.read_csv('AudioSet.csv')

# Apply the processing function to the 'normalized_text' column
df['normalized_text'] = df['normalized_text'].apply(process_text)

# Save the updated DataFrame back to the CSV file
df.to_csv('AudioSet.csv', index=False)

replacements = [
    ("՚", "?"),
    ('՛', ""),
    ('՝', ""),
    ("«", "\""),
    ("»", "\""),
    ("՞", "?"),
    ("ա", "a"),
    ("բ", "b"),
    ("գ", "g"),
    ("դ", "d"),
    ("զ",	"z"),
    ("է", "e"),
    ("ը", "e'"),
    ("թ", "t'"),
    ("ժ",	"jh"),
    ("ի",	"i"),
    ("լ",	"l"),
    ("խ",	"kh"),
    ("ծ",	"ts"),
    ("կ",	"k"),
    ("հ",	"h"),
    ("ձ",	"dz"),
    ("ղ",	"gh"),
    ("ճ",	"ch"),
    ("մ",	"m"),
    ("յ",	"y"),
    ("ն",	"n"),
    ("շ",	"sh"),
    ("չ",	"ch'"),
    ("պ",	"p"),
    ("ջ",	"j"),
    ("ռ",	"r"),
    ("ս",	"s"),
    ("վ",	"v"),
    ("տ",	"t"),
    ("ր",	"r"),
    ("ց",	"ts'"),
    ("ւ",	""),
    ("փ",	"p'"),
    ("ք",	"k'"),
    ("և",	"yev"),
    ("օ",	"o"),
    ("ֆ",	"f"),
    ('։', "."),
    ('–', "-"),
    ('†', "e'"),
]


def cleanup_text(text):
    
    translator = str.maketrans("", "", string.punctuation)

    text = text.translate(translator).lower()
    text = text.lower()
    
    normalized_text = text

    normalized_text = normalized_text.replace("ու", "u")
    normalized_text = normalized_text.replace("եւ", "u")
    normalized_text = normalized_text.replace("եվ", "u")

    # Handle 'ո' at the beginning of a word
    normalized_text = normalized_text.replace(" ո", " vo")

    # Handle 'ո' in the middle of a word
    normalized_text = normalized_text.replace("ո", "o")

    # Handle 'ե' at the beginning of a word
    normalized_text = normalized_text.replace(" ե", " ye")

    # Handle 'ե' in the middle of a word
    normalized_text = normalized_text.replace("ե", "e")

    # Apply other replacements
    for src, dst in replacements:
        normalized_text = normalized_text.replace(src, dst)

    inputs = normalized_text
    return inputs

def predict(text, speaker):
    if len(text.strip()) == 0:
        return (16000, np.zeros(0).astype(np.int16))

    text = process_text(text)
    
    text = cleanup_text({'normalized_text': text})['normalized_text']

    inputs = processor(text=text, return_tensors="pt")

    # limit input length
    input_ids = inputs["input_ids"]
    input_ids = input_ids[..., :model.config.max_text_positions]

    speaker_embedding = np.load(speaker_embeddings[speaker[:3]])

    speaker_embedding = torch.tensor(speaker_embedding).unsqueeze(0)

    speech = model.generate_speech(input_ids, speaker_embedding, vocoder=vocoder)

    speech = (speech.numpy() * 32767).astype(np.int16)
    return (16000, speech)


title = "SpeechT5: Speech Synthesis"

description = """
The <b>SpeechT5</b> model is pre-trained on text as well as speech inputs, with targets that are also a mix of text and speech.
By pre-training on text and speech at the same time, it learns unified representations for both, resulting in improved modeling capabilities.

SpeechT5 can be fine-tuned for different speech tasks. This space demonstrates the <b>text-to-speech</b> (TTS) checkpoint for the English language.

See also the <a href="https://huggingface.co./spaces/Matthijs/speecht5-asr-demo">speech recognition (ASR) demo</a>
and the <a href="https://huggingface.co./spaces/Matthijs/speecht5-vc-demo">voice conversion demo</a>.

Refer to <a href="https://colab.research.google.com/drive/1i7I5pzBcU3WDFarDnzweIj4-sVVoIUFJ">this Colab notebook</a> to learn how to fine-tune the SpeechT5 TTS model on your own dataset or language.

<b>How to use:</b> Enter some English text and choose a speaker. The output is a mel spectrogram, which is converted to a mono 16 kHz waveform by the
HiFi-GAN vocoder. Because the model always applies random dropout, each attempt will give slightly different results.
The <em>Surprise Me!</em> option creates a completely randomized speaker.
"""

article = """
<div style='margin:20px auto;'>

<p>References: <a href="https://arxiv.org/abs/2110.07205">SpeechT5 paper</a> |
<a href="https://github.com/microsoft/SpeechT5/">original GitHub</a> |
<a href="https://huggingface.co./mechanicalsea/speecht5-tts">original weights</a></p>

<pre>
@article{Ao2021SpeechT5,
  title   = {SpeechT5: Unified-Modal Encoder-Decoder Pre-training for Spoken Language Processing},
  author  = {Junyi Ao and Rui Wang and Long Zhou and Chengyi Wang and Shuo Ren and Yu Wu and Shujie Liu and Tom Ko and Qing Li and Yu Zhang and Zhihua Wei and Yao Qian and Jinyu Li and Furu Wei},
  eprint={2110.07205},
  archivePrefix={arXiv},
  primaryClass={eess.AS},
  year={2021}
}
</pre>

<p>Speaker embeddings were generated from <a href="http://www.festvox.org/cmu_arctic/">CMU ARCTIC</a> using <a href="https://huggingface.co./mechanicalsea/speecht5-vc/blob/main/manifest/utils/prep_cmu_arctic_spkemb.py">this script</a>.</p>

</div>
"""

examples = [
    ["It is not in the stars to hold our destiny but in ourselves.", "BDL (male)"],
    ["The octopus and Oliver went to the opera in October.", "CLB (female)"],
    ["She sells seashells by the seashore. I saw a kitten eating chicken in the kitchen.", "RMS (male)"],
    ["Brisk brave brigadiers brandished broad bright blades, blunderbusses, and bludgeons—balancing them badly.", "SLT (female)"],
    ["A synonym for cinnamon is a cinnamon synonym.", "BDL (male)"],
    ["How much wood would a woodchuck chuck if a woodchuck could chuck wood? He would chuck, he would, as much as he could, and chuck as much wood as a woodchuck would if a woodchuck could chuck wood.", "CLB (female)"],
]

gr.Interface(
    fn=predict,
    inputs=[
        gr.Text(label="Input Text"),
        gr.Radio(label="Speaker", choices=[
            "BDL (male)"
        ],
        value="BDL (male)"),
    ],
    outputs=[
        gr.Audio(label="Generated Speech", type="numpy"),
    ],
    title=title,
    description=description,
    article=article,
    examples=examples,
).launch()