Spaces:
Runtime error
Runtime error
File size: 4,139 Bytes
ae71d4b 830a7a9 6be28a0 830a7a9 908706b 830a7a9 ae71d4b 3e188e1 bbb7e65 3e188e1 bbb7e65 830a7a9 908706b 6742dfa 014aba2 6742dfa 3e188e1 ae71d4b 6be28a0 aa4c2b6 ae71d4b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 |
import gradio as gr
import librosa
import numpy as np
import torch
from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan
from datasets import load_dataset, Audio
dataset = load_dataset(
"divakaivan/glaswegian_audio"
)
dataset = dataset.cast_column("audio", Audio(sampling_rate=16000))['train']
from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech
processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
model = SpeechT5ForTextToSpeech.from_pretrained("divakaivan/glaswegian_tts")
tokenizer = processor.tokenizer
def extract_all_chars(batch):
all_text = " ".join(batch["transcription"])
vocab = list(set(all_text))
return {"vocab": [vocab], "all_text": [all_text]}
vocabs = dataset.map(
extract_all_chars,
batched=True,
batch_size=-1,
keep_in_memory=True,
remove_columns=dataset.column_names,
)
dataset_vocab = set(vocabs["vocab"][0])
tokenizer_vocab = {k for k,_ in tokenizer.get_vocab().items()}
replacements = [
('à', 'a'),
('ç', 'c'),
('è', 'e'),
('ë', 'e'),
('í', 'i'),
('ï', 'i'),
('ö', 'o'),
('ü', 'u'),
]
def cleanup_text(inputs):
for src, dst in replacements:
inputs["transcription"] = inputs["transcription"].replace(src, dst)
return inputs
dataset = dataset.map(cleanup_text)
import os
import torch
from speechbrain.inference.speaker import EncoderClassifier
spk_model_name = "speechbrain/spkrec-xvect-voxceleb"
device = "cuda" if torch.cuda.is_available() else "cpu"
speaker_model = EncoderClassifier.from_hparams(
source=spk_model_name,
run_opts={"device": device},
savedir=os.path.join("/tmp", spk_model_name),
)
def create_speaker_embedding(waveform):
with torch.no_grad():
speaker_embeddings = speaker_model.encode_batch(torch.tensor(waveform))
speaker_embeddings = torch.nn.functional.normalize(speaker_embeddings, dim=2)
speaker_embeddings = speaker_embeddings.squeeze().cpu().numpy()
return speaker_embeddings
def prepare_dataset(example):
# load the audio data; if necessary, this resamples the audio to 16kHz
audio = example["audio"]
# feature extraction and tokenization
example = processor(
text=example["transcription"],
audio_target=audio["array"],
sampling_rate=audio["sampling_rate"],
return_attention_mask=False,
)
# strip off the batch dimension
example["labels"] = example["labels"][0]
# use SpeechBrain to obtain x-vector
example["speaker_embeddings"] = create_speaker_embedding(audio["array"])
return example
processed_example = prepare_dataset(dataset[0])
from transformers import SpeechT5HifiGan
vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
spectrogram = torch.tensor(processed_example["labels"])
with torch.no_grad():
speech = vocoder(spectrogram)
dataset = dataset.map(
prepare_dataset, remove_columns=dataset.column_names,
)
dataset = dataset.train_test_split(test_size=0.1)
def predict(text, speaker):
if len(text.strip()) == 0:
return (16000, np.zeros(0).astype(np.int16))
inputs = processor(text=text, return_tensors="pt")
# limit input length
# input_ids = inputs["input_ids"]
# input_ids = input_ids[..., :model.config.max_text_positions]
### ### ###
example = dataset['test'][11]
speaker_embeddings = torch.tensor(example["speaker_embeddings"]).unsqueeze(0)
spectrogram = model.generate_speech(inputs["input_ids"], speaker_embeddings)
with torch.no_grad():
speech = vocoder(spectrogram)
speech = (speech.numpy() * 32767).astype(np.int16)
return (16000, speech)
title = "Glaswegian TTS"
article = "Model fine-tuned and gradle demo generated thanks to this notebook: https://colab.research.google.com/drive/1i7I5pzBcU3WDFarDnzweIj4-sVVoIUFJ#scrollTo=wm7B3zxrumfF"
gr.Interface(
fn=predict,
inputs=[
gr.Text(label="Input Text"),
],
outputs=[
gr.Audio(label="Generated Speech", type="numpy"),
],
title=title,
article=article,
).launch()
|