implement enjabements demo

InferenceInterfaces/Meta_FastSpeech2.py

@@ -40,9 +40,13 @@ class Meta_FastSpeech2(torch.nn.Module):
         self.text2phone = ArticulatoryCombinedTextFrontend(language=lang_id, add_silence_to_end=True, silent=True)
         self.lang_id = get_language_id(lang_id).to(self.device)
 
-    def forward(self, text, view=False, durations=None, pitch=None, energy=None):
-        with torch.no_grad():
-            phones = self.text2phone.string_to_tensor(text).to(torch.device(self.device))
+    def forward(self, text, view=False, durations=None, pitch=None, energy=None, phones = False):
+        with torch.inference_mode():
+            if phones is False:
+                phones = self.text2phone.string_to_tensor(text).to(torch.device(self.device))
+            else:
+                phones = self.text2phone.string_to_tensor(text, input_phonemes=True).to(torch.device(self.device))
+
             mel, durations, pitch, energy = self.phone2mel(phones,
                                                            return_duration_pitch_energy=True,
                                                            utterance_embedding=self.default_utterance_embedding,

app.py

@@ -1,18 +1,15 @@
-import os
-
 import gradio as gr
 import numpy as np
-import soundfile as sf
 import torch
-
+import math
 from Preprocessing.ArticulatoryCombinedTextFrontend import ArticulatoryCombinedTextFrontend
-from Preprocessing.AudioPreprocessor import AudioPreprocessor
 from TrainingInterfaces.Text_to_Spectrogram.AutoAligner.Aligner import Aligner
 from TrainingInterfaces.Text_to_Spectrogram.FastSpeech2.DurationCalculator import DurationCalculator
 from run_utterance_cloner import UtteranceCloner
 
-os.system("pip uninstall -y gradio")
-os.system("pip install gradio==2.7.5.2")
+
+# os.system("pip uninstall -y gradio")
+# os.system("pip install gradio==2.7.5.2")
 
 
 def float2pcm(sig, dtype='int16'):
@@ -36,148 +33,173 @@ class TTS_Interface:
     def __init__(self):
         self.device = "cuda" if torch.cuda.is_available() else "cpu"
         self.utterance_cloner = UtteranceCloner(device=self.device)
-        self.speaker_path_lookup = {
-            "Voice 1": "reference_audios/voice_1.flac",
-            "Voice 2": "reference_audios/voice_2.wav",
-            "Voice 3": "reference_audios/voice_3.wav",
-            }
+        self.utterance_cloner.tts.set_language("de")
         self.acoustic_model = Aligner()
         self.acoustic_model.load_state_dict(torch.load("Models/Aligner/aligner.pt", map_location='cpu')["asr_model"])
         self.acoustic_model = self.acoustic_model.to(self.device)
         self.dc = DurationCalculator(reduction_factor=1)
         self.tf = ArticulatoryCombinedTextFrontend(language="en")
-
-    def read(self, prompt, speaker_1, speaker_2, speaker_3):
-        if prompt == "Hello, here is the first sentence. And here comes the second one. I think three sentences is enough to get the point across, right?":
-            reference_audio = "reference_audios/clone_me_1.wav"
-        elif prompt == "I am excited! And my prosody is rather flat. And this sentence is shocking!":
-            reference_audio = "reference_audios/clone_me_2.wav"
-        elif prompt == "Don't do it! But I want to! Then go ahead.":
-            reference_audio = "reference_audios/clone_me_3.wav"
-        elif prompt == "How many examples do I realistically need? How about five? That should do it!":
-            reference_audio = "reference_audios/clone_me_4.wav"
-        elif prompt == "Betty Botter bought some butter, but she said the butters bitter. If I put it in my batter, it will make my batter bitter. But a bit of better butter will make my batter better.":
-            reference_audio = "reference_audios/clone_me_5.wav"
-
-        text_list = prompt.replace(".", ".|").replace("?", "?|").replace("!", "!|").split("|")
-        # we don't split on the punctuation marks because we want to retain them.
-
-        self.split_audio(reference_audio, text_list)
-        # at this point, split_1.wav, split_2.wav and split_3.wav should exist.
-
-        self.utterance_cloner.tts.set_utterance_embedding(self.speaker_path_lookup[speaker_1])
-        part_1 = self.utterance_cloner.clone_utterance(path_to_reference_audio="split_1.wav",
-                                                       reference_transcription=text_list[0],
-                                                       clone_speaker_identity=False,
-                                                       lang="en")
-
-        self.utterance_cloner.tts.set_utterance_embedding(self.speaker_path_lookup[speaker_2])
-        part_2 = self.utterance_cloner.clone_utterance(path_to_reference_audio="split_2.wav",
-                                                       reference_transcription=text_list[1],
-                                                       clone_speaker_identity=False,
-                                                       lang="en")
-
-        self.utterance_cloner.tts.set_utterance_embedding(self.speaker_path_lookup[speaker_3])
-        part_3 = self.utterance_cloner.clone_utterance(path_to_reference_audio="split_3.wav",
-                                                       reference_transcription=text_list[2],
-                                                       clone_speaker_identity=False,
-                                                       lang="en")
-
-        return "alignment.png", reference_audio, (48000, float2pcm(torch.cat([part_1, part_2, part_3], dim=0).numpy()))
-
-    def split_audio(self, path_to_audio, text_list):
-        # extract audio
-        audio, sr = sf.read(path_to_audio)
-        ap = AudioPreprocessor(input_sr=sr, output_sr=16000, melspec_buckets=80, hop_length=256, n_fft=1024, cut_silence=False)
-        norm_wave = ap.audio_to_wave_tensor(normalize=True, audio=audio)
-        melspec = ap.audio_to_mel_spec_tensor(audio=norm_wave, normalize=False, explicit_sampling_rate=16000).transpose(0, 1)
-
-        # extract phonemes
-        lines = list()
-        for segment in text_list:
-            if segment.strip() != "":
-                lines.append(self.tf.string_to_tensor(segment, handle_missing=False).squeeze())
-        # postprocess phonemes: [~ sentence ~ #] --> [sentence ~] except for the first one, which is [~ sentence ~]
-        processed_lines = list()
-        for index, line in enumerate(lines):
-            if index == 0:
-                processed_lines.append(line[:-1])
-            else:
-                processed_lines.append(line[1:-1])
-        lines = processed_lines
-        joined_phonemes = torch.cat(lines, dim=0)
-
-        # get durations of each phone in audio as average of an ensemble
-        alignment_paths = list()
-        ensemble_of_durations = list()
-        for ensemble in range(2):
-            alignment_paths.append(self.acoustic_model.inference(mel=melspec.to(self.device),
-                                                                 tokens=joined_phonemes.to(self.device),
-                                                                 save_img_for_debug="alignment.png" if ensemble == 1 else None,
-                                                                 return_ctc=False))
-        for alignment_path in alignment_paths:
-            ensemble_of_durations.append(self.dc(torch.LongTensor(alignment_path), vis=None).squeeze())
-        durations = list()
-        for i, _ in enumerate(ensemble_of_durations[0]):
-            duration_of_phone = list()
-            for ensemble_member in ensemble_of_durations:
-                duration_of_phone.append(ensemble_member.squeeze()[i])
-            durations.append(sum(duration_of_phone) / len(duration_of_phone))
-
-        # cut audio according to duration sum of each line in transcript
-        line_lens = [len(x) for x in lines]
-        index = 0
-        segment_durations = list()
-        for num_phones in line_lens:
-            segment_durations.append(sum(durations[index: index + num_phones]))
-            index += num_phones
-        spec_to_wave_factor = len(norm_wave) / sum(segment_durations)
-        wave_segment_lens = [int(x * spec_to_wave_factor) for x in segment_durations]
-        start_index = 0
-        wave_segments = list()
-        for index, segment_len in enumerate(wave_segment_lens):
-            if index == len(wave_segment_lens) - 1:
-                wave_segments.append(norm_wave[start_index:])
-            else:
-                wave_segments.append(norm_wave[start_index: start_index + segment_len])
-                start_index += segment_len
-
-        # write the audio segments into new files
-        for index, wave_segment in enumerate(wave_segments):
-            sf.write(f"split_{index + 1}.wav", wave_segment, 16000)
-
-
-meta_model = TTS_Interface()
-article = "<p style='text-align: left'>This is still a work in progress, models will be exchanged for better ones as soon as they are done. More diverse training data can help with more exact cloning. For example we are still trying to incorporate more singing data. </p><p style='text-align: center'><a href='https://github.com/DigitalPhonetics/IMS-Toucan' target='_blank'>Click here to learn more about the IMS Toucan Speech Synthesis Toolkit</a></p>"
-
-iface = gr.Interface(fn=meta_model.read,
-                     inputs=[gr.inputs.Dropdown(
-                         ["Hello, here is the first sentence. And here comes the second one. I think three sentences is enough to get the point across, right?",
-                          "I am excited! And my prosody is rather flat. And this sentence is shocking!",
-                          "Don't do it! But I want to! Then go ahead.",
-                          "How many examples do I realistically need? How about five? That should do it!",
-                          "Betty Botter bought some butter, but she said the butters bitter. If I put it in my batter, it will make my batter bitter. But a bit of better butter will make my batter better."],
+        self.text = "Quellen hattest du ihm, hattest dem Flüchtigen, kühle Schatten geschenkt, und die Gestade sahen, all ihm nach, und es bebte, aus den Wellen ihr lieblich Bild."
+        reference_audio = "reference_audios/2.wav"
+        self.duration, self.pitch, self.energy, _, _ = self.utterance_cloner.extract_prosody(self.text, reference_audio, lang="de", on_line_fine_tune=False)
+        self.phones = self.utterance_cloner.tts.text2phone.get_phone_string(self.text)
+
+        #######
+        self.utterance_cloner.tts.set_utterance_embedding("reference_audios/german_male.wav")
+        self.current_voice = "male"
+        self.cloned_speech_male = self.utterance_cloner.tts(self.phones,
+                                                            view=False,
+                                                            durations=self.duration,
+                                                            pitch=self.pitch,
+                                                            energy=self.energy,
+                                                            phones=True).cpu().numpy()
+        self.utterance_cloner.tts.set_utterance_embedding("reference_audios/german_female.wav")
+        self.current_voice = "female"
+        self.cloned_speech_female = self.utterance_cloner.tts(self.phones,
+                                                              view=False,
+                                                              durations=self.duration,
+                                                              pitch=self.pitch,
+                                                              energy=self.energy,
+                                                              phones=True).cpu().numpy()
+
+        #######
+        self.utterance_cloner.tts.set_utterance_embedding("reference_audios/german_male.wav")
+        self.current_voice = "male"
+        self.reg_speech_male = self.utterance_cloner.tts(
+            "Quellen hattest du ihm, hattest dem Flüchtigen kühle Schatten geschenkt, und die Gestade sahen all ihm nach, und es bebte aus den Wellen ihr lieblich Bild.",
+            view=False).cpu().numpy()
+        self.utterance_cloner.tts.set_utterance_embedding("reference_audios/german_female.wav")
+        self.current_voice = "female"
+        self.reg_speech_female = self.utterance_cloner.tts(
+            "Quellen hattest du ihm, hattest dem Flüchtigen kühle Schatten geschenkt, und die Gestade sahen all ihm nach, und es bebte aus den Wellen ihr lieblich Bild.",
+            view=False).cpu().numpy()
+
+    def read(self, _, speaker, lengthening, pause_dur, pitch_up):
+
+        if speaker == "Female Voice" and self.current_voice != "female":
+            self.utterance_cloner.tts.set_utterance_embedding("reference_audios/german_female.wav")
+            self.current_voice = "female"
+        elif speaker == "Male Voice" and self.current_voice != "male":
+            self.utterance_cloner.tts.set_utterance_embedding("reference_audios/german_male.wav")
+            self.current_voice = "male"
+
+        duration = self.duration.clone()
+        # lengthening
+        lenghtening_candidates = [  # ('f', 27),
+            # ('l', 28),
+            ('ʏ', 29),
+            ('ç', 30),
+            # ('t', 31),
+            ('ɪ', 32),
+            # ('ɡ', 33),
+            ('ə', 34),
+            ('n', 35),
+
+            # ('z', 66),
+            ('ɑ', 67),
+            # ('ə', 68),
+            ('n', 69),
+
+            # ('b', 84),
+            ('e', 85),
+            # ('p', 86),
+            # ('t', 87),
+            ('ə', 88)
+            ]
+
+        for lenghtening_candidate in lenghtening_candidates:
+            duration[lenghtening_candidate[1]] = duration[lenghtening_candidate[1]] + lengthening
+
+        # pauses
+        pause_candidates = [('~', 36),
+                            ('~', 70),
+                            ('~', 89)]
+
+        for pause_candidate in pause_candidates:
+            duration[pause_candidate[1]] = duration[pause_candidate[1]] + pause_dur
+
+        pitch = self.pitch.clone()
+        # pitch raise
+
+        pitch_candidates = [  # ('k', 37),
+            ('y', 38),
+            ('l', 39),
+            ('ə', 40),
+            ('ʃ', 41),
+            ('a', 42),
+            ('t', 43),
+            # ('ə', 44),
+            # ('n', 45),
+
+            ('a', 71),
+            ('l', 72),
+
+            ('v', 96),
+            ('ɛ', 97),
+            ('l', 98),
+            # ('ə', 99),
+            # ('n', 100)
+            ]
+
+        for pitch_candidate in pitch_candidates:
+            pitch[pitch_candidate[1]] = pitch[pitch_candidate[1]] + pitch_up
+
+        fixme = [('f', 27),
+                 ('l', 28),
+                 ('ʏ', 29),
+                 ('ç', 30),
+                 ('t', 31),
+                 ('ɪ', 32),
+                 ('ɡ', 33),
+                 ('ə', 34),
+                 ('n', 35)
+                 ]
+        for pitch_candidate in fixme:
+            pitch[pitch_candidate[1]] = pitch[pitch_candidate[1]] - abs(pitch_up)
+
+        manipulated_speech = self.utterance_cloner.tts(self.phones,
+                                                       view=False,
+                                                       durations=duration,
+                                                       pitch=pitch,
+                                                       energy=self.energy,
+                                                       phones=True).cpu()
+
+        if self.current_voice == "female":
+            cloned_speech = self.cloned_speech_female
+            reg_speech = self.reg_speech_female
+        else:
+            cloned_speech = self.cloned_speech_male
+            reg_speech = self.reg_speech_male
+
+        return (48000, float2pcm(reg_speech)), (48000, float2pcm(cloned_speech)), (48000, float2pcm(manipulated_speech.numpy()))
+
+
+poem_model = TTS_Interface()
+article = "<p style='text-align: left'>This is still a work in progress, models will be exchanged for better ones as soon as they are done. More diverse training data can help with more exact cloning and more controllability. For example we are still trying to incorporate more singing data. </p><p style='text-align: center'><a href='https://github.com/DigitalPhonetics/IMS-Toucan' target='_blank'>Click here to learn more about the IMS Toucan Speech Synthesis Toolkit</a></p>"
+
+iface = gr.Interface(fn=poem_model.read,
+                     inputs=[gr.inputs.Dropdown([
+                         "Quellen hattest du ihm, hattest dem Flüchtigen // kühle Schatten geschenkt, und die Gestade sahn // all ihm nach, und es bebte // aus den Wellen ihr lieblich Bild."],
                          type="value",
-                         default="Hello, here is the first sentence. And here comes the second one. I think three sentences is enough to get the point across, right?",
-                         label="Select which utterance should be customized"),
-                         gr.inputs.Dropdown(["Voice 1",
-                                             "Voice 2",
-                                             "Voice 3"], type="value", default="Voice 1", label="Speaker selection for the first sentence"),
-                         gr.inputs.Dropdown(["Voice 1",
-                                             "Voice 2",
-                                             "Voice 3"], type="value", default="Voice 2", label="Speaker selection for the second sentence"),
-                         gr.inputs.Dropdown(["Voice 1",
-                                             "Voice 2",
-                                             "Voice 3"], type="value", default="Voice 3", label="Speaker selection for the third sentence")],
-                     outputs=[gr.outputs.Image(label="Alignment of Phonemes to Audio"),
-                              gr.outputs.Audio(type="file", label="Original Audio"),
-                              gr.outputs.Audio(type="numpy", label="Customized Audio")],
+                         default="Quellen hattest du ihm, hattest dem Flüchtigen // kühle Schatten geschenkt, und die Gestade sahn // all ihm nach, und es bebte // aus den Wellen ihr lieblich Bild.",
+                         label="Poem Transcript"),
+                         gr.inputs.Dropdown(["Female Voice", "Male Voice"],
+                                            type="value",
+                                            default="Female Voice",
+                                            label="Select a Speaker"),
+                         gr.inputs.Slider(minimum=0, maximum=4, step=1, default=2, label="Lengthening on verse end"),
+                         gr.inputs.Slider(minimum=0, maximum=20, step=1, default=8, label="Length of Pause after verse end"),
+                         gr.inputs.Slider(minimum=-0.4, maximum=0.4, step=0.01, default=0.2, label="Raise Pitch on new verse")
+                         ],
+                     outputs=[gr.outputs.Audio(type="numpy", label="Poem read with prose reading"),
+                              gr.outputs.Audio(type="numpy", label="Poem cloned from a reference"),
+                              gr.outputs.Audio(type="numpy", label="Poem after human-in-the-loop adjustments")],
                      layout="vertical",
-                     title="IMS Toucan Speech Customization through Voice Cloning Demo",
+                     title="PoeticTTS - Customizing Poetry for Literary Studies",
                      thumbnail="Utility/toucan.png",
                      theme="default",
                      allow_flagging="never",
                      allow_screenshot=False,
-                     description="In this demo, an audio is split automatically into individual sentences. Then each of the sentences is re-synthesized into speech with the exact same prosody, but with a voice that you can choose. This allows customizing any existing read speech while retaining as much from the original reading as possible.",
+                     description="Customize how a poem is read by a text-to-speech system with intuitive high-level controls. You can control markers of syntactic phrasing ",
                      article=article)
 iface.launch(enable_queue=True)

run_utterance_cloner.py

@@ -82,8 +82,6 @@ class UtteranceCloner:
                 torch.nn.utils.clip_grad_norm_(acoustic_model.parameters(), 1.0)
                 optim_asr.step()
             acoustic_model.eval()
-        torch.save({"asr_model": acoustic_model.state_dict()},
-                   os.path.join(os.path.join("Models", "Aligner", "aligner.pt")))
 
         alignment_path = acoustic_model.inference(mel=melspec.to(self.device),
                                                   tokens=text.to(self.device),