3 apis

src/predict.py

@@ -129,6 +129,10 @@ class Predictor:
             sigma_schedule=KarrasSchedule(sigma_min=0.0001, sigma_max=3.0, rho=9.0), # empirical parameters
             clamp=False
         )
+        self.base_speaker_tts = BaseSpeakerTTS(f'{self.ckpt_base}/config.json', device=self.device)
+        self.base_speaker_tts.load_ckpt(f'{self.ckpt_base}/checkpoint.pth')
+        self.tone_color_converter = ToneColorConverter(f'{self.ckpt_converter}/config.json', device=self.device)
+        self.tone_color_converter.load_ckpt(f'{self.ckpt_converter}/checkpoint.pth')
 
 
     def predict(self,s3_url,passage,method_type='voice_clone'):
@@ -158,14 +162,86 @@ class Predictor:
             result = self.process_audio_file(processed_seg_dir,passage,model,sampler)
             final_output = os.path.join(results_dir,f"{gen_id}-voice-clone-1.wav")
             sf.write(final_output,result,24000)
+
+            base_speaker_tts,tone_color_converter = self.base_speaker_tts,self.tone_color_converter
+            reference_speaker = local_file_path
+            target_se, audio_name = se_extractor.get_se(reference_speaker, tone_color_converter, target_dir=openvoice_dir, vad=False)
+            src_path = os.path.join(results_dir,f"{gen_id}-tmp.wav")
+            openvoice_output = os.path.join(results_dir,f"{gen_id}-2.wav")
+            base_speaker_tts.tts(passage,src_path,speaker='default',language='English',speed=1.0)
+
+            source_se = torch.load(f'{self.ckpt_base}/en_default_se.pth').to(self.device)
+            tone_color_converter.convert(audio_src_path=src_path,src_se=source_se,tgt_se=target_se,output_path=openvoice_output,message='')
+            (new_sr, wav1) = self._fn(openvoice_output,"Midpoint",32,0.5)
+            denoised_openvoice_output = os.path.join(results_dir,f"{gen_id}-voice-clone-2.wav")
+            sf.write(denoised_openvoice_output,wav1,new_sr)
+
             
            
             mp3_final_output_1 = str(final_output).replace('wav','mp3')
+            mp3_final_output_2 = str(denoised_openvoice_output).replace('wav','mp3')
             self.convert_wav_to_mp3(final_output,mp3_final_output_1)
+            self.convert_wav_to_mp3(denoised_openvoice_output,mp3_final_output_2)
             print(mp3_final_output_1)
+            print(mp3_final_output_2)
+            
             self.upload_file_to_s3(mp3_final_output_1,'demovidelyusergenerations',f"{gen_id}-voice-clone-1.mp3")
-            return {"voice_clone_1":f"https://demovidelyusergenerations.s3.amazonaws.com/{gen_id}-voice-clone-1.mp3"}
+            self.upload_file_to_s3(mp3_final_output_2,'demovidelyusergenerations',f"{gen_id}-voice-clone-2.mp3")
+            shutil.rmtree(os.path.join(output_dir,gen_id))
+            return {"voice_clone_1":f"https://demovidelyusergenerations.s3.amazonaws.com/{gen_id}-voice-clone-1.mp3",
+                    "voice_clone_2":f"https://demovidelyusergenerations.s3.amazonaws.com/{gen_id}-voice-clone-2.mp3"
+                    }
+        if method_type == 'voice_clone_with_emotions':        
+            base_speaker_tts,tone_color_converter = self.base_speaker_tts,self.tone_color_converter
+            reference_speaker = local_file_path
+            target_se, audio_name = se_extractor.get_se(reference_speaker, tone_color_converter, target_dir=openvoice_dir, vad=False)
+            src_path = os.path.join(results_dir,f"{gen_id}-tmp-emotions.wav")
+            openvoice_output = os.path.join(results_dir,f"{gen_id}-4.wav")
+            base_speaker_tts.tts(passage,src_path,speaker='default',language='English',speed=1.0,use_emotions=True)
+            source_se = torch.load(f'{self.ckpt_base}/en_style_se.pth').to(self.device)
+            tone_color_converter.convert(audio_src_path=src_path,src_se=source_se,tgt_se=target_se,output_path=openvoice_output,message='')
+            (new_sr, wav1) = self._fn(openvoice_output,"Midpoint",32,0.5)
+            denoised_openvoice_output = os.path.join(results_dir,f"{gen_id}-with-emotions.wav")
+            sf.write(denoised_openvoice_output,wav1,new_sr)
+            
+            mp3_final_output_1 = str(denoised_openvoice_output).replace('wav','mp3')
+            self.convert_wav_to_mp3(denoised_openvoice_output,mp3_final_output_1)
+            print(mp3_final_output_1)
+            self.upload_file_to_s3(mp3_final_output_1,'demovidelyusergenerations',f"{gen_id}-voice-with-emotions.mp3")
+            shutil.rmtree(os.path.join(output_dir,gen_id))
+            return {"voice_clone_with_emotions":f"https://demovidelyusergenerations.s3.amazonaws.com/{gen_id}-voice-with-emotions.mp3"
+                    }
         
+        if method_type == 'voice_clone_multi_lang':
+            #voice clone with multi-lingugal
+            _,tone_color_converter = self.base_speaker_tts,self.tone_color_converter
+            reference_speaker = local_file_path
+            target_se, audio_name = se_extractor.get_se(reference_speaker, tone_color_converter, target_dir=openvoice_dir, vad=False)
+            src_path = 'openai_source_output.mp3'
+            source_se, audio_name = se_extractor.get_se(src_path, tone_color_converter, vad=True)
+
+            
+            client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+            response = client.audio.speech.create(
+                model="tts-1",
+                voice="fable",
+                input=passage
+            )
+
+            openai_multi_lang_path = os.path.join(results_dir,f"{gen_id}-openai-gen.wav")
+            response.stream_to_file(openai_multi_lang_path)
+            multi_lang_with_voice_clone_path = os.path.join(results_dir,f"{gen_id}-voice-clone-multi-lang.wav")
+
+            source_se, audio_name = se_extractor.get_se(src_path, tone_color_converter, vad=True)
+            self.tone_color_converter.convert(audio_src_path=openai_multi_lang_path, src_se=source_se, tgt_se=target_se, output_path=multi_lang_with_voice_clone_path,message='')
+
+            mp3_final_output_1 = str(multi_lang_with_voice_clone_path).replace('wav','mp3')
+            convert_wav_to_mp3(multi_lang_with_voice_clone_path,mp3_final_output_1)
+            print(mp3_final_output_1)
+            upload_file_to_s3(mp3_final_output_1,'demovidelyusergenerations',f"{gen_id}-voice-clone-multi-lang.mp3")
+            return {"voice_clone_with_emotions":f"https://demovidelyusergenerations.s3.amazonaws.com/{gen_id}-voice-clone-multi-lang.mp3"
+            }
+    
 
     def _fn(self,path, solver, nfe, tau):
         if path is None:
@@ -322,7 +398,7 @@ class Predictor:
                                     s_prev, 
                                     s_ref, 
                                     alpha = 0, 
-                                    beta = 0.3,  # make it more suitable for the text
+                                    beta = 0.1,  
                                     t = 0.7, 
                                     diffusion_steps=10, embedding_scale=1)
             wavs.append(wav)

src/processed/9ac5dfd2-1477-4903-adfc-1cc4d0351977/raw/039cf8da-75b8-474d-affa-fc84066c3fa3.wav

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:68f1fdaa436c8072a3d58f8234507be22e12302c77b78ee19b1a911168f96d33
-size 3098668

src/processed/9ac5dfd2-1477-4903-adfc-1cc4d0351977/results/9ac5dfd2-1477-4903-adfc-1cc4d0351977-voice-clone-1.mp3

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b639c38bce187bdaa36582877934f9896fe87fa4a05731ed2e5f1ce9bf794820
-size 1261173

src/processed/9ac5dfd2-1477-4903-adfc-1cc4d0351977/results/9ac5dfd2-1477-4903-adfc-1cc4d0351977-voice-clone-1.wav

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6dd8186db2bab0a68d1b6c432f1226c26b38bafb993a77734b0d79f4b32433c3
-size 4954644

src/processed/9ac5dfd2-1477-4903-adfc-1cc4d0351977/segments/039cf8da-75b8-474d-affa-fc84066c3fa3/wavs/039cf8da-75b8-474d-affa-fc84066c3fa3_seg0.wav

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:312b3f6534a92d88ea1d1fdbe12ded45c29360d602a74864d2787329b4dbeddd
-size 774616

src/processed/9ac5dfd2-1477-4903-adfc-1cc4d0351977/segments/039cf8da-75b8-474d-affa-fc84066c3fa3/wavs/039cf8da-75b8-474d-affa-fc84066c3fa3_seg1.wav

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dbff348a94a69ed3875f4a89e589b542da96af938248748de0e62b416fe76aa4
-size 774704

src/processed/9ac5dfd2-1477-4903-adfc-1cc4d0351977/segments/039cf8da-75b8-474d-affa-fc84066c3fa3/wavs/039cf8da-75b8-474d-affa-fc84066c3fa3_seg2.wav

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dcca412270a4dd74f260396a72772cdfbcdc557a9c66cf20b9bfec2b350778f1
-size 774704

src/processed/9ac5dfd2-1477-4903-adfc-1cc4d0351977/segments/039cf8da-75b8-474d-affa-fc84066c3fa3/wavs/039cf8da-75b8-474d-affa-fc84066c3fa3_seg3.wav

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d0ee46e6a9974651b2f2f5c350f7ea2a3e3590b5197394292ad31e51b1fc4bca
-size 774618

src/rp_handler.py

@@ -25,7 +25,7 @@ def run_voice_clone_job(job):
     assert method_type in ["create_voice","voice_clone","voice_clone_with_emotions","voice_clone_with_multi_lang"]
     s3_url = job_input['s3_url']
     passage = job_input['passage']
-    processed_urls = MODEL.predict(s3_url,passage)
+    processed_urls = MODEL.predict(s3_url,passage,method_type)
 
     return processed_urls