Upload 4 files

lib/infer.py

@@ -0,0 +1,211 @@
+import os
+import torch
+from multiprocessing import cpu_count
+from lib.infer.modules import VC
+from split_audio import split_silence_nonsilent, adjust_audio_lengths, combine_silence_nonsilent
+
+class Configs:
+    def __init__(self, device, is_half):
+        self.device = device
+        self.is_half = is_half
+        self.n_cpu = 0
+        self.gpu_name = None
+        self.gpu_mem = None
+        self.x_pad, self.x_query, self.x_center, self.x_max = self.device_config()
+
+    def device_config(self) -> tuple:
+        if torch.cuda.is_available():
+            i_device = int(self.device.split(":")[-1])
+            self.gpu_name = torch.cuda.get_device_name(i_device)
+            #if (
+#                    ("16" in self.gpu_name and "V100" not in self.gpu_name.upper())
+#                    or "P40" in self.gpu_name.upper()
+#                    or "1060" in self.gpu_name
+#                    or "1070" in self.gpu_name
+#                    or "1080" in self.gpu_name
+#            ):
+#                print("16 series/10 series P40 forced single precision")
+#                self.is_half = False
+#                for config_file in ["32k.json", "40k.json", "48k.json"]:
+#                    with open(BASE_DIR / "src" / "configs" / config_file, "r") as f:
+#                        strr = f.read().replace("true", "false")
+#                    with open(BASE_DIR / "src" / "configs" / config_file, "w") as f:
+#                        f.write(strr)
+#                with open(BASE_DIR / "src" / "trainset_preprocess_pipeline_print.py", "r") as f:
+#                    strr = f.read().replace("3.7", "3.0")
+#                with open(BASE_DIR / "src" / "trainset_preprocess_pipeline_print.py", "w") as f:
+#                    f.write(strr)
+#            else:
+#                self.gpu_name = None
+#            self.gpu_mem = int(
+#                torch.cuda.get_device_properties(i_device).total_memory
+#                / 1024
+#                / 1024
+#                / 1024
+#                + 0.4
+#            )
+#            if self.gpu_mem <= 4:
+#                with open(BASE_DIR / "src" / "trainset_preprocess_pipeline_print.py", "r") as f:
+#                    strr = f.read().replace("3.7", "3.0")
+#                with open(BASE_DIR / "src" / "trainset_preprocess_pipeline_print.py", "w") as f:
+#                    f.write(strr)
+        elif torch.backends.mps.is_available():
+            print("No supported N-card found, use MPS for inference")
+            self.device = "mps"
+        else:
+            print("No supported N-card found, use CPU for inference")
+            self.device = "cpu"
+            self.is_half = True
+
+        if self.n_cpu == 0:
+            self.n_cpu = cpu_count()
+
+        if self.is_half:
+            # 6G memory config
+            x_pad = 3
+            x_query = 10
+            x_center = 60
+            x_max = 65
+        else:
+            # 5G memory config
+            x_pad = 1
+            x_query = 6
+            x_center = 38
+            x_max = 41
+
+        if self.gpu_mem != None and self.gpu_mem <= 4:
+            x_pad = 1
+            x_query = 5
+            x_center = 30
+            x_max = 32
+
+        return x_pad, x_query, x_center, x_max
+
+def get_model(voice_model):
+    model_dir = os.path.join(models_dir, voice_model)
+    model_filename, index_filename = None, None
+    for file in os.listdir(model_dir):
+        ext = os.path.splitext(file)[1]
+        if ext == '.pth':
+            model_filename = file
+        if ext == '.index':
+            index_filename = file
+
+    if model_filename is None:
+        print(f'No model file exists in {models_dir}.')
+        return None, None
+
+    return os.path.join(model_dir, model_filename), os.path.join(model_dir, index_filename) if index_filename else ''
+
+def infer_audio(
+    model_name,
+    audio_path,
+    f0_change=0,
+    f0_method="rmvpe+",
+    min_pitch="50",
+    max_pitch="1100",
+    crepe_hop_length=128,
+    index_rate=0.75,
+    filter_radius=3,
+    rms_mix_rate=0.25,
+    protect=0.33,
+    split_infer=False,
+    min_silence=500,
+    silence_threshold=-50,
+    seek_step=1,
+    keep_silence=100,
+    do_formant=False,
+    quefrency=0,
+    timbre=1,
+    f0_autotune=False,
+    audio_format="wav",
+    resample_sr=0
+    ):
+    configs = Configs('cuda:0', True)
+    vc = VC(configs)
+    pth_path, index_path = get_model(model_name)
+    vc_data = vc.get_vc(pth_path, protect, 0.5)
+    
+    if split_infer:
+        inferred_files = []
+        temp_dir = os.path.join(main_dir, "seperate", "temp")
+        os.makedirs(temp_dir, exist_ok=True)
+        print("Splitting audio to silence and nonsilent segments.")
+        silence_files, nonsilent_files = split_silence_nonsilent(audio_path, min_silence, silence_threshold, seek_step, keep_silence)
+        print(f"Total silence segments: {len(silence_files)}.\nTotal nonsilent segments: {len(nonsilent_files)}.")
+        for i, nonsilent_file in enumerate(nonsilent_files):
+            print(f"Inferring nonsilent audio {i+1}")
+            inference_info, audio_data, output_path = vc.vc_single(
+            0,
+            audio_path,
+            f0_up_key,
+            f0_method,
+            index_path,
+            index_path,
+            index_rate,
+            filter_radius,
+            resample_sr,
+            rms_mix_rate,
+            protect,
+            audio_format,
+            crepe_hop_length,
+            do_formant,
+            quefrency,
+            timbre,
+            f0_min_pitch,
+            f0_max_pitch,
+            f0_autotune
+            )
+            if inference_info[0] == "Success.":
+                print("Inference ran successfully.")
+                print(inference_info[1])
+                print("Times:\nnpy: %.2fs f0: %.2fs infer: %.2fs\nTotal time: %.2fs" % (inference_info[2]*))
+            else:
+                print(f"An error occurred while processing.\n{inference_info[0]}")
+                return None
+            shutil.move(output_path, temp_dir)
+            inferred_files.append(os.path.join(temp_dir, os.path.basename(output_path)))
+        print("Adjusting inferred audio lengths.")
+        adjusted_inferred_files = adjust_audio_lengths(nonsilent_files, inferred_files)
+        print("Combining silence and inferred audios.")
+        output_count = 1
+        while True:
+            output_path = os.path.join(os.getcwd(), "output", f"{os.path.splitext(os.path.basename(audio_path))[0]}{model_name}{f0_method.capitalize()}_{output_count}.{audio_format}")
+            if not os.path.exists(output_path):
+                break
+            output_count += 1
+        inferred_audio = combine_silence_nonsilent(silence_files, adjusted_inferred_files, keep_silence, output_path)
+        shutil.rmtree(os.path.join(main_dir, "seperate", "temp"))
+    else:
+        inference_info, audio_data, output_path = vc.vc_single(
+            0,
+            audio_path,
+            f0_up_key,
+            f0_method,
+            index_path,
+            index_path,
+            index_rate,
+            filter_radius,
+            resample_sr,
+            rms_mix_rate,
+            protect,
+            audio_format,
+            crepe_hop_length,
+            do_formant,
+            quefrency,
+            timbre,
+            f0_min_pitch,
+            f0_max_pitch,
+            f0_autotune
+        )
+        if inference_info[0] == "Success.":
+            print("Inference ran successfully.")
+            print(inference_info[1])
+            print("Times:\nnpy: %.2fs f0: %.2fs infer: %.2fs\nTotal time: %.2fs" % (inference_info[2]*))
+        else:
+            print(f"An error occurred while processing.\n{inference_info[0]}")
+            return None
+    
+    del configs, vc
+    gc.collect()
+    return output_path

lib/modules.py

@@ -0,0 +1,578 @@
+import os, sys
+import traceback
+import logging
+now_dir = os.getcwd()
+sys.path.append(now_dir)
+logger = logging.getLogger(__name__)
+import numpy as np
+import soundfile as sf
+import torch
+from io import BytesIO
+from lib.infer_libs.audio import load_audio
+from lib.infer_libs.audio import wav2
+from lib.infer_libs.infer_pack.models import (
+    SynthesizerTrnMs256NSFsid,
+    SynthesizerTrnMs256NSFsid_nono,
+    SynthesizerTrnMs768NSFsid,
+    SynthesizerTrnMs768NSFsid_nono,
+)
+from lib.pipeline import Pipeline
+import time
+import glob
+from shutil import move
+from fairseq import checkpoint_utils
+
+sup_audioext = {
+    "wav",
+    "mp3",
+    "flac",
+    "ogg",
+    "opus",
+    "m4a",
+    "mp4",
+    "aac",
+    "alac",
+    "wma",
+    "aiff",
+    "webm",
+    "ac3",
+}
+
+def note_to_hz(note_name):
+    try:
+        SEMITONES = {'C': -9, 'C#': -8, 'D': -7, 'D#': -6, 'E': -5, 'F': -4, 'F#': -3, 'G': -2, 'G#': -1, 'A': 0, 'A#': 1, 'B': 2}
+        pitch_class, octave = note_name[:-1], int(note_name[-1])
+        semitone = SEMITONES[pitch_class]
+        note_number = 12 * (octave - 4) + semitone
+        frequency = 440.0 * (2.0 ** (1.0/12)) ** note_number
+        return frequency
+    except:
+        return None
+
+def load_hubert(hubert_model_path, config):
+    models, _, _ = checkpoint_utils.load_model_ensemble_and_task(
+        [hubert_model_path],
+        suffix="",
+    )
+    hubert_model = models[0]
+    hubert_model = hubert_model.to(config.device)
+    if config.is_half:
+        hubert_model = hubert_model.half()
+    else:
+        hubert_model = hubert_model.float()
+    return hubert_model.eval()
+
+class VC:
+    def __init__(self, config):
+        self.n_spk = None
+        self.tgt_sr = None
+        self.net_g = None
+        self.pipeline = None
+        self.cpt = None
+        self.version = None
+        self.if_f0 = None
+        self.version = None
+        self.hubert_model = None
+
+        self.config = config
+
+    def get_vc(self, sid, *to_return_protect):
+        logger.info("Get sid: " + sid)
+
+        to_return_protect0 = {
+            "visible": self.if_f0 != 0,
+            "value": to_return_protect[0]
+            if self.if_f0 != 0 and to_return_protect
+            else 0.5,
+            "__type__": "update",
+        }
+        to_return_protect1 = {
+            "visible": self.if_f0 != 0,
+            "value": to_return_protect[1]
+            if self.if_f0 != 0 and to_return_protect
+            else 0.33,
+            "__type__": "update",
+        }
+
+        if sid == "" or sid == []:
+            if self.hubert_model is not None:  # 考虑到轮询, 需要加个判断看是否 sid 是由有模型切换到无模型的
+                logger.info("Clean model cache")
+                del (
+                    self.net_g,
+                    self.n_spk,
+                    self.vc,
+                    self.hubert_model,
+                    self.tgt_sr,
+                )  # ,cpt
+                self.hubert_model = (
+                    self.net_g
+                ) = self.n_spk = self.vc = self.hubert_model = self.tgt_sr = None
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+                ###楼下不这么折腾清理不干净
+                self.if_f0 = self.cpt.get("f0", 1)
+                self.version = self.cpt.get("version", "v1")
+                if self.version == "v1":
+                    if self.if_f0 == 1:
+                        self.net_g = SynthesizerTrnMs256NSFsid(
+                            *self.cpt["config"], is_half=self.config.is_half
+                        )
+                    else:
+                        self.net_g = SynthesizerTrnMs256NSFsid_nono(*self.cpt["config"])
+                elif self.version == "v2":
+                    if self.if_f0 == 1:
+                        self.net_g = SynthesizerTrnMs768NSFsid(
+                            *self.cpt["config"], is_half=self.config.is_half
+                        )
+                    else:
+                        self.net_g = SynthesizerTrnMs768NSFsid_nono(*self.cpt["config"])
+                del self.net_g, self.cpt
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+            return (
+                {"visible": False, "__type__": "update"},
+                {
+                    "visible": True,
+                    "value": to_return_protect0,
+                    "__type__": "update",
+                },
+                {
+                    "visible": True,
+                    "value": to_return_protect1,
+                    "__type__": "update",
+                },
+                "",
+                "",
+            )
+        #person = f'{os.getenv("weight_root")}/{sid}'
+        person = f'{sid}'
+        #logger.info(f"Loading: {person}")
+        logger.info(f"Loading...")
+        self.cpt = torch.load(person, map_location="cpu")
+        self.tgt_sr = self.cpt["config"][-1]
+        self.cpt["config"][-3] = self.cpt["weight"]["emb_g.weight"].shape[0]  # n_spk
+        self.if_f0 = self.cpt.get("f0", 1)
+        self.version = self.cpt.get("version", "v1")
+
+        synthesizer_class = {
+            ("v1", 1): SynthesizerTrnMs256NSFsid,
+            ("v1", 0): SynthesizerTrnMs256NSFsid_nono,
+            ("v2", 1): SynthesizerTrnMs768NSFsid,
+            ("v2", 0): SynthesizerTrnMs768NSFsid_nono,
+        }
+
+        self.net_g = synthesizer_class.get(
+            (self.version, self.if_f0), SynthesizerTrnMs256NSFsid
+        )(*self.cpt["config"], is_half=self.config.is_half)
+
+        del self.net_g.enc_q
+
+        self.net_g.load_state_dict(self.cpt["weight"], strict=False)
+        self.net_g.eval().to(self.config.device)
+        if self.config.is_half:
+            self.net_g = self.net_g.half()
+        else:
+            self.net_g = self.net_g.float()
+
+        self.pipeline = Pipeline(self.tgt_sr, self.config)
+        n_spk = self.cpt["config"][-3]
+        #index = {"value": get_index_path_from_model(sid), "__type__": "update"}
+        #logger.info("Select index: " + index["value"])
+
+        return (
+            (
+                {"visible": False, "maximum": n_spk, "__type__": "update"},
+                to_return_protect0,
+                to_return_protect1
+            )
+            if to_return_protect
+            else {"visible": False, "maximum": n_spk, "__type__": "update"}
+        )
+
+    def vc_single_dont_save(
+        self,
+        sid,
+        input_audio_path1,
+        f0_up_key,
+        f0_method,
+        file_index,
+        file_index2,
+        index_rate,
+        filter_radius,
+        resample_sr,
+        rms_mix_rate,
+        protect,
+        crepe_hop_length,
+        do_formant,
+        quefrency,
+        timbre,
+        f0_min,
+        f0_max,
+        f0_autotune,
+        hubert_model_path = "assets/hubert/hubert_base.pt"
+    ):
+        """
+        Performs inference without saving
+    
+        Parameters:
+        - sid (int)
+        - input_audio_path1 (str)
+        - f0_up_key (int)
+        - f0_method (str)
+        - file_index (str)
+        - file_index2 (str)
+        - index_rate (float)
+        - filter_radius (int)
+        - resample_sr (int)
+        - rms_mix_rate (float)
+        - protect (float)
+        - crepe_hop_length (int)
+        - do_formant (bool)
+        - quefrency (float)
+        - timbre (float)
+        - f0_min (str)
+        - f0_max (str)
+        - f0_autotune (bool)
+        - hubert_model_path (str)
+
+        Returns:
+        Tuple(Tuple(status, index_info, times), Tuple(sr, data)):
+            - Tuple(status, index_info, times):
+                - status (str): either "Success." or an error
+                - index_info (str): index path if used
+                - times (list): [npy_time, f0_time, infer_time, total_time]
+            - Tuple(sr, data): Audio data results.
+        """
+        global total_time
+        total_time = 0
+        start_time = time.time()
+        
+        if not input_audio_path1:
+            return "You need to upload an audio", None
+        
+        if not os.path.exists(input_audio_path1):
+            return "Audio was not properly selected or doesn't exist", None
+        
+        f0_up_key = int(f0_up_key)
+        if not isdigit(f0_min):
+            f0_min = note_to_hz(f0_min)
+            if f0_min:
+                print(f"Converted Min pitch: freq - {f0_min}")
+            else:
+                f0_min = 50
+                print("Invalid minimum pitch note. Defaulting to 50hz.")
+        else:
+            f0_min = float(f0_min)
+        if not isdigit(f0_max):
+            f0_max = note_to_hz(f0_max)
+            if f0_max:
+                print(f"Converted Max pitch: freq - {f0_max}")
+            else:
+                f0_max = 1100
+                print("Invalid maximum pitch note. Defaulting to 1100hz.")
+        else:
+            f0_max = float(f0_max)
+        
+        try:
+            print(f"Attempting to load {input_audio_path1}....")
+            audio = load_audio(file=input_audio_path1,
+                               sr=16000,
+                               DoFormant=do_formant,
+                               Quefrency=quefrency,
+                               Timbre=timbre)
+            
+            audio_max = np.abs(audio).max() / 0.95
+            if audio_max > 1:
+                audio /= audio_max
+            times = [0, 0, 0]
+
+            if self.hubert_model is None:
+                self.hubert_model = load_hubert(hubert_model_path, self.config)
+
+            try:
+                self.if_f0 = self.cpt.get("f0", 1)
+            except NameError:
+                message = "Model was not properly selected"
+                print(message)
+                return message, None
+            
+            if file_index and not file_index == "" and isinstance(file_index, str):
+                file_index = file_index.strip(" ") \
+                .strip('"') \
+                .strip("\n") \
+                .strip('"') \
+                .strip(" ") \
+                .replace("trained", "added")
+            elif file_index2:
+                file_index = file_index2
+            else:
+                file_index = ""  
+
+            try:
+                audio_opt = self.pipeline.pipeline(
+                    self.hubert_model,
+                    self.net_g,
+                    sid,
+                    audio,
+                    input_audio_path1,
+                    times,
+                    f0_up_key,
+                    f0_method,
+                    file_index,
+                    index_rate,
+                    self.if_f0,
+                    filter_radius,
+                    self.tgt_sr,
+                    resample_sr,
+                    rms_mix_rate,
+                    self.version,
+                    protect,
+                    crepe_hop_length,
+                    f0_autotune,
+                    f0_min=f0_min,
+                    f0_max=f0_max
+                    )
+            except AssertionError:
+                message = "Mismatching index version detected (v1 with v2, or v2 with v1)."
+                print(message)
+                return message, None
+            except NameError:
+                message = "RVC libraries are still loading. Please try again in a few seconds."
+                print(message)
+                return message, None
+
+            if self.tgt_sr != resample_sr >= 16000:
+                tgt_sr = resample_sr
+            else:
+                tgt_sr = self.tgt_sr
+            index_info = (
+                "Index: %s." % file_index
+                if isinstance(file_index, str) and os.path.exists(file_index)
+                else "Index not used."
+            )
+            end_time = time.time()
+            total_time = end_time - start_time
+            times.append(total_time)
+            return (
+                ("Success.", index_info, times),
+                (tgt_sr, audio_opt),
+            )
+        except:
+            info = traceback.format_exc()
+            logger.warn(info)
+            return (
+                (info, None, [None, None, None, None]),
+                (None, None)
+            )
+
+    def vc_single(
+        self,
+        sid,
+        input_audio_path1,
+        f0_up_key,
+        f0_file,
+        f0_method,
+        file_index,
+        file_index2,
+        index_rate,
+        filter_radius,
+        resample_sr,
+        rms_mix_rate,
+        protect,
+        format1,
+        crepe_hop_length,
+        do_formant,
+        quefrency,
+        timbre,
+        f0_min,
+        f0_max,
+        f0_autotune,
+        hubert_model_path = "assets/hubert/hubert_base.pt"
+    ):
+        """
+        Performs inference with saving
+    
+        Parameters:
+        - sid (int)
+        - input_audio_path1 (str)
+        - f0_up_key (int)
+        - f0_method (str)
+        - file_index (str)
+        - file_index2 (str)
+        - index_rate (float)
+        - filter_radius (int)
+        - resample_sr (int)
+        - rms_mix_rate (float)
+        - protect (float)
+        - format1 (str)
+        - crepe_hop_length (int)
+        - do_formant (bool)
+        - quefrency (float)
+        - timbre (float)
+        - f0_min (str)
+        - f0_max (str)
+        - f0_autotune (bool)
+        - hubert_model_path (str)
+
+        Returns:
+        Tuple(Tuple(status, index_info, times), Tuple(sr, data), output_path):
+            - Tuple(status, index_info, times):
+                - status (str): either "Success." or an error
+                - index_info (str): index path if used
+                - times (list): [npy_time, f0_time, infer_time, total_time]
+            - Tuple(sr, data): Audio data results.
+            - output_path (str): Audio results path
+        """
+        global total_time
+        total_time = 0
+        start_time = time.time()
+        
+        if not input_audio_path1:
+            return "You need to upload an audio", None, None
+        
+        if not os.path.exists(input_audio_path1):
+            return "Audio was not properly selected or doesn't exist", None, None
+
+        f0_up_key = int(f0_up_key)
+        if not isdigit(f0_min):
+            f0_min = note_to_hz(f0_min)
+            if f0_min:
+                print(f"Converted Min pitch: freq - {f0_min}")
+            else:
+                f0_min = 50
+                print("Invalid minimum pitch note. Defaulting to 50hz.")
+        else:
+            f0_min = float(f0_min)
+        if not isdigit(f0_max):
+            f0_max = note_to_hz(f0_max)
+            if f0_max:
+                print(f"Converted Max pitch: freq - {f0_max}")
+            else:
+                f0_max = 1100
+                print("Invalid maximum pitch note. Defaulting to 1100hz.")
+        else:
+            f0_max = float(f0_max)
+
+        try:
+            print(f"Attempting to load {input_audio_path1}...")
+            audio = load_audio(file=input_audio_path1,
+                               sr=16000,
+                               DoFormant=do_formant,
+                               Quefrency=quefrency,
+                               Timbre=timbre)
+            
+            audio_max = np.abs(audio).max() / 0.95
+            if audio_max > 1:
+                audio /= audio_max
+            times = [0, 0, 0]
+
+            if self.hubert_model is None:
+                self.hubert_model = load_hubert(hubert_model_path, self.config)
+
+            try:
+                self.if_f0 = self.cpt.get("f0", 1)
+            except NameError:
+                message = "Model was not properly selected"
+                print(message)
+                return message, None
+            if file_index and not file_index == "" and isinstance(file_index, str):
+                file_index = file_index.strip(" ") \
+                .strip('"') \
+                .strip("\n") \
+                .strip('"') \
+                .strip(" ") \
+                .replace("trained", "added")
+            elif file_index2:
+                file_index = file_index2
+            else:
+                file_index = ""
+
+            try:
+                audio_opt = self.pipeline.pipeline(
+                    self.hubert_model,
+                    self.net_g,
+                    sid,
+                    audio,
+                    input_audio_path1,
+                    times,
+                    f0_up_key,
+                    f0_method,
+                    file_index,
+                    index_rate,
+                    self.if_f0,
+                    filter_radius,
+                    self.tgt_sr,
+                    resample_sr,
+                    rms_mix_rate,
+                    self.version,
+                    protect,
+                    crepe_hop_length,
+                    f0_autotune,
+                    f0_min=f0_min,
+                    f0_max=f0_max
+                    )
+            except AssertionError:
+                message = "Mismatching index version detected (v1 with v2, or v2 with v1)."
+                print(message)
+                return message, None
+            except NameError:
+                message = "RVC libraries are still loading. Please try again in a few seconds."
+                print(message)
+                return message, None
+
+            if self.tgt_sr != resample_sr >= 16000:
+                tgt_sr = resample_sr
+            else:
+                tgt_sr = self.tgt_sr
+            index_info = (
+                "Index: %s." % file_index
+                if isinstance(file_index, str) and os.path.exists(file_index)
+                else "Index not used."
+            )
+            
+            opt_root = os.path.join(os.getcwd(), "output")
+            os.makedirs(opt_root, exist_ok=True)
+            output_count = 1
+            
+            while True:
+                opt_filename = f"{os.path.splitext(os.path.basename(input_audio_path1))[0]}{os.path.basename(os.path.dirname(file_index))}{f0_method.capitalize()}_{output_count}.{format1}"
+                current_output_path = os.path.join(opt_root, opt_filename)
+                if not os.path.exists(current_output_path):
+                    break
+                output_count += 1
+            try:
+                if format1 in ["wav", "flac"]:
+                    sf.write(
+                        current_output_path,
+                        audio_opt,
+                        self.tgt_sr,
+                    )
+                else:
+                    with BytesIO() as wavf:
+                        sf.write(
+                            wavf,
+                            audio_opt,
+                            self.tgt_sr,
+                            format="wav"
+                        )
+                        wavf.seek(0, 0)
+                        with open(current_output_path, "wb") as outf:
+                                wav2(wavf, outf, format1)
+            except:
+                info = traceback.format_exc()
+            end_time = time.time()
+            total_time = end_time - start_time
+            times.append(total_time)
+            return (
+                ("Success.", index_info, times),
+                (tgt_sr, audio_opt),
+                current_output_path
+            )
+        except:
+            info = traceback.format_exc()
+            logger.warn(info)
+            return (
+                (info, None, [None, None, None, None]),
+                (None, None),
+                None
+            )

lib/pipeline.py

@@ -0,0 +1,766 @@
+import os
+import sys
+import gc
+import traceback
+import logging
+
+logger = logging.getLogger(__name__)
+
+from functools import lru_cache
+from time import time as ttime
+from torch import Tensor
+import faiss
+import librosa
+import numpy as np
+import parselmouth
+import pyworld
+import torch.nn.functional as F
+from scipy import signal
+from tqdm import tqdm
+
+import random
+now_dir = os.getcwd()
+sys.path.append(now_dir)
+import re
+from functools import partial
+bh, ah = signal.butter(N=5, Wn=48, btype="high", fs=16000)
+
+input_audio_path2wav = {}
+import torchcrepe  # Fork Feature. Crepe algo for training and preprocess
+import torch
+from lib.infer_libs.rmvpe import RMVPE
+from lib.infer_libs.fcpe import FCPE
+
+@lru_cache
+def cache_harvest_f0(input_audio_path, fs, f0max, f0min, frame_period):
+    audio = input_audio_path2wav[input_audio_path]
+    f0, t = pyworld.harvest(
+        audio,
+        fs=fs,
+        f0_ceil=f0max,
+        f0_floor=f0min,
+        frame_period=frame_period,
+    )
+    f0 = pyworld.stonemask(audio, f0, t, fs)
+    return f0
+
+
+def change_rms(data1, sr1, data2, sr2, rate):  # 1是输入音频，2是输出音频,rate是2的占比
+    # print(data1.max(),data2.max())
+    rms1 = librosa.feature.rms(
+        y=data1, frame_length=sr1 // 2 * 2, hop_length=sr1 // 2
+    )  # 每半秒一个点
+    rms2 = librosa.feature.rms(y=data2, frame_length=sr2 // 2 * 2, hop_length=sr2 // 2)
+    rms1 = torch.from_numpy(rms1)
+    rms1 = F.interpolate(
+        rms1.unsqueeze(0), size=data2.shape[0], mode="linear"
+    ).squeeze()
+    rms2 = torch.from_numpy(rms2)
+    rms2 = F.interpolate(
+        rms2.unsqueeze(0), size=data2.shape[0], mode="linear"
+    ).squeeze()
+    rms2 = torch.max(rms2, torch.zeros_like(rms2) + 1e-6)
+    data2 *= (
+        torch.pow(rms1, torch.tensor(1 - rate))
+        * torch.pow(rms2, torch.tensor(rate - 1))
+    ).numpy()
+    return data2
+
+
+class Pipeline(object):
+    def __init__(self, tgt_sr, config):
+        self.x_pad, self.x_query, self.x_center, self.x_max, self.is_half = (
+            config.x_pad,
+            config.x_query,
+            config.x_center,
+            config.x_max,
+            config.is_half,
+        )
+        self.sr = 16000  # hubert输入采样率
+        self.window = 160  # 每帧点数
+        self.t_pad = self.sr * self.x_pad  # 每条前后pad时间
+        self.t_pad_tgt = tgt_sr * self.x_pad
+        self.t_pad2 = self.t_pad * 2
+        self.t_query = self.sr * self.x_query  # 查询切点前后查询时间
+        self.t_center = self.sr * self.x_center  # 查询切点位置
+        self.t_max = self.sr * self.x_max  # 免查询时长阈值
+        self.device = config.device
+        self.model_rmvpe = RMVPE("%s/rmvpe.pt" % os.environ["rmvpe_root"], is_half=self.is_half, device=self.device)
+
+        self.note_dict = [
+            65.41, 69.30, 73.42, 77.78, 82.41, 87.31,
+            92.50, 98.00, 103.83, 110.00, 116.54, 123.47,
+            130.81, 138.59, 146.83, 155.56, 164.81, 174.61,
+            185.00, 196.00, 207.65, 220.00, 233.08, 246.94,
+            261.63, 277.18, 293.66, 311.13, 329.63, 349.23,
+            369.99, 392.00, 415.30, 440.00, 466.16, 493.88,
+            523.25, 554.37, 587.33, 622.25, 659.25, 698.46,
+            739.99, 783.99, 830.61, 880.00, 932.33, 987.77,
+            1046.50, 1108.73, 1174.66, 1244.51, 1318.51, 1396.91,
+            1479.98, 1567.98, 1661.22, 1760.00, 1864.66, 1975.53,
+            2093.00, 2217.46, 2349.32, 2489.02, 2637.02, 2793.83,
+            2959.96, 3135.96, 3322.44, 3520.00, 3729.31, 3951.07
+        ]
+
+    # Fork Feature: Get the best torch device to use for f0 algorithms that require a torch device. Will return the type (torch.device)
+    def get_optimal_torch_device(self, index: int = 0) -> torch.device:
+        if torch.cuda.is_available():
+            return torch.device(
+                f"cuda:{index % torch.cuda.device_count()}"
+            )  # Very fast
+        elif torch.backends.mps.is_available():
+            return torch.device("mps")
+        return torch.device("cpu")
+
+    # Fork Feature: Compute f0 with the crepe method
+    def get_f0_crepe_computation(
+        self,
+        x,
+        f0_min,
+        f0_max,
+        p_len,
+        *args,  # 512 before. Hop length changes the speed that the voice jumps to a different dramatic pitch. Lower hop lengths means more pitch accuracy but longer inference time.
+        **kwargs,  # Either use crepe-tiny "tiny" or crepe "full". Default is full
+    ):
+        x = x.astype(
+            np.float32
+        )  # fixes the F.conv2D exception. We needed to convert double to float.
+        x /= np.quantile(np.abs(x), 0.999)
+        torch_device = self.get_optimal_torch_device()
+        audio = torch.from_numpy(x).to(torch_device, copy=True)
+        audio = torch.unsqueeze(audio, dim=0)
+        if audio.ndim == 2 and audio.shape[0] > 1:
+            audio = torch.mean(audio, dim=0, keepdim=True).detach()
+        audio = audio.detach()
+        hop_length = kwargs.get('crepe_hop_length', 160)
+        model = kwargs.get('model', 'full') 
+        print("Initiating prediction with a crepe_hop_length of: " + str(hop_length))
+        pitch: Tensor = torchcrepe.predict(
+            audio,
+            self.sr,
+            hop_length,
+            f0_min,
+            f0_max,
+            model,
+            batch_size=hop_length * 2,
+            device=torch_device,
+            pad=True,
+        )
+        p_len = p_len or x.shape[0] // hop_length
+        # Resize the pitch for final f0
+        source = np.array(pitch.squeeze(0).cpu().float().numpy())
+        source[source < 0.001] = np.nan
+        target = np.interp(
+            np.arange(0, len(source) * p_len, len(source)) / p_len,
+            np.arange(0, len(source)),
+            source,
+        )
+        f0 = np.nan_to_num(target)
+        return f0  # Resized f0
+    
+    def get_f0_official_crepe_computation(
+        self,
+        x,
+        f0_min,
+        f0_max,
+        *args,
+        **kwargs
+    ):
+        # Pick a batch size that doesn't cause memory errors on your gpu
+        batch_size = 512
+        # Compute pitch using first gpu
+        audio = torch.tensor(np.copy(x))[None].float()
+        model = kwargs.get('model', 'full')
+        f0, pd = torchcrepe.predict(
+            audio,
+            self.sr,
+            self.window,
+            f0_min,
+            f0_max,
+            model,
+            batch_size=batch_size,
+            device=self.device,
+            return_periodicity=True,
+        )
+        pd = torchcrepe.filter.median(pd, 3)
+        f0 = torchcrepe.filter.mean(f0, 3)
+        f0[pd < 0.1] = 0
+        f0 = f0[0].cpu().numpy()
+        return f0
+
+    # Fork Feature: Compute pYIN f0 method
+    def get_f0_pyin_computation(self, x, f0_min, f0_max):
+        y, sr = librosa.load(x, sr=self.sr, mono=True)
+        f0, _, _ = librosa.pyin(y, fmin=f0_min, fmax=f0_max, sr=self.sr)
+        f0 = f0[1:]  # Get rid of extra first frame
+        return f0
+
+    def get_pm(self, x, p_len, *args, **kwargs):
+        f0 = parselmouth.Sound(x, self.sr).to_pitch_ac(
+            time_step=160 / 16000,
+            voicing_threshold=0.6,
+            pitch_floor=kwargs.get('f0_min'),
+            pitch_ceiling=kwargs.get('f0_max'),
+        ).selected_array["frequency"]
+        
+        return np.pad(
+            f0,
+            [[max(0, (p_len - len(f0) + 1) // 2), max(0, p_len - len(f0) - (p_len - len(f0) + 1) // 2)]],
+            mode="constant"
+        )
+
+    def get_harvest(self, x, *args, **kwargs):
+        f0_spectral = pyworld.harvest(
+            x.astype(np.double),
+            fs=self.sr,
+            f0_ceil=kwargs.get('f0_max'),
+            f0_floor=kwargs.get('f0_min'),
+            frame_period=1000 * kwargs.get('hop_length', 160) / self.sr,
+        )
+        return pyworld.stonemask(x.astype(np.double), *f0_spectral, self.sr)
+
+    def get_dio(self, x, *args, **kwargs):
+        f0_spectral = pyworld.dio(
+            x.astype(np.double),
+            fs=self.sr,
+            f0_ceil=kwargs.get('f0_max'),
+            f0_floor=kwargs.get('f0_min'),
+            frame_period=1000 * kwargs.get('hop_length', 160) / self.sr,
+        )
+        return pyworld.stonemask(x.astype(np.double), *f0_spectral, self.sr)
+
+
+    def get_rmvpe(self, x, *args, **kwargs):
+        if not hasattr(self, "model_rmvpe"):
+            from lib.infer.infer_libs.rmvpe import RMVPE
+            
+            logger.info(
+                "Loading rmvpe model,%s" % "%s/rmvpe.pt" % os.environ["rmvpe_root"]
+            )
+            self.model_rmvpe = RMVPE(
+                "%s/rmvpe.pt" % os.environ["rmvpe_root"],
+                is_half=self.is_half,
+                device=self.device,
+            )
+        f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03)
+
+        if "privateuseone" in str(self.device):  # clean ortruntime memory
+                del self.model_rmvpe.model
+                del self.model_rmvpe
+                logger.info("Cleaning ortruntime memory")
+            
+        return f0
+    
+
+    def get_pitch_dependant_rmvpe(self, x, f0_min=1, f0_max=40000, *args, **kwargs):
+        if not hasattr(self, "model_rmvpe"):
+            from lib.infer.infer_libs.rmvpe import RMVPE
+            
+            logger.info(
+                "Loading rmvpe model,%s" % "%s/rmvpe.pt" % os.environ["rmvpe_root"]
+            )
+            self.model_rmvpe = RMVPE(
+                "%s/rmvpe.pt" % os.environ["rmvpe_root"],
+                is_half=self.is_half,
+                device=self.device,
+            )
+        f0 = self.model_rmvpe.infer_from_audio_with_pitch(x, thred=0.03, f0_min=f0_min, f0_max=f0_max)   
+        if "privateuseone" in str(self.device):  # clean ortruntime memory
+                del self.model_rmvpe.model
+                del self.model_rmvpe
+                logger.info("Cleaning ortruntime memory")
+            
+        return f0
+        
+    def get_fcpe(self, x, f0_min, f0_max, p_len, *args, **kwargs):
+        self.model_fcpe = FCPEF0Predictor("%s/fcpe.pt" % os.environ["fcpe_root"], f0_min=f0_min, f0_max=f0_max, dtype=torch.float32, device=self.device, sampling_rate=self.sr, threshold=0.03)
+        f0 = self.model_fcpe.compute_f0(x, p_len=p_len)
+        del self.model_fcpe
+        gc.collect()
+        return f0
+
+    def autotune_f0(self, f0):
+        autotuned_f0 = []
+        for freq in f0:
+            closest_notes = [x for x in self.note_dict if abs(x - freq) == min(abs(n - freq) for n in self.note_dict)]
+            autotuned_f0.append(random.choice(closest_notes))
+        return np.array(autotuned_f0, np.float64)
+
+
+    # Fork Feature: Acquire median hybrid f0 estimation calculation
+    def get_f0_hybrid_computation(
+        self,
+        methods_str,
+        input_audio_path,
+        x,
+        f0_min,
+        f0_max,
+        p_len,
+        filter_radius,
+        crepe_hop_length,
+        time_step,
+    ):
+        # Get various f0 methods from input to use in the computation stack
+        methods_str = re.search('hybrid\[(.+)\]', methods_str)
+        if methods_str:  # Ensure a match was found
+            methods = [method.strip() for method in methods_str.group(1).split('+')]
+        f0_computation_stack = []
+
+        print("Calculating f0 pitch estimations for methods: %s" % str(methods))
+        x = x.astype(np.float32)
+        x /= np.quantile(np.abs(x), 0.999)
+        # Get f0 calculations for all methods specified
+        for method in methods:
+            f0 = None
+            if method == "pm":
+                f0 = self.get_pm(x, p_len=p_len)
+            elif method == "crepe":
+                f0 = self.get_f0_official_crepe_computation(x, f0_min, f0_max, model="full")
+                f0 = f0[1:]
+            elif method == "crepe-tiny":
+                f0 = self.get_f0_official_crepe_computation(x, f0_min, f0_max, model="tiny")
+                f0 = f0[1:]  # Get rid of extra first frame
+            elif method == "mangio-crepe":
+                f0 = self.get_f0_crepe_computation(
+                    x, f0_min, f0_max, p_len, crepe_hop_length=crepe_hop_length
+                )
+            elif method == "mangio-crepe-tiny":
+                f0 = self.get_f0_crepe_computation(
+                    x, f0_min, f0_max, p_len, crepe_hop_length=crepe_hop_length, model="tiny"
+                )
+            elif method == "harvest":
+                f0 = self.get_harvest(x)
+                f0 = f0[1:]
+            elif method == "dio":
+                f0 = self.get_dio(x)
+                f0 = f0[1:]
+            elif method == "rmvpe":
+                f0 = self.get_rmvpe(x)
+                f0 = f0[1:]
+            elif method == "fcpe":
+                f0 = self.get_fcpe(x, f0_min=f0_min, f0_max=f0_max, p_len=p_len)
+            elif method == "pyin":
+                f0 = self.get_f0_pyin_computation(input_audio_path, f0_min, f0_max)
+            # Push method to the stack
+            f0_computation_stack.append(f0)
+
+        for fc in f0_computation_stack:
+            print(len(fc))
+
+        print("Calculating hybrid median f0 from the stack of: %s" % str(methods))
+        f0_median_hybrid = None
+        if len(f0_computation_stack) == 1:
+            f0_median_hybrid = f0_computation_stack[0]
+        else:
+            f0_median_hybrid = np.nanmedian(f0_computation_stack, axis=0)
+        return f0_median_hybrid
+    
+    def get_f0(
+        self,
+        input_audio_path,
+        x,
+        p_len,
+        f0_up_key,
+        f0_method,
+        filter_radius,
+        crepe_hop_length,
+        f0_autotune,
+        inp_f0=None,
+        f0_min=50,
+        f0_max=1100,
+    ):
+        global input_audio_path2wav
+        time_step = self.window / self.sr * 1000
+        f0_min = f0_min
+        f0_max = f0_max
+        f0_mel_min = 1127 * np.log(1 + f0_min / 700)
+        f0_mel_max = 1127 * np.log(1 + f0_max / 700)
+
+        if f0_method == "pm":
+            f0 = (
+                parselmouth.Sound(x, self.sr)
+                .to_pitch_ac(
+                    time_step=time_step / 1000,
+                    voicing_threshold=0.6,
+                    pitch_floor=f0_min,
+                    pitch_ceiling=f0_max,
+                )
+                .selected_array["frequency"]
+            )
+            pad_size = (p_len - len(f0) + 1) // 2
+            if pad_size > 0 or p_len - len(f0) - pad_size > 0:
+                f0 = np.pad(
+                    f0, [[pad_size, p_len - len(f0) - pad_size]], mode="constant"
+                )
+        elif f0_method == "harvest":
+            input_audio_path2wav[input_audio_path] = x.astype(np.double)
+            f0 = cache_harvest_f0(input_audio_path, self.sr, f0_max, f0_min, 10)
+            if filter_radius > 2:
+                f0 = signal.medfilt(f0, 3)
+        elif f0_method == "dio":  # Potentially Buggy?
+            f0, t = pyworld.dio(
+                x.astype(np.double),
+                fs=self.sr,
+                f0_ceil=f0_max,
+                f0_floor=f0_min,
+                frame_period=10,
+            )
+            f0 = pyworld.stonemask(x.astype(np.double), f0, t, self.sr)
+            f0 = signal.medfilt(f0, 3)
+        elif f0_method == "crepe":
+            model = "full"
+            # Pick a batch size that doesn't cause memory errors on your gpu
+            batch_size = 512
+            # Compute pitch using first gpu
+            audio = torch.tensor(np.copy(x))[None].float()
+            f0, pd = torchcrepe.predict(
+                audio,
+                self.sr,
+                self.window,
+                f0_min,
+                f0_max,
+                model,
+                batch_size=batch_size,
+                device=self.device,
+                return_periodicity=True,
+            )
+            pd = torchcrepe.filter.median(pd, 3)
+            f0 = torchcrepe.filter.mean(f0, 3)
+            f0[pd < 0.1] = 0
+            f0 = f0[0].cpu().numpy()
+        elif f0_method == "crepe-tiny":
+            f0 = self.get_f0_official_crepe_computation(x, f0_min, f0_max, model="tiny")
+        elif f0_method == "mangio-crepe":
+            f0 = self.get_f0_crepe_computation(
+                x, f0_min, f0_max, p_len, crepe_hop_length=crepe_hop_length
+            )
+        elif f0_method == "mangio-crepe-tiny":
+            f0 = self.get_f0_crepe_computation(
+                x, f0_min, f0_max, p_len, crepe_hop_length=crepe_hop_length, model="tiny"
+            )
+        elif f0_method == "rmvpe":
+            if not hasattr(self, "model_rmvpe"):
+                from lib.infer.infer_libs.rmvpe import RMVPE
+
+                logger.info(
+                    "Loading rmvpe model,%s" % "%s/rmvpe.pt" % os.environ["rmvpe_root"]
+                )
+                self.model_rmvpe = RMVPE(
+                    "%s/rmvpe.pt" % os.environ["rmvpe_root"],
+                    is_half=self.is_half,
+                    device=self.device,
+                )
+            f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03)
+
+            if "privateuseone" in str(self.device):  # clean ortruntime memory
+                del self.model_rmvpe.model
+                del self.model_rmvpe
+                logger.info("Cleaning ortruntime memory")
+        elif f0_method == "rmvpe+":
+            params = {'x': x, 'p_len': p_len, 'f0_up_key': f0_up_key, 'f0_min': f0_min, 
+                      'f0_max': f0_max, 'time_step': time_step, 'filter_radius': filter_radius, 
+                      'crepe_hop_length': crepe_hop_length, 'model': "full"
+                      }
+            f0 = self.get_pitch_dependant_rmvpe(**params)
+        elif f0_method == "pyin":
+            f0 = self.get_f0_pyin_computation(input_audio_path, f0_min, f0_max)
+        elif f0_method == "fcpe":
+            f0 = self.get_fcpe(x, f0_min=f0_min, f0_max=f0_max, p_len=p_len)
+        elif "hybrid" in f0_method:
+            # Perform hybrid median pitch estimation
+            input_audio_path2wav[input_audio_path] = x.astype(np.double)
+            f0 = self.get_f0_hybrid_computation(
+                f0_method,
+                input_audio_path,
+                x,
+                f0_min,
+                f0_max,
+                p_len,
+                filter_radius,
+                crepe_hop_length,
+                time_step,
+            )
+        #print("Autotune:", f0_autotune)
+        if f0_autotune == True:
+            print("Autotune:", f0_autotune)
+            f0 = self.autotune_f0(f0)
+
+        f0 *= pow(2, f0_up_key / 12)
+        # with open("test.txt","w")as f:f.write("\n".join([str(i)for i in f0.tolist()]))
+        tf0 = self.sr // self.window  # 每秒f0点数
+        if inp_f0 is not None:
+            delta_t = np.round(
+                (inp_f0[:, 0].max() - inp_f0[:, 0].min()) * tf0 + 1
+            ).astype("int16")
+            replace_f0 = np.interp(
+                list(range(delta_t)), inp_f0[:, 0] * 100, inp_f0[:, 1]
+            )
+            shape = f0[self.x_pad * tf0 : self.x_pad * tf0 + len(replace_f0)].shape[0]
+            f0[self.x_pad * tf0 : self.x_pad * tf0 + len(replace_f0)] = replace_f0[
+                :shape
+            ]
+        # with open("test_opt.txt","w")as f:f.write("\n".join([str(i)for i in f0.tolist()]))
+        f0bak = f0.copy()
+        f0_mel = 1127 * np.log(1 + f0 / 700)
+        f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / (
+            f0_mel_max - f0_mel_min
+        ) + 1
+        f0_mel[f0_mel <= 1] = 1
+        f0_mel[f0_mel > 255] = 255
+        f0_coarse = np.rint(f0_mel).astype(np.int32)
+        return f0_coarse, f0bak  # 1-0
+
+    def vc(
+        self,
+        model,
+        net_g,
+        sid,
+        audio0,
+        pitch,
+        pitchf,
+        times,
+        index,
+        big_npy,
+        index_rate,
+        version,
+        protect,
+    ):  # ,file_index,file_big_npy
+        feats = torch.from_numpy(audio0)
+        if self.is_half:
+            feats = feats.half()
+        else:
+            feats = feats.float()
+        if feats.dim() == 2:  # double channels
+            feats = feats.mean(-1)
+        assert feats.dim() == 1, feats.dim()
+        feats = feats.view(1, -1)
+        padding_mask = torch.BoolTensor(feats.shape).to(self.device).fill_(False)
+
+        inputs = {
+            "source": feats.to(self.device),
+            "padding_mask": padding_mask,
+            "output_layer": 9 if version == "v1" else 12,
+        }
+        t0 = ttime()
+        with torch.no_grad():
+            logits = model.extract_features(**inputs)
+            feats = model.final_proj(logits[0]) if version == "v1" else logits[0]
+        if protect < 0.5 and pitch is not None and pitchf is not None:
+            feats0 = feats.clone()
+        if (
+            not isinstance(index, type(None))
+            and not isinstance(big_npy, type(None))
+            and index_rate != 0
+        ):
+            npy = feats[0].cpu().numpy()
+            if self.is_half:
+                npy = npy.astype("float32")
+
+            # _, I = index.search(npy, 1)
+            # npy = big_npy[I.squeeze()]
+
+            score, ix = index.search(npy, k=8)
+            weight = np.square(1 / score)
+            weight /= weight.sum(axis=1, keepdims=True)
+            npy = np.sum(big_npy[ix] * np.expand_dims(weight, axis=2), axis=1)
+
+            if self.is_half:
+                npy = npy.astype("float16")
+            feats = (
+                torch.from_numpy(npy).unsqueeze(0).to(self.device) * index_rate
+                + (1 - index_rate) * feats
+            )
+
+        feats = F.interpolate(feats.permute(0, 2, 1), scale_factor=2).permute(0, 2, 1)
+        if protect < 0.5 and pitch is not None and pitchf is not None:
+            feats0 = F.interpolate(feats0.permute(0, 2, 1), scale_factor=2).permute(
+                0, 2, 1
+            )
+        t1 = ttime()
+        p_len = audio0.shape[0] // self.window
+        if feats.shape[1] < p_len:
+            p_len = feats.shape[1]
+            if pitch is not None and pitchf is not None:
+                pitch = pitch[:, :p_len]
+                pitchf = pitchf[:, :p_len]
+
+        if protect < 0.5 and pitch is not None and pitchf is not None:
+            pitchff = pitchf.clone()
+            pitchff[pitchf > 0] = 1
+            pitchff[pitchf < 1] = protect
+            pitchff = pitchff.unsqueeze(-1)
+            feats = feats * pitchff + feats0 * (1 - pitchff)
+            feats = feats.to(feats0.dtype)
+        p_len = torch.tensor([p_len], device=self.device).long()
+        with torch.no_grad():
+            hasp = pitch is not None and pitchf is not None
+            arg = (feats, p_len, pitch, pitchf, sid) if hasp else (feats, p_len, sid)
+            audio1 = (net_g.infer(*arg)[0][0, 0]).data.cpu().float().numpy()
+            del hasp, arg
+        del feats, p_len, padding_mask
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        t2 = ttime()
+        times[0] += t1 - t0
+        times[2] += t2 - t1
+        return audio1
+    def process_t(self, t, s, window, audio_pad, pitch, pitchf, times, index, big_npy, index_rate, version, protect, t_pad_tgt, if_f0, sid, model, net_g):
+        t = t // window * window
+        if if_f0 == 1:
+            return self.vc(
+                model,
+                net_g,
+                sid,
+                audio_pad[s : t + t_pad_tgt + window],
+                pitch[:, s // window : (t + t_pad_tgt) // window],
+                pitchf[:, s // window : (t + t_pad_tgt) // window],
+                times,
+                index,
+                big_npy,
+                index_rate,
+                version,
+                protect,
+            )[t_pad_tgt : -t_pad_tgt]
+        else:
+            return self.vc(
+                model,
+                net_g,
+                sid,
+                audio_pad[s : t + t_pad_tgt + window],
+                None,
+                None,
+                times,
+                index,
+                big_npy,
+                index_rate,
+                version,
+                protect,
+            )[t_pad_tgt : -t_pad_tgt]
+
+
+    def pipeline(
+        self,
+        model,
+        net_g,
+        sid,
+        audio,
+        input_audio_path,
+        times,
+        f0_up_key,
+        f0_method,
+        file_index,
+        index_rate,
+        if_f0,
+        filter_radius,
+        tgt_sr,
+        resample_sr,
+        rms_mix_rate,
+        version,
+        protect,
+        crepe_hop_length, 
+        f0_autotune, 
+        f0_min=50, 
+        f0_max=1100
+    ):
+        if (
+            file_index != ""
+            and isinstance(file_index, str)
+            # and file_big_npy != ""
+            # and os.path.exists(file_big_npy) == True
+            and os.path.exists(file_index)
+            and index_rate != 0
+        ):
+            try:
+                index = faiss.read_index(file_index)
+                # big_npy = np.load(file_big_npy)
+                big_npy = index.reconstruct_n(0, index.ntotal)
+            except:
+                traceback.print_exc()
+                index = big_npy = None
+        else:
+            index = big_npy = None
+        audio = signal.filtfilt(bh, ah, audio)
+        audio_pad = np.pad(audio, (self.window // 2, self.window // 2), mode="reflect")
+        opt_ts = []
+        if audio_pad.shape[0] > self.t_max:
+            audio_sum = np.zeros_like(audio)
+            for i in range(self.window):
+                audio_sum += audio_pad[i : i - self.window]
+            for t in range(self.t_center, audio.shape[0], self.t_center):
+                opt_ts.append(
+                    t
+                    - self.t_query
+                    + np.where(
+                        np.abs(audio_sum[t - self.t_query : t + self.t_query])
+                        == np.abs(audio_sum[t - self.t_query : t + self.t_query]).min()
+                    )[0][0]
+                )
+        s = 0
+        audio_opt = []
+        t = None
+        t1 = ttime()
+        audio_pad = np.pad(audio, (self.t_pad, self.t_pad), mode="reflect")
+        p_len = audio_pad.shape[0] // self.window
+        inp_f0 = None
+        
+        sid = torch.tensor(sid, device=self.device).unsqueeze(0).long()
+        pitch, pitchf = None, None
+        if if_f0:
+            pitch, pitchf = self.get_f0(
+                input_audio_path,
+                audio_pad,
+                p_len,
+                f0_up_key,
+                f0_method,
+                filter_radius, 
+                crepe_hop_length, 
+                f0_autotune,
+                inp_f0, 
+                f0_min, 
+                f0_max
+            )
+            pitch = pitch[:p_len]
+            pitchf = pitchf[:p_len]
+            if "mps" not in str(self.device) or "xpu" not in str(self.device):
+                pitchf = pitchf.astype(np.float32)
+            pitch = torch.tensor(pitch, device=self.device).unsqueeze(0).long()
+            pitchf = torch.tensor(pitchf, device=self.device).unsqueeze(0).float()
+        t2 = ttime()
+        times[1] += t2 - t1
+
+        with tqdm(total=len(opt_ts), desc="Processing", unit="window") as pbar:
+            for i, t in enumerate(opt_ts):
+                t = t // self.window * self.window
+                start = s
+                end = t + self.t_pad2 + self.window
+                audio_slice = audio_pad[start:end]
+                pitch_slice = pitch[:, start // self.window:end // self.window] if if_f0 else None
+                pitchf_slice = pitchf[:, start // self.window:end // self.window] if if_f0 else None
+                audio_opt.append(self.vc(model, net_g, sid, audio_slice, pitch_slice, pitchf_slice, times, index, big_npy, index_rate, version, protect)[self.t_pad_tgt : -self.t_pad_tgt])
+                s = t
+                pbar.update(1)
+                pbar.refresh()
+
+        audio_slice = audio_pad[t:]
+        pitch_slice = pitch[:, t // self.window:] if if_f0 and t is not None else pitch
+        pitchf_slice = pitchf[:, t // self.window:] if if_f0 and t is not None else pitchf
+        audio_opt.append(self.vc(model, net_g, sid, audio_slice, pitch_slice, pitchf_slice, times, index, big_npy, index_rate, version, protect)[self.t_pad_tgt : -self.t_pad_tgt])
+        
+        audio_opt = np.concatenate(audio_opt)
+        if rms_mix_rate != 1:
+            audio_opt = change_rms(audio, 16000, audio_opt, tgt_sr, rms_mix_rate)
+        if tgt_sr != resample_sr >= 16000:
+            audio_opt = librosa.resample(
+                audio_opt, orig_sr=tgt_sr, target_sr=resample_sr
+            )
+        audio_max = np.abs(audio_opt).max() / 0.99
+        max_int16 = 32768
+        if audio_max > 1:
+            max_int16 /= audio_max
+        audio_opt = (audio_opt * max_int16).astype(np.int16)
+        del pitch, pitchf, sid
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+
+        print("Returning completed audio...")
+        return audio_opt

lib/split_audio.py

@@ -0,0 +1,91 @@
+import os
+from pydub import AudioSegment
+from pydub.silence import detect_silence, detect_nonsilent
+
+SEPERATE_DIR = os.path.join(os.getcwd(), "seperate")
+TEMP_DIR = os.path.join(SEPERATE_DIR, "temp")
+cache = {}
+
+os.makedirs(SEPERATE_DIR, exist_ok=True)
+os.makedirs(TEMP_DIR, exist_ok=True)
+
+def cache_result(func):
+    def wrapper(*args, **kwargs):
+        key = (args, frozenset(kwargs.items()))
+        if key in cache:
+            return cache[key]
+        else:
+            result = func(*args, **kwargs)
+            cache[key] = result
+            return result
+    return wrapper   
+
+def get_non_silent(audio_name, audio, min_silence, silence_thresh, seek_step, keep_silence):
+    """
+    Function to get non-silent parts of the audio.
+    """
+    nonsilent_ranges = detect_nonsilent(audio, min_silence_len=min_silence, silence_thresh=silence_thresh, seek_step=seek_step)
+    nonsilent_files = []
+    for index, range in enumerate(nonsilent_ranges):
+        nonsilent_name = os.path.join(SEPERATE_DIR, f"{audio_name}_min{min_silence}_t{silence_thresh}_ss{seek_step}_ks{keep_silence}", f"nonsilent{index}-{audio_name}.wav")
+        start, end = range[0] - keep_silence, range[1] + keep_silence
+        audio[start:end].export(nonsilent_name, format="wav")
+        nonsilent_files.append(nonsilent_name)
+    return nonsilent_files
+
+def get_silence(audio_name, audio, min_silence, silence_thresh, seek_step, keep_silence):
+    """
+    Function to get silent parts of the audio.
+    """
+    silence_ranges = detect_silence(audio, min_silence_len=min_silence, silence_thresh=silence_thresh, seek_step=seek_step)
+    silence_files = []
+    for index, range in enumerate(silence_ranges):
+        silence_name = os.path.join(SEPERATE_DIR, f"{audio_name}_min{min_silence}_t{silence_thresh}_ss{seek_step}_ks{keep_silence}", f"silence{index}-{audio_name}.wav")
+        start, end = range[0] + keep_silence, range[1] - keep_silence
+        audio[start:end].export(silence_name, format="wav")
+        silence_files.append(silence_name)
+    return silence_files
+
+@cache_result
+def split_silence_nonsilent(input_path, min_silence=500, silence_thresh=-40, seek_step=1, keep_silence=100):
+    """
+    Function to split the audio into silent and non-silent parts.
+    """
+    audio_name = os.path.splitext(os.path.basename(input_path))[0]
+    os.makedirs(os.path.join(SEPERATE_DIR, f"{audio_name}_min{min_silence}_t{silence_thresh}_ss{seek_step}_ks{keep_silence}"), exist_ok=True)
+    audio = AudioSegment.silent(duration=1000) + AudioSegment.from_file(input_path) + AudioSegment.silent(duration=1000)
+    silence_files = get_silence(audio_name, audio, min_silence, silence_thresh, seek_step, keep_silence)
+    nonsilent_files = get_non_silent(audio_name, audio, min_silence, silence_thresh, seek_step, keep_silence)
+    return silence_files, nonsilent_files
+
+def adjust_audio_lengths(original_audios, inferred_audios):
+    """
+    Function to adjust the lengths of the inferred audio files list to match the original audio files length.
+    """
+    adjusted_audios = []
+    for original_audio, inferred_audio in zip(original_audios, inferred_audios):
+        audio_1 = AudioSegment.from_file(original_audio)
+        audio_2 = AudioSegment.from_file(inferred_audio)
+        
+        if len(audio_1) > len(audio_2):
+            audio_2 += AudioSegment.silent(duration=len(audio_1) - len(audio_2))
+        else:
+            audio_2 = audio_2[:len(audio_1)]
+        
+        adjusted_file = os.path.join(TEMP_DIR, f"adjusted-{os.path.basename(inferred_audio)}")
+        audio_2.export(adjusted_file, format="wav")
+        adjusted_audios.append(adjusted_file)
+    
+    return adjusted_audios
+
+def combine_silence_nonsilent(silence_files, nonsilent_files, keep_silence, output):
+    """
+    Function to combine the silent and non-silent parts of the audio.
+    """
+    combined = AudioSegment.empty()
+    for silence, nonsilent in zip(silence_files, nonsilent_files):
+        combined += AudioSegment.from_wav(silence) + AudioSegment.from_wav(nonsilent)
+    combined += AudioSegment.from_wav(silence_files[-1])
+    combined = AudioSegment.silent(duration=keep_silence) + combined[1000:-1000] + AudioSegment.silent(duration=keep_silence)
+    combined.export(output, format="wav")
+    return output