update

examples/mpnet_aishell/run.sh

@@ -19,7 +19,8 @@ sh run.sh --stage 5 --stop_stage 5 --system_version centos --file_folder_name fi
 
 sh run.sh --stage 1 --stop_stage 2 --system_version centos --file_folder_name file_dir --final_model_name mpnet-nx-speech-20250224 \
 --noise_dir "/data/tianxing/HuggingDatasets/nx_noise/data/noise" \
---speech_dir "/data/tianxing/HuggingDatasets/nx_noise/data/speech"
+--speech_dir "/data/tianxing/HuggingDatasets/nx_noise/data/speech" \
+--max_epochs 1
 
 
 END

log.py

@@ -0,0 +1,229 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import logging
+from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler
+import os
+
+
+def setup_size_rotating(log_directory: str):
+    fmt = "%(asctime)s - %(name)s - %(levelname)s  %(filename)s:%(lineno)d >  %(message)s"
+
+    stream_handler = logging.StreamHandler()
+    stream_handler.setLevel(logging.INFO)
+    stream_handler.setFormatter(logging.Formatter(fmt))
+
+    # main
+    main_logger = logging.getLogger("main")
+    main_logger.addHandler(stream_handler)
+    main_info_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "main.log"),
+        maxBytes=100*1024*1024,  # 100MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    main_info_file_handler.setLevel(logging.INFO)
+    main_info_file_handler.setFormatter(logging.Formatter(fmt))
+    main_logger.addHandler(main_info_file_handler)
+
+    # http
+    http_logger = logging.getLogger("http")
+    http_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "http.log"),
+        maxBytes=100*1024*1024,  # 100MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    http_file_handler.setLevel(logging.DEBUG)
+    http_file_handler.setFormatter(logging.Formatter(fmt))
+    http_logger.addHandler(http_file_handler)
+
+    # api
+    api_logger = logging.getLogger("api")
+    api_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "api.log"),
+        maxBytes=10*1024*1024,  # 10MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    api_file_handler.setLevel(logging.DEBUG)
+    api_file_handler.setFormatter(logging.Formatter(fmt))
+    api_logger.addHandler(api_file_handler)
+
+    # toolbox
+    toolbox_logger = logging.getLogger("toolbox")
+    toolbox_logger.addHandler(stream_handler)
+    toolbox_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "toolbox.log"),
+        maxBytes=10*1024*1024,  # 10MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    toolbox_file_handler.setLevel(logging.DEBUG)
+    toolbox_file_handler.setFormatter(logging.Formatter(fmt))
+    toolbox_logger.addHandler(toolbox_file_handler)
+
+    # alarm
+    alarm_logger = logging.getLogger("alarm")
+    alarm_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "alarm.log"),
+        maxBytes=1*1024*1024,  # 1MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    alarm_file_handler.setLevel(logging.DEBUG)
+    alarm_file_handler.setFormatter(logging.Formatter(fmt))
+    alarm_logger.addHandler(alarm_file_handler)
+
+    debug_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "debug.log"),
+        maxBytes=1*1024*1024,  # 1MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    debug_file_handler.setLevel(logging.DEBUG)
+    debug_file_handler.setFormatter(logging.Formatter(fmt))
+
+    info_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "info.log"),
+        maxBytes=1*1024*1024,  # 1MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    info_file_handler.setLevel(logging.INFO)
+    info_file_handler.setFormatter(logging.Formatter(fmt))
+
+    error_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "error.log"),
+        maxBytes=1*1024*1024,  # 1MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    error_file_handler.setLevel(logging.ERROR)
+    error_file_handler.setFormatter(logging.Formatter(fmt))
+
+    logging.basicConfig(
+        level=logging.DEBUG,
+        datefmt="%a, %d %b %Y %H:%M:%S",
+        handlers=[
+            debug_file_handler,
+            info_file_handler,
+            error_file_handler,
+        ]
+    )
+
+
+def setup_time_rotating(log_directory: str):
+    fmt = "%(asctime)s - %(name)s - %(levelname)s  %(filename)s:%(lineno)d >  %(message)s"
+
+    stream_handler = logging.StreamHandler()
+    stream_handler.setLevel(logging.INFO)
+    stream_handler.setFormatter(logging.Formatter(fmt))
+
+    # main
+    main_logger = logging.getLogger("main")
+    main_logger.addHandler(stream_handler)
+    main_info_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "main.log"),
+        encoding="utf-8",
+        when="midnight",
+        interval=1,
+        backupCount=7
+    )
+    main_info_file_handler.setLevel(logging.INFO)
+    main_info_file_handler.setFormatter(logging.Formatter(fmt))
+    main_logger.addHandler(main_info_file_handler)
+
+    # http
+    http_logger = logging.getLogger("http")
+    http_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "http.log"),
+        encoding='utf-8',
+        when="midnight",
+        interval=1,
+        backupCount=7
+    )
+    http_file_handler.setLevel(logging.DEBUG)
+    http_file_handler.setFormatter(logging.Formatter(fmt))
+    http_logger.addHandler(http_file_handler)
+
+    # api
+    api_logger = logging.getLogger("api")
+    api_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "api.log"),
+        encoding='utf-8',
+        when="midnight",
+        interval=1,
+        backupCount=7
+    )
+    api_file_handler.setLevel(logging.DEBUG)
+    api_file_handler.setFormatter(logging.Formatter(fmt))
+    api_logger.addHandler(api_file_handler)
+
+    # toolbox
+    toolbox_logger = logging.getLogger("toolbox")
+    toolbox_file_handler = RotatingFileHandler(
+        filename=os.path.join(log_directory, "toolbox.log"),
+        maxBytes=10*1024*1024,  # 10MB
+        encoding="utf-8",
+        backupCount=2,
+    )
+    toolbox_file_handler.setLevel(logging.DEBUG)
+    toolbox_file_handler.setFormatter(logging.Formatter(fmt))
+    toolbox_logger.addHandler(toolbox_file_handler)
+
+    # alarm
+    alarm_logger = logging.getLogger("alarm")
+    alarm_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "alarm.log"),
+        encoding="utf-8",
+        when="midnight",
+        interval=1,
+        backupCount=7
+    )
+    alarm_file_handler.setLevel(logging.DEBUG)
+    alarm_file_handler.setFormatter(logging.Formatter(fmt))
+    alarm_logger.addHandler(alarm_file_handler)
+
+    debug_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "debug.log"),
+        encoding="utf-8",
+        when="D",
+        interval=1,
+        backupCount=7
+    )
+    debug_file_handler.setLevel(logging.DEBUG)
+    debug_file_handler.setFormatter(logging.Formatter(fmt))
+
+    info_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "info.log"),
+        encoding="utf-8",
+        when="D",
+        interval=1,
+        backupCount=7
+    )
+    info_file_handler.setLevel(logging.INFO)
+    info_file_handler.setFormatter(logging.Formatter(fmt))
+
+    error_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "error.log"),
+        encoding="utf-8",
+        when="D",
+        interval=1,
+        backupCount=7
+    )
+    error_file_handler.setLevel(logging.ERROR)
+    error_file_handler.setFormatter(logging.Formatter(fmt))
+
+    logging.basicConfig(
+        level=logging.DEBUG,
+        datefmt="%a, %d %b %Y %H:%M:%S",
+        handlers=[
+            debug_file_handler,
+            info_file_handler,
+            error_file_handler,
+        ]
+    )
+
+
+if __name__ == "__main__":
+    pass

main.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 import argparse
+import logging
 from pathlib import Path
 import platform
 import shutil
@@ -9,11 +10,16 @@ import zipfile
 import gradio as gr
 from huggingface_hub import snapshot_download
 import numpy as np
-import torch
 
-from project_settings import environment, project_path
+import log
+from project_settings import environment, project_path, log_directory
+from toolbox.os.command import Command
 from toolbox.torchaudio.models.mpnet.inference_mpnet import InferenceMPNet
 
+log.setup_size_rotating(log_directory=log_directory)
+
+logger = logging.getLogger("main")
+
 
 def get_args():
     parser = argparse.ArgumentParser()
@@ -48,11 +54,16 @@ def get_args():
     return args
 
 
+def shell(cmd: str):
+    return Command.popen(cmd)
+
+
 denoise_engines = dict()
 
 
 def when_click_denoise_button(noisy_audio_t, engine: str):
     sample_rate, signal = noisy_audio_t
+    logger.info(f"run denoise; engine: {engine}, sample_rate: {sample_rate}, signal dtype: {signal.dtype}, signal shape: {signal.shape}")
 
     noisy_audio = np.array(signal / (1 << 15), dtype=np.float32)
 
@@ -143,6 +154,17 @@ def main():
                     cache_mode="lazy",
                 )
 
+            with gr.TabItem("shell"):
+                shell_text = gr.Textbox(label="cmd")
+                shell_button = gr.Button("run")
+                shell_output = gr.Textbox(label="output")
+
+                shell_button.click(
+                    shell,
+                    inputs=[shell_text,],
+                    outputs=[shell_output],
+                )
+
     # http://127.0.0.1:7864/
     blocks.queue().launch(
         share=False if platform.system() == "Windows" else False,

project_settings.py

@@ -12,8 +12,8 @@ project_path = Path(project_path)
 log_directory = project_path / "logs"
 log_directory.mkdir(parents=True, exist_ok=True)
 
-temp_directory = project_path / "temp"
-temp_directory.mkdir(parents=True, exist_ok=True)
+# temp_directory = project_path / "temp"
+# temp_directory.mkdir(parents=True, exist_ok=True)
 
 environment = EnvironmentManager(
     path=os.path.join(project_path, "dotenv"),