{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [ "update_desc = '''\n", "欢迎使用 stable-diffusion-webui 便捷启动工具\n", "\n", "此脚本理论上可以在任何jupyter环境运行，但仅在 google colab 和 kaggle 测试。 已经无法在 google colab 免费实例上运行。\n", "此脚本的【前置脚本】发布地址 https://www.kaggle.com/code/yiyiooo/stable-diffusion-webui-novelai-sdxl。\n", "此脚本需配合 【前置脚本】 的脚本附带的配置项才能正常启动。\n", "此脚本的内容会自动更新，你无需更新【前置脚本】就能获取到最新的功能。\n", "增加一个说明：为解决加密图片的解密和浏览的问题，开发了一个独立应用，可以在这里下载 https://github.com/viyiviyi/encrypt_gallery/releases/\n", "加密插件可解决生成nsfw图片后被平台封号问题\n", "\n", "路径说明\n", "* 为了解决平台差异，所有的安装目录和文件输出目录都被重新指定，如果你需要在配置中访问这些目录，请查看以下说明\n", "*\n", "* 可以使用 $install_path 或 {install_path} 来访问安装目录，写在字符串内也会生效\n", "* $output_path 或 {output_path} 可以访问输出目录\n", "* 如果你需要安装在自定义目录，也可以设置这些值 如: install_path = '新的路径'\n", "* 可自定义方法 on_before_start 并在方法内写上启动前需要的逻辑来实现在webui启动前执行自定义逻辑\n", "* 可以增加配置 multi_case = True 来控制是否使用多卡\n", "* 如果需要显示更多控制台输出 需配置 hidden_console_info = False\n", "\n", "********* 特别提示 *********\n", "* huggingface 的下载链接需要增加 ?download=true 参数才能正确下载到文件，如果你需要的文件里的下载链接没有这个参数，请自行增加。\n", "'''" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [ "from pathlib import Path\n", "import os\n", "import time\n", "import re\n", "import subprocess\n", "import threading\n", "import sys\n", "import socket\n", "import torch\n", "from typing import List\n", "import uuid\n", "import asyncio\n", "from urllib import request" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [ "# 内置参数默认值，当上下文有参数时可覆盖默认值\n", "_runing = False\n", "\n", "_useFrpc = locals().get('useFrpc') or globals().get('useFrpc') or True\n", "\n", "_useNgrok = locals().get('useNgrok') or globals().get('useNgrok') or True\n", "\n", "_reLoad = locals().get('reLoad') or globals().get('reLoad') or False\n", " \n", "_before_downloading = locals().get('before_downloading') or globals().get('before_downloading') or ''\n", "\n", "_async_downloading = locals().get('async_downloading') or globals().get('async_downloading') or ''\n", "\n", "_before_start_sync_downloading = locals().get('before_start_sync_downloading') or globals().get('before_start_sync_downloading') or ''\n", "\n", "_server_port = locals().get('server_port') or globals().get('server_port') or 7860\n", " \n", "_sd_git_repo = locals().get('sd_git_repo') or globals().get('sd_git_repo') or 'https://github.com/viyiviyi/stable-diffusion-webui.git -b local' \n", "_sd_git_repo = _sd_git_repo\\\n", " .replace('{sdwui}','stable-diffusion-webui')\\\n", " .replace('{wui}',\"webui\")\n", " \n", "_sd_config_git_repu = locals().get('sd_config_git_repu') or globals().get('sd_config_git_repu') or 'https://github.com/viyiviyi/sd-configs.git'\n", "_sd_config_git_repu = _sd_config_git_repu\\\n", " .replace('{sdwui}','stable-diffusion-webui')\\\n", " .replace('{wui}',\"webui\")\n", " \n", " \n", "_huggingface_token = locals().get('huggingface_token') or globals().get('huggingface_token') or '{input_path}/configs/huggingface_token.txt'\n", "_huggingface_token = _huggingface_token\\\n", " .replace('{sdwui}','stable-diffusion-webui')\\\n", " .replace('{wui}',\"webui\")\n", " \n", "_huggingface_repo = locals().get('huggingface_repo') or globals().get('huggingface_repo') or ''\n", "_huggingface_repo = _huggingface_repo\\\n", " .replace('{sdwui}','stable-diffusion-webui')\\\n", " .replace('{wui}',\"webui\")\n", "\n", "_link_instead_of_copy = locals().get('link_instead_of_copy') or globals().get('link_instead_of_copy') or True\n", " \n", "show_shell_info = locals().get('hidden_console_info') or globals().get('hidden_console_info')\n", "if show_shell_info is None: show_shell_info = False\n", "\n", "_multi_case = locals().get('multi_case') or globals().get('multi_case') or False\n", " \n", "_skip_start = locals().get('skip_start') or globals().get('skip_start') or True\n", "\n", "def before_start():\n", " pass\n", "\n", "def main_start():\n", " pass\n", "\n", "_on_before_start = locals().get('on_before_start') or globals().get('on_before_start') or before_start \n", "_skip_webui = locals().get('skip_webui') or globals().get('skip_webui') or False\n", " \n", "run_by_none_device = False\n", "\n", "_proxy_path = locals().get('proxy_path') or globals().get('proxy_path') or {}\n", "\n", "_sub_path = locals().get('sub_path') or globals().get('sub_path') or ['/','/1/']\n", "if len(_sub_path) != 2:\n", " _sub_path = ['/','/1/']\n", " \n", "_config_args:dict[str, str] = locals().get('config_args') or globals().get('config_args') or {}" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [ "\n", "def run(command, cwd=None, desc=None, errdesc=None, custom_env=None,try_error:bool=True) -> str:\n", " global show_shell_info\n", " if desc is not None:\n", " print(desc)\n", "\n", " run_kwargs = {\n", " \"args\": command,\n", " \"shell\": True,\n", " \"cwd\": cwd,\n", " \"env\": os.environ if custom_env is None else custom_env,\n", " \"encoding\": 'utf8',\n", " \"errors\": 'ignore',\n", " }\n", "\n", " if not show_shell_info:\n", " run_kwargs[\"stdout\"] = run_kwargs[\"stderr\"] = subprocess.PIPE\n", "\n", " result = subprocess.run(**run_kwargs)\n", "\n", " if result.returncode != 0:\n", " error_bits = [\n", " f\"{errdesc or 'Error running command'}.\",\n", " f\"Command: {command}\",\n", " f\"Error code: {result.returncode}\",\n", " ]\n", " if result.stdout:\n", " error_bits.append(f\"stdout: {result.stdout}\")\n", " if result.stderr:\n", " error_bits.append(f\"stderr: {result.stderr}\")\n", " if try_error:\n", " print((RuntimeError(\"\\n\".join(error_bits))))\n", " else:\n", " raise RuntimeError(\"\\n\".join(error_bits))\n", "\n", " if show_shell_info:\n", " print((result.stdout or \"\"))\n", " return (result.stdout or \"\")\n", "\n", "def mkdirs(path, exist_ok=True):\n", " if path and not Path(path).exists():\n", " os.makedirs(path,exist_ok=exist_ok)\n", "\n", "\n", "# 检查网络\n", "def check_service(host, port):\n", " try:\n", " socket.create_connection((host, port), timeout=5)\n", " return True\n", " except socket.error:\n", " return False\n", "\n", "\n", "# 检查gpu是否存在\n", "def check_gpu():\n", " if not run_by_none_device and torch.cuda.device_count() == 0:\n", " raise Exception('当前环境没有GPU')\n", "\n", "\n", "def echoToFile(content:str,path:str):\n", " if path.find('/') >= 0:\n", " _path = '/'.join(path.split('/')[:-1])\n", " run(f'''mkdir -p {_path}''')\n", " with open(path,'w') as sh:\n", " sh.write(content)\n", " \n", "def get_freefrp_confog(local_port):\n", " rd_str = uuid.uuid1()\n", " return (f'''\n", "[common]\n", "server_addr = frp.freefrp.net\n", "server_port = 7000\n", "token = freefrp.net\n", "\n", "[{rd_str}_http]\n", "type = http\n", "local_ip = 127.0.0.1\n", "local_port = {local_port}\n", "custom_domains = {rd_str}.frp.eaias.com\n", "''',f'http://{rd_str}.frp.eaias.com')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [ "\n", "_install_path = f\"/kaggle\" if os.path.exists('/kaggle/') else f\"{os.environ['HOME']}/sd_webui\" # 安装目录\n", "_output_path = '/kaggle/working' if os.path.exists('/kaggle/working/') else f\"{os.environ['HOME']}/.sdwui/Output\" # 输出目录 如果使用google云盘 会在google云盘增加sdwebui/Output\n", "_input_path = '/kaggle/input' # 输入目录\n", "_ui_dir_name = 'sd_main_dir'\n", "\n", "_install_path = locals().get('install_path') or globals().get('install_path') or _install_path\n", "_output_path = locals().get('output_path') or globals().get('output_path') or _output_path\n", "_input_path = locals().get('input_path') or globals().get('input_path') or _input_path\n", "_ui_dir_name = locals().get('ui_dir_name') or globals().get('ui_dir_name') or _ui_dir_name\n", "\n", "install_path = _install_path\n", "output_path = _output_path\n", "input_path = _input_path\n", "ui_dir_name = _ui_dir_name\n", " \n", "google_drive = '' \n", "\n", "\n", "_useGooglrDrive = locals().get('useGooglrDrive') or globals().get('useGooglrDrive') or True\n", "\n", "# 连接谷歌云\n", "try:\n", " if _useGooglrDrive:\n", " from google.colab import drive\n", " drive.mount(f'~/google_drive')\n", " google_drive = f\"{os.environ['HOME']}/google_drive/MyDrive\"\n", " _output_path = f'{google_drive}/sdwebui/Output'\n", " _input_path = f'{google_drive}/sdwebui/Input'\n", " run(f'''mkdir -p {_input_path}''')\n", " print('''\n", "已经链接到谷歌云盘\n", "已在云盘创建Input和Output目录\n", " ''')\n", "except:\n", " _useGooglrDrive = False\n", "\n", "run(f'''mkdir -p {_install_path}''')\n", "run(f'''mkdir -p {_output_path}''')\n", "\n", "\n", "os.environ['install_path'] = _install_path\n", "os.environ['output_path'] = _output_path\n", "os.environ['google_drive'] = google_drive\n", "os.environ['input_path'] = _input_path\n", "\n", "space_string = ' \\n\\r\\t\\'\\\",'\n", "\n", "def replace_path(input_str:str):\n", " if not input_str: return ''\n", " for key in _config_args:\n", " input_str = input_str.replace(key,_config_args[key])\n", " \n", " if not (locals().get('use_comfyui') or globals().get('use_comfyui') or False):\n", " input_str = input_str.replace('https://github.com/comfyanonymous/ComfyUI.git','https://github.com/comfyanonymous/ComfyUI.git')\n", " \n", " return input_str.replace('$install_path',_install_path)\\\n", " .replace('{install_path}',_install_path)\\\n", " .replace('$input_path',_input_path)\\\n", " .replace('{input_path}',_input_path)\\\n", " .replace('$output_path',_output_path)\\\n", " .replace('{output_path}',_output_path)\\\n", " .replace('{sdwui}','stable-diffusion-webui')\\\n", " .replace('{wui}',\"webui\")\\\n", " .strip()\n", "\n", "\n", "def config_reader(conf:str):\n", " conf = conf or \"\"\n", " args = [replace_path(item.split('#')[0].strip(space_string)) for item in conf.split('\\n') if item.strip(space_string)]\n", " return [item.strip() for item in args if item.strip()]\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_kg_hide-input": true, "id": "i3LhnwYHLCtC", "trusted": true }, "outputs": [], "source": [ "ngrokTokenFile = os.path.join(_input_path,'configs/ngrok_token.txt') # 非必填 存放ngrokToken的文件的路径\n", "frpcConfigFile = os.path.join(_input_path,'configs/frpc_koishi.ini') # 非必填 frp 配置文件\n", "# ss证书目录 下载nginx的版本，把pem格式改成crt格式\n", "frpcSSLFFlies = [os.path.join(_input_path,'configs/koishi_ssl')]\n", "if 'frp_ssl_dir' in locals() or 'frp_ssl_dir' in globals():\n", " frpcSSLFFlies = frpcSSLFFlies + config_reader(locals().get('frp_ssl_dir') or globals().get('frp_ssl_dir'))\n", "# frpc 文件目录 如果目录不存在，会自动下载，也可以在数据集搜索 viyiviyi/utils 添加\n", "frpcExePath = os.path.join(_input_path,'utils-tools/frpc')\n", "# 其他需要加载的webui启动参数 写到【参数列表】这个配置去\n", "otherArgs = '--xformers'\n", "if 'sd_start_args' in locals() or 'sd_start_args' in globals():\n", " otherArgs = ' '.join([item for item in config_reader(locals().get('sd_start_args') or globals().get('sd_start_args')) if item != '--no-gradio-queue'])\n", "venvPath = os.path.join(_input_path,'sd-webui-venv/venv.tar.bak') # 安装好的python环境 sd-webui-venv是一个公开是数据集 可以搜索添加\n", "\n", "# 用于使用kaggle api的token文件 参考 https://www.kaggle.com/docs/api\n", "# 此文件用于自动上传koishi的相关配置 也可以用于保存重要的输出文件\n", "kaggleApiTokenFile = os.path.join(_input_path,'configs/kaggle.json')\n", "\n", "requirements = []\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_kg_hide-input": true, "id": "a_GtG2ayLCtD", "trusted": true }, "outputs": [], "source": [ "# 这下面的是用于初始化一些值或者环境变量的，轻易别改\n", "_setting_file = replace_path(locals().get('setting_file') or globals().get('setting_file') or 'config.json')\n", "\n", "_ui_config_file = replace_path(locals().get('ui_config_file') or globals().get('ui_config_file') or 'ui-config.json')\n", "\n", "# 设置文件路径\n", "if Path(f\"{os.environ['HOME']}/google_drive/MyDrive\").exists():\n", " if _setting_file == '/kaggle/working/configs/config.json':\n", " _setting_file = os.path.join(_output_path,'configs/config.json')\n", " if _ui_config_file == '/kaggle/working/configs/ui-config.json':\n", " _ui_config_file = os.path.join(_output_path,'configs/ui-config.json')\n", " \n", "frpcStartArg = ''\n", "freefrp_url = ''\n", "_frp_temp_config_file = ''\n", "_frp_config_or_file = replace_path(locals().get('frp_config_or_file') or globals().get('frp_config_or_file')) or frpcConfigFile\n", "run(f'''mkdir -p {_install_path}/configFiles''')\n", "if _frp_config_or_file:\n", " if '[common]' in _frp_config_or_file:\n", " echoToFile(_frp_config_or_file,f'{_install_path}/configFiles/temp_frpc_webui.ini')\n", " _frp_temp_config_file = f'{_install_path}/configFiles/temp_frpc_webui.ini'\n", " elif '.ini' in _frp_config_or_file:\n", " _frp_temp_config_file = _frp_config_or_file.strip()\n", " \n", " if _frp_temp_config_file:\n", " if Path(_frp_temp_config_file).exists():\n", " run(f'''cp -f {_frp_temp_config_file} {_install_path}/configFiles/frpc_webui.ini''')\n", " run(f'''sed -i \"s/local_port = .*/local_port = {_server_port}/g\" {_install_path}/configFiles/frpc_webui.ini''')\n", " frpcStartArg = f' -c {_install_path}/configFiles/frpc_webui.ini'\n", " elif _frp_config_or_file.strip().startswith('-f'):\n", " frpcStartArg = _frp_config_or_file.strip()\n", " \n", "if not frpcStartArg:\n", " conf,url = get_freefrp_confog(_server_port)\n", " echoToFile(conf,f'{_install_path}/configFiles/frpc_webui.ini')\n", " freefrp_url = url\n", " frpcStartArg = f' -c {_install_path}/configFiles/frpc_webui.ini'\n", "\n", "ngrokToken=''\n", "_ngrok_config_or_file = replace_path(locals().get('ngrok_config_or_file') or globals().get('ngrok_config_or_file')) or ngrokTokenFile\n", "if _ngrok_config_or_file:\n", " if Path(_ngrok_config_or_file.strip()).exists():\n", " ngrokTokenFile = _ngrok_config_or_file.strip()\n", " if Path(ngrokTokenFile).exists():\n", " with open(ngrokTokenFile,encoding = \"utf-8\") as nkfile:\n", " ngrokToken = nkfile.readline()\n", " elif not _ngrok_config_or_file.strip().startswith('/'):\n", " ngrokToken=_ngrok_config_or_file.strip()\n", " \n", "if not Path(venvPath).exists():\n", " venvPath = os.path.join(_input_path,'sd-webui-venv/venv.zip')\n", " \n", "huggingface_headers:dict = None " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def _init_conf():\n", " global _useFrpc\n", " global _useNgrok\n", " global _reLoad\n", " global _before_downloading\n", " global _async_downloading\n", " global _before_start_sync_downloading\n", " global _server_port\n", " global _sd_git_repo\n", " global _sd_config_git_repu\n", " global _huggingface_token\n", " global _huggingface_repo\n", " global _link_instead_of_copy\n", " global show_shell_info\n", " global _multi_case\n", " global _skip_start\n", " global _on_before_start\n", " global _skip_webui\n", " global _proxy_path\n", " global _sub_path\n", " global _config_args\n", " global _install_path\n", " global _output_path\n", " global _input_path\n", " global _ui_dir_name\n", " \n", " global ngrokTokenFile\n", " global frpcConfigFile\n", " global frpcSSLFFlies\n", " global frpcExePath\n", " global otherArgs\n", " global _setting_file\n", " global _ui_config_file\n", " global _frp_temp_config_file\n", " global _frp_config_or_file\n", " global ngrokToken\n", " global venvPath\n", " \n", " _useFrpc = locals().get('useFrpc') or globals().get('useFrpc') or True\n", " _useNgrok = locals().get('useNgrok') or globals().get('useNgrok') or True\n", " _reLoad = locals().get('reLoad') or globals().get('reLoad') or False\n", " _before_downloading = locals().get('before_downloading') or globals().get('before_downloading') or ''\n", " _async_downloading = locals().get('async_downloading') or globals().get('async_downloading') or ''\n", " _before_start_sync_downloading = locals().get('before_start_sync_downloading') or globals().get('before_start_sync_downloading') or ''\n", " _server_port = locals().get('server_port') or globals().get('server_port') or 7860\n", " _sd_git_repo = locals().get('sd_git_repo') or globals().get('sd_git_repo') or 'https://github.com/viyiviyi/stable-diffusion-webui.git -b local' \n", " _sd_git_repo = _sd_git_repo\\\n", " .replace('{sdwui}','stable-diffusion-webui')\\\n", " .replace('{wui}',\"webui\") \n", " _sd_config_git_repu = locals().get('sd_config_git_repu') or globals().get('sd_config_git_repu') or 'https://github.com/viyiviyi/sd-configs.git'\n", " _sd_config_git_repu = _sd_config_git_repu\\\n", " .replace('{sdwui}','stable-diffusion-webui')\\\n", " .replace('{wui}',\"webui\")\n", " _huggingface_token = locals().get('huggingface_token') or globals().get('huggingface_token') or '{input_path}/configs/huggingface_token.txt'\n", " _huggingface_token = _huggingface_token\\\n", " .replace('{sdwui}','stable-diffusion-webui')\\\n", " .replace('{wui}',\"webui\")\n", " _huggingface_repo = locals().get('huggingface_repo') or globals().get('huggingface_repo') or ''\n", " _huggingface_repo = _huggingface_repo\\\n", " .replace('{sdwui}','stable-diffusion-webui')\\\n", " .replace('{wui}',\"webui\")\n", " _link_instead_of_copy = locals().get('link_instead_of_copy') or globals().get('link_instead_of_copy') or True\n", " show_shell_info = locals().get('hidden_console_info') or globals().get('hidden_console_info')\n", " if show_shell_info is None: show_shell_info = False\n", " else: show_shell_info = not show_shell_info\n", " _multi_case = locals().get('multi_case') or globals().get('multi_case') or False\n", " _skip_start = locals().get('skip_start') or globals().get('skip_start') or True\n", " _on_before_start = locals().get('on_before_start') or globals().get('on_before_start') or before_start \n", " _skip_webui = locals().get('skip_webui') or globals().get('skip_webui') or False\n", " _proxy_path = locals().get('proxy_path') or globals().get('proxy_path') or {}\n", " _sub_path = locals().get('sub_path') or globals().get('sub_path') or ['/','/1/']\n", " if len(_sub_path) != 2:\n", " _sub_path = ['/','/1/']\n", " \n", " _config_args = locals().get('config_args') or globals().get('config_args') or {}\n", " \n", " _install_path = locals().get('install_path') or globals().get('install_path') or _install_path\n", " _output_path = locals().get('output_path') or globals().get('output_path') or _output_path\n", " _input_path = locals().get('input_path') or globals().get('input_path') or _input_path\n", " _ui_dir_name = locals().get('ui_dir_name') or globals().get('ui_dir_name') or _ui_dir_name\n", " \n", " ngrokTokenFile = os.path.join(_input_path,'configs/ngrok_token.txt') # 非必填 存放ngrokToken的文件的路径\n", " frpcConfigFile = os.path.join(_input_path,'configs/frpc_koishi.ini') # 非必填 frp 配置文件\n", " # ss证书目录 下载nginx的版本，把pem格式改成crt格式\n", " frpcSSLFFlies = [os.path.join(_input_path,'configs/koishi_ssl')]\n", " if 'frp_ssl_dir' in locals() or 'frp_ssl_dir' in globals():\n", " frpcSSLFFlies = frpcSSLFFlies + config_reader(locals().get('frp_ssl_dir') or globals().get('frp_ssl_dir'))\n", " # frpc 文件目录 如果目录不存在，会自动下载，也可以在数据集搜索 viyiviyi/utils 添加\n", " frpcExePath = os.path.join(_input_path,'utils-tools/frpc_v0.51.0')\n", " # 其他需要加载的webui启动参数 写到【参数列表】这个配置去\n", " otherArgs = '--xformers'\n", " if 'sd_start_args' in locals() or 'sd_start_args' in globals():\n", " otherArgs = ' '.join([item for item in config_reader(locals().get('sd_start_args') or globals().get('sd_start_args')) if item != '--no-gradio-queue'])\n", "\n", " # 这下面的是用于初始化一些值或者环境变量的，轻易别改\n", " _setting_file = replace_path(locals().get('setting_file') or globals().get('setting_file') or 'config.json')\n", "\n", " _ui_config_file = replace_path(locals().get('ui_config_file') or globals().get('ui_config_file') or 'ui-config.json')\n", "\n", " # 设置文件路径\n", " if Path(f\"{os.environ['HOME']}/google_drive/MyDrive\").exists():\n", " if _setting_file == '/kaggle/working/configs/config.json':\n", " _setting_file = os.path.join(_output_path,'configs/config.json')\n", " if _ui_config_file == '/kaggle/working/configs/ui-config.json':\n", " _ui_config_file = os.path.join(_output_path,'configs/ui-config.json')\n", " \n", " frpcStartArg = ''\n", " freefrp_url = ''\n", " _frp_temp_config_file = ''\n", " _frp_config_or_file = replace_path(locals().get('frp_config_or_file') or globals().get('frp_config_or_file')) or frpcConfigFile\n", " run(f'''mkdir -p {_install_path}/configFiles''')\n", " if _frp_config_or_file:\n", " if '[common]' in _frp_config_or_file:\n", " echoToFile(_frp_config_or_file,f'{_install_path}/configFiles/temp_frpc_webui.ini')\n", " _frp_temp_config_file = f'{_install_path}/configFiles/temp_frpc_webui.ini'\n", " elif '.ini' in _frp_config_or_file:\n", " _frp_temp_config_file = _frp_config_or_file.strip()\n", " \n", " if _frp_temp_config_file:\n", " if Path(_frp_temp_config_file).exists():\n", " run(f'''cp -f {_frp_temp_config_file} {_install_path}/configFiles/frpc_webui.ini''')\n", " run(f'''sed -i \"s/local_port = .*/local_port = {_server_port}/g\" {_install_path}/configFiles/frpc_webui.ini''')\n", " frpcStartArg = f' -c {_install_path}/configFiles/frpc_webui.ini'\n", " elif _frp_config_or_file.strip().startswith('-f'):\n", " frpcStartArg = _frp_config_or_file.strip()\n", " \n", " if not frpcStartArg:\n", " conf,url = get_freefrp_confog(_server_port)\n", " echoToFile(conf,f'{_install_path}/configFiles/frpc_webui.ini')\n", " freefrp_url = url\n", " frpcStartArg = f' -c {_install_path}/configFiles/frpc_webui.ini'\n", "\n", " ngrokToken=''\n", " _ngrok_config_or_file = replace_path(locals().get('ngrok_config_or_file') or globals().get('ngrok_config_or_file')) or ngrokTokenFile\n", " if _ngrok_config_or_file:\n", " if Path(_ngrok_config_or_file.strip()).exists():\n", " ngrokTokenFile = _ngrok_config_or_file.strip()\n", " if Path(ngrokTokenFile).exists():\n", " with open(ngrokTokenFile,encoding = \"utf-8\") as nkfile:\n", " ngrokToken = nkfile.readline()\n", " elif not _ngrok_config_or_file.strip().startswith('/'):\n", " ngrokToken=_ngrok_config_or_file.strip()\n", " \n", " if not Path(venvPath).exists():\n", " venvPath = os.path.join(_input_path,'sd-webui-venv/venv.zip')\n", " " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 文件下载工具\n", "\n", "---\n", "\n", "link_or_download_flie(config:str, skip_url:bool=False, _link_instead_of_copy:bool=True, base_path:str = '',sync:bool=False,thread_num:int=None)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [ "import concurrent.futures\n", "import importlib\n", "import os\n", "import pprint\n", "import re\n", "import venv\n", "from pathlib import Path\n", "from typing import List\n", "\n", "import requests\n", "\n", "show_shell_info = False\n", "\n", "def is_installed(package):\n", " try:\n", " spec = importlib.util.find_spec(package)\n", " except ModuleNotFoundError:\n", " return False\n", "\n", " return spec is not None\n", "\n", "def download_file(url:str, filename:str, dist_path:str, cache_path = '',_link_instead_of_copy:bool=True,headers={}):\n", " startTicks = time.time()\n", " # 获取文件的真实文件名\n", " if not filename:\n", " with requests.get(url, stream=True,headers=headers) as r:\n", " if 'Content-Disposition' in r.headers:\n", " filename = r.headers['Content-Disposition'].split('filename=')[1].strip('\"')\n", " r.close()\n", " if not filename and re.search(r'/[^/]+\\.[^/]+$',url):\n", " filename = url.split('/')[-1].split('?')[0]\n", " \n", " filename = re.sub(r'[\\\\/:*?\"<>|;]', '', filename)\n", " filename = re.sub(r'[\\s\\t]+', '_', filename)\n", " \n", " print(f'下载 {filename} url: {url} --> {dist_path}')\n", " \n", " # 创建目录\n", " if cache_path and not Path(cache_path).exists():\n", " os.makedirs(cache_path,exist_ok=True)\n", " if dist_path and not Path(dist_path).exists():\n", " os.makedirs(dist_path,exist_ok=True)\n", " \n", " # 拼接文件的完整路径\n", " filepath = os.path.join(dist_path, filename)\n", "\n", " if cache_path:\n", " cache_path = os.path.join(cache_path, filename)\n", " \n", " # 判断文件是否已存在\n", " if Path(filepath).exists():\n", " print(f'文件 {filename} 已存在 {dist_path}')\n", " return\n", " \n", " if cache_path and Path(cache_path).exists():\n", " run(f'cp -n -r -f {\"-s\" if _link_instead_of_copy else \"\"} {cache_path} {dist_path}')\n", " print(f'文件缓存 {cache_path} --> {dist_path}')\n", " return\n", " # 下载文件\n", " size = 0\n", " with requests.get(url, stream=True, headers=headers) as r:\n", " r.raise_for_status()\n", " with open(cache_path or filepath, 'wb') as f:\n", " for chunk in r.iter_content(chunk_size=1024*1024):\n", " if chunk:\n", " size += len(chunk)\n", " f.write(chunk)\n", " # 如果使用了缓存目录 需要复制或链接文件到目标目录\n", " if cache_path:\n", " run(f'cp -n -r -f {\"-s\" if _link_instead_of_copy else \"\"} {cache_path} {dist_path}')\n", " ticks = time.time()\n", " print(f'下载完成 {filename} --> {dist_path} 大小{round(size/1024/1024,2)}M 耗时:{round(ticks - startTicks,2)}秒')\n", " \n", "def download_git(url, dist_path, cache_path = '',_link_instead_of_copy:bool=True):\n", " if not Path(dist_path).exists():\n", " os.makedirs(dist_path,exist_ok=True)\n", " if show_shell_info:\n", " print(f'git 下载 {url} --> {dist_path}')\n", " if cache_path and not Path(cache_path).exists():\n", " os.makedirs(cache_path,exist_ok=True)\n", " run(f'git clone {url}',cwd = cache_path)\n", " if cache_path:\n", " run(f'cp -n -r -f {cache_path}/* {dist_path}')\n", " else:\n", " run(f'git clone {url}',cwd = dist_path)\n", " if show_shell_info:\n", " print(f'git 下载完成 {url} --> {dist_path}')\n", " \n", " \n", "def download_huggingface(url:str, filename:str, dist_path, cache_path = '',_link_instead_of_copy:bool=True):\n", " fileReg = r'^https:\\/\\/huggingface.co(\\/([^\\/]+\\/)?[^\\/]+\\/[^\\/]+\\/(resolve|blob)\\/[^\\/]+\\/|[^\\.]+\\.[^\\.]+$|download=true)'\n", " def isFile(url:str):\n", " if re.match(fileReg,url):\n", " return True\n", " return False\n", " if isFile(url):\n", " download_file(url,filename,dist_path,cache_path,_link_instead_of_copy,headers=huggingface_headers)\n", " else:\n", " download_git(url,dist_path,cache_path,_link_instead_of_copy)\n", " \n", "# 加入文件到下载列表\n", "def pause_url(url:str,dist_path:str):\n", " file_name = ''\n", " if re.match(r'^[^:]+:(https?|ftps?)://', url, flags=0):\n", " file_name = re.findall(r'^[^:]+:',url)[0][:-1]\n", " url = url[len(file_name)+1:]\n", " if not re.match(r'^(https?|ftps?)://',url):\n", " return\n", " file_name = re.sub(r'\\s+','_',file_name or '')\n", " path_hash = str(hash(url)).replace('-','')\n", " \n", " return {'file_name':file_name,'path_hash':path_hash,'url':url,'dist_path':dist_path}\n", "\n", "def download_urls(download_list:List[dict],sync:bool=False,thread_num:int=5, \n", " cache_path:str=os.path.join(os.environ['HOME'],'.cache','download_util'),\n", " _link_instead_of_copy:bool=True,is_await:bool=False):\n", " if sync:\n", " for conf in download_list:\n", " cache_dir = os.path.join(cache_path,conf['path_hash'])\n", " if conf['url'].startswith('https://github.com'):\n", " try:\n", " download_git(conf['url'],conf['dist_path'],cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy)\n", " except:\n", " pass\n", " continue\n", " if conf['url'].startswith('https://huggingface.co.'):\n", " try:\n", " download_huggingface(conf['url'],conf['file_name'],conf['dist_path'],cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy)\n", " except:\n", " pass\n", " continue\n", " if conf['url'].startswith('https://civitai.com'):\n", " if not re.search(r'token=.+', conf['url']):\n", " if conf['url'].find('?') == -1:\n", " conf['url'] = conf['url']+'?token=fee8bb78b75566eddfd04d061996185c'\n", " else:\n", " conf['url'] = conf['url']+'&token=fee8bb78b75566eddfd04d061996185c'\n", " try:\n", " download_file(conf['url'],conf['file_name'],conf['dist_path'],cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy)\n", " except:\n", " pass\n", " else:\n", " executor = concurrent.futures.ThreadPoolExecutor(max_workers=thread_num)\n", " futures = []\n", " for conf in download_list:\n", " cache_dir = os.path.join(cache_path,conf['path_hash'])\n", " if conf['url'].startswith('https://github.com'):\n", " futures.append(executor.submit(download_git, conf['url'],conf['dist_path'],\n", " cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy))\n", " continue\n", " if conf['url'].startswith('https://huggingface.co.'):\n", " futures.append(executor.submit(download_huggingface,conf['url'],conf['file_name'],conf['dist_path'],cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy))\n", " continue\n", " if conf['url'].startswith('https://civitai.com'):\n", " if not re.search(r'token=.+', conf['url']):\n", " if conf['url'].find('?') == -1:\n", " conf['url'] = conf['url']+'?token=fee8bb78b75566eddfd04d061996185c'\n", " else:\n", " conf['url'] = conf['url']+'&token=fee8bb78b75566eddfd04d061996185c'\n", " futures.append(executor.submit(download_file, conf['url'],conf['file_name'],conf['dist_path'],\n", " cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy))\n", " if is_await:\n", " concurrent.futures.wait(futures)\n", " \n", " \n", "def parse_config(config:str):\n", " space_string = ' \\n\\r\\t\\'\\\",'\n", " other_flie_list = [item.split('#')[0].strip(space_string) for item in config.split('\\n') if item.strip(space_string)]\n", " other_flie_list = [item.strip() for item in other_flie_list if item.strip()]\n", " other_flie_list_store = {}\n", " other_flie_list_store_name='default'\n", " other_flie_list_store_list_cache=[]\n", " \n", " for item in other_flie_list:\n", " if item.startswith('[') and item.endswith(']'):\n", " if not other_flie_list_store_name == 'default':\n", " other_flie_list_store[other_flie_list_store_name]=other_flie_list_store_list_cache\n", " other_flie_list_store_list_cache = []\n", " other_flie_list_store_name = item[1:-1]\n", " else:\n", " other_flie_list_store_list_cache.append(item)\n", " other_flie_list_store[other_flie_list_store_name]=other_flie_list_store_list_cache\n", " \n", " return other_flie_list_store\n", "\n", "\n", "def link_or_download_flie(config:str, skip_url:bool=False, _link_instead_of_copy:bool=True, base_path:str = '',\n", " sync:bool=False,thread_num:int=None, is_await:bool=False):\n", " store:dict[str,List[str]] = parse_config(config)\n", " download_list = []\n", " for dist_dir in store.keys():\n", " dist_path = os.path.join(base_path,dist_dir)\n", " os.makedirs(dist_path,exist_ok=True)\n", " for path in store[dist_dir]:\n", " if 'https://' in path or 'http://' in path:\n", " if skip_url:\n", " continue\n", " if sync:\n", " try:\n", " download_urls([pause_url(path,dist_path)],_link_instead_of_copy = _link_instead_of_copy, sync=sync)\n", " except:\n", " pass\n", " continue\n", " download_list.append(pause_url(path,dist_path))\n", " else:\n", " run(f'cp -n -r -f {\"-s\" if _link_instead_of_copy else \"\"} {path} {dist_path}')\n", " if show_shell_info:\n", " print(f'{\"链接\" if _link_instead_of_copy else \"复制\"} {path} --> {dist_path}')\n", " run(f'rm -f {dist_path}/\\*.* ')\n", " if not skip_url:\n", " if show_shell_info:\n", " pprint.pprint(download_list)\n", " try:\n", " download_urls(download_list,_link_instead_of_copy = _link_instead_of_copy, sync=sync, thread_num=thread_num or 3,is_await=is_await)\n", " except:\n", " pass" ] }, { "cell_type": "markdown", "metadata": { "id": "p0uS-BLULCtD" }, "source": [ "## kaggle public API\n", "\n", "**不能使用%cd这种会改变当前工作目录的命令，会导致和其他线程冲突**\n", "\n", "---" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_kg_hide-input": true, "id": "m8FJi4j0LCtD", "trusted": true }, "outputs": [], "source": [ "# 安装kaggle的api token文件\n", "def initKaggleConfig():\n", " if Path('~/.kaggle/kaggle.json').exists():\n", " return True\n", " if Path(kaggleApiTokenFile).exists():\n", " run(f'''mkdir -p ~/.kaggle/''')\n", " run('cp '+kaggleApiTokenFile+' ~/.kaggle/kaggle.json')\n", " run(f'''chmod 600 ~/.kaggle/kaggle.json''')\n", " return True\n", " print('缺少kaggle的apiToken文件，访问：https://www.kaggle.com/你的kaggle用户名/account 获取')\n", " return False\n", "\n", "def getUserName():\n", " if not initKaggleConfig(): return\n", " import kaggle\n", " return kaggle.KaggleApi().read_config_file()['username']\n", "\n", "def createOrUpdateDataSet(path:str,datasetName:str):\n", " if not initKaggleConfig(): return\n", " print('创建或更新数据集 '+datasetName)\n", " import kaggle\n", " run(f'mkdir -p {_install_path}/kaggle_cache')\n", " run(f'rm -rf {_install_path}/kaggle_cache/*')\n", " datasetDirPath = _install_path+'/kaggle_cache/'+datasetName\n", " run('mkdir -p '+datasetDirPath)\n", " run('cp -f '+path+' '+datasetDirPath+'/')\n", " username = getUserName()\n", " print(\"kaggle username:\"+username)\n", " datasetPath = username+'/'+datasetName\n", " datasetList = kaggle.api.dataset_list(mine=True,search=datasetPath)\n", " print(datasetList)\n", " if len(datasetList) == 0 or datasetPath not in [str(d) for d in datasetList]: # 创建 create\n", " run('kaggle datasets init -p' + datasetDirPath)\n", " metadataFile = datasetDirPath+'/dataset-metadata.json'\n", " run('sed -i s/INSERT_TITLE_HERE/'+ datasetName + '/g ' + metadataFile)\n", " run('sed -i s/INSERT_SLUG_HERE/'+ datasetName + '/g ' + metadataFile)\n", " run('cat '+metadataFile)\n", " run('kaggle datasets create -p '+datasetDirPath)\n", " print('create database done')\n", " else:\n", " kaggle.api.dataset_metadata(datasetPath,datasetDirPath)\n", " kaggle.api.dataset_create_version(datasetDirPath, 'auto update',dir_mode='zip')\n", " print('upload database done')\n", "\n", "def downloadDatasetFiles(datasetName:str,outputPath:str):\n", " if not initKaggleConfig(): return\n", " print('下载数据集文件 '+datasetName)\n", " import kaggle\n", " username = getUserName()\n", " datasetPath = username+'/'+datasetName\n", " datasetList = kaggle.api.dataset_list(mine=True,search=datasetPath)\n", " if datasetPath not in [str(d) for d in datasetList]:\n", " return False\n", " run('mkdir -p '+outputPath)\n", " kaggle.api.dataset_download_files(datasetPath,path=outputPath,unzip=True)\n", " return True\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 同步文件夹到 huggingface\n", "\n", "---" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [ "# 文件夹与 huggingface 同步\n", "if _huggingface_token and _huggingface_repo:\n", " if not is_installed('watchdog'):\n", " requirements.append('watchdog')\n", " if not is_installed('huggingface_hub'):\n", " requirements.append('huggingface_hub')\n", " else:\n", " try:\n", " from huggingface_hub import HfApi,login,snapshot_download\n", " except:\n", " requirements.append('huggingface_hub')\n", "\n", "huggingface_is_init = False\n", "\n", "def init_huggingface():\n", " if not _huggingface_token:\n", " return False\n", "\n", " global huggingface_headers\n", " global huggingface_is_init\n", " \n", " from huggingface_hub import login\n", " token = replace_path(_huggingface_token)\n", " if not _huggingface_token.startswith('hf_') and Path(token).exists():\n", " with open(token,encoding = \"utf-8\") as nkfile:\n", " token = nkfile.readline()\n", " if not token.startswith('hf_'):\n", " print('huggingface token 不正确，请将 token 或 仅存放token 的txt文件路径填入 _huggingface_token 配置')\n", " return False\n", " login(token,add_to_git_credential=True)\n", " huggingface_headers = {'Authorization': 'Bearer '+token}\n", " print('huggingface token 已经加载，可以下载私有仓库或文件')\n", " \n", " if not _huggingface_repo:\n", " print('huggingface 同步收藏图片功能不会启动，可增加配置项 huggingface_token = \"token\" 和 huggingface_repo = “仓库id” 后启用 huggingface 同步收藏图片功能')\n", " return False\n", " huggingface_is_init = True\n", " return True\n", "\n", "\n", "def download__huggingface_repo(repo_id:str,dist_directory:str=None,repo_type='dataset',callback=None):\n", " if not huggingface_is_init:\n", " print('huggingface 相关功能未初始化 请调用 init_huggingface() 初始化')\n", " \n", " if not dist_directory:\n", " dist_directory = f'{_install_path}/{_ui_dir_name}/log'\n", " \n", " print('下载收藏的图片')\n", " if not Path(f'{_install_path}/cache/huggingface/huggingface_repo').exists():\n", " mkdirs(f'{_install_path}/cache/huggingface')\n", " repo_path = ''\n", " if repo_type == 'dataset':\n", " repo_path = 'datasets'\n", " if repo_type == 'space':\n", " repo_path = 'spaces'\n", " if repo_path:\n", " run(f'git clone https://huggingface.co./{repo_path}/{repo_id} huggingface_repo',cwd=f'{_install_path}/cache/huggingface')\n", " else:\n", " run(f'git clone https://huggingface.co./{repo_id} huggingface_repo',cwd=f'{_install_path}/cache/huggingface')\n", " if Path(f'{_install_path}/cache/huggingface/huggingface_repo').exists():\n", " run(f'cp -r -f -n -s {_install_path}/cache/huggingface/huggingface_repo/* {dist_directory}')\n", " if callback:\n", " callback()\n", "\n", "def start_sync_log_to_huggingface(repo_id:str,directory_to_watch:str=None,repo_type='dataset'):\n", " if not huggingface_is_init:\n", " print('huggingface 相关功能未初始化 请调用 init_huggingface() 初始化')\n", " \n", " from watchdog.observers import Observer\n", " from watchdog.events import FileSystemEventHandler\n", " from huggingface_hub import HfApi,login,snapshot_download\n", " \n", " # 配置监视的目录和 Hugging Face 仓库信息\n", " class FileChangeHandler(FileSystemEventHandler):\n", " def __init__(self, api, repo_id, repo_type,directory_to_watch):\n", " self.api = api\n", " self.repo_id = repo_id\n", " self.repo_type = repo_type\n", " self.directory_to_watch = directory_to_watch\n", " def on_created(self, event):\n", " if not event.is_directory:\n", " # 上传新文件到 Hugging Face 仓库\n", " file_path = event.src_path\n", " file_name:str = os.path.basename(file_path)\n", " print(file_name)\n", " if file_name[file_name.rindex('.'):] not in ['.png','.jpg','.txt','.webp','.jpeg']: return\n", " print(file_name,'>>','huggingface')\n", " try:\n", " self.api.upload_file(\n", " path_or_fileobj=file_path,\n", " path_in_repo=file_path.replace(self.directory_to_watch,''),\n", " repo_id=self.repo_id,\n", " repo_type=self.repo_type,\n", " )\n", " except IOError as error:\n", " print(error)\n", "\n", " def on_deleted(self, event):\n", " if not event.is_directory:\n", " # 从 Hugging Face 仓库删除文件\n", " file_path = event.src_path\n", " file_name = os.path.basename(file_path)\n", " if file_name[file_name.rindex('.'):] not in ['.png','.jpg','.txt','.webp','.jpeg']: return\n", " try:\n", " self.api.delete_file(\n", " path_in_repo=file_path.replace(self.directory_to_watch,''),\n", " repo_id=self.repo_id,\n", " repo_type=self.repo_type,\n", " )\n", " except IOError as error:\n", " print(error)\n", "\n", " def on_modified(self, event):\n", " if not event.is_directory:\n", " # 更新 Hugging Face 仓库中的文件\n", " file_path = event.src_path\n", " file_name = os.path.basename(file_path)\n", " if file_name[file_name.rindex('.'):] not in ['.png','.jpg','.txt','.webp','.jpeg']: return\n", " try:\n", " self.api.upload_file(\n", " path_or_fileobj=file_path,\n", " path_in_repo=file_path.replace(self.directory_to_watch,''),\n", " repo_id=self.repo_id,\n", " repo_type=self.repo_type,\n", " )\n", " except IOError as error:\n", " print(error)\n", "\n", " def on_moved(self, event):\n", " if not event.is_directory:\n", " file_path = event.dest_path\n", " file_name = os.path.basename(file_path)\n", " if file_name[file_name.rindex('.'):] not in ['.png','.jpg','.txt','.webp','.jpeg']: return\n", " if event.dest_path.startswith(self.directory_to_watch):\n", " try:\n", " self.api.upload_file(\n", " path_or_fileobj=file_path,\n", " path_in_repo=file_path.replace(self.directory_to_watch,''),\n", " repo_id=self.repo_id,\n", " repo_type=self.repo_type,\n", " )\n", " except IOError as error:\n", " print(error)\n", "\n", " api = HfApi()\n", " \n", " if not directory_to_watch:\n", " directory_to_watch = f'{_install_path}/{_ui_dir_name}/log'\n", " # 创建观察者对象并注册文件变化处理程序\n", " event_handler = FileChangeHandler(api,repo_id,repo_type,directory_to_watch)\n", " observer = Observer()\n", " observer.schedule(event_handler, directory_to_watch, recursive=True)\n", "\n", " # 启动观察者\n", " observer.name = \"solo_directory_to_watch\"\n", " print(f'启动收藏图片文件夹监听，并自动同步到 huggingface {repo_type} : {repo_id}')\n", " observer.start()" ] }, { "cell_type": "markdown", "metadata": { "id": "sswa04veLCtE" }, "source": [ "## 工具函数\n", "**不能使用%cd这种会改变当前工作目录的命令，会导致和其他线程冲突**\n", "\n", "---" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_kg_hide-input": true, "trusted": true }, "outputs": [], "source": [ "\n", "def zipPath(path:str,zipName:str,format='tar'):\n", " if path.startswith('$install_path'):\n", " path = path.replace('$install_path',_install_path)\n", " if path.startswith('$output_path'):\n", " path = path.replace('$install_path',_output_path)\n", " if not path.startswith('/'):\n", " path = f'{_install_path}/{_ui_dir_name}/{path}'\n", " if Path(path).exists():\n", " if 'tar' == format:\n", " run(f'tar -cf {_output_path}/'+ zipName +'.tar -C '+ path +' . ')\n", " elif 'gz' == format:\n", " run(f'tar -czf {_output_path}/'+ zipName +'.tar.gz -C '+ path +' . ')\n", " return\n", " print('指定的目录不存在：'+path)\n", " \n", "def get_folder_list(directory:str): \n", " folder_list = [] \n", " for item in os.listdir(directory): \n", " if os.path.isdir(os.path.join(directory, item)): \n", " folder_list.append(item) \n", " return folder_list \n", "def read_text_file(file_path:str):\n", " if not Path(file_path).exists(): return ''\n", " with open(file_path,\"r\") as f:\n", " text = file.read()\n", " if text: return text.strip()\n", " return ''" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 内网穿透\n", "\n", "---" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_kg_hide-input": true, "_kg_hide-output": true, "id": "coqQvTSLLCtE", "trusted": true }, "outputs": [], "source": [ "def printUrl(url,name=''):\n", " print(f'{name} 访问地址：{url}')\n", " for key in sorted(_proxy_path.keys(), key=len)[::-1]:\n", " print(f'{name} 本地服务：{_proxy_path[key]} 访问地址：{url}{key}')\n", "# ngrok\n", "def startNgrok(ngrokToken:str,ngrokLocalPort:int):\n", " if not is_installed('pyngrok'):\n", " run('pip install pyngrok')\n", " from pyngrok import conf, ngrok\n", " try:\n", " conf.get_default().auth_token = ngrokToken\n", " conf.get_default().monitor_thread = False\n", " ssh_tunnels = ngrok.get_tunnels(conf.get_default())\n", " url = ''\n", " if len(ssh_tunnels) == 0:\n", " ssh_tunnel = ngrok.connect(ngrokLocalPort)\n", " url = ssh_tunnel.public_url\n", " print('ngrok 访问地址：'+ssh_tunnel.public_url)\n", " else:\n", " print('ngrok 访问地址：'+ssh_tunnels[0].public_url)\n", " url = ssh_tunnels[0].public_url\n", " printUrl(url,'ngrok')\n", " def auto_request_ngrok():\n", " if url:\n", " while(_runing):\n", " time.sleep(60*1)\n", " try:\n", " res = requests.get(url+'/',headers={\"ngrok-skip-browser-warning\" : \"1\"},timeout=10)\n", " except:\n", " ''\n", " # print('自动调用ngrok链接以保存链接不会断开',res.status_code)\n", "\n", " # threading.Thread(target = auto_request_ngrok,daemon=True,name='solo_auto_request_ngrok').start()\n", " except:\n", " print('启动ngrok出错')\n", " \n", "def startFrpc(name,configFile):\n", " if not Path(f'{_install_path}/frpc/frpc').exists():\n", " installFrpExe()\n", " if freefrp_url:\n", " printUrl(freefrp_url,'freefrp')\n", " echoToFile(f'''\n", "cd {_install_path}/frpc/\n", "{_install_path}/frpc/frpc {configFile}\n", "''',f'{_install_path}/frpc/start.sh')\n", " get_ipython().system(f'''bash {_install_path}/frpc/start.sh''')\n", " \n", "def installFrpExe():\n", " if _useFrpc:\n", " print('安装frpc')\n", " run(f'mkdir -p {_install_path}/frpc')\n", " if Path(frpcExePath).exists():\n", " run(f'cp -f -n {frpcExePath} {_install_path}/frpc/frpc')\n", " else:\n", " run(f'wget \"https://huggingface.co./datasets/ACCA225/Frp/resolve/main/frpc\" -O {_install_path}/frpc/frpc')\n", " \n", " for ssl in frpcSSLFFlies:\n", " if Path(ssl).exists():\n", " run(f'cp -f -n {ssl}/* {_install_path}/frpc/')\n", " run(f'chmod +x {_install_path}/frpc/frpc')\n", " run(f'{_install_path}/frpc/frpc -v')\n", "\n", "def startProxy():\n", " if _useNgrok:\n", " startNgrok(ngrokToken,_server_port)\n", " if _useFrpc:\n", " startFrpc('frpc_proxy',frpcStartArg)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## NGINX 反向代理\n", "\n", "---" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_kg_hide-input": true, "_kg_hide-output": true, "trusted": true }, "outputs": [], "source": [ "\n", "# nginx 反向代理配置文件\n", "def localProxy():\n", " def getProxyLocation(subPath:str, localServer:str):\n", " return '''\n", " location '''+ subPath +'''\n", " {\n", " proxy_pass '''+ localServer +''';\n", " \n", " client_max_body_size 1000m;\n", " proxy_set_header Host $http_host;\n", " proxy_set_header X-Real-IP $remote_addr;\n", " proxy_set_header X-Real-Port $remote_port;\n", " proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n", " proxy_set_header X-Forwarded-Proto $scheme;\n", " proxy_set_header X-Forwarded-Host $host;\n", " proxy_set_header X-Forwarded-Port $server_port;\n", " proxy_set_header REMOTE-HOST $remote_addr;\n", " proxy_connect_timeout 3000s;\n", " proxy_send_timeout 3000s;\n", " proxy_read_timeout 3000s;\n", " proxy_http_version 1.1;\n", " proxy_set_header Upgrade $http_upgrade;\n", " proxy_set_header Connection 'upgrade';\n", " add_header Access-Control-Allow-Origin * always;\n", " add_header Access-Control-Allow-Headers *;\n", " add_header Access-Control-Allow-Methods \"GET, POST, PUT, OPTIONS\";\n", " }\n", " \n", " '''\n", " \n", " conf = '''\n", "server\n", "{\n", " listen '''+str(_server_port)+''';\n", " listen [::]:'''+str(_server_port)+''';\n", " server_name 127.0.0.1 localhost 0.0.0.0 \"\";\n", " \n", " fastcgi_send_timeout 3000s;\n", " fastcgi_read_timeout 3000s;\n", " fastcgi_connect_timeout 3000s;\n", " \n", " if ($request_method = OPTIONS) {\n", " return 200;\n", " }\n", " \n", " '''+ ''.join([getProxyLocation(key,_proxy_path[key]) for key in sorted(_proxy_path.keys(), key=len)[::-1]]) +'''\n", "}\n", "'''\n", " echoToFile(conf,'/etc/nginx/conf.d/proxy_nginx.conf')\n", " if not check_service('localhost',_server_port):\n", " run(f'''nginx -c /etc/nginx/nginx.conf''')\n", " run(f'''nginx -s reload''')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 线程清理工具\n", "\n", "---\n", "\n", "清理线程名以 solo_ 开头的所有线程" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_kg_hide-input": true, "trusted": true }, "outputs": [], "source": [ "import inspect\n", "import ctypes\n", "\n", "def _async_raise(tid, exctype):\n", " \"\"\"raises the exception, performs cleanup if needed\"\"\"\n", " tid = ctypes.c_long(tid)\n", " if not inspect.isclass(exctype):\n", " exctype = type(exctype)\n", " res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))\n", " if res == 0:\n", " raise ValueError(\"invalid thread id\")\n", " elif res != 1:\n", " # \"\"\"if it returns a number greater than one, you're in trouble,\n", " # and you should call it again with exc=NULL to revert the effect\"\"\"\n", " ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)\n", " raise SystemError(\"PyThreadState_SetAsyncExc failed\")\n", "\n", "def stop_thread(thread):\n", " _async_raise(thread.ident, SystemExit)\n", "\n", "def stop_solo_threads():\n", " global _runing\n", " _runing = False\n", " # 获取当前所有活动的线程\n", " threads = threading.enumerate()\n", " # 关闭之前创建的子线程\n", " for thread in threads:\n", " if thread.name.startswith('solo_'):\n", " print(f'结束线程：{thread.name}')\n", " try:\n", " stop_thread(thread)\n", " except socket.error:\n", " print(f'结束线程：{thread.name} 执行失败')" ] }, { "cell_type": "markdown", "metadata": { "id": "Ve3p8oOkLCtE" }, "source": [ "# webui 安装和配置函数\n", "---" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_kg_hide-input": true, "id": "GTjyBJihLCtE", "trusted": true }, "outputs": [], "source": [ "envInstalled=False\n", "quickStart = True\n", "#安装\n", "def install():\n", " print('安装')\n", " os.chdir(f'''{_install_path}''')\n", " run(f'''git lfs install''')\n", " run(f'''git config --global credential.helper store''')\n", " for requirement in requirements:\n", " run(f'pip install {requirement}')\n", " if _reLoad:\n", " run(f'''rm -rf {_install_path}/{_ui_dir_name}''')\n", " if Path(f\"{_ui_dir_name}\").exists():\n", " os.chdir(f'''{_install_path}/{_ui_dir_name}/''')\n", " run(f'''git checkout .''')\n", " run(f'''git pull''')\n", " else:\n", " run(f'''git clone --recursive {_sd_git_repo} {_ui_dir_name}''')\n", " if not Path(f'''{_install_path}/{_ui_dir_name}''').exists():\n", " print('sd-webui主程序安装失败，请检查 sd_git_repo 配置的值是否正确')\n", " sys.exit(0)\n", " os.chdir(f'''{_install_path}/{_ui_dir_name}''')\n", " print('安装 完成')\n", "\n", "# 链接输出目录\n", "def link_dir():\n", " print('链接输出目录')\n", " # 链接图片输出目录\n", " run(f'''mkdir -p {_output_path}/outputs''')\n", " run(f'''rm -rf {_install_path}/{_ui_dir_name}/outputs''')\n", " run(f'''ln -s -r {_output_path}/outputs {_install_path}/{_ui_dir_name}/''')\n", " # 输出收藏目录\n", " run(f'''mkdir -p {_output_path}/log''')\n", " run(f'''rm -rf {_install_path}/{_ui_dir_name}/log''')\n", " run(f'''ln -s -r {_output_path}/log {_install_path}/{_ui_dir_name}/''')\n", " # 链接训练输出目录 文件夹链接会导致功能不能用\n", " run(f'''rm -rf {_install_path}/{_ui_dir_name}/textual_inversion''')\n", " run(f'''mkdir -p {_output_path}/textual_inversion/''')\n", " run(f'''ln -s -r {_output_path}/textual_inversion {_install_path}/{_ui_dir_name}/''')\n", " print('链接输出目录 完成') \n", "\n", "def install_optimizing():\n", " run('add-apt-repository ppa:deadsnakes/ppa -y')\n", " run('apt update -y')\n", " run('apt install nginx -y')\n", " # run('apt install python3.10 -y')\n", " \n", "#安装依赖\n", "def install_dependencies():\n", " print('安装需要的python环境')\n", " global envInstalled\n", " global venvPath\n", " if Path(f'{_install_path}/{_ui_dir_name}/venv').exists():\n", " print('跳过安装python环境')\n", " envInstalled = True\n", " return\n", " \n", " if quickStart:\n", " if not Path(venvPath).exists():\n", " mkdirs(f'{_install_path}/venv_cache',True)\n", " if not Path(f'{_install_path}/venv_cache/venv.tar.bak').exists():\n", " print('下载 venv.zip')\n", " download_file('https://huggingface.co./viyi/sdwui/resolve/main/venv.zip','venv.zip',f'{_install_path}/venv_cache')\n", " run(f'''unzip {_install_path}/venv_cache/venv.zip -d {_install_path}/venv_cache''')\n", " venvPath = f'{_install_path}/venv_cache/venv.tar.bak'\n", " run(f'''rm -rf {_install_path}/venv_cache/venv.zip''')\n", " elif venvPath.endswith('.zip'):\n", " mkdirs(f'{_install_path}/venv_cache',True)\n", " run(f'''unzip {venvPath} -d {_install_path}/venv_cache''')\n", " venvPath = f'{_install_path}/venv_cache/venv.tar.bak'\n", " print('解压环境')\n", " mkdirs(f'{_install_path}/{_ui_dir_name}/venv')\n", "# run('python3.10 -m venv venv',cwd=f'{_install_path}/{_ui_dir_name}')\n", " run(f'tar -xf {venvPath} -C ./venv',cwd=f'{_install_path}/{_ui_dir_name}')\n", " run(f'rm -f {_install_path}/{_ui_dir_name}/venv/bin/pip*')\n", " run(f'rm -f {_install_path}/{_ui_dir_name}/venv/bin/python*')\n", " venv.create(f'{_install_path}/{_ui_dir_name}/venv')\n", " if not Path(f'{_install_path}/{_ui_dir_name}/venv/bin/pip').exists():\n", " run('curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py')\n", " run(f'{_install_path}/{_ui_dir_name}/venv/bin/python3 get-pip.py')\n", "\n", " get_ipython().system(f'''{_install_path}/{_ui_dir_name}/venv/bin/python3 -V''')\n", " get_ipython().system(f'''{_install_path}/{_ui_dir_name}/venv/bin/python3 -m pip -V''')\n", "\n", " envInstalled = True\n", " print('安装需要的python环境 完成')\n", " \n", "# 个性化配置 \n", "def use_config():\n", " print('使用自定义配置 包括tag翻译 \\n')\n", " run(f'''mkdir -p {_install_path}/temp''')\n", " run(f'git clone {_sd_config_git_repu} sd-configs',cwd=f'{_install_path}/temp')\n", " run(f'cp -r -f -n {_install_path}/temp/sd-configs/dist/* {_install_path}/{_ui_dir_name}')\n", " if not Path(_ui_config_file).exists() and _ui_config_file != 'ui-config.json': # ui配置文件\n", " run(f'''mkdir -p {_ui_config_file[:_ui_config_file.rfind('/')]}''')\n", " run(f'cp -f -n {_install_path}/{_ui_dir_name}/ui-config.json {_ui_config_file}')\n", " if not Path(_setting_file).exists() and _setting_file != 'config.json': # 设置配置文件\n", " run(f'''mkdir -p {_setting_file[:_setting_file.rfind('/')]}''')\n", " run(f'cp -f -n {_install_path}/{_ui_dir_name}/config.json {_setting_file}')\n", "\n", "def copy_last_log_to_images():\n", " if not Path(f'{_install_path}/{_ui_dir_name}/log/images').exists(): mkdirs(f'{_install_path}/{_ui_dir_name}/log/images')\n", " print('复制编号最大的一张收藏图到输出目录，用于保持编号，否则会出现收藏的图片被覆盖的情况')\n", " img_list = os.listdir(f'{_install_path}/{_ui_dir_name}/log/images')\n", " last_img_path = ''\n", " last_img_num = 0\n", " for img in img_list:\n", " if re.findall(r'^\\d+-',str(img)):\n", " num = int(re.findall(r'^\\d+-',str(img))[0][:-1])\n", " if num > last_img_num:\n", " last_img_path = img\n", " last_img_num = num\n", " \n", " if not last_img_path: return\n", " \n", " print(f'{_install_path}/{_ui_dir_name}/log/images/{last_img_path} {_install_path}/{_ui_dir_name}/outputs/txt2img-images')\n", " run(f'''mkdir -p {_install_path}/{_ui_dir_name}/outputs/txt2img-images''')\n", " run(f'''cp -f {_install_path}/{_ui_dir_name}/log/images/{last_img_path} {_install_path}/{_ui_dir_name}/outputs/txt2img-images/''')\n", " \n", " print(f'{_install_path}/{_ui_dir_name}/log/images/{last_img_path} {_install_path}/{_ui_dir_name}/outputs/img2img-images')\n", " run(f'''mkdir -p {_install_path}/{_ui_dir_name}/outputs/img2img-images''')\n", " run(f'''cp -f {_install_path}/{_ui_dir_name}/log/images/{last_img_path} {_install_path}/{_ui_dir_name}/outputs/img2img-images/''')\n", " \n", "def start_webui(i):\n", " # 只要不爆内存，其他方式关闭后会再次重启 访问地址会发生变化\n", " print(i,'--port',str(_server_port+1+i))\n", " if i>0:\n", " print(f'使用第{i+1}张显卡启动第{i+1}个服务，通过frpc或nrgok地址后加{_sub_path[i]}进行访问')\n", "\n", " restart_times = 0\n", " last_restart_time = time.time()\n", " while _runing:\n", " os.chdir(f'{_install_path}/{_ui_dir_name}')\n", " root_path = _sub_path[i]\n", " if root_path.endswith('/'): root_path = root_path[:-1]\n", " get_ipython().system(f'''venv/bin/python3 launch.py --device-id={i} --port {str(_server_port+1+i)} --subpath={_sub_path[i]}''')\n", " print('10秒后重启服务')\n", " if time.time() - last_restart_time < 60:\n", " restart_times = restart_times + 1\n", " else:\n", " restart_times = 0\n", " last_restart_time = time.time()\n", " if restart_times >3 :\n", " # 如果180秒内重启了3此，将不再自动重启\n", " break\n", " time.sleep(10)\n", " \n", "# 启动\n", "def start():\n", " print('启动webui')\n", " os.chdir(f'''{_install_path}/{_ui_dir_name}''')\n", " args = ''\n", " if _ui_config_file is not None and _ui_config_file != '' and Path(_ui_config_file).exists(): # ui配置文件\n", " args += ' --ui-config-file=' + _ui_config_file\n", " if _setting_file is not None and _setting_file != '' and Path(_setting_file).exists(): # 设置配置文件\n", " args += ' --ui-settings-file=' + _setting_file\n", " args += ' ' + otherArgs\n", " os.environ['COMMANDLINE_ARGS']=args\n", " run(f'''echo COMMANDLINE_ARGS=$COMMANDLINE_ARGS''')\n", " os.environ['REQS_FILE']='requirements_versions.txt'\n", "\n", " with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:\n", " for i in range(torch.cuda.device_count() if _multi_case else 1):\n", " executor.submit(start_webui,i)\n", " while _runing and not check_service('localhost',str(_server_port+1+i)): # 当当前服务启动完成才允许退出此次循环\n", " time.sleep(5)\n", " if not _runing: break\n", " time.sleep(10)" ] }, { "cell_type": "markdown", "metadata": { "id": "qLvsk8ByLCtF" }, "source": [ "# 入口函数\n", "---" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_kg_hide-input": true, "id": "IOKjaMlcLCtF", "trusted": true }, "outputs": [], "source": [ "\n", "# 启动非webui相关的的内容，加快启动速度\n", "def main():\n", " global envInstalled\n", " global huggingface_is_init\n", " global _runing\n", " _init_conf()\n", " stop_solo_threads()\n", " print('启动...')\n", " startTicks = time.time()\n", " time.sleep(5)\n", " _runing = True\n", " isInstall = True if os.getenv('IsInstall','False') == 'True' else False\n", " _proxy_path[_sub_path[0]] = f'http://127.0.0.1:{_server_port+1}/'\n", " _proxy_path[_sub_path[1]] = f'http://127.0.0.1:{_server_port+2}/'\n", " proxy_thread = threading.Thread(target = startProxy, daemon=True, name='solo_startProxy')\n", " proxy_thread.start()\n", " if isInstall is False or _reLoad: \n", " print('安装运行环境')\n", " install()\n", " link_dir()\n", " init_huggingface()\n", " install_optimizing()\n", " if not _skip_webui:\n", " threading.Thread(target = install_dependencies,daemon=True,name='solo_install_dependencies').start()\n", " else: envInstalled = True\n", " link_or_download_flie(replace_path(_async_downloading), _link_instead_of_copy=_link_instead_of_copy,\n", " base_path=f'{_install_path}/{_ui_dir_name}')\n", " if huggingface_is_init:\n", " threading.Thread(target = download__huggingface_repo,daemon=True,\n", " args=([_huggingface_repo]),\n", " kwargs={\"callback\":copy_last_log_to_images},\n", " name='solo_download__huggingface_repo').start()\n", " \n", " link_or_download_flie(replace_path(_before_downloading), _link_instead_of_copy=_link_instead_of_copy,\n", " base_path=f'{_install_path}/{_ui_dir_name}',is_await=True,sync=True)\n", " t = 0\n", " while _runing and not envInstalled:\n", " if t%10==0:\n", " print('等待python环境安装...')\n", " t = t+1\n", " time.sleep(1)\n", " use_config()\n", " os.environ['IsInstall'] = 'True'\n", " else:\n", " envInstalled = True\n", " localProxy()\n", " link_or_download_flie(replace_path(_before_start_sync_downloading), _link_instead_of_copy=_link_instead_of_copy,\n", " base_path=f'{_install_path}/{_ui_dir_name}',sync=True)\n", " if init_huggingface():\n", " start_sync_log_to_huggingface(_huggingface_repo)\n", " ticks = time.time()\n", " _on_before_start()\n", " print(\"加载耗时:\",(ticks - startTicks),\"秒\")\n", " if _skip_webui:\n", " print('跳过webui启动')\n", " proxy_thread.join()\n", " else:\n", " start()\n" ] }, { "cell_type": "markdown", "metadata": { "id": "0oaCRs2gLCtF" }, "source": [ "# 执行区域\n", "---" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_kg_hide-output": true, "id": "O3DR0DWHLCtF", "scrolled": true, "trusted": true }, "outputs": [], "source": [ "# 启动\n", "# _reLoad = True\n", "# hidden_console_info = False\n", "# run_by_none_device = True\n", "# show_shell_info = True\n", "\n", "print(f'当前sd的安装路径是：{_install_path}/{_ui_dir_name}')\n", "print(f'当前图片保存路径是：{_output_path}')\n", "print(f'当前数据集路径是：{_input_path}')\n", "\n", "print(update_desc)\n", "\n", "if _skip_start:\n", " print('已跳过自动启动，可手动执行 main() 进行启动。')\n", " print('''推荐的启动代码：\n", "try:\n", " check_gpu() # 检查是否存在gpu\n", " main()\n", "except KeyboardInterrupt:\n", " stop_solo_threads() # 中断后自动停止后台线程 （有部分功能在后台线程中运行）\n", " ''')\n", "else:\n", " try:\n", " check_gpu()\n", " main()\n", " except KeyboardInterrupt:\n", " stop_solo_threads()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "trusted": true }, "outputs": [], "source": [] } ], "metadata": { "kaggle": { "accelerator": "nvidiaTeslaT4", "dataSources": [ { "datasetId": 2716934, "sourceId": 6167400, "sourceType": "datasetVersion" }, { "datasetId": 3654544, "sourceId": 6346544, "sourceType": "datasetVersion" }, { "datasetId": 2962375, "sourceId": 6720235, "sourceType": "datasetVersion" }, { "datasetId": 3074484, "sourceId": 6817788, "sourceType": "datasetVersion" } ], "isGpuEnabled": true, "isInternetEnabled": true, "language": "python", "sourceType": "notebook" }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.13" } }, "nbformat": 4, "nbformat_minor": 4 }