魏然 首次上传于 2024/04/13

app.py

@@ -0,0 +1,203 @@
+import argparse
+import gradio as gr
+import json
+from textwrap import dedent
+from task_plan import task_plan_list
+import subprocess
+from datetime import datetime
+
+
+def read_specific_line(file_path, line_number):
+    try:
+        with open(file_path, 'r') as file:
+            for current_line, line in enumerate(file, start=1):
+                if current_line == line_number:
+                    return line.strip()
+        return None  # 如果文件不足指定的行数，返回None
+    except FileNotFoundError:
+        return None  # 如果文件不存在，返回None
+
+
+def get_folder_paths(file_path):
+    source_folder = read_specific_line(file_path, 2)
+    target_folder = read_specific_line(file_path, 3)
+
+    if source_folder is None or target_folder is None:
+        return "错误信息：文件不存在。", None, None
+    elif source_folder == "" or target_folder == "":
+        return "错误信息：文件中的第2行或第3行为空。", None, None
+    else:
+        return None, source_folder, target_folder
+
+
+def read_specific_line(file_path, line_number):
+    with open(file_path, "r") as file:
+        for i, line in enumerate(file, 1):
+            if i == line_number:
+                return line.strip()
+
+
+def repo_pull(url):
+    # 从远程仓库拉取代码的操作
+    subprocess.check_output(["python3", "repo_pull.py", "--repo_url",
+                             str(url).replace(".git", "").replace("git", "").replace("clone", "").replace(" ",
+                                                                                                          "")])
+    yield
+
+
+# 将代码推送到远程仓库的操作
+def repo_push(hf_repo_id, tmp1, tmp2, tmp3, tmp4, select_task_plan):
+    task_plan_index = list(task_plan_list.keys()).index(select_task_plan)
+    if task_plan_index >= 0 and task_plan_index <= 5:
+        if task_plan_index == 0:
+            # 克隆目标仓库到本地
+            subprocess.check_output(["python3", "repo_pull.py", "--repo_url",
+                                     str(hf_repo_id).replace(".git", "").replace("git", "").replace("clone",
+                                                                                                    "").replace(" ",
+                                                                                                                "")])
+
+            # # 打开文件并读取第2行数据
+            file_path = ".folderinfo"
+            # # 读取指定行数的数据
+            # source_folder = read_specific_line(file_path, 1)
+            # target_folder = read_specific_line(file_path, 2)
+
+            error, source_folder, target_folder = get_folder_paths(file_path)
+
+            if error:
+                print(error)  # 这将打印出错误信息
+            else:
+                print(f"源文件夹: {source_folder}")
+                print(f"目标文件夹: {target_folder}")
+
+            # 获取当前日期和时间
+            current_datetime = datetime.now()
+            # 格式化日期和时间
+
+            formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
+            commit_message = f"魏然 首次上传于 {formatted_datetime}"
+            # subprocess.check_output(
+            #     f"cd {target_folder} && cp -rf ../{str(source_folder).replace('./', '')}/* . && git add -A . && git commit -m \"{commit_message}\"",
+            #     shell=True)
+
+
+        elif task_plan_index == 1:
+            print("选中了 1")
+        elif task_plan_index == 2:
+            print("选中了 2")
+        elif task_plan_index == 3:
+            print("选中了 3")
+        elif task_plan_index == 4:
+            print("选中了 4")
+        else:
+            print("选中了 5")
+    else:
+        print("task_plan_index 不在范围内")
+
+    yield
+
+
+# 更新进度
+def load_set(wisemodel_link, hf_repo_id, tmp1, tmp2, tmp3, tmp4, select_task_plan, progress=gr.Progress()):
+    tasks = [repo_pull(wisemodel_link),
+             repo_push(hf_repo_id, tmp1, tmp2, tmp3, tmp4, select_task_plan)]  # 将 repo_pull 和 repo_push 添加到任务列表中
+    total_tasks = len(tasks)
+    for i, task in enumerate(tasks, start=1):
+        progress(i / total_tasks, desc=f"{i} / {total_tasks} ")
+        for _ in task:
+            pass
+    return "下载完成！"
+
+
+# 创建解析命令行参数的函数
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--language", type=str, default="zh-CN",
+                        help="Language option, e.g., 'en-US' or 'zh-CN' or ja-JP or ko-KR or th-TH support now!")
+    return parser.parse_args()
+
+
+# 加载语言对应的JSON文件
+def load_language_json(language):
+    filename = f"locales/{language}.json"
+    try:
+        with open(filename, "r", encoding="utf-8") as file:
+            return json.load(file)
+    except FileNotFoundError:
+        print(f"Error: File '{filename}' not found.")
+        return {}
+
+
+# 加载JSON以支持国际化
+args = parse_args()
+language_json = load_language_json(args.language)
+
+# # 获取对应语言的文本信息
+info = language_json.get("info", {})
+_title = info.get("page_title", {})
+_about = info.get("about", {})
+_wisemodel_lable_title = info.get("wisemodel_lable_title", {})
+_hf_repo_id_title = info.get("hf_repo_id_title", {})
+_button = info.get("button", {})
+_clear = info.get("clear", {})
+_setting = info.get("setting", {})
+_HuggingFace_Token_title = info.get("HuggingFace_Token_title", {})
+_HuggingFace_Token_placeholder = info.get("HuggingFace_Token_placeholder", {})
+_WiseModel_Token_title = info.get("WiseModel_Token_title", {})
+_WiseModel_Token_placeholder = info.get("WiseModel_Token_placeholder", {})
+_Logs = info.get("Logs", {})
+_task_plan = info.get("task_plan", {})
+_source_user_id = info.get("source_user_id", {})
+_target_user_id = info.get("target_user_id", {})
+_source_user_placeholder = info.get("source_user_placeholder", {})
+_target_user_placeholder = info.get("target_user_placeholder", {})
+#
+with gr.Blocks(title=_title, theme=gr.themes.Soft(text_size="sm")) as demo:
+    with gr.Accordion("📒 " + _about, open=False):
+        _ = f""" 这个空间适用于将 WiseModel 仓库同步转移到 Hugging Face 仓库。\n
+                **请首先确保你是仓库的所有者或者拥有者同意你这样做！**\n
+                # 如何使用这个空间?\n
+                - 复制这个空间并提供 WiseModel token（可选）和你的 读/写 HF token（必填）\n
+                - 在Hugging Face上创建你的目标模型仓库。这一步需要手动完成，空间不会为你创建一个空的仓库。\n
+                - 在你自己的私有空间中填写以下信息。\n
+                - 点击提交按钮，然后观察日志中的输出进展。\n
+                - 重新创建修改自己的 README.md 文件（因为元数据与 HF 不兼容）
+            """
+        gr.Markdown(dedent(_))
+
+    wisemodel_link = gr.Textbox(label=_wisemodel_lable_title)
+    hf_repo_id = gr.Textbox(
+        label=_hf_repo_id_title)
+    with gr.Accordion("🔧 " + _setting, open=False):
+        with gr.Column(scale=12):
+            select_task_plan = gr.Dropdown(choices=list(task_plan_list.keys()), allow_custom_value=True, value="",
+                                           label=_task_plan)
+            with gr.Row():
+                tmp1 = gr.Textbox(
+                    label=_HuggingFace_Token_title,
+                    placeholder=_HuggingFace_Token_placeholder,
+                    type="password",  # 设置为密码类型，不显示明文
+                )
+                tmp2 = gr.Textbox(
+                    label=_WiseModel_Token_title,
+                    placeholder=_WiseModel_Token_placeholder,
+                    type="password",  # 设置为密码类型，不显示明文
+                )
+            with gr.Row():
+                tmp3 = gr.Textbox(
+                    label=_source_user_id,
+                    placeholder=_source_user_placeholder,
+                )
+                tmp4 = gr.Textbox(
+                    label=_target_user_id,
+                    placeholder=_target_user_placeholder,
+                )
+    with gr.Row():
+        with gr.Column(scale=100):
+            button = gr.Button(_button, variant="primary")
+        with gr.Column(scale=1):
+            clear = gr.Button(_clear)
+    output = gr.Textbox(label=_Logs)
+    button.click(load_set, [wisemodel_link, hf_repo_id, tmp1, tmp2, tmp3, tmp4, select_task_plan], [output])
+if __name__ == "__main__":
+    demo.launch(show_error=True, share=True, server_name="0.0.0.0", server_port=7860, debug=True)

locales/en-US.json

@@ -0,0 +1,21 @@
+{
+    "info": {
+        "page_title": "GitSync Master",
+        "about": "About",
+        "wisemodel_lable_title": "Paste your [Source] repository address here:",
+        "hf_repo_id_title": "Paste your [Target] repository address here:",
+        "button": "Clone",
+        "clear": "\uD83D\uDDD1\uFE0F Clear",
+        "setting": "More Settings",
+        "HuggingFace_Token_title": "[Source] Repository Token:",
+        "HuggingFace_Token_placeholder": "Enter your [Source] repository access token here, if you don't have one, create it by logging into the source repository!",
+        "WiseModel_Token_title": "[Target] Repository Token:",
+        "WiseModel_Token_placeholder": "Enter your [Target] repository access token here, if you don't have one, create it by logging into the target repository!",
+        "Logs": "Logs",
+        "task_plan": "Please select a task plan",
+        "source_user_placeholder": "Enter the source repository account here",
+        "target_user_placeholder": "Enter the target repository account here",
+        "source_user_id": "[Source] Repository Account:",
+        "target_user_id": "[Target] Repository Account:"
+    }
+}

locales/ja-JP.json

@@ -0,0 +1,21 @@
+{
+    "info": {
+        "page_title": "GitSync Master",
+        "about": "約",
+        "wisemodel_lable_title": "ここにあなたの[ソース]リポジトリのアドレスを貼り付けてください：",
+        "hf_repo_id_title": "ここにあなたの[ターゲット]リポジトリのアドレスを貼り付けてください：",
+        "button": "クローン",
+        "clear": "\uD83D\uDDD1\uFE0F クリア",
+        "setting": "さらなる設定",
+        "HuggingFace_Token_title": "[ソース]リポジトリトークン：",
+        "HuggingFace_Token_placeholder": "[ソース]リポジトリのアクセストークンをここに入力してください。お持ちでない場合は、ソースリポジトリにログインして作成してください！",
+        "WiseModel_Token_title": "[ターゲット]リポジトリトークン：",
+        "WiseModel_Token_placeholder": "[ターゲット]リポジトリのアクセストークンをここに入力してください。お持ちでない場合は、ターゲットリポジトリにログインして作成してください！",
+        "Logs": "ログ",
+        "task_plan": "タスクプランを選択してください",
+        "source_user_placeholder": "ここにソースリポジトリアカウントを入力してください",
+        "target_user_placeholder": "ここにターゲットリポジトリアカウントを入力してください",
+        "source_user_id": "[ソース]リポジトリアカウント：",
+        "target_user_id": "[ターゲット]リポジトリアカウント："
+    }
+}

locales/ko-KR.json

@@ -0,0 +1,21 @@
+{
+    "info": {
+        "page_title": "GitSync Master",
+        "about": "소개",
+        "wisemodel_lable_title": "여기에 귀하의 [소스] 저장소 주소를 붙여 넣으십시오:",
+        "hf_repo_id_title": "여기에 귀하의 [대상] 저장소 주소를 붙여 넣으십시오:",
+        "button": "복제",
+        "clear": "\uD83D\uDDD1\uFE0F 지우기",
+        "setting": "추가 설정",
+        "HuggingFace_Token_title": "[소스] 저장소 토큰:",
+        "HuggingFace_Token_placeholder": "여기에 귀하의 [소스] 저장소 액세스 토큰을 입력하십시오. 토큰이 없는 경우 소스 저장소에 로그인하여 생성하십시오!",
+        "WiseModel_Token_title": "[대상] 저장소 토큰:",
+        "WiseModel_Token_placeholder": "여기에 귀하의 [대상] 저장소 액세스 토큰을 입력하십시오. 토큰이 없는 경우 대상 저장소에 로그인하여 생성하십시오!",
+        "Logs": "로그",
+        "task_plan": "작업 계획을 선택하십시오",
+        "source_user_placeholder": "소스 저장소 계정을 여기에 입력하십시오",
+        "target_user_placeholder": "대상 저장소 계정을 여기에 입력하십시오",
+        "source_user_id": "[소스] 저장소 계정:",
+        "target_user_id": "[대상] 저장소 계정:"
+    }
+}

locales/th-TH.json

@@ -0,0 +1,21 @@
+{
+    "info": {
+        "page_title": "GitSync Master",
+        "about": "เกี่ยวกับ",
+        "wisemodel_lable_title": "วางที่อยู่เฉพาะของคุณ [Source] ที่นี่:",
+        "hf_repo_id_title": "วางที่อยู่เฉพาะของคุณ [Target] ที่นี่:",
+        "button": "โคลน",
+        "clear": "\uD83D\uDDD1\uFE0F ล้าง",
+        "setting": "การตั้งค่าเพิ่มเติม",
+        "HuggingFace_Token_title": "โทเค็นเข้าถึง[Source] ที่นี่:",
+        "HuggingFace_Token_placeholder": "ป้อนโทเค็นเข้าถึง [Source] ที่นี่ หากคุณไม่มี ให้สร้างโดยเข้าสู่ระบบที่เกี่ยวข้อง!",
+        "WiseModel_Token_title": "โทเค็นเข้าถึง[Target] ที่นี่:",
+        "WiseModel_Token_placeholder": "ป้อนโทเค็นเข้าถึง [Target] ที่นี่ หากคุณไม่มี ให้สร้างโดยเข้าสู่ระบบที่เกี่ยวข้อง!",
+        "Logs": "บันทึก",
+        "task_plan": "กรุณาเลือกแผนงานงาน",
+        "source_user_placeholder": "ป้อนบัญชีที่เกี่ยวข้องกับ Source ที่นี่",
+        "target_user_placeholder": "ป้อนบัญชีที่เกี่ยวข้องกับ Target ที่นี่",
+        "source_user_id": "บัญชี[Source] ที่นี่:",
+        "target_user_id": "บัญชี[Target] ที่นี่:"
+    }
+}

locales/zh-CN.json

@@ -0,0 +1,21 @@
+{
+    "info": {
+        "page_title": "GitSync Master",
+        "about": "相关信息",
+        "wisemodel_lable_title": "此处粘贴你的（模型/数据集/代码）[源] 仓库地址：",
+        "hf_repo_id_title": "此处粘贴你的（模型/数据集/代码）[目标] 仓库地址：",
+        "button": "克隆",
+        "clear": "\uD83D\uDDD1\uFE0F 清空",
+        "setting": "更多设置",
+        "HuggingFace_Token_title": "[源] 仓库令牌:",
+        "HuggingFace_Token_placeholder": "此处输入 [源] 仓库访问令牌，如果没有自行登陆源仓库创建一个！",
+        "WiseModel_Token_title": "[目标] 仓库令牌:",
+        "WiseModel_Token_placeholder": "此处输入 [目标] 仓库访问令牌，如果没有自行登陆目标仓库创建一个！",
+        "Logs": "日志",
+        "task_plan": "请选择一个任务方案",
+        "source_user_placeholder": "在这里输入源仓库账号",
+        "target_user_placeholder": "在这里输入目标仓库账号",
+        "source_user_id": "[源]仓库账号:",
+        "target_user_id": "[目标]仓库账号:"
+    }
+}

repo_pull.py

@@ -0,0 +1,66 @@
+import os
+import subprocess
+import sys
+import hashlib
+import random
+import string
+
+
+def get_file_count(repo_path):
+    # 使用git命令获取仓库中的文件数量
+    result = subprocess.run(['git', 'ls-files'], cwd=repo_path, capture_output=True, text=True)
+    files = result.stdout.split('\n')
+    # 过滤掉空字符串
+    files = list(filter(None, files))
+    return len(files)
+
+
+def generate_random_string(length):
+    # 生成随机字符串
+    return ''.join(random.choices(string.ascii_lowercase + string.digits, k=length))
+
+
+def clone_repository_without_progress(repo_url):
+    # 生成随机的目标路径
+    destination_path = "./" + generate_random_string(6)
+
+    # 克隆仓库
+    try:
+        subprocess.run(['git', 'clone', '--recursive', repo_url, destination_path], check=True)
+        print("Repository cloned successfully!")
+    except Exception as e:
+        print(f"An error occurred while cloning the repository: {e}")
+        return
+
+    # 获取文件总数
+    total_files = get_file_count(destination_path)
+
+    # 遍历仓库中的文件，不再显示进度条
+    for root, dirs, files in os.walk(destination_path):
+        for file in files:
+            pass
+
+    # 获取第一层文件夹名字
+    first_layer_folder = os.path.basename(destination_path)
+    # 使用哈希值作为新文件夹名
+    folder_hash = hashlib.md5(first_layer_folder.encode()).hexdigest()[:6]
+    new_folder_name = f"{folder_hash}"
+    new_folder_path = os.path.join(os.path.dirname(destination_path), new_folder_name)
+    # 重命名第一层文件夹
+    os.rename(destination_path, new_folder_path)
+    # 记录文件夹路径
+    with open('.folderinfo', 'a') as f:
+        f.write(new_folder_path + '\n')
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Usage: python3 script.py --repo_url <repository_url>")
+        sys.exit(1)
+
+    if sys.argv[1] != "--repo_url":
+        print("Usage: python3 script.py --repo_url <repository_url>")
+        sys.exit(1)
+
+    repo_url = sys.argv[2]
+    clone_repository_without_progress(repo_url)

task_plan.py

@@ -0,0 +1,7 @@
+task_plan_list = {'[源]huggingface.co 到 [目标]wisemodel.cn': 'HF2WM',
+                  '[源]wisemodel.cn 到 [目标]huggingface.co': 'WM2HF',
+                  '[源]github.com 到 [目标]huggingface.co': 'GIT2HF',
+                  '[源]huggingface.co 到 [目标]github.com': 'HF2GITHUB',
+                  '[源]github.com 到 [目标]gitee.cn': 'GITHUB2GITEE',
+                  '[源]gitee.cn 到 [目标]github.com': 'GITEE2GITHUB',
+                  }