Duplicate from mikeee/llama-2-70b-guanaco-qlora-ggml

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,13 @@
+call-activate.bat
+okteto.yml
+okteto-up.bat
+install-sw.sh
+install-sw1.sh
+start-sshd.sh
+pyproject.toml
+models
+.ruff_cache
+run-nodemon.sh
+app-.py
+nodemon.json
+node_modules

.husky/pre-commit

@@ -0,0 +1,6 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+# npm test
+ruff . --fix
+black .

.ruff.toml

@@ -0,0 +1,18 @@
+# Assume Python 3.10.
+target-version = "py310"
+# Decrease the maximum line length to 79 characters.
+line-length = 300
+
+# pyflakes, pycodestyle, isort
+# flake8 YTT, pydocstyle D, pylint PLC
+select = ["F", "E", "W", "I001", "YTT", "D", "PLC"]
+# select = ["ALL"]
+
+# D100 Missing docstring in public module
+# D103 Missing docstring in public function
+# D101 Missing docstring in public class
+# `multi-line-summary-first-line` (D212)
+# `one-blank-line-before-class` (D203)
+extend-ignore = ["D100", "D103", "D101", "D212", "D203"]
+
+exclude = [".venv"]

.stignore

@@ -0,0 +1,103 @@
+models
+*.bin
+.git
+# Byte-compiled / optimized / DLL files
+__pycache__
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build
+develop-eggs
+dist
+downloads
+eggs
+.eggs
+lib
+lib64
+parts
+sdist
+var
+wheels
+pip-wheel-metadata
+share/python-wheels
+*.egg-info
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build
+
+# PyBuilder
+target
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env
+venv
+ENV
+env.bak
+venv.bak
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mypy
+.mypy_cache
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre

README.md

@@ -0,0 +1,13 @@
+---
+title: llama-2-7b-or-13b-ggml
+emoji: 🚀
+colorFrom: green
+colorTo: green
+sdk: gradio
+sdk_version: 3.39.0
+app_file: app.py
+pinned: true
+duplicated_from: mikeee/llama-2-70b-guanaco-qlora-ggml
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,389 @@
+"""Run codes."""
+# pylint: disable=line-too-long, broad-exception-caught, invalid-name, missing-function-docstring, too-many-instance-attributes, missing-class-docstring
+# ruff: noqa: E501
+import os
+import platform
+import random
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from textwrap import dedent
+
+# from types import SimpleNamespace
+import gradio as gr
+import psutil
+from about_time import about_time
+from ctransformers import AutoModelForCausalLM
+from dl_hf_model import dl_hf_model
+from loguru import logger
+
+from examples_list import examples_list
+
+url = "https://huggingface.co/TheBloke/llama-2-13B-Guanaco-QLoRA-GGML/blob/main/llama-2-13b-guanaco-qlora.ggmlv3.q4_K_S.bin"  # 8.14G
+
+LLM = None
+
+if "forindo" in platform.node():  # deploy 70b model locally
+    # url = "https://huggingface.co/TheBloke/llama-2-70b-Guanaco-QLoRA-GGML/blob/main/llama-2-70b-guanaco-qlora.ggmlv3.q3_K_S.bin"  # 29.7G
+    # model_loc = "/home/mu2018/github/langchain-llama-2-70b-guanaco-qlora-ggml/models/llama-2-70b-guanaco-qlora.ggmlv3.q3_K_S.bin"
+    _ = """
+    url = "https://huggingface.co/TheBloke/StableBeluga2-70B-GGML/blob/main/stablebeluga2-70b.ggmlv3.q3_K_S.bin"
+    try:
+        model_loc, file_size = dl_hf_model(url)
+        logger.info(f"done load llm {model_loc=} {file_size=}G")
+    except Exception as exc_:
+        logger.error(exc_)
+        raise SystemExit(1) from exc_
+    # """
+    model_loc = "models/stablebeluga2-70b.ggmlv3.q3_K_S.bin"
+    assert Path(model_loc).exists(), f"Make sure {model_loc=} exists."
+else:
+    try:
+        logger.debug(f" dl {url}")
+        model_loc, file_size = dl_hf_model(url)
+        logger.info(f"done load llm {model_loc=} {file_size=}G")
+    except Exception as exc_:
+        logger.error(exc_)
+        raise SystemExit(1) from exc_
+
+# raise SystemExit(0)
+
+# Prompt template: Guanaco
+# {past_history}
+prompt_template = """You are a helpful assistant. Let's think step by step.
+### Human:
+{question}
+### Assistant:"""
+human_prefix = "### Human"
+ai_prefix = "### Assistant"
+stop_list = [f"{human_prefix}:"]
+
+if "beluga" in model_loc.lower():
+    prompt_template = dedent(
+        """
+    ### System:
+    You are Stable Beluga, an AI that follows instructions extremely well. Help as much as you can.
+    Let's think step by step.
+
+    ### User: {question}
+
+    ### Assistant:
+    """
+    ).lstrip()
+    human_prefix = "### User"
+    ai_prefix = "### Assistant"
+    stop_list = [f"{human_prefix}:"]
+
+_ = psutil.cpu_count(logical=False) - 1
+cpu_count: int = int(_) if _ else 1
+logger.debug(f"{cpu_count=}")
+
+
+logger.debug(f"{model_loc=}")
+LLM = AutoModelForCausalLM.from_pretrained(
+    model_loc,
+    model_type="llama",
+    threads=cpu_count,
+)
+
+os.environ["TZ"] = "Asia/Shanghai"
+try:
+    time.tzset()  # type: ignore # pylint: disable=no-member
+except Exception:
+    # Windows
+    logger.warning("Windows, cant run time.tzset()")
+
+
+@dataclass
+class GenerationConfig:
+    temperature: float = 0.7
+    top_k: int = 50
+    top_p: float = 0.9
+    repetition_penalty: float = 1.0
+    max_new_tokens: int = 512
+    seed: int = 42
+    reset: bool = False
+    stream: bool = True
+    threads: int = cpu_count
+    stop: list[str] = field(default_factory=lambda: stop_list)
+
+
+def generate(
+    question: str,
+    llm=LLM,
+    config: GenerationConfig = GenerationConfig(),
+):
+    """Run model inference, will return a Generator if streaming is true."""
+    # _ = prompt_template.format(question=question)
+    # print(_)
+
+    prompt = prompt_template.format(question=question)
+
+    return llm(
+        prompt,
+        **asdict(config),
+    )
+
+
+logger.debug(f"{asdict(GenerationConfig())=}")
+
+
+def user(user_message, history):
+    # return user_message, history + [[user_message, None]]
+    if history is None:
+        history = []
+    history.append([user_message, None])
+    return user_message, history  # keep user_message
+
+
+def user1(user_message, history):
+    # return user_message, history + [[user_message, None]]
+    if history is None:
+        history = []
+    history.append([user_message, None])
+    return "", history  # clear user_message
+
+
+def bot_(history):
+    user_message = history[-1][0]
+    resp = random.choice(["How are you?", "I love you", "I'm very hungry"])
+    bot_message = user_message + ": " + resp
+    history[-1][1] = ""
+    for character in bot_message:
+        history[-1][1] += character
+        time.sleep(0.02)
+        yield history
+
+    history[-1][1] = resp
+    yield history
+
+
+def bot(history):
+    user_message = ""
+    try:
+        user_message = history[-1][0]
+    except Exception as exc:
+        logger.error(exc)
+    response = []
+
+    logger.debug(f"{user_message=}")
+
+    with about_time() as atime:  # type: ignore
+        flag = 1
+        prefix = ""
+        then = time.time()
+
+        logger.debug("about to generate")
+
+        config = GenerationConfig(reset=True)
+        for elm in generate(user_message, config=config):
+            if flag == 1:
+                logger.debug("in the loop")
+                prefix = f"({time.time() - then:.2f}s) "
+                flag = 0
+                print(prefix, end="", flush=True)
+                logger.debug(f"{prefix=}")
+            print(elm, end="", flush=True)
+            # logger.debug(f"{elm}")
+
+            response.append(elm)
+            history[-1][1] = prefix + "".join(response)
+            yield history
+
+    _ = (
+        f"(time elapsed: {atime.duration_human}, "  # type: ignore
+        f"{atime.duration/len(''.join(response)):.2f}s/char)"  # type: ignore
+    )
+
+    history[-1][1] = "".join(response) + f"\n{_}"
+    yield history
+
+
+def predict_api(prompt):
+    logger.debug(f"{prompt=}")
+    try:
+        # user_prompt = prompt
+        config = GenerationConfig(
+            temperature=0.2,
+            top_k=10,
+            top_p=0.9,
+            repetition_penalty=1.0,
+            max_new_tokens=512,  # adjust as needed
+            seed=42,
+            reset=True,  # reset history (cache)
+            stream=False,
+            # threads=cpu_count,
+            # stop=prompt_prefix[1:2],
+        )
+
+        response = generate(
+            prompt,
+            config=config,
+        )
+
+        logger.debug(f"api: {response=}")
+    except Exception as exc:
+        logger.error(exc)
+        response = f"{exc=}"
+    # bot = {"inputs": [response]}
+    # bot = [(prompt, response)]
+
+    return response
+
+
+css = """
+    .importantButton {
+        background: linear-gradient(45deg, #7e0570,#5d1c99, #6e00ff) !important;
+        border: none !important;
+    }
+    .importantButton:hover {
+        background: linear-gradient(45deg, #ff00e0,#8500ff, #6e00ff) !important;
+        border: none !important;
+    }
+    .disclaimer {font-variant-caps: all-small-caps; font-size: xx-small;}
+    .xsmall {font-size: x-small;}
+"""
+
+logger.info("start block")
+
+with gr.Blocks(
+    title=f"{Path(model_loc).name}",
+    # theme=gr.themes.Soft(text_size="sm", spacing_size="sm"),
+    theme=gr.themes.Glass(text_size="sm", spacing_size="sm"),
+    css=css,
+) as block:
+    # buff_var = gr.State("")
+    with gr.Accordion("🎈 Info", open=False):
+        gr.Markdown(
+            f"""<h5><center>{Path(model_loc).name}</center></h4>
+            Most examples are meant for another model.
+            You probably should try to test
+            some related prompts.""",
+            elem_classes="xsmall",
+        )
+
+    # chatbot = gr.Chatbot().style(height=700)  # 500
+    chatbot = gr.Chatbot(height=500)
+
+    # buff = gr.Textbox(show_label=False, visible=True)
+
+    with gr.Row():
+        with gr.Column(scale=5):
+            msg = gr.Textbox(
+                label="Chat Message Box",
+                placeholder="Ask me anything (press Shift+Enter or click Submit to send)",
+                show_label=False,
+                # container=False,
+                lines=6,
+                max_lines=30,
+                show_copy_button=True,
+                # ).style(container=False)
+            )
+        with gr.Column(scale=1, min_width=50):
+            with gr.Row():
+                submit = gr.Button("Submit", elem_classes="xsmall")
+                stop = gr.Button("Stop", visible=True)
+                clear = gr.Button("Clear History", visible=True)
+    with gr.Row(visible=False):
+        with gr.Accordion("Advanced Options:", open=False):
+            with gr.Row():
+                with gr.Column(scale=2):
+                    system = gr.Textbox(
+                        label="System Prompt",
+                        value=prompt_template,
+                        show_label=False,
+                        container=False,
+                        # ).style(container=False)
+                    )
+                with gr.Column():
+                    with gr.Row():
+                        change = gr.Button("Change System Prompt")
+                        reset = gr.Button("Reset System Prompt")
+
+    with gr.Accordion("Example Inputs", open=True):
+        examples = gr.Examples(
+            examples=examples_list,
+            inputs=[msg],
+            examples_per_page=40,
+        )
+
+    # with gr.Row():
+    with gr.Accordion("Disclaimer", open=False):
+        _ = Path(model_loc).name
+        gr.Markdown(
+            f"Disclaimer: {_} can produce factually incorrect output, and should not be relied on to produce "
+            "factually accurate information. {_} was trained on various public datasets; while great efforts "
+            "have been taken to clean the pretraining data, it is possible that this model could generate lewd, "
+            "biased, or otherwise offensive outputs.",
+            elem_classes=["disclaimer"],
+        )
+
+    msg_submit_event = msg.submit(
+        # fn=conversation.user_turn,
+        fn=user,
+        inputs=[msg, chatbot],
+        outputs=[msg, chatbot],
+        queue=True,
+        show_progress="full",
+        # api_name=None,
+    ).then(bot, chatbot, chatbot, queue=True)
+    submit_click_event = submit.click(
+        # fn=lambda x, y: ("",) + user(x, y)[1:],  # clear msg
+        fn=user1,  # clear msg
+        inputs=[msg, chatbot],
+        outputs=[msg, chatbot],
+        queue=True,
+        # queue=False,
+        show_progress="full",
+        # api_name=None,
+    ).then(bot, chatbot, chatbot, queue=True)
+    stop.click(
+        fn=None,
+        inputs=None,
+        outputs=None,
+        cancels=[msg_submit_event, submit_click_event],
+        queue=False,
+    )
+    clear.click(lambda: None, None, chatbot, queue=False)
+
+    with gr.Accordion("For Chat/Translation API", open=False, visible=False):
+        input_text = gr.Text()
+        api_btn = gr.Button("Go", variant="primary")
+        out_text = gr.Text()
+
+    api_btn.click(
+        predict_api,
+        input_text,
+        out_text,
+        api_name="api",
+    )
+
+    # block.load(update_buff, [], buff, every=1)
+    # block.load(update_buff, [buff_var], [buff_var, buff], every=1)
+
+# concurrency_count=5, max_size=20
+# max_size=36, concurrency_count=14
+# CPU cpu_count=2 16G, model 7G
+# CPU UPGRADE cpu_count=8 32G, model 7G
+
+# does not work
+_ = """
+# _ = int(psutil.virtual_memory().total / 10**9 // file_size - 1)
+# concurrency_count = max(_, 1)
+if psutil.cpu_count(logical=False) >= 8:
+    # concurrency_count = max(int(32 / file_size) - 1, 1)
+else:
+    # concurrency_count = max(int(16 / file_size) - 1, 1)
+# """
+
+# default concurrency_count = 1
+# block.queue(concurrency_count=concurrency_count, max_size=5).launch(debug=True)
+
+server_port = 7860
+if "forindo" in platform.node():
+    server_port = 7861
+block.queue(max_size=5).launch(
+    debug=True, server_name="0.0.0.0", server_port=server_port
+)
+
+# block.queue(max_size=5).launch(debug=True, server_name="0.0.0.0")

dl_model.py

@@ -0,0 +1,73 @@
+"""Download modles."""
+# pylint: disable=invalid-name, broad-exception-caught, line-too-long
+from typing import Optional
+
+import typer
+from dl_hf_model import dl_hf_model
+from loguru import logger
+
+url = "https://huggingface.co/TheBloke/Upstage-Llama-2-70B-instruct-v2-GGML/blob/main/upstage-llama-2-70b-instruct-v2.ggmlv3.q3_K_S.bin"
+
+
+__version__ = "0.0.1"
+
+app = typer.Typer(
+    name="dl-mode",
+    add_completion=False,
+    help="donwload models from hf and save to a dir (default models)",
+)
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        typer.echo(
+            f"{app.info.name} v.{__version__} -- download models for given url(s)"
+        )
+        raise typer.Exit()
+
+
+@app.command()
+def main(
+    urls: str = typer.Argument(  # pylint: disable=unused-argument
+        "",
+        help=f"one or more urls (default {url})",
+        show_default=False,
+    ),
+    version: Optional[bool] = typer.Option(  # pylint: disable=unused-argument
+        None,
+        "--version",
+        "-v",
+        "-V",
+        help="Show version info and exit.",
+        callback=_version_callback,
+        is_eager=True,
+    ),
+    model_dir: Optional[str] = typer.Option(
+        None,
+        "--mode-dir",
+        help="dir to save downloaded models (default models)",
+    ),
+):
+    """Download a model or model given url(s)."""
+    logger.trace(f"{urls}")
+    if model_dir is None:
+        model_dir = "models"
+    if isinstance(urls, str):
+        urls.split()
+
+    url_list = urls[:]
+    if not urls:
+        url_list = [url]
+    try:
+        for elm in url_list:
+            dl_hf_model(elm)
+    except Exception as exc:
+        logger.error(exc)
+        raise typer.Exit()
+
+
+if __name__ == "__main__":
+    try:
+        app()
+    except Exception as exc_:
+        logger.error(exc_)

examples_list.py

@@ -0,0 +1,43 @@
+etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """
+examples_list = [
+    ["What NFL team won the Super Bowl in the year Justin Bieber was born?"],
+    [
+        "What NFL team won the Super Bowl in the year Justin Bieber was born? Think step by step."
+    ],
+    ["How to pick a lock? Provide detailed steps."],
+    [
+        "If it takes 10 hours to dry 10 clothes,  assuming all the clothes are hung together at the same time for drying , then how long will it take to dry a cloth?"
+    ],
+    [
+        "If it takes 10 hours to dry 10 clothes,  assuming all the clothes are hung together at the same time for drying , then how long will it take to dry a cloth? Think step by step."
+    ],
+    ["is infinity + 1 bigger than infinity?"],
+    ["Explain the plot of Cinderella in a sentence."],
+    [
+        "How long does it take to become proficient in French, and what are the best methods for retaining information?"
+    ],
+    ["What are some common mistakes to avoid when writing code?"],
+    ["Build a prompt to generate a beautiful portrait of a horse"],
+    ["Suggest four metaphors to describe the benefits of AI"],
+    ["Write a pop song about leaving home for the sandy beaches."],
+    ["Write a summary demonstrating my ability to tame lions"],
+    ["鲁迅和周树人什么关系？ 说中文。"],
+    ["鲁迅和周树人什么关系？"],
+    ["鲁迅和周树人什么关系？ 用英文回答。"],
+    ["从前有一头牛，这头牛后面有什么？"],
+    ["正无穷大加一大于正无穷大吗？"],
+    ["正无穷大加正无穷大大于正无穷大吗？"],
+    ["-2的平方根等于什么？"],
+    ["树上有5只鸟，猎人开枪打死了一只。树上还有几只鸟？"],
+    ["树上有11只鸟，猎人开枪打死了一只。树上还有几只鸟？提示：需考虑鸟可能受惊吓飞走。"],
+    ["以红楼梦的行文风格写一张委婉的请假条。不少于320字。"],
+    [f"Translate ths following to Chinese. List 2 variants: \n{etext}"],
+    [f"{etext} 翻成中文，列出3个版本。"],
+    [f"{etext} \n 翻成中文，保留原意，但使用文学性的语言。不要写解释。列出3个版本。"],
+    ["假定 1 + 2 = 4, 试求 7 + 8。"],
+    ["给出判断一个数是不是质数的 javascript 码。"],
+    ["给出实现python 里 range(10)的 javascript 码。"],
+    ["给出实现python 里 [*(range(10)]的 javascript 码。"],
+    ["Erkläre die Handlung von Cinderella in einem Satz."],
+    ["Erkläre die Handlung von Cinderella in einem Satz. Auf Deutsch."],
+]

package.json

@@ -0,0 +1,14 @@
+{
+  "name": "llama-2-70b-guanaco-qlora-ggml",
+  "version": "1.0.0",
+  "main": "index.js",
+  "repository": "https://huggingface.co/spaces/mikeee/llama-2-70b-guanaco-qlora-ggml",
+  "author": "ffreemt <[email protected]>",
+  "license": "MIT",
+  "devDependencies": {
+    "husky": "^8.0.0"
+  },
+  "scripts": {
+    "prepare": "husky install"
+  }
+}

requirements.txt

@@ -0,0 +1,8 @@
+ctransformers  # ==0.2.10 0.2.13
+transformers  # ==4.30.2
+# huggingface_hub
+gradio
+loguru
+about-time
+psutil
+dl-hf-model

run-app.sh

@@ -0,0 +1,2 @@
+export GRADIO_SERVER_NAME=0.0.0.0
+nodemon -w app.py -x python app.py

yarn.lock

@@ -0,0 +1,8 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+husky@^8.0.0:
+  version "8.0.3"
+  resolved "https://registry.npmmirror.com/husky/-/husky-8.0.3.tgz#4936d7212e46d1dea28fef29bb3a108872cd9184"
+  integrity sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg==