app3.py

import os
from flask import Flask, render_template, redirect, url_for, request, flash
from flask_sqlalchemy import SQLAlchemy
from flask_login import LoginManager, UserMixin, login_user, login_required, logout_user, current_user
from werkzeug.security import generate_password_hash, check_password_hash
from transformers import AutoTokenizer
from optimum.intel.openvino import OVModelForCausalLM
import gradio as gr
from threading import Thread
from time import perf_counter
from typing import List
import numpy as np

app = Flask(__name__)
app.config['SECRET_KEY'] = 'your_secret_key'
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///users.db'
db = SQLAlchemy(app)
login_manager = LoginManager()
login_manager.init_app(app)
login_manager.login_view = 'login'

class User(db.Model, UserMixin):
    id = db.Column(db.Integer, primary_key=True)
    username = db.Column(db.String(80), unique=True, nullable=False)
    password = db.Column(db.String(120), nullable=False)

    def __repr__(self):
        return '<User %r>' % self.username

# Create the database tables
with app.app_context():
    db.create_all()

@login_manager.user_loader
def load_user(user_id):
    return User.query.get(int(user_id))

@app.route('/', methods=['GET', 'POST'])
def signup():
    if request.method == 'POST':
        username = request.form['username']
        password = request.form['password']
        hashed_password = generate_password_hash(password, method='pbkdf2:sha256')

        new_user = User(username=username, password=hashed_password)
        db.session.add(new_user)
        db.session.commit()
        flash('Signup successful!', 'success')
        return redirect(url_for('login'))

    return render_template('signup.html')

@app.route('/login', methods=['GET', 'POST'])
def login():
    if request.method == 'POST':
        username = request.form['username']
        password = request.form['password']
        user = User.query.filter_by(username=username).first()
        if user and check_password_hash(user.password, password):
            login_user(user)
            return redirect(url_for('dashboard'))
        flash('Invalid username or password', 'danger')

    return render_template('login.html')

@app.route('/dashboard')
@login_required
def dashboard():
    return render_template('dashboard.html', name=current_user.username)

@app.route('/logout')
@login_required
def logout():
    logout_user()
    return redirect(url_for('login'))

# Gradio app integration
model_dir = "C:/Users/KIIT/OneDrive/Desktop/INTEL/phi-2/INT8_compressed_weights"
model_name = "susnato/phi-2"
ov_config = {"PERFORMANCE_HINT": "LATENCY", "NUM_STREAMS": "1", "CACHE_DIR": ""}
tokenizer = AutoTokenizer.from_pretrained(model_name)
ov_model = OVModelForCausalLM.from_pretrained(model_dir, device="CPU", ov_config=ov_config)

prompt_template = "{instruction}"
end_key_token_id = tokenizer.eos_token_id
pad_token_id = tokenizer.pad_token_id

def estimate_latency(current_time, current_perf_text, new_gen_text, per_token_time, num_tokens):
    num_current_toks = len(tokenizer.encode(new_gen_text))
    num_tokens += num_current_toks
    per_token_time.append(num_current_toks / current_time)
    if len(per_token_time) > 10 and len(per_token_time) % 4 == 0:
        current_bucket = per_token_time[:-10]
        return f"Average generation speed: {np.mean(current_bucket):.2f} tokens/s. Total generated tokens: {num_tokens}", num_tokens
    return current_perf_text, num_tokens

def run_generation(user_text, top_p, temperature, top_k, max_new_tokens, perf_text):
    prompt_text = prompt_template.format(instruction=user_text)
    model_inputs = tokenizer(prompt_text, return_tensors="pt")

    streamer = gr.utils.TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
    generate_kwargs = dict(
        model_inputs,
        streamer=streamer,
        max_new_tokens=max_new_tokens,
        do_sample=True,
        top_p=top_p,
        temperature=temperature,
        top_k=top_k,
        eos_token_id=end_key_token_id,
        pad_token_id=pad_token_id,
    )
    t = Thread(target=ov_model.generate, kwargs=generate_kwargs)
    t.start()

    model_output = ""
    per_token_time = []
    num_tokens = 0
    start = perf_counter()
    for new_text in streamer:
        current_time = perf_counter() - start
        model_output += new_text
        perf_text, num_tokens = estimate_latency(current_time, perf_text, new_text, per_token_time, num_tokens)
        yield model_output, perf_text
        start = perf_counter()
    return model_output, perf_text

def reset_textbox(instruction, response, perf):
    return "", "", ""

examples = [
    "Give me a recipe for pizza with pineapple",
    "Write me a tweet about the new OpenVINO release",
    "Explain the difference between CPU and GPU",
    "Give five ideas for a great weekend with family",
    "Do Androids dream of Electric sheep?",
    "Who is Dolly?",
    "Please give me advice on how to write resume?",
    "Name 3 advantages to being a cat",
    "Write instructions on how to become a good AI engineer",
    "Write a love letter to my best friend",
]

@app.route('/gradio')
@login_required
def gradio():
    with gr.Blocks() as demo:
        gr.Markdown("# Question Answering with Model and OpenVINO.\nProvide instruction which describes a task below or select among predefined examples and model writes response that performs requested task.")
        with gr.Row():
            with gr.Column(scale=4):
                user_text = gr.Textbox(placeholder="Write an email about an alpaca that likes flan", label="User instruction")
                model_output = gr.Textbox(label="Model response", interactive=False)
                performance = gr.Textbox(label="Performance", lines=1, interactive=False)
                with gr.Column(scale=1):
                    button_clear = gr.Button(value="Clear")
                    button_submit = gr.Button(value="Submit")
                gr.Examples(examples, user_text)
            with gr.Column(scale=1):
                max_new_tokens = gr.Slider(minimum=1, maximum=1000, value=256, step=1, interactive=True, label="Max New Tokens")
                top_p = gr.Slider(minimum=0.05, maximum=1.0, value=0.92, step=0.05, interactive=True, label="Top-p (nucleus sampling)")
                top_k = gr.Slider(minimum=0, maximum=50, value=0, step=1, interactive=True, label="Top-k")
                temperature = gr.Slider(minimum=0.1, maximum=5.0, value=0.8, step=0.1, interactive=True, label="Temperature")

        user_text.submit(run_generation, [user_text, top_p, temperature, top_k, max_new_tokens, performance], [model_output, performance])
        button_submit.click(run_generation, [user_text, top_p, temperature, top_k, max_new_tokens, performance], [model_output, performance])
        button_clear.click(reset_textbox, [user_text, model_output, performance], [user_text, model_output, performance])

    return demo.launch(share=True)

if __name__ == '__main__':
    app.run(debug=True)