app.py

import os
import re
import nltk
import copy
import time
import msgpack
import joblib
import validators
import numpy as np
import streamlit as st
from typing import List
from numpy import ndarray
from qdrant_client import QdrantClient, models
from llama_cpp import Llama, GGML_TYPE_I8
from optimum_encoder import OptimumEncoder
from unstructured.partition.auto import partition
from statistical_chunker import StatisticalChunker
from fastembed import SparseEmbedding, SparseTextEmbedding
from unstructured.nlp.tokenize import download_nltk_packages
from scipy.sparse import csr_matrix, save_npz, load_npz, vstack
from langchain_community.document_loaders import WikipediaLoader, WebBaseLoader
from qdrant_client.models import (
    NamedSparseVector,
    NamedVector,
    SparseVector,
    PointStruct,
    ScoredPoint,
    Prefetch,
    FusionQuery,
    Fusion
)

def make_points(texts: List[str], metadatas: List[dict], dense: List[ndarray], sparse: List[SparseEmbedding])-> List[PointStruct]:
    points = []
    for idx, (text, metadata, sparse_vector, dense_vector) in enumerate(zip(texts, metadatas, sparse, dense)):
        sparse_vec = SparseVector(indices=sparse_vector.indices.tolist(), values=sparse_vector.values.tolist())
        point = PointStruct(
            id=idx,
            vector={
                "text-sparse": sparse_vec,
                "text-dense": dense_vector,
            },
            payload={
                "text": text,
                "metadata": metadata
            }
        )
        points.append(point)
    return points

def transform_query(query: str) -> str:
    """ For retrieval, add the prompt for query (not for documents).
    """
    return f'Represent this sentence for searching relevant passages: {query}'

def query_hybrid_search(query: str, client: QdrantClient, collection_name: str, dense_model: OptimumEncoder, sparse_model: SparseTextEmbedding):
    dense_embeddings = dense_model([transform_query(query)], 1, convert_to_numpy=True)[0]
    sparse_embeddings = list(sparse_model.query_embed(query))[0]
    
    return client.query_points(
        collection_name=collection_name,
        prefetch=[
            Prefetch(query=sparse_embeddings.as_object(), using="text-sparse", limit=10),
            Prefetch(query=dense_embeddings.tolist(), using="text-dense", limit=10)
        ],
        query=FusionQuery(fusion=Fusion.RRF),
        limit=3
    )

def main(query: str, client: QdrantClient, collection_name: str, llm: Llama, dense_model: OptimumEncoder, sparse_model: SparseTextEmbedding):
    scored_points = query_hybrid_search(query, client, collection_name, dense_model, sparse_model).points

    docs = [(scored_point.payload['text'], scored_point.payload['metadata']) for scored_point in scored_points]
    contents, metadatas = [list(t) for t in zip(*docs)]

    context = "\n".join(contents)
    
    seen_values = set()
    result_metadatas = "\n\n".join(
        f'{value}'
        for metadata in metadatas
        for key, value in metadata.items()
        if (value not in seen_values and not seen_values.add(value))
    )
    print(f'QA_PROMPT : {st.session_state.qa_prompt}')
    response = llm.create_chat_completion(
        messages = [
          {"role": "system", "content": "You are a helpful assistant."},
          {
              "role": "user",
              "content": st.session_state.qa_prompt(query, context)
          }
      ], temperature=0, frequency_penalty=0.2, presence_penalty=0.4, top_p=0.2)

    text = response["choices"][0]["message"]['content']
    print(f'TEXT: {text}')

    output = llm.create_chat_completion(
        messages = [
          {"role": "system", "content": """Act like a professional summary writer. You have been providing summarization services for various types of documents, including academic papers, legal texts, and business reports, for over 20 years. 
          Your expertise includes extracting key points and important details concisely without adding unnecessary introductory phrases."""
          },
          {
              "role": "user",
              "content": f"""Write a summary of the following text delimited by triple backquotes. Ensure the summary covers the key points of the text. Do not introduce the summary with sentences like "Here is the summary:" or similar. The summary should be detailed, precise, and directly convey the essential information from the text.
              
              ```{text}```
              
              Take a deep breath and work on this problem step-by-step."""
          }
      ], temperature=0.7, max_tokens=3000)
    
    answer = output['choices'][0]['message']['content']
    answer_with_metadatas = f"{answer}\n\n\nSource(s) :\n\n{result_metadatas}"
    
    print(f'OUTPUT: {output}')

    if st.session_state.documents_only:
        return answer if 'no_answer' in text else answer_with_metadatas
    else:
        return f'Internal Knowledge :\n\n{answer}' if 'knowledge_topic' in text else f'Documents Based :\n\n{answer_with_metadatas}'

@st.cache_resource
def load_models_and_documents():
    with st.spinner('Load models...'):
        llm = Llama.from_pretrained(
            repo_id="MaziyarPanahi/Llama-3-8B-Instruct-32k-v0.1-GGUF",
            filename="*Q8_0.gguf",
            verbose=True,
            chat_format="llama-3",
            n_ctx=16384,
            n_gpu_layers=32,
            flash_attn=True,
            type_k=GGML_TYPE_I8,
            type_v=GGML_TYPE_I8
        )

        dense_model = OptimumEncoder(
            device="cuda",
            cache_dir=os.getenv('HF_HOME')
        )
        
        sparse_model = SparseTextEmbedding(
            'Qdrant/bm42-all-minilm-l6-v2-attentions',
            cache_dir=os.getenv('HF_HOME'),
            providers=['CPUExecutionProvider']
        )

        download_nltk_packages()
        
    client = QdrantClient(':memory:')
    collection_name = 'collection_demo'
    
    client.create_collection(
        collection_name,
        {
            "text-dense": models.VectorParams(
                size=1024,
                distance=models.Distance.COSINE,
                on_disk=False
            )
        },
        {
            "text-sparse": models.SparseVectorParams(
                index=models.SparseIndexParams(
                    on_disk=False
                ),
                modifier=models.Modifier.IDF
            )
        },
        2,
        optimizers_config=models.OptimizersConfigDiff(
            indexing_threshold=0,
            default_segment_number=4
        ),
        hnsw_config=models.HnswConfigDiff(
            on_disk=False, 
            m=64,
            ef_construct=512
        )
    )
        
    with st.spinner('Parse and chunk documents...'):
        name = 'action_rpg'
        embeddings_path = os.path.join(os.getenv('HF_HOME'), 'embeddings')

        texts_path = os.path.join(embeddings_path, name + '_texts.msgpack')
        metadatas_path = os.path.join(embeddings_path, name + '_metadatas.msgpack')
        dense_path = os.path.join(embeddings_path, name + '_dense.npz')
        sparse_path = os.path.join(embeddings_path, name + '_sparse.npz')

        if not os.path.exists(embeddings_path):
            os.mkdir(embeddings_path)

            docs_1 = WikipediaLoader(query='Action-RPG').load()
            docs_2 = WikipediaLoader(query='Real-time strategy').load()
            docs_3 = WikipediaLoader(query='First-person shooter').load()
            docs_4 = WikipediaLoader(query='Multiplayer online battle arena').load()
            docs_5 = WikipediaLoader(query='List of video game genres').load()
            docs = docs_1 + docs_2 + docs_3 + docs_4 + docs_5

            texts, metadatas = [], []
            for doc in docs:
                texts.append(doc.page_content)
                del doc.metadata['title']
                del doc.metadata['summary']
                metadatas.append(doc.metadata)
            
            docs_texts, docs_metadatas, dense_embeddings, sparse_embeddings = chunk_documents(texts, metadatas, dense_model, sparse_model)

            with open(texts_path, "wb") as outfile_texts:
                packed_texts = msgpack.packb(docs_texts, use_bin_type=True)
                outfile_texts.write(packed_texts)

            with open(metadatas_path, "wb") as outfile_metadatas:
                packed_metadatas = msgpack.packb(docs_metadatas, use_bin_type=True)
                outfile_metadatas.write(packed_metadatas)
            
            np.savez_compressed(dense_path, *dense_embeddings)
            
            max_index = 0
            for embedding in sparse_embeddings:
                if embedding.indices.size > 0:
                    max_index = max(max_index, np.max(embedding.indices))
        
            sparse_matrices = []
            for embedding in sparse_embeddings:
                data = embedding.values
                indices = embedding.indices
                indptr = np.array([0, len(data)])
                matrix = csr_matrix((data, indices, indptr), shape=(1, max_index + 1))
                sparse_matrices.append(matrix)
            
            combined_sparse_matrix = vstack(sparse_matrices)
            save_npz(sparse_path, combined_sparse_matrix)
        else:
            with open(texts_path, "rb") as data_file_texts:
                byte_data_texts = data_file_texts.read()
                
            with open(metadatas_path, "rb") as data_file_metadatas:
                byte_data_metadatas = data_file_metadatas.read()
                
            docs_texts = msgpack.unpackb(byte_data_texts, raw=False)
            docs_metadatas = msgpack.unpackb(byte_data_metadatas, raw=False)

            dense_embeddings = list(np.load(dense_path).values())

            sparse_embeddings = []
            loaded_sparse_matrix = load_npz(sparse_path)
        
            for i in range(loaded_sparse_matrix.shape[0]):
                row = loaded_sparse_matrix.getrow(i)
                values = row.data
                indices = row.indices
                embedding = SparseEmbedding(values, indices)
                sparse_embeddings.append(embedding)

    with st.spinner('Save documents...'):
        client.upsert(
            collection_name,
            make_points(
                docs_texts, 
                docs_metadatas,
                dense_embeddings, 
                sparse_embeddings
            )
        )
    client.update_collection(
        collection_name=collection_name,
        optimizer_config=models.OptimizersConfigDiff(indexing_threshold=20000)
    )

    return client, collection_name, llm, dense_model, sparse_model

def chunk_documents(texts: List[str], metadatas: List[dict], dense_model: OptimumEncoder, sparse_model: SparseTextEmbedding):
    import time
    text_splitter = StatisticalChunker(
        dense_model
    )
    start = time.time()
    chunks = text_splitter(docs=texts, metadatas=metadatas)
    end = time.time()
    final = end - start
    print(f'FINAL CHUNKING TIME: {final}')

    documents_and_metadatas = [(chunk.content, chunk.metadata) for sub_chunk in chunks for chunk in sub_chunk]
    documents, metadatas_docs = [list(t) for t in zip(*documents_and_metadatas)]
    print(f'CHUNKS : {documents}')

    start_dense = time.time()
    dense_embeddings = dense_model(documents, 32, convert_to_numpy=True)
    end_dense = time.time()
    final_dense = end_dense - start_dense
    print(f'DENSE TIME: {final_dense}')

    start_sparse = time.time()
    sparse_embeddings = list(sparse_model.embed(documents, 32, 0))
    end_sparse = time.time()
    final_sparse = end_sparse - start_sparse
    print(f'SPARSE TIME: {final_sparse}')
    
    return documents, metadatas_docs, dense_embeddings, sparse_embeddings

def on_change_documents_only():
    st.session_state.qa_prompt = lambda query, context: (
        f"""You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, reply with 'no_answer'. Use three sentences maximum and keep the answer concise.
        
        Question: {query} 
              
        Context: {context} 
              
        Answer:"""
        
        if st.session_state.documents_only else 
        f"""If the context is not relevant, please answer the question by using your own knowledge about the topic.
        If you decide to provide information using your own knowledge or general knowledge, write 'knowledge_topic' at the top of your answer
        
        {context}

        Question: {query}"""
    )
    
    st.session_state.tooltip = 'The AI answer your questions only considering the documents provided' if st.session_state.documents_only else """The AI answer your questions considering the documents provided, and if it doesn't found the answer in them, try to find in its own internal knowledge"""
    
    
if __name__ == '__main__':
    st.set_page_config(page_title="Multipurpose AI Agent",layout="wide")
    st.title("Multipurpose AI Agent")

    if "qa_prompt" not in st.session_state:
        st.session_state.qa_prompt = lambda query, context: (
            f"""You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, reply with 'no_answer'. Use three sentences maximum and keep the answer concise.
            
            Question: {query} 
              
            Context: {context} 
              
            Answer:"""
        )

    if "tooltip" not in st.session_state:
        st.session_state.tooltip = 'The AI answer your questions only considering the documents provided'
    
    client, collection_name, llm, dense_model, sparse_model = load_models_and_documents()

    if "messages" not in st.session_state:
        st.session_state.messages = []

    for message in st.session_state.messages:
        with st.chat_message(message["role"]):
            st.markdown(message["content"])

    if prompt := st.chat_input("Message Video Game Assistant"):
        st.chat_message("user").markdown(prompt)
        st.session_state.messages.append({"role": "user", "content": prompt})

        ai_response = main(prompt, client, collection_name, llm, dense_model, sparse_model)
        with st.chat_message("assistant"):
            message_placeholder = st.empty()
            full_response = ""
            for chunk in re.split(r'(\s+)', ai_response):
                full_response += chunk + " "
                time.sleep(0.01)
                message_placeholder.write(full_response + "▌")
            message_placeholder.write(full_response)
        st.session_state.messages.append({"role": "assistant", "content": full_response})

    url = st.sidebar.text_input("Scrape an URL link :")
    if validators.url(url):
        docs = WebBaseLoader(url).load()
        print(f'WebBaseLoader: {docs[0].metadata}')
        
        texts, metadatas = [], []
        for doc in docs:
            texts.append(doc.page_content)
            del doc.metadata['title']
            del doc.metadata['description']
            del doc.metadata['language']
            metadatas.append(doc.metadata)

        texts, metadatas, dense_embeddings, sparse_embeddings = chunk_documents(texts, metadatas, dense_model, sparse_model)
        
        client.upsert(
            collection_name,
            make_points(
                texts,
                metadatas,
                dense_embeddings, 
                sparse_embeddings
            )
        )
        st.sidebar.success("URL content uploaded and ready!")

    uploaded_files = st.sidebar.file_uploader("Upload a file :", accept_multiple_files=True, type=['docx', 'doc', 'odt', 'pptx', 'ppt', 'xlsx', 'csv', 'tsv', 'eml', 'msg', 'rtf', 'epub', 'html', 'xml', 'pdf', 'png', 'jpg', 'heic','txt'])
    print(f'uploaded-files : {uploaded_files}')
    for uploaded_file in uploaded_files:
        print('count')
        elements = partition(file=uploaded_file,
                             strategy='hi_res',
                             skip_infer_table_types=['png', 'pdf', 'jpg', 'xls', 'xlsx', 'heic'],
                             hi_res_model_name='yolox',
                             include_page_breaks=True
                            )
        metadata_dict = {"source": uploaded_file.name}
        
        texts, metadatas = [], []
        for elem in elements:
            texts.append(elem.text)
            metadatas.append(metadata_dict)

        texts, metadatas, dense_embeddings, sparse_embeddings = chunk_documents(texts, metadatas, dense_model, sparse_model)
        
        client.upsert(
            collection_name,
            make_points(
                texts,
                metadatas,
                dense_embeddings, 
                sparse_embeddings
            )
        )
        st.sidebar.success("Document content uploaded and ready!")

    tooltip = 'The AI answer your questions only considering the documents provided'
    
    st.sidebar.toggle(
        label="""Enable 'Documents-Only' Mode""", 
        value=True, 
        on_change=on_change_documents_only, 
        key="documents_only",
        help=st.session_state.tooltip
    )