import os import re import nltk import json import copy import time import validators import numpy as np import streamlit as st from typing import List from numpy import ndarray from llama_cpp import Llama from splade_encoder import SpladeEncoder from optimum_encoder import OptimumEncoder from unstructured.partition.auto import partition from unstructured.nlp.tokenize import download_nltk_packages from langchain_experimental.text_splitter import SemanticChunker from langchain_community.document_loaders import WikipediaLoader, WebBaseLoader from pymilvus import ( MilvusClient, Collection, CollectionSchema, FieldSchema, DataType, AnnSearchRequest, RRFRanker ) def transform_query(query: str) -> str: """ For retrieval, add the prompt for query (not for documents). """ return f'Represent this sentence for searching relevant passages: {query}' def query_hybrid_search(query: str, col: Collection, dense_model: OptimumEncoder, sparse_model: SpladeEncoder): dense_embeddings = dense_model.embed_query(transform_query(query), convert_to_numpy=True)[0] sparse_embeddings = sparse_model([query], 1) sparse_req = AnnSearchRequest( query_embeddings["sparse"], "sparse_vector", {"metric_type": "IP"}, limit=10 ) dense_req = AnnSearchRequest( query_embeddings["dense"], "dense_vector", {"metric_type": "COSINE"}, limit=10 ) return col.hybrid_search( [sparse_req, dense_req], rerank=RRFRanker(), limit=3, output_fields=['text', 'metadata'] ) def main(query: str, collection: Collection, llm: Llama, dense_model: OptimumEncoder, sparse_model: SpladeEncoder): collection.load() search_results = query_hybrid_search(query, collection, dense_model, sparse_model) print(search_results[0]) contents, metadatas = [], [] for search_result in search_results: contents.append(search_result) context = "\n".join(contents) seen_values = set() result_metadatas = "\n\n".join( f'{value}' for metadata in metadatas for key, value in metadata.items() if (value not in seen_values and not seen_values.add(value)) ) print(f'QA_PROMPT : {st.session_state.qa_prompt}') response = llm.create_chat_completion( messages = [ {"role": "system", "content": "You are a helpful assistant."}, { "role": "user", "content": st.session_state.qa_prompt(query, context) } ], temperature=0, frequency_penalty=0.2, presence_penalty=0.4, top_p=0.2) text = response["choices"][0]["message"]['content'] print(f'TEXT: {text}') output = llm.create_chat_completion( messages = [ {"role": "system", "content": """Act like a professional summary writer. You have been providing summarization services for various types of documents, including academic papers, legal texts, and business reports, for over 20 years. Your expertise includes extracting key points and important details concisely without adding unnecessary introductory phrases.""" }, { "role": "user", "content": f"""Write a summary of the following text delimited by triple backquotes. Ensure the summary covers the key points of the text. Do not introduce the summary with sentences like "Here is the summary:" or similar. The summary should be detailed, precise, and directly convey the essential information from the text. ```{text}``` Take a deep breath and work on this problem step-by-step.""" } ], temperature=0.7, max_tokens=3000) answer = output['choices'][0]['message']['content'] answer_with_metadatas = f"{answer}\n\n\nSource(s) :\n\n{result_metadatas}" print(f'OUTPUT: {output}') if st.session_state.documents_only: return answer if 'no_answer' in text else answer_with_metadatas else: return f'Internal Knowledge :\n\n{answer}' if 'knowledge_topic' in text else f'Documents Based :\n\n{answer_with_metadatas}' @st.cache_resource def load_models_and_documents(): with st.spinner('Load models...'): llm = Llama.from_pretrained( repo_id="MaziyarPanahi/Llama-3-8B-Instruct-32k-v0.1-GGUF", filename="*Q8_0.gguf", verbose=True, chat_format="llama-3", n_ctx=16384, n_gpu_layers=33, flash_attn=True ) dense_model = OptimumEncoder( device="cuda", cache_dir=os.getenv('HF_HOME') ) sparse_model = SpladeEncoder( model_name='devve1/Splade_PP_en_v2_onnx', device='cuda' ) download_nltk_packages() path_db = os.path.join(os.getenv('HF_HOME'), 'database_demo.db') if not os.path.exists(path_db): download_nltk_packages() MilvusClient(path_db) collection_name = 'collection_demo' fields = [ FieldSchema(name="pk", dtype=DataType.VARCHAR, is_primary=True, auto_id=True, max_length=100), FieldSchema(name="text", dtype=DataType.VARCHAR, max_length=512), FieldSchema(name="metadata", dtype=DataType.JSON), FieldSchema(name="sparse_vector", dtype=DataType.SPARSE_FLOAT_VECTOR), FieldSchema(name="dense_vector", dtype=DataType.FLOAT_VECTOR, dim=1024) ] schema = CollectionSchema(fields, '') col = Collection(collection_name, schema, consistency_level="Strong") sparse_index = {"index_type": "SPARSE_INVERTED_INDEX", "metric_type": "IP"} dense_index = {"index_type": "HNSW", "metric_type": "COSINE"} col.create_index("sparse_vector", sparse_index) col.create_index("dense_vector", dense_index) col.alter_index( index_name=("dense_vector"), extra_params={"mmap.enabled": True} ) with st.spinner('Parse and chunk documents...'): docs_1 = WikipediaLoader(query='Action-RPG').load() docs_2 = WikipediaLoader(query='Real-time strategy').load() docs_3 = WikipediaLoader(query='First-person shooter').load() docs_4 = WikipediaLoader(query='Multiplayer online battle arena').load() docs_5 = WikipediaLoader(query='List of video game genres').load() docs = docs_1 + docs_2 + docs_3 + docs_4 + docs_5 texts, metadatas = [], [] for doc in docs: texts.append(doc.page_content) del doc.metadata['title'] del doc.metadata['summary'] metadatas.append(doc.metadata) docs_texts, docs_metadatas, dense_embeddings, sparse_embeddings = chunk_documents(texts, metadatas, dense_model, sparse_model) # with open(texts_path, "wb") as outfile_texts: # packed_texts = msgpack.packb(docs_texts, use_bin_type=True) # outfile_texts.write(packed_texts) # with open(metadatas_path, "wb") as outfile_metadatas: # packed_metadatas = msgpack.packb(docs_metadatas, use_bin_type=True) # outfile_metadatas.write(packed_metadatas) # np.savez_compressed(dense_path, *dense_embeddings) # sp.save_npz(sparse_path, sparse_embeddings) entities = [ docs_texts, docs_metadatas, sparse_embeddings, dense_embeddings ] col.insert(entities) col.flush() # else: # with open(texts_path, "rb") as data_file_texts: # byte_data_texts = data_file_texts.read() # with open(metadatas_path, "rb") as data_file_metadatas: # byte_data_metadatas = data_file_metadatas.read() # docs_texts = msgpack.unpackb(byte_data_texts, raw=False) # docs_metadatas = msgpack.unpackb(byte_data_metadatas, raw=False) # dense_embeddings = list(np.load(dense_path, mmap_mode='r').values()) # sparse_embeddings = load_npz(sparse_path) return col, llm, dense_model, sparse_model def chunk_documents(texts: List[str], metadatas: List[dict], dense_model: OptimumEncoder, sparse_model: SpladeEncoder): import time text_splitter = SemanticChunker( dense_model, breakpoint_threshold_type='standard_deviation' ) docs = text_splitter.create_documents(texts, metadatas) documents, metadatas_docs = [], [] for doc in docs: documents.append(doc.page_content) metadatas_docs.append(json.dumps(doc.metadata)) start_dense = time.time() dense_embeddings = dense_model.embed_documents(documents, convert_to_numpy=True) end_dense = time.time() final_dense = end_dense - start_dense print(f'DENSE TIME: {final_dense}') start_sparse = time.time() sparse_embeddings = sparse_model(documents) end_sparse = time.time() final_sparse = end_sparse - start_sparse print(f'SPARSE TIME: {final_sparse}') return documents, metadatas_docs, dense_embeddings, sparse_embeddings def on_change_documents_only(): st.session_state.qa_prompt = lambda query, context: ( f"""You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, say that you dont' know and reply with 'no_answer'. If you don't know when you are not sure, ask for clarification and reply with 'no_answer'. Avoid mentioning that you obtained the information from the context. Use three sentences maximum and keep the answer concise. Question: {query} Context: {context} Answer:""" if st.session_state.documents_only else f"""If the context is not relevant, please answer the question by using your own knowledge about the topic. If you decide to provide information using your own knowledge or general knowledge, write 'knowledge_topic' at the top of your answer {context} Question: {query}""" ) st.session_state.tooltip = 'The AI answer your questions only considering the documents provided' if st.session_state.documents_only else """The AI answer your questions considering the documents provided, and if it doesn't found the answer in them, try to find in its own internal knowledge""" if __name__ == '__main__': st.set_page_config(page_title="Multipurpose AI Agent",layout="wide") st.title("Multipurpose AI Agent") if "qa_prompt" not in st.session_state: st.session_state.qa_prompt = lambda query, context: ( f"""You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, reply with 'no_answer'. Use three sentences maximum and keep the answer concise. Question: {query} Context: {context} Answer:""" ) if "tooltip" not in st.session_state: st.session_state.tooltip = 'The AI answer your questions only considering the documents provided' collection, llm, dense_model, sparse_model = load_models_and_documents() if "messages" not in st.session_state: st.session_state.messages = [] for message in st.session_state.messages: with st.chat_message(message["role"]): st.markdown(message["content"]) if prompt := st.chat_input("Message Video Game Assistant"): st.chat_message("user").markdown(prompt) st.session_state.messages.append({"role": "user", "content": prompt}) ai_response = main(prompt, collection, llm, dense_model, sparse_model) with st.chat_message("assistant"): message_placeholder = st.empty() full_response = "" for chunk in re.split(r'(\s+)', ai_response): full_response += chunk + " " time.sleep(0.01) message_placeholder.write(full_response + "▌") message_placeholder.write(full_response) st.session_state.messages.append({"role": "assistant", "content": full_response}) url = st.sidebar.text_input("Scrape an URL link :") if validators.url(url): docs = WebBaseLoader(url).load() print(f'WebBaseLoader: {docs[0].metadata}') texts, metadatas = [], [] for doc in docs: texts.append(doc.page_content) del doc.metadata['title'] del doc.metadata['description'] del doc.metadata['language'] metadatas.append(doc.metadata) texts, metadatas, dense_embeddings, sparse_embeddings = chunk_documents(texts, metadatas, dense_model, sparse_model) entities = [ texts, metadatas, sparse_embeddings, dense_embeddings ] col.insert(entities) st.sidebar.success("URL content uploaded and ready!") uploaded_files = st.sidebar.file_uploader("Upload a file :", accept_multiple_files=True, type=['docx', 'doc', 'pptx', 'ppt', 'xlsx', 'txt']) print(f'uploaded-files : {uploaded_files}') for uploaded_file in uploaded_files: print('count') elements = partition(file=uploaded_file, strategy='fast', skip_infer_table_types=['png', 'pdf', 'jpg', 'xls', 'xlsx', 'heic'], include_page_breaks=True ) metadata_dict = {"source": uploaded_file.name} texts, metadatas = [], [] for elem in elements: texts.append(elem.text) metadatas.append(metadata_dict) texts, metadatas, dense_embeddings, sparse_embeddings = chunk_documents(texts, metadatas, dense_model, sparse_model) entities = [ texts, metadatas, sparse_embeddings, dense_embeddings ] col.insert(entities) st.sidebar.success("Document content uploaded and ready!") tooltip = 'The AI answer your questions only considering the documents provided' st.sidebar.toggle( label="""Enable 'Documents-Only' Mode""", value=True, on_change=on_change_documents_only, key="documents_only", help=st.session_state.tooltip )