spaces

.gitattributes

@@ -32,3 +32,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/dnd_spell.pdf filter=lfs diff=lfs merge=lfs -text
+db_spells/index/index_abb61b80-18ff-4301-8b99-627ee62ef944.bin filter=lfs diff=lfs merge=lfs -text
+db_spells/chroma-collections.parquet filter=lfs diff=lfs merge=lfs -text
+db_spells/chroma-embeddings.parquet filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Harrison Chase
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,13 +1,53 @@
 ---
-title: Dnd Qa
-emoji: 🚀
-colorFrom: red
-colorTo: indigo
+title: DnD QA Bot
+emoji: 🗡️
+colorFrom: purple
+colorTo: yellow
 sdk: streamlit
 sdk_version: 1.19.0
-app_file: app.py
-pinned: false
+app_file: main.py
 license: agpl-3.0
+python: 3.10.9
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🗡️D&D Spell QA Bot🗡️
+
+This is a chatbot that can answer questions about **Dungeon and Dragons spells** based on this [database](https://www.aidedd.org/dnd-filters/spells-5e.php) and built with LangChain and OpenAI API. Usefull to find informations quickly instead of browsing through 50 pages of PDF.
+The creator of this bot is **[Corentin Meyer (@corentinm_py)](https://twitter.com/corentinm_py)**.  
+💪 This bot it based on Notion Question-Answering demo from [LangChain](https://github.com/hwchase17/langchain)
+
+# 🌲 Environment Setup
+
+In order to set your environment up to run the code here, first install all requirements and then launch streamlit app:
+
+```shell
+python -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+streamlit run main.py
+```
+
+Then set your OpenAI API key (if you don't have one, get one [here](https://beta.openai.com/playground))
+
+```shell
+export OPENAI_API_KEY=....
+```
+
+## 🚀 Code to deploy on StreamLit
+
+The code to run the StreamLit app is in `main.py`.
+Note that when setting up your StreamLit app you should make sure to add `OPENAI_API_KEY` as a secret environment variable.
+
+## 🧑 Reproduce the embedding and stuff
+
+Run the following command to ingest the data.
+
+```shell
+python ingest.py
+```
+
+Boom! Now you're done, and you can ask it questions like:
+
+```shell
+python qa.py "What's the size of tsunami spell ?"
+```

awk.sh

@@ -0,0 +1 @@
+awk '{B[NR%3]=$0} NR>2{ print B[(NR+1)%3]} /^level/ {print ""}  END {print B[(NR+2)%3]; print B[(NR+3)%3]}' dnd_spell.txt > dnd_spell_split.txt

data/dnd_spell.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b89814c7fe82a5b318fbf0aace794533ed9a4501e28dec72bb1eccd113b5027
+size 2307284

db_spells/chroma-collections.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:085aa69d06c9189b9a861beb67583680dc3ada6e910077896432aed1739558c5
+size 557

db_spells/chroma-embeddings.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a15ac51b1f91a063e3e4b3ddfd8e22d4fd0df42d527e5b8459f26d0edc486de6
+size 3148551

db_spells/index/id_to_uuid_abb61b80-18ff-4301-8b99-627ee62ef944.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfc6dbe52b6e9d5c12281a1d253f9950a4dbaf6cba9d5b421ea0608e1d3817b3
+size 10498

db_spells/index/index_abb61b80-18ff-4301-8b99-627ee62ef944.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16ed23c2bb32432023b2665ee2f0eb9b812f24959266df681a128a5aec1679ec
+size 2076836

db_spells/index/index_metadata_abb61b80-18ff-4301-8b99-627ee62ef944.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47a774357132affe4f362595e475f07233eed4702f998ad5cd1e3b67455dbaa4
+size 74

db_spells/index/uuid_to_id_abb61b80-18ff-4301-8b99-627ee62ef944.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08c067aeda07cadf4c22b364e1b009e6352806af610923970c00c9bc227b5a9b
+size 12300

ingest.py

@@ -0,0 +1,35 @@
+#%%
+from langchain.embeddings.openai import OpenAIEmbeddings
+
+from langchain.vectorstores import Chroma
+from langchain.text_splitter import CharacterTextSplitter
+
+persist_directory = "db_spells"
+with open("data/dnd_spell_split.txt") as f:
+    dnd_spell = f.read()
+text_splitter = CharacterTextSplitter(
+    separator="\n\n",
+    chunk_size=1000,
+    chunk_overlap=0,
+    length_function=len,
+)
+texts = text_splitter.split_text(dnd_spell)
+
+docs = text_splitter.create_documents([dnd_spell])
+embeddings = OpenAIEmbeddings()
+
+metadatas = []
+for i in texts:
+    source = i.split("\n")[0]
+    metadatas.append({"source": f"Spell {source} in dnd_spell_split.txt"})
+#%%
+docsearch = Chroma.from_texts(
+    texts,
+    embeddings,
+    persist_directory=persist_directory,
+    metadatas=metadatas,
+)
+docsearch.persist()
+docsearch = None
+
+# %%

main.py

@@ -0,0 +1,100 @@
+"""Python file to serve as the frontend"""
+import streamlit as st
+from streamlit_chat import message
+
+from langchain.chains import VectorDBQAWithSourcesChain
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import Chroma
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts.chat import (
+    ChatPromptTemplate,
+    SystemMessagePromptTemplate,
+    HumanMessagePromptTemplate,
+)
+
+st.set_page_config(page_title="D&D 🗡️ Spell QA Bot", page_icon="🗡️")
+
+# Load the LangChain.
+system_template = """Use the following pieces of context to answer the users question. 
+If you don't know the answer, just say that you don't know, don't try to make up an answer.
+ALWAYS return a "SOURCES" part in your answer.
+The "SOURCES" part should be a reference to the source of the document from which you got your answer.
+
+Example of your response should be:
+
+```
+The answer is foo
+SOURCES: xyz
+```
+
+Begin!
+----------------
+{summaries}"""
+messages = [
+    SystemMessagePromptTemplate.from_template(system_template),
+    HumanMessagePromptTemplate.from_template("{question}"),
+]
+prompt = ChatPromptTemplate.from_messages(messages)
+
+
+@st.cache_resource
+def load_chroma():
+    persist_directory = "db_spells"
+    embeddings = OpenAIEmbeddings()
+    vectordb = Chroma(
+        persist_directory=persist_directory, embedding_function=embeddings
+    )
+    return vectordb
+
+
+vectordb = load_chroma()
+chain_type_kwargs = {"prompt": prompt}
+chain = VectorDBQAWithSourcesChain.from_chain_type(
+    ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0),
+    chain_type="stuff",
+    vectorstore=vectordb,
+    chain_type_kwargs=chain_type_kwargs,
+)
+
+
+# From here down is all the StreamLit UI.
+st.header("D&D 🗡️ Spell QA Bot")
+st.markdown(
+    """
+This is a chatbot that can answer questions about **Dungeon and Dragons spells** based on this [database](https://www.aidedd.org/dnd-filters/spells-5e.php) and built with LangChain and OpenAI API.  
+The creator of this bot is **[Corentin Meyer (@corentinm_py)](https://twitter.com/corentinm_py)**.  
+Try by yourself by typing something like: "What's the size of tsunami spell ?"
+"""
+)
+
+if "generated" not in st.session_state:
+    st.session_state["generated"] = []
+
+if "past" not in st.session_state:
+    st.session_state["past"] = []
+
+
+def get_text():
+    input_text = st.text_input(
+        "You: ", "What's the size of tsunami spell ?", key="input"
+    )
+    return input_text
+
+
+user_input = get_text()
+
+if user_input:
+    result = chain(
+        {"question": user_input},
+        return_only_outputs=True,
+    )
+    output = f"Answer: {result['answer']}\nSources: {result['sources']}"
+
+    st.session_state.past.append(user_input)
+    st.session_state.generated.append(output)
+
+if st.session_state["generated"]:
+
+    for i in range(len(st.session_state["generated"]) - 1, -1, -1):
+        message(st.session_state["generated"][i], key=str(i))
+        message(st.session_state["past"][i], is_user=True, key=str(i) + "_user")

qa.py

@@ -0,0 +1,55 @@
+"""Ask a question to the database."""
+#%%
+from langchain.chains import VectorDBQAWithSourcesChain
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import Chroma
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts.chat import (
+    ChatPromptTemplate,
+    SystemMessagePromptTemplate,
+    HumanMessagePromptTemplate,
+)
+import argparse
+
+system_template = """Use the following pieces of context to answer the users question. 
+If you don't know the answer, just say that you don't know, don't try to make up an answer.
+ALWAYS return a "SOURCES" part in your answer.
+The "SOURCES" part should be a reference to the source of the document from which you got your answer.
+
+Example of your response should be:
+
+```
+The answer is foo
+SOURCES: xyz
+```
+
+Begin!
+----------------
+{summaries}"""
+messages = [
+    SystemMessagePromptTemplate.from_template(system_template),
+    HumanMessagePromptTemplate.from_template("{question}"),
+]
+prompt = ChatPromptTemplate.from_messages(messages)
+
+persist_directory = "db_spells"
+embeddings = OpenAIEmbeddings()
+vectordb = Chroma(persist_directory=persist_directory, embedding_function=embeddings)
+chain_type_kwargs = {"prompt": prompt}
+chain = VectorDBQAWithSourcesChain.from_chain_type(
+    ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0),
+    chain_type="stuff",
+    vectorstore=vectordb,
+    chain_type_kwargs=chain_type_kwargs,
+)
+
+parser = argparse.ArgumentParser(description="Ask a question to the DB.")
+parser.add_argument("question", type=str, help="The question to ask the DB")
+args = parser.parse_args()
+
+result = chain(
+    {"question": args.question},
+    return_only_outputs=True,
+)
+print(f"Answer: {result['answer']}")
+print(f"Sources: {result['sources']}")

requirements.txt

@@ -0,0 +1,8 @@
+langchain
+openai
+streamlit
+streamlit-chat
+ipykernel
+tiktoken
+chromadb
+ipykernel