Update app.py

app.py

@@ -26,31 +26,26 @@ for filename in os.listdir(directory):
         docs = loader.load()
         documents.extend(docs)
 
-# Split the documents
-text_splitter = RecursiveCharacterTextSplitter(
-    chunk_size=500,
-    chunk_overlap=40,
-    length_function=len,
-    is_separator_regex=False
-)
-rag_documents = text_splitter.split_documents(documents)
-
-# # Alternative chunking: Tokens (more accurate for OpenAI models)
-# token_text_splitter = CharacterTextSplitter.from_tiktoken_encoder(
-#     encoding="cl100k_base", chunk_size=100, chunk_overlap=0
-# )
-# token_rag_documents = token_text_splitter.split_documents(documents)
-# # TO DO ^^ test
-
 # Split the documents by character
-text_splitter = CharacterTextSplitter(
+character_text_splitter = CharacterTextSplitter(
     separator="\n\n",
     chunk_size=1000,
     chunk_overlap=200,
     length_function=len,
     is_separator_regex=False,
 )
-character_rag_documents = text_splitter.split_documents(documents)
+rag_documents = character_text_splitter.split_documents(documents)
+
+# Split the documents recursively
+recursive_text_splitter = RecursiveCharacterTextSplitter(
+    chunk_size=500,
+    chunk_overlap=40,
+    length_function=len,
+    is_separator_regex=False
+)
+# rag_documents = recursive_text_splitter.split_documents(documents)
+
+
 
 embedding = OpenAIEmbeddings(model="text-embedding-3-small")