Update app.py

app.py

@@ -35,12 +35,22 @@ text_splitter = RecursiveCharacterTextSplitter(
 )
 rag_documents = text_splitter.split_documents(documents)
 
-# Alternative chunking: Tokens (more accurate for OpenAI models)
-token_text_splitter = CharacterTextSplitter.from_tiktoken_encoder(
-    encoding="cl100k_base", chunk_size=100, chunk_overlap=0
+# # Alternative chunking: Tokens (more accurate for OpenAI models)
+# token_text_splitter = CharacterTextSplitter.from_tiktoken_encoder(
+#     encoding="cl100k_base", chunk_size=100, chunk_overlap=0
+# )
+# token_rag_documents = token_text_splitter.split_documents(documents)
+# # TO DO ^^ test
+
+# Split the documents by character
+text_splitter = CharacterTextSplitter(
+    separator="\n\n",
+    chunk_size=1000,
+    chunk_overlap=200,
+    length_function=len,
+    is_separator_regex=False,
 )
-token_rag_documents = token_text_splitter.split_documents(documents)
-# TO DO ^^ test
+character_rag_documents = text_splitter.split_documents(documents)
 
 embedding = OpenAIEmbeddings(model="text-embedding-3-small")