Spaces:
Running
Running
File size: 2,072 Bytes
7beb366 2624a11 7bc489f 72762da 7bc489f 72762da 7bc489f 72762da 7bc489f 72762da 2624a11 72762da 5023418 2624a11 e5beda5 2624a11 72762da 7beb366 72762da 2624a11 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 |
from langchain_community.document_loaders import PyMuPDFLoader, TextLoader, UnstructuredURLLoader
from langchain_community.vectorstores import Qdrant
import os
import requests
def process_file(file):
# save the file temporarily
temp_file = "./"+file.path
with open(temp_file, "wb") as file:
file.write(file.content)
documents = []
if file.path.endswith(".pdf"):
loader = PyMuPDF(temp_file)
docs = loader.load()
documents.extend(docs)
else:
loader = TextLoader(temp_file)
docs = loader.load()
documents.extend(docs)
return documents
def load_documents_from_url(url):
try:
# Check if it's a PDF
if url.endswith(".pdf"):
try:
loader = PyMuPDFLoader(url)
return loader.load()
except Exception as e:
print(f"Error loading PDF from {url}: {e}")
return None
# Fetch the content and check for video pages
try:
response = requests.head(url, timeout=10) # Timeout for fetching headers
content_type = response.headers.get('Content-Type', '')
except Exception as e:
print(f"Error fetching headers from {url}: {e}")
return None
# Ignore video content (flagged for now)
if 'video' in content_type:
return None
if 'youtube' in url:
return None
# Otherwise, treat it as an HTML page
try:
loader = UnstructuredURLLoader([url])
return loader.load()
except Exception as e:
print(f"Error loading HTML from {url}: {e}")
return None
except Exception as e:
print(f"General error loading from {url}: {e}")
return None
def add_to_qdrant(documents, embeddings, qdrant_client, collection_name):
Qdrant.from_documents(
documents,
embeddings,
url=qdrant_client.url,
prefer_grpc=True,
collection_name=collection_name,
) |