app/api/api_file.py

from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form, Request, Query, status
from fastapi.responses import StreamingResponse
import os
import logging
import uuid
from datetime import datetime

from pydantic import BaseModel, Field
from typing import Optional, List, Any
from urllib.parse import urlparse
import shutil
# from app.wrapper.llm_wrapper import *
from app.crud.process_file import load_file_with_markitdown, process_uploaded_file

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


def is_url(path: str) -> bool:
    """
    Determines if the given path is a URL.

    Args:
        path (str): The path or URL to check.

    Returns:
        bool: True if it's a URL, False otherwise.
    """
    try:
        result = urlparse(path)
        return all([result.scheme, result.netloc])
    except Exception:
        return False


file_router = APIRouter()

# Configure logging to file with date-based filenames
log_filename = f"document_logs_{datetime.now().strftime('%Y-%m-%d')}.txt"
file_handler = logging.FileHandler(log_filename)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
file_handler.setFormatter(formatter)

# Create a logger for document processing
doc_logger = logging.getLogger('document_logger')
doc_logger.setLevel(logging.INFO)
doc_logger.addHandler(file_handler)

# Also configure the general logger if not already configured
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

from app.search.rag_pipeline import RAGSystem
from sentence_transformers import SentenceTransformer


@file_router.post("/load_file_with_markdown/")
async def load_file_with_markdown(request: Request, filepaths: List[str]):
    try:
        # Ensure RAG system is initialized
        try:
            rag_system =  request.app.state.rag_system
            if rag_system is None:
                raise AttributeError("RAG system is not initialized in app state")
        except AttributeError:
            logger.error("RAG system is not initialized in app state")
            raise HTTPException(status_code=500, detail="RAG system not initialized in app state")

        
        processed_files = []
        pages = []
        
         # Process each file path or URL
        for path in filepaths:
            if is_url(path):
                logger.info(f"Processing URL: {path}")
                try:
                    # Generate a unique UUID for the document
                    doc_id = str(uuid.uuid4())

                    # Process the URL
                    document = await process_uploaded_file(id=doc_id, file_path=path, rag_system=rag_system)

                    # Append the document details to pages
                    pages.append({
                        "metadata": {"title": document.title},
                        "page_content": document.text_content,
                    })

                    logger.info(f"Successfully processed URL: {path} with ID: {doc_id}")

                    # Log the ID and a 100-character snippet of the document
                    snippet = document.text_content[:100].replace('\n', ' ').replace('\r', ' ')
                    # Ensure 'doc_logger' is defined; if not, use 'logger' or define 'doc_logger'
                    doc_logger.info(f"ID: {doc_id}_{document.title}, Snippet: {snippet}")


                except Exception as e:
                    logger.error(f"Error processing URL {path}: {str(e)}")
                    processed_files.append({"path": path, "status": "error", "message": str(e)})

            else:
                logger.info(f"Processing local file: {path}")
                if os.path.exists(path):
                    try:
                        # Generate a unique UUID for the document
                        doc_id = str(uuid.uuid4())

                        # Process the local file
                        document = await process_uploaded_file(id=doc_id, file_path=path, rag_system=rag_system)

                        # Append the document details to pages
                        pages.append({
                            "metadata": {"title": document.title},
                            "page_content": document.text_content,
                        })

                        logger.info(f"Successfully processed file: {path} with ID: {doc_id}")

                        # Log the ID and a 100-character snippet of the document
                        snippet = document.text_content[:100].replace('\n', ' ').replace('\r', ' ')
                        # Ensure 'doc_logger' is defined; if not, use 'logger' or define 'doc_logger'
                        logger.info(f"ID: {doc_id}, Snippet: {snippet}")

                    except Exception as e:
                        logger.error(f"Error processing file {path}: {str(e)}")
                        processed_files.append({"path": path, "status": "error", "message": str(e)})
                else:
                    logger.error(f"File path does not exist: {path}")
                    processed_files.append({"path": path, "status": "not found"})
        
        # Get total tokens from RAG system
        total_tokens = rag_system.get_total_tokens() if hasattr(rag_system, "get_total_tokens") else 0
        
        return {
            "message": "File processing completed",
            "total_tokens": total_tokens,
            "document_count": len(filepaths),
            "pages": pages,
            "errors": processed_files,  # Include details about files that couldn't be processed
        }
    
    except Exception as e:
        logger.exception("Unexpected error during file processing")
        raise HTTPException(status_code=500, detail=f"An unexpected error occurred: {str(e)}")
    
async def load_file_with_markdown_function(filepaths: List[str],
    rag_system: Any):
    try:
        # Ensure RAG system is initialized
        try:
            rag_system =  rag_system
        except AttributeError:
            logger.error("RAG system is not initialized in app state")
            raise HTTPException(status_code=500, detail="RAG system not initialized in app state")

        
        processed_files = []
        pages = []
        
         # Process each file path or URL
        for path in filepaths:
            if is_url(path):
                logger.info(f"Processing URL: {path}")
                try:
                    # Generate a unique UUID for the document
                    doc_id = str(uuid.uuid4())

                    # Process the URL
                    document = await process_uploaded_file(id=doc_id, file_path=path, rag_system=rag_system)

                    # Append the document details to pages
                    pages.append({
                        "metadata": {"title": document.title},
                        "page_content": document.text_content,
                    })

                    logger.info(f"Successfully processed URL: {path} with ID: {doc_id}")

                    # Log the ID and a 100-character snippet of the document
                    snippet = document.text_content[:100].replace('\n', ' ').replace('\r', ' ')
                    # Ensure 'doc_logger' is defined; if not, use 'logger' or define 'doc_logger'
                    doc_logger(f"ID: {doc_id}, Snippet: {snippet}")
                    logger.info(f"ID: {doc_id}, Snippet: {snippet}")

                except Exception as e:
                    logger.error(f"Error processing URL {path}: {str(e)}")
                    processed_files.append({"path": path, "status": "error", "message": str(e)})

            else:
                logger.info(f"Processing local file: {path}")
                if os.path.exists(path):
                    try:
                        # Generate a unique UUID for the document
                        doc_id = str(uuid.uuid4())

                        # Process the local file
                        document = await process_uploaded_file(id=doc_id, file_path=path, rag_system=rag_system)

                        # Append the document details to pages
                        pages.append({
                            "metadata": {"title": document.title},
                            "page_content": document.text_content,
                        })

                        logger.info(f"Successfully processed file: {path} with ID: {doc_id}")

                        # Log the ID and a 100-character snippet of the document
                        snippet = document.text_content[:100].replace('\n', ' ').replace('\r', ' ')
                        # Ensure 'doc_logger' is defined; if not, use 'logger' or define 'doc_logger'
                        logger.info(f"ID: {doc_id}, Snippet: {snippet}")

                    except Exception as e:
                        logger.error(f"Error processing file {path}: {str(e)}")
                        processed_files.append({"path": path, "status": "error", "message": str(e)})
                else:
                    logger.error(f"File path does not exist: {path}")
                    processed_files.append({"path": path, "status": "not found"})
        
        # Get total tokens from RAG system
        total_tokens = rag_system.get_total_tokens() if hasattr(rag_system, "get_total_tokens") else 0
        
        return {
            "message": "File processing completed",
            "total_tokens": total_tokens,
            "document_count": len(filepaths),
            "pages": pages,
            "errors": processed_files,  # Include details about files that couldn't be processed
        }
    
    except Exception as e:
        logger.exception("Unexpected error during file processing")
        raise HTTPException(status_code=500, detail=f"An unexpected error occurred: {str(e)}")

@file_router.get("/document_exists/{doc_id}", status_code=status.HTTP_200_OK)
async def document_exists(request: Request, doc_id: str):
    try:
        rag_system = request.app.state.rag_system
    except AttributeError:
        logger.error("RAG system is not initialized in app state")
        raise HTTPException(status_code=500, detail="RAG system not initialized in app state")
    
    exists = doc_id in rag_system.doc_ids
    return {"document_id": doc_id, "exists": exists}

@file_router.delete("/delete_document/{doc_id}", status_code=status.HTTP_200_OK)
async def delete_document(request: Request, doc_id: str):
    try:
        rag_system = request.app.state.rag_system
    except AttributeError:
        logger.error("RAG system is not initialized in app state")
        raise HTTPException(status_code=500, detail="RAG system not initialized in app state")
    
    try:
        rag_system.delete_document(doc_id)
        logger.info(f"Deleted document with ID: {doc_id}")
        return {"message": f"Document with ID {doc_id} has been deleted."}
    except Exception as e:
        logger.error(f"Error deleting document with ID {doc_id}: {str(e)}")
        raise HTTPException(status_code=500, detail=f"Failed to delete document: {str(e)}")