import spaces
import streamlit as st
from PIL import Image
import torch
from transformers import (
DonutProcessor,
VisionEncoderDecoderModel,
LayoutLMv3Processor,
LayoutLMv3ForSequenceClassification,
AutoProcessor,
AutoModelForCausalLM
)
from ultralytics import YOLO
import io
import base64
import json
from datetime import datetime
@st.cache_resource
def load_model(model_name):
"""Load the selected model and processor
Args:
model_name (str): Name of the model to load ("Donut", "LayoutLMv3", or "OmniParser")
Returns:
dict: Dictionary containing model components
"""
try:
if model_name == "Donut":
processor = DonutProcessor.from_pretrained("naver-clova-ix/donut-base")
model = VisionEncoderDecoderModel.from_pretrained("naver-clova-ix/donut-base")
# Configure Donut specific parameters
model.config.decoder_start_token_id = processor.tokenizer.bos_token_id
model.config.pad_token_id = processor.tokenizer.pad_token_id
model.config.vocab_size = len(processor.tokenizer)
return {'model': model, 'processor': processor}
elif model_name == "LayoutLMv3":
processor = LayoutLMv3Processor.from_pretrained("microsoft/layoutlmv3-base")
model = LayoutLMv3ForSequenceClassification.from_pretrained("microsoft/layoutlmv3-base")
return {'model': model, 'processor': processor}
elif model_name == "OmniParser":
# Load YOLO model for icon detection
yolo_model = YOLO("microsoft/OmniParser")
# Load Florence-2 processor and model for captioning
processor = AutoProcessor.from_pretrained(
"microsoft/Florence-2-base",
trust_remote_code=True
)
# Load the captioning model
caption_model = AutoModelForCausalLM.from_pretrained(
"microsoft/OmniParser",
trust_remote_code=True
)
return {
'yolo': yolo_model,
'processor': processor,
'model': caption_model
}
else:
raise ValueError(f"Unknown model name: {model_name}")
except Exception as e:
st.error(f"Error loading model {model_name}: {str(e)}")
return None
@spaces.GPU
@torch.inference_mode()
def analyze_document(image, model_name, models_dict):
"""Analyze document using selected model
Args:
image (PIL.Image): Input image to analyze
model_name (str): Name of the model to use ("Donut", "LayoutLMv3", or "OmniParser")
models_dict (dict): Dictionary containing loaded model components
Returns:
dict: Analysis results including detected elements, text, and/or coordinates
"""
try:
if models_dict is None:
return {"error": "Model failed to load", "type": "model_error"}
if model_name == "OmniParser":
# Configure detection parameters
box_threshold = 0.05 # Confidence threshold for detection
iou_threshold = 0.1 # IoU threshold for NMS
# Save image temporarily for YOLO processing
temp_path = "temp_image.png"
image.save(temp_path)
# Run YOLO detection
yolo_results = models_dict['yolo'](
temp_path,
conf=box_threshold,
iou=iou_threshold
)
# Process detections and generate captions
results = []
for det in yolo_results[0].boxes.data:
x1, y1, x2, y2, conf, cls = det
# Get region of interest
roi = image.crop((int(x1), int(y1), int(x2), int(y2)))
# Generate caption using the model
inputs = models_dict['processor'](
images=roi,
return_tensors="pt"
)
outputs = models_dict['model'].generate(
**inputs,
max_length=50,
num_beams=4,
temperature=0.7
)
caption = models_dict['processor'].decode(outputs[0], skip_special_tokens=True)
results.append({
"bbox": [float(x) for x in [x1, y1, x2, y2]],
"confidence": float(conf),
"class": int(cls),
"caption": caption
})
# Clean up temporary file
if os.path.exists(temp_path):
os.remove(temp_path)
return {
"detected_elements": len(results),
"elements": results
}
elif model_name == "Donut":
# Process image with Donut
pixel_values = models_dict['processor'](image, return_tensors="pt").pixel_values
task_prompt = "analyze the document and extract information"
decoder_input_ids = models_dict['processor'].tokenizer(
task_prompt,
add_special_tokens=False,
return_tensors="pt"
).input_ids
outputs = models_dict['model'].generate(
pixel_values,
decoder_input_ids=decoder_input_ids,
max_length=512,
early_stopping=True,
pad_token_id=models_dict['processor'].tokenizer.pad_token_id,
eos_token_id=models_dict['processor'].tokenizer.eos_token_id,
use_cache=True,
num_beams=4,
bad_words_ids=[[models_dict['processor'].tokenizer.unk_token_id]],
return_dict_in_generate=True
)
sequence = models_dict['processor'].batch_decode(outputs.sequences)[0]
sequence = sequence.replace(task_prompt, "").replace("", "").strip()
try:
result = json.loads(sequence)
except json.JSONDecodeError:
result = {"raw_text": sequence}
return result
elif model_name == "LayoutLMv3":
# Process image with LayoutLMv3
encoded_inputs = models_dict['processor'](
image,
return_tensors="pt",
add_special_tokens=True,
return_offsets_mapping=True
)
outputs = models_dict['model'](**encoded_inputs)
predictions = outputs.logits.argmax(-1).squeeze().tolist()
# Convert predictions to labels
words = models_dict['processor'].tokenizer.convert_ids_to_tokens(
encoded_inputs.input_ids.squeeze().tolist()
)
result = {
"predictions": [
{
"text": word,
"label": pred
}
for word, pred in zip(words, predictions)
if word not in ["", "", ""]
],
"confidence_scores": outputs.logits.softmax(-1).max(-1).values.squeeze().tolist()
}
return result
else:
return {"error": f"Unknown model: {model_name}", "type": "model_error"}
except Exception as e:
import traceback
error_details = traceback.format_exc()
return {
"error": str(e),
"type": "processing_error",
"details": error_details
}
# Set page config with improved layout
st.set_page_config(
page_title="Document Analysis Comparison",
layout="wide",
initial_sidebar_state="expanded"
)
# Add custom CSS for better styling
st.markdown("""
""", unsafe_allow_html=True)
# Title and description
st.title("Document Understanding Model Comparison")
st.markdown("""
Compare different models for document analysis and understanding.
Upload an image and select a model to analyze it.
""")
# Create two columns for layout
col1, col2 = st.columns([1, 1])
with col1:
# File uploader with improved error handling
uploaded_file = st.file_uploader(
"Choose a document image",
type=['png', 'jpg', 'jpeg', 'pdf'],
help="Supported formats: PNG, JPEG, PDF"
)
if uploaded_file is not None:
try:
# Display uploaded image
image = Image.open(uploaded_file)
st.image(image, caption='Uploaded Document', use_column_width=True)
except Exception as e:
st.error(f"Error loading image: {str(e)}")
with col2:
# Model selection with detailed information
model_info = {
"Donut": {
"description": "Best for structured OCR and document format understanding",
"memory": "6-8GB",
"strengths": ["Structured OCR", "Memory efficient", "Good with fixed formats"],
"best_for": ["Invoices", "Forms", "Structured documents", "Tables"]
},
"LayoutLMv3": {
"description": "Strong layout understanding with reasoning capabilities",
"memory": "12-15GB",
"strengths": ["Layout understanding", "Reasoning", "Pre-trained knowledge"],
"best_for": ["Complex documents", "Mixed layouts", "Documents with tables", "Multi-column text"]
},
"OmniParser": {
"description": "General screen parsing tool for UI understanding",
"memory": "8-10GB",
"strengths": ["UI element detection", "Interactive element recognition", "Function description"],
"best_for": ["Screenshots", "UI analysis", "Interactive elements", "Web interfaces"]
}
}
selected_model = st.selectbox(
"Select Model",
list(model_info.keys())
)
# Display enhanced model information
st.markdown("### Model Details")
with st.expander("Model Information", expanded=True):
st.markdown(f"**Description:** {model_info[selected_model]['description']}")
st.markdown(f"**Memory Required:** {model_info[selected_model]['memory']}")
st.markdown("**Strengths:**")
for strength in model_info[selected_model]['strengths']:
st.markdown(f"- {strength}")
st.markdown("**Best For:**")
for use_case in model_info[selected_model]['best_for']:
st.markdown(f"- {use_case}")
# Inside the analysis section, replace the existing if-block with:
if uploaded_file is not None and selected_model:
if st.button("Analyze Document", help="Click to start document analysis"):
# Create two columns for results and debug info
result_col, debug_col = st.columns([1, 1])
with st.spinner('Processing...'):
try:
# Create a progress bar in results column
with result_col:
st.markdown("### Analysis Progress")
progress_bar = st.progress(0)
# Initialize debug column
with debug_col:
st.markdown("### Debug Information")
debug_container = st.empty()
def update_debug(message, level="info"):
"""Update debug information with timestamp"""
timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
color = {
"info": "blue",
"warning": "orange",
"error": "red",
"success": "green"
}.get(level, "black")
return f"[{timestamp}] {message}
"
debug_messages = []
def add_debug(message, level="info"):
debug_messages.append(update_debug(message, level))
debug_container.markdown(
"\n".join(debug_messages),
unsafe_allow_html=True
)
# Load model with progress update
with result_col:
progress_bar.progress(25)
st.info("Loading model...")
add_debug(f"Loading {selected_model} model and processor...")
model, processor = load_model(selected_model)
if model is None or processor is None:
with result_col:
st.error("Failed to load model. Please try again.")
add_debug("Model loading failed!", "error")
else:
add_debug("Model loaded successfully", "success")
add_debug(f"Model device: {next(model.parameters()).device}")
add_debug(f"Model memory usage: {torch.cuda.max_memory_allocated()/1024**2:.2f}MB") if torch.cuda.is_available() else None
# Update progress
with result_col:
progress_bar.progress(50)
st.info("Analyzing document...")
# Log image details
add_debug(f"Image size: {image.size}")
add_debug(f"Image mode: {image.mode}")
# Analyze document
add_debug("Starting document analysis...")
results = analyze_document(image, selected_model, model, processor)
add_debug("Analysis completed", "success")
# Update progress
with result_col:
progress_bar.progress(75)
st.markdown("### Analysis Results")
if isinstance(results, dict) and "error" in results:
st.error(f"Analysis Error: {results['error']}")
add_debug(f"Analysis error: {results['error']}", "error")
else:
# Pretty print the results in results column
st.json(results)
# Show detailed results breakdown in debug column
add_debug("Results breakdown:", "info")
if isinstance(results, dict):
for key, value in results.items():
add_debug(f"- {key}: {type(value)}")
else:
add_debug(f"Result type: {type(results)}")
# Complete progress
progress_bar.progress(100)
st.success("Analysis completed!")
# Final debug info
add_debug("Process completed successfully", "success")
with debug_col:
if torch.cuda.is_available():
st.markdown("### Resource Usage")
st.markdown(f"""
- GPU Memory: {torch.cuda.max_memory_allocated()/1024**2:.2f}MB
- GPU Utilization: {torch.cuda.utilization()}%
""")
except Exception as e:
with result_col:
st.error(f"Error during analysis: {str(e)}")
add_debug(f"Error: {str(e)}", "error")
add_debug(f"Error type: {type(e)}", "error")
if hasattr(e, '__traceback__'):
add_debug("Traceback available in logs", "warning")
# Add improved information about usage and limitations
st.markdown("""
---
### Usage Notes:
- Different models excel at different types of documents
- Processing time and memory requirements vary by model
- Image quality significantly affects results
- Some models may require specific document formats
""")
# Add performance metrics section
if st.checkbox("Show Performance Metrics"):
st.markdown("""
### Model Performance Metrics
| Model | Avg. Processing Time | Memory Usage | Accuracy* |
|-------|---------------------|--------------|-----------|
| Donut | 2-3 seconds | 6-8GB | 85-90% |
| LayoutLMv3 | 3-4 seconds | 12-15GB | 88-93% |
| OmniParser | 2-3 seconds | 8-10GB | 85-90% |
*Accuracy varies based on document type and quality
""")
# Add a footer with version and contact information
st.markdown("---")
st.markdown("""
v1.1 - Created with Streamlit
\nPowered by Hugging Face Spaces 🤗
""")
# Add model selection guidance
if st.checkbox("Show Model Selection Guide"):
st.markdown("""
### How to Choose the Right Model
1. **Donut**: Choose for structured documents with clear layouts
2. **LayoutLMv3**: Best for documents with complex layouts and relationships
3. **OmniParser**: Best for UI elements and screen parsing
""")