# utils/depth_estimation.py import torch import numpy as np from PIL import Image import open3d as o3d from transformers import DPTImageProcessor, DPTForDepthEstimation from pathlib import Path import logging logging.getLogger("transformers.modeling_utils").setLevel(logging.ERROR) from utils.image_utils import ( resize_image_with_aspect_ratio ) # Load models once during module import image_processor = DPTImageProcessor.from_pretrained("Intel/dpt-large") depth_model = DPTForDepthEstimation.from_pretrained("Intel/dpt-large", ignore_mismatched_sizes=True) def estimate_depth(image): # Ensure image is in RGB mode if image.mode != "RGB": image = image.convert("RGB") # Resize the image for the model image_resized = image.resize( (image.width, image.height), Image.Resampling.LANCZOS ) # Prepare image for the model encoding = image_processor(image_resized, return_tensors="pt") # Forward pass with torch.no_grad(): outputs = depth_model(**encoding) predicted_depth = outputs.predicted_depth # Interpolate to original size prediction = torch.nn.functional.interpolate( predicted_depth.unsqueeze(1), size=(image.height, image.width), mode="bicubic", align_corners=False, ).squeeze() # Convert to depth image output = prediction.cpu().numpy() depth_min = output.min() depth_max = output.max() max_val = (2**8) - 1 # Normalize and convert to 8-bit image depth_image = max_val * (output - depth_min) / (depth_max - depth_min) depth_image = depth_image.astype("uint8") depth_pil = Image.fromarray(depth_image) return depth_pil, output def create_3d_model(rgb_image, depth_array, voxel_size_factor=0.01): depth_o3d = o3d.geometry.Image(depth_array.astype(np.float32)) rgb_o3d = o3d.geometry.Image(np.array(rgb_image)) rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth( rgb_o3d, depth_o3d, convert_rgb_to_intensity=False ) # Create a point cloud from the RGBD image camera_intrinsic = o3d.camera.PinholeCameraIntrinsic( rgb_image.width, rgb_image.height, fx=1.0, fy=1.0, cx=rgb_image.width / 2.0, cy=rgb_image.height / 2.0, ) pcd = o3d.geometry.PointCloud.create_from_rgbd_image( rgbd_image, camera_intrinsic ) # Voxel downsample voxel_size = max(pcd.get_max_bound() - pcd.get_min_bound()) * voxel_size_factor voxel_grid = o3d.geometry.VoxelGrid.create_from_point_cloud(pcd, voxel_size=voxel_size) # Save the 3D model to a temporary file temp_dir = Path.cwd() / "temp_models" temp_dir.mkdir(exist_ok=True) model_path = temp_dir / "model.ply" o3d.io.write_voxel_grid(str(model_path), voxel_grid) return str(model_path) def generate_depth_and_3d(input_image_path, voxel_size_factor): image = Image.open(input_image_path).convert("RGB") resized_image = resize_image_with_aspect_ratio(image, 2688, 1680) depth_image, depth_array = estimate_depth(resized_image) model_path = create_3d_model(resized_image, depth_array, voxel_size_factor=voxel_size_factor) return depth_image, model_path def generate_depth_button_click(depth_image_source, voxel_size_factor, input_image, output_image, overlay_image, bordered_image_output): if depth_image_source == "Input Image": image_path = input_image elif depth_image_source == "Output Image": image_path = output_image elif depth_image_source == "Image with Margins": image_path = bordered_image_output else: image_path = overlay_image return generate_depth_and_3d(image_path, voxel_size_factor)