diff --git a/README.md b/README.md index 0118817f997acec32f4540eeb3468c1da7949140..ac0abfbfdd4c402e266931abb1fc0cae89a09c43 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,54 @@ ---- -title: Splatt3r -emoji: 🐠 -colorFrom: blue -colorTo: green -sdk: gradio -sdk_version: 4.41.0 -app_file: app.py -pinned: false ---- - -Check out the configuration reference at https://huggingface.co./docs/hub/spaces-config-reference +# Splatt3R: Zero-shot Gaussian Splatting from Uncalibarated Image Pairs + +Official implementation of `Zero-shot Gaussian Splatting from Uncalibarated Image Pairs` + +Links removed for anonymity: +[Project Page](), [Splatt3R arXiv]() + +## Installation + +1. Clone Splatt3R +```bash +git clone +cd splatt3r +``` + +2. Setup Anaconda Environment +```bash +conda env create -f environment.yml +pip install git+https://github.com/dcharatan/diff-gaussian-rasterization-modified +``` + +3. (Optional) Compile the CUDA kernels for RoPE (as in MASt3R and CroCo v2) + +```bash +cd src/dust3r_src/croco/models/curope/ +python setup.py build_ext --inplace +cd ../../../../../ +``` + +## Checkpoints + +We train our model using the pretrained `MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric` checkpoint from the MASt3R authors, available from [the MASt3R GitHub repo](https://github.com/naver/mast3r). This checkpoint is placed at the file path `checkpoints/MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric.pth`. + +A pretrained Splatt3R model can be downloaded [here]() (redacted link). + +## Data + +We use ScanNet++ to train our model. We download the data from the [official ScanNet++ homepage](https://kaldir.vc.in.tum.de/scannetpp/) and process the data using SplaTAM's modified version of [the ScanNet++ toolkit](https://github.com/Nik-V9/scannetpp). We save the processed data to the 'processed' subfolder of the ScanNet++ root directory. + +Our generated test coverage files, and our training and testing splits, can be downloaded [here]() (redacted link), and placed in `data/scannetpp`. + +## Demo + +The Gradio demo can be run using `python demo.py `, replacing `` with the trained network path. A checkpoint will be available for the public release of this code. + +This demo generates a `.ply` file that represents the scene, which can be downloaded and rendered using online 3D Gaussian Splatting viewers such as [here](https://projects.markkellogg.org/threejs/demo_gaussian_splats_3d.php?art=1&cu=0,-1,0&cp=0,1,0&cla=1,0,0&aa=false&2d=false&sh=0) or [here](https://playcanvas.com/supersplat/editor). + +## Training + +Our training run can be recreated by running `python main.py configs/main.yaml`. Other configurations can be found in the `configs` folder. + +## BibTeX + +Forthcoming arXiv citation \ No newline at end of file diff --git a/ablations.py b/ablations.py new file mode 100644 index 0000000000000000000000000000000000000000..8f2d2a3db2910536a759905641b4323f96d72090 --- /dev/null +++ b/ablations.py @@ -0,0 +1,75 @@ +from main import * + + +def default_run(): + + # Setup the workspace (eg. load the config, create a directory for results at config.save_dir, etc.) + config_location = "configs/main.yaml" + config = workspace.load_config(config_location, None) + if os.getenv("LOCAL_RANK", '0') == '0': + config = workspace.create_workspace(config) + + # Run the experiment + run_experiment(config) + + +def with_mast3r_loss(): + + # Setup the workspace (eg. load the config, create a directory for results at config.save_dir, etc.) + config_location = "configs/with_mast3r_loss.yaml" + config = workspace.load_config(config_location, None) + if os.getenv("LOCAL_RANK", '0') == '0': + config = workspace.create_workspace(config) + + # Run the experiment + run_experiment(config) + + +def without_masking(): + + # Setup the workspace (eg. load the config, create a directory for results at config.save_dir, etc.) + config_location = "configs/without_masking.yaml" + config = workspace.load_config(config_location, None) + if os.getenv("LOCAL_RANK", '0') == '0': + config = workspace.create_workspace(config) + + # Run the experiment + run_experiment(config) + + +def without_lpips_loss(): + + # Setup the workspace (eg. load the config, create a directory for results at config.save_dir, etc.) + config_location = "configs/without_lpips_loss.yaml" + config = workspace.load_config(config_location, None) + if os.getenv("LOCAL_RANK", '0') == '0': + config = workspace.create_workspace(config) + + # Run the experiment + run_experiment(config) + + +def without_offset(): + + # Setup the workspace (eg. load the config, create a directory for results at config.save_dir, etc.) + config_location = "configs/without_offset.yaml" + config = workspace.load_config(config_location, None) + if os.getenv("LOCAL_RANK", '0') == '0': + config = workspace.create_workspace(config) + + # Run the experiment + run_experiment(config) + + +if __name__ == "__main__": + + # Somewhat hacky way to fetch the function corresponding to the ablation we want to run + ablation_name = sys.argv[1] + ablation_function = locals().get(ablation_name) + + # Run the ablation if it exists + if ablation_function: + ablation_function() + else: + raise NotImplementedError( + f"Ablation name '{sys.argv[1]}' not recognised") diff --git a/configs/debug.yaml b/configs/debug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6185e43d990e339ebfc9d74c6fd4a3738035914 --- /dev/null +++ b/configs/debug.yaml @@ -0,0 +1,17 @@ +include: ['main.yaml'] + +save_dir: './results/debug/${name}/' + +devices: [0] + +loggers: + use_wandb: True + +data: + root: '/media/brandon/anubis09/scannetpp' + batch_size: 2 + num_workers: 8 + epochs_per_train_epoch: 10 + +opt: + epochs: 1 \ No newline at end of file diff --git a/configs/main.yaml b/configs/main.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ea15a081cad718d5672081672b74dbc873ceed0 --- /dev/null +++ b/configs/main.yaml @@ -0,0 +1,42 @@ +name: '%Y-%m-%d-%H-%M-%S' + +save_dir: './results/${name}/' + +# Environment +seed: 0 +devices: 'auto' + +# Loggers +use_profiler: False +loggers: + use_csv_logger: True + use_wandb: True + +# Model +use_pretrained: True +pretrained_mast3r_path: './checkpoints/MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric.pth' + +# Data +data: + root: '/home/bras5602/data/scannetpp' + batch_size: 12 + num_workers: 16 + resolution: [512, 512] + epochs_per_train_epoch: 100 # How many times to sample from each scene each training epoch (helps avoid unnecessary Pytorch Lightning overhead) + +# Optimization +opt: + epochs: 20 + lr: 0.00001 + weight_decay: 0.05 + gradient_clip_val: 0.5 + +loss: + mse_loss_weight: 1.0 + lpips_loss_weight: 0.25 + mast3r_loss_weight: Null + apply_mask: True + average_over_mask: True + +use_offsets: True +sh_degree: 1 \ No newline at end of file diff --git a/configs/with_mast3r_loss.yaml b/configs/with_mast3r_loss.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ce0dfd1c20094e93cc305d6c6436ee90ab7510d --- /dev/null +++ b/configs/with_mast3r_loss.yaml @@ -0,0 +1,6 @@ +include: ['main.yaml'] + +name: 'with_mast3r_loss/%Y-%m-%d-%H-%M-%S' + +loss: + mast3r_loss_weight: 0.05 \ No newline at end of file diff --git a/configs/without_lpips_loss.yaml b/configs/without_lpips_loss.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b222f5215221756c6743642143e36be7938e3b2a --- /dev/null +++ b/configs/without_lpips_loss.yaml @@ -0,0 +1,6 @@ +include: ['main.yaml'] + +name: 'without_lpips_loss/%Y-%m-%d-%H-%M-%S' + +loss: + lpips_loss_weight: 0.0 \ No newline at end of file diff --git a/configs/without_masking.yaml b/configs/without_masking.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e97bc62eba22214bccfb2d74249e903f9bbd6ac --- /dev/null +++ b/configs/without_masking.yaml @@ -0,0 +1,7 @@ +include: ['main.yaml'] + +name: 'without_masking/%Y-%m-%d-%H-%M-%S' + +loss: + apply_mask: False + average_over_mask: False \ No newline at end of file diff --git a/configs/without_offset.yaml b/configs/without_offset.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2d41543d5fc40469bc7dbd7f6d93d40c62ad122 --- /dev/null +++ b/configs/without_offset.yaml @@ -0,0 +1,5 @@ +include: ['main.yaml'] + +name: 'without_offset/%Y-%m-%d-%H-%M-%S' + +use_offsets: False \ No newline at end of file diff --git a/data/data.py b/data/data.py new file mode 100644 index 0000000000000000000000000000000000000000..e20f0f48719c66824b4bc596b43e16cf51395435 --- /dev/null +++ b/data/data.py @@ -0,0 +1,205 @@ +import random + +import numpy as np +import PIL +import torch +import torchvision + +from src.mast3r_src.dust3r.dust3r.datasets.utils.transforms import ImgNorm +from src.mast3r_src.dust3r.dust3r.utils.geometry import depthmap_to_absolute_camera_coordinates, geotrf +from src.mast3r_src.dust3r.dust3r.utils.misc import invalid_to_zeros +import src.mast3r_src.dust3r.dust3r.datasets.utils.cropping as cropping + + +def crop_resize_if_necessary(image, depthmap, intrinsics, resolution): + """Adapted from DUST3R's Co3D dataset implementation""" + + if not isinstance(image, PIL.Image.Image): + image = PIL.Image.fromarray(image) + + # Downscale with lanczos interpolation so that image.size == resolution cropping centered on the principal point + # The new window will be a rectangle of size (2*min_margin_x, 2*min_margin_y) centered on (cx,cy) + W, H = image.size + cx, cy = intrinsics[:2, 2].round().astype(int) + min_margin_x = min(cx, W - cx) + min_margin_y = min(cy, H - cy) + assert min_margin_x > W / 5 + assert min_margin_y > H / 5 + l, t = cx - min_margin_x, cy - min_margin_y + r, b = cx + min_margin_x, cy + min_margin_y + crop_bbox = (l, t, r, b) + image, depthmap, intrinsics = cropping.crop_image_depthmap(image, depthmap, intrinsics, crop_bbox) + + # High-quality Lanczos down-scaling + target_resolution = np.array(resolution) + image, depthmap, intrinsics = cropping.rescale_image_depthmap(image, depthmap, intrinsics, target_resolution) + + # Actual cropping (if necessary) with bilinear interpolation + intrinsics2 = cropping.camera_matrix_of_crop(intrinsics, image.size, resolution, offset_factor=0.5) + crop_bbox = cropping.bbox_from_intrinsics_in_out(intrinsics, intrinsics2, resolution) + image, depthmap, intrinsics2 = cropping.crop_image_depthmap(image, depthmap, intrinsics, crop_bbox) + + return image, depthmap, intrinsics2 + + +class DUST3RSplattingDataset(torch.utils.data.Dataset): + + def __init__(self, data, coverage, resolution, num_epochs_per_epoch=1, alpha=0.3, beta=0.3): + + super(DUST3RSplattingDataset, self).__init__() + self.data = data + self.coverage = coverage + + self.num_context_views = 2 + self.num_target_views = 3 + + self.resolution = resolution + self.transform = ImgNorm + self.org_transform = torchvision.transforms.ToTensor() + self.num_epochs_per_epoch = num_epochs_per_epoch + + self.alpha = alpha + self.beta = beta + + def __getitem__(self, idx): + + sequence = self.data.sequences[idx // self.num_epochs_per_epoch] + sequence_length = len(self.data.color_paths[sequence]) + + context_views, target_views = self.sample(sequence, self.num_target_views, self.alpha, self.beta) + + views = {"context": [], "target": [], "scene": sequence} + + # Fetch the context views + for c_view in context_views: + + assert c_view < sequence_length, f"Invalid view index: {c_view}, sequence length: {sequence_length}, c_views: {context_views}" + + view = self.data.get_view(sequence, c_view, self.resolution) + + # Transform the input + view['img'] = self.transform(view['original_img']) + view['original_img'] = self.org_transform(view['original_img']) + + # Create the point cloud and validity mask + pts3d, valid_mask = depthmap_to_absolute_camera_coordinates(**view) + view['pts3d'] = pts3d + view['valid_mask'] = valid_mask & np.isfinite(pts3d).all(axis=-1) + assert view['valid_mask'].any(), f"Invalid mask for sequence: {sequence}, view: {c_view}" + + views['context'].append(view) + + # Fetch the target views + for t_view in target_views: + + view = self.data.get_view(sequence, t_view, self.resolution) + view['original_img'] = self.org_transform(view['original_img']) + views['target'].append(view) + + return views + + def __len__(self): + + return len(self.data.sequences) * self.num_epochs_per_epoch + + def sample(self, sequence, num_target_views, context_overlap_threshold=0.5, target_overlap_threshold=0.6): + + first_context_view = random.randint(0, len(self.data.color_paths[sequence]) - 1) + + # Pick a second context view that has sufficient overlap with the first context view + valid_second_context_views = [] + for frame in range(len(self.data.color_paths[sequence])): + if frame == first_context_view: + continue + overlap = self.coverage[sequence][first_context_view][frame] + if overlap > context_overlap_threshold: + valid_second_context_views.append(frame) + if len(valid_second_context_views) > 0: + second_context_view = random.choice(valid_second_context_views) + + # If there are no valid second context views, pick the best one + else: + best_view = None + best_overlap = None + for frame in range(len(self.data.color_paths[sequence])): + if frame == first_context_view: + continue + overlap = self.coverage[sequence][first_context_view][frame] + if best_view is None or overlap > best_overlap: + best_view = frame + best_overlap = overlap + second_context_view = best_view + + # Pick the target views + valid_target_views = [] + for frame in range(len(self.data.color_paths[sequence])): + if frame == first_context_view or frame == second_context_view: + continue + overlap_max = max( + self.coverage[sequence][first_context_view][frame], + self.coverage[sequence][second_context_view][frame] + ) + if overlap_max > target_overlap_threshold: + valid_target_views.append(frame) + if len(valid_target_views) >= num_target_views: + target_views = random.sample(valid_target_views, num_target_views) + + # If there are not enough valid target views, pick the best ones + else: + overlaps = [] + for frame in range(len(self.data.color_paths[sequence])): + if frame == first_context_view or frame == second_context_view: + continue + overlap = max( + self.coverage[sequence][first_context_view][frame], + self.coverage[sequence][second_context_view][frame] + ) + overlaps.append((frame, overlap)) + overlaps.sort(key=lambda x: x[1], reverse=True) + target_views = [frame for frame, _ in overlaps[:num_target_views]] + + return [first_context_view, second_context_view], target_views + + +class DUST3RSplattingTestDataset(torch.utils.data.Dataset): + + def __init__(self, data, samples, resolution): + + self.data = data + self.samples = samples + + self.resolution = resolution + self.transform = ImgNorm + self.org_transform = torchvision.transforms.ToTensor() + + def get_view(self, sequence, c_view): + + view = self.data.get_view(sequence, c_view, self.resolution) + + # Transform the input + view['img'] = self.transform(view['original_img']) + view['original_img'] = self.org_transform(view['original_img']) + + # Create the point cloud and validity mask + pts3d, valid_mask = depthmap_to_absolute_camera_coordinates(**view) + view['pts3d'] = pts3d + view['valid_mask'] = valid_mask & np.isfinite(pts3d).all(axis=-1) + assert view['valid_mask'].any(), f"Invalid mask for sequence: {sequence}, view: {c_view}" + + return view + + def __getitem__(self, idx): + + sequence, c_view_1, c_view_2, target_view = self.samples[idx] + c_view_1, c_view_2, target_view = int(c_view_1), int(c_view_2), int(target_view) + fetched_c_view_1 = self.get_view(sequence, c_view_1) + fetched_c_view_2 = self.get_view(sequence, c_view_2) + fetched_target_view = self.get_view(sequence, target_view) + + views = {"context": [fetched_c_view_1, fetched_c_view_2], "target": [fetched_target_view], "scene": sequence} + + return views + + def __len__(self): + + return len(self.samples) diff --git a/data/scannetpp/scannetpp.py b/data/scannetpp/scannetpp.py new file mode 100644 index 0000000000000000000000000000000000000000..3ba8e047da9d56b9f1b04e3375e53b7628124d1b --- /dev/null +++ b/data/scannetpp/scannetpp.py @@ -0,0 +1,187 @@ +import json +import logging +import os +import sys + +import cv2 +import numpy as np + +# Add dust3r to the sys.path +sys.path.append('src/dust3r_src') +from data.data import crop_resize_if_necessary, DUST3RSplattingDataset, DUST3RSplattingTestDataset +from src.mast3r_src.dust3r.dust3r.utils.image import imread_cv2 + +logger = logging.getLogger(__name__) + + +class ScanNetPPData(): + + def __init__(self, root, stage): + + self.root = root + self.stage = stage + self.png_depth_scale = 1000.0 + + # Dictionaries to store the data for each scene + self.color_paths = {} + self.depth_paths = {} + self.intrinsics = {} + self.c2ws = {} + + # Fetch the sequences to use + if stage == "train": + sequence_file = os.path.join(self.root, "raw", "splits", "nvs_sem_train.txt") + bad_scenes = ['303745abc7'] + elif stage == "val" or stage == "test": + sequence_file = os.path.join(self.root, "raw", "splits", "nvs_sem_val.txt") + bad_scenes = ['cc5237fd77'] + with open(sequence_file, "r") as f: + self.sequences = f.read().splitlines() + + # Remove scenes that have frames with no valid depths + logger.info(f"Removing scenes that have frames with no valid depths: {bad_scenes}") + self.sequences = [s for s in self.sequences if s not in bad_scenes] + + P = np.array([ + [1, 0, 0, 0], + [0, -1, 0, 0], + [0, 0, -1, 0], + [0, 0, 0, 1]] + ).astype(np.float32) + + # Collect information for every sequence + scenes_with_no_good_frames = [] + for sequence in self.sequences: + + input_raw_folder = os.path.join(self.root, 'raw', 'data', sequence) + input_processed_folder = os.path.join(self.root, 'processed', sequence) + + # Load Train & Test Splits + frame_file = os.path.join(input_raw_folder, "dslr", "train_test_lists.json") + with open(frame_file, "r") as f: + train_test_list = json.load(f) + + # Camera Metadata + cams_metadata_path = f"{input_processed_folder}/dslr/nerfstudio/transforms_undistorted.json" + with open(cams_metadata_path, "r") as f: + cams_metadata = json.load(f) + + # Load the nerfstudio/transforms.json file to check whether each image is blurry + nerfstudio_transforms_path = f"{input_raw_folder}/dslr/nerfstudio/transforms.json" + with open(nerfstudio_transforms_path, "r") as f: + nerfstudio_transforms = json.load(f) + + # Create a reverse mapping from image name to the frame information and nerfstudio transform + # (as transforms_undistorted.json does not store the frames in the same order as train_test_lists.json) + file_path_to_frame_metadata = {} + file_path_to_nerfstudio_transform = {} + for frame in cams_metadata["frames"]: + file_path_to_frame_metadata[frame["file_path"]] = frame + for frame in nerfstudio_transforms["frames"]: + file_path_to_nerfstudio_transform[frame["file_path"]] = frame + + # Fetch the pose for every frame + sequence_color_paths = [] + sequence_depth_paths = [] + sequence_c2ws = [] + for train_file_name in train_test_list["train"]: + is_bad = file_path_to_nerfstudio_transform[train_file_name]["is_bad"] + if is_bad: + continue + sequence_color_paths.append(f"{input_processed_folder}/dslr/undistorted_images/{train_file_name}") + sequence_depth_paths.append(f"{input_processed_folder}/dslr/undistorted_depths/{train_file_name.replace('.JPG', '.png')}") + frame_metadata = file_path_to_frame_metadata[train_file_name] + c2w = np.array(frame_metadata["transform_matrix"], dtype=np.float32) + c2w = P @ c2w @ P.T + sequence_c2ws.append(c2w) + + if len(sequence_color_paths) == 0: + logger.info(f"No good frames for sequence: {sequence}") + scenes_with_no_good_frames.append(sequence) + continue + + # Get the intrinsics data for the frame + K = np.eye(4, dtype=np.float32) + K[0, 0] = cams_metadata["fl_x"] + K[1, 1] = cams_metadata["fl_y"] + K[0, 2] = cams_metadata["cx"] + K[1, 2] = cams_metadata["cy"] + + self.color_paths[sequence] = sequence_color_paths + self.depth_paths[sequence] = sequence_depth_paths + self.c2ws[sequence] = sequence_c2ws + self.intrinsics[sequence] = K + + # Remove scenes with no good frames + self.sequences = [s for s in self.sequences if s not in scenes_with_no_good_frames] + + def get_view(self, sequence, view_idx, resolution): + + # RGB Image + rgb_path = self.color_paths[sequence][view_idx] + rgb_image = imread_cv2(rgb_path) + + # Depthmap + depth_path = self.depth_paths[sequence][view_idx] + depthmap = imread_cv2(depth_path, cv2.IMREAD_UNCHANGED) + depthmap = depthmap.astype(np.float32) + depthmap = depthmap / self.png_depth_scale + + # C2W Pose + c2w = self.c2ws[sequence][view_idx] + + # Camera Intrinsics + intrinsics = self.intrinsics[sequence] + + # Resize + rgb_image, depthmap, intrinsics = crop_resize_if_necessary( + rgb_image, depthmap, intrinsics, resolution + ) + + view = { + 'original_img': rgb_image, + 'depthmap': depthmap, + 'camera_pose': c2w, + 'camera_intrinsics': intrinsics, + 'dataset': 'scannet++', + 'label': f"scannet++/{sequence}", + 'instance': f'{view_idx}', + 'is_metric_scale': True, + 'sky_mask': depthmap <= 0.0, + } + return view + + +def get_scannet_dataset(root, stage, resolution, num_epochs_per_epoch=1): + + data = ScanNetPPData(root, stage) + + coverage = {} + for sequence in data.sequences: + with open(f'./data/scannetpp/coverage/{sequence}.json', 'r') as f: + sequence_coverage = json.load(f) + coverage[sequence] = sequence_coverage[sequence] + + dataset = DUST3RSplattingDataset( + data, + coverage, + resolution, + num_epochs_per_epoch=num_epochs_per_epoch, + ) + + return dataset + + +def get_scannet_test_dataset(root, alpha, beta, resolution, use_every_n_sample=100): + + data = ScanNetPPData(root, 'val') + + samples_file = f'data/scannetpp/test_set_{alpha}_{beta}.json' + print(f"Loading samples from: {samples_file}") + with open(samples_file, 'r') as f: + samples = json.load(f) + samples = samples[::use_every_n_sample] + + dataset = DUST3RSplattingTestDataset(data, samples, resolution) + + return dataset diff --git a/demo.py b/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..efcb9a325683e2935bd2d2b7fc921fc459cf5cdb --- /dev/null +++ b/demo.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# The MASt3R Gradio demo, modified for predicting 3D Gaussian Splats + +# --- Original License --- +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +import functools +import os +import sys +import tempfile + +import gradio +import torch + +sys.path.append('src/mast3r_src') +sys.path.append('src/mast3r_src/dust3r') +sys.path.append('src/pixelsplat_src') +from dust3r.utils.image import load_images +from mast3r.utils.misc import hash_md5 +import main +import utils.export as export + + +def get_reconstructed_scene(outdir, model, device, silent, image_size, ios_mode, filelist): + + if ios_mode: + filelist = [f[0] for f in filelist] + if len(filelist) == 1: + filelist = [filelist[0], filelist[0]] + assert len(filelist) == 2, "Please provide two images" + imgs = load_images(filelist, size=image_size, verbose=not silent) + + for img in imgs: + img['img'] = img['img'].to(device) + img['original_img'] = img['original_img'].to(device) + img['true_shape'] = torch.from_numpy(img['true_shape']) + + output = model(imgs[0], imgs[1]) + + pred1, pred2 = output + plyfile = os.path.join(outdir, 'gaussians.ply') + export.save_as_ply(pred1, pred2, plyfile) + return plyfile + +if __name__ == '__main__': + + weights_path = sys.argv[1] + + image_size = 512 + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + server_name = '127.0.0.1' + server_port = None + share = True + silent = False + ios_mode = True + + model = main.MAST3RGaussians.load_from_checkpoint(weights_path, device) + chkpt_tag = hash_md5(weights_path) + + # Define example inputs and their corresponding precalculated outputs + examples = [ + ["assets/demo_examples/scannet++_1_img_1.jpg", "assets/demo_examples/scannet++_1_img_2.jpg", "assets/demo_examples/scannet++_1.ply"], + ["assets/demo_examples/scannet++_2_img_1.jpg", "assets/demo_examples/scannet++_2_img_2.jpg", "assets/demo_examples/scannet++_2.ply"], + ["assets/demo_examples/scannet++_3_img_1.jpg", "assets/demo_examples/scannet++_3_img_2.jpg", "assets/demo_examples/scannet++_3.ply"], + ["assets/demo_examples/scannet++_4_img_1.jpg", "assets/demo_examples/scannet++_4_img_2.jpg", "assets/demo_examples/scannet++_4.ply"], + ["assets/demo_examples/scannet++_5_img_1.jpg", "assets/demo_examples/scannet++_5_img_2.jpg", "assets/demo_examples/scannet++_5.ply"], + ["assets/demo_examples/scannet++_6_img_1.jpg", "assets/demo_examples/scannet++_6_img_2.jpg", "assets/demo_examples/scannet++_6.ply"], + ["assets/demo_examples/scannet++_7_img_1.jpg", "assets/demo_examples/scannet++_7_img_2.jpg", "assets/demo_examples/scannet++_7.ply"], + ["assets/demo_examples/scannet++_8_img_1.jpg", "assets/demo_examples/scannet++_8_img_2.jpg", "assets/demo_examples/scannet++_8.ply"], + ["assets/demo_examples/in_the_wild_1_img_1.jpg", "assets/demo_examples/in_the_wild_1_img_2.jpg", "assets/demo_examples/in_the_wild_1.ply"], + ["assets/demo_examples/in_the_wild_2_img_1.jpg", "assets/demo_examples/in_the_wild_2_img_2.jpg", "assets/demo_examples/in_the_wild_2.ply"], + ["assets/demo_examples/in_the_wild_3_img_1.jpg", "assets/demo_examples/in_the_wild_3_img_2.jpg", "assets/demo_examples/in_the_wild_3.ply"], + ["assets/demo_examples/in_the_wild_4_img_1.jpg", "assets/demo_examples/in_the_wild_4_img_2.jpg", "assets/demo_examples/in_the_wild_4.ply"], + ["assets/demo_examples/in_the_wild_5_img_1.jpg", "assets/demo_examples/in_the_wild_5_img_2.jpg", "assets/demo_examples/in_the_wild_5.ply"], + ["assets/demo_examples/in_the_wild_6_img_1.jpg", "assets/demo_examples/in_the_wild_6_img_2.jpg", "assets/demo_examples/in_the_wild_6.ply"], + ["assets/demo_examples/in_the_wild_7_img_1.jpg", "assets/demo_examples/in_the_wild_7_img_2.jpg", "assets/demo_examples/in_the_wild_7.ply"], + ["assets/demo_examples/in_the_wild_8_img_1.jpg", "assets/demo_examples/in_the_wild_8_img_2.jpg", "assets/demo_examples/in_the_wild_8.ply"], + ] + + with tempfile.TemporaryDirectory(suffix='_mast3r_gradio_demo') as tmpdirname: + + cache_path = os.path.join(tmpdirname, chkpt_tag) + os.makedirs(cache_path, exist_ok=True) + + recon_fun = functools.partial(get_reconstructed_scene, tmpdirname, model, device, silent, image_size, ios_mode) + + if not ios_mode: + for i in range(len(examples)): + examples[i].insert(2, (examples[i][0], examples[i][1])) + + css = """.gradio-container {margin: 0 !important; min-width: 100%};""" + with gradio.Blocks(css=css, title="Splatt3R Demo") as demo: + + gradio.HTML('

Splatt3R Demo

') + + with gradio.Column(): + gradio.Markdown(''' + Please upload exactly one or two images below to be used for reconstruction. + If non-square images are uploaded, they will be cropped to squares for reconstruction. + ''') + if ios_mode: + inputfiles = gradio.Gallery(type="filepath") + else: + inputfiles = gradio.File(file_count="multiple") + run_btn = gradio.Button("Run") + gradio.Markdown(''' + ## Output + Below we show the generated 3D Gaussian Splat. + There may be a short delay as the reconstruction needs to be downloaded before rendering. + The arrow in the top right of the window below can be used to download the .ply for rendering with other viewers, + such as [here](https://projects.markkellogg.org/threejs/demo_gaussian_splats_3d.php?art=1&cu=0,-1,0&cp=0,1,0&cla=1,0,0&aa=false&2d=false&sh=0) or [here](https://playcanvas.com/supersplat/editor) + ''') + outmodel = gradio.Model3D( + clear_color=[1.0, 1.0, 1.0, 0.0], + ) + run_btn.click(fn=recon_fun, inputs=[inputfiles], outputs=[outmodel]) + + gradio.Markdown(''' + ## Examples + A gallery of examples generated from ScanNet++ and from 'in the wild' images taken with a mobile phone. + ''') + + snapshot_1 = gradio.Image(None, visible=False) + snapshot_2 = gradio.Image(None, visible=False) + if ios_mode: + gradio.Examples( + examples=examples, + inputs=[snapshot_1, snapshot_2, outmodel], + examples_per_page=5 + ) + else: + gradio.Examples( + examples=examples, + inputs=[snapshot_1, snapshot_2, inputfiles, outmodel], + examples_per_page=5 + ) + + demo.launch(share=share, server_name=server_name, server_port=server_port) diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000000000000000000000000000000000000..dc58137055484ded952aeee072aa137fcac5394d --- /dev/null +++ b/environment.yml @@ -0,0 +1,453 @@ +name: mast3r +channels: + - anaconda + - pytorch + - nvidia + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - _sysroot_linux-64_curr_repodata_hack=3=haa98f57_10 + - aiohttp=3.9.5=py311h5eee18b_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - ansi2html=1.9.1=py311h06a4308_0 + - antlr-python-runtime=4.9.3=pyhd8ed1ab_1 + - aom=3.9.1=hac33072_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - assimp=5.4.1=h8343317_0 + - binutils=2.38=h1680402_1 + - binutils_impl_linux-64=2.38=h2a08ee3_1 + - binutils_linux-64=2.38.0=hc2dff05_0 + - blas=1.0=mkl + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.5=hc2324a3_1 + - brotli=1.0.9=he6710b0_2 + - brotli-bin=1.1.0=hd590300_1 + - brotli-python=1.0.9=py311h6a678d5_8 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h5eee18b_6 + - c-ares=1.32.3=h4bc722e_0 + - c-blosc2=2.14.4=hb4ffafa_1 + - ca-certificates=2024.7.4=hbcca054_0 + - cccl=2.4.0=h7ab4013_0 + - certifi=2024.7.4=py311h06a4308_0 + - cfitsio=3.470=h5893167_7 + - charls=2.4.2=h59595ed_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - cmake=3.14.0=h52cb24c_0 + - colorama=0.4.6=pyhd8ed1ab_0 + - cuda=12.1.0=0 + - cuda-cccl=12.5.39=ha770c72_0 + - cuda-cccl_linux-64=12.5.39=ha770c72_0 + - cuda-command-line-tools=12.1.1=0 + - cuda-compiler=12.4.1=h6a678d5_1 + - cuda-cudart=12.1.105=0 + - cuda-cudart-dev=12.1.105=0 + - cuda-cudart-static=12.5.82=0 + - cuda-cudart-static_linux-64=12.5.82=0 + - cuda-cuobjdump=12.4.127=0 + - cuda-cupti=12.1.105=0 + - cuda-cuxxfilt=12.4.127=0 + - cuda-demo-suite=12.4.127=0 + - cuda-documentation=12.4.127=0 + - cuda-driver-dev=12.5.82=0 + - cuda-driver-dev_linux-64=12.5.82=h85509e4_0 + - cuda-gdb=12.5.82=0 + - cuda-libraries=12.1.0=0 + - cuda-libraries-dev=12.5.1=0 + - cuda-libraries-static=12.5.1=ha770c72_0 + - cuda-nsight=12.5.82=ha770c72_0 + - cuda-nsight-compute=12.4.1=0 + - cuda-nvcc=12.4.131=0 + - cuda-nvdisasm=12.5.39=0 + - cuda-nvml-dev=12.5.82=0 + - cuda-nvprof=12.5.82=0 + - cuda-nvprune=12.4.127=0 + - cuda-nvrtc=12.1.105=0 + - cuda-nvrtc-dev=12.1.105=0 + - cuda-nvrtc-static=12.5.82=0 + - cuda-nvtx=12.1.105=0 + - cuda-nvvp=12.5.82=0 + - cuda-opencl=12.5.39=0 + - cuda-opencl-dev=12.5.39=0 + - cuda-profiler-api=12.5.39=ha770c72_0 + - cuda-runtime=12.1.0=0 + - cuda-sanitizer-api=12.5.81=0 + - cuda-toolkit=12.1.0=0 + - cuda-tools=12.1.1=ha770c72_0 + - cuda-version=12.5=3 + - cuda-visual-tools=12.1.1=0 + - cytoolz=0.12.2=py311h5eee18b_0 + - dash=2.14.2=py311h06a4308_0 + - dask-core=2024.5.0=py311h06a4308_0 + - dav1d=1.2.1=h5eee18b_0 + - dbus=1.13.18=hb2f20db_0 + - docker-pycreds=0.4.0=pyhd3eb1b0_0 + - double-conversion=3.3.0=h59595ed_0 + - eigen=3.4.0=hdb19cb5_0 + - elfutils=0.189=hde5d1a3_0 + - embree=3.13.0=habf647b_1 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=4.3=hf484d3e_0 + - filelock=3.13.1=py311h06a4308_0 + - flask=3.0.3=py311h06a4308_0 + - flask-compress=1.13=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_1 + - fontconfig=2.14.2=h14ed4e7_0 + - freetype=2.12.1=h4a9f257_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2024.6.1=pyhff2d567_0 + - gcc_impl_linux-64=11.2.0=h1234567_1 + - gcc_linux-64=11.2.0=h5c386dc_0 + - gds-tools=1.6.1.9=0 + - giflib=5.2.2=hd590300_0 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.43=py311h06a4308_0 + - gl2ps=1.4.2=h70c0345_1 + - glew=2.1.0=h295c915_3 + - glfw=3.4=hd590300_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.6.15=he1e5248_0 + - gxx_impl_linux-64=11.2.0=h1234567_1 + - gxx_linux-64=11.2.0=hc2dff05_0 + - hdf4=4.2.15=h2a13503_7 + - hdf5=1.14.3=nompi_hdf9ad27_105 + - icu=73.2=h59595ed_0 + - idna=3.7=py311h06a4308_0 + - imagecodecs=2024.6.1=py311h60053b1_0 + - importlib-metadata=7.0.1=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - itsdangerous=2.2.0=py311h06a4308_0 + - jinja2=3.1.4=py311h06a4308_0 + - jsoncpp=1.9.5=h4bd325d_1 + - jxrlib=1.1=h7b6447c_2 + - kernel-headers_linux-64=3.10.0=h57e8cba_10 + - keyutils=1.6.1=h166bdaf_0 + - krb5=1.21.3=h659f571_0 + - lame=3.100=h7b6447c_0 + - lazy_loader=0.4=py311h06a4308_0 + - lcms2=2.16=hb7c19ff_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=4.0.0=h27087fc_0 + - libabseil=20240116.2=cxx17_h6a678d5_0 + - libaec=1.1.3=h59595ed_0 + - libarchive=3.6.2=h039dbb9_1 + - libavif=1.1.0=h9b56c87_0 + - libavif16=1.1.0=h9b56c87_0 + - libblas=3.9.0=1_h86c2bf4_netlib + - libboost=1.84.0=hba137d9_3 + - libbrotlicommon=1.1.0=hd590300_1 + - libbrotlidec=1.1.0=hd590300_1 + - libbrotlienc=1.1.0=hd590300_1 + - libcblas=3.9.0=6_ha36c22a_netlib + - libcublas=12.1.0.26=0 + - libcublas-dev=12.1.0.26=0 + - libcublas-static=12.5.3.2=0 + - libcufft=11.0.2.4=0 + - libcufft-dev=11.0.2.4=0 + - libcufft-static=11.2.3.61=0 + - libcufile=1.10.1.7=0 + - libcufile-dev=1.10.1.7=0 + - libcufile-static=1.10.1.7=0 + - libcurand=10.3.6.82=0 + - libcurand-dev=10.3.6.82=0 + - libcurand-static=10.3.6.82=0 + - libcurl=8.8.0=hca28451_1 + - libcusolver=11.4.4.55=0 + - libcusolver-dev=11.4.4.55=0 + - libcusolver-static=11.6.3.83=0 + - libcusparse=12.0.2.55=0 + - libcusparse-dev=12.0.2.55=0 + - libcusparse-static=12.5.1.3=0 + - libdeflate=1.20=hd590300_0 + - libdrm=2.4.122=h4ab18f5_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_1 + - libgcc-devel_linux-64=11.2.0=h1234567_1 + - libgcc-ng=14.1.0=h77fa898_0 + - libgfortran-ng=14.1.0=h69a702a_0 + - libgfortran5=14.1.0=hc5f4f2c_0 + - libglib=2.78.4=hdc74915_0 + - libglu=9.0.0=hf484d3e_1 + - libgomp=14.1.0=h77fa898_0 + - libhwloc=2.11.1=default_hecaa2ac_1000 + - libhwy=1.1.0=h00ab1b0_0 + - libiconv=1.16=h5eee18b_3 + - libidn2=2.3.4=h5eee18b_0 + - libjpeg-turbo=3.0.3=h5eee18b_0 + - libjxl=0.10.3=h66b40c8_0 + - liblapack=3.9.0=6_ha36c22a_netlib + - liblapacke=3.9.0=6_ha36c22a_netlib + - libllvm17=17.0.6=hc9c083f_0 + - liblzf=3.6=hd590300_0 + - libmicrohttpd=0.9.76=h5eee18b_0 + - libnetcdf=4.9.2=nompi_h135f659_114 + - libnghttp2=1.58.0=h47da74e_1 + - libnpp=12.0.2.50=0 + - libnpp-dev=12.0.2.50=0 + - libnpp-static=12.3.0.159=0 + - libnsl=2.0.1=hd590300_0 + - libnvfatbin=12.5.82=0 + - libnvfatbin-dev=12.5.82=0 + - libnvfatbin-static=12.5.82=0 + - libnvjitlink=12.1.105=0 + - libnvjitlink-dev=12.1.105=0 + - libnvjitlink-static=12.5.82=0 + - libnvjpeg=12.1.1.14=0 + - libnvjpeg-dev=12.1.1.14=0 + - libnvjpeg-static=12.3.2.81=ha770c72_0 + - libnvvm-samples=12.1.105=0 + - libogg=1.3.5=h27cfd23_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.43=h2797004_0 + - libprotobuf=4.25.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsqlite=3.46.0=hde9e2c9_0 + - libssh2=1.11.0=h251f7ec_0 + - libstdcxx-devel_linux-64=11.2.0=h1234567_1 + - libstdcxx-ng=14.1.0=hc0a3c3a_0 + - libtasn1=4.19.0=h5eee18b_0 + - libtheora=1.1.1=h7f8727e_3 + - libtiff=4.6.0=h1dd3fc0_3 + - libunistring=0.9.10=h27cfd23_0 + - libuuid=2.38.1=h0b41bf4_0 + - libvorbis=1.3.7=h7b6447c_0 + - libwebp-base=1.4.0=hd590300_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.7.0=h662e7e4_0 + - libxml2=2.13.1=hfdd30dd_1 + - libzip=1.10.1=h2629f0a_3 + - libzlib=1.2.13=h4ab18f5_6 + - libzopfli=1.0.3=he6710b0_0 + - lightning=2.3.2=pyhd8ed1ab_0 + - lightning-utilities=0.11.3.post0=pyhd8ed1ab_0 + - llvm-openmp=14.0.6=h9e868ea_0 + - locket=1.0.0=py311h06a4308_0 + - loguru=0.5.3=py311h06a4308_4 + - lz4-c=1.9.4=h6a678d5_1 + - lzo=2.10=h7b6447c_2 + - markupsafe=2.1.3=py311h5eee18b_0 + - mesalib=23.3.2=h6b56f8e_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.7.3=hbbd107a_1 + - networkx=3.3=py311h06a4308_0 + - nlohmann_json=3.11.2=h6a678d5_0 + - nsight-compute=2024.2.0.16=2 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - omegaconf=2.3.0=pyhd8ed1ab_0 + - open3d=0.18.0=py311hcec1c9b_3 + - openh264=2.1.1=h4ff587b_0 + - openjpeg=2.5.2=h488ebb8_0 + - openssl=3.3.1=h4ab18f5_1 + - packaging=24.1=pyhd8ed1ab_0 + - partd=1.4.1=py311h06a4308_0 + - pathtools=0.1.2=pyhd3eb1b0_1 + - pcre2=10.42=hebb0a14_1 + - pillow=10.3.0=py311h18e6fac_0 + - pip=24.0=py311h06a4308_0 + - plotly=5.22.0=py311h92b7b1e_0 + - plyfile=1.0.3=pyhd8ed1ab_0 + - proj=9.3.1=he5811b7_0 + - protobuf=4.25.3=py311h12ddb61_0 + - psutil=5.9.0=py311h5eee18b_0 + - pugixml=1.14=h59595ed_0 + - pybind11-abi=4=hd3eb1b0_1 + - pysocks=1.7.1=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python_abi=3.11=4_cp311 + - pytorch=2.3.1=py3.11_cuda12.1_cudnn8.9.2_0 + - pytorch-cuda=12.1=ha16c6d3_5 + - pytorch-lightning=2.3.3=pyhd8ed1ab_0 + - pytorch-mutex=1.0=cuda + - pywavelets=1.5.0=py311hf4808d0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - qhull=2020.2=hdb19cb5_2 + - rav1e=0.6.6=he8a937b_2 + - readline=8.2=h5eee18b_0 + - requests=2.32.2=py311h06a4308_0 + - retrying=1.3.3=pyhd3eb1b0_2 + - rhash=1.4.3=hdbd6064_0 + - scikit-image=0.20.0=py311h6a678d5_0 + - sentry-sdk=1.9.0=py311h06a4308_0 + - setproctitle=1.2.2=py311h5eee18b_0 + - setuptools=69.5.1=py311h06a4308_0 + - six=1.16.0=pyhd3eb1b0_1 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.2.1=ha2e4443_0 + - sqlite=3.45.3=h5eee18b_0 + - svt-av1=2.1.2=hac33072_0 + - sympy=1.12.1=pyh04b8f61_3 + - sysroot_linux-64=2.17=h57e8cba_10 + - tbb=2021.12.0=h434a139_3 + - tbb-devel=2021.12.0=hfcbfbdb_3 + - tenacity=8.2.3=py311h06a4308_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinyobjloader=1.0.7=h59595ed_2 + - tk=8.6.14=h39e8969_0 + - torchmetrics=1.4.0.post0=pyhd8ed1ab_0 + - torchtriton=2.3.1=py311 + - torchvision=0.18.1=py311_cu121 + - tqdm=4.66.4=pyhd8ed1ab_0 + - typing-extensions=4.11.0=py311h06a4308_0 + - typing_extensions=4.11.0=py311h06a4308_0 + - urllib3=2.2.2=py311h06a4308_0 + - utfcpp=3.2.1=h06a4308_0 + - vtk-base=9.2.6=osmesa_py311h1234567_123 + - wandb=0.16.6=pyhd8ed1ab_0 + - wayland=1.22.0=h8c25dac_1 + - werkzeug=3.0.3=py311h06a4308_0 + - wheel=0.43.0=py311h06a4308_0 + - wslink=2.1.1=pyhd8ed1ab_0 + - xkeyboard-config=2.42=h4ab18f5_0 + - xorg-damageproto=1.2.1=h7f98852_1002 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-glproto=1.4.17=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxau=1.0.11=hd590300_0 + - xorg-libxdamage=1.1.5=h7f98852_1 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxinerama=1.1.5=h27087fc_0 + - xorg-libxrandr=1.5.2=h7f98852_1 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-libxt=1.3.0=hd590300_1 + - xorg-randrproto=1.5.0=h7f98852_1001 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-util-macros=1.19.0=h27cfd23_2 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xf86vidmodeproto=2.3.1=h7f98852_1002 + - xorg-xproto=7.0.31=h27cfd23_1007 + - xz=5.4.6=h5eee18b_1 + - yaml=0.2.5=h7b6447c_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.1=hac33072_1 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=h4ab18f5_6 + - zlib-ng=2.0.7=h5eee18b_0 + - zstd=1.5.6=ha6fb4c9_0 + - pip: + - absl-py==2.1.0 + - aiofiles==23.2.1 + - altair==5.3.0 + - annotated-types==0.7.0 + - anyio==4.4.0 + - attrs==23.2.0 + - bracex==2.4 + - build==1.2.1 + - clarabel==0.9.0 + - contourpy==1.2.1 + - cvxpy==1.5.2 + - cycler==0.12.1 + # - diff-gaussian-rasterization==0.0.0 + - dnspython==2.6.1 + - ecos==2.0.14 + - einops==0.8.0 + - email-validator==2.2.0 + - fastapi==0.111.0 + - fastapi-cli==0.0.4 + - ffmpy==0.3.2 + - fonttools==4.53.1 + - freetype-py==2.4.0 + - gradio==4.37.2 + - gradio-client==1.0.2 + - grpcio==1.64.1 + - h11==0.14.0 + - httpcore==1.0.5 + - httptools==0.6.1 + - httpx==0.27.0 + - huggingface-hub==0.23.4 + - imageio==2.34.2 + - importlib-resources==6.4.0 + - jaxtyping==0.2.33 + - joblib==1.4.2 + - jsonschema==4.23.0 + - jsonschema-specifications==2023.12.1 + - kapture==1.1.10 + - kapture-localization==1.1.10 + - kiwisolver==1.4.5 + - llvmlite==0.43.0 + - lpips==0.1.4 + - markdown==3.6 + - markdown-it-py==3.0.0 + - matplotlib==3.9.1 + - mdurl==0.1.2 + - numba==0.60.0 + - numpy-quaternion==2023.0.4 + - opencv-python==4.10.0.84 + - orjson==3.10.6 + - osqp==0.6.7.post0 + - pandas==2.2.2 + - piexif==1.1.3 + - pillow-heif==0.17.0 + - poselib==2.0.2 + - pycolmap==0.6.1 + - pydantic==2.8.2 + - pydantic-core==2.20.1 + - pydub==0.25.1 + - pyglet==1.5.29 + - pygments==2.18.0 + - pyopengl==3.1.0 + - pyparsing==3.1.2 + - pyproject-hooks==1.1.0 + - pyrender==0.1.45 + - python-dateutil==2.9.0.post0 + - python-dotenv==1.0.1 + - python-multipart==0.0.9 + - pytz==2024.1 + - qdldl==0.1.7.post4 + - referencing==0.35.1 + - rich==13.7.1 + - roma==1.5.0 + - rpds-py==0.19.0 + - ruff==0.5.1 + - safetensors==0.4.3 + - scikit-learn==1.5.1 + - scipy==1.14.0 + - scs==3.2.6 + - semantic-version==2.10.0 + - shellingham==1.5.4 + - sniffio==1.3.1 + - starlette==0.37.2 + - tabulate==0.9.0 + - tensorboard==2.17.0 + - tensorboard-data-server==0.7.2 + - threadpoolctl==3.5.0 + - tomlkit==0.12.0 + - toolz==0.12.1 + - trimesh==4.4.3 + - typeguard==2.13.3 + - typer==0.12.3 + - tzdata==2024.1 + - ujson==5.10.0 + - uvicorn==0.30.1 + - uvloop==0.19.0 + - watchfiles==0.22.0 + - wcmatch==8.5.2 + - websockets==11.0.3 +prefix: /media/brandon/HDD/anaconda3/envs/mast3r diff --git a/main.py b/main.py new file mode 100644 index 0000000000000000000000000000000000000000..61d32e41ac21bac2e303080af8130ed9f0c64c06 --- /dev/null +++ b/main.py @@ -0,0 +1,429 @@ +import json +import os +import sys + +import einops +import lightning as L +import lpips +import omegaconf +import torch +import wandb + +# Add MAST3R and PixelSplat to the sys.path to prevent issues during importing +sys.path.append('src/pixelsplat_src') +sys.path.append('src/mast3r_src') +sys.path.append('src/mast3r_src/dust3r') +from src.mast3r_src.dust3r.dust3r.losses import L21 +from src.mast3r_src.mast3r.losses import ConfLoss, Regr3D +import data.scannetpp.scannetpp as scannetpp +import src.mast3r_src.mast3r.model as mast3r_model +import src.pixelsplat_src.benchmarker as benchmarker +import src.pixelsplat_src.decoder_splatting_cuda as pixelsplat_decoder +import utils.compute_ssim as compute_ssim +import utils.export as export +import utils.geometry as geometry +import utils.loss_mask as loss_mask +import utils.sh_utils as sh_utils +import workspace + + +class MAST3RGaussians(L.LightningModule): + + def __init__(self, config): + + super().__init__() + + # Save the config + self.config = config + + # The encoder which we use to predict the 3D points and Gaussians, + # trained as a modified MAST3R model. The model's configuration is + # primarily defined by the pretrained checkpoint that we load, see + # MASt3R's README.md + self.encoder = mast3r_model.AsymmetricMASt3R( + pos_embed='RoPE100', + patch_embed_cls='ManyAR_PatchEmbed', + img_size=(512, 512), + head_type='gaussian_head', + output_mode='pts3d+gaussian+desc24', + depth_mode=('exp', -mast3r_model.inf, mast3r_model.inf), + conf_mode=('exp', 1, mast3r_model.inf), + enc_embed_dim=1024, + enc_depth=24, + enc_num_heads=16, + dec_embed_dim=768, + dec_depth=12, + dec_num_heads=12, + two_confs=True, + use_offsets=config.use_offsets, + sh_degree=config.sh_degree if hasattr(config, 'sh_degree') else 1 + ) + self.encoder.requires_grad_(False) + self.encoder.downstream_head1.gaussian_dpt.dpt.requires_grad_(True) + self.encoder.downstream_head2.gaussian_dpt.dpt.requires_grad_(True) + + # The decoder which we use to render the predicted Gaussians into + # images, lightly modified from PixelSplat + self.decoder = pixelsplat_decoder.DecoderSplattingCUDA( + background_color=[0.0, 0.0, 0.0] + ) + + self.benchmarker = benchmarker.Benchmarker() + + # Loss criteria + if config.loss.average_over_mask: + self.lpips_criterion = lpips.LPIPS('vgg', spatial=True) + else: + self.lpips_criterion = lpips.LPIPS('vgg') + + if config.loss.mast3r_loss_weight is not None: + self.mast3r_criterion = ConfLoss(Regr3D(L21, norm_mode='?avg_dis'), alpha=0.2) + self.encoder.downstream_head1.requires_grad_(True) + self.encoder.downstream_head2.requires_grad_(True) + + self.save_hyperparameters() + + def forward(self, view1, view2): + + # Freeze the encoder and decoder + with torch.no_grad(): + (shape1, shape2), (feat1, feat2), (pos1, pos2) = self.encoder._encode_symmetrized(view1, view2) + dec1, dec2 = self.encoder._decoder(feat1, pos1, feat2, pos2) + + # Train the downstream heads + pred1 = self.encoder._downstream_head(1, [tok.float() for tok in dec1], shape1) + pred2 = self.encoder._downstream_head(2, [tok.float() for tok in dec2], shape2) + + pred1['covariances'] = geometry.build_covariance(pred1['scales'], pred1['rotations']) + pred2['covariances'] = geometry.build_covariance(pred2['scales'], pred2['rotations']) + + learn_residual = True + if learn_residual: + new_sh1 = torch.zeros_like(pred1['sh']) + new_sh2 = torch.zeros_like(pred2['sh']) + new_sh1[..., 0] = sh_utils.RGB2SH(einops.rearrange(view1['original_img'], 'b c h w -> b h w c')) + new_sh2[..., 0] = sh_utils.RGB2SH(einops.rearrange(view2['original_img'], 'b c h w -> b h w c')) + pred1['sh'] = pred1['sh'] + new_sh1 + pred2['sh'] = pred2['sh'] + new_sh2 + + # Update the keys to make clear that pts3d and means are in view1's frame + pred2['pts3d_in_other_view'] = pred2.pop('pts3d') + pred2['means_in_other_view'] = pred2.pop('means') + + return pred1, pred2 + + def training_step(self, batch, batch_idx): + + _, _, h, w = batch["context"][0]["img"].shape + view1, view2 = batch['context'] + + # Predict using the encoder/decoder and calculate the loss + pred1, pred2 = self.forward(view1, view2) + color, _ = self.decoder(batch, pred1, pred2, (h, w)) + + # Calculate losses + mask = loss_mask.calculate_loss_mask(batch) + loss, mse, lpips = self.calculate_loss( + batch, view1, view2, pred1, pred2, color, mask, + apply_mask=self.config.loss.apply_mask, + average_over_mask=self.config.loss.average_over_mask, + calculate_ssim=False + ) + + # Log losses + self.log_metrics('train', loss, mse, lpips) + return loss + + def validation_step(self, batch, batch_idx): + + _, _, h, w = batch["context"][0]["img"].shape + view1, view2 = batch['context'] + + # Predict using the encoder/decoder and calculate the loss + pred1, pred2 = self.forward(view1, view2) + color, _ = self.decoder(batch, pred1, pred2, (h, w)) + + # Calculate losses + mask = loss_mask.calculate_loss_mask(batch) + loss, mse, lpips = self.calculate_loss( + batch, view1, view2, pred1, pred2, color, mask, + apply_mask=self.config.loss.apply_mask, + average_over_mask=self.config.loss.average_over_mask, + calculate_ssim=False + ) + + # Log losses + self.log_metrics('val', loss, mse, lpips) + return loss + + def test_step(self, batch, batch_idx): + + _, _, h, w = batch["context"][0]["img"].shape + view1, view2 = batch['context'] + num_targets = len(batch['target']) + + # Predict using the encoder/decoder and calculate the loss + with self.benchmarker.time("encoder"): + pred1, pred2 = self.forward(view1, view2) + with self.benchmarker.time("decoder", num_calls=num_targets): + color, _ = self.decoder(batch, pred1, pred2, (h, w)) + + # Calculate losses + mask = loss_mask.calculate_loss_mask(batch) + loss, mse, lpips, ssim = self.calculate_loss( + batch, view1, view2, pred1, pred2, color, mask, + apply_mask=self.config.loss.apply_mask, + average_over_mask=self.config.loss.average_over_mask, + calculate_ssim=True + ) + + # Log losses + self.log_metrics('test', loss, mse, lpips, ssim=ssim) + return loss + + def on_test_end(self): + benchmark_file_path = os.path.join(self.config.save_dir, "benchmark.json") + self.benchmarker.dump(os.path.join(benchmark_file_path)) + + def calculate_loss(self, batch, view1, view2, pred1, pred2, color, mask, apply_mask=True, average_over_mask=True, calculate_ssim=False): + + target_color = torch.stack([target_view['original_img'] for target_view in batch['target']], dim=1) + predicted_color = color + + if apply_mask: + assert mask.sum() > 0, "There are no valid pixels in the mask!" + target_color = target_color * mask[..., None, :, :] + predicted_color = predicted_color * mask[..., None, :, :] + + flattened_color = einops.rearrange(predicted_color, 'b v c h w -> (b v) c h w') + flattened_target_color = einops.rearrange(target_color, 'b v c h w -> (b v) c h w') + flattened_mask = einops.rearrange(mask, 'b v h w -> (b v) h w') + + # MSE loss + rgb_l2_loss = (predicted_color - target_color) ** 2 + if average_over_mask: + mse_loss = (rgb_l2_loss * mask[:, None, ...]).sum() / mask.sum() + else: + mse_loss = rgb_l2_loss.mean() + + # LPIPS loss + lpips_loss = self.lpips_criterion(flattened_target_color, flattened_color, normalize=True) + if average_over_mask: + lpips_loss = (lpips_loss * flattened_mask[:, None, ...]).sum() / flattened_mask.sum() + else: + lpips_loss = lpips_loss.mean() + + # Calculate the total loss + loss = 0 + loss += self.config.loss.mse_loss_weight * mse_loss + loss += self.config.loss.lpips_loss_weight * lpips_loss + + # MAST3R Loss + if self.config.loss.mast3r_loss_weight is not None: + mast3r_loss = self.mast3r_criterion(view1, view2, pred1, pred2)[0] + loss += self.config.loss.mast3r_loss_weight * mast3r_loss + + # Masked SSIM + if calculate_ssim: + if average_over_mask: + ssim_val = compute_ssim.compute_ssim(flattened_target_color, flattened_color, full=True) + ssim_val = (ssim_val * flattened_mask[:, None, ...]).sum() / flattened_mask.sum() + else: + ssim_val = compute_ssim.compute_ssim(flattened_target_color, flattened_color, full=False) + ssim_val = ssim_val.mean() + return loss, mse_loss, lpips_loss, ssim_val + + return loss, mse_loss, lpips_loss + + def log_metrics(self, prefix, loss, mse, lpips, ssim=None): + values = { + f'{prefix}/loss': loss, + f'{prefix}/mse': mse, + f'{prefix}/psnr': -10.0 * mse.log10(), + f'{prefix}/lpips': lpips, + } + + if ssim is not None: + values[f'{prefix}/ssim'] = ssim + + prog_bar = prefix != 'val' + sync_dist = prefix != 'train' + self.log_dict(values, prog_bar=prog_bar, sync_dist=sync_dist, batch_size=self.config.data.batch_size) + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.encoder.parameters(), lr=self.config.opt.lr) + scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [self.config.opt.epochs // 2], gamma=0.1) + return { + "optimizer": optimizer, + "lr_scheduler": { + "scheduler": scheduler, + "interval": "epoch", + "frequency": 1, + }, + } + + +def run_experiment(config): + + # Set the seed + L.seed_everything(config.seed, workers=True) + + # Set up loggers + os.makedirs(os.path.join(config.save_dir, config.name), exist_ok=True) + loggers = [] + if config.loggers.use_csv_logger: + csv_logger = L.pytorch.loggers.CSVLogger( + save_dir=config.save_dir, + name=config.name + ) + loggers.append(csv_logger) + if config.loggers.use_wandb: + wandb_logger = L.pytorch.loggers.WandbLogger( + project='gaussian_zero', + name=config.name, + save_dir=config.save_dir, + config=omegaconf.OmegaConf.to_container(config), + ) + if wandb.run is not None: + wandb.run.log_code(".") + loggers.append(wandb_logger) + + # Set up profiler + if config.use_profiler: + profiler = L.pytorch.profilers.PyTorchProfiler( + dirpath=config.save_dir, + filename='trace', + export_to_chrome=True, + schedule=torch.profiler.schedule(wait=0, warmup=1, active=3), + on_trace_ready=torch.profiler.tensorboard_trace_handler(config.save_dir), + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA + ], + profile_memory=True, + with_stack=True + ) + else: + profiler = None + + # Model + print('Loading Model') + model = MAST3RGaussians(config) + if config.use_pretrained: + ckpt = torch.load(config.pretrained_mast3r_path) + _ = model.encoder.load_state_dict(ckpt['model'], strict=False) + del ckpt + + # Training Datasets + print(f'Building Datasets') + train_dataset = scannetpp.get_scannet_dataset( + config.data.root, + 'train', + config.data.resolution, + num_epochs_per_epoch=config.data.epochs_per_train_epoch, + ) + data_loader_train = torch.utils.data.DataLoader( + train_dataset, + shuffle=True, + batch_size=config.data.batch_size, + num_workers=config.data.num_workers, + ) + + val_dataset = scannetpp.get_scannet_test_dataset( + config.data.root, + alpha=0.5, + beta=0.5, + resolution=config.data.resolution, + use_every_n_sample=100, + ) + data_loader_val = torch.utils.data.DataLoader( + val_dataset, + shuffle=False, + batch_size=config.data.batch_size, + num_workers=config.data.num_workers, + ) + + # Training + print('Training') + trainer = L.Trainer( + accelerator="gpu", + benchmark=True, + callbacks=[ + L.pytorch.callbacks.LearningRateMonitor(logging_interval='epoch', log_momentum=True), + export.SaveBatchData(save_dir=config.save_dir), + ], + check_val_every_n_epoch=1, + default_root_dir=config.save_dir, + devices=config.devices, + gradient_clip_val=config.opt.gradient_clip_val, + log_every_n_steps=10, + logger=loggers, + max_epochs=config.opt.epochs, + profiler=profiler, + strategy="ddp_find_unused_parameters_true" if len(config.devices) > 1 else "auto", + ) + trainer.fit(model, train_dataloaders=data_loader_train, val_dataloaders=data_loader_val) + + # Testing + original_save_dir = config.save_dir + results = {} + for alpha, beta in ((0.9, 0.9), (0.7, 0.7), (0.5, 0.5), (0.3, 0.3)): + + test_dataset = scannetpp.get_scannet_test_dataset( + config.data.root, + alpha=alpha, + beta=beta, + resolution=config.data.resolution, + use_every_n_sample=10 + ) + data_loader_test = torch.utils.data.DataLoader( + test_dataset, + shuffle=False, + batch_size=config.data.batch_size, + num_workers=config.data.num_workers, + ) + + masking_configs = ((True, False), (True, True)) + for apply_mask, average_over_mask in masking_configs: + + new_save_dir = os.path.join( + original_save_dir, + f'alpha_{alpha}_beta_{beta}_apply_mask_{apply_mask}_average_over_mask_{average_over_mask}' + ) + os.makedirs(new_save_dir, exist_ok=True) + model.config.save_dir = new_save_dir + + L.seed_everything(config.seed, workers=True) + + # Training + trainer = L.Trainer( + accelerator="gpu", + benchmark=True, + callbacks=[export.SaveBatchData(save_dir=config.save_dir),], + default_root_dir=config.save_dir, + devices=config.devices, + log_every_n_steps=10, + strategy="ddp_find_unused_parameters_true" if len(config.devices) > 1 else "auto", + ) + + model.lpips_criterion = lpips.LPIPS('vgg', spatial=average_over_mask) + model.config.loss.apply_mask = apply_mask + model.config.loss.average_over_mask = average_over_mask + res = trainer.test(model, dataloaders=data_loader_test) + results[f"alpha: {alpha}, beta: {beta}, apply_mask: {apply_mask}, average_over_mask: {average_over_mask}"] = res + + # Save the results + save_path = os.path.join(original_save_dir, 'results.json') + with open(save_path, 'w') as f: + json.dump(results, f) + + +if __name__ == "__main__": + + # Setup the workspace (eg. load the config, create a directory for results at config.save_dir, etc.) + config = workspace.load_config(sys.argv[1], sys.argv[2:]) + if os.getenv("LOCAL_RANK", '0') == '0': + config = workspace.create_workspace(config) + + # Run training + run_experiment(config) diff --git a/src/mast3r_src/CHECKPOINTS_NOTICE b/src/mast3r_src/CHECKPOINTS_NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..040aed77ec78156cb6c1af8da1652e13ade10bcd --- /dev/null +++ b/src/mast3r_src/CHECKPOINTS_NOTICE @@ -0,0 +1,1376 @@ +MASt3R +Copyright 2024-present NAVER Corp. + +This project's checkpoints were trained on datasets with separate license terms. +Your use of theses checkpoints is subject to the terms and conditions of the following licenses. + +=== +pretrained model: +DUSt3R: DUSt3R_ViTLarge_BaseDecoder_512_dpt +https://github.com/naver/dust3r + +In particular, from the croco training set: + +3D_Street_View +https://github.com/amir32002/3D_Street_View/blob/master/LICENSE +This dataset is made freely available to academic and non-academic entities for non-commercial purposes such as academic research, teaching, scientific publications, or personal experimentation. Permission is granted to use the data given that you agree: + +1. That the dataset comes "AS IS", without express or implied warranty. Although every effort has been made to ensure accuracy, we do not accept any responsibility for errors or omissions. + +2. That you include a reference to the Dataset in any work that makes use of the dataset. For research papers, cite our publication as listed on our website. + +3. That you do not distribute this dataset or modified versions. It is permissible to distribute derivative works in as far as they are abstract representations of this dataset (such as models trained on it or additional annotations that do not directly include any of our data) and do not allow to recover the dataset or something similar in character. + +4. That you may not use the dataset or any derivative work for commercial purposes as, for example, licensing or selling the data, or using the data with a purpose to procure a commercial gain. +That all rights not expressly granted to you are reserved by us. + +In addition, using the dataset is subject to the following standard terms: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +Indoor Visual Localization datasets (IndoorVL) +https://challenge.naverlabs.com/kapture/GangnamStation_LICENSE.txt +https://challenge.naverlabs.com/kapture/HyundaiDepartmentStore_LICENSE.txt + +LICENSE.txt +Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 (modified ver.) +International Public License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution-NonCommercial-NoDerivatives 4.0 International Public +License ("Public License"). To the extent this Public License may be +interpreted as a contract, You are granted the Licensed Rights in +consideration of Your acceptance of these terms and conditions, and the +Licensor grants You such rights in consideration of benefits the +Licensor receives from making the Licensed Material available under +these terms and conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + c. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + d. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + e. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + f. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + g. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + h. NonCommercial means not primarily intended for or directed towards + commercial advantage or monetary compensation. For purposes of + this Public License, the exchange of the Licensed Material for + other material subject to Copyright and Similar Rights by digital + file-sharing or similar means is NonCommercial provided there is + no payment of monetary compensation in connection with the + exchange. + + i. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + j. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + k. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + l. Research purpose means to publish research achievements in a research paper + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part, for NonCommercial purposes only; and + + b. produce and reproduce, but not Share, Adapted Material + for NonCommercial purposes only. + + c. reproduce and share the Adapted Matrerial, in part, + for Research purposes only. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties, including when + the Licensed Material is used other than for NonCommercial + purposes. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material(including in a research paper), + You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + For the avoidance of doubt, You do not have permission under + this Public License to Share Adapted Material. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database for NonCommercial purposes + only and provided You do not Share Adapted Material; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + +=== +CO3Dv2 + +Creative Commons Attribution-NonCommercial 4.0 International Public +License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution-NonCommercial 4.0 International Public License ("Public +License"). To the extent this Public License may be interpreted as a +contract, You are granted the Licensed Rights in consideration of Your +acceptance of these terms and conditions, and the Licensor grants You +such rights in consideration of benefits the Licensor receives from +making the Licensed Material available under these terms and +conditions. + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + d. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + e. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + f. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + g. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + h. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + i. NonCommercial means not primarily intended for or directed towards + commercial advantage or monetary compensation. For purposes of + this Public License, the exchange of the Licensed Material for + other material subject to Copyright and Similar Rights by digital + file-sharing or similar means is NonCommercial provided there is + no payment of monetary compensation in connection with the + exchange. + + j. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + k. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + l. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part, for NonCommercial purposes only; and + + b. produce, reproduce, and Share Adapted Material for + NonCommercial purposes only. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties, including when + the Licensed Material is used other than for NonCommercial + purposes. + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database for NonCommercial purposes + only; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + +=== +ARKitScenes +Creative Commons Attribution-NonCommercial-ShareAlike 4.0: https://creativecommons.org/licenses/by-nc-sa/4.0/ + +=== +ScanNet++ +https://kaldir.vc.in.tum.de/scannetpp/static/scannetpp-terms-of-use.pdf + +=== +BlendedMVS +Creative Commons Attribution 4.0 International: http://creativecommons.org/licenses/by/4.0/ + +=== +Habitat-Sim +HM3D +https://matterport.com/fr/legal/matterport-end-user-license-agreement-academic-use-model-data + +ScanNet +https://kaldir.vc.in.tum.de/scannet/ScanNet_TOS.pdf + +Replica +Before Facebook Technologies, LLC (“FB”) is able to offer you (“Researcher” or +“You”) access to the Replica Dataset (the “Dataset”), please read the following +agreement (“Agreement”). + +By accessing, and in exchange for receiving permission to access, the Dataset, +Researcher hereby agrees to the following terms and conditions: +1. Researcher may use, modify, improve and/or publish the Dataset only in +connection with a research or educational purpose that is non-commercial or +not-for-profit in nature, and not for any other purpose. +1. Researcher may provide research associates and colleagues with access to the +Dataset provided that they first agree to be bound by these terms and +conditions. +1. Researcher may use the Dataset in the scope of their employment at a +for-profit or commercial entity provided that Researcher complies with Section 1 +of this Agreement. If Researcher is employed by a for-profit or commercial +entity, Researcher's employer shall also be bound by these terms and conditions, +and Researcher hereby represents that they are fully authorized to enter into +this agreement on behalf of such employer. +1. THE DATASET IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL FB OR ANY +CONTRIBUTOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE DATASET OR THE USE OR OTHER DEALINGS IN THE DATASET. +1. The law of the State of California shall apply to all disputes related to +this Dataset. + +ReplicaCAD +Creative Commons Attribution 4.0 International (CC BY 4.0): https://creativecommons.org/licenses/by/4.0/ + +habitat-sim +MIT License + +Copyright (c) Meta Platforms, Inc. and its affiliates. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +=== +MegaDepth +MIT License + +Copyright (c) 2018 Zhengqi Li + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=== +StaticThings3D + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +=== +WildRGB-D +https://github.com/wildrgbd/wildrgbd/ +MIT License + +Copyright (c) 2024 rowdataset + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=== +TartanAir +Creative Commons Attribution 4.0 International License: http://creativecommons.org/licenses/by/4.0/ + +=== +UnrealStereo4K +https://github.com/fabiotosi92/SMD-Nets +MIT License + +Copyright (c) 2021 Fabio Tosi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +=== +Virtual KITTI 2 +Creative Commons Attribution-NonCommercial-ShareAlike 3.0: http://creativecommons.org/licenses/by-nc-sa/3.0/legalcode + +=== +DL3DV +DL3DV-10K Term of use and Creative Commons Attribution-NonCommercial 4.0 International License. + +Terms of Use + +Researcher shall use the Dataset only for non-commercial research and educational purposes. +DL3DV-10K organization makes no representations or warranties regarding the dataset, including but not limited to warranties of non-infringement or fitness for a particular purpose. +Researcher accepts full responsibility for his/her/their use of the Dataset and shall defend and indemnify DL3DV-10K organization, including its members, employees, Trustees, officers and agents, against any and all claims arising from Researcher's use of the Dataset, including but not limited to Researcher's use of any copies of copyrighted 3D models that he/she/they may create from the dataset. +Researcher may provide research associates and colleagues with access to the Dataset, after receiving entity has also agreed to and signed these terms and conditions. Sharing the data otherwise is strictly prohibited. +Following General Data Protection Regulation, Researcher must ensure that they can delete all person-specific data upon request. +DL3DV-10K organization reserves the right to terminate Researcher's access to the Dataset at any time. +If Researcher is employed by a for-profit, commercial entity, Researcher's employer shall also be bound by these terms and conditions, and Researcher hereby represents that he/she/they is/are fully authorized to enter into this agreement on behalf of such employer. +The law of the Indiana State shall apply to all disputes under this agreement. + +Creative Commons Attribution-NonCommercial 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-NonCommercial 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. + +Section 1 -- Definitions. + +a. Adapted Material means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. + +b. Adapter's License means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. + +c. Copyright and Similar Rights means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights. d. Effective Technological Measures means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. + +e. Exceptions and Limitations means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. + +f. Licensed Material means the artistic or literary work, database, or other material to which the Licensor applied this Public License. + +g. Licensed Rights means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. + +h. Licensor means the individual(s) or entity(ies) granting rights under this Public License. + +i. NonCommercial means not primarily intended for or directed towards commercial advantage or monetary compensation. For purposes of this Public License, the exchange of the Licensed Material for other material subject to Copyright and Similar Rights by digital file-sharing or similar means is NonCommercial provided there is no payment of monetary compensation in connection with the exchange. + +j. Share means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. + +k. Sui Generis Database Rights means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. + +l. You means the individual or entity exercising the Licensed Rights under this Public License. Your has a corresponding meaning. + +Section 2 -- Scope. + +a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part, for NonCommercial purposes only; and + + b. produce, reproduce, and Share Adapted Material for + NonCommercial purposes only. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + +b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties, including when + the Licensed Material is used other than for NonCommercial + purposes. + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the following conditions. + +a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: + +a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database for NonCommercial purposes only; + +b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material; and + +c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + +a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + +b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + +c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. + +Section 6 -- Term and Termination. + +a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. + +b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + +c. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. + +d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. + +Section 7 -- Other Terms and Conditions. + +a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. + +b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. + +Section 8 -- Interpretation. + +a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. + +b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. + +c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. + +d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. + +=== +Niantic Map Free Relocalization Dataset License Agreement +This Niantic Map Free Relocalization Dataset License Agreement ("Agreement") is an agreement between you and Niantic, Inc. (“Niantic” or “we”). By downloading or otherwise using Niantic’s Map-Free Relocalization dataset or dataset-derived materials (collectively, the "Dataset") you agree to: + +1. Purpose and Restrictions. You may only use the Dataset only for non-commercial purposes, such as academic research at educational and not-for-profit research institutions, teaching, public demonstrations, and personal experimentation. Non-commercial use expressly excludes any profit-making or commercial activities, including without limitation sale, license, manufacture or development of commercial products, use in commercially-sponsored research, use at a laboratory or other facility owned or controlled (whether in whole or in part) by a commercial entity, provision of consulting service, use for or on behalf of any commercial entity, and use in consulting service, use for or on behalf of any commercial entity, use in research where a commercial party obtains rights to research results or any other benefit. Notwithstanding the foregoing restrictions, you can use this Dataset for publishing comparison results for academic papers, including retraining your models on this Dataset. + +2. License. Subject to this Agreement, Niantic grants you a non-exclusive, non-transferable, non-sublicensable right to download and use the Dataset for the purpose stated in Section 1 of this Agreement. All rights not expressly granted to you in this Agreement are reserved. + +3. Condition of Use. You must not use the Dataset in a way that could diminish, tarnish, or in any way harm Niantic’s reputation or image. + +4. No Warranties. The Dataset comes “as is”, and you will use it at your own risk. Niantic makes no representations or warranties regarding the Dataset, including but not limited to warranties of non-infringement or fitness for a particular purpose. Neither Niantic nor any contributor to the Dataset will be liable for any damages related to the Dataset or this Agreement, including direct, indirect, special, consequential or incidental damages, to the maximum extent the law permits, no matter what legal theory they are based on. We are not obligated to (and will not) provide technical support for the Dataset. + +5. Indemnity. You accept full responsibility for your use of the Dataset and shall defend and indemnify Niantic, including its employees, officers and agents, against any and all claims arising from your use of the Dataset. + +6. Removal. Niantic reserves the right to remove access to the Dataset at any time without cause. If you have downloaded a copy of the Dataset prior to such removal, you may use such a copy subject to this Agreement, but you may not distribute your copy. + +7. Termination. This Agreement will terminate immediately upon your commercial use of the Dataset. + +8. Authorized Representative. If you are employed by a for-profit, commercial entity, your employer shall also be bound by the terms and conditions of this Agreement, and you hereby represent that you are fully authorized to enter into this Agreement on behalf of such employer. + +9. Survivability. Sections 2, 4, 5, 6, 7, 8, 9, and 10 of this Agreement survive the termination of this Agreement. + +10. Misc. This Agreement is governed and construed in all respects in accordance with the laws of the State of California, USA without regard to conflicts of law. If any provision of this Agreement is deemed unenforceable or contrary to law, the rest of this Agreement shall remain in full effect and enforceable. If you do not agree to this Agreement, do not download or use the Dataset. The Dataset is protected by copyright and other intellectual property laws and is licensed, not sold. + +=== +NVIDIA Source Code License for SegFormer + +1. Definitions + +“Licensor” means any person or entity that distributes its Work. + +“Software” means the original work of authorship made available under this License. + +“Work” means the Software and any additions to or derivative works of the Software that are made available under +this License. + +The terms “reproduce,” “reproduction,” “derivative works,” and “distribution” have the meaning as provided under +U.S. copyright law; provided, however, that for the purposes of this License, derivative works shall not include +works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work. + +Works, including the Software, are “made available” under this License by including in or with the Work either +(a) a copyright notice referencing the applicability of this License to the Work, or (b) a copy of this License. + +2. License Grant + +2.1 Copyright Grant. Subject to the terms and conditions of this License, each Licensor grants to you a perpetual, +worldwide, non-exclusive, royalty-free, copyright license to reproduce, prepare derivative works of, publicly +display, publicly perform, sublicense and distribute its Work and any resulting derivative works in any form. + +3. Limitations + +3.1 Redistribution. You may reproduce or distribute the Work only if (a) you do so under this License, (b) you +include a complete copy of this License with your distribution, and (c) you retain without modification any +copyright, patent, trademark, or attribution notices that are present in the Work. + +3.2 Derivative Works. You may specify that additional or different terms apply to the use, reproduction, and +distribution of your derivative works of the Work (“Your Terms”) only if (a) Your Terms provide that the use +limitation in Section 3.3 applies to your derivative works, and (b) you identify the specific derivative works +that are subject to Your Terms. Notwithstanding Your Terms, this License (including the redistribution +requirements in Section 3.1) will continue to apply to the Work itself. + +3.3 Use Limitation. The Work and any derivative works thereof only may be used or intended for use +non-commercially. Notwithstanding the foregoing, NVIDIA and its affiliates may use the Work and any derivative +works commercially. As used herein, “non-commercially” means for research or evaluation purposes only. + +3.4 Patent Claims. If you bring or threaten to bring a patent claim against any Licensor (including any claim, +cross-claim or counterclaim in a lawsuit) to enforce any patents that you allege are infringed by any Work, then +your rights under this License from such Licensor (including the grant in Section 2.1) will terminate immediately. + +3.5 Trademarks. This License does not grant any rights to use any Licensor’s or its affiliates’ names, logos, +or trademarks, except as necessary to reproduce the notices described in this License. + +3.6 Termination. If you violate any term of this License, then your rights under this License (including the +grant in Section 2.1) will terminate immediately. + +4. Disclaimer of Warranty. + +THE WORK IS PROVIDED “AS IS” WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING +WARRANTIES OR CONDITIONS OF M ERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON-INFRINGEMENT. YOU +BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER THIS LICENSE. + +5. Limitation of Liability. + +EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL THEORY, WHETHER IN TORT (INCLUDING +NEGLIGENCE), CONTRACT, OR OTHERWISE SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, +INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF OR RELATED TO THIS LICENSE, THE USE OR +INABILITY TO USE THE WORK (INCLUDING BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, LOST PROFITS OR +DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER COMM ERCIAL DAMAGES OR LOSSES), EVEN IF THE LICENSOR HAS BEEN +ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +=== +CosXL License Agreement + + +STABILITY AI NON-COMMERCIAL RESEARCH COMMUNITY LICENSE AGREEMENT Dated: April 7th, 2024 +By clicking “I Accept” below or by using or distributing any portion or element of the Models, Software, Software Products or Derivative Works, you agree to the terms of this License. If you do not agree to this License, then you do not have any rights to use the Software Products or Derivative Works through this License, and you must immediately cease using the Software Products or Derivative Works. If you are agreeing to be bound by the terms of this License on behalf of your employer or other entity, you represent and warrant to Stability AI that you have full legal authority to bind your employer or such entity to this License. If you do not have the requisite authority, you may not accept the License or access the Software Products or Derivative Works on behalf of your employer or other entity. +"Agreement" means this Stable Non-Commercial Research Community License Agreement. +“AUP” means the Stability AI Acceptable Use Policy available at https://stability.ai/use-policy, as may be updated from time to time. +"Derivative Work(s)” means (a) any derivative work of the Software Products as recognized by U.S. copyright laws and (b) any modifications to a Model, and any other model created which is based on or derived from the Model or the Model’s output. For clarity, Derivative Works do not include the output of any Model. +“Documentation” means any specifications, manuals, documentation, and other written information provided by Stability AI related to the Software. +"Licensee" or "you" means you, or your employer or any other person or entity (if you are entering into this Agreement on such person or entity's behalf), of the age required under applicable laws, rules or regulations to provide legal consent and that has legal authority to bind your employer or such other person or entity if you are entering in this Agreement on their behalf. +“Model(s)" means, collectively, Stability AI’s proprietary models and algorithms, including machine-learning models, trained model weights and other elements of the foregoing, made available under this Agreement. +“Non-Commercial Uses” means exercising any of the rights granted herein for the purpose of research or non-commercial purposes. Non-Commercial Uses does not include any production use of the Software Products or any Derivative Works. +"Stability AI" or "we" means Stability AI Ltd. and its affiliates. + +"Software" means Stability AI’s proprietary software made available under this Agreement. +“Software Products” means the Models, Software and Documentation, individually or in any combination. + + License Rights and Redistribution. + a. Subject to your compliance with this Agreement, the AUP (which is hereby incorporated herein by reference), and the Documentation, Stability AI grants you a non-exclusive, worldwide, non-transferable, non-sublicensable, revocable, royalty free and limited license under Stability AI’s intellectual property or other rights owned or controlled by Stability AI embodied in the Software Products to use, reproduce, distribute, and create Derivative Works of, the Software Products, in each case for Non-Commercial Uses only. + b. You may not use the Software Products or Derivative Works to enable third parties to use the Software Products or Derivative Works as part of your hosted service or via your APIs, whether you are adding substantial additional functionality thereto or not. Merely distributing the Software Products or Derivative Works for download online without offering any related service (ex. by distributing the Models on HuggingFace) is not a violation of this subsection. If you wish to use the Software Products or any Derivative Works for commercial or production use or you wish to make the Software Products or any Derivative Works available to third parties via your hosted service or your APIs, contact Stability AI at https://stability.ai/contact. + c. If you distribute or make the Software Products, or any Derivative Works thereof, available to a third party, the Software Products, Derivative Works, or any portion thereof, respectively, will remain subject to this Agreement and you must (i) provide a copy of this Agreement to such third party, and (ii) retain the following attribution notice within a "Notice" text file distributed as a part of such copies: "This Stability AI Model is licensed under the Stability AI Non-Commercial Research Community License, Copyright (c) Stability AI Ltd. All Rights Reserved.” If you create a Derivative Work of a Software Product, you may add your own attribution notices to the Notice file included with the Software Product, provided that you clearly indicate which attributions apply to the Software Product and you must state in the NOTICE file that you changed the Software Product and how it was modified. + Disclaimer of Warranty. UNLESS REQUIRED BY APPLICABLE LAW, THE SOFTWARE PRODUCTS AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING OR REDISTRIBUTING THE SOFTWARE PRODUCTS, DERIVATIVE WORKS OR ANY OUTPUT OR RESULTS AND ASSUME ANY RISKS ASSOCIATED WITH YOUR USE OF THE SOFTWARE PRODUCTS, DERIVATIVE WORKS AND ANY OUTPUT AND RESULTS. 3. Limitation of Liability. IN NO EVENT WILL STABILITY AI OR ITS AFFILIATES BE LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, ARISING OUT OF THIS AGREEMENT, FOR ANY LOST PROFITS OR ANY DIRECT, INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL, EXEMPLARY OR PUNITIVE DAMAGES, EVEN IF STABILITY AI OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF ANY OF THE FOREGOING. 4. Intellectual Property. + a. No trademark licenses are granted under this Agreement, and in connection with the Software Products or Derivative Works, neither Stability AI nor Licensee may use any name or mark owned by or associated with the other or any of its affiliates, except as required for reasonable and customary use in describing and redistributing the Software Products or Derivative Works. + b. Subject to Stability AI’s ownership of the Software Products and Derivative Works made by or for Stability AI, with respect to any Derivative Works that are made by you, as between you and Stability AI, you are and will be the owner of such Derivative Works + c. If you institute litigation or other proceedings against Stability AI (including a cross-claim or counterclaim in a lawsuit) alleging that the Software Products, Derivative Works or associated outputs or results, or any portion of any of the foregoing, constitutes infringement of intellectual property or other rights owned or licensable by you, then any licenses granted to you under this Agreement shall terminate as of the date such litigation or claim is filed or instituted. You will indemnify and hold harmless Stability AI from and against any claim by any third party arising out of or related to your use or distribution of the Software Products or Derivative Works in violation of this Agreement. + Term and Termination. The term of this Agreement will commence upon your acceptance of this Agreement or access to the Software Products and will continue in full force and effect until terminated in accordance with the terms and conditions herein. Stability AI may terminate this Agreement if you are in breach of any term or condition of this Agreement. Upon termination of this Agreement, you shall delete and cease use of any Software Products or Derivative Works. Sections 2-4 shall survive the termination of this Agreement. + Governing Law. This Agreement will be governed by and construed in accordance with the laws of the United States and the State of California without regard to choice of law + principles. + diff --git a/src/mast3r_src/LICENSE b/src/mast3r_src/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..a97986e3a8ddd49973959f6c748dfa8b881b64d3 --- /dev/null +++ b/src/mast3r_src/LICENSE @@ -0,0 +1,7 @@ +DUSt3R, Copyright (c) 2024-present Naver Corporation, is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 license. + +A summary of the CC BY-NC-SA 4.0 license is located here: + https://creativecommons.org/licenses/by-nc-sa/4.0/ + +The CC BY-NC-SA 4.0 license is located here: + https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode diff --git a/src/mast3r_src/NOTICE b/src/mast3r_src/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..86583416b75cc1749cac38d437b376842975ca06 --- /dev/null +++ b/src/mast3r_src/NOTICE @@ -0,0 +1,103 @@ +MASt3R +Copyright 2024-present NAVER Corp. + +This project contains subcomponents with separate copyright notices and license terms. +Your use of the source code for these subcomponents is subject to the terms and conditions of the following licenses. + +==== + +naver/dust3r +https://github.com/naver/dust3r/ + +Creative Commons Attribution-NonCommercial-ShareAlike 4.0 + +==== + +naver/croco +https://github.com/naver/croco/ + +Creative Commons Attribution-NonCommercial-ShareAlike 4.0 + +==== + +pytorch/pytorch +https://github.com/pytorch/pytorch + +From PyTorch: + +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + +From Caffe2: + +Copyright (c) 2016-present, Facebook Inc. All rights reserved. + +All contributions by Facebook: +Copyright (c) 2016 Facebook Inc. + +All contributions by Google: +Copyright (c) 2015 Google Inc. +All rights reserved. + +All contributions by Yangqing Jia: +Copyright (c) 2015 Yangqing Jia +All rights reserved. + +All contributions by Kakao Brain: +Copyright 2019-2020 Kakao Brain + +All contributions by Cruise LLC: +Copyright (c) 2022 Cruise LLC. +All rights reserved. + +All contributions from Caffe: +Copyright(c) 2013, 2014, 2015, the respective contributors +All rights reserved. + +All other contributions: +Copyright(c) 2015, 2016 the respective contributors +All rights reserved. + +Caffe2 uses a copyright model similar to Caffe: each contributor holds +copyright over their contributions to Caffe2. The project versioning records +all such contribution and copyright details. If a contributor wants to further +mark their specific copyright on a particular contribution, they should +indicate their copyright solely in the commit message of the change when it is +committed. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America + and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + diff --git a/src/mast3r_src/README.md b/src/mast3r_src/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7eef329100d32f76c707bf0c489db115895c274e --- /dev/null +++ b/src/mast3r_src/README.md @@ -0,0 +1,316 @@ +![banner](assets/mast3r.jpg) + +Official implementation of `Grounding Image Matching in 3D with MASt3R` +[[Project page](https://dust3r.europe.naverlabs.com/)], [[MASt3R arxiv](https://arxiv.org/abs/2406.09756)], [[DUSt3R arxiv](https://arxiv.org/abs/2312.14132)] + +![Example of matching results obtained from MASt3R](assets/examples.jpg) + +![High level overview of MASt3R's architecture](assets/mast3r_archi.jpg) + +```bibtex +@misc{mast3r_arxiv24, + title={Grounding Image Matching in 3D with MASt3R}, + author={Vincent Leroy and Yohann Cabon and Jerome Revaud}, + year={2024}, + eprint={2406.09756}, + archivePrefix={arXiv}, + primaryClass={cs.CV} +} + +@inproceedings{dust3r_cvpr24, + title={DUSt3R: Geometric 3D Vision Made Easy}, + author={Shuzhe Wang and Vincent Leroy and Yohann Cabon and Boris Chidlovskii and Jerome Revaud}, + booktitle = {CVPR}, + year = {2024} +} +``` + +## Table of Contents + +- [Table of Contents](#table-of-contents) +- [License](#license) +- [Get Started](#get-started) + - [Installation](#installation) + - [Checkpoints](#checkpoints) + - [Interactive demo](#interactive-demo) + - [Interactive demo with docker](#interactive-demo-with-docker) +- [Usage](#usage) +- [Training](#training) + - [Datasets](#datasets) + - [Demo](#demo) + - [Our Hyperparameters](#our-hyperparameters) +- [Visual Localization](#visual-localization) + - [Dataset Preparation](#dataset-preparation) + - [Example Commands](#example-commands) + +## License + +The code is distributed under the CC BY-NC-SA 4.0 License. +See [LICENSE](LICENSE) for more information. + +```python +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +``` + +## Get Started + +### Installation + +1. Clone MASt3R. +```bash +git clone --recursive https://github.com/naver/mast3r +cd mast3r +# if you have already cloned mast3r: +# git submodule update --init --recursive +``` + +2. Create the environment, here we show an example using conda. +```bash +conda create -n mast3r python=3.11 cmake=3.14.0 +conda activate mast3r +conda install pytorch torchvision pytorch-cuda=12.1 -c pytorch -c nvidia # use the correct version of cuda for your system +pip install -r requirements.txt +pip install -r dust3r/requirements.txt +# Optional: you can also install additional packages to: +# - add support for HEIC images +# - add required packages for visloc.py +pip install -r dust3r/requirements_optional.txt +``` + +3. Optional, compile the cuda kernels for RoPE (as in CroCo v2). +```bash +# DUST3R relies on RoPE positional embeddings for which you can compile some cuda kernels for faster runtime. +cd dust3r/croco/models/curope/ +python setup.py build_ext --inplace +cd ../../../../ +``` + + +### Checkpoints + +You can obtain the checkpoints by two ways: + +1) You can use our huggingface_hub integration: the models will be downloaded automatically. + +2) Otherwise, We provide several pre-trained models: + +| Modelname | Training resolutions | Head | Encoder | Decoder | +|-------------|----------------------|------|---------|---------| +| [`MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric`](https://download.europe.naverlabs.com/ComputerVision/MASt3R/MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric.pth) | 512x384, 512x336, 512x288, 512x256, 512x160 | CatMLP+DPT | ViT-L | ViT-B | + +You can check the hyperparameters we used to train these models in the [section: Our Hyperparameters](#our-hyperparameters) +Make sure to check license of the datasets we used. + +To download a specific model, for example `MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric.pth`: +```bash +mkdir -p checkpoints/ +wget https://download.europe.naverlabs.com/ComputerVision/MASt3R/MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric.pth -P checkpoints/ +``` + +For these checkpoints, make sure to agree to the license of all the training datasets we used, in addition to CC-BY-NC-SA 4.0. +The mapfree dataset license in particular is very restrictive. For more information, check [CHECKPOINTS_NOTICE](CHECKPOINTS_NOTICE). + + +### Interactive demo + +There are two demos available: + +``` +demo.py is the updated demo for MASt3R. It uses our new sparse global alignment method that allows you to reconstruct larger scenes + +python3 demo.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric + +# Use --weights to load a checkpoint from a local file, eg --weights checkpoints/MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric.pth +# Use --local_network to make it accessible on the local network, or --server_name to specify the url manually +# Use --server_port to change the port, by default it will search for an available port starting at 7860 +# Use --device to use a different device, by default it's "cuda" + +demo_dust3r_ga.py is the same demo as in dust3r (+ compatibility for MASt3R models) +see https://github.com/naver/dust3r?tab=readme-ov-file#interactive-demo for details +``` +### Interactive demo with docker + +TODO + +![demo](assets/demo.jpg) + +## Usage + +```python +from mast3r.model import AsymmetricMASt3R +from mast3r.fast_nn import fast_reciprocal_NNs + +import mast3r.utils.path_to_dust3r +from dust3r.inference import inference +from dust3r.utils.image import load_images + +if __name__ == '__main__': + device = 'cuda' + schedule = 'cosine' + lr = 0.01 + niter = 300 + + model_name = "naver/MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric" + # you can put the path to a local checkpoint in model_name if needed + model = AsymmetricMASt3R.from_pretrained(model_name).to(device) + images = load_images(['dust3r/croco/assets/Chateau1.png', 'dust3r/croco/assets/Chateau2.png'], size=512) + output = inference([tuple(images)], model, device, batch_size=1, verbose=False) + + # at this stage, you have the raw dust3r predictions + view1, pred1 = output['view1'], output['pred1'] + view2, pred2 = output['view2'], output['pred2'] + + desc1, desc2 = pred1['desc'].squeeze(0).detach(), pred2['desc'].squeeze(0).detach() + + # find 2D-2D matches between the two images + matches_im0, matches_im1 = fast_reciprocal_NNs(desc1, desc2, subsample_or_initxy1=8, + device=device, dist='dot', block_size=2**13) + + # ignore small border around the edge + H0, W0 = view1['true_shape'][0] + valid_matches_im0 = (matches_im0[:, 0] >= 3) & (matches_im0[:, 0] < int(W0) - 3) & ( + matches_im0[:, 1] >= 3) & (matches_im0[:, 1] < int(H0) - 3) + + H1, W1 = view2['true_shape'][0] + valid_matches_im1 = (matches_im1[:, 0] >= 3) & (matches_im1[:, 0] < int(W1) - 3) & ( + matches_im1[:, 1] >= 3) & (matches_im1[:, 1] < int(H1) - 3) + + valid_matches = valid_matches_im0 & valid_matches_im1 + matches_im0, matches_im1 = matches_im0[valid_matches], matches_im1[valid_matches] + + # visualize a few matches + import numpy as np + import torch + import torchvision.transforms.functional + from matplotlib import pyplot as pl + + n_viz = 20 + num_matches = matches_im0.shape[0] + match_idx_to_viz = np.round(np.linspace(0, num_matches - 1, n_viz)).astype(int) + viz_matches_im0, viz_matches_im1 = matches_im0[match_idx_to_viz], matches_im1[match_idx_to_viz] + + image_mean = torch.as_tensor([0.5, 0.5, 0.5], device='cpu').reshape(1, 3, 1, 1) + image_std = torch.as_tensor([0.5, 0.5, 0.5], device='cpu').reshape(1, 3, 1, 1) + + viz_imgs = [] + for i, view in enumerate([view1, view2]): + rgb_tensor = view['img'] * image_std + image_mean + viz_imgs.append(rgb_tensor.squeeze(0).permute(1, 2, 0).cpu().numpy()) + + H0, W0, H1, W1 = *viz_imgs[0].shape[:2], *viz_imgs[1].shape[:2] + img0 = np.pad(viz_imgs[0], ((0, max(H1 - H0, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img1 = np.pad(viz_imgs[1], ((0, max(H0 - H1, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img = np.concatenate((img0, img1), axis=1) + pl.figure() + pl.imshow(img) + cmap = pl.get_cmap('jet') + for i in range(n_viz): + (x0, y0), (x1, y1) = viz_matches_im0[i].T, viz_matches_im1[i].T + pl.plot([x0, x1 + W0], [y0, y1], '-+', color=cmap(i / (n_viz - 1)), scalex=False, scaley=False) + pl.show(block=True) +``` +![matching example on croco pair](assets/matching.jpg) + +## Training + +In this section, we present a short demonstration to get started with training MASt3R. + +### Datasets + +See [Datasets section in DUSt3R](https://github.com/naver/dust3r/tree/datasets?tab=readme-ov-file#datasets) + +### Demo + +Like for the DUSt3R training demo, we're going to download and prepare the same subset of [CO3Dv2](https://github.com/facebookresearch/co3d) - [Creative Commons Attribution-NonCommercial 4.0 International](https://github.com/facebookresearch/co3d/blob/main/LICENSE) and launch the training code on it. +It is the exact same process as DUSt3R. +The demo model will be trained for a few epochs on a very small dataset. +It will not be very good. + +```bash +# download and prepare the co3d subset +mkdir -p data/co3d_subset +cd data/co3d_subset +git clone https://github.com/facebookresearch/co3d +cd co3d +python3 ./co3d/download_dataset.py --download_folder ../ --single_sequence_subset +rm ../*.zip +cd ../../.. + +python3 datasets_preprocess/preprocess_co3d.py --co3d_dir data/co3d_subset --output_dir data/co3d_subset_processed --single_sequence_subset + +# download the pretrained dust3r checkpoint +mkdir -p checkpoints/ +wget https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth -P checkpoints/ + +# for this example we'll do fewer epochs, for the actual hyperparameters we used in the paper, see the next section: "Our Hyperparameters" +torchrun --nproc_per_node=4 train.py \ + --train_dataset "1000 @ Co3d(split='train', ROOT='data/co3d_subset_processed', aug_crop='auto', aug_monocular=0.005, aug_rot90='diff', mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], n_corres=8192, nneg=0.5, transform=ColorJitter)" \ + --test_dataset "100 @ Co3d(split='test', ROOT='data/co3d_subset_processed', resolution=(512,384), n_corres=1024, seed=777)" \ + --model "AsymmetricMASt3R(pos_embed='RoPE100', patch_embed_cls='ManyAR_PatchEmbed', img_size=(512, 512), head_type='catmlp+dpt', output_mode='pts3d+desc24', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12, two_confs=True)" \ + --train_criterion "ConfLoss(Regr3D(L21, norm_mode='?avg_dis'), alpha=0.2) + 0.075*ConfMatchingLoss(MatchingLoss(InfoNCE(mode='proper', temperature=0.05), negatives_padding=0, blocksize=8192), alpha=10.0, confmode='mean')" \ + --test_criterion "Regr3D_ScaleShiftInv(L21, norm_mode='?avg_dis', gt_scale=True, sky_loss_value=0) + -1.*MatchingLoss(APLoss(nq='torch', fp=torch.float16), negatives_padding=12288)" \ + --pretrained "checkpoints/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth" \ + --lr 0.0001 --min_lr 1e-06 --warmup_epochs 1 --epochs 10 --batch_size 4 --accum_iter 4 \ + --save_freq 1 --keep_freq 5 --eval_freq 1 \ + --output_dir "checkpoints/mast3r_demo" + +``` + +### Our Hyperparameters +We didn't release all the training datasets, but here are the commands we used for training our models: + +```bash +# MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric - train mast3r with metric regression and matching loss +# we used cosxl to generate variations of DL3DV: "foggy", "night", "rainy", "snow", "sunny" but we were not convinced by it. + +torchrun --nproc_per_node=8 train.py \ + --train_dataset "57_000 @ Habitat512(1_000_000, split='train', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 68_400 @ BlendedMVS(split='train', mask_sky=True, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 68_400 @ MegaDepth(split='train', mask_sky=True, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 45_600 @ ARKitScenes(split='train', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 22_800 @ Co3d(split='train', mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 22_800 @ StaticThings3D(mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 45_600 @ ScanNetpp(split='train', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 45_600 @ TartanAir(pairs_subset='', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 4_560 @ UnrealStereo4K(resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 1_140 @ VirtualKitti(optical_center_is_centered=True, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 22_800 @ WildRgbd(split='train', mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 145_920 @ NianticMapFree(split='train', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 57_000 @ DL3DV(split='nlight', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 57_000 @ DL3DV(split='not-nlight', cosxl_augmentations=None, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5) + 34_200 @ InternalUnreleasedDataset(resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], aug_crop='auto', aug_monocular=0.005, transform=ColorJitter, n_corres=8192, nneg=0.5)" \ + --test_dataset "Habitat512(1_000, split='val', resolution=(512,384), seed=777, n_corres=1024) + 1_000 @ BlendedMVS(split='val', resolution=(512,384), mask_sky=True, seed=777, n_corres=1024) + 1_000 @ ARKitScenes(split='test', resolution=(512,384), seed=777, n_corres=1024) + 1_000 @ MegaDepth(split='val', mask_sky=True, resolution=(512,336), seed=777, n_corres=1024) + 1_000 @ Co3d(split='test', resolution=(512,384), mask_bg='rand', seed=777, n_corres=1024)" \ + --model "AsymmetricMASt3R(pos_embed='RoPE100', patch_embed_cls='ManyAR_PatchEmbed', img_size=(512, 512), head_type='catmlp+dpt', output_mode='pts3d+desc24', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12, two_confs=True, desc_conf_mode=('exp', 0, inf))" \ + --train_criterion "ConfLoss(Regr3D(L21, norm_mode='?avg_dis'), alpha=0.2, loss_in_log=False) + 0.075*ConfMatchingLoss(MatchingLoss(InfoNCE(mode='proper', temperature=0.05), negatives_padding=0, blocksize=8192), alpha=10.0, confmode='mean')" \ + --test_criterion "Regr3D(L21, norm_mode='?avg_dis', gt_scale=True, sky_loss_value=0) + -1.*MatchingLoss(APLoss(nq='torch', fp=torch.float16), negatives_padding=12288)" \ + --pretrained "checkpoints/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth" \ + --lr 0.0001 --min_lr 1e-06 --warmup_epochs 8 --epochs 50 --batch_size 4 --accum_iter 2 \ + --save_freq 1 --keep_freq 5 --eval_freq 1 --print_freq=10 \ + --output_dir "checkpoints/MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric" + +``` + +## Visual Localization + +### Dataset preparation + +See [Visloc section in DUSt3R](https://github.com/naver/dust3r/tree/dust3r_visloc#dataset-preparation) + +### Example Commands + +With `visloc.py` you can run our visual localization experiments on Aachen-Day-Night, InLoc, Cambridge Landmarks and 7 Scenes. + + +```bash +# Aachen-Day-Night-v1.1: +# scene in 'day' 'night' +# scene can also be 'all' +python3 visloc.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric --dataset "VislocAachenDayNight('/path/to/prepared/Aachen-Day-Night-v1.1/', subscene='${scene}', pairsfile='fire_top50', topk=20)" --pixel_tol 5 --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/Aachen-Day-Night-v1.1/${scene}/loc + +# or with coarse to fine: + +python3 visloc.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric --dataset "VislocAachenDayNight('/path/to/prepared/Aachen-Day-Night-v1.1/', subscene='${scene}', pairsfile='fire_top50', topk=20)" --pixel_tol 5 --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/Aachen-Day-Night-v1.1/${scene}/loc --coarse_to_fine --max_batch_size 48 --c2f_crop_with_homography + +# InLoc +python3 visloc.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric --dataset "VislocInLoc('/path/to/prepared/InLoc/', pairsfile='pairs-query-netvlad40-temporal', topk=20)" --pixel_tol 5 --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/InLoc/loc + +# or with coarse to fine: + +python3 visloc.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric --dataset "VislocInLoc('/path/to/prepared/InLoc/', pairsfile='pairs-query-netvlad40-temporal', topk=20)" --pixel_tol 5 --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/InLoc/loc --coarse_to_fine --max_image_size 1200 --max_batch_size 48 --c2f_crop_with_homography + +# 7-scenes: +# scene in 'chess' 'fire' 'heads' 'office' 'pumpkin' 'redkitchen' 'stairs' +python3 visloc.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric --dataset "VislocSevenScenes('/path/to/prepared/7-scenes/', subscene='${scene}', pairsfile='APGeM-LM18_top20', topk=1)" --pixel_tol 5 --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/7-scenes/${scene}/loc + +# Cambridge Landmarks: +# scene in 'ShopFacade' 'GreatCourt' 'KingsCollege' 'OldHospital' 'StMarysChurch' +python3 visloc.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric --dataset "VislocCambridgeLandmarks('/path/to/prepared/Cambridge_Landmarks/', subscene='${scene}', pairsfile='APGeM-LM18_top20', topk=1)" --pixel_tol 5 --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/Cambridge_Landmarks/${scene}/loc + +``` diff --git a/src/mast3r_src/assets/NLE_tower/01D90321-69C8-439F-B0B0-E87E7634741C-83120-000041DAE419D7AE.jpg b/src/mast3r_src/assets/NLE_tower/01D90321-69C8-439F-B0B0-E87E7634741C-83120-000041DAE419D7AE.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fd6b2d4edcfb5b277350a7ab0948a144f01f809c Binary files /dev/null and b/src/mast3r_src/assets/NLE_tower/01D90321-69C8-439F-B0B0-E87E7634741C-83120-000041DAE419D7AE.jpg differ diff --git a/src/mast3r_src/assets/NLE_tower/1AD85EF5-B651-4291-A5C0-7BDB7D966384-83120-000041DADF639E09.jpg b/src/mast3r_src/assets/NLE_tower/1AD85EF5-B651-4291-A5C0-7BDB7D966384-83120-000041DADF639E09.jpg new file mode 100644 index 0000000000000000000000000000000000000000..999cc01f92686bdab008e91004a457769613845f Binary files /dev/null and b/src/mast3r_src/assets/NLE_tower/1AD85EF5-B651-4291-A5C0-7BDB7D966384-83120-000041DADF639E09.jpg differ diff --git a/src/mast3r_src/assets/NLE_tower/2679C386-1DC0-4443-81B5-93D7EDE4AB37-83120-000041DADB2EA917.jpg b/src/mast3r_src/assets/NLE_tower/2679C386-1DC0-4443-81B5-93D7EDE4AB37-83120-000041DADB2EA917.jpg new file mode 100644 index 0000000000000000000000000000000000000000..79e8bf4c76d6b281db4ebb1ad8e08ef4ec03b66a Binary files /dev/null and b/src/mast3r_src/assets/NLE_tower/2679C386-1DC0-4443-81B5-93D7EDE4AB37-83120-000041DADB2EA917.jpg differ diff --git a/src/mast3r_src/assets/NLE_tower/28EDBB63-B9F9-42FB-AC86-4852A33ED71B-83120-000041DAF22407A1.jpg b/src/mast3r_src/assets/NLE_tower/28EDBB63-B9F9-42FB-AC86-4852A33ED71B-83120-000041DAF22407A1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ba840a9d53c7c48087d080a98033ebe7354ebd84 Binary files /dev/null and b/src/mast3r_src/assets/NLE_tower/28EDBB63-B9F9-42FB-AC86-4852A33ED71B-83120-000041DAF22407A1.jpg differ diff --git a/src/mast3r_src/assets/NLE_tower/91E9B685-7A7D-42D7-B933-23A800EE4129-83120-000041DAE12C8176.jpg b/src/mast3r_src/assets/NLE_tower/91E9B685-7A7D-42D7-B933-23A800EE4129-83120-000041DAE12C8176.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6c261b1cd68d26158b33e8874cd2530dbcf1e75f Binary files /dev/null and b/src/mast3r_src/assets/NLE_tower/91E9B685-7A7D-42D7-B933-23A800EE4129-83120-000041DAE12C8176.jpg differ diff --git a/src/mast3r_src/assets/NLE_tower/CDBBD885-54C3-4EB4-9181-226059A60EE0-83120-000041DAE0C3D612.jpg b/src/mast3r_src/assets/NLE_tower/CDBBD885-54C3-4EB4-9181-226059A60EE0-83120-000041DAE0C3D612.jpg new file mode 100644 index 0000000000000000000000000000000000000000..224d0f7992d19e87464d430e1ee9c0469e9a4028 Binary files /dev/null and b/src/mast3r_src/assets/NLE_tower/CDBBD885-54C3-4EB4-9181-226059A60EE0-83120-000041DAE0C3D612.jpg differ diff --git a/src/mast3r_src/assets/NLE_tower/FF5599FD-768B-431A-AB83-BDA5FB44CB9D-83120-000041DADDE35483.jpg b/src/mast3r_src/assets/NLE_tower/FF5599FD-768B-431A-AB83-BDA5FB44CB9D-83120-000041DADDE35483.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9974efd5907393daa5ae47a047daec9698866a38 Binary files /dev/null and b/src/mast3r_src/assets/NLE_tower/FF5599FD-768B-431A-AB83-BDA5FB44CB9D-83120-000041DADDE35483.jpg differ diff --git a/src/mast3r_src/assets/demo.jpg b/src/mast3r_src/assets/demo.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0c3c6560eca4b1cb41e8c29b52c7f25e9afd5173 Binary files /dev/null and b/src/mast3r_src/assets/demo.jpg differ diff --git a/src/mast3r_src/assets/examples.jpg b/src/mast3r_src/assets/examples.jpg new file mode 100644 index 0000000000000000000000000000000000000000..aa40b208a5107269aed3058ccd5315457ec14676 Binary files /dev/null and b/src/mast3r_src/assets/examples.jpg differ diff --git a/src/mast3r_src/assets/mast3r.jpg b/src/mast3r_src/assets/mast3r.jpg new file mode 100644 index 0000000000000000000000000000000000000000..de4a0ff4a8727f20030d673c0fcad653fc7e56d4 Binary files /dev/null and b/src/mast3r_src/assets/mast3r.jpg differ diff --git a/src/mast3r_src/assets/mast3r_archi.jpg b/src/mast3r_src/assets/mast3r_archi.jpg new file mode 100644 index 0000000000000000000000000000000000000000..126a371c984498587437c5f05ca842c5fda63cc2 Binary files /dev/null and b/src/mast3r_src/assets/mast3r_archi.jpg differ diff --git a/src/mast3r_src/assets/matching.jpg b/src/mast3r_src/assets/matching.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c35487745cd0e01ef1c3f5a72127cbf429840e59 Binary files /dev/null and b/src/mast3r_src/assets/matching.jpg differ diff --git a/src/mast3r_src/demo.py b/src/mast3r_src/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..ec15736ed249a95838afc63d11742394c31e629c --- /dev/null +++ b/src/mast3r_src/demo.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# gradio demo +# -------------------------------------------------------- +import math +import gradio +import os +import torch +import numpy as np +import tempfile +import functools +import trimesh +import copy +from scipy.spatial.transform import Rotation + +from mast3r.cloud_opt.sparse_ga import sparse_global_alignment +from mast3r.cloud_opt.tsdf_optimizer import TSDFPostProcess + +from mast3r.model import AsymmetricMASt3R +from mast3r.utils.misc import hash_md5 +import mast3r.utils.path_to_dust3r # noqa +from dust3r.image_pairs import make_pairs +from dust3r.utils.image import load_images +from dust3r.utils.device import to_numpy +from dust3r.viz import add_scene_cam, CAM_COLORS, OPENGL, pts3d_to_trimesh, cat_meshes +from dust3r.demo import get_args_parser as dust3r_get_args_parser + +import matplotlib.pyplot as pl +pl.ion() + +torch.backends.cuda.matmul.allow_tf32 = True # for gpu >= Ampere and pytorch >= 1.12 +batch_size = 1 + + +def get_args_parser(): + parser = dust3r_get_args_parser() + parser.add_argument('--share', action='store_true') + + actions = parser._actions + for action in actions: + if action.dest == 'model_name': + action.choices = ["MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric"] + # change defaults + parser.prog = 'mast3r demo' + return parser + + +def _convert_scene_output_to_glb(outdir, imgs, pts3d, mask, focals, cams2world, cam_size=0.05, + cam_color=None, as_pointcloud=False, + transparent_cams=False, silent=False): + assert len(pts3d) == len(mask) <= len(imgs) <= len(cams2world) == len(focals) + pts3d = to_numpy(pts3d) + imgs = to_numpy(imgs) + focals = to_numpy(focals) + cams2world = to_numpy(cams2world) + + scene = trimesh.Scene() + + # full pointcloud + if as_pointcloud: + pts = np.concatenate([p[m.ravel()] for p, m in zip(pts3d, mask)]) + col = np.concatenate([p[m] for p, m in zip(imgs, mask)]) + pct = trimesh.PointCloud(pts.reshape(-1, 3), colors=col.reshape(-1, 3)) + scene.add_geometry(pct) + else: + meshes = [] + for i in range(len(imgs)): + meshes.append(pts3d_to_trimesh(imgs[i], pts3d[i].reshape(imgs[i].shape), mask[i])) + mesh = trimesh.Trimesh(**cat_meshes(meshes)) + scene.add_geometry(mesh) + + # add each camera + for i, pose_c2w in enumerate(cams2world): + if isinstance(cam_color, list): + camera_edge_color = cam_color[i] + else: + camera_edge_color = cam_color or CAM_COLORS[i % len(CAM_COLORS)] + add_scene_cam(scene, pose_c2w, camera_edge_color, + None if transparent_cams else imgs[i], focals[i], + imsize=imgs[i].shape[1::-1], screen_width=cam_size) + + rot = np.eye(4) + rot[:3, :3] = Rotation.from_euler('y', np.deg2rad(180)).as_matrix() + scene.apply_transform(np.linalg.inv(cams2world[0] @ OPENGL @ rot)) + outfile = os.path.join(outdir, 'scene.glb') + if not silent: + print('(exporting 3D scene to', outfile, ')') + scene.export(file_obj=outfile) + return outfile + + +def get_3D_model_from_scene(outdir, silent, scene, min_conf_thr=2, as_pointcloud=False, mask_sky=False, + clean_depth=False, transparent_cams=False, cam_size=0.05, TSDF_thresh=0): + """ + extract 3D_model (glb file) from a reconstructed scene + """ + if scene is None: + return None + + # get optimized values from scene + rgbimg = scene.imgs + focals = scene.get_focals().cpu() + cams2world = scene.get_im_poses().cpu() + + # 3D pointcloud from depthmap, poses and intrinsics + if TSDF_thresh > 0: + tsdf = TSDFPostProcess(scene, TSDF_thresh=TSDF_thresh) + pts3d, _, confs = to_numpy(tsdf.get_dense_pts3d(clean_depth=clean_depth)) + else: + pts3d, _, confs = to_numpy(scene.get_dense_pts3d(clean_depth=clean_depth)) + msk = to_numpy([c > min_conf_thr for c in confs]) + return _convert_scene_output_to_glb(outdir, rgbimg, pts3d, msk, focals, cams2world, as_pointcloud=as_pointcloud, + transparent_cams=transparent_cams, cam_size=cam_size, silent=silent) + + +def get_reconstructed_scene(outdir, model, device, silent, image_size, filelist, optim_level, lr1, niter1, lr2, niter2, + min_conf_thr, matching_conf_thr, as_pointcloud, mask_sky, clean_depth, transparent_cams, + cam_size, scenegraph_type, winsize, win_cyclic, refid, TSDF_thresh, shared_intrinsics, + **kw): + """ + from a list of images, run mast3r inference, sparse global aligner. + then run get_3D_model_from_scene + """ + imgs = load_images(filelist, size=image_size, verbose=not silent) + if len(imgs) == 1: + imgs = [imgs[0], copy.deepcopy(imgs[0])] + imgs[1]['idx'] = 1 + filelist = [filelist[0], filelist[0] + '_2'] + + scene_graph_params = [scenegraph_type] + if scenegraph_type in ["swin", "logwin"]: + scene_graph_params.append(str(winsize)) + elif scenegraph_type == "oneref": + scene_graph_params.append(str(refid)) + if scenegraph_type in ["swin", "logwin"] and not win_cyclic: + scene_graph_params.append('noncyclic') + scene_graph = '-'.join(scene_graph_params) + pairs = make_pairs(imgs, scene_graph=scene_graph, prefilter=None, symmetrize=True) + if optim_level == 'coarse': + niter2 = 0 + # Sparse GA (forward mast3r -> matching -> 3D optim -> 2D refinement -> triangulation) + scene = sparse_global_alignment(filelist, pairs, os.path.join(outdir, 'cache'), + model, lr1=lr1, niter1=niter1, lr2=lr2, niter2=niter2, device=device, + opt_depth='depth' in optim_level, shared_intrinsics=shared_intrinsics, + matching_conf_thr=matching_conf_thr, **kw) + outfile = get_3D_model_from_scene(outdir, silent, scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh) + return scene, outfile + + +def set_scenegraph_options(inputfiles, win_cyclic, refid, scenegraph_type): + num_files = len(inputfiles) if inputfiles is not None else 1 + show_win_controls = scenegraph_type in ["swin", "logwin"] + show_winsize = scenegraph_type in ["swin", "logwin"] + show_cyclic = scenegraph_type in ["swin", "logwin"] + max_winsize, min_winsize = 1, 1 + if scenegraph_type == "swin": + if win_cyclic: + max_winsize = max(1, math.ceil((num_files - 1) / 2)) + else: + max_winsize = num_files - 1 + elif scenegraph_type == "logwin": + if win_cyclic: + half_size = math.ceil((num_files - 1) / 2) + max_winsize = max(1, math.ceil(math.log(half_size, 2))) + else: + max_winsize = max(1, math.ceil(math.log(num_files, 2))) + winsize = gradio.Slider(label="Scene Graph: Window Size", value=max_winsize, + minimum=min_winsize, maximum=max_winsize, step=1, visible=show_winsize) + win_cyclic = gradio.Checkbox(value=win_cyclic, label="Cyclic sequence", visible=show_cyclic) + win_col = gradio.Column(visible=show_win_controls) + refid = gradio.Slider(label="Scene Graph: Id", value=0, minimum=0, + maximum=num_files - 1, step=1, visible=scenegraph_type == 'oneref') + return win_col, winsize, win_cyclic, refid + + +def main_demo(tmpdirname, model, device, image_size, server_name, server_port, silent=False, share=False): + if not silent: + print('Outputing stuff in', tmpdirname) + + recon_fun = functools.partial(get_reconstructed_scene, tmpdirname, model, device, silent, image_size) + model_from_scene_fun = functools.partial(get_3D_model_from_scene, tmpdirname, silent) + with gradio.Blocks(css=""".gradio-container {margin: 0 !important; min-width: 100%};""", title="MASt3R Demo") as demo: + # scene state is save so that you can change conf_thr, cam_size... without rerunning the inference + scene = gradio.State(None) + gradio.HTML('

MASt3R Demo

') + with gradio.Column(): + inputfiles = gradio.File(file_count="multiple") + with gradio.Row(): + with gradio.Column(): + with gradio.Row(): + lr1 = gradio.Slider(label="Coarse LR", value=0.07, minimum=0.01, maximum=0.2, step=0.01) + niter1 = gradio.Number(value=500, precision=0, minimum=0, maximum=10_000, + label="num_iterations", info="For coarse alignment!") + lr2 = gradio.Slider(label="Fine LR", value=0.014, minimum=0.005, maximum=0.05, step=0.001) + niter2 = gradio.Number(value=200, precision=0, minimum=0, maximum=100_000, + label="num_iterations", info="For refinement!") + optim_level = gradio.Dropdown(["coarse", "refine", "refine+depth"], + value='refine', label="OptLevel", + info="Optimization level") + with gradio.Row(): + matching_conf_thr = gradio.Slider(label="Matching Confidence Thr", value=5., + minimum=0., maximum=30., step=0.1, + info="Before Fallback to Regr3D!") + shared_intrinsics = gradio.Checkbox(value=False, label="Shared intrinsics", + info="Only optimize one set of intrinsics for all views") + scenegraph_type = gradio.Dropdown(["complete", "swin", "logwin", "oneref"], + value='complete', label="Scenegraph", + info="Define how to make pairs", + interactive=True) + with gradio.Column(visible=False) as win_col: + winsize = gradio.Slider(label="Scene Graph: Window Size", value=1, + minimum=1, maximum=1, step=1) + win_cyclic = gradio.Checkbox(value=False, label="Cyclic sequence") + refid = gradio.Slider(label="Scene Graph: Id", value=0, + minimum=0, maximum=0, step=1, visible=False) + + run_btn = gradio.Button("Run") + + with gradio.Row(): + # adjust the confidence threshold + min_conf_thr = gradio.Slider(label="min_conf_thr", value=1.5, minimum=0.0, maximum=10, step=0.1) + # adjust the camera size in the output pointcloud + cam_size = gradio.Slider(label="cam_size", value=0.2, minimum=0.001, maximum=1.0, step=0.001) + TSDF_thresh = gradio.Slider(label="TSDF Threshold", value=0., minimum=0., maximum=1., step=0.01) + with gradio.Row(): + as_pointcloud = gradio.Checkbox(value=True, label="As pointcloud") + # two post process implemented + mask_sky = gradio.Checkbox(value=False, label="Mask sky") + clean_depth = gradio.Checkbox(value=True, label="Clean-up depthmaps") + transparent_cams = gradio.Checkbox(value=False, label="Transparent cameras") + + outmodel = gradio.Model3D() + + # events + scenegraph_type.change(set_scenegraph_options, + inputs=[inputfiles, win_cyclic, refid, scenegraph_type], + outputs=[win_col, winsize, win_cyclic, refid]) + inputfiles.change(set_scenegraph_options, + inputs=[inputfiles, win_cyclic, refid, scenegraph_type], + outputs=[win_col, winsize, win_cyclic, refid]) + win_cyclic.change(set_scenegraph_options, + inputs=[inputfiles, win_cyclic, refid, scenegraph_type], + outputs=[win_col, winsize, win_cyclic, refid]) + run_btn.click(fn=recon_fun, + inputs=[inputfiles, optim_level, lr1, niter1, lr2, niter2, min_conf_thr, matching_conf_thr, + as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size, + scenegraph_type, winsize, win_cyclic, refid, TSDF_thresh, shared_intrinsics], + outputs=[scene, outmodel]) + min_conf_thr.release(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh], + outputs=outmodel) + cam_size.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh], + outputs=outmodel) + TSDF_thresh.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh], + outputs=outmodel) + as_pointcloud.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh], + outputs=outmodel) + mask_sky.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh], + outputs=outmodel) + clean_depth.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh], + outputs=outmodel) + transparent_cams.change(model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size, TSDF_thresh], + outputs=outmodel) + demo.launch(share=True, server_name=server_name, server_port=server_port) + + +if __name__ == '__main__': + parser = get_args_parser() + args = parser.parse_args() + + if args.server_name is not None: + server_name = args.server_name + else: + server_name = '0.0.0.0' if args.local_network else '127.0.0.1' + + if args.weights is not None: + weights_path = args.weights + else: + weights_path = "naver/" + args.model_name + + model = AsymmetricMASt3R.from_pretrained(weights_path).to(args.device) + chkpt_tag = hash_md5(weights_path) + + # mast3r will write the 3D model inside tmpdirname/chkpt_tag + if args.tmp_dir is not None: + tmpdirname = args.tmp_dir + cache_path = os.path.join(tmpdirname, chkpt_tag) + os.makedirs(cache_path, exist_ok=True) + main_demo(cache_path, model, args.device, args.image_size, server_name, args.server_port, silent=args.silent, + share=args.share) + else: + with tempfile.TemporaryDirectory(suffix='_mast3r_gradio_demo') as tmpdirname: + cache_path = os.path.join(tmpdirname, chkpt_tag) + os.makedirs(cache_path, exist_ok=True) + main_demo(tmpdirname, model, args.device, args.image_size, + server_name, args.server_port, silent=args.silent, + share=args.share) diff --git a/src/mast3r_src/demo_dust3r_ga.py b/src/mast3r_src/demo_dust3r_ga.py new file mode 100644 index 0000000000000000000000000000000000000000..31d5be0501949e2393ae389d2fe7ac16cf3651dc --- /dev/null +++ b/src/mast3r_src/demo_dust3r_ga.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# mast3r gradio demo executable +# -------------------------------------------------------- +import os +import torch +import tempfile + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.model import AsymmetricCroCo3DStereo +from mast3r.model import AsymmetricMASt3R +from dust3r.demo import get_args_parser as dust3r_get_args_parser +from dust3r.demo import main_demo + +import matplotlib.pyplot as pl +pl.ion() + +torch.backends.cuda.matmul.allow_tf32 = True # for gpu >= Ampere and pytorch >= 1.12 + + +def get_args_parser(): + parser = dust3r_get_args_parser() + + actions = parser._actions + for action in actions: + if action.dest == 'model_name': + action.choices.append('MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric') + # change defaults + parser.prog = 'mast3r demo' + return parser + + +if __name__ == '__main__': + parser = get_args_parser() + args = parser.parse_args() + + if args.tmp_dir is not None: + tmp_path = args.tmp_dir + os.makedirs(tmp_path, exist_ok=True) + tempfile.tempdir = tmp_path + + if args.server_name is not None: + server_name = args.server_name + else: + server_name = '0.0.0.0' if args.local_network else '127.0.0.1' + + if args.weights is not None: + weights_path = args.weights + else: + weights_path = "naver/" + args.model_name + + try: + model = AsymmetricMASt3R.from_pretrained(weights_path).to(args.device) + except Exception as e: + model = AsymmetricCroCo3DStereo.from_pretrained(weights_path).to(args.device) + + # dust3r will write the 3D model inside tmpdirname + with tempfile.TemporaryDirectory(suffix='dust3r_gradio_demo') as tmpdirname: + if not args.silent: + print('Outputing stuff in', tmpdirname) + main_demo(tmpdirname, model, args.device, args.image_size, server_name, args.server_port, silent=args.silent) diff --git a/src/mast3r_src/dust3r/.gitignore b/src/mast3r_src/dust3r/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..194e236cbd708160926c3513b4232285eb47b029 --- /dev/null +++ b/src/mast3r_src/dust3r/.gitignore @@ -0,0 +1,132 @@ +data/ +checkpoints/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/src/mast3r_src/dust3r/.gitmodules b/src/mast3r_src/dust3r/.gitmodules new file mode 100644 index 0000000000000000000000000000000000000000..c950ef981a8d2e47599dd7acbbe1bf8de9a42aca --- /dev/null +++ b/src/mast3r_src/dust3r/.gitmodules @@ -0,0 +1,3 @@ +[submodule "croco"] + path = croco + url = https://github.com/naver/croco diff --git a/src/mast3r_src/dust3r/LICENSE b/src/mast3r_src/dust3r/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..a97986e3a8ddd49973959f6c748dfa8b881b64d3 --- /dev/null +++ b/src/mast3r_src/dust3r/LICENSE @@ -0,0 +1,7 @@ +DUSt3R, Copyright (c) 2024-present Naver Corporation, is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 license. + +A summary of the CC BY-NC-SA 4.0 license is located here: + https://creativecommons.org/licenses/by-nc-sa/4.0/ + +The CC BY-NC-SA 4.0 license is located here: + https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode diff --git a/src/mast3r_src/dust3r/NOTICE b/src/mast3r_src/dust3r/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..81da544dd534c5465361f35cf6a5a0cfff7c1d3f --- /dev/null +++ b/src/mast3r_src/dust3r/NOTICE @@ -0,0 +1,12 @@ +DUSt3R +Copyright 2024-present NAVER Corp. + +This project contains subcomponents with separate copyright notices and license terms. +Your use of the source code for these subcomponents is subject to the terms and conditions of the following licenses. + +==== + +naver/croco +https://github.com/naver/croco/ + +Creative Commons Attribution-NonCommercial-ShareAlike 4.0 diff --git a/src/mast3r_src/dust3r/README.md b/src/mast3r_src/dust3r/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6df7772a18830d1249d1f9992cf6c28e3e794993 --- /dev/null +++ b/src/mast3r_src/dust3r/README.md @@ -0,0 +1,388 @@ +![demo](assets/dust3r.jpg) + +Official implementation of `DUSt3R: Geometric 3D Vision Made Easy` +[[Project page](https://dust3r.europe.naverlabs.com/)], [[DUSt3R arxiv](https://arxiv.org/abs/2312.14132)] + +![Example of reconstruction from two images](assets/pipeline1.jpg) + +![High level overview of DUSt3R capabilities](assets/dust3r_archi.jpg) + +```bibtex +@inproceedings{dust3r_cvpr24, + title={DUSt3R: Geometric 3D Vision Made Easy}, + author={Shuzhe Wang and Vincent Leroy and Yohann Cabon and Boris Chidlovskii and Jerome Revaud}, + booktitle = {CVPR}, + year = {2024} +} + +@misc{dust3r_arxiv23, + title={DUSt3R: Geometric 3D Vision Made Easy}, + author={Shuzhe Wang and Vincent Leroy and Yohann Cabon and Boris Chidlovskii and Jerome Revaud}, + year={2023}, + eprint={2312.14132}, + archivePrefix={arXiv}, + primaryClass={cs.CV} +} +``` + +## Table of Contents + +- [Table of Contents](#table-of-contents) +- [License](#license) +- [Get Started](#get-started) + - [Installation](#installation) + - [Checkpoints](#checkpoints) + - [Interactive demo](#interactive-demo) + - [Interactive demo with docker](#interactive-demo-with-docker) +- [Usage](#usage) +- [Training](#training) + - [Datasets](#datasets) + - [Demo](#demo) + - [Our Hyperparameters](#our-hyperparameters) + +## License + +The code is distributed under the CC BY-NC-SA 4.0 License. +See [LICENSE](LICENSE) for more information. + +```python +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +``` + +## Get Started + +### Installation + +1. Clone DUSt3R. +```bash +git clone --recursive https://github.com/naver/dust3r +cd dust3r +# if you have already cloned dust3r: +# git submodule update --init --recursive +``` + +2. Create the environment, here we show an example using conda. +```bash +conda create -n dust3r python=3.11 cmake=3.14.0 +conda activate dust3r +conda install pytorch torchvision pytorch-cuda=12.1 -c pytorch -c nvidia # use the correct version of cuda for your system +pip install -r requirements.txt +# Optional: you can also install additional packages to: +# - add support for HEIC images +# - add pyrender, used to render depthmap in some datasets preprocessing +# - add required packages for visloc.py +pip install -r requirements_optional.txt +``` + +3. Optional, compile the cuda kernels for RoPE (as in CroCo v2). +```bash +# DUST3R relies on RoPE positional embeddings for which you can compile some cuda kernels for faster runtime. +cd croco/models/curope/ +python setup.py build_ext --inplace +cd ../../../ +``` + +### Checkpoints + +You can obtain the checkpoints by two ways: + +1) You can use our huggingface_hub integration: the models will be downloaded automatically. + +2) Otherwise, We provide several pre-trained models: + +| Modelname | Training resolutions | Head | Encoder | Decoder | +|-------------|----------------------|------|---------|---------| +| [`DUSt3R_ViTLarge_BaseDecoder_224_linear.pth`](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_224_linear.pth) | 224x224 | Linear | ViT-L | ViT-B | +| [`DUSt3R_ViTLarge_BaseDecoder_512_linear.pth`](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_512_linear.pth) | 512x384, 512x336, 512x288, 512x256, 512x160 | Linear | ViT-L | ViT-B | +| [`DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth`](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth) | 512x384, 512x336, 512x288, 512x256, 512x160 | DPT | ViT-L | ViT-B | + +You can check the hyperparameters we used to train these models in the [section: Our Hyperparameters](#our-hyperparameters) + +To download a specific model, for example `DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth`: +```bash +mkdir -p checkpoints/ +wget https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth -P checkpoints/ +``` + +For the checkpoints, make sure to agree to the license of all the public training datasets and base checkpoints we used, in addition to CC-BY-NC-SA 4.0. Again, see [section: Our Hyperparameters](#our-hyperparameters) for details. + +### Interactive demo + +In this demo, you should be able run DUSt3R on your machine to reconstruct a scene. +First select images that depicts the same scene. + +You can adjust the global alignment schedule and its number of iterations. + +> [!NOTE] +> If you selected one or two images, the global alignment procedure will be skipped (mode=GlobalAlignerMode.PairViewer) + +Hit "Run" and wait. +When the global alignment ends, the reconstruction appears. +Use the slider "min_conf_thr" to show or remove low confidence areas. + +```bash +python3 demo.py --model_name DUSt3R_ViTLarge_BaseDecoder_512_dpt + +# Use --weights to load a checkpoint from a local file, eg --weights checkpoints/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth +# Use --image_size to select the correct resolution for the selected checkpoint. 512 (default) or 224 +# Use --local_network to make it accessible on the local network, or --server_name to specify the url manually +# Use --server_port to change the port, by default it will search for an available port starting at 7860 +# Use --device to use a different device, by default it's "cuda" +``` + +### Interactive demo with docker + +To run DUSt3R using Docker, including with NVIDIA CUDA support, follow these instructions: + +1. **Install Docker**: If not already installed, download and install `docker` and `docker compose` from the [Docker website](https://www.docker.com/get-started). + +2. **Install NVIDIA Docker Toolkit**: For GPU support, install the NVIDIA Docker toolkit from the [Nvidia website](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). + +3. **Build the Docker image and run it**: `cd` into the `./docker` directory and run the following commands: + +```bash +cd docker +bash run.sh --with-cuda --model_name="DUSt3R_ViTLarge_BaseDecoder_512_dpt" +``` + +Or if you want to run the demo without CUDA support, run the following command: + +```bash +cd docker +bash run.sh --model_name="DUSt3R_ViTLarge_BaseDecoder_512_dpt" +``` + +By default, `demo.py` is lanched with the option `--local_network`. +Visit `http://localhost:7860/` to access the web UI (or replace `localhost` with the machine's name to access it from the network). + +`run.sh` will launch docker-compose using either the [docker-compose-cuda.yml](docker/docker-compose-cuda.yml) or [docker-compose-cpu.ym](docker/docker-compose-cpu.yml) config file, then it starts the demo using [entrypoint.sh](docker/files/entrypoint.sh). + + +![demo](assets/demo.jpg) + +## Usage + +```python +from dust3r.inference import inference +from dust3r.model import AsymmetricCroCo3DStereo +from dust3r.utils.image import load_images +from dust3r.image_pairs import make_pairs +from dust3r.cloud_opt import global_aligner, GlobalAlignerMode + +if __name__ == '__main__': + device = 'cuda' + batch_size = 1 + schedule = 'cosine' + lr = 0.01 + niter = 300 + + model_name = "naver/DUSt3R_ViTLarge_BaseDecoder_512_dpt" + # you can put the path to a local checkpoint in model_name if needed + model = AsymmetricCroCo3DStereo.from_pretrained(model_name).to(device) + # load_images can take a list of images or a directory + images = load_images(['croco/assets/Chateau1.png', 'croco/assets/Chateau2.png'], size=512) + pairs = make_pairs(images, scene_graph='complete', prefilter=None, symmetrize=True) + output = inference(pairs, model, device, batch_size=batch_size) + + # at this stage, you have the raw dust3r predictions + view1, pred1 = output['view1'], output['pred1'] + view2, pred2 = output['view2'], output['pred2'] + # here, view1, pred1, view2, pred2 are dicts of lists of len(2) + # -> because we symmetrize we have (im1, im2) and (im2, im1) pairs + # in each view you have: + # an integer image identifier: view1['idx'] and view2['idx'] + # the img: view1['img'] and view2['img'] + # the image shape: view1['true_shape'] and view2['true_shape'] + # an instance string output by the dataloader: view1['instance'] and view2['instance'] + # pred1 and pred2 contains the confidence values: pred1['conf'] and pred2['conf'] + # pred1 contains 3D points for view1['img'] in view1['img'] space: pred1['pts3d'] + # pred2 contains 3D points for view2['img'] in view1['img'] space: pred2['pts3d_in_other_view'] + + # next we'll use the global_aligner to align the predictions + # depending on your task, you may be fine with the raw output and not need it + # with only two input images, you could use GlobalAlignerMode.PairViewer: it would just convert the output + # if using GlobalAlignerMode.PairViewer, no need to run compute_global_alignment + scene = global_aligner(output, device=device, mode=GlobalAlignerMode.PointCloudOptimizer) + loss = scene.compute_global_alignment(init="mst", niter=niter, schedule=schedule, lr=lr) + + # retrieve useful values from scene: + imgs = scene.imgs + focals = scene.get_focals() + poses = scene.get_im_poses() + pts3d = scene.get_pts3d() + confidence_masks = scene.get_masks() + + # visualize reconstruction + scene.show() + + # find 2D-2D matches between the two images + from dust3r.utils.geometry import find_reciprocal_matches, xy_grid + pts2d_list, pts3d_list = [], [] + for i in range(2): + conf_i = confidence_masks[i].cpu().numpy() + pts2d_list.append(xy_grid(*imgs[i].shape[:2][::-1])[conf_i]) # imgs[i].shape[:2] = (H, W) + pts3d_list.append(pts3d[i].detach().cpu().numpy()[conf_i]) + reciprocal_in_P2, nn2_in_P1, num_matches = find_reciprocal_matches(*pts3d_list) + print(f'found {num_matches} matches') + matches_im1 = pts2d_list[1][reciprocal_in_P2] + matches_im0 = pts2d_list[0][nn2_in_P1][reciprocal_in_P2] + + # visualize a few matches + import numpy as np + from matplotlib import pyplot as pl + n_viz = 10 + match_idx_to_viz = np.round(np.linspace(0, num_matches-1, n_viz)).astype(int) + viz_matches_im0, viz_matches_im1 = matches_im0[match_idx_to_viz], matches_im1[match_idx_to_viz] + + H0, W0, H1, W1 = *imgs[0].shape[:2], *imgs[1].shape[:2] + img0 = np.pad(imgs[0], ((0, max(H1 - H0, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img1 = np.pad(imgs[1], ((0, max(H0 - H1, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img = np.concatenate((img0, img1), axis=1) + pl.figure() + pl.imshow(img) + cmap = pl.get_cmap('jet') + for i in range(n_viz): + (x0, y0), (x1, y1) = viz_matches_im0[i].T, viz_matches_im1[i].T + pl.plot([x0, x1 + W0], [y0, y1], '-+', color=cmap(i / (n_viz - 1)), scalex=False, scaley=False) + pl.show(block=True) + +``` +![matching example on croco pair](assets/matching.jpg) + +## Training + +In this section, we present a short demonstration to get started with training DUSt3R. + +### Datasets +At this moment, we have added the following training datasets: + - [CO3Dv2](https://github.com/facebookresearch/co3d) - [Creative Commons Attribution-NonCommercial 4.0 International](https://github.com/facebookresearch/co3d/blob/main/LICENSE) + - [ARKitScenes](https://github.com/apple/ARKitScenes) - [Creative Commons Attribution-NonCommercial-ShareAlike 4.0](https://github.com/apple/ARKitScenes/tree/main?tab=readme-ov-file#license) + - [ScanNet++](https://kaldir.vc.in.tum.de/scannetpp/) - [non-commercial research and educational purposes](https://kaldir.vc.in.tum.de/scannetpp/static/scannetpp-terms-of-use.pdf) + - [BlendedMVS](https://github.com/YoYo000/BlendedMVS) - [Creative Commons Attribution 4.0 International License](https://creativecommons.org/licenses/by/4.0/) + - [WayMo Open dataset](https://github.com/waymo-research/waymo-open-dataset) - [Non-Commercial Use](https://waymo.com/open/terms/) + - [Habitat-Sim](https://github.com/facebookresearch/habitat-sim/blob/main/DATASETS.md) + - [MegaDepth](https://www.cs.cornell.edu/projects/megadepth/) + - [StaticThings3D](https://github.com/lmb-freiburg/robustmvd/blob/master/rmvd/data/README.md#staticthings3d) + - [WildRGB-D](https://github.com/wildrgbd/wildrgbd/) + +For each dataset, we provide a preprocessing script in the `datasets_preprocess` directory and an archive containing the list of pairs when needed. +You have to download the datasets yourself from their official sources, agree to their license, download our list of pairs, and run the preprocessing script. + +Links: + +[ARKitScenes pairs](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/arkitscenes_pairs.zip) +[ScanNet++ pairs](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/scannetpp_pairs.zip) +[BlendedMVS pairs](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/blendedmvs_pairs.npy) +[WayMo Open dataset pairs](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/waymo_pairs.npz) +[Habitat metadata](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/habitat_5views_v1_512x512_metadata.tar.gz) +[MegaDepth pairs](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/megadepth_pairs.npz) +[StaticThings3D pairs](https://download.europe.naverlabs.com/ComputerVision/DUSt3R/staticthings_pairs.npy) + +> [!NOTE] +> They are not strictly equivalent to what was used to train DUSt3R, but they should be close enough. + +### Demo +For this training demo, we're going to download and prepare a subset of [CO3Dv2](https://github.com/facebookresearch/co3d) - [Creative Commons Attribution-NonCommercial 4.0 International](https://github.com/facebookresearch/co3d/blob/main/LICENSE) and launch the training code on it. +The demo model will be trained for a few epochs on a very small dataset. +It will not be very good. + +```bash +# download and prepare the co3d subset +mkdir -p data/co3d_subset +cd data/co3d_subset +git clone https://github.com/facebookresearch/co3d +cd co3d +python3 ./co3d/download_dataset.py --download_folder ../ --single_sequence_subset +rm ../*.zip +cd ../../.. + +python3 datasets_preprocess/preprocess_co3d.py --co3d_dir data/co3d_subset --output_dir data/co3d_subset_processed --single_sequence_subset + +# download the pretrained croco v2 checkpoint +mkdir -p checkpoints/ +wget https://download.europe.naverlabs.com/ComputerVision/CroCo/CroCo_V2_ViTLarge_BaseDecoder.pth -P checkpoints/ + +# the training of dust3r is done in 3 steps. +# for this example we'll do fewer epochs, for the actual hyperparameters we used in the paper, see the next section: "Our Hyperparameters" +# step 1 - train dust3r for 224 resolution +torchrun --nproc_per_node=4 train.py \ + --train_dataset "1000 @ Co3d(split='train', ROOT='data/co3d_subset_processed', aug_crop=16, mask_bg='rand', resolution=224, transform=ColorJitter)" \ + --test_dataset "100 @ Co3d(split='test', ROOT='data/co3d_subset_processed', resolution=224, seed=777)" \ + --model "AsymmetricCroCo3DStereo(pos_embed='RoPE100', img_size=(224, 224), head_type='linear', output_mode='pts3d', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12)" \ + --train_criterion "ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)" \ + --test_criterion "Regr3D_ScaleShiftInv(L21, gt_scale=True)" \ + --pretrained "checkpoints/CroCo_V2_ViTLarge_BaseDecoder.pth" \ + --lr 0.0001 --min_lr 1e-06 --warmup_epochs 1 --epochs 10 --batch_size 16 --accum_iter 1 \ + --save_freq 1 --keep_freq 5 --eval_freq 1 \ + --output_dir "checkpoints/dust3r_demo_224" + +# step 2 - train dust3r for 512 resolution +torchrun --nproc_per_node=4 train.py \ + --train_dataset "1000 @ Co3d(split='train', ROOT='data/co3d_subset_processed', aug_crop=16, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter)" \ + --test_dataset "100 @ Co3d(split='test', ROOT='data/co3d_subset_processed', resolution=(512,384), seed=777)" \ + --model "AsymmetricCroCo3DStereo(pos_embed='RoPE100', patch_embed_cls='ManyAR_PatchEmbed', img_size=(512, 512), head_type='linear', output_mode='pts3d', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12)" \ + --train_criterion "ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)" \ + --test_criterion "Regr3D_ScaleShiftInv(L21, gt_scale=True)" \ + --pretrained "checkpoints/dust3r_demo_224/checkpoint-best.pth" \ + --lr 0.0001 --min_lr 1e-06 --warmup_epochs 1 --epochs 10 --batch_size 4 --accum_iter 4 \ + --save_freq 1 --keep_freq 5 --eval_freq 1 \ + --output_dir "checkpoints/dust3r_demo_512" + +# step 3 - train dust3r for 512 resolution with dpt +torchrun --nproc_per_node=4 train.py \ + --train_dataset "1000 @ Co3d(split='train', ROOT='data/co3d_subset_processed', aug_crop=16, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter)" \ + --test_dataset "100 @ Co3d(split='test', ROOT='data/co3d_subset_processed', resolution=(512,384), seed=777)" \ + --model "AsymmetricCroCo3DStereo(pos_embed='RoPE100', patch_embed_cls='ManyAR_PatchEmbed', img_size=(512, 512), head_type='dpt', output_mode='pts3d', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12)" \ + --train_criterion "ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)" \ + --test_criterion "Regr3D_ScaleShiftInv(L21, gt_scale=True)" \ + --pretrained "checkpoints/dust3r_demo_512/checkpoint-best.pth" \ + --lr 0.0001 --min_lr 1e-06 --warmup_epochs 1 --epochs 10 --batch_size 2 --accum_iter 8 \ + --save_freq 1 --keep_freq 5 --eval_freq 1 \ + --output_dir "checkpoints/dust3r_demo_512dpt" + +``` + +### Our Hyperparameters + +Here are the commands we used for training the models: + +```bash +# NOTE: ROOT path omitted for datasets +# 224 linear +torchrun --nproc_per_node 8 train.py \ + --train_dataset=" + 100_000 @ Habitat(1_000_000, split='train', aug_crop=16, resolution=224, transform=ColorJitter) + 100_000 @ BlendedMVS(split='train', aug_crop=16, resolution=224, transform=ColorJitter) + 100_000 @ MegaDepth(split='train', aug_crop=16, resolution=224, transform=ColorJitter) + 100_000 @ ARKitScenes(aug_crop=256, resolution=224, transform=ColorJitter) + 100_000 @ Co3d(split='train', aug_crop=16, mask_bg='rand', resolution=224, transform=ColorJitter) + 100_000 @ StaticThings3D(aug_crop=256, mask_bg='rand', resolution=224, transform=ColorJitter) + 100_000 @ ScanNetpp(split='train', aug_crop=256, resolution=224, transform=ColorJitter) + 100_000 @ InternalUnreleasedDataset(aug_crop=128, resolution=224, transform=ColorJitter) " \ + --test_dataset=" Habitat(1_000, split='val', resolution=224, seed=777) + 1_000 @ BlendedMVS(split='val', resolution=224, seed=777) + 1_000 @ MegaDepth(split='val', resolution=224, seed=777) + 1_000 @ Co3d(split='test', mask_bg='rand', resolution=224, seed=777) " \ + --train_criterion="ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)" \ + --test_criterion="Regr3D_ScaleShiftInv(L21, gt_scale=True)" \ + --model="AsymmetricCroCo3DStereo(pos_embed='RoPE100', img_size=(224, 224), head_type='linear', output_mode='pts3d', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12)" \ + --pretrained="checkpoints/CroCo_V2_ViTLarge_BaseDecoder.pth" \ + --lr=0.0001 --min_lr=1e-06 --warmup_epochs=10 --epochs=100 --batch_size=16 --accum_iter=1 \ + --save_freq=5 --keep_freq=10 --eval_freq=1 \ + --output_dir="checkpoints/dust3r_224" + +# 512 linear +torchrun --nproc_per_node 8 train.py \ + --train_dataset=" + 10_000 @ Habitat(1_000_000, split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ BlendedMVS(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ MegaDepth(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ARKitScenes(aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ Co3d(split='train', aug_crop=16, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ StaticThings3D(aug_crop=256, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ScanNetpp(split='train', aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ InternalUnreleasedDataset(aug_crop=128, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) " \ + --test_dataset=" Habitat(1_000, split='val', resolution=(512,384), seed=777) + 1_000 @ BlendedMVS(split='val', resolution=(512,384), seed=777) + 1_000 @ MegaDepth(split='val', resolution=(512,336), seed=777) + 1_000 @ Co3d(split='test', resolution=(512,384), seed=777) " \ + --train_criterion="ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)" \ + --test_criterion="Regr3D_ScaleShiftInv(L21, gt_scale=True)" \ + --model="AsymmetricCroCo3DStereo(pos_embed='RoPE100', patch_embed_cls='ManyAR_PatchEmbed', img_size=(512, 512), head_type='linear', output_mode='pts3d', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12)" \ + --pretrained="checkpoints/dust3r_224/checkpoint-best.pth" \ + --lr=0.0001 --min_lr=1e-06 --warmup_epochs=20 --epochs=100 --batch_size=4 --accum_iter=2 \ + --save_freq=10 --keep_freq=10 --eval_freq=1 --print_freq=10 \ + --output_dir="checkpoints/dust3r_512" + +# 512 dpt +torchrun --nproc_per_node 8 train.py \ + --train_dataset=" + 10_000 @ Habitat(1_000_000, split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ BlendedMVS(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ MegaDepth(split='train', aug_crop=16, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ARKitScenes(aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ Co3d(split='train', aug_crop=16, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ StaticThings3D(aug_crop=256, mask_bg='rand', resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ ScanNetpp(split='train', aug_crop=256, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) + 10_000 @ InternalUnreleasedDataset(aug_crop=128, resolution=[(512, 384), (512, 336), (512, 288), (512, 256), (512, 160)], transform=ColorJitter) " \ + --test_dataset=" Habitat(1_000, split='val', resolution=(512,384), seed=777) + 1_000 @ BlendedMVS(split='val', resolution=(512,384), seed=777) + 1_000 @ MegaDepth(split='val', resolution=(512,336), seed=777) + 1_000 @ Co3d(split='test', resolution=(512,384), seed=777) " \ + --train_criterion="ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)" \ + --test_criterion="Regr3D_ScaleShiftInv(L21, gt_scale=True)" \ + --model="AsymmetricCroCo3DStereo(pos_embed='RoPE100', patch_embed_cls='ManyAR_PatchEmbed', img_size=(512, 512), head_type='dpt', output_mode='pts3d', depth_mode=('exp', -inf, inf), conf_mode=('exp', 1, inf), enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_depth=12, dec_num_heads=12)" \ + --pretrained="checkpoints/dust3r_512/checkpoint-best.pth" \ + --lr=0.0001 --min_lr=1e-06 --warmup_epochs=15 --epochs=90 --batch_size=4 --accum_iter=2 \ + --save_freq=5 --keep_freq=10 --eval_freq=1 --print_freq=10 \ + --output_dir="checkpoints/dust3r_512dpt" + +``` diff --git a/src/mast3r_src/dust3r/assets/demo.jpg b/src/mast3r_src/dust3r/assets/demo.jpg new file mode 100644 index 0000000000000000000000000000000000000000..684b4b08729e1ec3e3a221db8531ff06f0d6dd54 Binary files /dev/null and b/src/mast3r_src/dust3r/assets/demo.jpg differ diff --git a/src/mast3r_src/dust3r/assets/dust3r.jpg b/src/mast3r_src/dust3r/assets/dust3r.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2f65b18fdb613950a683186b2b0fbcbbbcad82e4 Binary files /dev/null and b/src/mast3r_src/dust3r/assets/dust3r.jpg differ diff --git a/src/mast3r_src/dust3r/assets/dust3r_archi.jpg b/src/mast3r_src/dust3r/assets/dust3r_archi.jpg new file mode 100644 index 0000000000000000000000000000000000000000..332de7f7dfd78ef70b9cf3defcebafec1e1a8d6e Binary files /dev/null and b/src/mast3r_src/dust3r/assets/dust3r_archi.jpg differ diff --git a/src/mast3r_src/dust3r/assets/matching.jpg b/src/mast3r_src/dust3r/assets/matching.jpg new file mode 100644 index 0000000000000000000000000000000000000000..68e0e739d7dc25c8ccc6b5dd164022e9a84b2162 Binary files /dev/null and b/src/mast3r_src/dust3r/assets/matching.jpg differ diff --git a/src/mast3r_src/dust3r/assets/pipeline1.jpg b/src/mast3r_src/dust3r/assets/pipeline1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5a0fc1e800b92fae577d6293ad50c8ee1815c3e8 Binary files /dev/null and b/src/mast3r_src/dust3r/assets/pipeline1.jpg differ diff --git a/src/mast3r_src/dust3r/croco/LICENSE b/src/mast3r_src/dust3r/croco/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d9b84b1a65f9db6d8920a9048d162f52ba3ea56d --- /dev/null +++ b/src/mast3r_src/dust3r/croco/LICENSE @@ -0,0 +1,52 @@ +CroCo, Copyright (c) 2022-present Naver Corporation, is licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 license. + +A summary of the CC BY-NC-SA 4.0 license is located here: + https://creativecommons.org/licenses/by-nc-sa/4.0/ + +The CC BY-NC-SA 4.0 license is located here: + https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode + + +SEE NOTICE BELOW WITH RESPECT TO THE FILE: models/pos_embed.py, models/blocks.py + +*************************** + +NOTICE WITH RESPECT TO THE FILE: models/pos_embed.py + +This software is being redistributed in a modifiled form. The original form is available here: + +https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py + +This software in this file incorporates parts of the following software available here: + +Transformer: https://github.com/tensorflow/models/blob/master/official/legacy/transformer/model_utils.py +available under the following license: https://github.com/tensorflow/models/blob/master/LICENSE + +MoCo v3: https://github.com/facebookresearch/moco-v3 +available under the following license: https://github.com/facebookresearch/moco-v3/blob/main/LICENSE + +DeiT: https://github.com/facebookresearch/deit +available under the following license: https://github.com/facebookresearch/deit/blob/main/LICENSE + + +ORIGINAL COPYRIGHT NOTICE AND PERMISSION NOTICE AVAILABLE HERE IS REPRODUCE BELOW: + +https://github.com/facebookresearch/mae/blob/main/LICENSE + +Attribution-NonCommercial 4.0 International + +*************************** + +NOTICE WITH RESPECT TO THE FILE: models/blocks.py + +This software is being redistributed in a modifiled form. The original form is available here: + +https://github.com/rwightman/pytorch-image-models + +ORIGINAL COPYRIGHT NOTICE AND PERMISSION NOTICE AVAILABLE HERE IS REPRODUCE BELOW: + +https://github.com/rwightman/pytorch-image-models/blob/master/LICENSE + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/NOTICE b/src/mast3r_src/dust3r/croco/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..d51bb365036c12d428d6e3a4fd00885756d5261c --- /dev/null +++ b/src/mast3r_src/dust3r/croco/NOTICE @@ -0,0 +1,21 @@ +CroCo +Copyright 2022-present NAVER Corp. + +This project contains subcomponents with separate copyright notices and license terms. +Your use of the source code for these subcomponents is subject to the terms and conditions of the following licenses. + +==== + +facebookresearch/mae +https://github.com/facebookresearch/mae + +Attribution-NonCommercial 4.0 International + +==== + +rwightman/pytorch-image-models +https://github.com/rwightman/pytorch-image-models + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/README.MD b/src/mast3r_src/dust3r/croco/README.MD new file mode 100644 index 0000000000000000000000000000000000000000..38e33b001a60bd16749317fb297acd60f28a6f1b --- /dev/null +++ b/src/mast3r_src/dust3r/croco/README.MD @@ -0,0 +1,124 @@ +# CroCo + CroCo v2 / CroCo-Stereo / CroCo-Flow + +[[`CroCo arXiv`](https://arxiv.org/abs/2210.10716)] [[`CroCo v2 arXiv`](https://arxiv.org/abs/2211.10408)] [[`project page and demo`](https://croco.europe.naverlabs.com/)] + +This repository contains the code for our CroCo model presented in our NeurIPS'22 paper [CroCo: Self-Supervised Pre-training for 3D Vision Tasks by Cross-View Completion](https://openreview.net/pdf?id=wZEfHUM5ri) and its follow-up extension published at ICCV'23 [Improved Cross-view Completion Pre-training for Stereo Matching and Optical Flow](https://openaccess.thecvf.com/content/ICCV2023/html/Weinzaepfel_CroCo_v2_Improved_Cross-view_Completion_Pre-training_for_Stereo_Matching_and_ICCV_2023_paper.html), refered to as CroCo v2: + +![image](assets/arch.jpg) + +```bibtex +@inproceedings{croco, + title={{CroCo: Self-Supervised Pre-training for 3D Vision Tasks by Cross-View Completion}}, + author={{Weinzaepfel, Philippe and Leroy, Vincent and Lucas, Thomas and Br\'egier, Romain and Cabon, Yohann and Arora, Vaibhav and Antsfeld, Leonid and Chidlovskii, Boris and Csurka, Gabriela and Revaud J\'er\^ome}}, + booktitle={{NeurIPS}}, + year={2022} +} + +@inproceedings{croco_v2, + title={{CroCo v2: Improved Cross-view Completion Pre-training for Stereo Matching and Optical Flow}}, + author={Weinzaepfel, Philippe and Lucas, Thomas and Leroy, Vincent and Cabon, Yohann and Arora, Vaibhav and Br{\'e}gier, Romain and Csurka, Gabriela and Antsfeld, Leonid and Chidlovskii, Boris and Revaud, J{\'e}r{\^o}me}, + booktitle={ICCV}, + year={2023} +} +``` + +## License + +The code is distributed under the CC BY-NC-SA 4.0 License. See [LICENSE](LICENSE) for more information. +Some components are based on code from [MAE](https://github.com/facebookresearch/mae) released under the CC BY-NC-SA 4.0 License and [timm](https://github.com/rwightman/pytorch-image-models) released under the Apache 2.0 License. +Some components for stereo matching and optical flow are based on code from [unimatch](https://github.com/autonomousvision/unimatch) released under the MIT license. + +## Preparation + +1. Install dependencies on a machine with a NVidia GPU using e.g. conda. Note that `habitat-sim` is required only for the interactive demo and the synthetic pre-training data generation. If you don't plan to use it, you can ignore the line installing it and use a more recent python version. + +```bash +conda create -n croco python=3.7 cmake=3.14.0 +conda activate croco +conda install habitat-sim headless -c conda-forge -c aihabitat +conda install pytorch torchvision -c pytorch +conda install notebook ipykernel matplotlib +conda install ipywidgets widgetsnbextension +conda install scikit-learn tqdm quaternion opencv # only for pretraining / habitat data generation + +``` + +2. Compile cuda kernels for RoPE + +CroCo v2 relies on RoPE positional embeddings for which you need to compile some cuda kernels. +```bash +cd models/curope/ +python setup.py build_ext --inplace +cd ../../ +``` + +This can be a bit long as we compile for all cuda architectures, feel free to update L9 of `models/curope/setup.py` to compile for specific architectures only. +You might also need to set the environment `CUDA_HOME` in case you use a custom cuda installation. + +In case you cannot provide, we also provide a slow pytorch version, which will be automatically loaded. + +3. Download pre-trained model + +We provide several pre-trained models: + +| modelname | pre-training data | pos. embed. | Encoder | Decoder | +|------------------------------------------------------------------------------------------------------------------------------------|-------------------|-------------|---------|---------| +| [`CroCo.pth`](https://download.europe.naverlabs.com/ComputerVision/CroCo/CroCo.pth) | Habitat | cosine | ViT-B | Small | +| [`CroCo_V2_ViTBase_SmallDecoder.pth`](https://download.europe.naverlabs.com/ComputerVision/CroCo/CroCo_V2_ViTBase_SmallDecoder.pth) | Habitat + real | RoPE | ViT-B | Small | +| [`CroCo_V2_ViTBase_BaseDecoder.pth`](https://download.europe.naverlabs.com/ComputerVision/CroCo/CroCo_V2_ViTBase_BaseDecoder.pth) | Habitat + real | RoPE | ViT-B | Base | +| [`CroCo_V2_ViTLarge_BaseDecoder.pth`](https://download.europe.naverlabs.com/ComputerVision/CroCo/CroCo_V2_ViTLarge_BaseDecoder.pth) | Habitat + real | RoPE | ViT-L | Base | + +To download a specific model, i.e., the first one (`CroCo.pth`) +```bash +mkdir -p pretrained_models/ +wget https://download.europe.naverlabs.com/ComputerVision/CroCo/CroCo.pth -P pretrained_models/ +``` + +## Reconstruction example + +Simply run after downloading the `CroCo_V2_ViTLarge_BaseDecoder` pretrained model (or update the corresponding line in `demo.py`) +```bash +python demo.py +``` + +## Interactive demonstration of cross-view completion reconstruction on the Habitat simulator + +First download the test scene from Habitat: +```bash +python -m habitat_sim.utils.datasets_download --uids habitat_test_scenes --data-path habitat-sim-data/ +``` + +Then, run the Notebook demo `interactive_demo.ipynb`. + +In this demo, you should be able to sample a random reference viewpoint from an [Habitat](https://github.com/facebookresearch/habitat-sim) test scene. Use the sliders to change viewpoint and select a masked target view to reconstruct using CroCo. +![croco_interactive_demo](https://user-images.githubusercontent.com/1822210/200516576-7937bc6a-55f8-49ed-8618-3ddf89433ea4.jpg) + +## Pre-training + +### CroCo + +To pre-train CroCo, please first generate the pre-training data from the Habitat simulator, following the instructions in [datasets/habitat_sim/README.MD](datasets/habitat_sim/README.MD) and then run the following command: +``` +torchrun --nproc_per_node=4 pretrain.py --output_dir ./output/pretraining/ +``` + +Our CroCo pre-training was launched on a single server with 4 GPUs. +It should take around 10 days with A100 or 15 days with V100 to do the 400 pre-training epochs, but decent performances are obtained earlier in training. +Note that, while the code contains the same scaling rule of the learning rate as MAE when changing the effective batch size, we did not experimented if it is valid in our case. +The first run can take a few minutes to start, to parse all available pre-training pairs. + +### CroCo v2 + +For CroCo v2 pre-training, in addition to the generation of the pre-training data from the Habitat simulator above, please pre-extract the crops from the real datasets following the instructions in [datasets/crops/README.MD](datasets/crops/README.MD). +Then, run the following command for the largest model (ViT-L encoder, Base decoder): +``` +torchrun --nproc_per_node=8 pretrain.py --model "CroCoNet(enc_embed_dim=1024, enc_depth=24, enc_num_heads=16, dec_embed_dim=768, dec_num_heads=12, dec_depth=12, pos_embed='RoPE100')" --dataset "habitat_release+ARKitScenes+MegaDepth+3DStreetView+IndoorVL" --warmup_epochs 12 --max_epoch 125 --epochs 250 --amp 0 --keep_freq 5 --output_dir ./output/pretraining_crocov2/ +``` + +Our CroCo v2 pre-training was launched on a single server with 8 GPUs for the largest model, and on a single server with 4 GPUs for the smaller ones, keeping a batch size of 64 per gpu in all cases. +The largest model should take around 12 days on A100. +Note that, while the code contains the same scaling rule of the learning rate as MAE when changing the effective batch size, we did not experimented if it is valid in our case. + +## Stereo matching and Optical flow downstream tasks + +For CroCo-Stereo and CroCo-Flow, please refer to [stereoflow/README.MD](stereoflow/README.MD). diff --git a/src/mast3r_src/dust3r/croco/assets/Chateau1.png b/src/mast3r_src/dust3r/croco/assets/Chateau1.png new file mode 100644 index 0000000000000000000000000000000000000000..d282fc6a51c00b8dd8267d5d507220ae253c2d65 Binary files /dev/null and b/src/mast3r_src/dust3r/croco/assets/Chateau1.png differ diff --git a/src/mast3r_src/dust3r/croco/assets/Chateau2.png b/src/mast3r_src/dust3r/croco/assets/Chateau2.png new file mode 100644 index 0000000000000000000000000000000000000000..722b2fc553ec089346722efb9445526ddfa8e7bd Binary files /dev/null and b/src/mast3r_src/dust3r/croco/assets/Chateau2.png differ diff --git a/src/mast3r_src/dust3r/croco/assets/arch.jpg b/src/mast3r_src/dust3r/croco/assets/arch.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3f5b032729ddc58c06d890a0ebda1749276070c4 Binary files /dev/null and b/src/mast3r_src/dust3r/croco/assets/arch.jpg differ diff --git a/src/mast3r_src/dust3r/croco/croco-stereo-flow-demo.ipynb b/src/mast3r_src/dust3r/croco/croco-stereo-flow-demo.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..2b00a7607ab5f82d1857041969bfec977e56b3e0 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/croco-stereo-flow-demo.ipynb @@ -0,0 +1,191 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "9bca0f41", + "metadata": {}, + "source": [ + "# Simple inference example with CroCo-Stereo or CroCo-Flow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80653ef7", + "metadata": {}, + "outputs": [], + "source": [ + "# Copyright (C) 2022-present Naver Corporation. All rights reserved.\n", + "# Licensed under CC BY-NC-SA 4.0 (non-commercial use only)." + ] + }, + { + "cell_type": "markdown", + "id": "4f033862", + "metadata": {}, + "source": [ + "First download the model(s) of your choice by running\n", + "```\n", + "bash stereoflow/download_model.sh crocostereo.pth\n", + "bash stereoflow/download_model.sh crocoflow.pth\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1fb2e392", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "use_gpu = torch.cuda.is_available() and torch.cuda.device_count()>0\n", + "device = torch.device('cuda:0' if use_gpu else 'cpu')\n", + "import matplotlib.pylab as plt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e0e25d77", + "metadata": {}, + "outputs": [], + "source": [ + "from stereoflow.test import _load_model_and_criterion\n", + "from stereoflow.engine import tiled_pred\n", + "from stereoflow.datasets_stereo import img_to_tensor, vis_disparity\n", + "from stereoflow.datasets_flow import flowToColor\n", + "tile_overlap=0.7 # recommended value, higher value can be slightly better but slower" + ] + }, + { + "cell_type": "markdown", + "id": "86a921f5", + "metadata": {}, + "source": [ + "### CroCo-Stereo example" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64e483cb", + "metadata": {}, + "outputs": [], + "source": [ + "image1 = np.asarray(Image.open(''))\n", + "image2 = np.asarray(Image.open(''))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0d04303", + "metadata": {}, + "outputs": [], + "source": [ + "model, _, cropsize, with_conf, task, tile_conf_mode = _load_model_and_criterion('stereoflow_models/crocostereo.pth', None, device)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47dc14b5", + "metadata": {}, + "outputs": [], + "source": [ + "im1 = img_to_tensor(image1).to(device).unsqueeze(0)\n", + "im2 = img_to_tensor(image2).to(device).unsqueeze(0)\n", + "with torch.inference_mode():\n", + " pred, _, _ = tiled_pred(model, None, im1, im2, None, conf_mode=tile_conf_mode, overlap=tile_overlap, crop=cropsize, with_conf=with_conf, return_time=False)\n", + "pred = pred.squeeze(0).squeeze(0).cpu().numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "583b9f16", + "metadata": {}, + "outputs": [], + "source": [ + "plt.imshow(vis_disparity(pred))\n", + "plt.axis('off')" + ] + }, + { + "cell_type": "markdown", + "id": "d2df5d70", + "metadata": {}, + "source": [ + "### CroCo-Flow example" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ee257a7", + "metadata": {}, + "outputs": [], + "source": [ + "image1 = np.asarray(Image.open(''))\n", + "image2 = np.asarray(Image.open(''))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5edccf0", + "metadata": {}, + "outputs": [], + "source": [ + "model, _, cropsize, with_conf, task, tile_conf_mode = _load_model_and_criterion('stereoflow_models/crocoflow.pth', None, device)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b19692c3", + "metadata": {}, + "outputs": [], + "source": [ + "im1 = img_to_tensor(image1).to(device).unsqueeze(0)\n", + "im2 = img_to_tensor(image2).to(device).unsqueeze(0)\n", + "with torch.inference_mode():\n", + " pred, _, _ = tiled_pred(model, None, im1, im2, None, conf_mode=tile_conf_mode, overlap=tile_overlap, crop=cropsize, with_conf=with_conf, return_time=False)\n", + "pred = pred.squeeze(0).permute(1,2,0).cpu().numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26f79db3", + "metadata": {}, + "outputs": [], + "source": [ + "plt.imshow(flowToColor(pred))\n", + "plt.axis('off')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/mast3r_src/dust3r/croco/datasets/__init__.py b/src/mast3r_src/dust3r/croco/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/mast3r_src/dust3r/croco/datasets/crops/README.MD b/src/mast3r_src/dust3r/croco/datasets/crops/README.MD new file mode 100644 index 0000000000000000000000000000000000000000..47ddabebb177644694ee247ae878173a3a16644f --- /dev/null +++ b/src/mast3r_src/dust3r/croco/datasets/crops/README.MD @@ -0,0 +1,104 @@ +## Generation of crops from the real datasets + +The instructions below allow to generate the crops used for pre-training CroCo v2 from the following real-world datasets: ARKitScenes, MegaDepth, 3DStreetView and IndoorVL. + +### Download the metadata of the crops to generate + +First, download the metadata and put them in `./data/`: +``` +mkdir -p data +cd data/ +wget https://download.europe.naverlabs.com/ComputerVision/CroCo/data/crop_metadata.zip +unzip crop_metadata.zip +rm crop_metadata.zip +cd .. +``` + +### Prepare the original datasets + +Second, download the original datasets in `./data/original_datasets/`. +``` +mkdir -p data/original_datasets +``` + +##### ARKitScenes + +Download the `raw` dataset from https://github.com/apple/ARKitScenes/blob/main/DATA.md and put it in `./data/original_datasets/ARKitScenes/`. +The resulting file structure should be like: +``` +./data/original_datasets/ARKitScenes/ +└───Training + └───40753679 + │ │ ultrawide + │ │ ... + └───40753686 + │ + ... +``` + +##### MegaDepth + +Download `MegaDepth v1 Dataset` from https://www.cs.cornell.edu/projects/megadepth/ and put it in `./data/original_datasets/MegaDepth/`. +The resulting file structure should be like: + +``` +./data/original_datasets/MegaDepth/ +└───0000 +│ └───images +│ │ │ 1000557903_87fa96b8a4_o.jpg +│ │ └ ... +│ └─── ... +└───0001 +│ │ +│ └ ... +└─── ... +``` + +##### 3DStreetView + +Download `3D_Street_View` dataset from https://github.com/amir32002/3D_Street_View and put it in `./data/original_datasets/3DStreetView/`. +The resulting file structure should be like: + +``` +./data/original_datasets/3DStreetView/ +└───dataset_aligned +│ └───0002 +│ │ │ 0000002_0000001_0000002_0000001.jpg +│ │ └ ... +│ └─── ... +└───dataset_unaligned +│ └───0003 +│ │ │ 0000003_0000001_0000002_0000001.jpg +│ │ └ ... +│ └─── ... +``` + +##### IndoorVL + +Download the `IndoorVL` datasets using [Kapture](https://github.com/naver/kapture). + +``` +pip install kapture +mkdir -p ./data/original_datasets/IndoorVL +cd ./data/original_datasets/IndoorVL +kapture_download_dataset.py update +kapture_download_dataset.py install "HyundaiDepartmentStore_*" +kapture_download_dataset.py install "GangnamStation_*" +cd - +``` + +### Extract the crops + +Now, extract the crops for each of the dataset: +``` +for dataset in ARKitScenes MegaDepth 3DStreetView IndoorVL; +do + python3 datasets/crops/extract_crops_from_images.py --crops ./data/crop_metadata/${dataset}/crops_release.txt --root-dir ./data/original_datasets/${dataset}/ --output-dir ./data/${dataset}_crops/ --imsize 256 --nthread 8 --max-subdir-levels 5 --ideal-number-pairs-in-dir 500; +done +``` + +##### Note for IndoorVL + +Due to some legal issues, we can only release 144,228 pairs out of the 1,593,689 pairs used in the paper. +To account for it in terms of number of pre-training iterations, the pre-training command in this repository uses 125 training epochs including 12 warm-up epochs and learning rate cosine schedule of 250, instead of 100, 10 and 200 respectively. +The impact on the performance is negligible. diff --git a/src/mast3r_src/dust3r/croco/datasets/crops/extract_crops_from_images.py b/src/mast3r_src/dust3r/croco/datasets/crops/extract_crops_from_images.py new file mode 100644 index 0000000000000000000000000000000000000000..eb66a0474ce44b54c44c08887cbafdb045b11ff3 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/datasets/crops/extract_crops_from_images.py @@ -0,0 +1,159 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Extracting crops for pre-training +# -------------------------------------------------------- + +import os +import argparse +from tqdm import tqdm +from PIL import Image +import functools +from multiprocessing import Pool +import math + + +def arg_parser(): + parser = argparse.ArgumentParser('Generate cropped image pairs from image crop list') + + parser.add_argument('--crops', type=str, required=True, help='crop file') + parser.add_argument('--root-dir', type=str, required=True, help='root directory') + parser.add_argument('--output-dir', type=str, required=True, help='output directory') + parser.add_argument('--imsize', type=int, default=256, help='size of the crops') + parser.add_argument('--nthread', type=int, required=True, help='number of simultaneous threads') + parser.add_argument('--max-subdir-levels', type=int, default=5, help='maximum number of subdirectories') + parser.add_argument('--ideal-number-pairs-in-dir', type=int, default=500, help='number of pairs stored in a dir') + return parser + + +def main(args): + listing_path = os.path.join(args.output_dir, 'listing.txt') + + print(f'Loading list of crops ... ({args.nthread} threads)') + crops, num_crops_to_generate = load_crop_file(args.crops) + + print(f'Preparing jobs ({len(crops)} candidate image pairs)...') + num_levels = min(math.ceil(math.log(num_crops_to_generate, args.ideal_number_pairs_in_dir)), args.max_subdir_levels) + num_pairs_in_dir = math.ceil(num_crops_to_generate ** (1/num_levels)) + + jobs = prepare_jobs(crops, num_levels, num_pairs_in_dir) + del crops + + os.makedirs(args.output_dir, exist_ok=True) + mmap = Pool(args.nthread).imap_unordered if args.nthread > 1 else map + call = functools.partial(save_image_crops, args) + + print(f"Generating cropped images to {args.output_dir} ...") + with open(listing_path, 'w') as listing: + listing.write('# pair_path\n') + for results in tqdm(mmap(call, jobs), total=len(jobs)): + for path in results: + listing.write(f'{path}\n') + print('Finished writing listing to', listing_path) + + +def load_crop_file(path): + data = open(path).read().splitlines() + pairs = [] + num_crops_to_generate = 0 + for line in tqdm(data): + if line.startswith('#'): + continue + line = line.split(', ') + if len(line) < 8: + img1, img2, rotation = line + pairs.append((img1, img2, int(rotation), [])) + else: + l1, r1, t1, b1, l2, r2, t2, b2 = map(int, line) + rect1, rect2 = (l1, t1, r1, b1), (l2, t2, r2, b2) + pairs[-1][-1].append((rect1, rect2)) + num_crops_to_generate += 1 + return pairs, num_crops_to_generate + + +def prepare_jobs(pairs, num_levels, num_pairs_in_dir): + jobs = [] + powers = [num_pairs_in_dir**level for level in reversed(range(num_levels))] + + def get_path(idx): + idx_array = [] + d = idx + for level in range(num_levels - 1): + idx_array.append(idx // powers[level]) + idx = idx % powers[level] + idx_array.append(d) + return '/'.join(map(lambda x: hex(x)[2:], idx_array)) + + idx = 0 + for pair_data in tqdm(pairs): + img1, img2, rotation, crops = pair_data + if -60 <= rotation and rotation <= 60: + rotation = 0 # most likely not a true rotation + paths = [get_path(idx + k) for k in range(len(crops))] + idx += len(crops) + jobs.append(((img1, img2), rotation, crops, paths)) + return jobs + + +def load_image(path): + try: + return Image.open(path).convert('RGB') + except Exception as e: + print('skipping', path, e) + raise OSError() + + +def save_image_crops(args, data): + # load images + img_pair, rot, crops, paths = data + try: + img1, img2 = [load_image(os.path.join(args.root_dir, impath)) for impath in img_pair] + except OSError as e: + return [] + + def area(sz): + return sz[0] * sz[1] + + tgt_size = (args.imsize, args.imsize) + + def prepare_crop(img, rect, rot=0): + # actual crop + img = img.crop(rect) + + # resize to desired size + interp = Image.Resampling.LANCZOS if area(img.size) > 4*area(tgt_size) else Image.Resampling.BICUBIC + img = img.resize(tgt_size, resample=interp) + + # rotate the image + rot90 = (round(rot/90) % 4) * 90 + if rot90 == 90: + img = img.transpose(Image.Transpose.ROTATE_90) + elif rot90 == 180: + img = img.transpose(Image.Transpose.ROTATE_180) + elif rot90 == 270: + img = img.transpose(Image.Transpose.ROTATE_270) + return img + + results = [] + for (rect1, rect2), path in zip(crops, paths): + crop1 = prepare_crop(img1, rect1) + crop2 = prepare_crop(img2, rect2, rot) + + fullpath1 = os.path.join(args.output_dir, path+'_1.jpg') + fullpath2 = os.path.join(args.output_dir, path+'_2.jpg') + os.makedirs(os.path.dirname(fullpath1), exist_ok=True) + + assert not os.path.isfile(fullpath1), fullpath1 + assert not os.path.isfile(fullpath2), fullpath2 + crop1.save(fullpath1) + crop2.save(fullpath2) + results.append(path) + + return results + + +if __name__ == '__main__': + args = arg_parser().parse_args() + main(args) + diff --git a/src/mast3r_src/dust3r/croco/datasets/habitat_sim/README.MD b/src/mast3r_src/dust3r/croco/datasets/habitat_sim/README.MD new file mode 100644 index 0000000000000000000000000000000000000000..a505781ff9eb91bce7f1d189e848f8ba1c560940 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/datasets/habitat_sim/README.MD @@ -0,0 +1,76 @@ +## Generation of synthetic image pairs using Habitat-Sim + +These instructions allow to generate pre-training pairs from the Habitat simulator. +As we did not save metadata of the pairs used in the original paper, they are not strictly the same, but these data use the same setting and are equivalent. + +### Download Habitat-Sim scenes +Download Habitat-Sim scenes: +- Download links can be found here: https://github.com/facebookresearch/habitat-sim/blob/main/DATASETS.md +- We used scenes from the HM3D, habitat-test-scenes, Replica, ReplicaCad and ScanNet datasets. +- Please put the scenes under `./data/habitat-sim-data/scene_datasets/` following the structure below, or update manually paths in `paths.py`. +``` +./data/ +└──habitat-sim-data/ + └──scene_datasets/ + ├──hm3d/ + ├──gibson/ + ├──habitat-test-scenes/ + ├──replica_cad_baked_lighting/ + ├──replica_cad/ + ├──ReplicaDataset/ + └──scannet/ +``` + +### Image pairs generation +We provide metadata to generate reproducible images pairs for pretraining and validation. +Experiments described in the paper used similar data, but whose generation was not reproducible at the time. + +Specifications: +- 256x256 resolution images, with 60 degrees field of view . +- Up to 1000 image pairs per scene. +- Number of scenes considered/number of images pairs per dataset: + - Scannet: 1097 scenes / 985 209 pairs + - HM3D: + - hm3d/train: 800 / 800k pairs + - hm3d/val: 100 scenes / 100k pairs + - hm3d/minival: 10 scenes / 10k pairs + - habitat-test-scenes: 3 scenes / 3k pairs + - replica_cad_baked_lighting: 13 scenes / 13k pairs + +- Scenes from hm3d/val and hm3d/minival pairs were not used for the pre-training but kept for validation purposes. + +Download metadata and extract it: +```bash +mkdir -p data/habitat_release_metadata/ +cd data/habitat_release_metadata/ +wget https://download.europe.naverlabs.com/ComputerVision/CroCo/data/habitat_release_metadata/multiview_habitat_metadata.tar.gz +tar -xvf multiview_habitat_metadata.tar.gz +cd ../.. +# Location of the metadata +METADATA_DIR="./data/habitat_release_metadata/multiview_habitat_metadata" +``` + +Generate image pairs from metadata: +- The following command will print a list of commandlines to generate image pairs for each scene: +```bash +# Target output directory +PAIRS_DATASET_DIR="./data/habitat_release/" +python datasets/habitat_sim/generate_from_metadata_files.py --input_dir=$METADATA_DIR --output_dir=$PAIRS_DATASET_DIR +``` +- One can launch multiple of such commands in parallel e.g. using GNU Parallel: +```bash +python datasets/habitat_sim/generate_from_metadata_files.py --input_dir=$METADATA_DIR --output_dir=$PAIRS_DATASET_DIR | parallel -j 16 +``` + +## Metadata generation + +Image pairs were randomly sampled using the following commands, whose outputs contain randomness and are thus not exactly reproducible: +```bash +# Print commandlines to generate image pairs from the different scenes available. +PAIRS_DATASET_DIR=MY_CUSTOM_PATH +python datasets/habitat_sim/generate_multiview_images.py --list_commands --output_dir=$PAIRS_DATASET_DIR + +# Once a dataset is generated, pack metadata files for reproducibility. +METADATA_DIR=MY_CUSTON_PATH +python datasets/habitat_sim/pack_metadata_files.py $PAIRS_DATASET_DIR $METADATA_DIR +``` diff --git a/src/mast3r_src/dust3r/croco/datasets/habitat_sim/__init__.py b/src/mast3r_src/dust3r/croco/datasets/habitat_sim/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/mast3r_src/dust3r/croco/datasets/habitat_sim/generate_from_metadata.py b/src/mast3r_src/dust3r/croco/datasets/habitat_sim/generate_from_metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..fbe0d399084359495250dc8184671ff498adfbf2 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/datasets/habitat_sim/generate_from_metadata.py @@ -0,0 +1,92 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +""" +Script to generate image pairs for a given scene reproducing poses provided in a metadata file. +""" +import os +from datasets.habitat_sim.multiview_habitat_sim_generator import MultiviewHabitatSimGenerator +from datasets.habitat_sim.paths import SCENES_DATASET +import argparse +import quaternion +import PIL.Image +import cv2 +import json +from tqdm import tqdm + +def generate_multiview_images_from_metadata(metadata_filename, + output_dir, + overload_params = dict(), + scene_datasets_paths=None, + exist_ok=False): + """ + Generate images from a metadata file for reproducibility purposes. + """ + # Reorder paths by decreasing label length, to avoid collisions when testing if a string by such label + if scene_datasets_paths is not None: + scene_datasets_paths = dict(sorted(scene_datasets_paths.items(), key= lambda x: len(x[0]), reverse=True)) + + with open(metadata_filename, 'r') as f: + input_metadata = json.load(f) + metadata = dict() + for key, value in input_metadata.items(): + # Optionally replace some paths + if key in ("scene_dataset_config_file", "scene", "navmesh") and value != "": + if scene_datasets_paths is not None: + for dataset_label, dataset_path in scene_datasets_paths.items(): + if value.startswith(dataset_label): + value = os.path.normpath(os.path.join(dataset_path, os.path.relpath(value, dataset_label))) + break + metadata[key] = value + + # Overload some parameters + for key, value in overload_params.items(): + metadata[key] = value + + generation_entries = dict([(key, value) for key, value in metadata.items() if not (key in ('multiviews', 'output_dir', 'generate_depth'))]) + generate_depth = metadata["generate_depth"] + + os.makedirs(output_dir, exist_ok=exist_ok) + + generator = MultiviewHabitatSimGenerator(**generation_entries) + + # Generate views + for idx_label, data in tqdm(metadata['multiviews'].items()): + positions = data["positions"] + orientations = data["orientations"] + n = len(positions) + for oidx in range(n): + observation = generator.render_viewpoint(positions[oidx], quaternion.from_float_array(orientations[oidx])) + observation_label = f"{oidx + 1}" # Leonid is indexing starting from 1 + # Color image saved using PIL + img = PIL.Image.fromarray(observation['color'][:,:,:3]) + filename = os.path.join(output_dir, f"{idx_label}_{observation_label}.jpeg") + img.save(filename) + if generate_depth: + # Depth image as EXR file + filename = os.path.join(output_dir, f"{idx_label}_{observation_label}_depth.exr") + cv2.imwrite(filename, observation['depth'], [cv2.IMWRITE_EXR_TYPE, cv2.IMWRITE_EXR_TYPE_HALF]) + # Camera parameters + camera_params = dict([(key, observation[key].tolist()) for key in ("camera_intrinsics", "R_cam2world", "t_cam2world")]) + filename = os.path.join(output_dir, f"{idx_label}_{observation_label}_camera_params.json") + with open(filename, "w") as f: + json.dump(camera_params, f) + # Save metadata + with open(os.path.join(output_dir, "metadata.json"), "w") as f: + json.dump(metadata, f) + + generator.close() + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--metadata_filename", required=True) + parser.add_argument("--output_dir", required=True) + args = parser.parse_args() + + generate_multiview_images_from_metadata(metadata_filename=args.metadata_filename, + output_dir=args.output_dir, + scene_datasets_paths=SCENES_DATASET, + overload_params=dict(), + exist_ok=True) + + \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/datasets/habitat_sim/generate_from_metadata_files.py b/src/mast3r_src/dust3r/croco/datasets/habitat_sim/generate_from_metadata_files.py new file mode 100644 index 0000000000000000000000000000000000000000..962ef849d8c31397b8622df4f2d9140175d78873 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/datasets/habitat_sim/generate_from_metadata_files.py @@ -0,0 +1,27 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +""" +Script generating commandlines to generate image pairs from metadata files. +""" +import os +import glob +from tqdm import tqdm +import argparse + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--input_dir", required=True) + parser.add_argument("--output_dir", required=True) + parser.add_argument("--prefix", default="", help="Commanline prefix, useful e.g. to setup environment.") + args = parser.parse_args() + + input_metadata_filenames = glob.iglob(f"{args.input_dir}/**/metadata.json", recursive=True) + + for metadata_filename in tqdm(input_metadata_filenames): + output_dir = os.path.join(args.output_dir, os.path.relpath(os.path.dirname(metadata_filename), args.input_dir)) + # Do not process the scene if the metadata file already exists + if os.path.exists(os.path.join(output_dir, "metadata.json")): + continue + commandline = f"{args.prefix}python datasets/habitat_sim/generate_from_metadata.py --metadata_filename={metadata_filename} --output_dir={output_dir}" + print(commandline) diff --git a/src/mast3r_src/dust3r/croco/datasets/habitat_sim/generate_multiview_images.py b/src/mast3r_src/dust3r/croco/datasets/habitat_sim/generate_multiview_images.py new file mode 100644 index 0000000000000000000000000000000000000000..421d49a1696474415940493296b3f2d982398850 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/datasets/habitat_sim/generate_multiview_images.py @@ -0,0 +1,177 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +import os +from tqdm import tqdm +import argparse +import PIL.Image +import numpy as np +import json +from datasets.habitat_sim.multiview_habitat_sim_generator import MultiviewHabitatSimGenerator, NoNaviguableSpaceError +from datasets.habitat_sim.paths import list_scenes_available +import cv2 +import quaternion +import shutil + +def generate_multiview_images_for_scene(scene_dataset_config_file, + scene, + navmesh, + output_dir, + views_count, + size, + exist_ok=False, + generate_depth=False, + **kwargs): + """ + Generate tuples of overlapping views for a given scene. + generate_depth: generate depth images and camera parameters. + """ + if os.path.exists(output_dir) and not exist_ok: + print(f"Scene {scene}: data already generated. Ignoring generation.") + return + try: + print(f"Scene {scene}: {size} multiview acquisitions to generate...") + os.makedirs(output_dir, exist_ok=exist_ok) + + metadata_filename = os.path.join(output_dir, "metadata.json") + + metadata_template = dict(scene_dataset_config_file=scene_dataset_config_file, + scene=scene, + navmesh=navmesh, + views_count=views_count, + size=size, + generate_depth=generate_depth, + **kwargs) + metadata_template["multiviews"] = dict() + + if os.path.exists(metadata_filename): + print("Metadata file already exists:", metadata_filename) + print("Loading already generated metadata file...") + with open(metadata_filename, "r") as f: + metadata = json.load(f) + + for key in metadata_template.keys(): + if key != "multiviews": + assert metadata_template[key] == metadata[key], f"existing file is inconsistent with the input parameters:\nKey: {key}\nmetadata: {metadata[key]}\ntemplate: {metadata_template[key]}." + else: + print("No temporary file found. Starting generation from scratch...") + metadata = metadata_template + + starting_id = len(metadata["multiviews"]) + print(f"Starting generation from index {starting_id}/{size}...") + if starting_id >= size: + print("Generation already done.") + return + + generator = MultiviewHabitatSimGenerator(scene_dataset_config_file=scene_dataset_config_file, + scene=scene, + navmesh=navmesh, + views_count = views_count, + size = size, + **kwargs) + + for idx in tqdm(range(starting_id, size)): + # Generate / re-generate the observations + try: + data = generator[idx] + observations = data["observations"] + positions = data["positions"] + orientations = data["orientations"] + + idx_label = f"{idx:08}" + for oidx, observation in enumerate(observations): + observation_label = f"{oidx + 1}" # Leonid is indexing starting from 1 + # Color image saved using PIL + img = PIL.Image.fromarray(observation['color'][:,:,:3]) + filename = os.path.join(output_dir, f"{idx_label}_{observation_label}.jpeg") + img.save(filename) + if generate_depth: + # Depth image as EXR file + filename = os.path.join(output_dir, f"{idx_label}_{observation_label}_depth.exr") + cv2.imwrite(filename, observation['depth'], [cv2.IMWRITE_EXR_TYPE, cv2.IMWRITE_EXR_TYPE_HALF]) + # Camera parameters + camera_params = dict([(key, observation[key].tolist()) for key in ("camera_intrinsics", "R_cam2world", "t_cam2world")]) + filename = os.path.join(output_dir, f"{idx_label}_{observation_label}_camera_params.json") + with open(filename, "w") as f: + json.dump(camera_params, f) + metadata["multiviews"][idx_label] = {"positions": positions.tolist(), + "orientations": orientations.tolist(), + "covisibility_ratios": data["covisibility_ratios"].tolist(), + "valid_fractions": data["valid_fractions"].tolist(), + "pairwise_visibility_ratios": data["pairwise_visibility_ratios"].tolist()} + except RecursionError: + print("Recursion error: unable to sample observations for this scene. We will stop there.") + break + + # Regularly save a temporary metadata file, in case we need to restart the generation + if idx % 10 == 0: + with open(metadata_filename, "w") as f: + json.dump(metadata, f) + + # Save metadata + with open(metadata_filename, "w") as f: + json.dump(metadata, f) + + generator.close() + except NoNaviguableSpaceError: + pass + +def create_commandline(scene_data, generate_depth, exist_ok=False): + """ + Create a commandline string to generate a scene. + """ + def my_formatting(val): + if val is None or val == "": + return '""' + else: + return val + commandline = f"""python {__file__} --scene {my_formatting(scene_data.scene)} + --scene_dataset_config_file {my_formatting(scene_data.scene_dataset_config_file)} + --navmesh {my_formatting(scene_data.navmesh)} + --output_dir {my_formatting(scene_data.output_dir)} + --generate_depth {int(generate_depth)} + --exist_ok {int(exist_ok)} + """ + commandline = " ".join(commandline.split()) + return commandline + +if __name__ == "__main__": + os.umask(2) + + parser = argparse.ArgumentParser(description="""Example of use -- listing commands to generate data for scenes available: + > python datasets/habitat_sim/generate_multiview_habitat_images.py --list_commands + """) + + parser.add_argument("--output_dir", type=str, required=True) + parser.add_argument("--list_commands", action='store_true', help="list commandlines to run if true") + parser.add_argument("--scene", type=str, default="") + parser.add_argument("--scene_dataset_config_file", type=str, default="") + parser.add_argument("--navmesh", type=str, default="") + + parser.add_argument("--generate_depth", type=int, default=1) + parser.add_argument("--exist_ok", type=int, default=0) + + kwargs = dict(resolution=(256,256), hfov=60, views_count = 2, size=1000) + + args = parser.parse_args() + generate_depth=bool(args.generate_depth) + exist_ok = bool(args.exist_ok) + + if args.list_commands: + # Listing scenes available... + scenes_data = list_scenes_available(base_output_dir=args.output_dir) + + for scene_data in scenes_data: + print(create_commandline(scene_data, generate_depth=generate_depth, exist_ok=exist_ok)) + else: + if args.scene == "" or args.output_dir == "": + print("Missing scene or output dir argument!") + print(parser.format_help()) + else: + generate_multiview_images_for_scene(scene=args.scene, + scene_dataset_config_file = args.scene_dataset_config_file, + navmesh = args.navmesh, + output_dir = args.output_dir, + exist_ok=exist_ok, + generate_depth=generate_depth, + **kwargs) \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/datasets/habitat_sim/multiview_habitat_sim_generator.py b/src/mast3r_src/dust3r/croco/datasets/habitat_sim/multiview_habitat_sim_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..91e5f923b836a645caf5d8e4aacc425047e3c144 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/datasets/habitat_sim/multiview_habitat_sim_generator.py @@ -0,0 +1,390 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +import os +import numpy as np +import quaternion +import habitat_sim +import json +from sklearn.neighbors import NearestNeighbors +import cv2 + +# OpenCV to habitat camera convention transformation +R_OPENCV2HABITAT = np.stack((habitat_sim.geo.RIGHT, -habitat_sim.geo.UP, habitat_sim.geo.FRONT), axis=0) +R_HABITAT2OPENCV = R_OPENCV2HABITAT.T +DEG2RAD = np.pi / 180 + +def compute_camera_intrinsics(height, width, hfov): + f = width/2 / np.tan(hfov/2 * np.pi/180) + cu, cv = width/2, height/2 + return f, cu, cv + +def compute_camera_pose_opencv_convention(camera_position, camera_orientation): + R_cam2world = quaternion.as_rotation_matrix(camera_orientation) @ R_OPENCV2HABITAT + t_cam2world = np.asarray(camera_position) + return R_cam2world, t_cam2world + +def compute_pointmap(depthmap, hfov): + """ Compute a HxWx3 pointmap in camera frame from a HxW depth map.""" + height, width = depthmap.shape + f, cu, cv = compute_camera_intrinsics(height, width, hfov) + # Cast depth map to point + z_cam = depthmap + u, v = np.meshgrid(range(width), range(height)) + x_cam = (u - cu) / f * z_cam + y_cam = (v - cv) / f * z_cam + X_cam = np.stack((x_cam, y_cam, z_cam), axis=-1) + return X_cam + +def compute_pointcloud(depthmap, hfov, camera_position, camera_rotation): + """Return a 3D point cloud corresponding to valid pixels of the depth map""" + R_cam2world, t_cam2world = compute_camera_pose_opencv_convention(camera_position, camera_rotation) + + X_cam = compute_pointmap(depthmap=depthmap, hfov=hfov) + valid_mask = (X_cam[:,:,2] != 0.0) + + X_cam = X_cam.reshape(-1, 3)[valid_mask.flatten()] + X_world = X_cam @ R_cam2world.T + t_cam2world.reshape(1, 3) + return X_world + +def compute_pointcloud_overlaps_scikit(pointcloud1, pointcloud2, distance_threshold, compute_symmetric=False): + """ + Compute 'overlapping' metrics based on a distance threshold between two point clouds. + """ + nbrs = NearestNeighbors(n_neighbors=1, algorithm = 'kd_tree').fit(pointcloud2) + distances, indices = nbrs.kneighbors(pointcloud1) + intersection1 = np.count_nonzero(distances.flatten() < distance_threshold) + + data = {"intersection1": intersection1, + "size1": len(pointcloud1)} + if compute_symmetric: + nbrs = NearestNeighbors(n_neighbors=1, algorithm = 'kd_tree').fit(pointcloud1) + distances, indices = nbrs.kneighbors(pointcloud2) + intersection2 = np.count_nonzero(distances.flatten() < distance_threshold) + data["intersection2"] = intersection2 + data["size2"] = len(pointcloud2) + + return data + +def _append_camera_parameters(observation, hfov, camera_location, camera_rotation): + """ + Add camera parameters to the observation dictionnary produced by Habitat-Sim + In-place modifications. + """ + R_cam2world, t_cam2world = compute_camera_pose_opencv_convention(camera_location, camera_rotation) + height, width = observation['depth'].shape + f, cu, cv = compute_camera_intrinsics(height, width, hfov) + K = np.asarray([[f, 0, cu], + [0, f, cv], + [0, 0, 1.0]]) + observation["camera_intrinsics"] = K + observation["t_cam2world"] = t_cam2world + observation["R_cam2world"] = R_cam2world + +def look_at(eye, center, up, return_cam2world=True): + """ + Return camera pose looking at a given center point. + Analogous of gluLookAt function, using OpenCV camera convention. + """ + z = center - eye + z /= np.linalg.norm(z, axis=-1, keepdims=True) + y = -up + y = y - np.sum(y * z, axis=-1, keepdims=True) * z + y /= np.linalg.norm(y, axis=-1, keepdims=True) + x = np.cross(y, z, axis=-1) + + if return_cam2world: + R = np.stack((x, y, z), axis=-1) + t = eye + else: + # World to camera transformation + # Transposed matrix + R = np.stack((x, y, z), axis=-2) + t = - np.einsum('...ij, ...j', R, eye) + return R, t + +def look_at_for_habitat(eye, center, up, return_cam2world=True): + R, t = look_at(eye, center, up) + orientation = quaternion.from_rotation_matrix(R @ R_OPENCV2HABITAT.T) + return orientation, t + +def generate_orientation_noise(pan_range, tilt_range, roll_range): + return (quaternion.from_rotation_vector(np.random.uniform(*pan_range) * DEG2RAD * habitat_sim.geo.UP) + * quaternion.from_rotation_vector(np.random.uniform(*tilt_range) * DEG2RAD * habitat_sim.geo.RIGHT) + * quaternion.from_rotation_vector(np.random.uniform(*roll_range) * DEG2RAD * habitat_sim.geo.FRONT)) + + +class NoNaviguableSpaceError(RuntimeError): + def __init__(self, *args): + super().__init__(*args) + +class MultiviewHabitatSimGenerator: + def __init__(self, + scene, + navmesh, + scene_dataset_config_file, + resolution = (240, 320), + views_count=2, + hfov = 60, + gpu_id = 0, + size = 10000, + minimum_covisibility = 0.5, + transform = None): + self.scene = scene + self.navmesh = navmesh + self.scene_dataset_config_file = scene_dataset_config_file + self.resolution = resolution + self.views_count = views_count + assert(self.views_count >= 1) + self.hfov = hfov + self.gpu_id = gpu_id + self.size = size + self.transform = transform + + # Noise added to camera orientation + self.pan_range = (-3, 3) + self.tilt_range = (-10, 10) + self.roll_range = (-5, 5) + + # Height range to sample cameras + self.height_range = (1.2, 1.8) + + # Random steps between the camera views + self.random_steps_count = 5 + self.random_step_variance = 2.0 + + # Minimum fraction of the scene which should be valid (well defined depth) + self.minimum_valid_fraction = 0.7 + + # Distance threshold to see to select pairs + self.distance_threshold = 0.05 + # Minimum IoU of a view point cloud with respect to the reference view to be kept. + self.minimum_covisibility = minimum_covisibility + + # Maximum number of retries. + self.max_attempts_count = 100 + + self.seed = None + self._lazy_initialization() + + def _lazy_initialization(self): + # Lazy random seeding and instantiation of the simulator to deal with multiprocessing properly + if self.seed == None: + # Re-seed numpy generator + np.random.seed() + self.seed = np.random.randint(2**32-1) + sim_cfg = habitat_sim.SimulatorConfiguration() + sim_cfg.scene_id = self.scene + if self.scene_dataset_config_file is not None and self.scene_dataset_config_file != "": + sim_cfg.scene_dataset_config_file = self.scene_dataset_config_file + sim_cfg.random_seed = self.seed + sim_cfg.load_semantic_mesh = False + sim_cfg.gpu_device_id = self.gpu_id + + depth_sensor_spec = habitat_sim.CameraSensorSpec() + depth_sensor_spec.uuid = "depth" + depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH + depth_sensor_spec.resolution = self.resolution + depth_sensor_spec.hfov = self.hfov + depth_sensor_spec.position = [0.0, 0.0, 0] + depth_sensor_spec.orientation + + rgb_sensor_spec = habitat_sim.CameraSensorSpec() + rgb_sensor_spec.uuid = "color" + rgb_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR + rgb_sensor_spec.resolution = self.resolution + rgb_sensor_spec.hfov = self.hfov + rgb_sensor_spec.position = [0.0, 0.0, 0] + agent_cfg = habitat_sim.agent.AgentConfiguration(sensor_specifications=[rgb_sensor_spec, depth_sensor_spec]) + + cfg = habitat_sim.Configuration(sim_cfg, [agent_cfg]) + self.sim = habitat_sim.Simulator(cfg) + if self.navmesh is not None and self.navmesh != "": + # Use pre-computed navmesh when available (usually better than those generated automatically) + self.sim.pathfinder.load_nav_mesh(self.navmesh) + + if not self.sim.pathfinder.is_loaded: + # Try to compute a navmesh + navmesh_settings = habitat_sim.NavMeshSettings() + navmesh_settings.set_defaults() + self.sim.recompute_navmesh(self.sim.pathfinder, navmesh_settings, True) + + # Ensure that the navmesh is not empty + if not self.sim.pathfinder.is_loaded: + raise NoNaviguableSpaceError(f"No naviguable location (scene: {self.scene} -- navmesh: {self.navmesh})") + + self.agent = self.sim.initialize_agent(agent_id=0) + + def close(self): + self.sim.close() + + def __del__(self): + self.sim.close() + + def __len__(self): + return self.size + + def sample_random_viewpoint(self): + """ Sample a random viewpoint using the navmesh """ + nav_point = self.sim.pathfinder.get_random_navigable_point() + + # Sample a random viewpoint height + viewpoint_height = np.random.uniform(*self.height_range) + viewpoint_position = nav_point + viewpoint_height * habitat_sim.geo.UP + viewpoint_orientation = quaternion.from_rotation_vector(np.random.uniform(0, 2 * np.pi) * habitat_sim.geo.UP) * generate_orientation_noise(self.pan_range, self.tilt_range, self.roll_range) + return viewpoint_position, viewpoint_orientation, nav_point + + def sample_other_random_viewpoint(self, observed_point, nav_point): + """ Sample a random viewpoint close to an existing one, using the navmesh and a reference observed point.""" + other_nav_point = nav_point + + walk_directions = self.random_step_variance * np.asarray([1,0,1]) + for i in range(self.random_steps_count): + temp = self.sim.pathfinder.snap_point(other_nav_point + walk_directions * np.random.normal(size=3)) + # Snapping may return nan when it fails + if not np.isnan(temp[0]): + other_nav_point = temp + + other_viewpoint_height = np.random.uniform(*self.height_range) + other_viewpoint_position = other_nav_point + other_viewpoint_height * habitat_sim.geo.UP + + # Set viewing direction towards the central point + rotation, position = look_at_for_habitat(eye=other_viewpoint_position, center=observed_point, up=habitat_sim.geo.UP, return_cam2world=True) + rotation = rotation * generate_orientation_noise(self.pan_range, self.tilt_range, self.roll_range) + return position, rotation, other_nav_point + + def is_other_pointcloud_overlapping(self, ref_pointcloud, other_pointcloud): + """ Check if a viewpoint is valid and overlaps significantly with a reference one. """ + # Observation + pixels_count = self.resolution[0] * self.resolution[1] + valid_fraction = len(other_pointcloud) / pixels_count + assert valid_fraction <= 1.0 and valid_fraction >= 0.0 + overlap = compute_pointcloud_overlaps_scikit(ref_pointcloud, other_pointcloud, self.distance_threshold, compute_symmetric=True) + covisibility = min(overlap["intersection1"] / pixels_count, overlap["intersection2"] / pixels_count) + is_valid = (valid_fraction >= self.minimum_valid_fraction) and (covisibility >= self.minimum_covisibility) + return is_valid, valid_fraction, covisibility + + def is_other_viewpoint_overlapping(self, ref_pointcloud, observation, position, rotation): + """ Check if a viewpoint is valid and overlaps significantly with a reference one. """ + # Observation + other_pointcloud = compute_pointcloud(observation['depth'], self.hfov, position, rotation) + return self.is_other_pointcloud_overlapping(ref_pointcloud, other_pointcloud) + + def render_viewpoint(self, viewpoint_position, viewpoint_orientation): + agent_state = habitat_sim.AgentState() + agent_state.position = viewpoint_position + agent_state.rotation = viewpoint_orientation + self.agent.set_state(agent_state) + viewpoint_observations = self.sim.get_sensor_observations(agent_ids=0) + _append_camera_parameters(viewpoint_observations, self.hfov, viewpoint_position, viewpoint_orientation) + return viewpoint_observations + + def __getitem__(self, useless_idx): + ref_position, ref_orientation, nav_point = self.sample_random_viewpoint() + ref_observations = self.render_viewpoint(ref_position, ref_orientation) + # Extract point cloud + ref_pointcloud = compute_pointcloud(depthmap=ref_observations['depth'], hfov=self.hfov, + camera_position=ref_position, camera_rotation=ref_orientation) + + pixels_count = self.resolution[0] * self.resolution[1] + ref_valid_fraction = len(ref_pointcloud) / pixels_count + assert ref_valid_fraction <= 1.0 and ref_valid_fraction >= 0.0 + if ref_valid_fraction < self.minimum_valid_fraction: + # This should produce a recursion error at some point when something is very wrong. + return self[0] + # Pick an reference observed point in the point cloud + observed_point = np.mean(ref_pointcloud, axis=0) + + # Add the first image as reference + viewpoints_observations = [ref_observations] + viewpoints_covisibility = [ref_valid_fraction] + viewpoints_positions = [ref_position] + viewpoints_orientations = [quaternion.as_float_array(ref_orientation)] + viewpoints_clouds = [ref_pointcloud] + viewpoints_valid_fractions = [ref_valid_fraction] + + for _ in range(self.views_count - 1): + # Generate an other viewpoint using some dummy random walk + successful_sampling = False + for sampling_attempt in range(self.max_attempts_count): + position, rotation, _ = self.sample_other_random_viewpoint(observed_point, nav_point) + # Observation + other_viewpoint_observations = self.render_viewpoint(position, rotation) + other_pointcloud = compute_pointcloud(other_viewpoint_observations['depth'], self.hfov, position, rotation) + + is_valid, valid_fraction, covisibility = self.is_other_pointcloud_overlapping(ref_pointcloud, other_pointcloud) + if is_valid: + successful_sampling = True + break + if not successful_sampling: + print("WARNING: Maximum number of attempts reached.") + # Dirty hack, try using a novel original viewpoint + return self[0] + viewpoints_observations.append(other_viewpoint_observations) + viewpoints_covisibility.append(covisibility) + viewpoints_positions.append(position) + viewpoints_orientations.append(quaternion.as_float_array(rotation)) # WXYZ convention for the quaternion encoding. + viewpoints_clouds.append(other_pointcloud) + viewpoints_valid_fractions.append(valid_fraction) + + # Estimate relations between all pairs of images + pairwise_visibility_ratios = np.ones((len(viewpoints_observations), len(viewpoints_observations))) + for i in range(len(viewpoints_observations)): + pairwise_visibility_ratios[i,i] = viewpoints_valid_fractions[i] + for j in range(i+1, len(viewpoints_observations)): + overlap = compute_pointcloud_overlaps_scikit(viewpoints_clouds[i], viewpoints_clouds[j], self.distance_threshold, compute_symmetric=True) + pairwise_visibility_ratios[i,j] = overlap['intersection1'] / pixels_count + pairwise_visibility_ratios[j,i] = overlap['intersection2'] / pixels_count + + # IoU is relative to the image 0 + data = {"observations": viewpoints_observations, + "positions": np.asarray(viewpoints_positions), + "orientations": np.asarray(viewpoints_orientations), + "covisibility_ratios": np.asarray(viewpoints_covisibility), + "valid_fractions": np.asarray(viewpoints_valid_fractions, dtype=float), + "pairwise_visibility_ratios": np.asarray(pairwise_visibility_ratios, dtype=float), + } + + if self.transform is not None: + data = self.transform(data) + return data + + def generate_random_spiral_trajectory(self, images_count = 100, max_radius=0.5, half_turns=5, use_constant_orientation=False): + """ + Return a list of images corresponding to a spiral trajectory from a random starting point. + Useful to generate nice visualisations. + Use an even number of half turns to get a nice "C1-continuous" loop effect + """ + ref_position, ref_orientation, navpoint = self.sample_random_viewpoint() + ref_observations = self.render_viewpoint(ref_position, ref_orientation) + ref_pointcloud = compute_pointcloud(depthmap=ref_observations['depth'], hfov=self.hfov, + camera_position=ref_position, camera_rotation=ref_orientation) + pixels_count = self.resolution[0] * self.resolution[1] + if len(ref_pointcloud) / pixels_count < self.minimum_valid_fraction: + # Dirty hack: ensure that the valid part of the image is significant + return self.generate_random_spiral_trajectory(images_count, max_radius, half_turns, use_constant_orientation) + + # Pick an observed point in the point cloud + observed_point = np.mean(ref_pointcloud, axis=0) + ref_R, ref_t = compute_camera_pose_opencv_convention(ref_position, ref_orientation) + + images = [] + is_valid = [] + # Spiral trajectory, use_constant orientation + for i, alpha in enumerate(np.linspace(0, 1, images_count)): + r = max_radius * np.abs(np.sin(alpha * np.pi)) # Increase then decrease the radius + theta = alpha * half_turns * np.pi + x = r * np.cos(theta) + y = r * np.sin(theta) + z = 0.0 + position = ref_position + (ref_R @ np.asarray([x, y, z]).reshape(3,1)).flatten() + if use_constant_orientation: + orientation = ref_orientation + else: + # trajectory looking at a mean point in front of the ref observation + orientation, position = look_at_for_habitat(eye=position, center=observed_point, up=habitat_sim.geo.UP) + observations = self.render_viewpoint(position, orientation) + images.append(observations['color'][...,:3]) + _is_valid, valid_fraction, iou = self.is_other_viewpoint_overlapping(ref_pointcloud, observations, position, orientation) + is_valid.append(_is_valid) + return images, np.all(is_valid) \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/datasets/habitat_sim/pack_metadata_files.py b/src/mast3r_src/dust3r/croco/datasets/habitat_sim/pack_metadata_files.py new file mode 100644 index 0000000000000000000000000000000000000000..10672a01f7dd615d3b4df37781f7f6f97e753ba6 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/datasets/habitat_sim/pack_metadata_files.py @@ -0,0 +1,69 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +""" +Utility script to pack metadata files of the dataset in order to be able to re-generate it elsewhere. +""" +import os +import glob +from tqdm import tqdm +import shutil +import json +from datasets.habitat_sim.paths import * +import argparse +import collections + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("input_dir") + parser.add_argument("output_dir") + args = parser.parse_args() + + input_dirname = args.input_dir + output_dirname = args.output_dir + + input_metadata_filenames = glob.iglob(f"{input_dirname}/**/metadata.json", recursive=True) + + images_count = collections.defaultdict(lambda : 0) + + os.makedirs(output_dirname) + for input_filename in tqdm(input_metadata_filenames): + # Ignore empty files + with open(input_filename, "r") as f: + original_metadata = json.load(f) + if "multiviews" not in original_metadata or len(original_metadata["multiviews"]) == 0: + print("No views in", input_filename) + continue + + relpath = os.path.relpath(input_filename, input_dirname) + print(relpath) + + # Copy metadata, while replacing scene paths by generic keys depending on the dataset, for portability. + # Data paths are sorted by decreasing length to avoid potential bugs due to paths starting by the same string pattern. + scenes_dataset_paths = dict(sorted(SCENES_DATASET.items(), key=lambda x: len(x[1]), reverse=True)) + metadata = dict() + for key, value in original_metadata.items(): + if key in ("scene_dataset_config_file", "scene", "navmesh") and value != "": + known_path = False + for dataset, dataset_path in scenes_dataset_paths.items(): + if value.startswith(dataset_path): + value = os.path.join(dataset, os.path.relpath(value, dataset_path)) + known_path = True + break + if not known_path: + raise KeyError("Unknown path:" + value) + metadata[key] = value + + # Compile some general statistics while packing data + scene_split = metadata["scene"].split("/") + upper_level = "/".join(scene_split[:2]) if scene_split[0] == "hm3d" else scene_split[0] + images_count[upper_level] += len(metadata["multiviews"]) + + output_filename = os.path.join(output_dirname, relpath) + os.makedirs(os.path.dirname(output_filename), exist_ok=True) + with open(output_filename, "w") as f: + json.dump(metadata, f) + + # Print statistics + print("Images count:") + for upper_level, count in images_count.items(): + print(f"- {upper_level}: {count}") \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/datasets/habitat_sim/paths.py b/src/mast3r_src/dust3r/croco/datasets/habitat_sim/paths.py new file mode 100644 index 0000000000000000000000000000000000000000..4d63b5fa29c274ddfeae084734a35ba66d7edee8 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/datasets/habitat_sim/paths.py @@ -0,0 +1,129 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +""" +Paths to Habitat-Sim scenes +""" + +import os +import json +import collections +from tqdm import tqdm + + +# Hardcoded path to the different scene datasets +SCENES_DATASET = { + "hm3d": "./data/habitat-sim-data/scene_datasets/hm3d/", + "gibson": "./data/habitat-sim-data/scene_datasets/gibson/", + "habitat-test-scenes": "./data/habitat-sim/scene_datasets/habitat-test-scenes/", + "replica_cad_baked_lighting": "./data/habitat-sim/scene_datasets/replica_cad_baked_lighting/", + "replica_cad": "./data/habitat-sim/scene_datasets/replica_cad/", + "replica": "./data/habitat-sim/scene_datasets/ReplicaDataset/", + "scannet": "./data/habitat-sim/scene_datasets/scannet/" +} + +SceneData = collections.namedtuple("SceneData", ["scene_dataset_config_file", "scene", "navmesh", "output_dir"]) + +def list_replicacad_scenes(base_output_dir, base_path=SCENES_DATASET["replica_cad"]): + scene_dataset_config_file = os.path.join(base_path, "replicaCAD.scene_dataset_config.json") + scenes = [f"apt_{i}" for i in range(6)] + ["empty_stage"] + navmeshes = [f"navmeshes/apt_{i}_static_furniture.navmesh" for i in range(6)] + ["empty_stage.navmesh"] + scenes_data = [] + for idx in range(len(scenes)): + output_dir = os.path.join(base_output_dir, "ReplicaCAD", scenes[idx]) + # Add scene + data = SceneData(scene_dataset_config_file=scene_dataset_config_file, + scene = scenes[idx] + ".scene_instance.json", + navmesh = os.path.join(base_path, navmeshes[idx]), + output_dir = output_dir) + scenes_data.append(data) + return scenes_data + +def list_replica_cad_baked_lighting_scenes(base_output_dir, base_path=SCENES_DATASET["replica_cad_baked_lighting"]): + scene_dataset_config_file = os.path.join(base_path, "replicaCAD_baked.scene_dataset_config.json") + scenes = sum([[f"Baked_sc{i}_staging_{j:02}" for i in range(5)] for j in range(21)], []) + navmeshes = ""#[f"navmeshes/apt_{i}_static_furniture.navmesh" for i in range(6)] + ["empty_stage.navmesh"] + scenes_data = [] + for idx in range(len(scenes)): + output_dir = os.path.join(base_output_dir, "replica_cad_baked_lighting", scenes[idx]) + data = SceneData(scene_dataset_config_file=scene_dataset_config_file, + scene = scenes[idx], + navmesh = "", + output_dir = output_dir) + scenes_data.append(data) + return scenes_data + +def list_replica_scenes(base_output_dir, base_path): + scenes_data = [] + for scene_id in os.listdir(base_path): + scene = os.path.join(base_path, scene_id, "mesh.ply") + navmesh = os.path.join(base_path, scene_id, "habitat/mesh_preseg_semantic.navmesh") # Not sure if I should use it + scene_dataset_config_file = "" + output_dir = os.path.join(base_output_dir, scene_id) + # Add scene only if it does not exist already, or if exist_ok + data = SceneData(scene_dataset_config_file = scene_dataset_config_file, + scene = scene, + navmesh = navmesh, + output_dir = output_dir) + scenes_data.append(data) + return scenes_data + + +def list_scenes(base_output_dir, base_path): + """ + Generic method iterating through a base_path folder to find scenes. + """ + scenes_data = [] + for root, dirs, files in os.walk(base_path, followlinks=True): + folder_scenes_data = [] + for file in files: + name, ext = os.path.splitext(file) + if ext == ".glb": + scene = os.path.join(root, name + ".glb") + navmesh = os.path.join(root, name + ".navmesh") + if not os.path.exists(navmesh): + navmesh = "" + relpath = os.path.relpath(root, base_path) + output_dir = os.path.abspath(os.path.join(base_output_dir, relpath, name)) + data = SceneData(scene_dataset_config_file="", + scene = scene, + navmesh = navmesh, + output_dir = output_dir) + folder_scenes_data.append(data) + + # Specific check for HM3D: + # When two meshesxxxx.basis.glb and xxxx.glb are present, use the 'basis' version. + basis_scenes = [data.scene[:-len(".basis.glb")] for data in folder_scenes_data if data.scene.endswith(".basis.glb")] + if len(basis_scenes) != 0: + folder_scenes_data = [data for data in folder_scenes_data if not (data.scene[:-len(".glb")] in basis_scenes)] + + scenes_data.extend(folder_scenes_data) + return scenes_data + +def list_scenes_available(base_output_dir, scenes_dataset_paths=SCENES_DATASET): + scenes_data = [] + + # HM3D + for split in ("minival", "train", "val", "examples"): + scenes_data += list_scenes(base_output_dir=os.path.join(base_output_dir, f"hm3d/{split}/"), + base_path=f"{scenes_dataset_paths['hm3d']}/{split}") + + # Gibson + scenes_data += list_scenes(base_output_dir=os.path.join(base_output_dir, "gibson"), + base_path=scenes_dataset_paths["gibson"]) + + # Habitat test scenes (just a few) + scenes_data += list_scenes(base_output_dir=os.path.join(base_output_dir, "habitat-test-scenes"), + base_path=scenes_dataset_paths["habitat-test-scenes"]) + + # ReplicaCAD (baked lightning) + scenes_data += list_replica_cad_baked_lighting_scenes(base_output_dir=base_output_dir) + + # ScanNet + scenes_data += list_scenes(base_output_dir=os.path.join(base_output_dir, "scannet"), + base_path=scenes_dataset_paths["scannet"]) + + # Replica + list_replica_scenes(base_output_dir=os.path.join(base_output_dir, "replica"), + base_path=scenes_dataset_paths["replica"]) + return scenes_data diff --git a/src/mast3r_src/dust3r/croco/datasets/pairs_dataset.py b/src/mast3r_src/dust3r/croco/datasets/pairs_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..9f107526b34e154d9013a9a7a0bde3d5ff6f581c --- /dev/null +++ b/src/mast3r_src/dust3r/croco/datasets/pairs_dataset.py @@ -0,0 +1,109 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +import os +from torch.utils.data import Dataset +from PIL import Image + +from datasets.transforms import get_pair_transforms + +def load_image(impath): + return Image.open(impath) + +def load_pairs_from_cache_file(fname, root=''): + assert os.path.isfile(fname), "cannot parse pairs from {:s}, file does not exist".format(fname) + with open(fname, 'r') as fid: + lines = fid.read().strip().splitlines() + pairs = [ (os.path.join(root,l.split()[0]), os.path.join(root,l.split()[1])) for l in lines] + return pairs + +def load_pairs_from_list_file(fname, root=''): + assert os.path.isfile(fname), "cannot parse pairs from {:s}, file does not exist".format(fname) + with open(fname, 'r') as fid: + lines = fid.read().strip().splitlines() + pairs = [ (os.path.join(root,l+'_1.jpg'), os.path.join(root,l+'_2.jpg')) for l in lines if not l.startswith('#')] + return pairs + + +def write_cache_file(fname, pairs, root=''): + if len(root)>0: + if not root.endswith('/'): root+='/' + assert os.path.isdir(root) + s = '' + for im1, im2 in pairs: + if len(root)>0: + assert im1.startswith(root), im1 + assert im2.startswith(root), im2 + s += '{:s} {:s}\n'.format(im1[len(root):], im2[len(root):]) + with open(fname, 'w') as fid: + fid.write(s[:-1]) + +def parse_and_cache_all_pairs(dname, data_dir='./data/'): + if dname=='habitat_release': + dirname = os.path.join(data_dir, 'habitat_release') + assert os.path.isdir(dirname), "cannot find folder for habitat_release pairs: "+dirname + cache_file = os.path.join(dirname, 'pairs.txt') + assert not os.path.isfile(cache_file), "cache file already exists: "+cache_file + + print('Parsing pairs for dataset: '+dname) + pairs = [] + for root, dirs, files in os.walk(dirname): + if 'val' in root: continue + dirs.sort() + pairs += [ (os.path.join(root,f), os.path.join(root,f[:-len('_1.jpeg')]+'_2.jpeg')) for f in sorted(files) if f.endswith('_1.jpeg')] + print('Found {:,} pairs'.format(len(pairs))) + print('Writing cache to: '+cache_file) + write_cache_file(cache_file, pairs, root=dirname) + + else: + raise NotImplementedError('Unknown dataset: '+dname) + +def dnames_to_image_pairs(dnames, data_dir='./data/'): + """ + dnames: list of datasets with image pairs, separated by + + """ + all_pairs = [] + for dname in dnames.split('+'): + if dname=='habitat_release': + dirname = os.path.join(data_dir, 'habitat_release') + assert os.path.isdir(dirname), "cannot find folder for habitat_release pairs: "+dirname + cache_file = os.path.join(dirname, 'pairs.txt') + assert os.path.isfile(cache_file), "cannot find cache file for habitat_release pairs, please first create the cache file, see instructions. "+cache_file + pairs = load_pairs_from_cache_file(cache_file, root=dirname) + elif dname in ['ARKitScenes', 'MegaDepth', '3DStreetView', 'IndoorVL']: + dirname = os.path.join(data_dir, dname+'_crops') + assert os.path.isdir(dirname), "cannot find folder for {:s} pairs: {:s}".format(dname, dirname) + list_file = os.path.join(dirname, 'listing.txt') + assert os.path.isfile(list_file), "cannot find list file for {:s} pairs, see instructions. {:s}".format(dname, list_file) + pairs = load_pairs_from_list_file(list_file, root=dirname) + print(' {:s}: {:,} pairs'.format(dname, len(pairs))) + all_pairs += pairs + if '+' in dnames: print(' Total: {:,} pairs'.format(len(all_pairs))) + return all_pairs + + +class PairsDataset(Dataset): + + def __init__(self, dnames, trfs='', totensor=True, normalize=True, data_dir='./data/'): + super().__init__() + self.image_pairs = dnames_to_image_pairs(dnames, data_dir=data_dir) + self.transforms = get_pair_transforms(transform_str=trfs, totensor=totensor, normalize=normalize) + + def __len__(self): + return len(self.image_pairs) + + def __getitem__(self, index): + im1path, im2path = self.image_pairs[index] + im1 = load_image(im1path) + im2 = load_image(im2path) + if self.transforms is not None: im1, im2 = self.transforms(im1, im2) + return im1, im2 + + +if __name__=="__main__": + import argparse + parser = argparse.ArgumentParser(prog="Computing and caching list of pairs for a given dataset") + parser.add_argument('--data_dir', default='./data/', type=str, help="path where data are stored") + parser.add_argument('--dataset', default='habitat_release', type=str, help="name of the dataset") + args = parser.parse_args() + parse_and_cache_all_pairs(dname=args.dataset, data_dir=args.data_dir) diff --git a/src/mast3r_src/dust3r/croco/datasets/transforms.py b/src/mast3r_src/dust3r/croco/datasets/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..216bac61f8254fd50e7f269ee80301f250a2d11e --- /dev/null +++ b/src/mast3r_src/dust3r/croco/datasets/transforms.py @@ -0,0 +1,95 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +import torch +import torchvision.transforms +import torchvision.transforms.functional as F + +# "Pair": apply a transform on a pair +# "Both": apply the exact same transform to both images + +class ComposePair(torchvision.transforms.Compose): + def __call__(self, img1, img2): + for t in self.transforms: + img1, img2 = t(img1, img2) + return img1, img2 + +class NormalizeBoth(torchvision.transforms.Normalize): + def forward(self, img1, img2): + img1 = super().forward(img1) + img2 = super().forward(img2) + return img1, img2 + +class ToTensorBoth(torchvision.transforms.ToTensor): + def __call__(self, img1, img2): + img1 = super().__call__(img1) + img2 = super().__call__(img2) + return img1, img2 + +class RandomCropPair(torchvision.transforms.RandomCrop): + # the crop will be intentionally different for the two images with this class + def forward(self, img1, img2): + img1 = super().forward(img1) + img2 = super().forward(img2) + return img1, img2 + +class ColorJitterPair(torchvision.transforms.ColorJitter): + # can be symmetric (same for both images) or assymetric (different jitter params for each image) depending on assymetric_prob + def __init__(self, assymetric_prob, **kwargs): + super().__init__(**kwargs) + self.assymetric_prob = assymetric_prob + def jitter_one(self, img, fn_idx, brightness_factor, contrast_factor, saturation_factor, hue_factor): + for fn_id in fn_idx: + if fn_id == 0 and brightness_factor is not None: + img = F.adjust_brightness(img, brightness_factor) + elif fn_id == 1 and contrast_factor is not None: + img = F.adjust_contrast(img, contrast_factor) + elif fn_id == 2 and saturation_factor is not None: + img = F.adjust_saturation(img, saturation_factor) + elif fn_id == 3 and hue_factor is not None: + img = F.adjust_hue(img, hue_factor) + return img + + def forward(self, img1, img2): + + fn_idx, brightness_factor, contrast_factor, saturation_factor, hue_factor = self.get_params( + self.brightness, self.contrast, self.saturation, self.hue + ) + img1 = self.jitter_one(img1, fn_idx, brightness_factor, contrast_factor, saturation_factor, hue_factor) + if torch.rand(1) < self.assymetric_prob: # assymetric: + fn_idx, brightness_factor, contrast_factor, saturation_factor, hue_factor = self.get_params( + self.brightness, self.contrast, self.saturation, self.hue + ) + img2 = self.jitter_one(img2, fn_idx, brightness_factor, contrast_factor, saturation_factor, hue_factor) + return img1, img2 + +def get_pair_transforms(transform_str, totensor=True, normalize=True): + # transform_str is eg crop224+color + trfs = [] + for s in transform_str.split('+'): + if s.startswith('crop'): + size = int(s[len('crop'):]) + trfs.append(RandomCropPair(size)) + elif s=='acolor': + trfs.append(ColorJitterPair(assymetric_prob=1.0, brightness=(0.6, 1.4), contrast=(0.6, 1.4), saturation=(0.6, 1.4), hue=0.0)) + elif s=='': # if transform_str was "" + pass + else: + raise NotImplementedError('Unknown augmentation: '+s) + + if totensor: + trfs.append( ToTensorBoth() ) + if normalize: + trfs.append( NormalizeBoth(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ) + + if len(trfs)==0: + return None + elif len(trfs)==1: + return trfs + else: + return ComposePair(trfs) + + + + + diff --git a/src/mast3r_src/dust3r/croco/demo.py b/src/mast3r_src/dust3r/croco/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..91b80ccc5c98c18e20d1ce782511aa824ef28f77 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/demo.py @@ -0,0 +1,55 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +import torch +from models.croco import CroCoNet +from PIL import Image +import torchvision.transforms +from torchvision.transforms import ToTensor, Normalize, Compose + +def main(): + device = torch.device('cuda:0' if torch.cuda.is_available() and torch.cuda.device_count()>0 else 'cpu') + + # load 224x224 images and transform them to tensor + imagenet_mean = [0.485, 0.456, 0.406] + imagenet_mean_tensor = torch.tensor(imagenet_mean).view(1,3,1,1).to(device, non_blocking=True) + imagenet_std = [0.229, 0.224, 0.225] + imagenet_std_tensor = torch.tensor(imagenet_std).view(1,3,1,1).to(device, non_blocking=True) + trfs = Compose([ToTensor(), Normalize(mean=imagenet_mean, std=imagenet_std)]) + image1 = trfs(Image.open('assets/Chateau1.png').convert('RGB')).to(device, non_blocking=True).unsqueeze(0) + image2 = trfs(Image.open('assets/Chateau2.png').convert('RGB')).to(device, non_blocking=True).unsqueeze(0) + + # load model + ckpt = torch.load('pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth', 'cpu') + model = CroCoNet( **ckpt.get('croco_kwargs',{})).to(device) + model.eval() + msg = model.load_state_dict(ckpt['model'], strict=True) + + # forward + with torch.inference_mode(): + out, mask, target = model(image1, image2) + + # the output is normalized, thus use the mean/std of the actual image to go back to RGB space + patchified = model.patchify(image1) + mean = patchified.mean(dim=-1, keepdim=True) + var = patchified.var(dim=-1, keepdim=True) + decoded_image = model.unpatchify(out * (var + 1.e-6)**.5 + mean) + # undo imagenet normalization, prepare masked image + decoded_image = decoded_image * imagenet_std_tensor + imagenet_mean_tensor + input_image = image1 * imagenet_std_tensor + imagenet_mean_tensor + ref_image = image2 * imagenet_std_tensor + imagenet_mean_tensor + image_masks = model.unpatchify(model.patchify(torch.ones_like(ref_image)) * mask[:,:,None]) + masked_input_image = ((1 - image_masks) * input_image) + + # make visualization + visualization = torch.cat((ref_image, masked_input_image, decoded_image, input_image), dim=3) # 4*(B, 3, H, W) -> B, 3, H, W*4 + B, C, H, W = visualization.shape + visualization = visualization.permute(1, 0, 2, 3).reshape(C, B*H, W) + visualization = torchvision.transforms.functional.to_pil_image(torch.clamp(visualization, 0, 1)) + fname = "demo_output.png" + visualization.save(fname) + print('Visualization save in '+fname) + + +if __name__=="__main__": + main() diff --git a/src/mast3r_src/dust3r/croco/interactive_demo.ipynb b/src/mast3r_src/dust3r/croco/interactive_demo.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..6cfc960af5baac9a69029c29a16eea4e24123a71 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/interactive_demo.ipynb @@ -0,0 +1,271 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Interactive demo of Cross-view Completion." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Copyright (C) 2022-present Naver Corporation. All rights reserved.\n", + "# Licensed under CC BY-NC-SA 4.0 (non-commercial use only)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import numpy as np\n", + "from models.croco import CroCoNet\n", + "from ipywidgets import interact, interactive, fixed, interact_manual\n", + "import ipywidgets as widgets\n", + "import matplotlib.pyplot as plt\n", + "import quaternion\n", + "import models.masking" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Load CroCo model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ckpt = torch.load('pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth', 'cpu')\n", + "model = CroCoNet( **ckpt.get('croco_kwargs',{}))\n", + "msg = model.load_state_dict(ckpt['model'], strict=True)\n", + "use_gpu = torch.cuda.is_available() and torch.cuda.device_count()>0\n", + "device = torch.device('cuda:0' if use_gpu else 'cpu')\n", + "model = model.eval()\n", + "model = model.to(device=device)\n", + "print(msg)\n", + "\n", + "def process_images(ref_image, target_image, masking_ratio, reconstruct_unmasked_patches=False):\n", + " \"\"\"\n", + " Perform Cross-View completion using two input images, specified using Numpy arrays.\n", + " \"\"\"\n", + " # Replace the mask generator\n", + " model.mask_generator = models.masking.RandomMask(model.patch_embed.num_patches, masking_ratio)\n", + "\n", + " # ImageNet-1k color normalization\n", + " imagenet_mean = torch.as_tensor([0.485, 0.456, 0.406]).reshape(1,3,1,1).to(device)\n", + " imagenet_std = torch.as_tensor([0.229, 0.224, 0.225]).reshape(1,3,1,1).to(device)\n", + "\n", + " normalize_input_colors = True\n", + " is_output_normalized = True\n", + " with torch.no_grad():\n", + " # Cast data to torch\n", + " target_image = (torch.as_tensor(target_image, dtype=torch.float, device=device).permute(2,0,1) / 255)[None]\n", + " ref_image = (torch.as_tensor(ref_image, dtype=torch.float, device=device).permute(2,0,1) / 255)[None]\n", + "\n", + " if normalize_input_colors:\n", + " ref_image = (ref_image - imagenet_mean) / imagenet_std\n", + " target_image = (target_image - imagenet_mean) / imagenet_std\n", + "\n", + " out, mask, _ = model(target_image, ref_image)\n", + " # # get target\n", + " if not is_output_normalized:\n", + " predicted_image = model.unpatchify(out)\n", + " else:\n", + " # The output only contains higher order information,\n", + " # we retrieve mean and standard deviation from the actual target image\n", + " patchified = model.patchify(target_image)\n", + " mean = patchified.mean(dim=-1, keepdim=True)\n", + " var = patchified.var(dim=-1, keepdim=True)\n", + " pred_renorm = out * (var + 1.e-6)**.5 + mean\n", + " predicted_image = model.unpatchify(pred_renorm)\n", + "\n", + " image_masks = model.unpatchify(model.patchify(torch.ones_like(ref_image)) * mask[:,:,None])\n", + " masked_target_image = (1 - image_masks) * target_image\n", + " \n", + " if not reconstruct_unmasked_patches:\n", + " # Replace unmasked patches by their actual values\n", + " predicted_image = predicted_image * image_masks + masked_target_image\n", + "\n", + " # Unapply color normalization\n", + " if normalize_input_colors:\n", + " predicted_image = predicted_image * imagenet_std + imagenet_mean\n", + " masked_target_image = masked_target_image * imagenet_std + imagenet_mean\n", + " \n", + " # Cast to Numpy\n", + " masked_target_image = np.asarray(torch.clamp(masked_target_image.squeeze(0).permute(1,2,0) * 255, 0, 255).cpu().numpy(), dtype=np.uint8)\n", + " predicted_image = np.asarray(torch.clamp(predicted_image.squeeze(0).permute(1,2,0) * 255, 0, 255).cpu().numpy(), dtype=np.uint8)\n", + " return masked_target_image, predicted_image" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Use the Habitat simulator to render images from arbitrary viewpoints (requires habitat_sim to be installed)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"MAGNUM_LOG\"]=\"quiet\"\n", + "os.environ[\"HABITAT_SIM_LOG\"]=\"quiet\"\n", + "import habitat_sim\n", + "\n", + "scene = \"habitat-sim-data/scene_datasets/habitat-test-scenes/skokloster-castle.glb\"\n", + "navmesh = \"habitat-sim-data/scene_datasets/habitat-test-scenes/skokloster-castle.navmesh\"\n", + "\n", + "sim_cfg = habitat_sim.SimulatorConfiguration()\n", + "if use_gpu: sim_cfg.gpu_device_id = 0\n", + "sim_cfg.scene_id = scene\n", + "sim_cfg.load_semantic_mesh = False\n", + "rgb_sensor_spec = habitat_sim.CameraSensorSpec()\n", + "rgb_sensor_spec.uuid = \"color\"\n", + "rgb_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR\n", + "rgb_sensor_spec.resolution = (224,224)\n", + "rgb_sensor_spec.hfov = 56.56\n", + "rgb_sensor_spec.position = [0.0, 0.0, 0.0]\n", + "rgb_sensor_spec.orientation = [0, 0, 0]\n", + "agent_cfg = habitat_sim.agent.AgentConfiguration(sensor_specifications=[rgb_sensor_spec])\n", + "\n", + "\n", + "cfg = habitat_sim.Configuration(sim_cfg, [agent_cfg])\n", + "sim = habitat_sim.Simulator(cfg)\n", + "if navmesh is not None:\n", + " sim.pathfinder.load_nav_mesh(navmesh)\n", + "agent = sim.initialize_agent(agent_id=0)\n", + "\n", + "def sample_random_viewpoint():\n", + " \"\"\" Sample a random viewpoint using the navmesh \"\"\"\n", + " nav_point = sim.pathfinder.get_random_navigable_point()\n", + " # Sample a random viewpoint height\n", + " viewpoint_height = np.random.uniform(1.0, 1.6)\n", + " viewpoint_position = nav_point + viewpoint_height * habitat_sim.geo.UP\n", + " viewpoint_orientation = quaternion.from_rotation_vector(np.random.uniform(-np.pi, np.pi) * habitat_sim.geo.UP)\n", + " return viewpoint_position, viewpoint_orientation\n", + "\n", + "def render_viewpoint(position, orientation):\n", + " agent_state = habitat_sim.AgentState()\n", + " agent_state.position = position\n", + " agent_state.rotation = orientation\n", + " agent.set_state(agent_state)\n", + " viewpoint_observations = sim.get_sensor_observations(agent_ids=0)\n", + " image = viewpoint_observations['color'][:,:,:3]\n", + " image = np.asarray(np.clip(1.5 * np.asarray(image, dtype=float), 0, 255), dtype=np.uint8)\n", + " return image" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Sample a random reference view" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ref_position, ref_orientation = sample_random_viewpoint()\n", + "ref_image = render_viewpoint(ref_position, ref_orientation)\n", + "plt.clf()\n", + "fig, axes = plt.subplots(1,1, squeeze=False, num=1)\n", + "axes[0,0].imshow(ref_image)\n", + "for ax in axes.flatten():\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Interactive cross-view completion using CroCo" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "reconstruct_unmasked_patches = False\n", + "\n", + "def show_demo(masking_ratio, x, y, z, panorama, elevation):\n", + " R = quaternion.as_rotation_matrix(ref_orientation)\n", + " target_position = ref_position + x * R[:,0] + y * R[:,1] + z * R[:,2]\n", + " target_orientation = (ref_orientation\n", + " * quaternion.from_rotation_vector(-elevation * np.pi/180 * habitat_sim.geo.LEFT) \n", + " * quaternion.from_rotation_vector(-panorama * np.pi/180 * habitat_sim.geo.UP))\n", + " \n", + " ref_image = render_viewpoint(ref_position, ref_orientation)\n", + " target_image = render_viewpoint(target_position, target_orientation)\n", + "\n", + " masked_target_image, predicted_image = process_images(ref_image, target_image, masking_ratio, reconstruct_unmasked_patches)\n", + "\n", + " fig, axes = plt.subplots(1,4, squeeze=True, dpi=300)\n", + " axes[0].imshow(ref_image)\n", + " axes[0].set_xlabel(\"Reference\")\n", + " axes[1].imshow(masked_target_image)\n", + " axes[1].set_xlabel(\"Masked target\")\n", + " axes[2].imshow(predicted_image)\n", + " axes[2].set_xlabel(\"Reconstruction\") \n", + " axes[3].imshow(target_image)\n", + " axes[3].set_xlabel(\"Target\")\n", + " for ax in axes.flatten():\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + "\n", + "interact(show_demo,\n", + " masking_ratio=widgets.FloatSlider(description='masking', value=0.9, min=0.0, max=1.0),\n", + " x=widgets.FloatSlider(value=0.0, min=-0.5, max=0.5, step=0.05),\n", + " y=widgets.FloatSlider(value=0.0, min=-0.5, max=0.5, step=0.05),\n", + " z=widgets.FloatSlider(value=0.0, min=-0.5, max=0.5, step=0.05),\n", + " panorama=widgets.FloatSlider(value=0.0, min=-20, max=20, step=0.5),\n", + " elevation=widgets.FloatSlider(value=0.0, min=-20, max=20, step=0.5));" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.13" + }, + "vscode": { + "interpreter": { + "hash": "f9237820cd248d7e07cb4fb9f0e4508a85d642f19d831560c0a4b61f3e907e67" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/mast3r_src/dust3r/croco/models/blocks.py b/src/mast3r_src/dust3r/croco/models/blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..18133524f0ae265b0bd8d062d7c9eeaa63858a9b --- /dev/null +++ b/src/mast3r_src/dust3r/croco/models/blocks.py @@ -0,0 +1,241 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + + +# -------------------------------------------------------- +# Main encoder/decoder blocks +# -------------------------------------------------------- +# References: +# timm +# https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py +# https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/layers/helpers.py +# https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/layers/drop.py +# https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/layers/mlp.py +# https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/layers/patch_embed.py + + +import torch +import torch.nn as nn + +from itertools import repeat +import collections.abc + + +def _ntuple(n): + def parse(x): + if isinstance(x, collections.abc.Iterable) and not isinstance(x, str): + return x + return tuple(repeat(x, n)) + return parse +to_2tuple = _ntuple(2) + +def drop_path(x, drop_prob: float = 0., training: bool = False, scale_by_keep: bool = True): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = x.new_empty(shape).bernoulli_(keep_prob) + if keep_prob > 0.0 and scale_by_keep: + random_tensor.div_(keep_prob) + return x * random_tensor + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + def __init__(self, drop_prob: float = 0., scale_by_keep: bool = True): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + self.scale_by_keep = scale_by_keep + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training, self.scale_by_keep) + + def extra_repr(self): + return f'drop_prob={round(self.drop_prob,3):0.3f}' + +class Mlp(nn.Module): + """ MLP as used in Vision Transformer, MLP-Mixer and related networks""" + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, bias=True, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + bias = to_2tuple(bias) + drop_probs = to_2tuple(drop) + + self.fc1 = nn.Linear(in_features, hidden_features, bias=bias[0]) + self.act = act_layer() + self.drop1 = nn.Dropout(drop_probs[0]) + self.fc2 = nn.Linear(hidden_features, out_features, bias=bias[1]) + self.drop2 = nn.Dropout(drop_probs[1]) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop1(x) + x = self.fc2(x) + x = self.drop2(x) + return x + +class Attention(nn.Module): + + def __init__(self, dim, rope=None, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0.): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + self.rope = rope + + def forward(self, x, xpos): + B, N, C = x.shape + + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).transpose(1,3) + q, k, v = [qkv[:,:,i] for i in range(3)] + # q,k,v = qkv.unbind(2) # make torchscript happy (cannot use tensor as tuple) + + if self.rope is not None: + q = self.rope(q, xpos) + k = self.rope(k, xpos) + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + +class Block(nn.Module): + + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, rope=None): + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention(dim, rope=rope, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def forward(self, x, xpos): + x = x + self.drop_path(self.attn(self.norm1(x), xpos)) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + +class CrossAttention(nn.Module): + + def __init__(self, dim, rope=None, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0.): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + + self.projq = nn.Linear(dim, dim, bias=qkv_bias) + self.projk = nn.Linear(dim, dim, bias=qkv_bias) + self.projv = nn.Linear(dim, dim, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + self.rope = rope + + def forward(self, query, key, value, qpos, kpos): + B, Nq, C = query.shape + Nk = key.shape[1] + Nv = value.shape[1] + + q = self.projq(query).reshape(B,Nq,self.num_heads, C// self.num_heads).permute(0, 2, 1, 3) + k = self.projk(key).reshape(B,Nk,self.num_heads, C// self.num_heads).permute(0, 2, 1, 3) + v = self.projv(value).reshape(B,Nv,self.num_heads, C// self.num_heads).permute(0, 2, 1, 3) + + if self.rope is not None: + q = self.rope(q, qpos) + k = self.rope(k, kpos) + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, Nq, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + +class DecoderBlock(nn.Module): + + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, norm_mem=True, rope=None): + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention(dim, rope=rope, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) + self.cross_attn = CrossAttention(dim, rope=rope, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + self.norm3 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + self.norm_y = norm_layer(dim) if norm_mem else nn.Identity() + + def forward(self, x, y, xpos, ypos): + x = x + self.drop_path(self.attn(self.norm1(x), xpos)) + y_ = self.norm_y(y) + x = x + self.drop_path(self.cross_attn(self.norm2(x), y_, y_, xpos, ypos)) + x = x + self.drop_path(self.mlp(self.norm3(x))) + return x, y + + +# patch embedding +class PositionGetter(object): + """ return positions of patches """ + + def __init__(self): + self.cache_positions = {} + + def __call__(self, b, h, w, device): + if not (h,w) in self.cache_positions: + x = torch.arange(w, device=device) + y = torch.arange(h, device=device) + self.cache_positions[h,w] = torch.cartesian_prod(y, x) # (h, w, 2) + pos = self.cache_positions[h,w].view(1, h*w, 2).expand(b, -1, 2).clone() + return pos + +class PatchEmbed(nn.Module): + """ just adding _init_weights + position getter compared to timm.models.layers.patch_embed.PatchEmbed""" + + def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + self.img_size = img_size + self.patch_size = patch_size + self.grid_size = (img_size[0] // patch_size[0], img_size[1] // patch_size[1]) + self.num_patches = self.grid_size[0] * self.grid_size[1] + self.flatten = flatten + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + self.position_getter = PositionGetter() + + def forward(self, x): + B, C, H, W = x.shape + torch._assert(H == self.img_size[0], f"Input image height ({H}) doesn't match model ({self.img_size[0]}).") + torch._assert(W == self.img_size[1], f"Input image width ({W}) doesn't match model ({self.img_size[1]}).") + x = self.proj(x) + pos = self.position_getter(B, x.size(2), x.size(3), x.device) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + x = self.norm(x) + return x, pos + + def _init_weights(self): + w = self.proj.weight.data + torch.nn.init.xavier_uniform_(w.view([w.shape[0], -1])) + diff --git a/src/mast3r_src/dust3r/croco/models/criterion.py b/src/mast3r_src/dust3r/croco/models/criterion.py new file mode 100644 index 0000000000000000000000000000000000000000..11696c40865344490f23796ea45e8fbd5e654731 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/models/criterion.py @@ -0,0 +1,37 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Criterion to train CroCo +# -------------------------------------------------------- +# References: +# MAE: https://github.com/facebookresearch/mae +# -------------------------------------------------------- + +import torch + +class MaskedMSE(torch.nn.Module): + + def __init__(self, norm_pix_loss=False, masked=True): + """ + norm_pix_loss: normalize each patch by their pixel mean and variance + masked: compute loss over the masked patches only + """ + super().__init__() + self.norm_pix_loss = norm_pix_loss + self.masked = masked + + def forward(self, pred, mask, target): + + if self.norm_pix_loss: + mean = target.mean(dim=-1, keepdim=True) + var = target.var(dim=-1, keepdim=True) + target = (target - mean) / (var + 1.e-6)**.5 + + loss = (pred - target) ** 2 + loss = loss.mean(dim=-1) # [N, L], mean loss per patch + if self.masked: + loss = (loss * mask).sum() / mask.sum() # mean loss on masked patches + else: + loss = loss.mean() # mean loss + return loss diff --git a/src/mast3r_src/dust3r/croco/models/croco.py b/src/mast3r_src/dust3r/croco/models/croco.py new file mode 100644 index 0000000000000000000000000000000000000000..14c68634152d75555b4c35c25af268394c5821fe --- /dev/null +++ b/src/mast3r_src/dust3r/croco/models/croco.py @@ -0,0 +1,249 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + + +# -------------------------------------------------------- +# CroCo model during pretraining +# -------------------------------------------------------- + + + +import torch +import torch.nn as nn +torch.backends.cuda.matmul.allow_tf32 = True # for gpu >= Ampere and pytorch >= 1.12 +from functools import partial + +from models.blocks import Block, DecoderBlock, PatchEmbed +from models.pos_embed import get_2d_sincos_pos_embed, RoPE2D +from models.masking import RandomMask + + +class CroCoNet(nn.Module): + + def __init__(self, + img_size=224, # input image size + patch_size=16, # patch_size + mask_ratio=0.9, # ratios of masked tokens + enc_embed_dim=768, # encoder feature dimension + enc_depth=12, # encoder depth + enc_num_heads=12, # encoder number of heads in the transformer block + dec_embed_dim=512, # decoder feature dimension + dec_depth=8, # decoder depth + dec_num_heads=16, # decoder number of heads in the transformer block + mlp_ratio=4, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + norm_im2_in_dec=True, # whether to apply normalization of the 'memory' = (second image) in the decoder + pos_embed='cosine', # positional embedding (either cosine or RoPE100) + ): + + super(CroCoNet, self).__init__() + + # patch embeddings (with initialization done as in MAE) + self._set_patch_embed(img_size, patch_size, enc_embed_dim) + + # mask generations + self._set_mask_generator(self.patch_embed.num_patches, mask_ratio) + + self.pos_embed = pos_embed + if pos_embed=='cosine': + # positional embedding of the encoder + enc_pos_embed = get_2d_sincos_pos_embed(enc_embed_dim, int(self.patch_embed.num_patches**.5), n_cls_token=0) + self.register_buffer('enc_pos_embed', torch.from_numpy(enc_pos_embed).float()) + # positional embedding of the decoder + dec_pos_embed = get_2d_sincos_pos_embed(dec_embed_dim, int(self.patch_embed.num_patches**.5), n_cls_token=0) + self.register_buffer('dec_pos_embed', torch.from_numpy(dec_pos_embed).float()) + # pos embedding in each block + self.rope = None # nothing for cosine + elif pos_embed.startswith('RoPE'): # eg RoPE100 + self.enc_pos_embed = None # nothing to add in the encoder with RoPE + self.dec_pos_embed = None # nothing to add in the decoder with RoPE + if RoPE2D is None: raise ImportError("Cannot find cuRoPE2D, please install it following the README instructions") + freq = float(pos_embed[len('RoPE'):]) + self.rope = RoPE2D(freq=freq) + else: + raise NotImplementedError('Unknown pos_embed '+pos_embed) + + # transformer for the encoder + self.enc_depth = enc_depth + self.enc_embed_dim = enc_embed_dim + self.enc_blocks = nn.ModuleList([ + Block(enc_embed_dim, enc_num_heads, mlp_ratio, qkv_bias=True, norm_layer=norm_layer, rope=self.rope) + for i in range(enc_depth)]) + self.enc_norm = norm_layer(enc_embed_dim) + + # masked tokens + self._set_mask_token(dec_embed_dim) + + # decoder + self._set_decoder(enc_embed_dim, dec_embed_dim, dec_num_heads, dec_depth, mlp_ratio, norm_layer, norm_im2_in_dec) + + # prediction head + self._set_prediction_head(dec_embed_dim, patch_size) + + # initializer weights + self.initialize_weights() + + def _set_patch_embed(self, img_size=224, patch_size=16, enc_embed_dim=768): + self.patch_embed = PatchEmbed(img_size, patch_size, 3, enc_embed_dim) + + def _set_mask_generator(self, num_patches, mask_ratio): + self.mask_generator = RandomMask(num_patches, mask_ratio) + + def _set_mask_token(self, dec_embed_dim): + self.mask_token = nn.Parameter(torch.zeros(1, 1, dec_embed_dim)) + + def _set_decoder(self, enc_embed_dim, dec_embed_dim, dec_num_heads, dec_depth, mlp_ratio, norm_layer, norm_im2_in_dec): + self.dec_depth = dec_depth + self.dec_embed_dim = dec_embed_dim + # transfer from encoder to decoder + self.decoder_embed = nn.Linear(enc_embed_dim, dec_embed_dim, bias=True) + # transformer for the decoder + self.dec_blocks = nn.ModuleList([ + DecoderBlock(dec_embed_dim, dec_num_heads, mlp_ratio=mlp_ratio, qkv_bias=True, norm_layer=norm_layer, norm_mem=norm_im2_in_dec, rope=self.rope) + for i in range(dec_depth)]) + # final norm layer + self.dec_norm = norm_layer(dec_embed_dim) + + def _set_prediction_head(self, dec_embed_dim, patch_size): + self.prediction_head = nn.Linear(dec_embed_dim, patch_size**2 * 3, bias=True) + + + def initialize_weights(self): + # patch embed + self.patch_embed._init_weights() + # mask tokens + if self.mask_token is not None: torch.nn.init.normal_(self.mask_token, std=.02) + # linears and layer norms + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + # we use xavier_uniform following official JAX ViT: + torch.nn.init.xavier_uniform_(m.weight) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def _encode_image(self, image, do_mask=False, return_all_blocks=False): + """ + image has B x 3 x img_size x img_size + do_mask: whether to perform masking or not + return_all_blocks: if True, return the features at the end of every block + instead of just the features from the last block (eg for some prediction heads) + """ + # embed the image into patches (x has size B x Npatches x C) + # and get position if each return patch (pos has size B x Npatches x 2) + x, pos = self.patch_embed(image) + # add positional embedding without cls token + if self.enc_pos_embed is not None: + x = x + self.enc_pos_embed[None,...] + # apply masking + B,N,C = x.size() + if do_mask: + masks = self.mask_generator(x) + x = x[~masks].view(B, -1, C) + posvis = pos[~masks].view(B, -1, 2) + else: + B,N,C = x.size() + masks = torch.zeros((B,N), dtype=bool) + posvis = pos + # now apply the transformer encoder and normalization + if return_all_blocks: + out = [] + for blk in self.enc_blocks: + x = blk(x, posvis) + out.append(x) + out[-1] = self.enc_norm(out[-1]) + return out, pos, masks + else: + for blk in self.enc_blocks: + x = blk(x, posvis) + x = self.enc_norm(x) + return x, pos, masks + + def _decoder(self, feat1, pos1, masks1, feat2, pos2, return_all_blocks=False): + """ + return_all_blocks: if True, return the features at the end of every block + instead of just the features from the last block (eg for some prediction heads) + + masks1 can be None => assume image1 fully visible + """ + # encoder to decoder layer + visf1 = self.decoder_embed(feat1) + f2 = self.decoder_embed(feat2) + # append masked tokens to the sequence + B,Nenc,C = visf1.size() + if masks1 is None: # downstreams + f1_ = visf1 + else: # pretraining + Ntotal = masks1.size(1) + f1_ = self.mask_token.repeat(B, Ntotal, 1).to(dtype=visf1.dtype) + f1_[~masks1] = visf1.view(B * Nenc, C) + # add positional embedding + if self.dec_pos_embed is not None: + f1_ = f1_ + self.dec_pos_embed + f2 = f2 + self.dec_pos_embed + # apply Transformer blocks + out = f1_ + out2 = f2 + if return_all_blocks: + _out, out = out, [] + for blk in self.dec_blocks: + _out, out2 = blk(_out, out2, pos1, pos2) + out.append(_out) + out[-1] = self.dec_norm(out[-1]) + else: + for blk in self.dec_blocks: + out, out2 = blk(out, out2, pos1, pos2) + out = self.dec_norm(out) + return out + + def patchify(self, imgs): + """ + imgs: (B, 3, H, W) + x: (B, L, patch_size**2 *3) + """ + p = self.patch_embed.patch_size[0] + assert imgs.shape[2] == imgs.shape[3] and imgs.shape[2] % p == 0 + + h = w = imgs.shape[2] // p + x = imgs.reshape(shape=(imgs.shape[0], 3, h, p, w, p)) + x = torch.einsum('nchpwq->nhwpqc', x) + x = x.reshape(shape=(imgs.shape[0], h * w, p**2 * 3)) + + return x + + def unpatchify(self, x, channels=3): + """ + x: (N, L, patch_size**2 *channels) + imgs: (N, 3, H, W) + """ + patch_size = self.patch_embed.patch_size[0] + h = w = int(x.shape[1]**.5) + assert h * w == x.shape[1] + x = x.reshape(shape=(x.shape[0], h, w, patch_size, patch_size, channels)) + x = torch.einsum('nhwpqc->nchpwq', x) + imgs = x.reshape(shape=(x.shape[0], channels, h * patch_size, h * patch_size)) + return imgs + + def forward(self, img1, img2): + """ + img1: tensor of size B x 3 x img_size x img_size + img2: tensor of size B x 3 x img_size x img_size + + out will be B x N x (3*patch_size*patch_size) + masks are also returned as B x N just in case + """ + # encoder of the masked first image + feat1, pos1, mask1 = self._encode_image(img1, do_mask=True) + # encoder of the second image + feat2, pos2, _ = self._encode_image(img2, do_mask=False) + # decoder + decfeat = self._decoder(feat1, pos1, mask1, feat2, pos2) + # prediction head + out = self.prediction_head(decfeat) + # get target + target = self.patchify(img1) + return out, mask1, target diff --git a/src/mast3r_src/dust3r/croco/models/croco_downstream.py b/src/mast3r_src/dust3r/croco/models/croco_downstream.py new file mode 100644 index 0000000000000000000000000000000000000000..159dfff4d2c1461bc235e21441b57ce1e2088f76 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/models/croco_downstream.py @@ -0,0 +1,122 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# CroCo model for downstream tasks +# -------------------------------------------------------- + +import torch + +from .croco import CroCoNet + + +def croco_args_from_ckpt(ckpt): + if 'croco_kwargs' in ckpt: # CroCo v2 released models + return ckpt['croco_kwargs'] + elif 'args' in ckpt and hasattr(ckpt['args'], 'model'): # pretrained using the official code release + s = ckpt['args'].model # eg "CroCoNet(enc_embed_dim=1024, enc_num_heads=16, enc_depth=24)" + assert s.startswith('CroCoNet(') + return eval('dict'+s[len('CroCoNet'):]) # transform it into the string of a dictionary and evaluate it + else: # CroCo v1 released models + return dict() + +class CroCoDownstreamMonocularEncoder(CroCoNet): + + def __init__(self, + head, + **kwargs): + """ Build network for monocular downstream task, only using the encoder. + It takes an extra argument head, that is called with the features + and a dictionary img_info containing 'width' and 'height' keys + The head is setup with the croconet arguments in this init function + NOTE: It works by *calling super().__init__() but with redefined setters + + """ + super(CroCoDownstreamMonocularEncoder, self).__init__(**kwargs) + head.setup(self) + self.head = head + + def _set_mask_generator(self, *args, **kwargs): + """ No mask generator """ + return + + def _set_mask_token(self, *args, **kwargs): + """ No mask token """ + self.mask_token = None + return + + def _set_decoder(self, *args, **kwargs): + """ No decoder """ + return + + def _set_prediction_head(self, *args, **kwargs): + """ No 'prediction head' for downstream tasks.""" + return + + def forward(self, img): + """ + img if of size batch_size x 3 x h x w + """ + B, C, H, W = img.size() + img_info = {'height': H, 'width': W} + need_all_layers = hasattr(self.head, 'return_all_blocks') and self.head.return_all_blocks + out, _, _ = self._encode_image(img, do_mask=False, return_all_blocks=need_all_layers) + return self.head(out, img_info) + + +class CroCoDownstreamBinocular(CroCoNet): + + def __init__(self, + head, + **kwargs): + """ Build network for binocular downstream task + It takes an extra argument head, that is called with the features + and a dictionary img_info containing 'width' and 'height' keys + The head is setup with the croconet arguments in this init function + """ + super(CroCoDownstreamBinocular, self).__init__(**kwargs) + head.setup(self) + self.head = head + + def _set_mask_generator(self, *args, **kwargs): + """ No mask generator """ + return + + def _set_mask_token(self, *args, **kwargs): + """ No mask token """ + self.mask_token = None + return + + def _set_prediction_head(self, *args, **kwargs): + """ No prediction head for downstream tasks, define your own head """ + return + + def encode_image_pairs(self, img1, img2, return_all_blocks=False): + """ run encoder for a pair of images + it is actually ~5% faster to concatenate the images along the batch dimension + than to encode them separately + """ + ## the two commented lines below is the naive version with separate encoding + #out, pos, _ = self._encode_image(img1, do_mask=False, return_all_blocks=return_all_blocks) + #out2, pos2, _ = self._encode_image(img2, do_mask=False, return_all_blocks=False) + ## and now the faster version + out, pos, _ = self._encode_image( torch.cat( (img1,img2), dim=0), do_mask=False, return_all_blocks=return_all_blocks ) + if return_all_blocks: + out,out2 = list(map(list, zip(*[o.chunk(2, dim=0) for o in out]))) + out2 = out2[-1] + else: + out,out2 = out.chunk(2, dim=0) + pos,pos2 = pos.chunk(2, dim=0) + return out, out2, pos, pos2 + + def forward(self, img1, img2): + B, C, H, W = img1.size() + img_info = {'height': H, 'width': W} + return_all_blocks = hasattr(self.head, 'return_all_blocks') and self.head.return_all_blocks + out, out2, pos, pos2 = self.encode_image_pairs(img1, img2, return_all_blocks=return_all_blocks) + if return_all_blocks: + decout = self._decoder(out[-1], pos, None, out2, pos2, return_all_blocks=return_all_blocks) + decout = out+decout + else: + decout = self._decoder(out, pos, None, out2, pos2, return_all_blocks=return_all_blocks) + return self.head(decout, img_info) \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/models/curope/__init__.py b/src/mast3r_src/dust3r/croco/models/curope/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..25e3d48a162760260826080f6366838e83e26878 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/models/curope/__init__.py @@ -0,0 +1,4 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +from .curope2d import cuRoPE2D diff --git a/src/mast3r_src/dust3r/croco/models/curope/curope.cpp b/src/mast3r_src/dust3r/croco/models/curope/curope.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8fe9058e05aa1bf3f37b0d970edc7312bc68455b --- /dev/null +++ b/src/mast3r_src/dust3r/croco/models/curope/curope.cpp @@ -0,0 +1,69 @@ +/* + Copyright (C) 2022-present Naver Corporation. All rights reserved. + Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +*/ + +#include + +// forward declaration +void rope_2d_cuda( torch::Tensor tokens, const torch::Tensor pos, const float base, const float fwd ); + +void rope_2d_cpu( torch::Tensor tokens, const torch::Tensor positions, const float base, const float fwd ) +{ + const int B = tokens.size(0); + const int N = tokens.size(1); + const int H = tokens.size(2); + const int D = tokens.size(3) / 4; + + auto tok = tokens.accessor(); + auto pos = positions.accessor(); + + for (int b = 0; b < B; b++) { + for (int x = 0; x < 2; x++) { // y and then x (2d) + for (int n = 0; n < N; n++) { + + // grab the token position + const int p = pos[b][n][x]; + + for (int h = 0; h < H; h++) { + for (int d = 0; d < D; d++) { + // grab the two values + float u = tok[b][n][h][d+0+x*2*D]; + float v = tok[b][n][h][d+D+x*2*D]; + + // grab the cos,sin + const float inv_freq = fwd * p / powf(base, d/float(D)); + float c = cosf(inv_freq); + float s = sinf(inv_freq); + + // write the result + tok[b][n][h][d+0+x*2*D] = u*c - v*s; + tok[b][n][h][d+D+x*2*D] = v*c + u*s; + } + } + } + } + } +} + +void rope_2d( torch::Tensor tokens, // B,N,H,D + const torch::Tensor positions, // B,N,2 + const float base, + const float fwd ) +{ + TORCH_CHECK(tokens.dim() == 4, "tokens must have 4 dimensions"); + TORCH_CHECK(positions.dim() == 3, "positions must have 3 dimensions"); + TORCH_CHECK(tokens.size(0) == positions.size(0), "batch size differs between tokens & positions"); + TORCH_CHECK(tokens.size(1) == positions.size(1), "seq_length differs between tokens & positions"); + TORCH_CHECK(positions.size(2) == 2, "positions.shape[2] must be equal to 2"); + TORCH_CHECK(tokens.is_cuda() == positions.is_cuda(), "tokens and positions are not on the same device" ); + + if (tokens.is_cuda()) + rope_2d_cuda( tokens, positions, base, fwd ); + else + rope_2d_cpu( tokens, positions, base, fwd ); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("rope_2d", &rope_2d, "RoPE 2d forward/backward"); +} diff --git a/src/mast3r_src/dust3r/croco/models/curope/curope2d.py b/src/mast3r_src/dust3r/croco/models/curope/curope2d.py new file mode 100644 index 0000000000000000000000000000000000000000..a49c12f8c529e9a889b5ac20c5767158f238e17d --- /dev/null +++ b/src/mast3r_src/dust3r/croco/models/curope/curope2d.py @@ -0,0 +1,40 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +import torch + +try: + import curope as _kernels # run `python setup.py install` +except ModuleNotFoundError: + from . import curope as _kernels # run `python setup.py build_ext --inplace` + + +class cuRoPE2D_func (torch.autograd.Function): + + @staticmethod + def forward(ctx, tokens, positions, base, F0=1): + ctx.save_for_backward(positions) + ctx.saved_base = base + ctx.saved_F0 = F0 + # tokens = tokens.clone() # uncomment this if inplace doesn't work + _kernels.rope_2d( tokens, positions, base, F0 ) + ctx.mark_dirty(tokens) + return tokens + + @staticmethod + def backward(ctx, grad_res): + positions, base, F0 = ctx.saved_tensors[0], ctx.saved_base, ctx.saved_F0 + _kernels.rope_2d( grad_res, positions, base, -F0 ) + ctx.mark_dirty(grad_res) + return grad_res, None, None, None + + +class cuRoPE2D(torch.nn.Module): + def __init__(self, freq=100.0, F0=1.0): + super().__init__() + self.base = freq + self.F0 = F0 + + def forward(self, tokens, positions): + cuRoPE2D_func.apply( tokens.transpose(1,2), positions, self.base, self.F0 ) + return tokens \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/models/curope/kernels.cu b/src/mast3r_src/dust3r/croco/models/curope/kernels.cu new file mode 100644 index 0000000000000000000000000000000000000000..7156cd1bb935cb1f0be45e58add53f9c21505c20 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/models/curope/kernels.cu @@ -0,0 +1,108 @@ +/* + Copyright (C) 2022-present Naver Corporation. All rights reserved. + Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +*/ + +#include +#include +#include +#include + +#define CHECK_CUDA(tensor) {\ + TORCH_CHECK((tensor).is_cuda(), #tensor " is not in cuda memory"); \ + TORCH_CHECK((tensor).is_contiguous(), #tensor " is not contiguous"); } +void CHECK_KERNEL() {auto error = cudaGetLastError(); TORCH_CHECK( error == cudaSuccess, cudaGetErrorString(error));} + + +template < typename scalar_t > +__global__ void rope_2d_cuda_kernel( + //scalar_t* __restrict__ tokens, + torch::PackedTensorAccessor32 tokens, + const int64_t* __restrict__ pos, + const float base, + const float fwd ) + // const int N, const int H, const int D ) +{ + // tokens shape = (B, N, H, D) + const int N = tokens.size(1); + const int H = tokens.size(2); + const int D = tokens.size(3); + + // each block update a single token, for all heads + // each thread takes care of a single output + extern __shared__ float shared[]; + float* shared_inv_freq = shared + D; + + const int b = blockIdx.x / N; + const int n = blockIdx.x % N; + + const int Q = D / 4; + // one token = [0..Q : Q..2Q : 2Q..3Q : 3Q..D] + // u_Y v_Y u_X v_X + + // shared memory: first, compute inv_freq + if (threadIdx.x < Q) + shared_inv_freq[threadIdx.x] = fwd / powf(base, threadIdx.x/float(Q)); + __syncthreads(); + + // start of X or Y part + const int X = threadIdx.x < D/2 ? 0 : 1; + const int m = (X*D/2) + (threadIdx.x % Q); // index of u_Y or u_X + + // grab the cos,sin appropriate for me + const float freq = pos[blockIdx.x*2+X] * shared_inv_freq[threadIdx.x % Q]; + const float cos = cosf(freq); + const float sin = sinf(freq); + /* + float* shared_cos_sin = shared + D + D/4; + if ((threadIdx.x % (D/2)) < Q) + shared_cos_sin[m+0] = cosf(freq); + else + shared_cos_sin[m+Q] = sinf(freq); + __syncthreads(); + const float cos = shared_cos_sin[m+0]; + const float sin = shared_cos_sin[m+Q]; + */ + + for (int h = 0; h < H; h++) + { + // then, load all the token for this head in shared memory + shared[threadIdx.x] = tokens[b][n][h][threadIdx.x]; + __syncthreads(); + + const float u = shared[m]; + const float v = shared[m+Q]; + + // write output + if ((threadIdx.x % (D/2)) < Q) + tokens[b][n][h][threadIdx.x] = u*cos - v*sin; + else + tokens[b][n][h][threadIdx.x] = v*cos + u*sin; + } +} + +void rope_2d_cuda( torch::Tensor tokens, const torch::Tensor pos, const float base, const float fwd ) +{ + const int B = tokens.size(0); // batch size + const int N = tokens.size(1); // sequence length + const int H = tokens.size(2); // number of heads + const int D = tokens.size(3); // dimension per head + + TORCH_CHECK(tokens.stride(3) == 1 && tokens.stride(2) == D, "tokens are not contiguous"); + TORCH_CHECK(pos.is_contiguous(), "positions are not contiguous"); + TORCH_CHECK(pos.size(0) == B && pos.size(1) == N && pos.size(2) == 2, "bad pos.shape"); + TORCH_CHECK(D % 4 == 0, "token dim must be multiple of 4"); + + // one block for each layer, one thread per local-max + const int THREADS_PER_BLOCK = D; + const int N_BLOCKS = B * N; // each block takes care of H*D values + const int SHARED_MEM = sizeof(float) * (D + D/4); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF(tokens.type(), "rope_2d_cuda", ([&] { + rope_2d_cuda_kernel <<>> ( + //tokens.data_ptr(), + tokens.packed_accessor32(), + pos.data_ptr(), + base, fwd); //, N, H, D ); + })); +} diff --git a/src/mast3r_src/dust3r/croco/models/curope/setup.py b/src/mast3r_src/dust3r/croco/models/curope/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..230632ed05e309200e8f93a3a852072333975009 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/models/curope/setup.py @@ -0,0 +1,34 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +from setuptools import setup +from torch import cuda +from torch.utils.cpp_extension import BuildExtension, CUDAExtension + +# compile for all possible CUDA architectures +all_cuda_archs = cuda.get_gencode_flags().replace('compute=','arch=').split() +# alternatively, you can list cuda archs that you want, eg: +# all_cuda_archs = [ + # '-gencode', 'arch=compute_70,code=sm_70', + # '-gencode', 'arch=compute_75,code=sm_75', + # '-gencode', 'arch=compute_80,code=sm_80', + # '-gencode', 'arch=compute_86,code=sm_86' +# ] + +setup( + name = 'curope', + ext_modules = [ + CUDAExtension( + name='curope', + sources=[ + "curope.cpp", + "kernels.cu", + ], + extra_compile_args = dict( + nvcc=['-O3','--ptxas-options=-v',"--use_fast_math"]+all_cuda_archs, + cxx=['-O3']) + ) + ], + cmdclass = { + 'build_ext': BuildExtension + }) diff --git a/src/mast3r_src/dust3r/croco/models/dpt_block.py b/src/mast3r_src/dust3r/croco/models/dpt_block.py new file mode 100644 index 0000000000000000000000000000000000000000..d4ddfb74e2769ceca88720d4c730e00afd71c763 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/models/dpt_block.py @@ -0,0 +1,450 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# DPT head for ViTs +# -------------------------------------------------------- +# References: +# https://github.com/isl-org/DPT +# https://github.com/EPFL-VILAB/MultiMAE/blob/main/multimae/output_adapters.py + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange, repeat +from typing import Union, Tuple, Iterable, List, Optional, Dict + +def pair(t): + return t if isinstance(t, tuple) else (t, t) + +def make_scratch(in_shape, out_shape, groups=1, expand=False): + scratch = nn.Module() + + out_shape1 = out_shape + out_shape2 = out_shape + out_shape3 = out_shape + out_shape4 = out_shape + if expand == True: + out_shape1 = out_shape + out_shape2 = out_shape * 2 + out_shape3 = out_shape * 4 + out_shape4 = out_shape * 8 + + scratch.layer1_rn = nn.Conv2d( + in_shape[0], + out_shape1, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + scratch.layer2_rn = nn.Conv2d( + in_shape[1], + out_shape2, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + scratch.layer3_rn = nn.Conv2d( + in_shape[2], + out_shape3, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + scratch.layer4_rn = nn.Conv2d( + in_shape[3], + out_shape4, + kernel_size=3, + stride=1, + padding=1, + bias=False, + groups=groups, + ) + + scratch.layer_rn = nn.ModuleList([ + scratch.layer1_rn, + scratch.layer2_rn, + scratch.layer3_rn, + scratch.layer4_rn, + ]) + + return scratch + +class ResidualConvUnit_custom(nn.Module): + """Residual convolution module.""" + + def __init__(self, features, activation, bn): + """Init. + Args: + features (int): number of features + """ + super().__init__() + + self.bn = bn + + self.groups = 1 + + self.conv1 = nn.Conv2d( + features, + features, + kernel_size=3, + stride=1, + padding=1, + bias=not self.bn, + groups=self.groups, + ) + + self.conv2 = nn.Conv2d( + features, + features, + kernel_size=3, + stride=1, + padding=1, + bias=not self.bn, + groups=self.groups, + ) + + if self.bn == True: + self.bn1 = nn.BatchNorm2d(features) + self.bn2 = nn.BatchNorm2d(features) + + self.activation = activation + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, x): + """Forward pass. + Args: + x (tensor): input + Returns: + tensor: output + """ + + out = self.activation(x) + out = self.conv1(out) + if self.bn == True: + out = self.bn1(out) + + out = self.activation(out) + out = self.conv2(out) + if self.bn == True: + out = self.bn2(out) + + if self.groups > 1: + out = self.conv_merge(out) + + return self.skip_add.add(out, x) + +class FeatureFusionBlock_custom(nn.Module): + """Feature fusion block.""" + + def __init__( + self, + features, + activation, + deconv=False, + bn=False, + expand=False, + align_corners=True, + width_ratio=1, + ): + """Init. + Args: + features (int): number of features + """ + super(FeatureFusionBlock_custom, self).__init__() + self.width_ratio = width_ratio + + self.deconv = deconv + self.align_corners = align_corners + + self.groups = 1 + + self.expand = expand + out_features = features + if self.expand == True: + out_features = features // 2 + + self.out_conv = nn.Conv2d( + features, + out_features, + kernel_size=1, + stride=1, + padding=0, + bias=True, + groups=1, + ) + + self.resConfUnit1 = ResidualConvUnit_custom(features, activation, bn) + self.resConfUnit2 = ResidualConvUnit_custom(features, activation, bn) + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, *xs): + """Forward pass. + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + res = self.resConfUnit1(xs[1]) + if self.width_ratio != 1: + res = F.interpolate(res, size=(output.shape[2], output.shape[3]), mode='bilinear') + + output = self.skip_add.add(output, res) + # output += res + + output = self.resConfUnit2(output) + + if self.width_ratio != 1: + # and output.shape[3] < self.width_ratio * output.shape[2] + #size=(image.shape[]) + if (output.shape[3] / output.shape[2]) < (2 / 3) * self.width_ratio: + shape = 3 * output.shape[3] + else: + shape = int(self.width_ratio * 2 * output.shape[2]) + output = F.interpolate(output, size=(2* output.shape[2], shape), mode='bilinear') + else: + output = nn.functional.interpolate(output, scale_factor=2, + mode="bilinear", align_corners=self.align_corners) + output = self.out_conv(output) + return output + +def make_fusion_block(features, use_bn, width_ratio=1): + return FeatureFusionBlock_custom( + features, + nn.ReLU(False), + deconv=False, + bn=use_bn, + expand=False, + align_corners=True, + width_ratio=width_ratio, + ) + +class Interpolate(nn.Module): + """Interpolation module.""" + + def __init__(self, scale_factor, mode, align_corners=False): + """Init. + Args: + scale_factor (float): scaling + mode (str): interpolation mode + """ + super(Interpolate, self).__init__() + + self.interp = nn.functional.interpolate + self.scale_factor = scale_factor + self.mode = mode + self.align_corners = align_corners + + def forward(self, x): + """Forward pass. + Args: + x (tensor): input + Returns: + tensor: interpolated data + """ + + x = self.interp( + x, + scale_factor=self.scale_factor, + mode=self.mode, + align_corners=self.align_corners, + ) + + return x + +class DPTOutputAdapter(nn.Module): + """DPT output adapter. + + :param num_cahnnels: Number of output channels + :param stride_level: tride level compared to the full-sized image. + E.g. 4 for 1/4th the size of the image. + :param patch_size_full: Int or tuple of the patch size over the full image size. + Patch size for smaller inputs will be computed accordingly. + :param hooks: Index of intermediate layers + :param layer_dims: Dimension of intermediate layers + :param feature_dim: Feature dimension + :param last_dim: out_channels/in_channels for the last two Conv2d when head_type == regression + :param use_bn: If set to True, activates batch norm + :param dim_tokens_enc: Dimension of tokens coming from encoder + """ + + def __init__(self, + num_channels: int = 1, + stride_level: int = 1, + patch_size: Union[int, Tuple[int, int]] = 16, + main_tasks: Iterable[str] = ('rgb',), + hooks: List[int] = [2, 5, 8, 11], + layer_dims: List[int] = [96, 192, 384, 768], + feature_dim: int = 256, + last_dim: int = 32, + use_bn: bool = False, + dim_tokens_enc: Optional[int] = None, + head_type: str = 'regression', + output_width_ratio=1, + **kwargs): + super().__init__() + self.num_channels = num_channels + self.stride_level = stride_level + self.patch_size = pair(patch_size) + self.main_tasks = main_tasks + self.hooks = hooks + self.layer_dims = layer_dims + self.feature_dim = feature_dim + self.dim_tokens_enc = dim_tokens_enc * len(self.main_tasks) if dim_tokens_enc is not None else None + self.head_type = head_type + + # Actual patch height and width, taking into account stride of input + self.P_H = max(1, self.patch_size[0] // stride_level) + self.P_W = max(1, self.patch_size[1] // stride_level) + + self.scratch = make_scratch(layer_dims, feature_dim, groups=1, expand=False) + + self.scratch.refinenet1 = make_fusion_block(feature_dim, use_bn, output_width_ratio) + self.scratch.refinenet2 = make_fusion_block(feature_dim, use_bn, output_width_ratio) + self.scratch.refinenet3 = make_fusion_block(feature_dim, use_bn, output_width_ratio) + self.scratch.refinenet4 = make_fusion_block(feature_dim, use_bn, output_width_ratio) + + if self.head_type == 'regression': + # The "DPTDepthModel" head + self.head = nn.Sequential( + nn.Conv2d(feature_dim, feature_dim // 2, kernel_size=3, stride=1, padding=1), + Interpolate(scale_factor=2, mode="bilinear", align_corners=True), + nn.Conv2d(feature_dim // 2, last_dim, kernel_size=3, stride=1, padding=1), + nn.ReLU(True), + nn.Conv2d(last_dim, self.num_channels, kernel_size=1, stride=1, padding=0) + ) + elif self.head_type == 'semseg': + # The "DPTSegmentationModel" head + self.head = nn.Sequential( + nn.Conv2d(feature_dim, feature_dim, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(feature_dim) if use_bn else nn.Identity(), + nn.ReLU(True), + nn.Dropout(0.1, False), + nn.Conv2d(feature_dim, self.num_channels, kernel_size=1), + Interpolate(scale_factor=2, mode="bilinear", align_corners=True), + ) + else: + raise ValueError('DPT head_type must be "regression" or "semseg".') + + if self.dim_tokens_enc is not None: + self.init(dim_tokens_enc=dim_tokens_enc) + + def init(self, dim_tokens_enc=768): + """ + Initialize parts of decoder that are dependent on dimension of encoder tokens. + Should be called when setting up MultiMAE. + + :param dim_tokens_enc: Dimension of tokens coming from encoder + """ + #print(dim_tokens_enc) + + # Set up activation postprocessing layers + if isinstance(dim_tokens_enc, int): + dim_tokens_enc = 4 * [dim_tokens_enc] + + self.dim_tokens_enc = [dt * len(self.main_tasks) for dt in dim_tokens_enc] + + self.act_1_postprocess = nn.Sequential( + nn.Conv2d( + in_channels=self.dim_tokens_enc[0], + out_channels=self.layer_dims[0], + kernel_size=1, stride=1, padding=0, + ), + nn.ConvTranspose2d( + in_channels=self.layer_dims[0], + out_channels=self.layer_dims[0], + kernel_size=4, stride=4, padding=0, + bias=True, dilation=1, groups=1, + ) + ) + + self.act_2_postprocess = nn.Sequential( + nn.Conv2d( + in_channels=self.dim_tokens_enc[1], + out_channels=self.layer_dims[1], + kernel_size=1, stride=1, padding=0, + ), + nn.ConvTranspose2d( + in_channels=self.layer_dims[1], + out_channels=self.layer_dims[1], + kernel_size=2, stride=2, padding=0, + bias=True, dilation=1, groups=1, + ) + ) + + self.act_3_postprocess = nn.Sequential( + nn.Conv2d( + in_channels=self.dim_tokens_enc[2], + out_channels=self.layer_dims[2], + kernel_size=1, stride=1, padding=0, + ) + ) + + self.act_4_postprocess = nn.Sequential( + nn.Conv2d( + in_channels=self.dim_tokens_enc[3], + out_channels=self.layer_dims[3], + kernel_size=1, stride=1, padding=0, + ), + nn.Conv2d( + in_channels=self.layer_dims[3], + out_channels=self.layer_dims[3], + kernel_size=3, stride=2, padding=1, + ) + ) + + self.act_postprocess = nn.ModuleList([ + self.act_1_postprocess, + self.act_2_postprocess, + self.act_3_postprocess, + self.act_4_postprocess + ]) + + def adapt_tokens(self, encoder_tokens): + # Adapt tokens + x = [] + x.append(encoder_tokens[:, :]) + x = torch.cat(x, dim=-1) + return x + + def forward(self, encoder_tokens: List[torch.Tensor], image_size): + #input_info: Dict): + assert self.dim_tokens_enc is not None, 'Need to call init(dim_tokens_enc) function first' + H, W = image_size + + # Number of patches in height and width + N_H = H // (self.stride_level * self.P_H) + N_W = W // (self.stride_level * self.P_W) + + # Hook decoder onto 4 layers from specified ViT layers + layers = [encoder_tokens[hook] for hook in self.hooks] + + # Extract only task-relevant tokens and ignore global tokens. + layers = [self.adapt_tokens(l) for l in layers] + + # Reshape tokens to spatial representation + layers = [rearrange(l, 'b (nh nw) c -> b c nh nw', nh=N_H, nw=N_W) for l in layers] + + layers = [self.act_postprocess[idx](l) for idx, l in enumerate(layers)] + # Project layers to chosen feature dim + layers = [self.scratch.layer_rn[idx](l) for idx, l in enumerate(layers)] + + # Fuse layers using refinement stages + path_4 = self.scratch.refinenet4(layers[3]) + path_3 = self.scratch.refinenet3(path_4, layers[2]) + path_2 = self.scratch.refinenet2(path_3, layers[1]) + path_1 = self.scratch.refinenet1(path_2, layers[0]) + + # Output head + out = self.head(path_1) + + return out diff --git a/src/mast3r_src/dust3r/croco/models/head_downstream.py b/src/mast3r_src/dust3r/croco/models/head_downstream.py new file mode 100644 index 0000000000000000000000000000000000000000..bd40c91ba244d6c3522c6efd4ed4d724b7bdc650 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/models/head_downstream.py @@ -0,0 +1,58 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# Heads for downstream tasks +# -------------------------------------------------------- + +""" +A head is a module where the __init__ defines only the head hyperparameters. +A method setup(croconet) takes a CroCoNet and set all layers according to the head and croconet attributes. +The forward takes the features as well as a dictionary img_info containing the keys 'width' and 'height' +""" + +import torch +import torch.nn as nn +from .dpt_block import DPTOutputAdapter + + +class PixelwiseTaskWithDPT(nn.Module): + """ DPT module for CroCo. + by default, hooks_idx will be equal to: + * for encoder-only: 4 equally spread layers + * for encoder+decoder: last encoder + 3 equally spread layers of the decoder + """ + + def __init__(self, *, hooks_idx=None, layer_dims=[96,192,384,768], + output_width_ratio=1, num_channels=1, postprocess=None, **kwargs): + super(PixelwiseTaskWithDPT, self).__init__() + self.return_all_blocks = True # backbone needs to return all layers + self.postprocess = postprocess + self.output_width_ratio = output_width_ratio + self.num_channels = num_channels + self.hooks_idx = hooks_idx + self.layer_dims = layer_dims + + def setup(self, croconet): + dpt_args = {'output_width_ratio': self.output_width_ratio, 'num_channels': self.num_channels} + if self.hooks_idx is None: + if hasattr(croconet, 'dec_blocks'): # encoder + decoder + step = {8: 3, 12: 4, 24: 8}[croconet.dec_depth] + hooks_idx = [croconet.dec_depth+croconet.enc_depth-1-i*step for i in range(3,-1,-1)] + else: # encoder only + step = croconet.enc_depth//4 + hooks_idx = [croconet.enc_depth-1-i*step for i in range(3,-1,-1)] + self.hooks_idx = hooks_idx + print(f' PixelwiseTaskWithDPT: automatically setting hook_idxs={self.hooks_idx}') + dpt_args['hooks'] = self.hooks_idx + dpt_args['layer_dims'] = self.layer_dims + self.dpt = DPTOutputAdapter(**dpt_args) + dim_tokens = [croconet.enc_embed_dim if hook0: + pos_embed = np.concatenate([np.zeros([n_cls_token, embed_dim]), pos_embed], axis=0) + return pos_embed + + +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): + assert embed_dim % 2 == 0 + + # use half of dimensions to encode grid_h + emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2) + emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2) + + emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) + return emb + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=float) + omega /= embed_dim / 2. + omega = 1. / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +# -------------------------------------------------------- +# Interpolate position embeddings for high-resolution +# References: +# MAE: https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py +# DeiT: https://github.com/facebookresearch/deit +# -------------------------------------------------------- +def interpolate_pos_embed(model, checkpoint_model): + if 'pos_embed' in checkpoint_model: + pos_embed_checkpoint = checkpoint_model['pos_embed'] + embedding_size = pos_embed_checkpoint.shape[-1] + num_patches = model.patch_embed.num_patches + num_extra_tokens = model.pos_embed.shape[-2] - num_patches + # height (== width) for the checkpoint position embedding + orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5) + # height (== width) for the new position embedding + new_size = int(num_patches ** 0.5) + # class_token and dist_token are kept unchanged + if orig_size != new_size: + print("Position interpolate from %dx%d to %dx%d" % (orig_size, orig_size, new_size, new_size)) + extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens] + # only the position tokens are interpolated + pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:] + pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2) + pos_tokens = torch.nn.functional.interpolate( + pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False) + pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2) + new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1) + checkpoint_model['pos_embed'] = new_pos_embed + + +#---------------------------------------------------------- +# RoPE2D: RoPE implementation in 2D +#---------------------------------------------------------- + +try: + from models.curope import cuRoPE2D + RoPE2D = cuRoPE2D +except ImportError: + print('Warning, cannot find cuda-compiled version of RoPE2D, using a slow pytorch version instead') + + class RoPE2D(torch.nn.Module): + + def __init__(self, freq=100.0, F0=1.0): + super().__init__() + self.base = freq + self.F0 = F0 + self.cache = {} + + def get_cos_sin(self, D, seq_len, device, dtype): + if (D,seq_len,device,dtype) not in self.cache: + inv_freq = 1.0 / (self.base ** (torch.arange(0, D, 2).float().to(device) / D)) + t = torch.arange(seq_len, device=device, dtype=inv_freq.dtype) + freqs = torch.einsum("i,j->ij", t, inv_freq).to(dtype) + freqs = torch.cat((freqs, freqs), dim=-1) + cos = freqs.cos() # (Seq, Dim) + sin = freqs.sin() + self.cache[D,seq_len,device,dtype] = (cos,sin) + return self.cache[D,seq_len,device,dtype] + + @staticmethod + def rotate_half(x): + x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + def apply_rope1d(self, tokens, pos1d, cos, sin): + assert pos1d.ndim==2 + cos = torch.nn.functional.embedding(pos1d, cos)[:, None, :, :] + sin = torch.nn.functional.embedding(pos1d, sin)[:, None, :, :] + return (tokens * cos) + (self.rotate_half(tokens) * sin) + + def forward(self, tokens, positions): + """ + input: + * tokens: batch_size x nheads x ntokens x dim + * positions: batch_size x ntokens x 2 (y and x position of each token) + output: + * tokens after appplying RoPE2D (batch_size x nheads x ntokens x dim) + """ + assert tokens.size(3)%2==0, "number of dimensions should be a multiple of two" + D = tokens.size(3) // 2 + assert positions.ndim==3 and positions.shape[-1] == 2 # Batch, Seq, 2 + cos, sin = self.get_cos_sin(D, int(positions.max())+1, tokens.device, tokens.dtype) + # split features into two along the feature dimension, and apply rope1d on each half + y, x = tokens.chunk(2, dim=-1) + y = self.apply_rope1d(y, positions[:,:,0], cos, sin) + x = self.apply_rope1d(x, positions[:,:,1], cos, sin) + tokens = torch.cat((y, x), dim=-1) + return tokens \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/pretrain.py b/src/mast3r_src/dust3r/croco/pretrain.py new file mode 100644 index 0000000000000000000000000000000000000000..2c45e488015ef5380c71d0381ff453fdb860759e --- /dev/null +++ b/src/mast3r_src/dust3r/croco/pretrain.py @@ -0,0 +1,254 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Pre-training CroCo +# -------------------------------------------------------- +# References: +# MAE: https://github.com/facebookresearch/mae +# DeiT: https://github.com/facebookresearch/deit +# BEiT: https://github.com/microsoft/unilm/tree/master/beit +# -------------------------------------------------------- +import argparse +import datetime +import json +import numpy as np +import os +import sys +import time +import math +from pathlib import Path +from typing import Iterable + +import torch +import torch.distributed as dist +import torch.backends.cudnn as cudnn +from torch.utils.tensorboard import SummaryWriter +import torchvision.transforms as transforms +import torchvision.datasets as datasets + +import utils.misc as misc +from utils.misc import NativeScalerWithGradNormCount as NativeScaler +from models.croco import CroCoNet +from models.criterion import MaskedMSE +from datasets.pairs_dataset import PairsDataset + + +def get_args_parser(): + parser = argparse.ArgumentParser('CroCo pre-training', add_help=False) + # model and criterion + parser.add_argument('--model', default='CroCoNet()', type=str, help="string containing the model to build") + parser.add_argument('--norm_pix_loss', default=1, choices=[0,1], help="apply per-patch mean/std normalization before applying the loss") + # dataset + parser.add_argument('--dataset', default='habitat_release', type=str, help="training set") + parser.add_argument('--transforms', default='crop224+acolor', type=str, help="transforms to apply") # in the paper, we also use some homography and rotation, but find later that they were not useful or even harmful + # training + parser.add_argument('--seed', default=0, type=int, help="Random seed") + parser.add_argument('--batch_size', default=64, type=int, help="Batch size per GPU (effective batch size is batch_size * accum_iter * # gpus") + parser.add_argument('--epochs', default=800, type=int, help="Maximum number of epochs for the scheduler") + parser.add_argument('--max_epoch', default=400, type=int, help="Stop training at this epoch") + parser.add_argument('--accum_iter', default=1, type=int, help="Accumulate gradient iterations (for increasing the effective batch size under memory constraints)") + parser.add_argument('--weight_decay', type=float, default=0.05, help="weight decay (default: 0.05)") + parser.add_argument('--lr', type=float, default=None, metavar='LR', help='learning rate (absolute lr)') + parser.add_argument('--blr', type=float, default=1.5e-4, metavar='LR', help='base learning rate: absolute_lr = base_lr * total_batch_size / 256') + parser.add_argument('--min_lr', type=float, default=0., metavar='LR', help='lower lr bound for cyclic schedulers that hit 0') + parser.add_argument('--warmup_epochs', type=int, default=40, metavar='N', help='epochs to warmup LR') + parser.add_argument('--amp', type=int, default=1, choices=[0,1], help="Use Automatic Mixed Precision for pretraining") + # others + parser.add_argument('--num_workers', default=8, type=int) + parser.add_argument('--world_size', default=1, type=int, help='number of distributed processes') + parser.add_argument('--local_rank', default=-1, type=int) + parser.add_argument('--dist_url', default='env://', help='url used to set up distributed training') + parser.add_argument('--save_freq', default=1, type=int, help='frequence (number of epochs) to save checkpoint in checkpoint-last.pth') + parser.add_argument('--keep_freq', default=20, type=int, help='frequence (number of epochs) to save checkpoint in checkpoint-%d.pth') + parser.add_argument('--print_freq', default=20, type=int, help='frequence (number of iterations) to print infos while training') + # paths + parser.add_argument('--output_dir', default='./output/', type=str, help="path where to save the output") + parser.add_argument('--data_dir', default='./data/', type=str, help="path where data are stored") + return parser + + + + +def main(args): + misc.init_distributed_mode(args) + global_rank = misc.get_rank() + world_size = misc.get_world_size() + + print("output_dir: "+args.output_dir) + if args.output_dir: + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + + # auto resume + last_ckpt_fname = os.path.join(args.output_dir, f'checkpoint-last.pth') + args.resume = last_ckpt_fname if os.path.isfile(last_ckpt_fname) else None + + print('job dir: {}'.format(os.path.dirname(os.path.realpath(__file__)))) + print("{}".format(args).replace(', ', ',\n')) + + device = "cuda" if torch.cuda.is_available() else "cpu" + device = torch.device(device) + + # fix the seed + seed = args.seed + misc.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + + cudnn.benchmark = True + + ## training dataset and loader + print('Building dataset for {:s} with transforms {:s}'.format(args.dataset, args.transforms)) + dataset = PairsDataset(args.dataset, trfs=args.transforms, data_dir=args.data_dir) + if world_size>1: + sampler_train = torch.utils.data.DistributedSampler( + dataset, num_replicas=world_size, rank=global_rank, shuffle=True + ) + print("Sampler_train = %s" % str(sampler_train)) + else: + sampler_train = torch.utils.data.RandomSampler(dataset) + data_loader_train = torch.utils.data.DataLoader( + dataset, sampler=sampler_train, + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=True, + ) + + ## model + print('Loading model: {:s}'.format(args.model)) + model = eval(args.model) + print('Loading criterion: MaskedMSE(norm_pix_loss={:s})'.format(str(bool(args.norm_pix_loss)))) + criterion = MaskedMSE(norm_pix_loss=bool(args.norm_pix_loss)) + + model.to(device) + model_without_ddp = model + print("Model = %s" % str(model_without_ddp)) + + eff_batch_size = args.batch_size * args.accum_iter * misc.get_world_size() + if args.lr is None: # only base_lr is specified + args.lr = args.blr * eff_batch_size / 256 + print("base lr: %.2e" % (args.lr * 256 / eff_batch_size)) + print("actual lr: %.2e" % args.lr) + print("accumulate grad iterations: %d" % args.accum_iter) + print("effective batch size: %d" % eff_batch_size) + + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], find_unused_parameters=True, static_graph=True) + model_without_ddp = model.module + + param_groups = misc.get_parameter_groups(model_without_ddp, args.weight_decay) # following timm: set wd as 0 for bias and norm layers + optimizer = torch.optim.AdamW(param_groups, lr=args.lr, betas=(0.9, 0.95)) + print(optimizer) + loss_scaler = NativeScaler() + + misc.load_model(args=args, model_without_ddp=model_without_ddp, optimizer=optimizer, loss_scaler=loss_scaler) + + if global_rank == 0 and args.output_dir is not None: + log_writer = SummaryWriter(log_dir=args.output_dir) + else: + log_writer = None + + print(f"Start training until {args.max_epoch} epochs") + start_time = time.time() + for epoch in range(args.start_epoch, args.max_epoch): + if world_size>1: + data_loader_train.sampler.set_epoch(epoch) + + train_stats = train_one_epoch( + model, criterion, data_loader_train, + optimizer, device, epoch, loss_scaler, + log_writer=log_writer, + args=args + ) + + if args.output_dir and epoch % args.save_freq == 0 : + misc.save_model( + args=args, model_without_ddp=model_without_ddp, optimizer=optimizer, + loss_scaler=loss_scaler, epoch=epoch, fname='last') + + if args.output_dir and (epoch % args.keep_freq == 0 or epoch + 1 == args.max_epoch) and (epoch>0 or args.max_epoch==1): + misc.save_model( + args=args, model_without_ddp=model_without_ddp, optimizer=optimizer, + loss_scaler=loss_scaler, epoch=epoch) + + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + 'epoch': epoch,} + + if args.output_dir and misc.is_main_process(): + if log_writer is not None: + log_writer.flush() + with open(os.path.join(args.output_dir, "log.txt"), mode="a", encoding="utf-8") as f: + f.write(json.dumps(log_stats) + "\n") + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + + + + +def train_one_epoch(model: torch.nn.Module, criterion: torch.nn.Module, + data_loader: Iterable, optimizer: torch.optim.Optimizer, + device: torch.device, epoch: int, loss_scaler, + log_writer=None, + args=None): + model.train(True) + metric_logger = misc.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', misc.SmoothedValue(window_size=1, fmt='{value:.6f}')) + header = 'Epoch: [{}]'.format(epoch) + accum_iter = args.accum_iter + + optimizer.zero_grad() + + if log_writer is not None: + print('log_dir: {}'.format(log_writer.log_dir)) + + for data_iter_step, (image1, image2) in enumerate(metric_logger.log_every(data_loader, args.print_freq, header)): + + # we use a per iteration lr scheduler + if data_iter_step % accum_iter == 0: + misc.adjust_learning_rate(optimizer, data_iter_step / len(data_loader) + epoch, args) + + image1 = image1.to(device, non_blocking=True) + image2 = image2.to(device, non_blocking=True) + with torch.cuda.amp.autocast(enabled=bool(args.amp)): + out, mask, target = model(image1, image2) + loss = criterion(out, mask, target) + + loss_value = loss.item() + + if not math.isfinite(loss_value): + print("Loss is {}, stopping training".format(loss_value)) + sys.exit(1) + + loss /= accum_iter + loss_scaler(loss, optimizer, parameters=model.parameters(), + update_grad=(data_iter_step + 1) % accum_iter == 0) + if (data_iter_step + 1) % accum_iter == 0: + optimizer.zero_grad() + + torch.cuda.synchronize() + + metric_logger.update(loss=loss_value) + + lr = optimizer.param_groups[0]["lr"] + metric_logger.update(lr=lr) + + loss_value_reduce = misc.all_reduce_mean(loss_value) + if log_writer is not None and ((data_iter_step + 1) % (accum_iter*args.print_freq)) == 0: + # x-axis is based on epoch_1000x in the tensorboard, calibrating differences curves when batch size changes + epoch_1000x = int((data_iter_step / len(data_loader) + epoch) * 1000) + log_writer.add_scalar('train_loss', loss_value_reduce, epoch_1000x) + log_writer.add_scalar('lr', lr, epoch_1000x) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger) + return {k: meter.global_avg for k, meter in metric_logger.meters.items()} + + + +if __name__ == '__main__': + args = get_args_parser() + args = args.parse_args() + main(args) diff --git a/src/mast3r_src/dust3r/croco/stereoflow/README.MD b/src/mast3r_src/dust3r/croco/stereoflow/README.MD new file mode 100644 index 0000000000000000000000000000000000000000..81595380fadd274b523e0cf77921b1b65cbedb34 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/stereoflow/README.MD @@ -0,0 +1,318 @@ +## CroCo-Stereo and CroCo-Flow + +This README explains how to use CroCo-Stereo and CroCo-Flow as well as how they were trained. +All commands should be launched from the root directory. + +### Simple inference example + +We provide a simple inference exemple for CroCo-Stereo and CroCo-Flow in the Totebook `croco-stereo-flow-demo.ipynb`. +Before running it, please download the trained models with: +``` +bash stereoflow/download_model.sh crocostereo.pth +bash stereoflow/download_model.sh crocoflow.pth +``` + +### Prepare data for training or evaluation + +Put the datasets used for training/evaluation in `./data/stereoflow` (or update the paths at the top of `stereoflow/datasets_stereo.py` and `stereoflow/datasets_flow.py`). +Please find below on the file structure should look for each dataset: +
+FlyingChairs + +``` +./data/stereoflow/FlyingChairs/ +└───chairs_split.txt +└───data/ + └─── ... +``` +
+ +
+MPI-Sintel + +``` +./data/stereoflow/MPI-Sintel/ +└───training/ +│ └───clean/ +│ └───final/ +│ └───flow/ +└───test/ + └───clean/ + └───final/ +``` +
+ +
+SceneFlow (including FlyingThings) + +``` +./data/stereoflow/SceneFlow/ +└───Driving/ +│ └───disparity/ +│ └───frames_cleanpass/ +│ └───frames_finalpass/ +└───FlyingThings/ +│ └───disparity/ +│ └───frames_cleanpass/ +│ └───frames_finalpass/ +│ └───optical_flow/ +└───Monkaa/ + └───disparity/ + └───frames_cleanpass/ + └───frames_finalpass/ +``` +
+ +
+TartanAir + +``` +./data/stereoflow/TartanAir/ +└───abandonedfactory/ +│ └───.../ +└───abandonedfactory_night/ +│ └───.../ +└───.../ +``` +
+ +
+Booster + +``` +./data/stereoflow/booster_gt/ +└───train/ + └───balanced/ + └───Bathroom/ + └───Bedroom/ + └───... +``` +
+ +
+CREStereo + +``` +./data/stereoflow/crenet_stereo_trainset/ +└───stereo_trainset/ + └───crestereo/ + └───hole/ + └───reflective/ + └───shapenet/ + └───tree/ +``` +
+ +
+ETH3D Two-view Low-res + +``` +./data/stereoflow/eth3d_lowres/ +└───test/ +│ └───lakeside_1l/ +│ └───... +└───train/ +│ └───delivery_area_1l/ +│ └───... +└───train_gt/ + └───delivery_area_1l/ + └───... +``` +
+ +
+KITTI 2012 + +``` +./data/stereoflow/kitti-stereo-2012/ +└───testing/ +│ └───colored_0/ +│ └───colored_1/ +└───training/ + └───colored_0/ + └───colored_1/ + └───disp_occ/ + └───flow_occ/ +``` +
+ +
+KITTI 2015 + +``` +./data/stereoflow/kitti-stereo-2015/ +└───testing/ +│ └───image_2/ +│ └───image_3/ +└───training/ + └───image_2/ + └───image_3/ + └───disp_occ_0/ + └───flow_occ/ +``` +
+ +
+Middlebury + +``` +./data/stereoflow/middlebury +└───2005/ +│ └───train/ +│ └───Art/ +│ └───... +└───2006/ +│ └───Aloe/ +│ └───Baby1/ +│ └───... +└───2014/ +│ └───Adirondack-imperfect/ +│ └───Adirondack-perfect/ +│ └───... +└───2021/ +│ └───data/ +│ └───artroom1/ +│ └───artroom2/ +│ └───... +└───MiddEval3_F/ + └───test/ + │ └───Australia/ + │ └───... + └───train/ + └───Adirondack/ + └───... +``` +
+ +
+Spring + +``` +./data/stereoflow/spring/ +└───test/ +│ └───0003/ +│ └───... +└───train/ + └───0001/ + └───... +``` +
+ + +### CroCo-Stereo + +##### Main model + +The main training of CroCo-Stereo was performed on a series of datasets, and it was used as it for Middlebury v3 benchmark. + +``` +# Download the model +bash stereoflow/download_model.sh crocostereo.pth +# Middlebury v3 submission +python stereoflow/test.py --model stereoflow_models/crocostereo.pth --dataset "MdEval3('all_full')" --save submission --tile_overlap 0.9 +# Training command that was used, using checkpoint-last.pth +python -u stereoflow/train.py stereo --criterion "LaplacianLossBounded2()" --dataset "CREStereo('train')+SceneFlow('train_allpass')+30*ETH3DLowRes('train')+50*Md05('train')+50*Md06('train')+50*Md14('train')+50*Md21('train')+50*MdEval3('train_full')+Booster('train_balanced')" --val_dataset "SceneFlow('test1of100_finalpass')+SceneFlow('test1of100_cleanpass')+ETH3DLowRes('subval')+Md05('subval')+Md06('subval')+Md14('subval')+Md21('subval')+MdEval3('subval_full')+Booster('subval_balanced')" --lr 3e-5 --batch_size 6 --epochs 32 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --output_dir xps/crocostereo/main/ +# or it can be launched on multiple gpus (while maintaining the effective batch size), e.g. on 3 gpus: +torchrun --nproc_per_node 3 stereoflow/train.py stereo --criterion "LaplacianLossBounded2()" --dataset "CREStereo('train')+SceneFlow('train_allpass')+30*ETH3DLowRes('train')+50*Md05('train')+50*Md06('train')+50*Md14('train')+50*Md21('train')+50*MdEval3('train_full')+Booster('train_balanced')" --val_dataset "SceneFlow('test1of100_finalpass')+SceneFlow('test1of100_cleanpass')+ETH3DLowRes('subval')+Md05('subval')+Md06('subval')+Md14('subval')+Md21('subval')+MdEval3('subval_full')+Booster('subval_balanced')" --lr 3e-5 --batch_size 2 --epochs 32 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --output_dir xps/crocostereo/main/ +``` + +For evaluation of validation set, we also provide the model trained on the `subtrain` subset of the training sets. + +``` +# Download the model +bash stereoflow/download_model.sh crocostereo_subtrain.pth +# Evaluation on validation sets +python stereoflow/test.py --model stereoflow_models/crocostereo_subtrain.pth --dataset "MdEval3('subval_full')+ETH3DLowRes('subval')+SceneFlow('test_finalpass')+SceneFlow('test_cleanpass')" --save metrics --tile_overlap 0.9 +# Training command that was used (same as above but on subtrain, using checkpoint-best.pth), can also be launched on multiple gpus +python -u stereoflow/train.py stereo --criterion "LaplacianLossBounded2()" --dataset "CREStereo('train')+SceneFlow('train_allpass')+30*ETH3DLowRes('subtrain')+50*Md05('subtrain')+50*Md06('subtrain')+50*Md14('subtrain')+50*Md21('subtrain')+50*MdEval3('subtrain_full')+Booster('subtrain_balanced')" --val_dataset "SceneFlow('test1of100_finalpass')+SceneFlow('test1of100_cleanpass')+ETH3DLowRes('subval')+Md05('subval')+Md06('subval')+Md14('subval')+Md21('subval')+MdEval3('subval_full')+Booster('subval_balanced')" --lr 3e-5 --batch_size 6 --epochs 32 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --output_dir xps/crocostereo/main_subtrain/ +``` + +##### Other models + +
+ Model for ETH3D + The model used for the submission on ETH3D is trained with the same command but using an unbounded Laplacian loss. + + # Download the model + bash stereoflow/download_model.sh crocostereo_eth3d.pth + # ETH3D submission + python stereoflow/test.py --model stereoflow_models/crocostereo_eth3d.pth --dataset "ETH3DLowRes('all')" --save submission --tile_overlap 0.9 + # Training command that was used + python -u stereoflow/train.py stereo --criterion "LaplacianLoss()" --tile_conf_mode conf_expbeta3 --dataset "CREStereo('train')+SceneFlow('train_allpass')+30*ETH3DLowRes('train')+50*Md05('train')+50*Md06('train')+50*Md14('train')+50*Md21('train')+50*MdEval3('train_full')+Booster('train_balanced')" --val_dataset "SceneFlow('test1of100_finalpass')+SceneFlow('test1of100_cleanpass')+ETH3DLowRes('subval')+Md05('subval')+Md06('subval')+Md14('subval')+Md21('subval')+MdEval3('subval_full')+Booster('subval_balanced')" --lr 3e-5 --batch_size 6 --epochs 32 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --output_dir xps/crocostereo/main_eth3d/ + +
+ +
+ Main model finetuned on Kitti + + # Download the model + bash stereoflow/download_model.sh crocostereo_finetune_kitti.pth + # Kitti submission + python stereoflow/test.py --model stereoflow_models/crocostereo_finetune_kitti.pth --dataset "Kitti15('test')" --save submission --tile_overlap 0.9 + # Training that was used + python -u stereoflow/train.py stereo --crop 352 1216 --criterion "LaplacianLossBounded2()" --dataset "Kitti12('train')+Kitti15('train')" --lr 3e-5 --batch_size 1 --accum_iter 6 --epochs 20 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --start_from stereoflow_models/crocostereo.pth --output_dir xps/crocostereo/finetune_kitti/ --save_every 5 +
+ +
+ Main model finetuned on Spring + + # Download the model + bash stereoflow/download_model.sh crocostereo_finetune_spring.pth + # Spring submission + python stereoflow/test.py --model stereoflow_models/crocostereo_finetune_spring.pth --dataset "Spring('test')" --save submission --tile_overlap 0.9 + # Training command that was used + python -u stereoflow/train.py stereo --criterion "LaplacianLossBounded2()" --dataset "Spring('train')" --lr 3e-5 --batch_size 6 --epochs 8 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --start_from stereoflow_models/crocostereo.pth --output_dir xps/crocostereo/finetune_spring/ +
+ +
+ Smaller models + To train CroCo-Stereo with smaller CroCo pretrained models, simply replace the --pretrained argument. To download the smaller CroCo-Stereo models based on CroCo v2 pretraining with ViT-Base encoder and Small encoder, use bash stereoflow/download_model.sh crocostereo_subtrain_vitb_smalldecoder.pth, and for the model with a ViT-Base encoder and a Base decoder, use bash stereoflow/download_model.sh crocostereo_subtrain_vitb_basedecoder.pth. +
+ + +### CroCo-Flow + +##### Main model + +The main training of CroCo-Flow was performed on the FlyingThings, FlyingChairs, MPI-Sintel and TartanAir datasets. +It was used for our submission to the MPI-Sintel benchmark. + +``` +# Download the model +bash stereoflow/download_model.sh crocoflow.pth +# Evaluation +python stereoflow/test.py --model stereoflow_models/crocoflow.pth --dataset "MPISintel('subval_cleanpass')+MPISintel('subval_finalpass')" --save metrics --tile_overlap 0.9 +# Sintel submission +python stereoflow/test.py --model stereoflow_models/crocoflow.pth --dataset "MPISintel('test_allpass')" --save submission --tile_overlap 0.9 +# Training command that was used, with checkpoint-best.pth +python -u stereoflow/train.py flow --criterion "LaplacianLossBounded()" --dataset "40*MPISintel('subtrain_cleanpass')+40*MPISintel('subtrain_finalpass')+4*FlyingThings('train_allpass')+4*FlyingChairs('train')+TartanAir('train')" --val_dataset "MPISintel('subval_cleanpass')+MPISintel('subval_finalpass')" --lr 2e-5 --batch_size 8 --epochs 240 --img_per_epoch 30000 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --output_dir xps/crocoflow/main/ +``` + +##### Other models + +
+ Main model finetuned on Kitti + + # Download the model + bash stereoflow/download_model.sh crocoflow_finetune_kitti.pth + # Kitti submission + python stereoflow/test.py --model stereoflow_models/crocoflow_finetune_kitti.pth --dataset "Kitti15('test')" --save submission --tile_overlap 0.99 + # Training that was used, with checkpoint-last.pth + python -u stereoflow/train.py flow --crop 352 1216 --criterion "LaplacianLossBounded()" --dataset "Kitti15('train')+Kitti12('train')" --lr 2e-5 --batch_size 1 --accum_iter 8 --epochs 150 --save_every 5 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --start_from stereoflow_models/crocoflow.pth --output_dir xps/crocoflow/finetune_kitti/ +
+ +
+ Main model finetuned on Spring + + # Download the model + bash stereoflow/download_model.sh crocoflow_finetune_spring.pth + # Spring submission + python stereoflow/test.py --model stereoflow_models/crocoflow_finetune_spring.pth --dataset "Spring('test')" --save submission --tile_overlap 0.9 + # Training command that was used, with checkpoint-last.pth + python -u stereoflow/train.py flow --criterion "LaplacianLossBounded()" --dataset "Spring('train')" --lr 2e-5 --batch_size 8 --epochs 12 --pretrained pretrained_models/CroCo_V2_ViTLarge_BaseDecoder.pth --start_from stereoflow_models/crocoflow.pth --output_dir xps/crocoflow/finetune_spring/ +
+ +
+ Smaller models + To train CroCo-Flow with smaller CroCo pretrained models, simply replace the --pretrained argument. To download the smaller CroCo-Flow models based on CroCo v2 pretraining with ViT-Base encoder and Small encoder, use bash stereoflow/download_model.sh crocoflow_vitb_smalldecoder.pth, and for the model with a ViT-Base encoder and a Base decoder, use bash stereoflow/download_model.sh crocoflow_vitb_basedecoder.pth. +
diff --git a/src/mast3r_src/dust3r/croco/stereoflow/augmentor.py b/src/mast3r_src/dust3r/croco/stereoflow/augmentor.py new file mode 100644 index 0000000000000000000000000000000000000000..69e6117151988d94cbc4b385e0d88e982133bf10 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/stereoflow/augmentor.py @@ -0,0 +1,290 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# Data augmentation for training stereo and flow +# -------------------------------------------------------- + +# References +# https://github.com/autonomousvision/unimatch/blob/master/dataloader/stereo/transforms.py +# https://github.com/autonomousvision/unimatch/blob/master/dataloader/flow/transforms.py + + +import numpy as np +import random +from PIL import Image + +import cv2 +cv2.setNumThreads(0) +cv2.ocl.setUseOpenCL(False) + +import torch +from torchvision.transforms import ColorJitter +import torchvision.transforms.functional as FF + +class StereoAugmentor(object): + + def __init__(self, crop_size, scale_prob=0.5, scale_xonly=True, lhth=800., lminscale=0.0, lmaxscale=1.0, hminscale=-0.2, hmaxscale=0.4, scale_interp_nearest=True, rightjitterprob=0.5, v_flip_prob=0.5, color_aug_asym=True, color_choice_prob=0.5): + self.crop_size = crop_size + self.scale_prob = scale_prob + self.scale_xonly = scale_xonly + self.lhth = lhth + self.lminscale = lminscale + self.lmaxscale = lmaxscale + self.hminscale = hminscale + self.hmaxscale = hmaxscale + self.scale_interp_nearest = scale_interp_nearest + self.rightjitterprob = rightjitterprob + self.v_flip_prob = v_flip_prob + self.color_aug_asym = color_aug_asym + self.color_choice_prob = color_choice_prob + + def _random_scale(self, img1, img2, disp): + ch,cw = self.crop_size + h,w = img1.shape[:2] + if self.scale_prob>0. and np.random.rand()1.: + scale_x = clip_scale + scale_y = scale_x if not self.scale_xonly else 1.0 + img1 = cv2.resize(img1, None, fx=scale_x, fy=scale_y, interpolation=cv2.INTER_LINEAR) + img2 = cv2.resize(img2, None, fx=scale_x, fy=scale_y, interpolation=cv2.INTER_LINEAR) + disp = cv2.resize(disp, None, fx=scale_x, fy=scale_y, interpolation=cv2.INTER_LINEAR if not self.scale_interp_nearest else cv2.INTER_NEAREST) * scale_x + return img1, img2, disp + + def _random_crop(self, img1, img2, disp): + h,w = img1.shape[:2] + ch,cw = self.crop_size + assert ch<=h and cw<=w, (img1.shape, h,w,ch,cw) + offset_x = np.random.randint(w - cw + 1) + offset_y = np.random.randint(h - ch + 1) + img1 = img1[offset_y:offset_y+ch,offset_x:offset_x+cw] + img2 = img2[offset_y:offset_y+ch,offset_x:offset_x+cw] + disp = disp[offset_y:offset_y+ch,offset_x:offset_x+cw] + return img1, img2, disp + + def _random_vflip(self, img1, img2, disp): + # vertical flip + if self.v_flip_prob>0 and np.random.rand() < self.v_flip_prob: + img1 = np.copy(np.flipud(img1)) + img2 = np.copy(np.flipud(img2)) + disp = np.copy(np.flipud(disp)) + return img1, img2, disp + + def _random_rotate_shift_right(self, img2): + if self.rightjitterprob>0. and np.random.rand() 0) & (xx < wd1) & (yy > 0) & (yy < ht1) + xx = xx[v] + yy = yy[v] + flow1 = flow1[v] + + flow = np.inf * np.ones([ht1, wd1, 2], dtype=np.float32) # invalid value every where, before we fill it with the correct ones + flow[yy, xx] = flow1 + return flow + + def spatial_transform(self, img1, img2, flow, dname): + + if np.random.rand() < self.spatial_aug_prob: + # randomly sample scale + ht, wd = img1.shape[:2] + clip_min_scale = np.maximum( + (self.crop_size[0] + 8) / float(ht), + (self.crop_size[1] + 8) / float(wd)) + min_scale, max_scale = self.min_scale, self.max_scale + scale = 2 ** np.random.uniform(self.min_scale, self.max_scale) + scale_x = scale + scale_y = scale + if np.random.rand() < self.stretch_prob: + scale_x *= 2 ** np.random.uniform(-self.max_stretch, self.max_stretch) + scale_y *= 2 ** np.random.uniform(-self.max_stretch, self.max_stretch) + scale_x = np.clip(scale_x, clip_min_scale, None) + scale_y = np.clip(scale_y, clip_min_scale, None) + # rescale the images + img1 = cv2.resize(img1, None, fx=scale_x, fy=scale_y, interpolation=cv2.INTER_LINEAR) + img2 = cv2.resize(img2, None, fx=scale_x, fy=scale_y, interpolation=cv2.INTER_LINEAR) + flow = self._resize_flow(flow, scale_x, scale_y, factor=2.0 if dname=='Spring' else 1.0) + elif dname=="Spring": + flow = self._resize_flow(flow, 1.0, 1.0, factor=2.0) + + if self.h_flip_prob>0. and np.random.rand() < self.h_flip_prob: # h-flip + img1 = img1[:, ::-1] + img2 = img2[:, ::-1] + flow = flow[:, ::-1] * [-1.0, 1.0] + + if self.v_flip_prob>0. and np.random.rand() < self.v_flip_prob: # v-flip + img1 = img1[::-1, :] + img2 = img2[::-1, :] + flow = flow[::-1, :] * [1.0, -1.0] + + # In case no cropping + if img1.shape[0] - self.crop_size[0] > 0: + y0 = np.random.randint(0, img1.shape[0] - self.crop_size[0]) + else: + y0 = 0 + if img1.shape[1] - self.crop_size[1] > 0: + x0 = np.random.randint(0, img1.shape[1] - self.crop_size[1]) + else: + x0 = 0 + + img1 = img1[y0:y0 + self.crop_size[0], x0:x0 + self.crop_size[1]] + img2 = img2[y0:y0 + self.crop_size[0], x0:x0 + self.crop_size[1]] + flow = flow[y0:y0 + self.crop_size[0], x0:x0 + self.crop_size[1]] + + return img1, img2, flow + + def __call__(self, img1, img2, flow, dname): + img1, img2, flow = self.spatial_transform(img1, img2, flow, dname) + img1, img2 = self.color_transform(img1, img2) + img1 = np.ascontiguousarray(img1) + img2 = np.ascontiguousarray(img2) + flow = np.ascontiguousarray(flow) + return img1, img2, flow \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/stereoflow/criterion.py b/src/mast3r_src/dust3r/croco/stereoflow/criterion.py new file mode 100644 index 0000000000000000000000000000000000000000..57792ebeeee34827b317a4d32b7445837bb33f17 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/stereoflow/criterion.py @@ -0,0 +1,251 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# Losses, metrics per batch, metrics per dataset +# -------------------------------------------------------- + +import torch +from torch import nn +import torch.nn.functional as F + +def _get_gtnorm(gt): + if gt.size(1)==1: # stereo + return gt + # flow + return torch.sqrt(torch.sum(gt**2, dim=1, keepdims=True)) # Bx1xHxW + +############ losses without confidence + +class L1Loss(nn.Module): + + def __init__(self, max_gtnorm=None): + super().__init__() + self.max_gtnorm = max_gtnorm + self.with_conf = False + + def _error(self, gt, predictions): + return torch.abs(gt-predictions) + + def forward(self, predictions, gt, inspect=False): + mask = torch.isfinite(gt) + if self.max_gtnorm is not None: + mask *= _get_gtnorm(gt).expand(-1,gt.size(1),-1,-1) which is a constant + + +class LaplacianLossBounded(nn.Module): # used for CroCo-Flow ; in the equation of the paper, we have a=1/b + def __init__(self, max_gtnorm=10000., a=0.25, b=4.): + super().__init__() + self.max_gtnorm = max_gtnorm + self.with_conf = True + self.a, self.b = a, b + + def forward(self, predictions, gt, conf): + mask = torch.isfinite(gt) + mask = mask[:,0,:,:] + if self.max_gtnorm is not None: mask *= _get_gtnorm(gt)[:,0,:,:] which is a constant + +class LaplacianLossBounded2(nn.Module): # used for CroCo-Stereo (except for ETH3D) ; in the equation of the paper, we have a=b + def __init__(self, max_gtnorm=None, a=3.0, b=3.0): + super().__init__() + self.max_gtnorm = max_gtnorm + self.with_conf = True + self.a, self.b = a, b + + def forward(self, predictions, gt, conf): + mask = torch.isfinite(gt) + mask = mask[:,0,:,:] + if self.max_gtnorm is not None: mask *= _get_gtnorm(gt)[:,0,:,:] which is a constant + +############## metrics per batch + +class StereoMetrics(nn.Module): + + def __init__(self, do_quantile=False): + super().__init__() + self.bad_ths = [0.5,1,2,3] + self.do_quantile = do_quantile + + def forward(self, predictions, gt): + B = predictions.size(0) + metrics = {} + gtcopy = gt.clone() + mask = torch.isfinite(gtcopy) + gtcopy[~mask] = 999999.0 # we make a copy and put a non-infinite value, such that it does not become nan once multiplied by the mask value 0 + Npx = mask.view(B,-1).sum(dim=1) + L1error = (torch.abs(gtcopy-predictions)*mask).view(B,-1) + L2error = (torch.square(gtcopy-predictions)*mask).view(B,-1) + # avgerr + metrics['avgerr'] = torch.mean(L1error.sum(dim=1)/Npx ) + # rmse + metrics['rmse'] = torch.sqrt(L2error.sum(dim=1)/Npx).mean(dim=0) + # err > t for t in [0.5,1,2,3] + for ths in self.bad_ths: + metrics['bad@{:.1f}'.format(ths)] = (((L1error>ths)* mask.view(B,-1)).sum(dim=1)/Npx).mean(dim=0) * 100 + return metrics + +class FlowMetrics(nn.Module): + def __init__(self): + super().__init__() + self.bad_ths = [1,3,5] + + def forward(self, predictions, gt): + B = predictions.size(0) + metrics = {} + mask = torch.isfinite(gt[:,0,:,:]) # both x and y would be infinite + Npx = mask.view(B,-1).sum(dim=1) + gtcopy = gt.clone() # to compute L1/L2 error, we need to have non-infinite value, the error computed at this locations will be ignored + gtcopy[:,0,:,:][~mask] = 999999.0 + gtcopy[:,1,:,:][~mask] = 999999.0 + L1error = (torch.abs(gtcopy-predictions).sum(dim=1)*mask).view(B,-1) + L2error = (torch.sqrt(torch.sum(torch.square(gtcopy-predictions),dim=1))*mask).view(B,-1) + metrics['L1err'] = torch.mean(L1error.sum(dim=1)/Npx ) + metrics['EPE'] = torch.mean(L2error.sum(dim=1)/Npx ) + for ths in self.bad_ths: + metrics['bad@{:.1f}'.format(ths)] = (((L2error>ths)* mask.view(B,-1)).sum(dim=1)/Npx).mean(dim=0) * 100 + return metrics + +############## metrics per dataset +## we update the average and maintain the number of pixels while adding data batch per batch +## at the beggining, call reset() +## after each batch, call add_batch(...) +## at the end: call get_results() + +class StereoDatasetMetrics(nn.Module): + + def __init__(self): + super().__init__() + self.bad_ths = [0.5,1,2,3] + + def reset(self): + self.agg_N = 0 # number of pixels so far + self.agg_L1err = torch.tensor(0.0) # L1 error so far + self.agg_Nbad = [0 for _ in self.bad_ths] # counter of bad pixels + self._metrics = None + + def add_batch(self, predictions, gt): + assert predictions.size(1)==1, predictions.size() + assert gt.size(1)==1, gt.size() + if gt.size(2)==predictions.size(2)*2 and gt.size(3)==predictions.size(3)*2: # special case for Spring ... + L1err = torch.minimum( torch.minimum( torch.minimum( + torch.sum(torch.abs(gt[:,:,0::2,0::2]-predictions),dim=1), + torch.sum(torch.abs(gt[:,:,1::2,0::2]-predictions),dim=1)), + torch.sum(torch.abs(gt[:,:,0::2,1::2]-predictions),dim=1)), + torch.sum(torch.abs(gt[:,:,1::2,1::2]-predictions),dim=1)) + valid = torch.isfinite(L1err) + else: + valid = torch.isfinite(gt[:,0,:,:]) # both x and y would be infinite + L1err = torch.sum(torch.abs(gt-predictions),dim=1) + N = valid.sum() + Nnew = self.agg_N + N + self.agg_L1err = float(self.agg_N)/Nnew * self.agg_L1err + L1err[valid].mean().cpu() * float(N)/Nnew + self.agg_N = Nnew + for i,th in enumerate(self.bad_ths): + self.agg_Nbad[i] += (L1err[valid]>th).sum().cpu() + + def _compute_metrics(self): + if self._metrics is not None: return + out = {} + out['L1err'] = self.agg_L1err.item() + for i,th in enumerate(self.bad_ths): + out['bad@{:.1f}'.format(th)] = (float(self.agg_Nbad[i]) / self.agg_N).item() * 100.0 + self._metrics = out + + def get_results(self): + self._compute_metrics() # to avoid recompute them multiple times + return self._metrics + +class FlowDatasetMetrics(nn.Module): + + def __init__(self): + super().__init__() + self.bad_ths = [0.5,1,3,5] + self.speed_ths = [(0,10),(10,40),(40,torch.inf)] + + def reset(self): + self.agg_N = 0 # number of pixels so far + self.agg_L1err = torch.tensor(0.0) # L1 error so far + self.agg_L2err = torch.tensor(0.0) # L2 (=EPE) error so far + self.agg_Nbad = [0 for _ in self.bad_ths] # counter of bad pixels + self.agg_EPEspeed = [torch.tensor(0.0) for _ in self.speed_ths] # EPE per speed bin so far + self.agg_Nspeed = [0 for _ in self.speed_ths] # N pixels per speed bin so far + self._metrics = None + self.pairname_results = {} + + def add_batch(self, predictions, gt): + assert predictions.size(1)==2, predictions.size() + assert gt.size(1)==2, gt.size() + if gt.size(2)==predictions.size(2)*2 and gt.size(3)==predictions.size(3)*2: # special case for Spring ... + L1err = torch.minimum( torch.minimum( torch.minimum( + torch.sum(torch.abs(gt[:,:,0::2,0::2]-predictions),dim=1), + torch.sum(torch.abs(gt[:,:,1::2,0::2]-predictions),dim=1)), + torch.sum(torch.abs(gt[:,:,0::2,1::2]-predictions),dim=1)), + torch.sum(torch.abs(gt[:,:,1::2,1::2]-predictions),dim=1)) + L2err = torch.minimum( torch.minimum( torch.minimum( + torch.sqrt(torch.sum(torch.square(gt[:,:,0::2,0::2]-predictions),dim=1)), + torch.sqrt(torch.sum(torch.square(gt[:,:,1::2,0::2]-predictions),dim=1))), + torch.sqrt(torch.sum(torch.square(gt[:,:,0::2,1::2]-predictions),dim=1))), + torch.sqrt(torch.sum(torch.square(gt[:,:,1::2,1::2]-predictions),dim=1))) + valid = torch.isfinite(L1err) + gtspeed = (torch.sqrt(torch.sum(torch.square(gt[:,:,0::2,0::2]),dim=1)) + torch.sqrt(torch.sum(torch.square(gt[:,:,0::2,1::2]),dim=1)) +\ + torch.sqrt(torch.sum(torch.square(gt[:,:,1::2,0::2]),dim=1)) + torch.sqrt(torch.sum(torch.square(gt[:,:,1::2,1::2]),dim=1)) ) / 4.0 # let's just average them + else: + valid = torch.isfinite(gt[:,0,:,:]) # both x and y would be infinite + L1err = torch.sum(torch.abs(gt-predictions),dim=1) + L2err = torch.sqrt(torch.sum(torch.square(gt-predictions),dim=1)) + gtspeed = torch.sqrt(torch.sum(torch.square(gt),dim=1)) + N = valid.sum() + Nnew = self.agg_N + N + self.agg_L1err = float(self.agg_N)/Nnew * self.agg_L1err + L1err[valid].mean().cpu() * float(N)/Nnew + self.agg_L2err = float(self.agg_N)/Nnew * self.agg_L2err + L2err[valid].mean().cpu() * float(N)/Nnew + self.agg_N = Nnew + for i,th in enumerate(self.bad_ths): + self.agg_Nbad[i] += (L2err[valid]>th).sum().cpu() + for i,(th1,th2) in enumerate(self.speed_ths): + vv = (gtspeed[valid]>=th1) * (gtspeed[valid] don't use batch_size>1 at test time) + self._prepare_data() + self._load_or_build_cache() + + def prepare_data(self): + """ + to be defined for each dataset + """ + raise NotImplementedError + + def __len__(self): + return len(self.pairnames) # each pairname is typically of the form (str, int1, int2) + + def __getitem__(self, index): + pairname = self.pairnames[index] + + # get filenames + img1name = self.pairname_to_img1name(pairname) + img2name = self.pairname_to_img2name(pairname) + flowname = self.pairname_to_flowname(pairname) if self.pairname_to_flowname is not None else None + + # load images and disparities + img1 = _read_img(img1name) + img2 = _read_img(img2name) + flow = self.load_flow(flowname) if flowname is not None else None + + # apply augmentations + if self.augmentor is not None: + img1, img2, flow = self.augmentor(img1, img2, flow, self.name) + + if self.totensor: + img1 = img_to_tensor(img1) + img2 = img_to_tensor(img2) + if flow is not None: + flow = flow_to_tensor(flow) + else: + flow = torch.tensor([]) # to allow dataloader batching with default collate_gn + pairname = str(pairname) # transform potential tuple to str to be able to batch it + + return img1, img2, flow, pairname + + def __rmul__(self, v): + self.rmul *= v + self.pairnames = v * self.pairnames + return self + + def __str__(self): + return f'{self.__class__.__name__}_{self.split}' + + def __repr__(self): + s = f'{self.__class__.__name__}(split={self.split}, augmentor={self.augmentor_str}, crop_size={str(self.crop_size)}, totensor={self.totensor})' + if self.rmul==1: + s+=f'\n\tnum pairs: {len(self.pairnames)}' + else: + s+=f'\n\tnum pairs: {len(self.pairnames)} ({len(self.pairnames)//self.rmul}x{self.rmul})' + return s + + def _set_root(self): + self.root = dataset_to_root[self.name] + assert os.path.isdir(self.root), f"could not find root directory for dataset {self.name}: {self.root}" + + def _load_or_build_cache(self): + cache_file = osp.join(cache_dir, self.name+'.pkl') + if osp.isfile(cache_file): + with open(cache_file, 'rb') as fid: + self.pairnames = pickle.load(fid)[self.split] + else: + tosave = self._build_cache() + os.makedirs(cache_dir, exist_ok=True) + with open(cache_file, 'wb') as fid: + pickle.dump(tosave, fid) + self.pairnames = tosave[self.split] + +class TartanAirDataset(FlowDataset): + + def _prepare_data(self): + self.name = "TartanAir" + self._set_root() + assert self.split in ['train'] + self.pairname_to_img1name = lambda pairname: osp.join(self.root, pairname[0], 'image_left/{:06d}_left.png'.format(pairname[1])) + self.pairname_to_img2name = lambda pairname: osp.join(self.root, pairname[0], 'image_left/{:06d}_left.png'.format(pairname[2])) + self.pairname_to_flowname = lambda pairname: osp.join(self.root, pairname[0], 'flow/{:06d}_{:06d}_flow.npy'.format(pairname[1],pairname[2])) + self.pairname_to_str = lambda pairname: os.path.join(pairname[0][pairname[0].find('/')+1:], '{:06d}_{:06d}'.format(pairname[1], pairname[2])) + self.load_flow = _read_numpy_flow + + def _build_cache(self): + seqs = sorted(os.listdir(self.root)) + pairs = [(osp.join(s,s,difficulty,Pxxx),int(a[:6]),int(a[:6])+1) for s in seqs for difficulty in ['Easy','Hard'] for Pxxx in sorted(os.listdir(osp.join(self.root,s,s,difficulty))) for a in sorted(os.listdir(osp.join(self.root,s,s,difficulty,Pxxx,'image_left/')))[:-1]] + assert len(pairs)==306268, "incorrect parsing of pairs in TartanAir" + tosave = {'train': pairs} + return tosave + +class FlyingChairsDataset(FlowDataset): + + def _prepare_data(self): + self.name = "FlyingChairs" + self._set_root() + assert self.split in ['train','val'] + self.pairname_to_img1name = lambda pairname: osp.join(self.root, 'data', pairname+'_img1.ppm') + self.pairname_to_img2name = lambda pairname: osp.join(self.root, 'data', pairname+'_img2.ppm') + self.pairname_to_flowname = lambda pairname: osp.join(self.root, 'data', pairname+'_flow.flo') + self.pairname_to_str = lambda pairname: pairname + self.load_flow = _read_flo_file + + def _build_cache(self): + split_file = osp.join(self.root, 'chairs_split.txt') + split_list = np.loadtxt(split_file, dtype=np.int32) + trainpairs = ['{:05d}'.format(i) for i in np.where(split_list==1)[0]+1] + valpairs = ['{:05d}'.format(i) for i in np.where(split_list==2)[0]+1] + assert len(trainpairs)==22232 and len(valpairs)==640, "incorrect parsing of pairs in MPI-Sintel" + tosave = {'train': trainpairs, 'val': valpairs} + return tosave + +class FlyingThingsDataset(FlowDataset): + + def _prepare_data(self): + self.name = "FlyingThings" + self._set_root() + assert self.split in [f'{set_}_{pass_}pass{camstr}' for set_ in ['train','test','test1024'] for camstr in ['','_rightcam'] for pass_ in ['clean','final','all']] + self.pairname_to_img1name = lambda pairname: osp.join(self.root, f'frames_{pairname[3]}pass', pairname[0].replace('into_future','').replace('into_past',''), '{:04d}.png'.format(pairname[1])) + self.pairname_to_img2name = lambda pairname: osp.join(self.root, f'frames_{pairname[3]}pass', pairname[0].replace('into_future','').replace('into_past',''), '{:04d}.png'.format(pairname[2])) + self.pairname_to_flowname = lambda pairname: osp.join(self.root, 'optical_flow', pairname[0], 'OpticalFlowInto{f:s}_{i:04d}_{c:s}.pfm'.format(f='Future' if 'future' in pairname[0] else 'Past', i=pairname[1], c='L' if 'left' in pairname[0] else 'R' )) + self.pairname_to_str = lambda pairname: os.path.join(pairname[3]+'pass', pairname[0], 'Into{f:s}_{i:04d}_{c:s}'.format(f='Future' if 'future' in pairname[0] else 'Past', i=pairname[1], c='L' if 'left' in pairname[0] else 'R' )) + self.load_flow = _read_pfm_flow + + def _build_cache(self): + tosave = {} + # train and test splits for the different passes + for set_ in ['train', 'test']: + sroot = osp.join(self.root, 'optical_flow', set_.upper()) + fname_to_i = lambda f: int(f[len('OpticalFlowIntoFuture_'):-len('_L.pfm')]) + pp = [(osp.join(set_.upper(), d, s, 'into_future/left'),fname_to_i(fname)) for d in sorted(os.listdir(sroot)) for s in sorted(os.listdir(osp.join(sroot,d))) for fname in sorted(os.listdir(osp.join(sroot,d, s, 'into_future/left')))[:-1]] + pairs = [(a,i,i+1) for a,i in pp] + pairs += [(a.replace('into_future','into_past'),i+1,i) for a,i in pp] + assert len(pairs)=={'train': 40302, 'test': 7866}[set_], "incorrect parsing of pairs Flying Things" + for cam in ['left','right']: + camstr = '' if cam=='left' else f'_{cam}cam' + for pass_ in ['final', 'clean']: + tosave[f'{set_}_{pass_}pass{camstr}'] = [(a.replace('left',cam),i,j,pass_) for a,i,j in pairs] + tosave[f'{set_}_allpass{camstr}'] = tosave[f'{set_}_cleanpass{camstr}'] + tosave[f'{set_}_finalpass{camstr}'] + # test1024: this is the same split as unimatch 'validation' split + # see https://github.com/autonomousvision/unimatch/blob/master/dataloader/flow/datasets.py#L229 + test1024_nsamples = 1024 + alltest_nsamples = len(tosave['test_cleanpass']) # 7866 + stride = alltest_nsamples // test1024_nsamples + remove = alltest_nsamples % test1024_nsamples + for cam in ['left','right']: + camstr = '' if cam=='left' else f'_{cam}cam' + for pass_ in ['final','clean']: + tosave[f'test1024_{pass_}pass{camstr}'] = sorted(tosave[f'test_{pass_}pass{camstr}'])[:-remove][::stride] # warning, it was not sorted before + assert len(tosave['test1024_cleanpass'])==1024, "incorrect parsing of pairs in Flying Things" + tosave[f'test1024_allpass{camstr}'] = tosave[f'test1024_cleanpass{camstr}'] + tosave[f'test1024_finalpass{camstr}'] + return tosave + + +class MPISintelDataset(FlowDataset): + + def _prepare_data(self): + self.name = "MPISintel" + self._set_root() + assert self.split in [s+'_'+p for s in ['train','test','subval','subtrain'] for p in ['cleanpass','finalpass','allpass']] + self.pairname_to_img1name = lambda pairname: osp.join(self.root, pairname[0], 'frame_{:04d}.png'.format(pairname[1])) + self.pairname_to_img2name = lambda pairname: osp.join(self.root, pairname[0], 'frame_{:04d}.png'.format(pairname[1]+1)) + self.pairname_to_flowname = lambda pairname: None if pairname[0].startswith('test/') else osp.join(self.root, pairname[0].replace('/clean/','/flow/').replace('/final/','/flow/'), 'frame_{:04d}.flo'.format(pairname[1])) + self.pairname_to_str = lambda pairname: osp.join(pairname[0], 'frame_{:04d}'.format(pairname[1])) + self.load_flow = _read_flo_file + + def _build_cache(self): + trainseqs = sorted(os.listdir(self.root+'training/clean')) + trainpairs = [ (osp.join('training/clean', s),i) for s in trainseqs for i in range(1, len(os.listdir(self.root+'training/clean/'+s)))] + subvalseqs = ['temple_2','temple_3'] + subtrainseqs = [s for s in trainseqs if s not in subvalseqs] + subvalpairs = [ (p,i) for p,i in trainpairs if any(s in p for s in subvalseqs)] + subtrainpairs = [ (p,i) for p,i in trainpairs if any(s in p for s in subtrainseqs)] + testseqs = sorted(os.listdir(self.root+'test/clean')) + testpairs = [ (osp.join('test/clean', s),i) for s in testseqs for i in range(1, len(os.listdir(self.root+'test/clean/'+s)))] + assert len(trainpairs)==1041 and len(testpairs)==552 and len(subvalpairs)==98 and len(subtrainpairs)==943, "incorrect parsing of pairs in MPI-Sintel" + tosave = {} + tosave['train_cleanpass'] = trainpairs + tosave['test_cleanpass'] = testpairs + tosave['subval_cleanpass'] = subvalpairs + tosave['subtrain_cleanpass'] = subtrainpairs + for t in ['train','test','subval','subtrain']: + tosave[t+'_finalpass'] = [(p.replace('/clean/','/final/'),i) for p,i in tosave[t+'_cleanpass']] + tosave[t+'_allpass'] = tosave[t+'_cleanpass'] + tosave[t+'_finalpass'] + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, _time): + assert prediction.shape[2]==2 + outfile = os.path.join(outdir, 'submission', self.pairname_to_str(pairname)+'.flo') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + writeFlowFile(prediction, outfile) + + def finalize_submission(self, outdir): + assert self.split == 'test_allpass' + bundle_exe = "/nfs/data/ffs-3d/datasets/StereoFlow/MPI-Sintel/bundler/linux-x64/bundler" # eg + if os.path.isfile(bundle_exe): + cmd = f'{bundle_exe} "{outdir}/submission/test/clean/" "{outdir}/submission/test/final" "{outdir}/submission/bundled.lzma"' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at: "{outdir}/submission/bundled.lzma"') + else: + print('Could not find bundler executable for submission.') + print('Please download it and run:') + print(f' "{outdir}/submission/test/clean/" "{outdir}/submission/test/final" "{outdir}/submission/bundled.lzma"') + +class SpringDataset(FlowDataset): + + def _prepare_data(self): + self.name = "Spring" + self._set_root() + assert self.split in ['train','test','subtrain','subval'] + self.pairname_to_img1name = lambda pairname: osp.join(self.root, pairname[0], pairname[1], 'frame_'+pairname[3], 'frame_{:s}_{:04d}.png'.format(pairname[3], pairname[4])) + self.pairname_to_img2name = lambda pairname: osp.join(self.root, pairname[0], pairname[1], 'frame_'+pairname[3], 'frame_{:s}_{:04d}.png'.format(pairname[3], pairname[4]+(1 if pairname[2]=='FW' else -1))) + self.pairname_to_flowname = lambda pairname: None if pairname[0]=='test' else osp.join(self.root, pairname[0], pairname[1], f'flow_{pairname[2]}_{pairname[3]}', f'flow_{pairname[2]}_{pairname[3]}_{pairname[4]:04d}.flo5') + self.pairname_to_str = lambda pairname: osp.join(pairname[0], pairname[1], f'flow_{pairname[2]}_{pairname[3]}', f'flow_{pairname[2]}_{pairname[3]}_{pairname[4]:04d}') + self.load_flow = _read_hdf5_flow + + def _build_cache(self): + # train + trainseqs = sorted(os.listdir( osp.join(self.root,'train'))) + trainpairs = [] + for leftright in ['left','right']: + for fwbw in ['FW','BW']: + trainpairs += [('train',s,fwbw,leftright,int(f[len(f'flow_{fwbw}_{leftright}_'):-len('.flo5')])) for s in trainseqs for f in sorted(os.listdir(osp.join(self.root,'train',s,f'flow_{fwbw}_{leftright}')))] + # test + testseqs = sorted(os.listdir( osp.join(self.root,'test'))) + testpairs = [] + for leftright in ['left','right']: + testpairs += [('test',s,'FW',leftright,int(f[len(f'frame_{leftright}_'):-len('.png')])) for s in testseqs for f in sorted(os.listdir(osp.join(self.root,'test',s,f'frame_{leftright}')))[:-1]] + testpairs += [('test',s,'BW',leftright,int(f[len(f'frame_{leftright}_'):-len('.png')])+1) for s in testseqs for f in sorted(os.listdir(osp.join(self.root,'test',s,f'frame_{leftright}')))[:-1]] + # subtrain / subval + subtrainpairs = [p for p in trainpairs if p[1]!='0041'] + subvalpairs = [p for p in trainpairs if p[1]=='0041'] + assert len(trainpairs)==19852 and len(testpairs)==3960 and len(subtrainpairs)==19472 and len(subvalpairs)==380, "incorrect parsing of pairs in Spring" + tosave = {'train': trainpairs, 'test': testpairs, 'subtrain': subtrainpairs, 'subval': subvalpairs} + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==3 + assert prediction.shape[2]==2 + assert prediction.dtype==np.float32 + outfile = osp.join(outdir, pairname[0], pairname[1], f'flow_{pairname[2]}_{pairname[3]}', f'flow_{pairname[2]}_{pairname[3]}_{pairname[4]:04d}.flo5') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + writeFlo5File(prediction, outfile) + + def finalize_submission(self, outdir): + assert self.split=='test' + exe = "{self.root}/flow_subsampling" + if os.path.isfile(exe): + cmd = f'cd "{outdir}/test"; {exe} .' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at {outdir}/test/flow_submission.hdf5') + else: + print('Could not find flow_subsampling executable for submission.') + print('Please download it and run:') + print(f'cd "{outdir}/test"; .') + + +class Kitti12Dataset(FlowDataset): + + def _prepare_data(self): + self.name = "Kitti12" + self._set_root() + assert self.split in ['train','test'] + self.pairname_to_img1name = lambda pairname: osp.join(self.root, pairname+'_10.png') + self.pairname_to_img2name = lambda pairname: osp.join(self.root, pairname+'_11.png') + self.pairname_to_flowname = None if self.split=='test' else lambda pairname: osp.join(self.root, pairname.replace('/colored_0/','/flow_occ/')+'_10.png') + self.pairname_to_str = lambda pairname: pairname.replace('/colored_0/','/') + self.load_flow = _read_kitti_flow + + def _build_cache(self): + trainseqs = ["training/colored_0/%06d"%(i) for i in range(194)] + testseqs = ["testing/colored_0/%06d"%(i) for i in range(195)] + assert len(trainseqs)==194 and len(testseqs)==195, "incorrect parsing of pairs in Kitti12" + tosave = {'train': trainseqs, 'test': testseqs} + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==3 + assert prediction.shape[2]==2 + outfile = os.path.join(outdir, pairname.split('/')[-1]+'_10.png') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + writeFlowKitti(outfile, prediction) + + def finalize_submission(self, outdir): + assert self.split=='test' + cmd = f'cd {outdir}/; zip -r "kitti12_flow_results.zip" .' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at {outdir}/kitti12_flow_results.zip') + + +class Kitti15Dataset(FlowDataset): + + def _prepare_data(self): + self.name = "Kitti15" + self._set_root() + assert self.split in ['train','subtrain','subval','test'] + self.pairname_to_img1name = lambda pairname: osp.join(self.root, pairname+'_10.png') + self.pairname_to_img2name = lambda pairname: osp.join(self.root, pairname+'_11.png') + self.pairname_to_flowname = None if self.split=='test' else lambda pairname: osp.join(self.root, pairname.replace('/image_2/','/flow_occ/')+'_10.png') + self.pairname_to_str = lambda pairname: pairname.replace('/image_2/','/') + self.load_flow = _read_kitti_flow + + def _build_cache(self): + trainseqs = ["training/image_2/%06d"%(i) for i in range(200)] + subtrainseqs = trainseqs[:-10] + subvalseqs = trainseqs[-10:] + testseqs = ["testing/image_2/%06d"%(i) for i in range(200)] + assert len(trainseqs)==200 and len(subtrainseqs)==190 and len(subvalseqs)==10 and len(testseqs)==200, "incorrect parsing of pairs in Kitti15" + tosave = {'train': trainseqs, 'subtrain': subtrainseqs, 'subval': subvalseqs, 'test': testseqs} + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==3 + assert prediction.shape[2]==2 + outfile = os.path.join(outdir, 'flow', pairname.split('/')[-1]+'_10.png') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + writeFlowKitti(outfile, prediction) + + def finalize_submission(self, outdir): + assert self.split=='test' + cmd = f'cd {outdir}/; zip -r "kitti15_flow_results.zip" flow' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at {outdir}/kitti15_flow_results.zip') + + +import cv2 +def _read_numpy_flow(filename): + return np.load(filename) + +def _read_pfm_flow(filename): + f, _ = _read_pfm(filename) + assert np.all(f[:,:,2]==0.0) + return np.ascontiguousarray(f[:,:,:2]) + +TAG_FLOAT = 202021.25 # tag to check the sanity of the file +TAG_STRING = 'PIEH' # string containing the tag +MIN_WIDTH = 1 +MAX_WIDTH = 99999 +MIN_HEIGHT = 1 +MAX_HEIGHT = 99999 +def readFlowFile(filename): + """ + readFlowFile() reads a flow file into a 2-band np.array. + if does not exist, an IOError is raised. + if does not finish by '.flo' or the tag, the width, the height or the file's size is illegal, an Expcetion is raised. + ---- PARAMETERS ---- + filename: string containg the name of the file to read a flow + ---- OUTPUTS ---- + a np.array of dimension (height x width x 2) containing the flow of type 'float32' + """ + + # check filename + if not filename.endswith(".flo"): + raise Exception("readFlowFile({:s}): filename must finish with '.flo'".format(filename)) + + # open the file and read it + with open(filename,'rb') as f: + # check tag + tag = struct.unpack('f',f.read(4))[0] + if tag != TAG_FLOAT: + raise Exception("flow_utils.readFlowFile({:s}): wrong tag".format(filename)) + # read dimension + w,h = struct.unpack('ii',f.read(8)) + if w < MIN_WIDTH or w > MAX_WIDTH: + raise Exception("flow_utils.readFlowFile({:s}: illegal width {:d}".format(filename,w)) + if h < MIN_HEIGHT or h > MAX_HEIGHT: + raise Exception("flow_utils.readFlowFile({:s}: illegal height {:d}".format(filename,h)) + flow = np.fromfile(f,'float32') + if not flow.shape == (h*w*2,): + raise Exception("flow_utils.readFlowFile({:s}: illegal size of the file".format(filename)) + flow.shape = (h,w,2) + return flow + +def writeFlowFile(flow,filename): + """ + writeFlowFile(flow,) write flow to the file . + if does not exist, an IOError is raised. + if does not finish with '.flo' or the flow has not 2 bands, an Exception is raised. + ---- PARAMETERS ---- + flow: np.array of dimension (height x width x 2) containing the flow to write + filename: string containg the name of the file to write a flow + """ + + # check filename + if not filename.endswith(".flo"): + raise Exception("flow_utils.writeFlowFile(,{:s}): filename must finish with '.flo'".format(filename)) + + if not flow.shape[2:] == (2,): + raise Exception("flow_utils.writeFlowFile(,{:s}): must have 2 bands".format(filename)) + + + # open the file and write it + with open(filename,'wb') as f: + # write TAG + f.write( TAG_STRING.encode('utf-8') ) + # write dimension + f.write( struct.pack('ii',flow.shape[1],flow.shape[0]) ) + # write the flow + + flow.astype(np.float32).tofile(f) + +_read_flo_file = readFlowFile + +def _read_kitti_flow(filename): + flow = cv2.imread(filename, cv2.IMREAD_ANYDEPTH | cv2.IMREAD_COLOR) + flow = flow[:, :, ::-1].astype(np.float32) + valid = flow[:, :, 2]>0 + flow = flow[:, :, :2] + flow = (flow - 2 ** 15) / 64.0 + flow[~valid,0] = np.inf + flow[~valid,1] = np.inf + return flow +_read_hd1k_flow = _read_kitti_flow + + +def writeFlowKitti(filename, uv): + uv = 64.0 * uv + 2 ** 15 + valid = np.ones([uv.shape[0], uv.shape[1], 1]) + uv = np.concatenate([uv, valid], axis=-1).astype(np.uint16) + cv2.imwrite(filename, uv[..., ::-1]) + +def writeFlo5File(flow, filename): + with h5py.File(filename, "w") as f: + f.create_dataset("flow", data=flow, compression="gzip", compression_opts=5) + +def _read_hdf5_flow(filename): + flow = np.asarray(h5py.File(filename)['flow']) + flow[np.isnan(flow)] = np.inf # make invalid values as +inf + return flow.astype(np.float32) + +# flow visualization +RY = 15 +YG = 6 +GC = 4 +CB = 11 +BM = 13 +MR = 6 +UNKNOWN_THRESH = 1e9 + +def colorTest(): + """ + flow_utils.colorTest(): display an example of image showing the color encoding scheme + """ + import matplotlib.pylab as plt + truerange = 1 + h,w = 151,151 + trange = truerange*1.04 + s2 = round(h/2) + x,y = np.meshgrid(range(w),range(h)) + u = x*trange/s2-trange + v = y*trange/s2-trange + img = _computeColor(np.concatenate((u[:,:,np.newaxis],v[:,:,np.newaxis]),2)/trange/np.sqrt(2)) + plt.imshow(img) + plt.axis('off') + plt.axhline(round(h/2),color='k') + plt.axvline(round(w/2),color='k') + +def flowToColor(flow, maxflow=None, maxmaxflow=None, saturate=False): + """ + flow_utils.flowToColor(flow): return a color code flow field, normalized based on the maximum l2-norm of the flow + flow_utils.flowToColor(flow,maxflow): return a color code flow field, normalized by maxflow + ---- PARAMETERS ---- + flow: flow to display of shape (height x width x 2) + maxflow (default:None): if given, normalize the flow by its value, otherwise by the flow norm + maxmaxflow (default:None): if given, normalize the flow by the max of its value and the flow norm + ---- OUTPUT ---- + an np.array of shape (height x width x 3) of type uint8 containing a color code of the flow + """ + h,w,n = flow.shape + # check size of flow + assert n == 2, "flow_utils.flowToColor(flow): flow must have 2 bands" + # fix unknown flow + unknown_idx = np.max(np.abs(flow),2)>UNKNOWN_THRESH + flow[unknown_idx] = 0.0 + # compute max flow if needed + if maxflow is None: + maxflow = flowMaxNorm(flow) + if maxmaxflow is not None: + maxflow = min(maxmaxflow, maxflow) + # normalize flow + eps = np.spacing(1) # minimum positive float value to avoid division by 0 + # compute the flow + img = _computeColor(flow/(maxflow+eps), saturate=saturate) + # put black pixels in unknown location + img[ np.tile( unknown_idx[:,:,np.newaxis],[1,1,3]) ] = 0.0 + return img + +def flowMaxNorm(flow): + """ + flow_utils.flowMaxNorm(flow): return the maximum of the l2-norm of the given flow + ---- PARAMETERS ---- + flow: the flow + + ---- OUTPUT ---- + a float containing the maximum of the l2-norm of the flow + """ + return np.max( np.sqrt( np.sum( np.square( flow ) , 2) ) ) + +def _computeColor(flow, saturate=True): + """ + flow_utils._computeColor(flow): compute color codes for the flow field flow + + ---- PARAMETERS ---- + flow: np.array of dimension (height x width x 2) containing the flow to display + ---- OUTPUTS ---- + an np.array of dimension (height x width x 3) containing the color conversion of the flow + """ + # set nan to 0 + nanidx = np.isnan(flow[:,:,0]) + flow[nanidx] = 0.0 + + # colorwheel + ncols = RY + YG + GC + CB + BM + MR + nchans = 3 + colorwheel = np.zeros((ncols,nchans),'uint8') + col = 0; + #RY + colorwheel[:RY,0] = 255 + colorwheel[:RY,1] = [(255*i) // RY for i in range(RY)] + col += RY + # YG + colorwheel[col:col+YG,0] = [255 - (255*i) // YG for i in range(YG)] + colorwheel[col:col+YG,1] = 255 + col += YG + # GC + colorwheel[col:col+GC,1] = 255 + colorwheel[col:col+GC,2] = [(255*i) // GC for i in range(GC)] + col += GC + # CB + colorwheel[col:col+CB,1] = [255 - (255*i) // CB for i in range(CB)] + colorwheel[col:col+CB,2] = 255 + col += CB + # BM + colorwheel[col:col+BM,0] = [(255*i) // BM for i in range(BM)] + colorwheel[col:col+BM,2] = 255 + col += BM + # MR + colorwheel[col:col+MR,0] = 255 + colorwheel[col:col+MR,2] = [255 - (255*i) // MR for i in range(MR)] + + # compute utility variables + rad = np.sqrt( np.sum( np.square(flow) , 2) ) # magnitude + a = np.arctan2( -flow[:,:,1] , -flow[:,:,0]) / np.pi # angle + fk = (a+1)/2 * (ncols-1) # map [-1,1] to [0,ncols-1] + k0 = np.floor(fk).astype('int') + k1 = k0+1 + k1[k1==ncols] = 0 + f = fk-k0 + + if not saturate: + rad = np.minimum(rad,1) + + # compute the image + img = np.zeros( (flow.shape[0],flow.shape[1],nchans), 'uint8' ) + for i in range(nchans): + tmp = colorwheel[:,i].astype('float') + col0 = tmp[k0]/255 + col1 = tmp[k1]/255 + col = (1-f)*col0 + f*col1 + idx = (rad <= 1) + col[idx] = 1-rad[idx]*(1-col[idx]) # increase saturation with radius + col[~idx] *= 0.75 # out of range + img[:,:,i] = (255*col*(1-nanidx.astype('float'))).astype('uint8') + + return img + +# flow dataset getter + +def get_train_dataset_flow(dataset_str, augmentor=True, crop_size=None): + dataset_str = dataset_str.replace('(','Dataset(') + if augmentor: + dataset_str = dataset_str.replace(')',', augmentor=True)') + if crop_size is not None: + dataset_str = dataset_str.replace(')',', crop_size={:s})'.format(str(crop_size))) + return eval(dataset_str) + +def get_test_datasets_flow(dataset_str): + dataset_str = dataset_str.replace('(','Dataset(') + return [eval(s) for s in dataset_str.split('+')] \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/stereoflow/datasets_stereo.py b/src/mast3r_src/dust3r/croco/stereoflow/datasets_stereo.py new file mode 100644 index 0000000000000000000000000000000000000000..dbdf841a6650afa71ae5782702902c79eba31a5c --- /dev/null +++ b/src/mast3r_src/dust3r/croco/stereoflow/datasets_stereo.py @@ -0,0 +1,674 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# Dataset structure for stereo +# -------------------------------------------------------- + +import sys, os +import os.path as osp +import pickle +import numpy as np +from PIL import Image +import json +import h5py +from glob import glob +import cv2 + +import torch +from torch.utils import data + +from .augmentor import StereoAugmentor + + + +dataset_to_root = { + 'CREStereo': './data/stereoflow//crenet_stereo_trainset/stereo_trainset/crestereo/', + 'SceneFlow': './data/stereoflow//SceneFlow/', + 'ETH3DLowRes': './data/stereoflow/eth3d_lowres/', + 'Booster': './data/stereoflow/booster_gt/', + 'Middlebury2021': './data/stereoflow/middlebury/2021/data/', + 'Middlebury2014': './data/stereoflow/middlebury/2014/', + 'Middlebury2006': './data/stereoflow/middlebury/2006/', + 'Middlebury2005': './data/stereoflow/middlebury/2005/train/', + 'MiddleburyEval3': './data/stereoflow/middlebury/MiddEval3/', + 'Spring': './data/stereoflow/spring/', + 'Kitti15': './data/stereoflow/kitti-stereo-2015/', + 'Kitti12': './data/stereoflow/kitti-stereo-2012/', +} +cache_dir = "./data/stereoflow/datasets_stereo_cache/" + + +in1k_mean = torch.tensor([0.485, 0.456, 0.406]).view(3,1,1) +in1k_std = torch.tensor([0.229, 0.224, 0.225]).view(3,1,1) +def img_to_tensor(img): + img = torch.from_numpy(img).permute(2, 0, 1).float() / 255. + img = (img-in1k_mean)/in1k_std + return img +def disp_to_tensor(disp): + return torch.from_numpy(disp)[None,:,:] + +class StereoDataset(data.Dataset): + + def __init__(self, split, augmentor=False, crop_size=None, totensor=True): + self.split = split + if not augmentor: assert crop_size is None + if crop_size: assert augmentor + self.crop_size = crop_size + self.augmentor_str = augmentor + self.augmentor = StereoAugmentor(crop_size) if augmentor else None + self.totensor = totensor + self.rmul = 1 # keep track of rmul + self.has_constant_resolution = True # whether the dataset has constant resolution or not (=> don't use batch_size>1 at test time) + self._prepare_data() + self._load_or_build_cache() + + def prepare_data(self): + """ + to be defined for each dataset + """ + raise NotImplementedError + + def __len__(self): + return len(self.pairnames) + + def __getitem__(self, index): + pairname = self.pairnames[index] + + # get filenames + Limgname = self.pairname_to_Limgname(pairname) + Rimgname = self.pairname_to_Rimgname(pairname) + Ldispname = self.pairname_to_Ldispname(pairname) if self.pairname_to_Ldispname is not None else None + + # load images and disparities + Limg = _read_img(Limgname) + Rimg = _read_img(Rimgname) + disp = self.load_disparity(Ldispname) if Ldispname is not None else None + + # sanity check + if disp is not None: assert np.all(disp>0) or self.name=="Spring", (self.name, pairname, Ldispname) + + # apply augmentations + if self.augmentor is not None: + Limg, Rimg, disp = self.augmentor(Limg, Rimg, disp, self.name) + + if self.totensor: + Limg = img_to_tensor(Limg) + Rimg = img_to_tensor(Rimg) + if disp is None: + disp = torch.tensor([]) # to allow dataloader batching with default collate_gn + else: + disp = disp_to_tensor(disp) + + return Limg, Rimg, disp, str(pairname) + + def __rmul__(self, v): + self.rmul *= v + self.pairnames = v * self.pairnames + return self + + def __str__(self): + return f'{self.__class__.__name__}_{self.split}' + + def __repr__(self): + s = f'{self.__class__.__name__}(split={self.split}, augmentor={self.augmentor_str}, crop_size={str(self.crop_size)}, totensor={self.totensor})' + if self.rmul==1: + s+=f'\n\tnum pairs: {len(self.pairnames)}' + else: + s+=f'\n\tnum pairs: {len(self.pairnames)} ({len(self.pairnames)//self.rmul}x{self.rmul})' + return s + + def _set_root(self): + self.root = dataset_to_root[self.name] + assert os.path.isdir(self.root), f"could not find root directory for dataset {self.name}: {self.root}" + + def _load_or_build_cache(self): + cache_file = osp.join(cache_dir, self.name+'.pkl') + if osp.isfile(cache_file): + with open(cache_file, 'rb') as fid: + self.pairnames = pickle.load(fid)[self.split] + else: + tosave = self._build_cache() + os.makedirs(cache_dir, exist_ok=True) + with open(cache_file, 'wb') as fid: + pickle.dump(tosave, fid) + self.pairnames = tosave[self.split] + +class CREStereoDataset(StereoDataset): + + def _prepare_data(self): + self.name = 'CREStereo' + self._set_root() + assert self.split in ['train'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname+'_left.jpg') + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname+'_right.jpg') + self.pairname_to_Ldispname = lambda pairname: osp.join(self.root, pairname+'_left.disp.png') + self.pairname_to_str = lambda pairname: pairname + self.load_disparity = _read_crestereo_disp + + + def _build_cache(self): + allpairs = [s+'/'+f[:-len('_left.jpg')] for s in sorted(os.listdir(self.root)) for f in sorted(os.listdir(self.root+'/'+s)) if f.endswith('_left.jpg')] + assert len(allpairs)==200000, "incorrect parsing of pairs in CreStereo" + tosave = {'train': allpairs} + return tosave + +class SceneFlowDataset(StereoDataset): + + def _prepare_data(self): + self.name = "SceneFlow" + self._set_root() + assert self.split in ['train_finalpass','train_cleanpass','train_allpass','test_finalpass','test_cleanpass','test_allpass','test1of100_cleanpass','test1of100_finalpass'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname) + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname).replace('/left/','/right/') + self.pairname_to_Ldispname = lambda pairname: osp.join(self.root, pairname).replace('/frames_finalpass/','/disparity/').replace('/frames_cleanpass/','/disparity/')[:-4]+'.pfm' + self.pairname_to_str = lambda pairname: pairname[:-4] + self.load_disparity = _read_sceneflow_disp + + def _build_cache(self): + trainpairs = [] + # driving + pairs = sorted(glob(self.root+'Driving/frames_finalpass/*/*/*/left/*.png')) + pairs = list(map(lambda x: x[len(self.root):], pairs)) + assert len(pairs) == 4400, "incorrect parsing of pairs in SceneFlow" + trainpairs += pairs + # monkaa + pairs = sorted(glob(self.root+'Monkaa/frames_finalpass/*/left/*.png')) + pairs = list(map(lambda x: x[len(self.root):], pairs)) + assert len(pairs) == 8664, "incorrect parsing of pairs in SceneFlow" + trainpairs += pairs + # flyingthings + pairs = sorted(glob(self.root+'FlyingThings/frames_finalpass/TRAIN/*/*/left/*.png')) + pairs = list(map(lambda x: x[len(self.root):], pairs)) + assert len(pairs) == 22390, "incorrect parsing of pairs in SceneFlow" + trainpairs += pairs + assert len(trainpairs) == 35454, "incorrect parsing of pairs in SceneFlow" + testpairs = sorted(glob(self.root+'FlyingThings/frames_finalpass/TEST/*/*/left/*.png')) + testpairs = list(map(lambda x: x[len(self.root):], testpairs)) + assert len(testpairs) == 4370, "incorrect parsing of pairs in SceneFlow" + test1of100pairs = testpairs[::100] + assert len(test1of100pairs) == 44, "incorrect parsing of pairs in SceneFlow" + # all + tosave = {'train_finalpass': trainpairs, + 'train_cleanpass': list(map(lambda x: x.replace('frames_finalpass','frames_cleanpass'), trainpairs)), + 'test_finalpass': testpairs, + 'test_cleanpass': list(map(lambda x: x.replace('frames_finalpass','frames_cleanpass'), testpairs)), + 'test1of100_finalpass': test1of100pairs, + 'test1of100_cleanpass': list(map(lambda x: x.replace('frames_finalpass','frames_cleanpass'), test1of100pairs)), + } + tosave['train_allpass'] = tosave['train_finalpass']+tosave['train_cleanpass'] + tosave['test_allpass'] = tosave['test_finalpass']+tosave['test_cleanpass'] + return tosave + +class Md21Dataset(StereoDataset): + + def _prepare_data(self): + self.name = "Middlebury2021" + self._set_root() + assert self.split in ['train','subtrain','subval'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname) + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname.replace('/im0','/im1')) + self.pairname_to_Ldispname = lambda pairname: osp.join(self.root, pairname.split('/')[0], 'disp0.pfm') + self.pairname_to_str = lambda pairname: pairname[:-4] + self.load_disparity = _read_middlebury_disp + + def _build_cache(self): + seqs = sorted(os.listdir(self.root)) + trainpairs = [] + for s in seqs: + #trainpairs += [s+'/im0.png'] # we should remove it, it is included as such in other lightings + trainpairs += [s+'/ambient/'+b+'/'+a for b in sorted(os.listdir(osp.join(self.root,s,'ambient'))) for a in sorted(os.listdir(osp.join(self.root,s,'ambient',b))) if a.startswith('im0')] + assert len(trainpairs)==355 + subtrainpairs = [p for p in trainpairs if any(p.startswith(s+'/') for s in seqs[:-2])] + subvalpairs = [p for p in trainpairs if any(p.startswith(s+'/') for s in seqs[-2:])] + assert len(subtrainpairs)==335 and len(subvalpairs)==20, "incorrect parsing of pairs in Middlebury 2021" + tosave = {'train': trainpairs, 'subtrain': subtrainpairs, 'subval': subvalpairs} + return tosave + +class Md14Dataset(StereoDataset): + + def _prepare_data(self): + self.name = "Middlebury2014" + self._set_root() + assert self.split in ['train','subtrain','subval'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, osp.dirname(pairname), 'im0.png') + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname) + self.pairname_to_Ldispname = lambda pairname: osp.join(self.root, osp.dirname(pairname), 'disp0.pfm') + self.pairname_to_str = lambda pairname: pairname[:-4] + self.load_disparity = _read_middlebury_disp + self.has_constant_resolution = False + + def _build_cache(self): + seqs = sorted(os.listdir(self.root)) + trainpairs = [] + for s in seqs: + trainpairs += [s+'/im1.png',s+'/im1E.png',s+'/im1L.png'] + assert len(trainpairs)==138 + valseqs = ['Umbrella-imperfect','Vintage-perfect'] + assert all(s in seqs for s in valseqs) + subtrainpairs = [p for p in trainpairs if not any(p.startswith(s+'/') for s in valseqs)] + subvalpairs = [p for p in trainpairs if any(p.startswith(s+'/') for s in valseqs)] + assert len(subtrainpairs)==132 and len(subvalpairs)==6, "incorrect parsing of pairs in Middlebury 2014" + tosave = {'train': trainpairs, 'subtrain': subtrainpairs, 'subval': subvalpairs} + return tosave + +class Md06Dataset(StereoDataset): + + def _prepare_data(self): + self.name = "Middlebury2006" + self._set_root() + assert self.split in ['train','subtrain','subval'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname) + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, osp.dirname(pairname), 'view5.png') + self.pairname_to_Ldispname = lambda pairname: osp.join(self.root, pairname.split('/')[0], 'disp1.png') + self.load_disparity = _read_middlebury20052006_disp + self.has_constant_resolution = False + + def _build_cache(self): + seqs = sorted(os.listdir(self.root)) + trainpairs = [] + for s in seqs: + for i in ['Illum1','Illum2','Illum3']: + for e in ['Exp0','Exp1','Exp2']: + trainpairs.append(osp.join(s,i,e,'view1.png')) + assert len(trainpairs)==189 + valseqs = ['Rocks1','Wood2'] + assert all(s in seqs for s in valseqs) + subtrainpairs = [p for p in trainpairs if not any(p.startswith(s+'/') for s in valseqs)] + subvalpairs = [p for p in trainpairs if any(p.startswith(s+'/') for s in valseqs)] + assert len(subtrainpairs)==171 and len(subvalpairs)==18, "incorrect parsing of pairs in Middlebury 2006" + tosave = {'train': trainpairs, 'subtrain': subtrainpairs, 'subval': subvalpairs} + return tosave + +class Md05Dataset(StereoDataset): + + def _prepare_data(self): + self.name = "Middlebury2005" + self._set_root() + assert self.split in ['train','subtrain','subval'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname) + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, osp.dirname(pairname), 'view5.png') + self.pairname_to_Ldispname = lambda pairname: osp.join(self.root, pairname.split('/')[0], 'disp1.png') + self.pairname_to_str = lambda pairname: pairname[:-4] + self.load_disparity = _read_middlebury20052006_disp + + def _build_cache(self): + seqs = sorted(os.listdir(self.root)) + trainpairs = [] + for s in seqs: + for i in ['Illum1','Illum2','Illum3']: + for e in ['Exp0','Exp1','Exp2']: + trainpairs.append(osp.join(s,i,e,'view1.png')) + assert len(trainpairs)==54, "incorrect parsing of pairs in Middlebury 2005" + valseqs = ['Reindeer'] + assert all(s in seqs for s in valseqs) + subtrainpairs = [p for p in trainpairs if not any(p.startswith(s+'/') for s in valseqs)] + subvalpairs = [p for p in trainpairs if any(p.startswith(s+'/') for s in valseqs)] + assert len(subtrainpairs)==45 and len(subvalpairs)==9, "incorrect parsing of pairs in Middlebury 2005" + tosave = {'train': trainpairs, 'subtrain': subtrainpairs, 'subval': subvalpairs} + return tosave + +class MdEval3Dataset(StereoDataset): + + def _prepare_data(self): + self.name = "MiddleburyEval3" + self._set_root() + assert self.split in [s+'_'+r for s in ['train','subtrain','subval','test','all'] for r in ['full','half','quarter']] + if self.split.endswith('_full'): + self.root = self.root.replace('/MiddEval3','/MiddEval3_F') + elif self.split.endswith('_half'): + self.root = self.root.replace('/MiddEval3','/MiddEval3_H') + else: + assert self.split.endswith('_quarter') + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname, 'im0.png') + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname, 'im1.png') + self.pairname_to_Ldispname = lambda pairname: None if pairname.startswith('test') else osp.join(self.root, pairname, 'disp0GT.pfm') + self.pairname_to_str = lambda pairname: pairname + self.load_disparity = _read_middlebury_disp + # for submission only + self.submission_methodname = "CroCo-Stereo" + self.submission_sresolution = 'F' if self.split.endswith('_full') else ('H' if self.split.endswith('_half') else 'Q') + + def _build_cache(self): + trainpairs = ['train/'+s for s in sorted(os.listdir(self.root+'train/'))] + testpairs = ['test/'+s for s in sorted(os.listdir(self.root+'test/'))] + subvalpairs = trainpairs[-1:] + subtrainpairs = trainpairs[:-1] + allpairs = trainpairs+testpairs + assert len(trainpairs)==15 and len(testpairs)==15 and len(subvalpairs)==1 and len(subtrainpairs)==14 and len(allpairs)==30, "incorrect parsing of pairs in Middlebury Eval v3" + tosave = {} + for r in ['full','half','quarter']: + tosave.update(**{'train_'+r: trainpairs, 'subtrain_'+r: subtrainpairs, 'subval_'+r: subvalpairs, 'test_'+r: testpairs, 'all_'+r: allpairs}) + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==2 + assert prediction.dtype==np.float32 + outfile = os.path.join(outdir, pairname.split('/')[0].replace('train','training')+self.submission_sresolution, pairname.split('/')[1], 'disp0'+self.submission_methodname+'.pfm') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + writePFM(outfile, prediction) + timefile = os.path.join( os.path.dirname(outfile), "time"+self.submission_methodname+'.txt') + with open(timefile, 'w') as fid: + fid.write(str(time)) + + def finalize_submission(self, outdir): + cmd = f'cd {outdir}/; zip -r "{self.submission_methodname}.zip" .' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at {outdir}/{self.submission_methodname}.zip') + +class ETH3DLowResDataset(StereoDataset): + + def _prepare_data(self): + self.name = "ETH3DLowRes" + self._set_root() + assert self.split in ['train','test','subtrain','subval','all'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname, 'im0.png') + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname, 'im1.png') + self.pairname_to_Ldispname = None if self.split=='test' else lambda pairname: None if pairname.startswith('test/') else osp.join(self.root, pairname.replace('train/','train_gt/'), 'disp0GT.pfm') + self.pairname_to_str = lambda pairname: pairname + self.load_disparity = _read_eth3d_disp + self.has_constant_resolution = False + + def _build_cache(self): + trainpairs = ['train/' + s for s in sorted(os.listdir(self.root+'train/'))] + testpairs = ['test/' + s for s in sorted(os.listdir(self.root+'test/'))] + assert len(trainpairs) == 27 and len(testpairs) == 20, "incorrect parsing of pairs in ETH3D Low Res" + subvalpairs = ['train/delivery_area_3s','train/electro_3l','train/playground_3l'] + assert all(p in trainpairs for p in subvalpairs) + subtrainpairs = [p for p in trainpairs if not p in subvalpairs] + assert len(subvalpairs)==3 and len(subtrainpairs)==24, "incorrect parsing of pairs in ETH3D Low Res" + tosave = {'train': trainpairs, 'test': testpairs, 'subtrain': subtrainpairs, 'subval': subvalpairs, 'all': trainpairs+testpairs} + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==2 + assert prediction.dtype==np.float32 + outfile = os.path.join(outdir, 'low_res_two_view', pairname.split('/')[1]+'.pfm') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + writePFM(outfile, prediction) + timefile = outfile[:-4]+'.txt' + with open(timefile, 'w') as fid: + fid.write('runtime '+str(time)) + + def finalize_submission(self, outdir): + cmd = f'cd {outdir}/; zip -r "eth3d_low_res_two_view_results.zip" low_res_two_view' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at {outdir}/eth3d_low_res_two_view_results.zip') + +class BoosterDataset(StereoDataset): + + def _prepare_data(self): + self.name = "Booster" + self._set_root() + assert self.split in ['train_balanced','test_balanced','subtrain_balanced','subval_balanced'] # we use only the balanced version + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname) + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname).replace('/camera_00/','/camera_02/') + self.pairname_to_Ldispname = lambda pairname: osp.join(self.root, osp.dirname(pairname), '../disp_00.npy') # same images with different colors, same gt per sequence + self.pairname_to_str = lambda pairname: pairname[:-4].replace('/camera_00/','/') + self.load_disparity = _read_booster_disp + + + def _build_cache(self): + trainseqs = sorted(os.listdir(self.root+'train/balanced')) + trainpairs = ['train/balanced/'+s+'/camera_00/'+imname for s in trainseqs for imname in sorted(os.listdir(self.root+'train/balanced/'+s+'/camera_00/'))] + testpairs = ['test/balanced/'+s+'/camera_00/'+imname for s in sorted(os.listdir(self.root+'test/balanced')) for imname in sorted(os.listdir(self.root+'test/balanced/'+s+'/camera_00/'))] + assert len(trainpairs) == 228 and len(testpairs) == 191 + subtrainpairs = [p for p in trainpairs if any(s in p for s in trainseqs[:-2])] + subvalpairs = [p for p in trainpairs if any(s in p for s in trainseqs[-2:])] + # warning: if we do validation split, we should split scenes!!! + tosave = {'train_balanced': trainpairs, 'test_balanced': testpairs, 'subtrain_balanced': subtrainpairs, 'subval_balanced': subvalpairs,} + return tosave + +class SpringDataset(StereoDataset): + + def _prepare_data(self): + self.name = "Spring" + self._set_root() + assert self.split in ['train', 'test', 'subtrain', 'subval'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname+'.png') + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname+'.png').replace('frame_right','').replace('frame_left','frame_right').replace('','frame_left') + self.pairname_to_Ldispname = lambda pairname: None if pairname.startswith('test') else osp.join(self.root, pairname+'.dsp5').replace('frame_left','disp1_left').replace('frame_right','disp1_right') + self.pairname_to_str = lambda pairname: pairname + self.load_disparity = _read_hdf5_disp + + def _build_cache(self): + trainseqs = sorted(os.listdir( osp.join(self.root,'train'))) + trainpairs = [osp.join('train',s,'frame_left',f[:-4]) for s in trainseqs for f in sorted(os.listdir(osp.join(self.root,'train',s,'frame_left')))] + testseqs = sorted(os.listdir( osp.join(self.root,'test'))) + testpairs = [osp.join('test',s,'frame_left',f[:-4]) for s in testseqs for f in sorted(os.listdir(osp.join(self.root,'test',s,'frame_left')))] + testpairs += [p.replace('frame_left','frame_right') for p in testpairs] + """maxnorm = {'0001': 32.88, '0002': 228.5, '0004': 298.2, '0005': 142.5, '0006': 113.6, '0007': 27.3, '0008': 554.5, '0009': 155.6, '0010': 126.1, '0011': 87.6, '0012': 303.2, '0013': 24.14, '0014': 82.56, '0015': 98.44, '0016': 156.9, '0017': 28.17, '0018': 21.03, '0020': 178.0, '0021': 58.06, '0022': 354.2, '0023': 8.79, '0024': 97.06, '0025': 55.16, '0026': 91.9, '0027': 156.6, '0030': 200.4, '0032': 58.66, '0033': 373.5, '0036': 149.4, '0037': 5.625, '0038': 37.0, '0039': 12.2, '0041': 453.5, '0043': 457.0, '0044': 379.5, '0045': 161.8, '0047': 105.44} # => let'use 0041""" + subtrainpairs = [p for p in trainpairs if p.split('/')[1]!='0041'] + subvalpairs = [p for p in trainpairs if p.split('/')[1]=='0041'] + assert len(trainpairs)==5000 and len(testpairs)==2000 and len(subtrainpairs)==4904 and len(subvalpairs)==96, "incorrect parsing of pairs in Spring" + tosave = {'train': trainpairs, 'test': testpairs, 'subtrain': subtrainpairs, 'subval': subvalpairs} + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==2 + assert prediction.dtype==np.float32 + outfile = os.path.join(outdir, pairname+'.dsp5').replace('frame_left','disp1_left').replace('frame_right','disp1_right') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + writeDsp5File(prediction, outfile) + + def finalize_submission(self, outdir): + assert self.split=='test' + exe = "{self.root}/disp1_subsampling" + if os.path.isfile(exe): + cmd = f'cd "{outdir}/test"; {exe} .' + print(cmd) + os.system(cmd) + else: + print('Could not find disp1_subsampling executable for submission.') + print('Please download it and run:') + print(f'cd "{outdir}/test"; .') + +class Kitti12Dataset(StereoDataset): + + def _prepare_data(self): + self.name = "Kitti12" + self._set_root() + assert self.split in ['train','test'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname+'_10.png') + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname.replace('/colored_0/','/colored_1/')+'_10.png') + self.pairname_to_Ldispname = None if self.split=='test' else lambda pairname: osp.join(self.root, pairname.replace('/colored_0/','/disp_occ/')+'_10.png') + self.pairname_to_str = lambda pairname: pairname.replace('/colored_0/','/') + self.load_disparity = _read_kitti_disp + + def _build_cache(self): + trainseqs = ["training/colored_0/%06d"%(i) for i in range(194)] + testseqs = ["testing/colored_0/%06d"%(i) for i in range(195)] + assert len(trainseqs)==194 and len(testseqs)==195, "incorrect parsing of pairs in Kitti12" + tosave = {'train': trainseqs, 'test': testseqs} + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==2 + assert prediction.dtype==np.float32 + outfile = os.path.join(outdir, pairname.split('/')[-1]+'_10.png') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + img = (prediction * 256).astype('uint16') + Image.fromarray(img).save(outfile) + + def finalize_submission(self, outdir): + assert self.split=='test' + cmd = f'cd {outdir}/; zip -r "kitti12_results.zip" .' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at {outdir}/kitti12_results.zip') + +class Kitti15Dataset(StereoDataset): + + def _prepare_data(self): + self.name = "Kitti15" + self._set_root() + assert self.split in ['train','subtrain','subval','test'] + self.pairname_to_Limgname = lambda pairname: osp.join(self.root, pairname+'_10.png') + self.pairname_to_Rimgname = lambda pairname: osp.join(self.root, pairname.replace('/image_2/','/image_3/')+'_10.png') + self.pairname_to_Ldispname = None if self.split=='test' else lambda pairname: osp.join(self.root, pairname.replace('/image_2/','/disp_occ_0/')+'_10.png') + self.pairname_to_str = lambda pairname: pairname.replace('/image_2/','/') + self.load_disparity = _read_kitti_disp + + def _build_cache(self): + trainseqs = ["training/image_2/%06d"%(i) for i in range(200)] + subtrainseqs = trainseqs[:-5] + subvalseqs = trainseqs[-5:] + testseqs = ["testing/image_2/%06d"%(i) for i in range(200)] + assert len(trainseqs)==200 and len(subtrainseqs)==195 and len(subvalseqs)==5 and len(testseqs)==200, "incorrect parsing of pairs in Kitti15" + tosave = {'train': trainseqs, 'subtrain': subtrainseqs, 'subval': subvalseqs, 'test': testseqs} + return tosave + + def submission_save_pairname(self, pairname, prediction, outdir, time): + assert prediction.ndim==2 + assert prediction.dtype==np.float32 + outfile = os.path.join(outdir, 'disp_0', pairname.split('/')[-1]+'_10.png') + os.makedirs( os.path.dirname(outfile), exist_ok=True) + img = (prediction * 256).astype('uint16') + Image.fromarray(img).save(outfile) + + def finalize_submission(self, outdir): + assert self.split=='test' + cmd = f'cd {outdir}/; zip -r "kitti15_results.zip" disp_0' + print(cmd) + os.system(cmd) + print(f'Done. Submission file at {outdir}/kitti15_results.zip') + + +### auxiliary functions + +def _read_img(filename): + # convert to RGB for scene flow finalpass data + img = np.asarray(Image.open(filename).convert('RGB')) + return img + +def _read_booster_disp(filename): + disp = np.load(filename) + disp[disp==0.0] = np.inf + return disp + +def _read_png_disp(filename, coef=1.0): + disp = np.asarray(Image.open(filename)) + disp = disp.astype(np.float32) / coef + disp[disp==0.0] = np.inf + return disp + +def _read_pfm_disp(filename): + disp = np.ascontiguousarray(_read_pfm(filename)[0]) + disp[disp<=0] = np.inf # eg /nfs/data/ffs-3d/datasets/middlebury/2014/Shopvac-imperfect/disp0.pfm + return disp + +def _read_npy_disp(filename): + return np.load(filename) + +def _read_crestereo_disp(filename): return _read_png_disp(filename, coef=32.0) +def _read_middlebury20052006_disp(filename): return _read_png_disp(filename, coef=1.0) +def _read_kitti_disp(filename): return _read_png_disp(filename, coef=256.0) +_read_sceneflow_disp = _read_pfm_disp +_read_eth3d_disp = _read_pfm_disp +_read_middlebury_disp = _read_pfm_disp +_read_carla_disp = _read_pfm_disp +_read_tartanair_disp = _read_npy_disp + +def _read_hdf5_disp(filename): + disp = np.asarray(h5py.File(filename)['disparity']) + disp[np.isnan(disp)] = np.inf # make invalid values as +inf + #disp[disp==0.0] = np.inf # make invalid values as +inf + return disp.astype(np.float32) + +import re +def _read_pfm(file): + file = open(file, 'rb') + + color = None + width = None + height = None + scale = None + endian = None + + header = file.readline().rstrip() + if header.decode("ascii") == 'PF': + color = True + elif header.decode("ascii") == 'Pf': + color = False + else: + raise Exception('Not a PFM file.') + + dim_match = re.match(r'^(\d+)\s(\d+)\s$', file.readline().decode("ascii")) + if dim_match: + width, height = list(map(int, dim_match.groups())) + else: + raise Exception('Malformed PFM header.') + + scale = float(file.readline().decode("ascii").rstrip()) + if scale < 0: # little-endian + endian = '<' + scale = -scale + else: + endian = '>' # big-endian + + data = np.fromfile(file, endian + 'f') + shape = (height, width, 3) if color else (height, width) + + data = np.reshape(data, shape) + data = np.flipud(data) + return data, scale + +def writePFM(file, image, scale=1): + file = open(file, 'wb') + + color = None + + if image.dtype.name != 'float32': + raise Exception('Image dtype must be float32.') + + image = np.flipud(image) + + if len(image.shape) == 3 and image.shape[2] == 3: # color image + color = True + elif len(image.shape) == 2 or len(image.shape) == 3 and image.shape[2] == 1: # greyscale + color = False + else: + raise Exception('Image must have H x W x 3, H x W x 1 or H x W dimensions.') + + file.write('PF\n' if color else 'Pf\n'.encode()) + file.write('%d %d\n'.encode() % (image.shape[1], image.shape[0])) + + endian = image.dtype.byteorder + + if endian == '<' or endian == '=' and sys.byteorder == 'little': + scale = -scale + + file.write('%f\n'.encode() % scale) + + image.tofile(file) + +def writeDsp5File(disp, filename): + with h5py.File(filename, "w") as f: + f.create_dataset("disparity", data=disp, compression="gzip", compression_opts=5) + + +# disp visualization + +def vis_disparity(disp, m=None, M=None): + if m is None: m = disp.min() + if M is None: M = disp.max() + disp_vis = (disp - m) / (M-m) * 255.0 + disp_vis = disp_vis.astype("uint8") + disp_vis = cv2.applyColorMap(disp_vis, cv2.COLORMAP_INFERNO) + return disp_vis + +# dataset getter + +def get_train_dataset_stereo(dataset_str, augmentor=True, crop_size=None): + dataset_str = dataset_str.replace('(','Dataset(') + if augmentor: + dataset_str = dataset_str.replace(')',', augmentor=True)') + if crop_size is not None: + dataset_str = dataset_str.replace(')',', crop_size={:s})'.format(str(crop_size))) + return eval(dataset_str) + +def get_test_datasets_stereo(dataset_str): + dataset_str = dataset_str.replace('(','Dataset(') + return [eval(s) for s in dataset_str.split('+')] \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/stereoflow/download_model.sh b/src/mast3r_src/dust3r/croco/stereoflow/download_model.sh new file mode 100644 index 0000000000000000000000000000000000000000..533119609108c5ec3c22ff79b10e9215c1ac5098 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/stereoflow/download_model.sh @@ -0,0 +1,12 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +model=$1 +outfile="stereoflow_models/${model}" +if [[ ! -f $outfile ]] +then + mkdir -p stereoflow_models/; + wget https://download.europe.naverlabs.com/ComputerVision/CroCo/StereoFlow_models/$1 -P stereoflow_models/; +else + echo "Model ${model} already downloaded in ${outfile}." +fi \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/stereoflow/engine.py b/src/mast3r_src/dust3r/croco/stereoflow/engine.py new file mode 100644 index 0000000000000000000000000000000000000000..c057346b99143bf6b9c4666a58215b2b91aca7a6 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/stereoflow/engine.py @@ -0,0 +1,280 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# Main function for training one epoch or testing +# -------------------------------------------------------- + +import math +import sys +from typing import Iterable +import numpy as np +import torch +import torchvision + +from utils import misc as misc + + +def split_prediction_conf(predictions, with_conf=False): + if not with_conf: + return predictions, None + conf = predictions[:,-1:,:,:] + predictions = predictions[:,:-1,:,:] + return predictions, conf + +def train_one_epoch(model: torch.nn.Module, criterion: torch.nn.Module, metrics: torch.nn.Module, + data_loader: Iterable, optimizer: torch.optim.Optimizer, + device: torch.device, epoch: int, loss_scaler, + log_writer=None, print_freq = 20, + args=None): + model.train(True) + metric_logger = misc.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', misc.SmoothedValue(window_size=1, fmt='{value:.6f}')) + header = 'Epoch: [{}]'.format(epoch) + + accum_iter = args.accum_iter + + optimizer.zero_grad() + + details = {} + + if log_writer is not None: + print('log_dir: {}'.format(log_writer.log_dir)) + + if args.img_per_epoch: + iter_per_epoch = args.img_per_epoch // args.batch_size + int(args.img_per_epoch % args.batch_size > 0) + assert len(data_loader) >= iter_per_epoch, 'Dataset is too small for so many iterations' + len_data_loader = iter_per_epoch + else: + len_data_loader, iter_per_epoch = len(data_loader), None + + for data_iter_step, (image1, image2, gt, pairname) in enumerate(metric_logger.log_every(data_loader, print_freq, header, max_iter=iter_per_epoch)): + + image1 = image1.to(device, non_blocking=True) + image2 = image2.to(device, non_blocking=True) + gt = gt.to(device, non_blocking=True) + + # we use a per iteration (instead of per epoch) lr scheduler + if data_iter_step % accum_iter == 0: + misc.adjust_learning_rate(optimizer, data_iter_step / len_data_loader + epoch, args) + + with torch.cuda.amp.autocast(enabled=bool(args.amp)): + prediction = model(image1, image2) + prediction, conf = split_prediction_conf(prediction, criterion.with_conf) + batch_metrics = metrics(prediction.detach(), gt) + loss = criterion(prediction, gt) if conf is None else criterion(prediction, gt, conf) + + loss_value = loss.item() + if not math.isfinite(loss_value): + print("Loss is {}, stopping training".format(loss_value)) + sys.exit(1) + + loss /= accum_iter + loss_scaler(loss, optimizer, parameters=model.parameters(), + update_grad=(data_iter_step + 1) % accum_iter == 0) + if (data_iter_step + 1) % accum_iter == 0: + optimizer.zero_grad() + + torch.cuda.synchronize() + + metric_logger.update(loss=loss_value) + for k,v in batch_metrics.items(): + metric_logger.update(**{k: v.item()}) + lr = optimizer.param_groups[0]["lr"] + metric_logger.update(lr=lr) + + #if args.dsitributed: loss_value_reduce = misc.all_reduce_mean(loss_value) + time_to_log = ((data_iter_step + 1) % (args.tboard_log_step * accum_iter) == 0 or data_iter_step == len_data_loader-1) + loss_value_reduce = misc.all_reduce_mean(loss_value) + if log_writer is not None and time_to_log: + epoch_1000x = int((data_iter_step / len_data_loader + epoch) * 1000) + # We use epoch_1000x as the x-axis in tensorboard. This calibrates different curves when batch size changes. + log_writer.add_scalar('train/loss', loss_value_reduce, epoch_1000x) + log_writer.add_scalar('lr', lr, epoch_1000x) + for k,v in batch_metrics.items(): + log_writer.add_scalar('train/'+k, v.item(), epoch_1000x) + + # gather the stats from all processes + #if args.distributed: metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger) + return {k: meter.global_avg for k, meter in metric_logger.meters.items()} + + +@torch.no_grad() +def validate_one_epoch(model: torch.nn.Module, + criterion: torch.nn.Module, + metrics: torch.nn.Module, + data_loaders: list[Iterable], + device: torch.device, + epoch: int, + log_writer=None, + args=None): + + model.eval() + metric_loggers = [] + header = 'Epoch: [{}]'.format(epoch) + print_freq = 20 + + conf_mode = args.tile_conf_mode + crop = args.crop + + if log_writer is not None: + print('log_dir: {}'.format(log_writer.log_dir)) + + results = {} + dnames = [] + image1, image2, gt, prediction = None, None, None, None + for didx, data_loader in enumerate(data_loaders): + dname = str(data_loader.dataset) + dnames.append(dname) + metric_loggers.append(misc.MetricLogger(delimiter=" ")) + for data_iter_step, (image1, image2, gt, pairname) in enumerate(metric_loggers[didx].log_every(data_loader, print_freq, header)): + image1 = image1.to(device, non_blocking=True) + image2 = image2.to(device, non_blocking=True) + gt = gt.to(device, non_blocking=True) + if dname.startswith('Spring'): + assert gt.size(2)==image1.size(2)*2 and gt.size(3)==image1.size(3)*2 + gt = (gt[:,:,0::2,0::2] + gt[:,:,0::2,1::2] + gt[:,:,1::2,0::2] + gt[:,:,1::2,1::2] ) / 4.0 # we approximate the gt based on the 2x upsampled ones + + with torch.inference_mode(): + prediction, tiled_loss, c = tiled_pred(model, criterion, image1, image2, gt, conf_mode=conf_mode, overlap=args.val_overlap, crop=crop, with_conf=criterion.with_conf) + batch_metrics = metrics(prediction.detach(), gt) + loss = criterion(prediction.detach(), gt) if not criterion.with_conf else criterion(prediction.detach(), gt, c) + loss_value = loss.item() + metric_loggers[didx].update(loss_tiled=tiled_loss.item()) + metric_loggers[didx].update(**{f'loss': loss_value}) + for k,v in batch_metrics.items(): + metric_loggers[didx].update(**{dname+'_' + k: v.item()}) + + results = {k: meter.global_avg for ml in metric_loggers for k, meter in ml.meters.items()} + if len(dnames)>1: + for k in batch_metrics.keys(): + results['AVG_'+k] = sum(results[dname+'_'+k] for dname in dnames) / len(dnames) + + if log_writer is not None : + epoch_1000x = int((1 + epoch) * 1000) + for k,v in results.items(): + log_writer.add_scalar('val/'+k, v, epoch_1000x) + + print("Averaged stats:", results) + return results + +import torch.nn.functional as F +def _resize_img(img, new_size): + return F.interpolate(img, size=new_size, mode='bicubic', align_corners=False) +def _resize_stereo_or_flow(data, new_size): + assert data.ndim==4 + assert data.size(1) in [1,2] + scale_x = new_size[1]/float(data.size(3)) + out = F.interpolate(data, size=new_size, mode='bicubic', align_corners=False) + out[:,0,:,:] *= scale_x + if out.size(1)==2: + scale_y = new_size[0]/float(data.size(2)) + out[:,1,:,:] *= scale_y + print(scale_x, new_size, data.shape) + return out + + +@torch.no_grad() +def tiled_pred(model, criterion, img1, img2, gt, + overlap=0.5, bad_crop_thr=0.05, + downscale=False, crop=512, ret='loss', + conf_mode='conf_expsigmoid_10_5', with_conf=False, + return_time=False): + + # for each image, we are going to run inference on many overlapping patches + # then, all predictions will be weighted-averaged + if gt is not None: + B, C, H, W = gt.shape + else: + B, _, H, W = img1.shape + C = model.head.num_channels-int(with_conf) + win_height, win_width = crop[0], crop[1] + + # upscale to be larger than the crop + do_change_scale = H= window and 0 <= overlap < 1, (total, window, overlap) + num_windows = 1 + int(np.ceil( (total - window) / ((1-overlap) * window) )) + offsets = np.linspace(0, total-window, num_windows).round().astype(int) + yield from (slice(x, x+window) for x in offsets) + +def _crop(img, sy, sx): + B, THREE, H, W = img.shape + if 0 <= sy.start and sy.stop <= H and 0 <= sx.start and sx.stop <= W: + return img[:,:,sy,sx] + l, r = max(0,-sx.start), max(0,sx.stop-W) + t, b = max(0,-sy.start), max(0,sy.stop-H) + img = torch.nn.functional.pad(img, (l,r,t,b), mode='constant') + return img[:, :, slice(sy.start+t,sy.stop+t), slice(sx.start+l,sx.stop+l)] \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/stereoflow/test.py b/src/mast3r_src/dust3r/croco/stereoflow/test.py new file mode 100644 index 0000000000000000000000000000000000000000..0248e56664c769752595af251e1eadcfa3a479d9 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/stereoflow/test.py @@ -0,0 +1,216 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# Main test function +# -------------------------------------------------------- + +import os +import argparse +import pickle +from PIL import Image +import numpy as np +from tqdm import tqdm + +import torch +from torch.utils.data import DataLoader + +import utils.misc as misc +from models.croco_downstream import CroCoDownstreamBinocular +from models.head_downstream import PixelwiseTaskWithDPT + +from stereoflow.criterion import * +from stereoflow.datasets_stereo import get_test_datasets_stereo +from stereoflow.datasets_flow import get_test_datasets_flow +from stereoflow.engine import tiled_pred + +from stereoflow.datasets_stereo import vis_disparity +from stereoflow.datasets_flow import flowToColor + +def get_args_parser(): + parser = argparse.ArgumentParser('Test CroCo models on stereo/flow', add_help=False) + # important argument + parser.add_argument('--model', required=True, type=str, help='Path to the model to evaluate') + parser.add_argument('--dataset', required=True, type=str, help="test dataset (there can be multiple dataset separated by a +)") + # tiling + parser.add_argument('--tile_conf_mode', type=str, default='', help='Weights for the tiling aggregation based on confidence (empty means use the formula from the loaded checkpoint') + parser.add_argument('--tile_overlap', type=float, default=0.7, help='overlap between tiles') + # save (it will automatically go to _/_) + parser.add_argument('--save', type=str, nargs='+', default=[], + help='what to save: \ + metrics (pickle file), \ + pred (raw prediction save as torch tensor), \ + visu (visualization in png of each prediction), \ + err10 (visualization in png of the error clamp at 10 for each prediction), \ + submission (submission file)') + # other (no impact) + parser.add_argument('--num_workers', default=4, type=int) + return parser + + +def _load_model_and_criterion(model_path, do_load_metrics, device): + print('loading model from', model_path) + assert os.path.isfile(model_path) + ckpt = torch.load(model_path, 'cpu') + + ckpt_args = ckpt['args'] + task = ckpt_args.task + tile_conf_mode = ckpt_args.tile_conf_mode + num_channels = {'stereo': 1, 'flow': 2}[task] + with_conf = eval(ckpt_args.criterion).with_conf + if with_conf: num_channels += 1 + print('head: PixelwiseTaskWithDPT()') + head = PixelwiseTaskWithDPT() + head.num_channels = num_channels + print('croco_args:', ckpt_args.croco_args) + model = CroCoDownstreamBinocular(head, **ckpt_args.croco_args) + msg = model.load_state_dict(ckpt['model'], strict=True) + model.eval() + model = model.to(device) + + if do_load_metrics: + if task=='stereo': + metrics = StereoDatasetMetrics().to(device) + else: + metrics = FlowDatasetMetrics().to(device) + else: + metrics = None + + return model, metrics, ckpt_args.crop, with_conf, task, tile_conf_mode + + +def _save_batch(pred, gt, pairnames, dataset, task, save, outdir, time, submission_dir=None): + + for i in range(len(pairnames)): + + pairname = eval(pairnames[i]) if pairnames[i].startswith('(') else pairnames[i] # unbatch pairname + fname = os.path.join(outdir, dataset.pairname_to_str(pairname)) + os.makedirs(os.path.dirname(fname), exist_ok=True) + + predi = pred[i,...] + if gt is not None: gti = gt[i,...] + + if 'pred' in save: + torch.save(predi.squeeze(0).cpu(), fname+'_pred.pth') + + if 'visu' in save: + if task=='stereo': + disparity = predi.permute((1,2,0)).squeeze(2).cpu().numpy() + m,M = None + if gt is not None: + mask = torch.isfinite(gti) + m = gt[mask].min() + M = gt[mask].max() + img_disparity = vis_disparity(disparity, m=m, M=M) + Image.fromarray(img_disparity).save(fname+'_pred.png') + else: + # normalize flowToColor according to the maxnorm of gt (or prediction if not available) + flowNorm = torch.sqrt(torch.sum( (gti if gt is not None else predi)**2, dim=0)).max().item() + imgflow = flowToColor(predi.permute((1,2,0)).cpu().numpy(), maxflow=flowNorm) + Image.fromarray(imgflow).save(fname+'_pred.png') + + if 'err10' in save: + assert gt is not None + L2err = torch.sqrt(torch.sum( (gti-predi)**2, dim=0)) + valid = torch.isfinite(gti[0,:,:]) + L2err[~valid] = 0.0 + L2err = torch.clamp(L2err, max=10.0) + red = (L2err*255.0/10.0).to(dtype=torch.uint8)[:,:,None] + zer = torch.zeros_like(red) + imgerr = torch.cat( (red,zer,zer), dim=2).cpu().numpy() + Image.fromarray(imgerr).save(fname+'_err10.png') + + if 'submission' in save: + assert submission_dir is not None + predi_np = predi.permute(1,2,0).squeeze(2).cpu().numpy() # transform into HxWx2 for flow or HxW for stereo + dataset.submission_save_pairname(pairname, predi_np, submission_dir, time) + +def main(args): + + # load the pretrained model and metrics + device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu') + model, metrics, cropsize, with_conf, task, tile_conf_mode = _load_model_and_criterion(args.model, 'metrics' in args.save, device) + if args.tile_conf_mode=='': args.tile_conf_mode = tile_conf_mode + + # load the datasets + datasets = (get_test_datasets_stereo if task=='stereo' else get_test_datasets_flow)(args.dataset) + dataloaders = [DataLoader(dataset, batch_size=1, shuffle=False, num_workers=args.num_workers, pin_memory=True, drop_last=False) for dataset in datasets] + + # run + for i,dataloader in enumerate(dataloaders): + dataset = datasets[i] + dstr = args.dataset.split('+')[i] + + outdir = args.model+'_'+misc.filename(dstr) + if 'metrics' in args.save and len(args.save)==1: + fname = os.path.join(outdir, f'conf_{args.tile_conf_mode}_overlap_{args.tile_overlap}.pkl') + if os.path.isfile(fname) and len(args.save)==1: + print(' metrics already compute in '+fname) + with open(fname, 'rb') as fid: + results = pickle.load(fid) + for k,v in results.items(): + print('{:s}: {:.3f}'.format(k, v)) + continue + + if 'submission' in args.save: + dirname = f'submission_conf_{args.tile_conf_mode}_overlap_{args.tile_overlap}' + submission_dir = os.path.join(outdir, dirname) + else: + submission_dir = None + + print('') + print('saving {:s} in {:s}'.format('+'.join(args.save), outdir)) + print(repr(dataset)) + + if metrics is not None: + metrics.reset() + + for data_iter_step, (image1, image2, gt, pairnames) in enumerate(tqdm(dataloader)): + + do_flip = (task=='stereo' and dstr.startswith('Spring') and any("right" in p for p in pairnames)) # we flip the images and will flip the prediction after as we assume img1 is on the left + + image1 = image1.to(device, non_blocking=True) + image2 = image2.to(device, non_blocking=True) + gt = gt.to(device, non_blocking=True) if gt.numel()>0 else None # special case for test time + if do_flip: + assert all("right" in p for p in pairnames) + image1 = image1.flip(dims=[3]) # this is already the right frame, let's flip it + image2 = image2.flip(dims=[3]) + gt = gt # that is ok + + with torch.inference_mode(): + pred, _, _, time = tiled_pred(model, None, image1, image2, None if dataset.name=='Spring' else gt, conf_mode=args.tile_conf_mode, overlap=args.tile_overlap, crop=cropsize, with_conf=with_conf, return_time=True) + + if do_flip: + pred = pred.flip(dims=[3]) + + if metrics is not None: + metrics.add_batch(pred, gt) + + if any(k in args.save for k in ['pred','visu','err10','submission']): + _save_batch(pred, gt, pairnames, dataset, task, args.save, outdir, time, submission_dir=submission_dir) + + + # print + if metrics is not None: + results = metrics.get_results() + for k,v in results.items(): + print('{:s}: {:.3f}'.format(k, v)) + + # save if needed + if 'metrics' in args.save: + os.makedirs(os.path.dirname(fname), exist_ok=True) + with open(fname, 'wb') as fid: + pickle.dump(results, fid) + print('metrics saved in', fname) + + # finalize submission if needed + if 'submission' in args.save: + dataset.finalize_submission(submission_dir) + + + +if __name__ == '__main__': + args = get_args_parser() + args = args.parse_args() + main(args) \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/stereoflow/train.py b/src/mast3r_src/dust3r/croco/stereoflow/train.py new file mode 100644 index 0000000000000000000000000000000000000000..91f2414ffbe5ecd547d31c0e2455478d402719d6 --- /dev/null +++ b/src/mast3r_src/dust3r/croco/stereoflow/train.py @@ -0,0 +1,253 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +# -------------------------------------------------------- +# Main training function +# -------------------------------------------------------- + +import argparse +import datetime +import json +import numpy as np +import os +import sys +import time + +import torch +import torch.distributed as dist +import torch.backends.cudnn as cudnn +from torch.utils.tensorboard import SummaryWriter +import torchvision.transforms as transforms +import torchvision.datasets as datasets +from torch.utils.data import DataLoader + +import utils +import utils.misc as misc +from utils.misc import NativeScalerWithGradNormCount as NativeScaler +from models.croco_downstream import CroCoDownstreamBinocular, croco_args_from_ckpt +from models.pos_embed import interpolate_pos_embed +from models.head_downstream import PixelwiseTaskWithDPT + +from stereoflow.datasets_stereo import get_train_dataset_stereo, get_test_datasets_stereo +from stereoflow.datasets_flow import get_train_dataset_flow, get_test_datasets_flow +from stereoflow.engine import train_one_epoch, validate_one_epoch +from stereoflow.criterion import * + + +def get_args_parser(): + # prepare subparsers + parser = argparse.ArgumentParser('Finetuning CroCo models on stereo or flow', add_help=False) + subparsers = parser.add_subparsers(title="Task (stereo or flow)", dest="task", required=True) + parser_stereo = subparsers.add_parser('stereo', help='Training stereo model') + parser_flow = subparsers.add_parser('flow', help='Training flow model') + def add_arg(name_or_flags, default=None, default_stereo=None, default_flow=None, **kwargs): + if default is not None: assert default_stereo is None and default_flow is None, "setting default makes default_stereo and default_flow disabled" + parser_stereo.add_argument(name_or_flags, default=default if default is not None else default_stereo, **kwargs) + parser_flow.add_argument(name_or_flags, default=default if default is not None else default_flow, **kwargs) + # output dir + add_arg('--output_dir', required=True, type=str, help='path where to save, if empty, automatically created') + # model + add_arg('--crop', type=int, nargs = '+', default_stereo=[352, 704], default_flow=[320, 384], help = "size of the random image crops used during training.") + add_arg('--pretrained', required=True, type=str, help="Load pretrained model (required as croco arguments come from there)") + # criterion + add_arg('--criterion', default_stereo='LaplacianLossBounded2()', default_flow='LaplacianLossBounded()', type=str, help='string to evaluate to get criterion') + add_arg('--bestmetric', default_stereo='avgerr', default_flow='EPE', type=str) + # dataset + add_arg('--dataset', type=str, required=True, help="training set") + # training + add_arg('--seed', default=0, type=int, help='seed') + add_arg('--batch_size', default_stereo=6, default_flow=8, type=int, help='Batch size per GPU (effective batch size is batch_size * accum_iter * # gpus') + add_arg('--epochs', default=32, type=int, help='number of training epochs') + add_arg('--img_per_epoch', type=int, default=None, help='Fix the number of images seen in an epoch (None means use all training pairs)') + add_arg('--accum_iter', default=1, type=int, help='Accumulate gradient iterations (for increasing the effective batch size under memory constraints)') + add_arg('--weight_decay', type=float, default=0.05, help='weight decay (default: 0.05)') + add_arg('--lr', type=float, default_stereo=3e-5, default_flow=2e-5, metavar='LR', help='learning rate (absolute lr)') + add_arg('--min_lr', type=float, default=0., metavar='LR', help='lower lr bound for cyclic schedulers that hit 0') + add_arg('--warmup_epochs', type=int, default=1, metavar='N', help='epochs to warmup LR') + add_arg('--optimizer', default='AdamW(param_groups, lr=args.lr, betas=(0.9, 0.95))', type=str, + help="Optimizer from torch.optim [ default: AdamW(param_groups, lr=args.lr, betas=(0.9, 0.95)) ]") + add_arg('--amp', default=0, type=int, choices=[0,1], help='enable automatic mixed precision training') + # validation + add_arg('--val_dataset', type=str, default='', help="Validation sets, multiple separated by + (empty string means that no validation is performed)") + add_arg('--tile_conf_mode', type=str, default_stereo='conf_expsigmoid_15_3', default_flow='conf_expsigmoid_10_5', help='Weights for tile aggregation') + add_arg('--val_overlap', default=0.7, type=float, help='Overlap value for the tiling') + # others + add_arg('--num_workers', default=8, type=int) + add_arg('--eval_every', type=int, default=1, help='Val loss evaluation frequency') + add_arg('--save_every', type=int, default=1, help='Save checkpoint frequency') + add_arg('--start_from', type=str, default=None, help='Start training using weights from an other model (eg for finetuning)') + add_arg('--tboard_log_step', type=int, default=100, help='Log to tboard every so many steps') + add_arg('--dist_url', default='env://', help='url used to set up distributed training') + + return parser + + +def main(args): + misc.init_distributed_mode(args) + global_rank = misc.get_rank() + num_tasks = misc.get_world_size() + + assert os.path.isfile(args.pretrained) + print("output_dir: "+args.output_dir) + os.makedirs(args.output_dir, exist_ok=True) + + # fix the seed for reproducibility + seed = args.seed + misc.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + cudnn.benchmark = True + + # Metrics / criterion + device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') + metrics = (StereoMetrics if args.task=='stereo' else FlowMetrics)().to(device) + criterion = eval(args.criterion).to(device) + print('Criterion: ', args.criterion) + + # Prepare model + assert os.path.isfile(args.pretrained) + ckpt = torch.load(args.pretrained, 'cpu') + croco_args = croco_args_from_ckpt(ckpt) + croco_args['img_size'] = (args.crop[0], args.crop[1]) + print('Croco args: '+str(croco_args)) + args.croco_args = croco_args # saved for test time + # prepare head + num_channels = {'stereo': 1, 'flow': 2}[args.task] + if criterion.with_conf: num_channels += 1 + print(f'Building head PixelwiseTaskWithDPT() with {num_channels} channel(s)') + head = PixelwiseTaskWithDPT() + head.num_channels = num_channels + # build model and load pretrained weights + model = CroCoDownstreamBinocular(head, **croco_args) + interpolate_pos_embed(model, ckpt['model']) + msg = model.load_state_dict(ckpt['model'], strict=False) + print(msg) + + total_params = sum(p.numel() for p in model.parameters()) + total_params_trainable = sum(p.numel() for p in model.parameters() if p.requires_grad) + print(f"Total params: {total_params}") + print(f"Total params trainable: {total_params_trainable}") + model_without_ddp = model.to(device) + + eff_batch_size = args.batch_size * args.accum_iter * misc.get_world_size() + print("lr: %.2e" % args.lr) + print("accumulate grad iterations: %d" % args.accum_iter) + print("effective batch size: %d" % eff_batch_size) + + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], static_graph=True) + model_without_ddp = model.module + + # following timm: set wd as 0 for bias and norm layers + param_groups = misc.get_parameter_groups(model_without_ddp, args.weight_decay) + optimizer = eval(f"torch.optim.{args.optimizer}") + print(optimizer) + loss_scaler = NativeScaler() + + # automatic restart + last_ckpt_fname = os.path.join(args.output_dir, f'checkpoint-last.pth') + args.resume = last_ckpt_fname if os.path.isfile(last_ckpt_fname) else None + + if not args.resume and args.start_from: + print(f"Starting from an other model's weights: {args.start_from}") + best_so_far = None + args.start_epoch = 0 + ckpt = torch.load(args.start_from, 'cpu') + msg = model_without_ddp.load_state_dict(ckpt['model'], strict=False) + print(msg) + else: + best_so_far = misc.load_model(args=args, model_without_ddp=model_without_ddp, optimizer=optimizer, loss_scaler=loss_scaler) + + if best_so_far is None: best_so_far = np.inf + + # tensorboard + log_writer = None + if global_rank == 0 and args.output_dir is not None: + log_writer = SummaryWriter(log_dir=args.output_dir, purge_step=args.start_epoch*1000) + + # dataset and loader + print('Building Train Data loader for dataset: ', args.dataset) + train_dataset = (get_train_dataset_stereo if args.task=='stereo' else get_train_dataset_flow)(args.dataset, crop_size=args.crop) + def _print_repr_dataset(d): + if isinstance(d, torch.utils.data.dataset.ConcatDataset): + for dd in d.datasets: + _print_repr_dataset(dd) + else: + print(repr(d)) + _print_repr_dataset(train_dataset) + print(' total length:', len(train_dataset)) + if args.distributed: + sampler_train = torch.utils.data.DistributedSampler( + train_dataset, num_replicas=num_tasks, rank=global_rank, shuffle=True + ) + else: + sampler_train = torch.utils.data.RandomSampler(train_dataset) + data_loader_train = torch.utils.data.DataLoader( + train_dataset, sampler=sampler_train, + batch_size=args.batch_size, + num_workers=args.num_workers, + pin_memory=True, + drop_last=True, + ) + if args.val_dataset=='': + data_loaders_val = None + else: + print('Building Val Data loader for datasets: ', args.val_dataset) + val_datasets = (get_test_datasets_stereo if args.task=='stereo' else get_test_datasets_flow)(args.val_dataset) + for val_dataset in val_datasets: print(repr(val_dataset)) + data_loaders_val = [DataLoader(val_dataset, batch_size=1, shuffle=False, num_workers=args.num_workers, pin_memory=True, drop_last=False) for val_dataset in val_datasets] + bestmetric = ("AVG_" if len(data_loaders_val)>1 else str(data_loaders_val[0].dataset)+'_')+args.bestmetric + + print(f"Start training for {args.epochs} epochs") + start_time = time.time() + # Training Loop + for epoch in range(args.start_epoch, args.epochs): + + if args.distributed: data_loader_train.sampler.set_epoch(epoch) + + # Train + epoch_start = time.time() + train_stats = train_one_epoch(model, criterion, metrics, data_loader_train, optimizer, device, epoch, loss_scaler, log_writer=log_writer, args=args) + epoch_time = time.time() - epoch_start + + if args.distributed: dist.barrier() + + # Validation (current naive implementation runs the validation on every gpu ... not smart ...) + if data_loaders_val is not None and args.eval_every > 0 and (epoch+1) % args.eval_every == 0: + val_epoch_start = time.time() + val_stats = validate_one_epoch(model, criterion, metrics, data_loaders_val, device, epoch, log_writer=log_writer, args=args) + val_epoch_time = time.time() - val_epoch_start + + val_best = val_stats[bestmetric] + + # Save best of all + if val_best <= best_so_far: + best_so_far = val_best + misc.save_model(args=args, model_without_ddp=model_without_ddp, optimizer=optimizer, loss_scaler=loss_scaler, epoch=epoch, best_so_far=best_so_far, fname='best') + + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + 'epoch': epoch, + **{f'val_{k}': v for k, v in val_stats.items()}} + else: + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + 'epoch': epoch,} + + if args.distributed: dist.barrier() + + # Save stuff + if args.output_dir and ((epoch+1) % args.save_every == 0 or epoch + 1 == args.epochs): + misc.save_model(args=args, model_without_ddp=model_without_ddp, optimizer=optimizer, loss_scaler=loss_scaler, epoch=epoch, best_so_far=best_so_far, fname='last') + + if args.output_dir: + if log_writer is not None: + log_writer.flush() + with open(os.path.join(args.output_dir, "log.txt"), mode="a", encoding="utf-8") as f: + f.write(json.dumps(log_stats) + "\n") + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + +if __name__ == '__main__': + args = get_args_parser() + args = args.parse_args() + main(args) \ No newline at end of file diff --git a/src/mast3r_src/dust3r/croco/utils/misc.py b/src/mast3r_src/dust3r/croco/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..132e102a662c987dce5282633cb8730b0e0d5c2d --- /dev/null +++ b/src/mast3r_src/dust3r/croco/utils/misc.py @@ -0,0 +1,463 @@ +# Copyright (C) 2022-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilitary functions for CroCo +# -------------------------------------------------------- +# References: +# MAE: https://github.com/facebookresearch/mae +# DeiT: https://github.com/facebookresearch/deit +# BEiT: https://github.com/microsoft/unilm/tree/master/beit +# -------------------------------------------------------- + +import builtins +import datetime +import os +import time +import math +import json +from collections import defaultdict, deque +from pathlib import Path +import numpy as np + +import torch +import torch.distributed as dist +from torch import inf + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float64, device='cuda') + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + return self.total / self.count + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, + avg=self.avg, + global_avg=self.global_avg, + max=self.max, + value=self.value) + + +class MetricLogger(object): + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if v is None: + continue + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format( + type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + loss_str.append( + "{}: {}".format(name, str(meter)) + ) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, print_freq, header=None, max_iter=None): + i = 0 + if not header: + header = '' + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt='{avg:.4f}') + data_time = SmoothedValue(fmt='{avg:.4f}') + len_iterable = min(len(iterable), max_iter) if max_iter else len(iterable) + space_fmt = ':' + str(len(str(len_iterable))) + 'd' + log_msg = [ + header, + '[{0' + space_fmt + '}/{1}]', + 'eta: {eta}', + '{meters}', + 'time: {time}', + 'data: {data}' + ] + if torch.cuda.is_available(): + log_msg.append('max mem: {memory:.0f}') + log_msg = self.delimiter.join(log_msg) + MB = 1024.0 * 1024.0 + for it,obj in enumerate(iterable): + data_time.update(time.time() - end) + yield obj + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == len_iterable - 1: + eta_seconds = iter_time.global_avg * (len_iterable - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + if torch.cuda.is_available(): + print(log_msg.format( + i, len_iterable, eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB)) + else: + print(log_msg.format( + i, len_iterable, eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time))) + i += 1 + end = time.time() + if max_iter and it >= max_iter: + break + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('{} Total time: {} ({:.4f} s / it)'.format( + header, total_time_str, total_time / len_iterable)) + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + builtin_print = builtins.print + + def print(*args, **kwargs): + force = kwargs.pop('force', False) + force = force or (get_world_size() > 8) + if is_master or force: + now = datetime.datetime.now().time() + builtin_print('[{}] '.format(now), end='') # print with time stamp + builtin_print(*args, **kwargs) + + builtins.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + nodist = args.nodist if hasattr(args,'nodist') else False + if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ and not nodist: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ['WORLD_SIZE']) + args.gpu = int(os.environ['LOCAL_RANK']) + else: + print('Not using distributed mode') + setup_for_distributed(is_master=True) # hack + args.distributed = False + return + + args.distributed = True + + torch.cuda.set_device(args.gpu) + args.dist_backend = 'nccl' + print('| distributed init (rank {}): {}, gpu {}'.format( + args.rank, args.dist_url, args.gpu), flush=True) + torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + torch.distributed.barrier() + setup_for_distributed(args.rank == 0) + + +class NativeScalerWithGradNormCount: + state_dict_key = "amp_scaler" + + def __init__(self, enabled=True): + self._scaler = torch.cuda.amp.GradScaler(enabled=enabled) + + def __call__(self, loss, optimizer, clip_grad=None, parameters=None, create_graph=False, update_grad=True): + self._scaler.scale(loss).backward(create_graph=create_graph) + if update_grad: + if clip_grad is not None: + assert parameters is not None + self._scaler.unscale_(optimizer) # unscale the gradients of optimizer's assigned params in-place + norm = torch.nn.utils.clip_grad_norm_(parameters, clip_grad) + else: + self._scaler.unscale_(optimizer) + norm = get_grad_norm_(parameters) + self._scaler.step(optimizer) + self._scaler.update() + else: + norm = None + return norm + + def state_dict(self): + return self._scaler.state_dict() + + def load_state_dict(self, state_dict): + self._scaler.load_state_dict(state_dict) + + +def get_grad_norm_(parameters, norm_type: float = 2.0) -> torch.Tensor: + if isinstance(parameters, torch.Tensor): + parameters = [parameters] + parameters = [p for p in parameters if p.grad is not None] + norm_type = float(norm_type) + if len(parameters) == 0: + return torch.tensor(0.) + device = parameters[0].grad.device + if norm_type == inf: + total_norm = max(p.grad.detach().abs().max().to(device) for p in parameters) + else: + total_norm = torch.norm(torch.stack([torch.norm(p.grad.detach(), norm_type).to(device) for p in parameters]), norm_type) + return total_norm + + + + +def save_model(args, epoch, model_without_ddp, optimizer, loss_scaler, fname=None, best_so_far=None): + output_dir = Path(args.output_dir) + if fname is None: fname = str(epoch) + checkpoint_path = output_dir / ('checkpoint-%s.pth' % fname) + to_save = { + 'model': model_without_ddp.state_dict(), + 'optimizer': optimizer.state_dict(), + 'scaler': loss_scaler.state_dict(), + 'args': args, + 'epoch': epoch, + } + if best_so_far is not None: to_save['best_so_far'] = best_so_far + print(f'>> Saving model to {checkpoint_path} ...') + save_on_master(to_save, checkpoint_path) + + +def load_model(args, model_without_ddp, optimizer, loss_scaler): + args.start_epoch = 0 + best_so_far = None + if args.resume is not None: + if args.resume.startswith('https'): + checkpoint = torch.hub.load_state_dict_from_url( + args.resume, map_location='cpu', check_hash=True) + else: + checkpoint = torch.load(args.resume, map_location='cpu') + print("Resume checkpoint %s" % args.resume) + model_without_ddp.load_state_dict(checkpoint['model'], strict=False) + args.start_epoch = checkpoint['epoch'] + 1 + optimizer.load_state_dict(checkpoint['optimizer']) + if 'scaler' in checkpoint: + loss_scaler.load_state_dict(checkpoint['scaler']) + if 'best_so_far' in checkpoint: + best_so_far = checkpoint['best_so_far'] + print(" & best_so_far={:g}".format(best_so_far)) + else: + print("") + print("With optim & sched! start_epoch={:d}".format(args.start_epoch), end='') + return best_so_far + +def all_reduce_mean(x): + world_size = get_world_size() + if world_size > 1: + x_reduce = torch.tensor(x).cuda() + dist.all_reduce(x_reduce) + x_reduce /= world_size + return x_reduce.item() + else: + return x + +def _replace(text, src, tgt, rm=''): + """ Advanced string replacement. + Given a text: + - replace all elements in src by the corresponding element in tgt + - remove all elements in rm + """ + if len(tgt) == 1: + tgt = tgt * len(src) + assert len(src) == len(tgt), f"'{src}' and '{tgt}' should have the same len" + for s,t in zip(src, tgt): + text = text.replace(s,t) + for c in rm: + text = text.replace(c,'') + return text + +def filename( obj ): + """ transform a python obj or cmd into a proper filename. + - \1 gets replaced by slash '/' + - \2 gets replaced by comma ',' + """ + if not isinstance(obj, str): + obj = repr(obj) + obj = str(obj).replace('()','') + obj = _replace(obj, '_,(*/\1\2','-__x%/,', rm=' )\'"') + assert all(len(s) < 256 for s in obj.split(os.sep)), 'filename too long (>256 characters):\n'+obj + return obj + +def _get_num_layer_for_vit(var_name, enc_depth, dec_depth): + if var_name in ("cls_token", "mask_token", "pos_embed", "global_tokens"): + return 0 + elif var_name.startswith("patch_embed"): + return 0 + elif var_name.startswith("enc_blocks"): + layer_id = int(var_name.split('.')[1]) + return layer_id + 1 + elif var_name.startswith('decoder_embed') or var_name.startswith('enc_norm'): # part of the last black + return enc_depth + elif var_name.startswith('dec_blocks'): + layer_id = int(var_name.split('.')[1]) + return enc_depth + layer_id + 1 + elif var_name.startswith('dec_norm'): # part of the last block + return enc_depth + dec_depth + elif any(var_name.startswith(k) for k in ['head','prediction_head']): + return enc_depth + dec_depth + 1 + else: + raise NotImplementedError(var_name) + +def get_parameter_groups(model, weight_decay, layer_decay=1.0, skip_list=(), no_lr_scale_list=[]): + parameter_group_names = {} + parameter_group_vars = {} + enc_depth, dec_depth = None, None + # prepare layer decay values + assert layer_decay==1.0 or 0.> wrote {fpath}') + + print(f'Loaded {len(list_subscenes)} sub-scenes') + + # separate scenes + list_scenes = defaultdict(list) + for scene in list_subscenes: + scene, id = os.path.split(scene) + list_scenes[scene].append(id) + + list_scenes = list(list_scenes.items()) + print(f'from {len(list_scenes)} scenes in total') + + np.random.shuffle(list_scenes) + train_scenes = list_scenes[len(list_scenes)//10:] + val_scenes = list_scenes[:len(list_scenes)//10] + + def write_scene_list(scenes, n, fpath): + sub_scenes = [os.path.join(scene, id) for scene, ids in scenes for id in ids] + np.random.shuffle(sub_scenes) + + if len(sub_scenes) < n: + return + + with open(fpath, 'w') as f: + f.write('\n'.join(sub_scenes[:n])) + print(f'>> wrote {fpath}') + + for n in n_scenes: + write_scene_list(train_scenes, n, os.path.join(habitat_root, f'Habitat_{n}_scenes_train.txt')) + write_scene_list(val_scenes, n//10, os.path.join(habitat_root, f'Habitat_{n//10}_scenes_val.txt')) + + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--root", required=True) + parser.add_argument("--n_scenes", nargs='+', default=[1_000, 1_000_000], type=int) + + args = parser.parse_args() + find_all_scenes(args.root, args.n_scenes) diff --git a/src/mast3r_src/dust3r/datasets_preprocess/habitat/habitat_renderer/__init__.py b/src/mast3r_src/dust3r/datasets_preprocess/habitat/habitat_renderer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a32692113d830ddc4af4e6ed608f222fbe062e6e --- /dev/null +++ b/src/mast3r_src/dust3r/datasets_preprocess/habitat/habitat_renderer/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). diff --git a/src/mast3r_src/dust3r/datasets_preprocess/habitat/habitat_renderer/habitat_sim_envmaps_renderer.py b/src/mast3r_src/dust3r/datasets_preprocess/habitat/habitat_renderer/habitat_sim_envmaps_renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..4a31f1174a234b900ecaa76705fa271baf8a5669 --- /dev/null +++ b/src/mast3r_src/dust3r/datasets_preprocess/habitat/habitat_renderer/habitat_sim_envmaps_renderer.py @@ -0,0 +1,170 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Render environment maps from 3D meshes using the Habitat Sim simulator. +# -------------------------------------------------------- +import numpy as np +import habitat_sim +import math +from habitat_renderer import projections + +# OpenCV to habitat camera convention transformation +R_OPENCV2HABITAT = np.stack((habitat_sim.geo.RIGHT, -habitat_sim.geo.UP, habitat_sim.geo.FRONT), axis=0) + +CUBEMAP_FACE_LABELS = ["left", "front", "right", "back", "up", "down"] +# Expressed while considering Habitat coordinates systems +CUBEMAP_FACE_ORIENTATIONS_ROTVEC = [ + [0, math.pi / 2, 0], # Left + [0, 0, 0], # Front + [0, - math.pi / 2, 0], # Right + [0, math.pi, 0], # Back + [math.pi / 2, 0, 0], # Up + [-math.pi / 2, 0, 0],] # Down + +class NoNaviguableSpaceError(RuntimeError): + def __init__(self, *args): + super().__init__(*args) + +class HabitatEnvironmentMapRenderer: + def __init__(self, + scene, + navmesh, + scene_dataset_config_file, + render_equirectangular=False, + equirectangular_resolution=(512, 1024), + render_cubemap=False, + cubemap_resolution=(512, 512), + render_depth=False, + gpu_id=0): + self.scene = scene + self.navmesh = navmesh + self.scene_dataset_config_file = scene_dataset_config_file + self.gpu_id = gpu_id + + self.render_equirectangular = render_equirectangular + self.equirectangular_resolution = equirectangular_resolution + self.equirectangular_projection = projections.EquirectangularProjection(*equirectangular_resolution) + # 3D unit ray associated to each pixel of the equirectangular map + equirectangular_rays = projections.get_projection_rays(self.equirectangular_projection) + # Not needed, but just in case. + equirectangular_rays /= np.linalg.norm(equirectangular_rays, axis=-1, keepdims=True) + # Depth map created by Habitat are produced by warping a cubemap, + # so the values do not correspond to distance to the center and need some scaling. + self.equirectangular_depth_scale_factors = 1.0 / np.max(np.abs(equirectangular_rays), axis=-1) + + self.render_cubemap = render_cubemap + self.cubemap_resolution = cubemap_resolution + + self.render_depth = render_depth + + self.seed = None + self._lazy_initialization() + + def _lazy_initialization(self): + # Lazy random seeding and instantiation of the simulator to deal with multiprocessing properly + if self.seed == None: + # Re-seed numpy generator + np.random.seed() + self.seed = np.random.randint(2**32-1) + sim_cfg = habitat_sim.SimulatorConfiguration() + sim_cfg.scene_id = self.scene + if self.scene_dataset_config_file is not None and self.scene_dataset_config_file != "": + sim_cfg.scene_dataset_config_file = self.scene_dataset_config_file + sim_cfg.random_seed = self.seed + sim_cfg.load_semantic_mesh = False + sim_cfg.gpu_device_id = self.gpu_id + + sensor_specifications = [] + + # Add cubemaps + if self.render_cubemap: + for face_id, orientation in enumerate(CUBEMAP_FACE_ORIENTATIONS_ROTVEC): + rgb_sensor_spec = habitat_sim.CameraSensorSpec() + rgb_sensor_spec.uuid = f"color_cubemap_{CUBEMAP_FACE_LABELS[face_id]}" + rgb_sensor_spec.sensor_type = habitat_sim.SensorType.COLOR + rgb_sensor_spec.resolution = self.cubemap_resolution + rgb_sensor_spec.hfov = 90 + rgb_sensor_spec.position = [0.0, 0.0, 0.0] + rgb_sensor_spec.orientation = orientation + sensor_specifications.append(rgb_sensor_spec) + + if self.render_depth: + depth_sensor_spec = habitat_sim.CameraSensorSpec() + depth_sensor_spec.uuid = f"depth_cubemap_{CUBEMAP_FACE_LABELS[face_id]}" + depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH + depth_sensor_spec.resolution = self.cubemap_resolution + depth_sensor_spec.hfov = 90 + depth_sensor_spec.position = [0.0, 0.0, 0.0] + depth_sensor_spec.orientation = orientation + sensor_specifications.append(depth_sensor_spec) + + # Add equirectangular map + if self.render_equirectangular: + rgb_sensor_spec = habitat_sim.bindings.EquirectangularSensorSpec() + rgb_sensor_spec.uuid = "color_equirectangular" + rgb_sensor_spec.resolution = self.equirectangular_resolution + rgb_sensor_spec.position = [0.0, 0.0, 0.0] + sensor_specifications.append(rgb_sensor_spec) + + if self.render_depth: + depth_sensor_spec = habitat_sim.bindings.EquirectangularSensorSpec() + depth_sensor_spec.uuid = "depth_equirectangular" + depth_sensor_spec.sensor_type = habitat_sim.SensorType.DEPTH + depth_sensor_spec.resolution = self.equirectangular_resolution + depth_sensor_spec.position = [0.0, 0.0, 0.0] + depth_sensor_spec.orientation + sensor_specifications.append(depth_sensor_spec) + + agent_cfg = habitat_sim.agent.AgentConfiguration(sensor_specifications=sensor_specifications) + + cfg = habitat_sim.Configuration(sim_cfg, [agent_cfg]) + self.sim = habitat_sim.Simulator(cfg) + if self.navmesh is not None and self.navmesh != "": + # Use pre-computed navmesh (the one generated automatically does some weird stuffs like going on top of the roof) + # See https://youtu.be/kunFMRJAu2U?t=1522 regarding navmeshes + self.sim.pathfinder.load_nav_mesh(self.navmesh) + + # Check that the navmesh is not empty + if not self.sim.pathfinder.is_loaded: + # Try to compute a navmesh + navmesh_settings = habitat_sim.NavMeshSettings() + navmesh_settings.set_defaults() + self.sim.recompute_navmesh(self.sim.pathfinder, navmesh_settings, True) + + # Check that the navmesh is not empty + if not self.sim.pathfinder.is_loaded: + raise NoNaviguableSpaceError(f"No naviguable location (scene: {self.scene} -- navmesh: {self.navmesh})") + + self.agent = self.sim.initialize_agent(agent_id=0) + + def close(self): + if hasattr(self, 'sim'): + self.sim.close() + + def __del__(self): + self.close() + + def render_viewpoint(self, viewpoint_position): + agent_state = habitat_sim.AgentState() + agent_state.position = viewpoint_position + # agent_state.rotation = viewpoint_orientation + self.agent.set_state(agent_state) + viewpoint_observations = self.sim.get_sensor_observations(agent_ids=0) + + try: + # Depth map values have been obtained using cubemap rendering internally, + # so they do not really correspond to distance to the viewpoint in practice + # and they need some scaling + viewpoint_observations["depth_equirectangular"] *= self.equirectangular_depth_scale_factors + except KeyError: + pass + + data = dict(observations=viewpoint_observations, position=viewpoint_position) + return data + + def up_direction(self): + return np.asarray(habitat_sim.geo.UP).tolist() + + def R_cam_to_world(self): + return R_OPENCV2HABITAT.tolist() diff --git a/src/mast3r_src/dust3r/datasets_preprocess/habitat/habitat_renderer/multiview_crop_generator.py b/src/mast3r_src/dust3r/datasets_preprocess/habitat/habitat_renderer/multiview_crop_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..b86238b44a5cdd7a2e30b9d64773c2388f9711c3 --- /dev/null +++ b/src/mast3r_src/dust3r/datasets_preprocess/habitat/habitat_renderer/multiview_crop_generator.py @@ -0,0 +1,93 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Generate pairs of crops from a dataset of environment maps. +# -------------------------------------------------------- +import os +import numpy as np +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" # noqa +import cv2 +import collections +from habitat_renderer import projections, projections_conversions +from habitat_renderer.habitat_sim_envmaps_renderer import HabitatEnvironmentMapRenderer + +ViewpointData = collections.namedtuple("ViewpointData", ["colormap", "distancemap", "pointmap", "position"]) + +class HabitatMultiviewCrops: + def __init__(self, + scene, + navmesh, + scene_dataset_config_file, + equirectangular_resolution=(400, 800), + crop_resolution=(240, 320), + pixel_jittering_iterations=5, + jittering_noise_level=1.0): + self.crop_resolution = crop_resolution + + self.pixel_jittering_iterations = pixel_jittering_iterations + self.jittering_noise_level = jittering_noise_level + + # Instanciate the low resolution habitat sim renderer + self.lowres_envmap_renderer = HabitatEnvironmentMapRenderer(scene=scene, + navmesh=navmesh, + scene_dataset_config_file=scene_dataset_config_file, + equirectangular_resolution=equirectangular_resolution, + render_depth=True, + render_equirectangular=True) + self.R_cam_to_world = np.asarray(self.lowres_envmap_renderer.R_cam_to_world()) + self.up_direction = np.asarray(self.lowres_envmap_renderer.up_direction()) + + # Projection applied by each environment map + self.envmap_height, self.envmap_width = self.lowres_envmap_renderer.equirectangular_resolution + base_projection = projections.EquirectangularProjection(self.envmap_height, self.envmap_width) + self.envmap_projection = projections.RotatedProjection(base_projection, self.R_cam_to_world.T) + # 3D Rays map associated to each envmap + self.envmap_rays = projections.get_projection_rays(self.envmap_projection) + + def compute_pointmap(self, distancemap, position): + # Point cloud associated to each ray + return self.envmap_rays * distancemap[:, :, None] + position + + def render_viewpoint_data(self, position): + data = self.lowres_envmap_renderer.render_viewpoint(np.asarray(position)) + colormap = data['observations']['color_equirectangular'][..., :3] # Ignore the alpha channel + distancemap = data['observations']['depth_equirectangular'] + pointmap = self.compute_pointmap(distancemap, position) + return ViewpointData(colormap=colormap, distancemap=distancemap, pointmap=pointmap, position=position) + + def extract_cropped_camera(self, projection, color_image, distancemap, pointmap, voxelmap=None): + remapper = projections_conversions.RemapProjection(input_projection=self.envmap_projection, output_projection=projection, + pixel_jittering_iterations=self.pixel_jittering_iterations, jittering_noise_level=self.jittering_noise_level) + cropped_color_image = remapper.convert( + color_image, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_WRAP, single_map=False) + cropped_distancemap = remapper.convert( + distancemap, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_WRAP, single_map=True) + cropped_pointmap = remapper.convert(pointmap, interpolation=cv2.INTER_NEAREST, + borderMode=cv2.BORDER_WRAP, single_map=True) + cropped_voxelmap = (None if voxelmap is None else + remapper.convert(voxelmap, interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_WRAP, single_map=True)) + # Convert the distance map into a depth map + cropped_depthmap = np.asarray( + cropped_distancemap / np.linalg.norm(remapper.output_rays, axis=-1), dtype=cropped_distancemap.dtype) + + return cropped_color_image, cropped_depthmap, cropped_pointmap, cropped_voxelmap + +def perspective_projection_to_dict(persp_projection, position): + """ + Serialization-like function.""" + camera_params = dict(camera_intrinsics=projections.colmap_to_opencv_intrinsics(persp_projection.base_projection.K).tolist(), + size=(persp_projection.base_projection.width, persp_projection.base_projection.height), + R_cam2world=persp_projection.R_to_base_projection.T.tolist(), + t_cam2world=position) + return camera_params + + +def dict_to_perspective_projection(camera_params): + K = projections.opencv_to_colmap_intrinsics(np.asarray(camera_params["camera_intrinsics"])) + size = camera_params["size"] + R_cam2world = np.asarray(camera_params["R_cam2world"]) + projection = projections.PerspectiveProjection(K, height=size[1], width=size[0]) + projection = projections.RotatedProjection(projection, R_to_base_projection=R_cam2world.T) + position = camera_params["t_cam2world"] + return projection, position \ No newline at end of file diff --git a/src/mast3r_src/dust3r/datasets_preprocess/habitat/habitat_renderer/projections.py b/src/mast3r_src/dust3r/datasets_preprocess/habitat/habitat_renderer/projections.py new file mode 100644 index 0000000000000000000000000000000000000000..4db1f79d23e23a8ba144b4357c4d4daf10cf8fab --- /dev/null +++ b/src/mast3r_src/dust3r/datasets_preprocess/habitat/habitat_renderer/projections.py @@ -0,0 +1,151 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Various 3D/2D projection utils, useful to sample virtual cameras. +# -------------------------------------------------------- +import numpy as np + +class EquirectangularProjection: + """ + Convention for the central pixel of the equirectangular map similar to OpenCV perspective model: + +X from left to right + +Y from top to bottom + +Z going outside the camera + EXCEPT that the top left corner of the image is assumed to have (0,0) coordinates (OpenCV assumes (-0.5,-0.5)) + """ + + def __init__(self, height, width): + self.height = height + self.width = width + self.u_scaling = (2 * np.pi) / self.width + self.v_scaling = np.pi / self.height + + def unproject(self, u, v): + """ + Args: + u, v: 2D coordinates + Returns: + unnormalized 3D rays. + """ + longitude = self.u_scaling * u - np.pi + minus_latitude = self.v_scaling * v - np.pi/2 + + cos_latitude = np.cos(minus_latitude) + x, z = np.sin(longitude) * cos_latitude, np.cos(longitude) * cos_latitude + y = np.sin(minus_latitude) + + rays = np.stack([x, y, z], axis=-1) + return rays + + def project(self, rays): + """ + Args: + rays: Bx3 array of 3D rays. + Returns: + u, v: tuple of 2D coordinates. + """ + rays = rays / np.linalg.norm(rays, axis=-1, keepdims=True) + x, y, z = [rays[..., i] for i in range(3)] + + longitude = np.arctan2(x, z) + minus_latitude = np.arcsin(y) + + u = (longitude + np.pi) * (1.0 / self.u_scaling) + v = (minus_latitude + np.pi/2) * (1.0 / self.v_scaling) + return u, v + + +class PerspectiveProjection: + """ + OpenCV convention: + World space: + +X from left to right + +Y from top to bottom + +Z going outside the camera + Pixel space: + +u from left to right + +v from top to bottom + EXCEPT that the top left corner of the image is assumed to have (0,0) coordinates (OpenCV assumes (-0.5,-0.5)). + """ + + def __init__(self, K, height, width): + self.height = height + self.width = width + self.K = K + self.Kinv = np.linalg.inv(K) + + def project(self, rays): + uv_homogeneous = np.einsum("ik, ...k -> ...i", self.K, rays) + uv = uv_homogeneous[..., :2] / uv_homogeneous[..., 2, None] + return uv[..., 0], uv[..., 1] + + def unproject(self, u, v): + uv_homogeneous = np.stack((u, v, np.ones_like(u)), axis=-1) + rays = np.einsum("ik, ...k -> ...i", self.Kinv, uv_homogeneous) + return rays + + +class RotatedProjection: + def __init__(self, base_projection, R_to_base_projection): + self.base_projection = base_projection + self.R_to_base_projection = R_to_base_projection + + @property + def width(self): + return self.base_projection.width + + @property + def height(self): + return self.base_projection.height + + def project(self, rays): + if self.R_to_base_projection is not None: + rays = np.einsum("ik, ...k -> ...i", self.R_to_base_projection, rays) + return self.base_projection.project(rays) + + def unproject(self, u, v): + rays = self.base_projection.unproject(u, v) + if self.R_to_base_projection is not None: + rays = np.einsum("ik, ...k -> ...i", self.R_to_base_projection.T, rays) + return rays + +def get_projection_rays(projection, noise_level=0): + """ + Return a 2D map of 3D rays corresponding to the projection. + If noise_level > 0, add some jittering noise to these rays. + """ + grid_u, grid_v = np.meshgrid(0.5 + np.arange(projection.width), 0.5 + np.arange(projection.height)) + if noise_level > 0: + grid_u += np.clip(0, noise_level * np.random.uniform(-0.5, 0.5, size=grid_u.shape), projection.width) + grid_v += np.clip(0, noise_level * np.random.uniform(-0.5, 0.5, size=grid_v.shape), projection.height) + return projection.unproject(grid_u, grid_v) + +def compute_camera_intrinsics(height, width, hfov): + f = width/2 / np.tan(hfov/2 * np.pi/180) + cu, cv = width/2, height/2 + return f, cu, cv + +def colmap_to_opencv_intrinsics(K): + """ + Modify camera intrinsics to follow a different convention. + Coordinates of the center of the top-left pixels are by default: + - (0.5, 0.5) in Colmap + - (0,0) in OpenCV + """ + K = K.copy() + K[0, 2] -= 0.5 + K[1, 2] -= 0.5 + return K + +def opencv_to_colmap_intrinsics(K): + """ + Modify camera intrinsics to follow a different convention. + Coordinates of the center of the top-left pixels are by default: + - (0.5, 0.5) in Colmap + - (0,0) in OpenCV + """ + K = K.copy() + K[0, 2] += 0.5 + K[1, 2] += 0.5 + return K \ No newline at end of file diff --git a/src/mast3r_src/dust3r/datasets_preprocess/habitat/habitat_renderer/projections_conversions.py b/src/mast3r_src/dust3r/datasets_preprocess/habitat/habitat_renderer/projections_conversions.py new file mode 100644 index 0000000000000000000000000000000000000000..4bcfed4066bbac62fa4254ea6417bf429b098b75 --- /dev/null +++ b/src/mast3r_src/dust3r/datasets_preprocess/habitat/habitat_renderer/projections_conversions.py @@ -0,0 +1,45 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Remap data from one projection to an other +# -------------------------------------------------------- +import numpy as np +import cv2 +from habitat_renderer import projections + +class RemapProjection: + def __init__(self, input_projection, output_projection, pixel_jittering_iterations=0, jittering_noise_level=0): + """ + Some naive random jittering can be introduced in the remapping to mitigate aliasing artecfacts. + """ + assert jittering_noise_level >= 0 + assert pixel_jittering_iterations >= 0 + + maps = [] + # Initial map + self.output_rays = projections.get_projection_rays(output_projection) + map_u, map_v = input_projection.project(self.output_rays) + map_u, map_v = np.asarray(map_u, dtype=np.float32), np.asarray(map_v, dtype=np.float32) + maps.append((map_u, map_v)) + + for _ in range(pixel_jittering_iterations): + # Define multiple mappings using some coordinates jittering to mitigate aliasing effects + crop_rays = projections.get_projection_rays(output_projection, jittering_noise_level) + map_u, map_v = input_projection.project(crop_rays) + map_u, map_v = np.asarray(map_u, dtype=np.float32), np.asarray(map_v, dtype=np.float32) + maps.append((map_u, map_v)) + self.maps = maps + + def convert(self, img, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_WRAP, single_map=False): + remapped = [] + for map_u, map_v in self.maps: + res = cv2.remap(img, map_u, map_v, interpolation=interpolation, borderMode=borderMode) + remapped.append(res) + if single_map: + break + if len(remapped) == 1: + res = remapped[0] + else: + res = np.asarray(np.mean(remapped, axis=0), dtype=img.dtype) + return res diff --git a/src/mast3r_src/dust3r/datasets_preprocess/habitat/preprocess_habitat.py b/src/mast3r_src/dust3r/datasets_preprocess/habitat/preprocess_habitat.py new file mode 100644 index 0000000000000000000000000000000000000000..cacbe2467a8e9629c2472b0e05fc0cf8326367e2 --- /dev/null +++ b/src/mast3r_src/dust3r/datasets_preprocess/habitat/preprocess_habitat.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# main executable for preprocessing habitat +# export METADATA_DIR="/path/to/habitat/5views_v1_512x512_metadata" +# export SCENES_DIR="/path/to/habitat/data/scene_datasets/" +# export OUTPUT_DIR="data/habitat_processed" +# export PYTHONPATH=$(pwd) +# python preprocess_habitat.py --scenes_dir=$SCENES_DIR --metadata_dir=$METADATA_DIR --output_dir=$OUTPUT_DIR | parallel -j 16 +# -------------------------------------------------------- +import os +import glob +import json +import os + +import PIL.Image +import json +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" # noqa +import cv2 +from habitat_renderer import multiview_crop_generator +from tqdm import tqdm + + +def preprocess_metadata(metadata_filename, + scenes_dir, + output_dir, + crop_resolution=[512, 512], + equirectangular_resolution=None, + fix_existing_dataset=False): + # Load data + with open(metadata_filename, "r") as f: + metadata = json.load(f) + + if metadata["scene_dataset_config_file"] == "": + scene = os.path.join(scenes_dir, metadata["scene"]) + scene_dataset_config_file = "" + else: + scene = metadata["scene"] + scene_dataset_config_file = os.path.join(scenes_dir, metadata["scene_dataset_config_file"]) + navmesh = None + + # Use 4 times the crop size as resolution for rendering the environment map. + max_res = max(crop_resolution) + + if equirectangular_resolution == None: + # Use 4 times the crop size as resolution for rendering the environment map. + max_res = max(crop_resolution) + equirectangular_resolution = (4*max_res, 8*max_res) + + print("equirectangular_resolution:", equirectangular_resolution) + + if os.path.exists(output_dir) and not fix_existing_dataset: + raise FileExistsError(output_dir) + + # Lazy initialization + highres_dataset = None + + for batch_label, batch in tqdm(metadata["view_batches"].items()): + for view_label, view_params in batch.items(): + + assert view_params["size"] == crop_resolution + label = f"{batch_label}_{view_label}" + + output_camera_params_filename = os.path.join(output_dir, f"{label}_camera_params.json") + if fix_existing_dataset and os.path.isfile(output_camera_params_filename): + # Skip generation if we are fixing a dataset and the corresponding output file already exists + continue + + # Lazy initialization + if highres_dataset is None: + highres_dataset = multiview_crop_generator.HabitatMultiviewCrops(scene=scene, + navmesh=navmesh, + scene_dataset_config_file=scene_dataset_config_file, + equirectangular_resolution=equirectangular_resolution, + crop_resolution=crop_resolution,) + os.makedirs(output_dir, exist_ok=bool(fix_existing_dataset)) + + # Generate a higher resolution crop + original_projection, position = multiview_crop_generator.dict_to_perspective_projection(view_params) + # Render an envmap at the given position + viewpoint_data = highres_dataset.render_viewpoint_data(position) + + projection = original_projection + colormap, depthmap, pointmap, _ = highres_dataset.extract_cropped_camera( + projection, viewpoint_data.colormap, viewpoint_data.distancemap, viewpoint_data.pointmap) + + camera_params = multiview_crop_generator.perspective_projection_to_dict(projection, position) + + # Color image + PIL.Image.fromarray(colormap).save(os.path.join(output_dir, f"{label}.jpeg")) + # Depth image + cv2.imwrite(os.path.join(output_dir, f"{label}_depth.exr"), + depthmap, [cv2.IMWRITE_EXR_TYPE, cv2.IMWRITE_EXR_TYPE_HALF]) + # Camera parameters + with open(output_camera_params_filename, "w") as f: + json.dump(camera_params, f) + + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--metadata_dir", required=True) + parser.add_argument("--scenes_dir", required=True) + parser.add_argument("--output_dir", required=True) + parser.add_argument("--metadata_filename", default="") + + args = parser.parse_args() + + if args.metadata_filename == "": + # Walk through the metadata dir to generate commandlines + for filename in glob.iglob(os.path.join(args.metadata_dir, "**/metadata.json"), recursive=True): + output_dir = os.path.join(args.output_dir, os.path.relpath(os.path.dirname(filename), args.metadata_dir)) + if not os.path.exists(output_dir): + commandline = f"python {__file__} --metadata_filename={filename} --metadata_dir={args.metadata_dir} --scenes_dir={args.scenes_dir} --output_dir={output_dir}" + print(commandline) + else: + preprocess_metadata(metadata_filename=args.metadata_filename, + scenes_dir=args.scenes_dir, + output_dir=args.output_dir) diff --git a/src/mast3r_src/dust3r/datasets_preprocess/path_to_root.py b/src/mast3r_src/dust3r/datasets_preprocess/path_to_root.py new file mode 100644 index 0000000000000000000000000000000000000000..6e076a17a408d0a9e043fbda2d73f1592e7cb71a --- /dev/null +++ b/src/mast3r_src/dust3r/datasets_preprocess/path_to_root.py @@ -0,0 +1,13 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# DUSt3R repo root import +# -------------------------------------------------------- + +import sys +import os.path as path +HERE_PATH = path.normpath(path.dirname(__file__)) +DUST3R_REPO_PATH = path.normpath(path.join(HERE_PATH, '../')) +# workaround for sibling import +sys.path.insert(0, DUST3R_REPO_PATH) diff --git a/src/mast3r_src/dust3r/datasets_preprocess/preprocess_arkitscenes.py b/src/mast3r_src/dust3r/datasets_preprocess/preprocess_arkitscenes.py new file mode 100644 index 0000000000000000000000000000000000000000..5dbc103a82d646293e1d81f5132683e2b08cd879 --- /dev/null +++ b/src/mast3r_src/dust3r/datasets_preprocess/preprocess_arkitscenes.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Script to pre-process the arkitscenes dataset. +# Usage: +# python3 datasets_preprocess/preprocess_arkitscenes.py --arkitscenes_dir /path/to/arkitscenes --precomputed_pairs /path/to/arkitscenes_pairs +# -------------------------------------------------------- +import os +import json +import os.path as osp +import decimal +import argparse +import math +from bisect import bisect_left +from PIL import Image +import numpy as np +import quaternion +from scipy import interpolate +import cv2 + + +def get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument('--arkitscenes_dir', required=True) + parser.add_argument('--precomputed_pairs', required=True) + parser.add_argument('--output_dir', default='data/arkitscenes_processed') + return parser + + +def value_to_decimal(value, decimal_places): + decimal.getcontext().rounding = decimal.ROUND_HALF_UP # define rounding method + return decimal.Decimal(str(float(value))).quantize(decimal.Decimal('1e-{}'.format(decimal_places))) + + +def closest(value, sorted_list): + index = bisect_left(sorted_list, value) + if index == 0: + return sorted_list[0] + elif index == len(sorted_list): + return sorted_list[-1] + else: + value_before = sorted_list[index - 1] + value_after = sorted_list[index] + if value_after - value < value - value_before: + return value_after + else: + return value_before + + +def get_up_vectors(pose_device_to_world): + return np.matmul(pose_device_to_world, np.array([[0.0], [-1.0], [0.0], [0.0]])) + + +def get_right_vectors(pose_device_to_world): + return np.matmul(pose_device_to_world, np.array([[1.0], [0.0], [0.0], [0.0]])) + + +def read_traj(traj_path): + quaternions = [] + poses = [] + timestamps = [] + poses_p_to_w = [] + with open(traj_path) as f: + traj_lines = f.readlines() + for line in traj_lines: + tokens = line.split() + assert len(tokens) == 7 + traj_timestamp = float(tokens[0]) + + timestamps_decimal_value = value_to_decimal(traj_timestamp, 3) + timestamps.append(float(timestamps_decimal_value)) # for spline interpolation + + angle_axis = [float(tokens[1]), float(tokens[2]), float(tokens[3])] + r_w_to_p, _ = cv2.Rodrigues(np.asarray(angle_axis)) + t_w_to_p = np.asarray([float(tokens[4]), float(tokens[5]), float(tokens[6])]) + + pose_w_to_p = np.eye(4) + pose_w_to_p[:3, :3] = r_w_to_p + pose_w_to_p[:3, 3] = t_w_to_p + + pose_p_to_w = np.linalg.inv(pose_w_to_p) + + r_p_to_w_as_quat = quaternion.from_rotation_matrix(pose_p_to_w[:3, :3]) + t_p_to_w = pose_p_to_w[:3, 3] + poses_p_to_w.append(pose_p_to_w) + poses.append(t_p_to_w) + quaternions.append(r_p_to_w_as_quat) + return timestamps, poses, quaternions, poses_p_to_w + + +def main(rootdir, pairsdir, outdir): + os.makedirs(outdir, exist_ok=True) + + subdirs = ['Test', 'Training'] + for subdir in subdirs: + if not osp.isdir(osp.join(rootdir, subdir)): + continue + # STEP 1: list all scenes + outsubdir = osp.join(outdir, subdir) + os.makedirs(outsubdir, exist_ok=True) + listfile = osp.join(pairsdir, subdir, 'scene_list.json') + with open(listfile, 'r') as f: + scene_dirs = json.load(f) + + valid_scenes = [] + for scene_subdir in scene_dirs: + out_scene_subdir = osp.join(outsubdir, scene_subdir) + os.makedirs(out_scene_subdir, exist_ok=True) + + scene_dir = osp.join(rootdir, subdir, scene_subdir) + depth_dir = osp.join(scene_dir, 'lowres_depth') + rgb_dir = osp.join(scene_dir, 'vga_wide') + intrinsics_dir = osp.join(scene_dir, 'vga_wide_intrinsics') + traj_path = osp.join(scene_dir, 'lowres_wide.traj') + + # STEP 2: read selected_pairs.npz + selected_pairs_path = osp.join(pairsdir, subdir, scene_subdir, 'selected_pairs.npz') + selected_npz = np.load(selected_pairs_path) + selection, pairs = selected_npz['selection'], selected_npz['pairs'] + selected_sky_direction_scene = str(selected_npz['sky_direction_scene'][0]) + if len(selection) == 0 or len(pairs) == 0: + # not a valid scene + continue + valid_scenes.append(scene_subdir) + + # STEP 3: parse the scene and export the list of valid (K, pose, rgb, depth) and convert images + scene_metadata_path = osp.join(out_scene_subdir, 'scene_metadata.npz') + if osp.isfile(scene_metadata_path): + continue + else: + print(f'parsing {scene_subdir}') + # loads traj + timestamps, poses, quaternions, poses_cam_to_world = read_traj(traj_path) + + poses = np.array(poses) + quaternions = np.array(quaternions, dtype=np.quaternion) + quaternions = quaternion.unflip_rotors(quaternions) + timestamps = np.array(timestamps) + + selected_images = [(basename, basename.split(".png")[0].split("_")[1]) for basename in selection] + timestamps_selected = [float(frame_id) for _, frame_id in selected_images] + + sky_direction_scene, trajectories, intrinsics, images = convert_scene_metadata(scene_subdir, + intrinsics_dir, + timestamps, + quaternions, + poses, + poses_cam_to_world, + selected_images, + timestamps_selected) + assert selected_sky_direction_scene == sky_direction_scene + + os.makedirs(os.path.join(out_scene_subdir, 'vga_wide'), exist_ok=True) + os.makedirs(os.path.join(out_scene_subdir, 'lowres_depth'), exist_ok=True) + assert isinstance(sky_direction_scene, str) + for basename in images: + img_out = os.path.join(out_scene_subdir, 'vga_wide', basename.replace('.png', '.jpg')) + depth_out = os.path.join(out_scene_subdir, 'lowres_depth', basename) + if osp.isfile(img_out) and osp.isfile(depth_out): + continue + + vga_wide_path = osp.join(rgb_dir, basename) + depth_path = osp.join(depth_dir, basename) + + img = Image.open(vga_wide_path) + depth = cv2.imread(depth_path, cv2.IMREAD_UNCHANGED) + + # rotate the image + if sky_direction_scene == 'RIGHT': + try: + img = img.transpose(Image.Transpose.ROTATE_90) + except Exception: + img = img.transpose(Image.ROTATE_90) + depth = cv2.rotate(depth, cv2.ROTATE_90_COUNTERCLOCKWISE) + elif sky_direction_scene == 'LEFT': + try: + img = img.transpose(Image.Transpose.ROTATE_270) + except Exception: + img = img.transpose(Image.ROTATE_270) + depth = cv2.rotate(depth, cv2.ROTATE_90_CLOCKWISE) + elif sky_direction_scene == 'DOWN': + try: + img = img.transpose(Image.Transpose.ROTATE_180) + except Exception: + img = img.transpose(Image.ROTATE_180) + depth = cv2.rotate(depth, cv2.ROTATE_180) + + W, H = img.size + if not osp.isfile(img_out): + img.save(img_out) + + depth = cv2.resize(depth, (W, H), interpolation=cv2.INTER_NEAREST_EXACT) + if not osp.isfile(depth_out): # avoid destroying the base dataset when you mess up the paths + cv2.imwrite(depth_out, depth) + + # save at the end + np.savez(scene_metadata_path, + trajectories=trajectories, + intrinsics=intrinsics, + images=images, + pairs=pairs) + + outlistfile = osp.join(outsubdir, 'scene_list.json') + with open(outlistfile, 'w') as f: + json.dump(valid_scenes, f) + + # STEP 5: concat all scene_metadata.npz into a single file + scene_data = {} + for scene_subdir in valid_scenes: + scene_metadata_path = osp.join(outsubdir, scene_subdir, 'scene_metadata.npz') + with np.load(scene_metadata_path) as data: + trajectories = data['trajectories'] + intrinsics = data['intrinsics'] + images = data['images'] + pairs = data['pairs'] + scene_data[scene_subdir] = {'trajectories': trajectories, + 'intrinsics': intrinsics, + 'images': images, + 'pairs': pairs} + offset = 0 + counts = [] + scenes = [] + sceneids = [] + images = [] + intrinsics = [] + trajectories = [] + pairs = [] + for scene_idx, (scene_subdir, data) in enumerate(scene_data.items()): + num_imgs = data['images'].shape[0] + img_pairs = data['pairs'] + + scenes.append(scene_subdir) + sceneids.extend([scene_idx] * num_imgs) + + images.append(data['images']) + + K = np.expand_dims(np.eye(3), 0).repeat(num_imgs, 0) + K[:, 0, 0] = [fx for _, _, fx, _, _, _ in data['intrinsics']] + K[:, 1, 1] = [fy for _, _, _, fy, _, _ in data['intrinsics']] + K[:, 0, 2] = [hw for _, _, _, _, hw, _ in data['intrinsics']] + K[:, 1, 2] = [hh for _, _, _, _, _, hh in data['intrinsics']] + + intrinsics.append(K) + trajectories.append(data['trajectories']) + + # offset pairs + img_pairs[:, 0:2] += offset + pairs.append(img_pairs) + counts.append(offset) + + offset += num_imgs + + images = np.concatenate(images, axis=0) + intrinsics = np.concatenate(intrinsics, axis=0) + trajectories = np.concatenate(trajectories, axis=0) + pairs = np.concatenate(pairs, axis=0) + np.savez(osp.join(outsubdir, 'all_metadata.npz'), + counts=counts, + scenes=scenes, + sceneids=sceneids, + images=images, + intrinsics=intrinsics, + trajectories=trajectories, + pairs=pairs) + + +def convert_scene_metadata(scene_subdir, intrinsics_dir, + timestamps, quaternions, poses, poses_cam_to_world, + selected_images, timestamps_selected): + # find scene orientation + sky_direction_scene, rotated_to_cam = find_scene_orientation(poses_cam_to_world) + + # find/compute pose for selected timestamps + # most images have a valid timestamp / exact pose associated + timestamps_selected = np.array(timestamps_selected) + spline = interpolate.interp1d(timestamps, poses, kind='linear', axis=0) + interpolated_rotations = quaternion.squad(quaternions, timestamps, timestamps_selected) + interpolated_positions = spline(timestamps_selected) + + trajectories = [] + intrinsics = [] + images = [] + for i, (basename, frame_id) in enumerate(selected_images): + intrinsic_fn = osp.join(intrinsics_dir, f"{scene_subdir}_{frame_id}.pincam") + if not osp.exists(intrinsic_fn): + intrinsic_fn = osp.join(intrinsics_dir, f"{scene_subdir}_{float(frame_id) - 0.001:.3f}.pincam") + if not osp.exists(intrinsic_fn): + intrinsic_fn = osp.join(intrinsics_dir, f"{scene_subdir}_{float(frame_id) + 0.001:.3f}.pincam") + assert osp.exists(intrinsic_fn) + w, h, fx, fy, hw, hh = np.loadtxt(intrinsic_fn) # PINHOLE + + pose = np.eye(4) + pose[:3, :3] = quaternion.as_rotation_matrix(interpolated_rotations[i]) + pose[:3, 3] = interpolated_positions[i] + + images.append(basename) + if sky_direction_scene == 'RIGHT' or sky_direction_scene == 'LEFT': + intrinsics.append([h, w, fy, fx, hh, hw]) # swapped intrinsics + else: + intrinsics.append([w, h, fx, fy, hw, hh]) + trajectories.append(pose @ rotated_to_cam) # pose_cam_to_world @ rotated_to_cam = rotated(cam) to world + + return sky_direction_scene, trajectories, intrinsics, images + + +def find_scene_orientation(poses_cam_to_world): + if len(poses_cam_to_world) > 0: + up_vector = sum(get_up_vectors(p) for p in poses_cam_to_world) / len(poses_cam_to_world) + right_vector = sum(get_right_vectors(p) for p in poses_cam_to_world) / len(poses_cam_to_world) + up_world = np.array([[0.0], [0.0], [1.0], [0.0]]) + else: + up_vector = np.array([[0.0], [-1.0], [0.0], [0.0]]) + right_vector = np.array([[1.0], [0.0], [0.0], [0.0]]) + up_world = np.array([[0.0], [0.0], [1.0], [0.0]]) + + # value between 0, 180 + device_up_to_world_up_angle = np.arccos(np.clip(np.dot(np.transpose(up_world), + up_vector), -1.0, 1.0)).item() * 180.0 / np.pi + device_right_to_world_up_angle = np.arccos(np.clip(np.dot(np.transpose(up_world), + right_vector), -1.0, 1.0)).item() * 180.0 / np.pi + + up_closest_to_90 = abs(device_up_to_world_up_angle - 90.0) < abs(device_right_to_world_up_angle - 90.0) + if up_closest_to_90: + assert abs(device_up_to_world_up_angle - 90.0) < 45.0 + # LEFT + if device_right_to_world_up_angle > 90.0: + sky_direction_scene = 'LEFT' + cam_to_rotated_q = quaternion.from_rotation_vector([0.0, 0.0, math.pi / 2.0]) + else: + # note that in metadata.csv RIGHT does not exist, but again it's not accurate... + # well, turns out there are scenes oriented like this + # for example Training/41124801 + sky_direction_scene = 'RIGHT' + cam_to_rotated_q = quaternion.from_rotation_vector([0.0, 0.0, -math.pi / 2.0]) + else: + # right is close to 90 + assert abs(device_right_to_world_up_angle - 90.0) < 45.0 + if device_up_to_world_up_angle > 90.0: + sky_direction_scene = 'DOWN' + cam_to_rotated_q = quaternion.from_rotation_vector([0.0, 0.0, math.pi]) + else: + sky_direction_scene = 'UP' + cam_to_rotated_q = quaternion.quaternion(1, 0, 0, 0) + cam_to_rotated = np.eye(4) + cam_to_rotated[:3, :3] = quaternion.as_rotation_matrix(cam_to_rotated_q) + rotated_to_cam = np.linalg.inv(cam_to_rotated) + return sky_direction_scene, rotated_to_cam + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + main(args.arkitscenes_dir, args.precomputed_pairs, args.output_dir) diff --git a/src/mast3r_src/dust3r/datasets_preprocess/preprocess_blendedMVS.py b/src/mast3r_src/dust3r/datasets_preprocess/preprocess_blendedMVS.py new file mode 100644 index 0000000000000000000000000000000000000000..d22793793c1219ebb1b3ba8eff51226c2b13f657 --- /dev/null +++ b/src/mast3r_src/dust3r/datasets_preprocess/preprocess_blendedMVS.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Preprocessing code for the BlendedMVS dataset +# dataset at https://github.com/YoYo000/BlendedMVS +# 1) Download BlendedMVS.zip +# 2) Download BlendedMVS+.zip +# 3) Download BlendedMVS++.zip +# 4) Unzip everything in the same /path/to/tmp/blendedMVS/ directory +# 5) python datasets_preprocess/preprocess_blendedMVS.py --blendedmvs_dir /path/to/tmp/blendedMVS/ +# -------------------------------------------------------- +import os +import os.path as osp +import re +from tqdm import tqdm +import numpy as np +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +import cv2 + +import path_to_root # noqa +from dust3r.utils.parallel import parallel_threads +from dust3r.datasets.utils import cropping # noqa + + +def get_parser(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--blendedmvs_dir', required=True) + parser.add_argument('--precomputed_pairs', required=True) + parser.add_argument('--output_dir', default='data/blendedmvs_processed') + return parser + + +def main(db_root, pairs_path, output_dir): + print('>> Listing all sequences') + sequences = [f for f in os.listdir(db_root) if len(f) == 24] + # should find 502 scenes + assert sequences, f'did not found any sequences at {db_root}' + print(f' (found {len(sequences)} sequences)') + + for i, seq in enumerate(tqdm(sequences)): + out_dir = osp.join(output_dir, seq) + os.makedirs(out_dir, exist_ok=True) + + # generate the crops + root = osp.join(db_root, seq) + cam_dir = osp.join(root, 'cams') + func_args = [(root, f[:-8], out_dir) for f in os.listdir(cam_dir) if not f.startswith('pair')] + parallel_threads(load_crop_and_save, func_args, star_args=True, leave=False) + + # verify that all pairs are there + pairs = np.load(pairs_path) + for seqh, seql, img1, img2, score in tqdm(pairs): + for view_index in [img1, img2]: + impath = osp.join(output_dir, f"{seqh:08x}{seql:016x}", f"{view_index:08n}.jpg") + assert osp.isfile(impath), f'missing image at {impath=}' + + print(f'>> Done, saved everything in {output_dir}/') + + +def load_crop_and_save(root, img, out_dir): + if osp.isfile(osp.join(out_dir, img + '.npz')): + return # already done + + # load everything + intrinsics_in, R_camin2world, t_camin2world = _load_pose(osp.join(root, 'cams', img + '_cam.txt')) + color_image_in = cv2.cvtColor(cv2.imread(osp.join(root, 'blended_images', img + + '.jpg'), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB) + depthmap_in = load_pfm_file(osp.join(root, 'rendered_depth_maps', img + '.pfm')) + + # do the crop + H, W = color_image_in.shape[:2] + assert H * 4 == W * 3 + image, depthmap, intrinsics_out, R_in2out = _crop_image(intrinsics_in, color_image_in, depthmap_in, (512, 384)) + + # write everything + image.save(osp.join(out_dir, img + '.jpg'), quality=80) + cv2.imwrite(osp.join(out_dir, img + '.exr'), depthmap) + + # New camera parameters + R_camout2world = R_camin2world @ R_in2out.T + t_camout2world = t_camin2world + np.savez(osp.join(out_dir, img + '.npz'), intrinsics=intrinsics_out, + R_cam2world=R_camout2world, t_cam2world=t_camout2world) + + +def _crop_image(intrinsics_in, color_image_in, depthmap_in, resolution_out=(800, 800)): + image, depthmap, intrinsics_out = cropping.rescale_image_depthmap( + color_image_in, depthmap_in, intrinsics_in, resolution_out) + R_in2out = np.eye(3) + return image, depthmap, intrinsics_out, R_in2out + + +def _load_pose(path, ret_44=False): + f = open(path) + RT = np.loadtxt(f, skiprows=1, max_rows=4, dtype=np.float32) + assert RT.shape == (4, 4) + RT = np.linalg.inv(RT) # world2cam to cam2world + + K = np.loadtxt(f, skiprows=2, max_rows=3, dtype=np.float32) + assert K.shape == (3, 3) + + if ret_44: + return K, RT + return K, RT[:3, :3], RT[:3, 3] # , depth_uint8_to_f32 + + +def load_pfm_file(file_path): + with open(file_path, 'rb') as file: + header = file.readline().decode('UTF-8').strip() + + if header == 'PF': + is_color = True + elif header == 'Pf': + is_color = False + else: + raise ValueError('The provided file is not a valid PFM file.') + + dimensions = re.match(r'^(\d+)\s(\d+)\s$', file.readline().decode('UTF-8')) + if dimensions: + img_width, img_height = map(int, dimensions.groups()) + else: + raise ValueError('Invalid PFM header format.') + + endian_scale = float(file.readline().decode('UTF-8').strip()) + if endian_scale < 0: + dtype = '= img_size * 3/4, and max dimension will be >= img_size")) + return parser + + +def convert_ndc_to_pinhole(focal_length, principal_point, image_size): + focal_length = np.array(focal_length) + principal_point = np.array(principal_point) + image_size_wh = np.array([image_size[1], image_size[0]]) + half_image_size = image_size_wh / 2 + rescale = half_image_size.min() + principal_point_px = half_image_size - principal_point * rescale + focal_length_px = focal_length * rescale + fx, fy = focal_length_px[0], focal_length_px[1] + cx, cy = principal_point_px[0], principal_point_px[1] + K = np.array([[fx, 0.0, cx], [0.0, fy, cy], [0.0, 0.0, 1.0]], dtype=np.float32) + return K + + +def opencv_from_cameras_projection(R, T, focal, p0, image_size): + R = torch.from_numpy(R)[None, :, :] + T = torch.from_numpy(T)[None, :] + focal = torch.from_numpy(focal)[None, :] + p0 = torch.from_numpy(p0)[None, :] + image_size = torch.from_numpy(image_size)[None, :] + + R_pytorch3d = R.clone() + T_pytorch3d = T.clone() + focal_pytorch3d = focal + p0_pytorch3d = p0 + T_pytorch3d[:, :2] *= -1 + R_pytorch3d[:, :, :2] *= -1 + tvec = T_pytorch3d + R = R_pytorch3d.permute(0, 2, 1) + + # Retype the image_size correctly and flip to width, height. + image_size_wh = image_size.to(R).flip(dims=(1,)) + + # NDC to screen conversion. + scale = image_size_wh.to(R).min(dim=1, keepdim=True)[0] / 2.0 + scale = scale.expand(-1, 2) + c0 = image_size_wh / 2.0 + + principal_point = -p0_pytorch3d * scale + c0 + focal_length = focal_pytorch3d * scale + + camera_matrix = torch.zeros_like(R) + camera_matrix[:, :2, 2] = principal_point + camera_matrix[:, 2, 2] = 1.0 + camera_matrix[:, 0, 0] = focal_length[:, 0] + camera_matrix[:, 1, 1] = focal_length[:, 1] + return R[0], tvec[0], camera_matrix[0] + + +def get_set_list(category_dir, split, is_single_sequence_subset=False): + listfiles = os.listdir(osp.join(category_dir, "set_lists")) + if is_single_sequence_subset: + # not all objects have manyview_dev + subset_list_files = [f for f in listfiles if "manyview_dev" in f] + else: + subset_list_files = [f for f in listfiles if f"fewview_train" in f] + + sequences_all = [] + for subset_list_file in subset_list_files: + with open(osp.join(category_dir, "set_lists", subset_list_file)) as f: + subset_lists_data = json.load(f) + sequences_all.extend(subset_lists_data[split]) + + return sequences_all + + +def prepare_sequences(category, co3d_dir, output_dir, img_size, split, min_quality, max_num_sequences_per_object, + seed, is_single_sequence_subset=False): + random.seed(seed) + category_dir = osp.join(co3d_dir, category) + category_output_dir = osp.join(output_dir, category) + sequences_all = get_set_list(category_dir, split, is_single_sequence_subset) + sequences_numbers = sorted(set(seq_name for seq_name, _, _ in sequences_all)) + + frame_file = osp.join(category_dir, "frame_annotations.jgz") + sequence_file = osp.join(category_dir, "sequence_annotations.jgz") + + with gzip.open(frame_file, "r") as fin: + frame_data = json.loads(fin.read()) + with gzip.open(sequence_file, "r") as fin: + sequence_data = json.loads(fin.read()) + + frame_data_processed = {} + for f_data in frame_data: + sequence_name = f_data["sequence_name"] + frame_data_processed.setdefault(sequence_name, {})[f_data["frame_number"]] = f_data + + good_quality_sequences = set() + for seq_data in sequence_data: + if seq_data["viewpoint_quality_score"] > min_quality: + good_quality_sequences.add(seq_data["sequence_name"]) + + sequences_numbers = [seq_name for seq_name in sequences_numbers if seq_name in good_quality_sequences] + if len(sequences_numbers) < max_num_sequences_per_object: + selected_sequences_numbers = sequences_numbers + else: + selected_sequences_numbers = random.sample(sequences_numbers, max_num_sequences_per_object) + + selected_sequences_numbers_dict = {seq_name: [] for seq_name in selected_sequences_numbers} + sequences_all = [(seq_name, frame_number, filepath) + for seq_name, frame_number, filepath in sequences_all + if seq_name in selected_sequences_numbers_dict] + + for seq_name, frame_number, filepath in tqdm(sequences_all): + frame_idx = int(filepath.split('/')[-1][5:-4]) + selected_sequences_numbers_dict[seq_name].append(frame_idx) + mask_path = filepath.replace("images", "masks").replace(".jpg", ".png") + frame_data = frame_data_processed[seq_name][frame_number] + focal_length = frame_data["viewpoint"]["focal_length"] + principal_point = frame_data["viewpoint"]["principal_point"] + image_size = frame_data["image"]["size"] + K = convert_ndc_to_pinhole(focal_length, principal_point, image_size) + R, tvec, camera_intrinsics = opencv_from_cameras_projection(np.array(frame_data["viewpoint"]["R"]), + np.array(frame_data["viewpoint"]["T"]), + np.array(focal_length), + np.array(principal_point), + np.array(image_size)) + + frame_data = frame_data_processed[seq_name][frame_number] + depth_path = os.path.join(co3d_dir, frame_data["depth"]["path"]) + assert frame_data["depth"]["scale_adjustment"] == 1.0 + image_path = os.path.join(co3d_dir, filepath) + mask_path_full = os.path.join(co3d_dir, mask_path) + + input_rgb_image = PIL.Image.open(image_path).convert('RGB') + input_mask = plt.imread(mask_path_full) + + with PIL.Image.open(depth_path) as depth_pil: + # the image is stored with 16-bit depth but PIL reads it as I (32 bit). + # we cast it to uint16, then reinterpret as float16, then cast to float32 + input_depthmap = ( + np.frombuffer(np.array(depth_pil, dtype=np.uint16), dtype=np.float16) + .astype(np.float32) + .reshape((depth_pil.size[1], depth_pil.size[0]))) + depth_mask = np.stack((input_depthmap, input_mask), axis=-1) + H, W = input_depthmap.shape + + camera_intrinsics = camera_intrinsics.numpy() + cx, cy = camera_intrinsics[:2, 2].round().astype(int) + min_margin_x = min(cx, W - cx) + min_margin_y = min(cy, H - cy) + + # the new window will be a rectangle of size (2*min_margin_x, 2*min_margin_y) centered on (cx,cy) + l, t = cx - min_margin_x, cy - min_margin_y + r, b = cx + min_margin_x, cy + min_margin_y + crop_bbox = (l, t, r, b) + input_rgb_image, depth_mask, input_camera_intrinsics = cropping.crop_image_depthmap( + input_rgb_image, depth_mask, camera_intrinsics, crop_bbox) + + # try to set the lower dimension to img_size * 3/4 -> img_size=512 => 384 + scale_final = ((img_size * 3 // 4) / min(H, W)) + 1e-8 + output_resolution = np.floor(np.array([W, H]) * scale_final).astype(int) + if max(output_resolution) < img_size: + # let's put the max dimension to img_size + scale_final = (img_size / max(H, W)) + 1e-8 + output_resolution = np.floor(np.array([W, H]) * scale_final).astype(int) + + input_rgb_image, depth_mask, input_camera_intrinsics = cropping.rescale_image_depthmap( + input_rgb_image, depth_mask, input_camera_intrinsics, output_resolution) + input_depthmap = depth_mask[:, :, 0] + input_mask = depth_mask[:, :, 1] + + # generate and adjust camera pose + camera_pose = np.eye(4, dtype=np.float32) + camera_pose[:3, :3] = R + camera_pose[:3, 3] = tvec + camera_pose = np.linalg.inv(camera_pose) + + # save crop images and depth, metadata + save_img_path = os.path.join(output_dir, filepath) + save_depth_path = os.path.join(output_dir, frame_data["depth"]["path"]) + save_mask_path = os.path.join(output_dir, mask_path) + os.makedirs(os.path.split(save_img_path)[0], exist_ok=True) + os.makedirs(os.path.split(save_depth_path)[0], exist_ok=True) + os.makedirs(os.path.split(save_mask_path)[0], exist_ok=True) + + input_rgb_image.save(save_img_path) + scaled_depth_map = (input_depthmap / np.max(input_depthmap) * 65535).astype(np.uint16) + cv2.imwrite(save_depth_path, scaled_depth_map) + cv2.imwrite(save_mask_path, (input_mask * 255).astype(np.uint8)) + + save_meta_path = save_img_path.replace('jpg', 'npz') + np.savez(save_meta_path, camera_intrinsics=input_camera_intrinsics, + camera_pose=camera_pose, maximum_depth=np.max(input_depthmap)) + + return selected_sequences_numbers_dict + + +if __name__ == "__main__": + parser = get_parser() + args = parser.parse_args() + assert args.co3d_dir != args.output_dir + if args.category is None: + if args.single_sequence_subset: + categories = SINGLE_SEQUENCE_CATEGORIES + else: + categories = CATEGORIES + else: + categories = [args.category] + os.makedirs(args.output_dir, exist_ok=True) + + for split in ['train', 'test']: + selected_sequences_path = os.path.join(args.output_dir, f'selected_seqs_{split}.json') + if os.path.isfile(selected_sequences_path): + continue + + all_selected_sequences = {} + for category in categories: + category_output_dir = osp.join(args.output_dir, category) + os.makedirs(category_output_dir, exist_ok=True) + category_selected_sequences_path = os.path.join(category_output_dir, f'selected_seqs_{split}.json') + if os.path.isfile(category_selected_sequences_path): + with open(category_selected_sequences_path, 'r') as fid: + category_selected_sequences = json.load(fid) + else: + print(f"Processing {split} - category = {category}") + category_selected_sequences = prepare_sequences( + category=category, + co3d_dir=args.co3d_dir, + output_dir=args.output_dir, + img_size=args.img_size, + split=split, + min_quality=args.min_quality, + max_num_sequences_per_object=args.num_sequences_per_object, + seed=args.seed + CATEGORIES_IDX[category], + is_single_sequence_subset=args.single_sequence_subset + ) + with open(category_selected_sequences_path, 'w') as file: + json.dump(category_selected_sequences, file) + + all_selected_sequences[category] = category_selected_sequences + with open(selected_sequences_path, 'w') as file: + json.dump(all_selected_sequences, file) diff --git a/src/mast3r_src/dust3r/datasets_preprocess/preprocess_megadepth.py b/src/mast3r_src/dust3r/datasets_preprocess/preprocess_megadepth.py new file mode 100644 index 0000000000000000000000000000000000000000..b07c0c5dff0cfd828f9ce4fd204cf2eaa22487f1 --- /dev/null +++ b/src/mast3r_src/dust3r/datasets_preprocess/preprocess_megadepth.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Preprocessing code for the MegaDepth dataset +# dataset at https://www.cs.cornell.edu/projects/megadepth/ +# -------------------------------------------------------- +import os +import os.path as osp +import collections +from tqdm import tqdm +import numpy as np +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +import cv2 +import h5py + +import path_to_root # noqa +from dust3r.utils.parallel import parallel_threads +from dust3r.datasets.utils import cropping # noqa + + +def get_parser(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--megadepth_dir', required=True) + parser.add_argument('--precomputed_pairs', required=True) + parser.add_argument('--output_dir', default='data/megadepth_processed') + return parser + + +def main(db_root, pairs_path, output_dir): + os.makedirs(output_dir, exist_ok=True) + + # load all pairs + data = np.load(pairs_path, allow_pickle=True) + scenes = data['scenes'] + images = data['images'] + pairs = data['pairs'] + + # enumerate all unique images + todo = collections.defaultdict(set) + for scene, im1, im2, score in pairs: + todo[scene].add(im1) + todo[scene].add(im2) + + # for each scene, load intrinsics and then parallel crops + for scene, im_idxs in tqdm(todo.items(), desc='Overall'): + scene, subscene = scenes[scene].split() + out_dir = osp.join(output_dir, scene, subscene) + os.makedirs(out_dir, exist_ok=True) + + # load all camera params + _, pose_w2cam, intrinsics = _load_kpts_and_poses(db_root, scene, subscene, intrinsics=True) + + in_dir = osp.join(db_root, scene, 'dense' + subscene) + args = [(in_dir, img, intrinsics[img], pose_w2cam[img], out_dir) + for img in [images[im_id] for im_id in im_idxs]] + parallel_threads(resize_one_image, args, star_args=True, front_num=0, leave=False, desc=f'{scene}/{subscene}') + + # save pairs + print('Done! prepared all pairs in', output_dir) + + +def resize_one_image(root, tag, K_pre_rectif, pose_w2cam, out_dir): + if osp.isfile(osp.join(out_dir, tag + '.npz')): + return + + # load image + img = cv2.cvtColor(cv2.imread(osp.join(root, 'imgs', tag), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB) + H, W = img.shape[:2] + + # load depth + with h5py.File(osp.join(root, 'depths', osp.splitext(tag)[0] + '.h5'), 'r') as hd5: + depthmap = np.asarray(hd5['depth']) + + # rectify = undistort the intrinsics + imsize_pre, K_pre, distortion = K_pre_rectif + imsize_post = img.shape[1::-1] + K_post = cv2.getOptimalNewCameraMatrix(K_pre, distortion, imsize_pre, alpha=0, + newImgSize=imsize_post, centerPrincipalPoint=True)[0] + + # downscale + img_out, depthmap_out, intrinsics_out, R_in2out = _downscale_image(K_post, img, depthmap, resolution_out=(800, 600)) + + # write everything + img_out.save(osp.join(out_dir, tag + '.jpg'), quality=90) + cv2.imwrite(osp.join(out_dir, tag + '.exr'), depthmap_out) + + camout2world = np.linalg.inv(pose_w2cam) + camout2world[:3, :3] = camout2world[:3, :3] @ R_in2out.T + np.savez(osp.join(out_dir, tag + '.npz'), intrinsics=intrinsics_out, cam2world=camout2world) + + +def _downscale_image(camera_intrinsics, image, depthmap, resolution_out=(512, 384)): + H, W = image.shape[:2] + resolution_out = sorted(resolution_out)[::+1 if W < H else -1] + + image, depthmap, intrinsics_out = cropping.rescale_image_depthmap( + image, depthmap, camera_intrinsics, resolution_out, force=False) + R_in2out = np.eye(3) + + return image, depthmap, intrinsics_out, R_in2out + + +def _load_kpts_and_poses(root, scene_id, subscene, z_only=False, intrinsics=False): + if intrinsics: + with open(os.path.join(root, scene_id, 'sparse', 'manhattan', subscene, 'cameras.txt'), 'r') as f: + raw = f.readlines()[3:] # skip the header + + camera_intrinsics = {} + for camera in raw: + camera = camera.split(' ') + width, height, focal, cx, cy, k0 = [float(elem) for elem in camera[2:]] + K = np.eye(3) + K[0, 0] = focal + K[1, 1] = focal + K[0, 2] = cx + K[1, 2] = cy + camera_intrinsics[int(camera[0])] = ((int(width), int(height)), K, (k0, 0, 0, 0)) + + with open(os.path.join(root, scene_id, 'sparse', 'manhattan', subscene, 'images.txt'), 'r') as f: + raw = f.read().splitlines()[4:] # skip the header + + extract_pose = colmap_raw_pose_to_principal_axis if z_only else colmap_raw_pose_to_RT + + poses = {} + points3D_idxs = {} + camera = [] + + for image, points in zip(raw[:: 2], raw[1:: 2]): + image = image.split(' ') + points = points.split(' ') + + image_id = image[-1] + camera.append(int(image[-2])) + + # find the principal axis + raw_pose = [float(elem) for elem in image[1: -2]] + poses[image_id] = extract_pose(raw_pose) + + current_points3D_idxs = {int(i) for i in points[2:: 3] if i != '-1'} + assert -1 not in current_points3D_idxs, bb() + points3D_idxs[image_id] = current_points3D_idxs + + if intrinsics: + image_intrinsics = {im_id: camera_intrinsics[cam] for im_id, cam in zip(poses, camera)} + return points3D_idxs, poses, image_intrinsics + else: + return points3D_idxs, poses + + +def colmap_raw_pose_to_principal_axis(image_pose): + qvec = image_pose[: 4] + qvec = qvec / np.linalg.norm(qvec) + w, x, y, z = qvec + z_axis = np.float32([ + 2 * x * z - 2 * y * w, + 2 * y * z + 2 * x * w, + 1 - 2 * x * x - 2 * y * y + ]) + return z_axis + + +def colmap_raw_pose_to_RT(image_pose): + qvec = image_pose[: 4] + qvec = qvec / np.linalg.norm(qvec) + w, x, y, z = qvec + R = np.array([ + [ + 1 - 2 * y * y - 2 * z * z, + 2 * x * y - 2 * z * w, + 2 * x * z + 2 * y * w + ], + [ + 2 * x * y + 2 * z * w, + 1 - 2 * x * x - 2 * z * z, + 2 * y * z - 2 * x * w + ], + [ + 2 * x * z - 2 * y * w, + 2 * y * z + 2 * x * w, + 1 - 2 * x * x - 2 * y * y + ] + ]) + # principal_axis.append(R[2, :]) + t = image_pose[4: 7] + # World-to-Camera pose + current_pose = np.eye(4) + current_pose[: 3, : 3] = R + current_pose[: 3, 3] = t + return current_pose + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + main(args.megadepth_dir, args.precomputed_pairs, args.output_dir) diff --git a/src/mast3r_src/dust3r/datasets_preprocess/preprocess_scannetpp.py b/src/mast3r_src/dust3r/datasets_preprocess/preprocess_scannetpp.py new file mode 100644 index 0000000000000000000000000000000000000000..34e26dc9474df16cf0736f71248d01b7853d4786 --- /dev/null +++ b/src/mast3r_src/dust3r/datasets_preprocess/preprocess_scannetpp.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Script to pre-process the scannet++ dataset. +# Usage: +# python3 datasets_preprocess/preprocess_scannetpp.py --scannetpp_dir /path/to/scannetpp --precomputed_pairs /path/to/scannetpp_pairs --pyopengl-platform egl +# -------------------------------------------------------- +import os +import argparse +import os.path as osp +import re +from tqdm import tqdm +import json +from scipy.spatial.transform import Rotation +import pyrender +import trimesh +import trimesh.exchange.ply +import numpy as np +import cv2 +import PIL.Image as Image + +from dust3r.datasets.utils.cropping import rescale_image_depthmap +import dust3r.utils.geometry as geometry + +inv = np.linalg.inv +norm = np.linalg.norm +REGEXPR_DSLR = re.compile(r'^DSC(?P\d+).JPG$') +REGEXPR_IPHONE = re.compile(r'frame_(?P\d+).jpg$') + +DEBUG_VIZ = None # 'iou' +if DEBUG_VIZ is not None: + import matplotlib.pyplot as plt # noqa + + +OPENGL_TO_OPENCV = np.float32([[1, 0, 0, 0], + [0, -1, 0, 0], + [0, 0, -1, 0], + [0, 0, 0, 1]]) + + +def get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument('--scannetpp_dir', required=True) + parser.add_argument('--precomputed_pairs', required=True) + parser.add_argument('--output_dir', default='data/scannetpp_processed') + parser.add_argument('--target_resolution', default=920, type=int, help="images resolution") + parser.add_argument('--pyopengl-platform', type=str, default='', help='PyOpenGL env variable') + return parser + + +def pose_from_qwxyz_txyz(elems): + qw, qx, qy, qz, tx, ty, tz = map(float, elems) + pose = np.eye(4) + pose[:3, :3] = Rotation.from_quat((qx, qy, qz, qw)).as_matrix() + pose[:3, 3] = (tx, ty, tz) + return np.linalg.inv(pose) # returns cam2world + + +def get_frame_number(name, cam_type='dslr'): + if cam_type == 'dslr': + regex_expr = REGEXPR_DSLR + elif cam_type == 'iphone': + regex_expr = REGEXPR_IPHONE + else: + raise NotImplementedError(f'wrong {cam_type=} for get_frame_number') + matches = re.match(regex_expr, name) + return matches['frameid'] + + +def load_sfm(sfm_dir, cam_type='dslr'): + # load cameras + with open(osp.join(sfm_dir, 'cameras.txt'), 'r') as f: + raw = f.read().splitlines()[3:] # skip header + + intrinsics = {} + for camera in tqdm(raw, position=1, leave=False): + camera = camera.split(' ') + intrinsics[int(camera[0])] = [camera[1]] + [float(cam) for cam in camera[2:]] + + # load images + with open(os.path.join(sfm_dir, 'images.txt'), 'r') as f: + raw = f.read().splitlines() + raw = [line for line in raw if not line.startswith('#')] # skip header + + img_idx = {} + img_infos = {} + for image, points in tqdm(zip(raw[0::2], raw[1::2]), total=len(raw) // 2, position=1, leave=False): + image = image.split(' ') + points = points.split(' ') + + idx = image[0] + img_name = image[-1] + assert img_name not in img_idx, 'duplicate db image: ' + img_name + img_idx[img_name] = idx # register image name + + current_points2D = {int(i): (float(x), float(y)) + for i, x, y in zip(points[2::3], points[0::3], points[1::3]) if i != '-1'} + img_infos[idx] = dict(intrinsics=intrinsics[int(image[-2])], + path=img_name, + frame_id=get_frame_number(img_name, cam_type), + cam_to_world=pose_from_qwxyz_txyz(image[1: -2]), + sparse_pts2d=current_points2D) + + # load 3D points + with open(os.path.join(sfm_dir, 'points3D.txt'), 'r') as f: + raw = f.read().splitlines() + raw = [line for line in raw if not line.startswith('#')] # skip header + + points3D = {} + observations = {idx: [] for idx in img_infos.keys()} + for point in tqdm(raw, position=1, leave=False): + point = point.split() + point_3d_idx = int(point[0]) + points3D[point_3d_idx] = tuple(map(float, point[1:4])) + if len(point) > 8: + for idx, point_2d_idx in zip(point[8::2], point[9::2]): + observations[idx].append((point_3d_idx, int(point_2d_idx))) + + return img_idx, img_infos, points3D, observations + + +def subsample_img_infos(img_infos, num_images, allowed_name_subset=None): + img_infos_val = [(idx, val) for idx, val in img_infos.items()] + if allowed_name_subset is not None: + img_infos_val = [(idx, val) for idx, val in img_infos_val if val['path'] in allowed_name_subset] + + if len(img_infos_val) > num_images: + img_infos_val = sorted(img_infos_val, key=lambda x: x[1]['frame_id']) + kept_idx = np.round(np.linspace(0, len(img_infos_val) - 1, num_images)).astype(int).tolist() + img_infos_val = [img_infos_val[idx] for idx in kept_idx] + return {idx: val for idx, val in img_infos_val} + + +def undistort_images(intrinsics, rgb, mask): + camera_type = intrinsics[0] + + width = int(intrinsics[1]) + height = int(intrinsics[2]) + fx = intrinsics[3] + fy = intrinsics[4] + cx = intrinsics[5] + cy = intrinsics[6] + distortion = np.array(intrinsics[7:]) + + K = np.zeros([3, 3]) + K[0, 0] = fx + K[0, 2] = cx + K[1, 1] = fy + K[1, 2] = cy + K[2, 2] = 1 + + K = geometry.colmap_to_opencv_intrinsics(K) + if camera_type == "OPENCV_FISHEYE": + assert len(distortion) == 4 + + new_K = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify( + K, + distortion, + (width, height), + np.eye(3), + balance=0.0, + ) + # Make the cx and cy to be the center of the image + new_K[0, 2] = width / 2.0 + new_K[1, 2] = height / 2.0 + + map1, map2 = cv2.fisheye.initUndistortRectifyMap(K, distortion, np.eye(3), new_K, (width, height), cv2.CV_32FC1) + else: + new_K, _ = cv2.getOptimalNewCameraMatrix(K, distortion, (width, height), 1, (width, height), True) + map1, map2 = cv2.initUndistortRectifyMap(K, distortion, np.eye(3), new_K, (width, height), cv2.CV_32FC1) + + undistorted_image = cv2.remap(rgb, map1, map2, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT_101) + undistorted_mask = cv2.remap(mask, map1, map2, interpolation=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, borderValue=255) + K = geometry.opencv_to_colmap_intrinsics(K) + return width, height, new_K, undistorted_image, undistorted_mask + + +def process_scenes(root, pairsdir, output_dir, target_resolution): + os.makedirs(output_dir, exist_ok=True) + + # default values from + # https://github.com/scannetpp/scannetpp/blob/main/common/configs/render.yml + znear = 0.05 + zfar = 20.0 + + listfile = osp.join(pairsdir, 'scene_list.json') + with open(listfile, 'r') as f: + scenes = json.load(f) + + # for each of these, we will select some dslr images and some iphone images + # we will undistort them and render their depth + renderer = pyrender.OffscreenRenderer(0, 0) + for scene in tqdm(scenes, position=0, leave=True): + data_dir = os.path.join(root, 'data', scene) + dir_dslr = os.path.join(data_dir, 'dslr') + dir_iphone = os.path.join(data_dir, 'iphone') + dir_scans = os.path.join(data_dir, 'scans') + + assert os.path.isdir(data_dir) and os.path.isdir(dir_dslr) \ + and os.path.isdir(dir_iphone) and os.path.isdir(dir_scans) + + output_dir_scene = os.path.join(output_dir, scene) + scene_metadata_path = osp.join(output_dir_scene, 'scene_metadata.npz') + if osp.isfile(scene_metadata_path): + continue + + pairs_dir_scene = os.path.join(pairsdir, scene) + pairs_dir_scene_selected_pairs = os.path.join(pairs_dir_scene, 'selected_pairs.npz') + assert osp.isfile(pairs_dir_scene_selected_pairs) + selected_npz = np.load(pairs_dir_scene_selected_pairs) + selection, pairs = selected_npz['selection'], selected_npz['pairs'] + + # set up the output paths + output_dir_scene_rgb = os.path.join(output_dir_scene, 'images') + output_dir_scene_depth = os.path.join(output_dir_scene, 'depth') + os.makedirs(output_dir_scene_rgb, exist_ok=True) + os.makedirs(output_dir_scene_depth, exist_ok=True) + + ply_path = os.path.join(dir_scans, 'mesh_aligned_0.05.ply') + + sfm_dir_dslr = os.path.join(dir_dslr, 'colmap') + rgb_dir_dslr = os.path.join(dir_dslr, 'resized_images') + mask_dir_dslr = os.path.join(dir_dslr, 'resized_anon_masks') + + sfm_dir_iphone = os.path.join(dir_iphone, 'colmap') + rgb_dir_iphone = os.path.join(dir_iphone, 'rgb') + mask_dir_iphone = os.path.join(dir_iphone, 'rgb_masks') + + # load the mesh + with open(ply_path, 'rb') as f: + mesh_kwargs = trimesh.exchange.ply.load_ply(f) + mesh_scene = trimesh.Trimesh(**mesh_kwargs) + + # read colmap reconstruction, we will only use the intrinsics and pose here + img_idx_dslr, img_infos_dslr, points3D_dslr, observations_dslr = load_sfm(sfm_dir_dslr, cam_type='dslr') + dslr_paths = { + "in_colmap": sfm_dir_dslr, + "in_rgb": rgb_dir_dslr, + "in_mask": mask_dir_dslr, + } + + img_idx_iphone, img_infos_iphone, points3D_iphone, observations_iphone = load_sfm( + sfm_dir_iphone, cam_type='iphone') + iphone_paths = { + "in_colmap": sfm_dir_iphone, + "in_rgb": rgb_dir_iphone, + "in_mask": mask_dir_iphone, + } + + mesh = pyrender.Mesh.from_trimesh(mesh_scene, smooth=False) + pyrender_scene = pyrender.Scene() + pyrender_scene.add(mesh) + + selection_dslr = [imgname + '.JPG' for imgname in selection if imgname.startswith('DSC')] + selection_iphone = [imgname + '.jpg' for imgname in selection if imgname.startswith('frame_')] + + # resize the image to a more manageable size and render depth + for selection_cam, img_idx, img_infos, paths_data in [(selection_dslr, img_idx_dslr, img_infos_dslr, dslr_paths), + (selection_iphone, img_idx_iphone, img_infos_iphone, iphone_paths)]: + rgb_dir = paths_data['in_rgb'] + mask_dir = paths_data['in_mask'] + for imgname in tqdm(selection_cam, position=1, leave=False): + imgidx = img_idx[imgname] + img_infos_idx = img_infos[imgidx] + rgb = np.array(Image.open(os.path.join(rgb_dir, img_infos_idx['path']))) + mask = np.array(Image.open(os.path.join(mask_dir, img_infos_idx['path'][:-3] + 'png'))) + + _, _, K, rgb, mask = undistort_images(img_infos_idx['intrinsics'], rgb, mask) + + # rescale_image_depthmap assumes opencv intrinsics + intrinsics = geometry.colmap_to_opencv_intrinsics(K) + image, mask, intrinsics = rescale_image_depthmap( + rgb, mask, intrinsics, (target_resolution, target_resolution * 3.0 / 4)) + + W, H = image.size + intrinsics = geometry.opencv_to_colmap_intrinsics(intrinsics) + + # update inpace img_infos_idx + img_infos_idx['intrinsics'] = intrinsics + rgb_outpath = os.path.join(output_dir_scene_rgb, img_infos_idx['path'][:-3] + 'jpg') + image.save(rgb_outpath) + + depth_outpath = os.path.join(output_dir_scene_depth, img_infos_idx['path'][:-3] + 'png') + # render depth image + renderer.viewport_width, renderer.viewport_height = W, H + fx, fy, cx, cy = intrinsics[0, 0], intrinsics[1, 1], intrinsics[0, 2], intrinsics[1, 2] + camera = pyrender.camera.IntrinsicsCamera(fx, fy, cx, cy, znear=znear, zfar=zfar) + camera_node = pyrender_scene.add(camera, pose=img_infos_idx['cam_to_world'] @ OPENGL_TO_OPENCV) + + depth = renderer.render(pyrender_scene, flags=pyrender.RenderFlags.DEPTH_ONLY) + pyrender_scene.remove_node(camera_node) # dont forget to remove camera + + depth = (depth * 1000).astype('uint16') + # invalidate depth from mask before saving + depth_mask = (mask < 255) + depth[depth_mask] = 0 + Image.fromarray(depth).save(depth_outpath) + + trajectories = [] + intrinsics = [] + for imgname in selection: + if imgname.startswith('DSC'): + imgidx = img_idx_dslr[imgname + '.JPG'] + img_infos_idx = img_infos_dslr[imgidx] + elif imgname.startswith('frame_'): + imgidx = img_idx_iphone[imgname + '.jpg'] + img_infos_idx = img_infos_iphone[imgidx] + else: + raise ValueError('invalid image name') + + intrinsics.append(img_infos_idx['intrinsics']) + trajectories.append(img_infos_idx['cam_to_world']) + + intrinsics = np.stack(intrinsics, axis=0) + trajectories = np.stack(trajectories, axis=0) + # save metadata for this scene + np.savez(scene_metadata_path, + trajectories=trajectories, + intrinsics=intrinsics, + images=selection, + pairs=pairs) + + del img_infos + del pyrender_scene + + # concat all scene_metadata.npz into a single file + scene_data = {} + for scene_subdir in scenes: + scene_metadata_path = osp.join(output_dir, scene_subdir, 'scene_metadata.npz') + with np.load(scene_metadata_path) as data: + trajectories = data['trajectories'] + intrinsics = data['intrinsics'] + images = data['images'] + pairs = data['pairs'] + scene_data[scene_subdir] = {'trajectories': trajectories, + 'intrinsics': intrinsics, + 'images': images, + 'pairs': pairs} + + offset = 0 + counts = [] + scenes = [] + sceneids = [] + images = [] + intrinsics = [] + trajectories = [] + pairs = [] + for scene_idx, (scene_subdir, data) in enumerate(scene_data.items()): + num_imgs = data['images'].shape[0] + img_pairs = data['pairs'] + + scenes.append(scene_subdir) + sceneids.extend([scene_idx] * num_imgs) + + images.append(data['images']) + + intrinsics.append(data['intrinsics']) + trajectories.append(data['trajectories']) + + # offset pairs + img_pairs[:, 0:2] += offset + pairs.append(img_pairs) + counts.append(offset) + + offset += num_imgs + + images = np.concatenate(images, axis=0) + intrinsics = np.concatenate(intrinsics, axis=0) + trajectories = np.concatenate(trajectories, axis=0) + pairs = np.concatenate(pairs, axis=0) + np.savez(osp.join(output_dir, 'all_metadata.npz'), + counts=counts, + scenes=scenes, + sceneids=sceneids, + images=images, + intrinsics=intrinsics, + trajectories=trajectories, + pairs=pairs) + print('all done') + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + if args.pyopengl_platform.strip(): + os.environ['PYOPENGL_PLATFORM'] = args.pyopengl_platform + process_scenes(args.scannetpp_dir, args.precomputed_pairs, args.output_dir, args.target_resolution) diff --git a/src/mast3r_src/dust3r/datasets_preprocess/preprocess_staticthings3d.py b/src/mast3r_src/dust3r/datasets_preprocess/preprocess_staticthings3d.py new file mode 100644 index 0000000000000000000000000000000000000000..ee3eec16321c14b12291699f1fee492b5a7d8b1c --- /dev/null +++ b/src/mast3r_src/dust3r/datasets_preprocess/preprocess_staticthings3d.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Preprocessing code for the StaticThings3D dataset +# dataset at https://github.com/lmb-freiburg/robustmvd/blob/master/rmvd/data/README.md#staticthings3d +# 1) Download StaticThings3D in /path/to/StaticThings3D/ +# with the script at https://github.com/lmb-freiburg/robustmvd/blob/master/rmvd/data/scripts/download_staticthings3d.sh +# --> depths.tar.bz2 frames_finalpass.tar.bz2 poses.tar.bz2 frames_cleanpass.tar.bz2 intrinsics.tar.bz2 +# 2) unzip everything in the same /path/to/StaticThings3D/ directory +# 5) python datasets_preprocess/preprocess_staticthings3d.py --StaticThings3D_dir /path/to/tmp/StaticThings3D/ +# -------------------------------------------------------- +import os +import os.path as osp +import re +from tqdm import tqdm +import numpy as np +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +import cv2 + +import path_to_root # noqa +from dust3r.utils.parallel import parallel_threads +from dust3r.datasets.utils import cropping # noqa + + +def get_parser(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--StaticThings3D_dir', required=True) + parser.add_argument('--precomputed_pairs', required=True) + parser.add_argument('--output_dir', default='data/staticthings3d_processed') + return parser + + +def main(db_root, pairs_path, output_dir): + all_scenes = _list_all_scenes(db_root) + + # crop images + args = [(db_root, osp.join(split, subsplit, seq), camera, f'{n:04d}', output_dir) + for split, subsplit, seq in all_scenes for camera in ['left', 'right'] for n in range(6, 16)] + parallel_threads(load_crop_and_save, args, star_args=True, front_num=1) + + # verify that all images are there + CAM = {b'l': 'left', b'r': 'right'} + pairs = np.load(pairs_path) + for scene, seq, cam1, im1, cam2, im2 in tqdm(pairs): + seq_path = osp.join('TRAIN', scene.decode('ascii'), f'{seq:04d}') + for cam, idx in [(CAM[cam1], im1), (CAM[cam2], im2)]: + for ext in ['clean', 'final']: + impath = osp.join(output_dir, seq_path, cam, f"{idx:04n}_{ext}.jpg") + assert osp.isfile(impath), f'missing an image at {impath=}' + + print(f'>> Saved all data to {output_dir}!') + + +def load_crop_and_save(db_root, relpath_, camera, num, out_dir): + relpath = osp.join(relpath_, camera, num) + if osp.isfile(osp.join(out_dir, relpath + '.npz')): + return + os.makedirs(osp.join(out_dir, relpath_, camera), exist_ok=True) + + # load everything + intrinsics_in = readFloat(osp.join(db_root, 'intrinsics', relpath_, num + '.float3')) + cam2world = np.linalg.inv(readFloat(osp.join(db_root, 'poses', relpath + '.float3'))) + depthmap_in = readFloat(osp.join(db_root, 'depths', relpath + '.float3')) + img_clean = cv2.cvtColor(cv2.imread(osp.join(db_root, 'frames_cleanpass', + relpath + '.png'), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB) + img_final = cv2.cvtColor(cv2.imread(osp.join(db_root, 'frames_finalpass', + relpath + '.png'), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB) + + # do the crop + assert img_clean.shape[:2] == (540, 960) + assert img_final.shape[:2] == (540, 960) + (clean_out, final_out), depthmap, intrinsics_out, R_in2out = _crop_image( + intrinsics_in, (img_clean, img_final), depthmap_in, (512, 384)) + + # write everything + clean_out.save(osp.join(out_dir, relpath + '_clean.jpg'), quality=80) + final_out.save(osp.join(out_dir, relpath + '_final.jpg'), quality=80) + cv2.imwrite(osp.join(out_dir, relpath + '.exr'), depthmap) + + # New camera parameters + cam2world[:3, :3] = cam2world[:3, :3] @ R_in2out.T + np.savez(osp.join(out_dir, relpath + '.npz'), intrinsics=intrinsics_out, cam2world=cam2world) + + +def _crop_image(intrinsics_in, color_image_in, depthmap_in, resolution_out=(512, 512)): + image, depthmap, intrinsics_out = cropping.rescale_image_depthmap( + color_image_in, depthmap_in, intrinsics_in, resolution_out) + R_in2out = np.eye(3) + return image, depthmap, intrinsics_out, R_in2out + + +def _list_all_scenes(path): + print('>> Listing all scenes') + + res = [] + for split in ['TRAIN']: + for subsplit in 'ABC': + for seq in os.listdir(osp.join(path, 'intrinsics', split, subsplit)): + res.append((split, subsplit, seq)) + print(f' (found ({len(res)}) scenes)') + assert res, f'Did not find anything at {path=}' + return res + + +def readFloat(name): + with open(name, 'rb') as f: + if (f.readline().decode("utf-8")) != 'float\n': + raise Exception('float file %s did not contain keyword' % name) + + dim = int(f.readline()) + + dims = [] + count = 1 + for i in range(0, dim): + d = int(f.readline()) + dims.append(d) + count *= d + + dims = list(reversed(dims)) + data = np.fromfile(f, np.float32, count).reshape(dims) + return data # Hxw or CxHxW NxCxHxW + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + main(args.StaticThings3D_dir, args.precomputed_pairs, args.output_dir) diff --git a/src/mast3r_src/dust3r/datasets_preprocess/preprocess_waymo.py b/src/mast3r_src/dust3r/datasets_preprocess/preprocess_waymo.py new file mode 100644 index 0000000000000000000000000000000000000000..203f337330a7e06e61d2fb9dd99647063967922d --- /dev/null +++ b/src/mast3r_src/dust3r/datasets_preprocess/preprocess_waymo.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Preprocessing code for the WayMo Open dataset +# dataset at https://github.com/waymo-research/waymo-open-dataset +# 1) Accept the license +# 2) download all training/*.tfrecord files from Perception Dataset, version 1.4.2 +# 3) put all .tfrecord files in '/path/to/waymo_dir' +# 4) install the waymo_open_dataset package with +# `python3 -m pip install gcsfs waymo-open-dataset-tf-2-12-0==1.6.4` +# 5) execute this script as `python preprocess_waymo.py --waymo_dir /path/to/waymo_dir` +# -------------------------------------------------------- +import sys +import os +import os.path as osp +import shutil +import json +from tqdm import tqdm +import PIL.Image +import numpy as np +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +import cv2 + +import tensorflow.compat.v1 as tf +tf.enable_eager_execution() + +import path_to_root # noqa +from dust3r.utils.geometry import geotrf, inv +from dust3r.utils.image import imread_cv2 +from dust3r.utils.parallel import parallel_processes as parallel_map +from dust3r.datasets.utils import cropping +from dust3r.viz import show_raw_pointcloud + + +def get_parser(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--waymo_dir', required=True) + parser.add_argument('--precomputed_pairs', required=True) + parser.add_argument('--output_dir', default='data/waymo_processed') + parser.add_argument('--workers', type=int, default=1) + return parser + + +def main(waymo_root, pairs_path, output_dir, workers=1): + extract_frames(waymo_root, output_dir, workers=workers) + make_crops(output_dir, workers=args.workers) + + # make sure all pairs are there + with np.load(pairs_path) as data: + scenes = data['scenes'] + frames = data['frames'] + pairs = data['pairs'] # (array of (scene_id, img1_id, img2_id) + + for scene_id, im1_id, im2_id in pairs: + for im_id in (im1_id, im2_id): + path = osp.join(output_dir, scenes[scene_id], frames[im_id] + '.jpg') + assert osp.isfile(path), f'Missing a file at {path=}\nDid you download all .tfrecord files?' + + shutil.rmtree(osp.join(output_dir, 'tmp')) + print('Done! all data generated at', output_dir) + + +def _list_sequences(db_root): + print('>> Looking for sequences in', db_root) + res = sorted(f for f in os.listdir(db_root) if f.endswith('.tfrecord')) + print(f' found {len(res)} sequences') + return res + + +def extract_frames(db_root, output_dir, workers=8): + sequences = _list_sequences(db_root) + output_dir = osp.join(output_dir, 'tmp') + print('>> outputing result to', output_dir) + args = [(db_root, output_dir, seq) for seq in sequences] + parallel_map(process_one_seq, args, star_args=True, workers=workers) + + +def process_one_seq(db_root, output_dir, seq): + out_dir = osp.join(output_dir, seq) + os.makedirs(out_dir, exist_ok=True) + calib_path = osp.join(out_dir, 'calib.json') + if osp.isfile(calib_path): + return + + try: + with tf.device('/CPU:0'): + calib, frames = extract_frames_one_seq(osp.join(db_root, seq)) + except RuntimeError: + print(f'/!\\ Error with sequence {seq} /!\\', file=sys.stderr) + return # nothing is saved + + for f, (frame_name, views) in enumerate(tqdm(frames, leave=False)): + for cam_idx, view in views.items(): + img = PIL.Image.fromarray(view.pop('img')) + img.save(osp.join(out_dir, f'{f:05d}_{cam_idx}.jpg')) + np.savez(osp.join(out_dir, f'{f:05d}_{cam_idx}.npz'), **view) + + with open(calib_path, 'w') as f: + json.dump(calib, f) + + +def extract_frames_one_seq(filename): + from waymo_open_dataset import dataset_pb2 as open_dataset + from waymo_open_dataset.utils import frame_utils + + print('>> Opening', filename) + dataset = tf.data.TFRecordDataset(filename, compression_type='') + + calib = None + frames = [] + + for data in tqdm(dataset, leave=False): + frame = open_dataset.Frame() + frame.ParseFromString(bytearray(data.numpy())) + + content = frame_utils.parse_range_image_and_camera_projection(frame) + range_images, camera_projections, _, range_image_top_pose = content + + views = {} + frames.append((frame.context.name, views)) + + # once in a sequence, read camera calibration info + if calib is None: + calib = [] + for cam in frame.context.camera_calibrations: + calib.append((cam.name, + dict(width=cam.width, + height=cam.height, + intrinsics=list(cam.intrinsic), + extrinsics=list(cam.extrinsic.transform)))) + + # convert LIDAR to pointcloud + points, cp_points = frame_utils.convert_range_image_to_point_cloud( + frame, + range_images, + camera_projections, + range_image_top_pose) + + # 3d points in vehicle frame. + points_all = np.concatenate(points, axis=0) + cp_points_all = np.concatenate(cp_points, axis=0) + + # The distance between lidar points and vehicle frame origin. + cp_points_all_tensor = tf.constant(cp_points_all, dtype=tf.int32) + + for i, image in enumerate(frame.images): + # select relevant 3D points for this view + mask = tf.equal(cp_points_all_tensor[..., 0], image.name) + cp_points_msk_tensor = tf.cast(tf.gather_nd(cp_points_all_tensor, tf.where(mask)), dtype=tf.float32) + + pose = np.asarray(image.pose.transform).reshape(4, 4) + timestamp = image.pose_timestamp + + rgb = tf.image.decode_jpeg(image.image).numpy() + + pix = cp_points_msk_tensor[..., 1:3].numpy().round().astype(np.int16) + pts3d = points_all[mask.numpy()] + + views[image.name] = dict(img=rgb, pose=pose, pixels=pix, pts3d=pts3d, timestamp=timestamp) + + if not 'show full point cloud': + show_raw_pointcloud([v['pts3d'] for v in views.values()], [v['img'] for v in views.values()]) + + return calib, frames + + +def make_crops(output_dir, workers=16, **kw): + tmp_dir = osp.join(output_dir, 'tmp') + sequences = _list_sequences(tmp_dir) + args = [(tmp_dir, output_dir, seq) for seq in sequences] + parallel_map(crop_one_seq, args, star_args=True, workers=workers, front_num=0) + + +def crop_one_seq(input_dir, output_dir, seq, resolution=512): + seq_dir = osp.join(input_dir, seq) + out_dir = osp.join(output_dir, seq) + if osp.isfile(osp.join(out_dir, '00100_1.jpg')): + return + os.makedirs(out_dir, exist_ok=True) + + # load calibration file + try: + with open(osp.join(seq_dir, 'calib.json')) as f: + calib = json.load(f) + except IOError: + print(f'/!\\ Error: Missing calib.json in sequence {seq} /!\\', file=sys.stderr) + return + + axes_transformation = np.array([ + [0, -1, 0, 0], + [0, 0, -1, 0], + [1, 0, 0, 0], + [0, 0, 0, 1]]) + + cam_K = {} + cam_distortion = {} + cam_res = {} + cam_to_car = {} + for cam_idx, cam_info in calib: + cam_idx = str(cam_idx) + cam_res[cam_idx] = (W, H) = (cam_info['width'], cam_info['height']) + f1, f2, cx, cy, k1, k2, p1, p2, k3 = cam_info['intrinsics'] + cam_K[cam_idx] = np.asarray([(f1, 0, cx), (0, f2, cy), (0, 0, 1)]) + cam_distortion[cam_idx] = np.asarray([k1, k2, p1, p2, k3]) + cam_to_car[cam_idx] = np.asarray(cam_info['extrinsics']).reshape(4, 4) # cam-to-vehicle + + frames = sorted(f[:-3] for f in os.listdir(seq_dir) if f.endswith('.jpg')) + + # from dust3r.viz import SceneViz + # viz = SceneViz() + + for frame in tqdm(frames, leave=False): + cam_idx = frame[-2] # cam index + assert cam_idx in '12345', f'bad {cam_idx=} in {frame=}' + data = np.load(osp.join(seq_dir, frame + 'npz')) + car_to_world = data['pose'] + W, H = cam_res[cam_idx] + + # load depthmap + pos2d = data['pixels'].round().astype(np.uint16) + x, y = pos2d.T + pts3d = data['pts3d'] # already in the car frame + pts3d = geotrf(axes_transformation @ inv(cam_to_car[cam_idx]), pts3d) + # X=LEFT_RIGHT y=ALTITUDE z=DEPTH + + # load image + image = imread_cv2(osp.join(seq_dir, frame + 'jpg')) + + # downscale image + output_resolution = (resolution, 1) if W > H else (1, resolution) + image, _, intrinsics2 = cropping.rescale_image_depthmap(image, None, cam_K[cam_idx], output_resolution) + image.save(osp.join(out_dir, frame + 'jpg'), quality=80) + + # save as an EXR file? yes it's smaller (and easier to load) + W, H = image.size + depthmap = np.zeros((H, W), dtype=np.float32) + pos2d = geotrf(intrinsics2 @ inv(cam_K[cam_idx]), pos2d).round().astype(np.int16) + x, y = pos2d.T + depthmap[y.clip(min=0, max=H - 1), x.clip(min=0, max=W - 1)] = pts3d[:, 2] + cv2.imwrite(osp.join(out_dir, frame + 'exr'), depthmap) + + # save camera parametes + cam2world = car_to_world @ cam_to_car[cam_idx] @ inv(axes_transformation) + np.savez(osp.join(out_dir, frame + 'npz'), intrinsics=intrinsics2, + cam2world=cam2world, distortion=cam_distortion[cam_idx]) + + # viz.add_rgbd(np.asarray(image), depthmap, intrinsics2, cam2world) + # viz.show() + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + main(args.waymo_dir, args.precomputed_pairs, args.output_dir, workers=args.workers) diff --git a/src/mast3r_src/dust3r/datasets_preprocess/preprocess_wildrgbd.py b/src/mast3r_src/dust3r/datasets_preprocess/preprocess_wildrgbd.py new file mode 100644 index 0000000000000000000000000000000000000000..ff3f0f7abb7d9ef43bba6a7c6cd6f4e652a8f510 --- /dev/null +++ b/src/mast3r_src/dust3r/datasets_preprocess/preprocess_wildrgbd.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Script to pre-process the WildRGB-D dataset. +# Usage: +# python3 datasets_preprocess/preprocess_wildrgbd.py --wildrgbd_dir /path/to/wildrgbd +# -------------------------------------------------------- + +import argparse +import random +import json +import os +import os.path as osp + +import PIL.Image +import numpy as np +import cv2 + +from tqdm.auto import tqdm +import matplotlib.pyplot as plt + +import path_to_root # noqa +import dust3r.datasets.utils.cropping as cropping # noqa +from dust3r.utils.image import imread_cv2 + + +def get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("--output_dir", type=str, default="data/wildrgbd_processed") + parser.add_argument("--wildrgbd_dir", type=str, required=True) + parser.add_argument("--train_num_sequences_per_object", type=int, default=50) + parser.add_argument("--test_num_sequences_per_object", type=int, default=10) + parser.add_argument("--num_frames", type=int, default=100) + parser.add_argument("--seed", type=int, default=42) + + parser.add_argument("--img_size", type=int, default=512, + help=("lower dimension will be >= img_size * 3/4, and max dimension will be >= img_size")) + return parser + + +def get_set_list(category_dir, split): + listfiles = ["camera_eval_list.json", "nvs_list.json"] + + sequences_all = {s: {k: set() for k in listfiles} for s in ['train', 'val']} + for listfile in listfiles: + with open(osp.join(category_dir, listfile)) as f: + subset_lists_data = json.load(f) + for s in ['train', 'val']: + sequences_all[s][listfile].update(subset_lists_data[s]) + train_intersection = set.intersection(*list(sequences_all['train'].values())) + if split == "train": + return train_intersection + else: + all_seqs = set.union(*list(sequences_all['train'].values()), *list(sequences_all['val'].values())) + return all_seqs.difference(train_intersection) + + +def prepare_sequences(category, wildrgbd_dir, output_dir, img_size, split, max_num_sequences_per_object, + output_num_frames, seed): + random.seed(seed) + category_dir = osp.join(wildrgbd_dir, category) + category_output_dir = osp.join(output_dir, category) + sequences_all = get_set_list(category_dir, split) + sequences_all = sorted(sequences_all) + + sequences_all_tmp = [] + for seq_name in sequences_all: + scene_dir = osp.join(wildrgbd_dir, category_dir, seq_name) + if not os.path.isdir(scene_dir): + print(f'{scene_dir} does not exist, skipped') + continue + sequences_all_tmp.append(seq_name) + sequences_all = sequences_all_tmp + if len(sequences_all) <= max_num_sequences_per_object: + selected_sequences = sequences_all + else: + selected_sequences = random.sample(sequences_all, max_num_sequences_per_object) + + selected_sequences_numbers_dict = {} + for seq_name in tqdm(selected_sequences, leave=False): + scene_dir = osp.join(category_dir, seq_name) + scene_output_dir = osp.join(category_output_dir, seq_name) + with open(osp.join(scene_dir, 'metadata'), 'r') as f: + metadata = json.load(f) + + K = np.array(metadata["K"]).reshape(3, 3).T + fx, fy, cx, cy = K[0, 0], K[1, 1], K[0, 2], K[1, 2] + w, h = metadata["w"], metadata["h"] + + camera_intrinsics = np.array( + [[fx, 0, cx], + [0, fy, cy], + [0, 0, 1]] + ) + camera_to_world_path = os.path.join(scene_dir, 'cam_poses.txt') + camera_to_world_content = np.genfromtxt(camera_to_world_path) + camera_to_world = camera_to_world_content[:, 1:].reshape(-1, 4, 4) + + frame_idx = camera_to_world_content[:, 0] + num_frames = frame_idx.shape[0] + assert num_frames >= output_num_frames + assert np.all(frame_idx == np.arange(num_frames)) + + # selected_sequences_numbers_dict[seq_name] = num_frames + + selected_frames = np.round(np.linspace(0, num_frames - 1, output_num_frames)).astype(int).tolist() + selected_sequences_numbers_dict[seq_name] = selected_frames + + for frame_id in tqdm(selected_frames): + depth_path = os.path.join(scene_dir, 'depth', f'{frame_id:0>5d}.png') + masks_path = os.path.join(scene_dir, 'masks', f'{frame_id:0>5d}.png') + rgb_path = os.path.join(scene_dir, 'rgb', f'{frame_id:0>5d}.png') + + input_rgb_image = PIL.Image.open(rgb_path).convert('RGB') + input_mask = plt.imread(masks_path) + input_depthmap = imread_cv2(depth_path, cv2.IMREAD_UNCHANGED).astype(np.float64) + depth_mask = np.stack((input_depthmap, input_mask), axis=-1) + H, W = input_depthmap.shape + + min_margin_x = min(cx, W - cx) + min_margin_y = min(cy, H - cy) + + # the new window will be a rectangle of size (2*min_margin_x, 2*min_margin_y) centered on (cx,cy) + l, t = int(cx - min_margin_x), int(cy - min_margin_y) + r, b = int(cx + min_margin_x), int(cy + min_margin_y) + crop_bbox = (l, t, r, b) + input_rgb_image, depth_mask, input_camera_intrinsics = cropping.crop_image_depthmap( + input_rgb_image, depth_mask, camera_intrinsics, crop_bbox) + + # try to set the lower dimension to img_size * 3/4 -> img_size=512 => 384 + scale_final = ((img_size * 3 // 4) / min(H, W)) + 1e-8 + output_resolution = np.floor(np.array([W, H]) * scale_final).astype(int) + if max(output_resolution) < img_size: + # let's put the max dimension to img_size + scale_final = (img_size / max(H, W)) + 1e-8 + output_resolution = np.floor(np.array([W, H]) * scale_final).astype(int) + + input_rgb_image, depth_mask, input_camera_intrinsics = cropping.rescale_image_depthmap( + input_rgb_image, depth_mask, input_camera_intrinsics, output_resolution) + input_depthmap = depth_mask[:, :, 0] + input_mask = depth_mask[:, :, 1] + + camera_pose = camera_to_world[frame_id] + + # save crop images and depth, metadata + save_img_path = os.path.join(scene_output_dir, 'rgb', f'{frame_id:0>5d}.jpg') + save_depth_path = os.path.join(scene_output_dir, 'depth', f'{frame_id:0>5d}.png') + save_mask_path = os.path.join(scene_output_dir, 'masks', f'{frame_id:0>5d}.png') + os.makedirs(os.path.split(save_img_path)[0], exist_ok=True) + os.makedirs(os.path.split(save_depth_path)[0], exist_ok=True) + os.makedirs(os.path.split(save_mask_path)[0], exist_ok=True) + + input_rgb_image.save(save_img_path) + cv2.imwrite(save_depth_path, input_depthmap.astype(np.uint16)) + cv2.imwrite(save_mask_path, (input_mask * 255).astype(np.uint8)) + + save_meta_path = os.path.join(scene_output_dir, 'metadata', f'{frame_id:0>5d}.npz') + os.makedirs(os.path.split(save_meta_path)[0], exist_ok=True) + np.savez(save_meta_path, camera_intrinsics=input_camera_intrinsics, + camera_pose=camera_pose) + + return selected_sequences_numbers_dict + + +if __name__ == "__main__": + parser = get_parser() + args = parser.parse_args() + assert args.wildrgbd_dir != args.output_dir + + categories = sorted([ + dirname for dirname in os.listdir(args.wildrgbd_dir) + if os.path.isdir(os.path.join(args.wildrgbd_dir, dirname, 'scenes')) + ]) + + os.makedirs(args.output_dir, exist_ok=True) + + splits_num_sequences_per_object = [args.train_num_sequences_per_object, args.test_num_sequences_per_object] + for split, num_sequences_per_object in zip(['train', 'test'], splits_num_sequences_per_object): + selected_sequences_path = os.path.join(args.output_dir, f'selected_seqs_{split}.json') + if os.path.isfile(selected_sequences_path): + continue + all_selected_sequences = {} + for category in categories: + category_output_dir = osp.join(args.output_dir, category) + os.makedirs(category_output_dir, exist_ok=True) + category_selected_sequences_path = os.path.join(category_output_dir, f'selected_seqs_{split}.json') + if os.path.isfile(category_selected_sequences_path): + with open(category_selected_sequences_path, 'r') as fid: + category_selected_sequences = json.load(fid) + else: + print(f"Processing {split} - category = {category}") + category_selected_sequences = prepare_sequences( + category=category, + wildrgbd_dir=args.wildrgbd_dir, + output_dir=args.output_dir, + img_size=args.img_size, + split=split, + max_num_sequences_per_object=num_sequences_per_object, + output_num_frames=args.num_frames, + seed=args.seed + int("category".encode('ascii').hex(), 16), + ) + with open(category_selected_sequences_path, 'w') as file: + json.dump(category_selected_sequences, file) + + all_selected_sequences[category] = category_selected_sequences + with open(selected_sequences_path, 'w') as file: + json.dump(all_selected_sequences, file) diff --git a/src/mast3r_src/dust3r/demo.py b/src/mast3r_src/dust3r/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..3c6d6a9cd8b2687be0a19c7b8a43942633d74310 --- /dev/null +++ b/src/mast3r_src/dust3r/demo.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# dust3r gradio demo executable +# -------------------------------------------------------- +import os +import torch +import tempfile + +from dust3r.model import AsymmetricCroCo3DStereo +from dust3r.demo import get_args_parser, main_demo + +import matplotlib.pyplot as pl +pl.ion() + +torch.backends.cuda.matmul.allow_tf32 = True # for gpu >= Ampere and pytorch >= 1.12 + +if __name__ == '__main__': + parser = get_args_parser() + args = parser.parse_args() + + if args.tmp_dir is not None: + tmp_path = args.tmp_dir + os.makedirs(tmp_path, exist_ok=True) + tempfile.tempdir = tmp_path + + if args.server_name is not None: + server_name = args.server_name + else: + server_name = '0.0.0.0' if args.local_network else '127.0.0.1' + + if args.weights is not None: + weights_path = args.weights + else: + weights_path = "naver/" + args.model_name + model = AsymmetricCroCo3DStereo.from_pretrained(weights_path).to(args.device) + + # dust3r will write the 3D model inside tmpdirname + with tempfile.TemporaryDirectory(suffix='dust3r_gradio_demo') as tmpdirname: + if not args.silent: + print('Outputing stuff in', tmpdirname) + main_demo(tmpdirname, model, args.device, args.image_size, server_name, args.server_port, silent=args.silent) diff --git a/src/mast3r_src/dust3r/docker/docker-compose-cpu.yml b/src/mast3r_src/dust3r/docker/docker-compose-cpu.yml new file mode 100644 index 0000000000000000000000000000000000000000..2015fd771e8b6246d288c03a38f6fbb3f17dff20 --- /dev/null +++ b/src/mast3r_src/dust3r/docker/docker-compose-cpu.yml @@ -0,0 +1,16 @@ +version: '3.8' +services: + dust3r-demo: + build: + context: ./files + dockerfile: cpu.Dockerfile + ports: + - "7860:7860" + volumes: + - ./files/checkpoints:/dust3r/checkpoints + environment: + - DEVICE=cpu + - MODEL=${MODEL:-DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth} + cap_add: + - IPC_LOCK + - SYS_RESOURCE diff --git a/src/mast3r_src/dust3r/docker/docker-compose-cuda.yml b/src/mast3r_src/dust3r/docker/docker-compose-cuda.yml new file mode 100644 index 0000000000000000000000000000000000000000..85710af953d669fe618273de6ce3a062a7a84cca --- /dev/null +++ b/src/mast3r_src/dust3r/docker/docker-compose-cuda.yml @@ -0,0 +1,23 @@ +version: '3.8' +services: + dust3r-demo: + build: + context: ./files + dockerfile: cuda.Dockerfile + ports: + - "7860:7860" + environment: + - DEVICE=cuda + - MODEL=${MODEL:-DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth} + volumes: + - ./files/checkpoints:/dust3r/checkpoints + cap_add: + - IPC_LOCK + - SYS_RESOURCE + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] diff --git a/src/mast3r_src/dust3r/docker/files/cpu.Dockerfile b/src/mast3r_src/dust3r/docker/files/cpu.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..c9ccc39682dd7c7723f447ff47f12531a593446f --- /dev/null +++ b/src/mast3r_src/dust3r/docker/files/cpu.Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.11-slim + +LABEL description="Docker container for DUSt3R with dependencies installed. CPU VERSION" + +ENV DEVICE="cpu" +ENV MODEL="DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth" +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + git \ + libgl1-mesa-glx \ + libegl1-mesa \ + libxrandr2 \ + libxrandr2 \ + libxss1 \ + libxcursor1 \ + libxcomposite1 \ + libasound2 \ + libxi6 \ + libxtst6 \ + libglib2.0-0 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN git clone --recursive https://github.com/naver/dust3r /dust3r +WORKDIR /dust3r + +RUN pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu +RUN pip install -r requirements.txt +RUN pip install -r requirements_optional.txt +RUN pip install opencv-python==4.8.0.74 + +WORKDIR /dust3r + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/src/mast3r_src/dust3r/docker/files/cuda.Dockerfile b/src/mast3r_src/dust3r/docker/files/cuda.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..a1d2edce1a5e7cee2fa3d66faf4f6ee019595267 --- /dev/null +++ b/src/mast3r_src/dust3r/docker/files/cuda.Dockerfile @@ -0,0 +1,27 @@ +FROM nvcr.io/nvidia/pytorch:24.01-py3 + +LABEL description="Docker container for DUSt3R with dependencies installed. CUDA VERSION" +ENV DEVICE="cuda" +ENV MODEL="DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth" +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + git=1:2.34.1-1ubuntu1.10 \ + libglib2.0-0=2.72.4-0ubuntu2.2 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN git clone --recursive https://github.com/naver/dust3r /dust3r +WORKDIR /dust3r +RUN pip install -r requirements.txt +RUN pip install -r requirements_optional.txt +RUN pip install opencv-python==4.8.0.74 + +WORKDIR /dust3r/croco/models/curope/ +RUN python setup.py build_ext --inplace + +WORKDIR /dust3r +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/src/mast3r_src/dust3r/docker/files/entrypoint.sh b/src/mast3r_src/dust3r/docker/files/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..9637072a0af071f927ca0481bcaa4b600644b8b5 --- /dev/null +++ b/src/mast3r_src/dust3r/docker/files/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -eux + +DEVICE=${DEVICE:-cuda} +MODEL=${MODEL:-DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth} + +exec python3 demo.py --weights "checkpoints/$MODEL" --device "$DEVICE" --local_network "$@" diff --git a/src/mast3r_src/dust3r/docker/run.sh b/src/mast3r_src/dust3r/docker/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..6c920363d607fc6019f10780d072edf49bee3046 --- /dev/null +++ b/src/mast3r_src/dust3r/docker/run.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +set -eux + +# Default model name +model_name="DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth" + +check_docker() { + if ! command -v docker &>/dev/null; then + echo "Docker could not be found. Please install Docker and try again." + exit 1 + fi +} + +download_model_checkpoint() { + if [ -f "./files/checkpoints/${model_name}" ]; then + echo "Model checkpoint ${model_name} already exists. Skipping download." + return + fi + echo "Downloading model checkpoint ${model_name}..." + wget "https://download.europe.naverlabs.com/ComputerVision/DUSt3R/${model_name}" -P ./files/checkpoints +} + +set_dcomp() { + if command -v docker-compose &>/dev/null; then + dcomp="docker-compose" + elif command -v docker &>/dev/null && docker compose version &>/dev/null; then + dcomp="docker compose" + else + echo "Docker Compose could not be found. Please install Docker Compose and try again." + exit 1 + fi +} + +run_docker() { + export MODEL=${model_name} + if [ "$with_cuda" -eq 1 ]; then + $dcomp -f docker-compose-cuda.yml up --build + else + $dcomp -f docker-compose-cpu.yml up --build + fi +} + +with_cuda=0 +for arg in "$@"; do + case $arg in + --with-cuda) + with_cuda=1 + ;; + --model_name=*) + model_name="${arg#*=}.pth" + ;; + *) + echo "Unknown parameter passed: $arg" + exit 1 + ;; + esac +done + + +main() { + check_docker + download_model_checkpoint + set_dcomp + run_docker +} + +main diff --git a/src/mast3r_src/dust3r/dust3r/__init__.py b/src/mast3r_src/dust3r/dust3r/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a32692113d830ddc4af4e6ed608f222fbe062e6e --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). diff --git a/src/mast3r_src/dust3r/dust3r/cloud_opt/__init__.py b/src/mast3r_src/dust3r/dust3r/cloud_opt/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..faf5cd279a317c1efb9ba947682992c0949c1bdc --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/cloud_opt/__init__.py @@ -0,0 +1,33 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# global alignment optimization wrapper function +# -------------------------------------------------------- +from enum import Enum + +from .optimizer import PointCloudOptimizer +from .modular_optimizer import ModularPointCloudOptimizer +from .pair_viewer import PairViewer + + +class GlobalAlignerMode(Enum): + PointCloudOptimizer = "PointCloudOptimizer" + ModularPointCloudOptimizer = "ModularPointCloudOptimizer" + PairViewer = "PairViewer" + + +def global_aligner(dust3r_output, device, mode=GlobalAlignerMode.PointCloudOptimizer, **optim_kw): + # extract all inputs + view1, view2, pred1, pred2 = [dust3r_output[k] for k in 'view1 view2 pred1 pred2'.split()] + # build the optimizer + if mode == GlobalAlignerMode.PointCloudOptimizer: + net = PointCloudOptimizer(view1, view2, pred1, pred2, **optim_kw).to(device) + elif mode == GlobalAlignerMode.ModularPointCloudOptimizer: + net = ModularPointCloudOptimizer(view1, view2, pred1, pred2, **optim_kw).to(device) + elif mode == GlobalAlignerMode.PairViewer: + net = PairViewer(view1, view2, pred1, pred2, **optim_kw).to(device) + else: + raise NotImplementedError(f'Unknown mode {mode}') + + return net diff --git a/src/mast3r_src/dust3r/dust3r/cloud_opt/base_opt.py b/src/mast3r_src/dust3r/dust3r/cloud_opt/base_opt.py new file mode 100644 index 0000000000000000000000000000000000000000..4d36e05bfca80509bced20add7c067987d538951 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/cloud_opt/base_opt.py @@ -0,0 +1,405 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Base class for the global alignement procedure +# -------------------------------------------------------- +from copy import deepcopy + +import numpy as np +import torch +import torch.nn as nn +import roma +from copy import deepcopy +import tqdm + +from dust3r.utils.geometry import inv, geotrf +from dust3r.utils.device import to_numpy +from dust3r.utils.image import rgb +from dust3r.viz import SceneViz, segment_sky, auto_cam_size +from dust3r.optim_factory import adjust_learning_rate_by_lr + +from dust3r.cloud_opt.commons import (edge_str, ALL_DISTS, NoGradParamDict, get_imshapes, signed_expm1, signed_log1p, + cosine_schedule, linear_schedule, get_conf_trf) +import dust3r.cloud_opt.init_im_poses as init_fun + + +class BasePCOptimizer (nn.Module): + """ Optimize a global scene, given a list of pairwise observations. + Graph node: images + Graph edges: observations = (pred1, pred2) + """ + + def __init__(self, *args, **kwargs): + if len(args) == 1 and len(kwargs) == 0: + other = deepcopy(args[0]) + attrs = '''edges is_symmetrized dist n_imgs pred_i pred_j imshapes + min_conf_thr conf_thr conf_i conf_j im_conf + base_scale norm_pw_scale POSE_DIM pw_poses + pw_adaptors pw_adaptors has_im_poses rand_pose imgs verbose'''.split() + self.__dict__.update({k: other[k] for k in attrs}) + else: + self._init_from_views(*args, **kwargs) + + def _init_from_views(self, view1, view2, pred1, pred2, + dist='l1', + conf='log', + min_conf_thr=3, + base_scale=0.5, + allow_pw_adaptors=False, + pw_break=20, + rand_pose=torch.randn, + iterationsCount=None, + verbose=True): + super().__init__() + if not isinstance(view1['idx'], list): + view1['idx'] = view1['idx'].tolist() + if not isinstance(view2['idx'], list): + view2['idx'] = view2['idx'].tolist() + self.edges = [(int(i), int(j)) for i, j in zip(view1['idx'], view2['idx'])] + self.is_symmetrized = set(self.edges) == {(j, i) for i, j in self.edges} + self.dist = ALL_DISTS[dist] + self.verbose = verbose + + self.n_imgs = self._check_edges() + + # input data + pred1_pts = pred1['pts3d'] + pred2_pts = pred2['pts3d_in_other_view'] + self.pred_i = NoGradParamDict({ij: pred1_pts[n] for n, ij in enumerate(self.str_edges)}) + self.pred_j = NoGradParamDict({ij: pred2_pts[n] for n, ij in enumerate(self.str_edges)}) + self.imshapes = get_imshapes(self.edges, pred1_pts, pred2_pts) + + # work in log-scale with conf + pred1_conf = pred1['conf'] + pred2_conf = pred2['conf'] + self.min_conf_thr = min_conf_thr + self.conf_trf = get_conf_trf(conf) + + self.conf_i = NoGradParamDict({ij: pred1_conf[n] for n, ij in enumerate(self.str_edges)}) + self.conf_j = NoGradParamDict({ij: pred2_conf[n] for n, ij in enumerate(self.str_edges)}) + self.im_conf = self._compute_img_conf(pred1_conf, pred2_conf) + for i in range(len(self.im_conf)): + self.im_conf[i].requires_grad = False + + # pairwise pose parameters + self.base_scale = base_scale + self.norm_pw_scale = True + self.pw_break = pw_break + self.POSE_DIM = 7 + self.pw_poses = nn.Parameter(rand_pose((self.n_edges, 1+self.POSE_DIM))) # pairwise poses + self.pw_adaptors = nn.Parameter(torch.zeros((self.n_edges, 2))) # slight xy/z adaptation + self.pw_adaptors.requires_grad_(allow_pw_adaptors) + self.has_im_poses = False + self.rand_pose = rand_pose + + # possibly store images for show_pointcloud + self.imgs = None + if 'img' in view1 and 'img' in view2: + imgs = [torch.zeros((3,)+hw) for hw in self.imshapes] + for v in range(len(self.edges)): + idx = view1['idx'][v] + imgs[idx] = view1['img'][v] + idx = view2['idx'][v] + imgs[idx] = view2['img'][v] + self.imgs = rgb(imgs) + + @property + def n_edges(self): + return len(self.edges) + + @property + def str_edges(self): + return [edge_str(i, j) for i, j in self.edges] + + @property + def imsizes(self): + return [(w, h) for h, w in self.imshapes] + + @property + def device(self): + return next(iter(self.parameters())).device + + def state_dict(self, trainable=True): + all_params = super().state_dict() + return {k: v for k, v in all_params.items() if k.startswith(('_', 'pred_i.', 'pred_j.', 'conf_i.', 'conf_j.')) != trainable} + + def load_state_dict(self, data): + return super().load_state_dict(self.state_dict(trainable=False) | data) + + def _check_edges(self): + indices = sorted({i for edge in self.edges for i in edge}) + assert indices == list(range(len(indices))), 'bad pair indices: missing values ' + return len(indices) + + @torch.no_grad() + def _compute_img_conf(self, pred1_conf, pred2_conf): + im_conf = nn.ParameterList([torch.zeros(hw, device=self.device) for hw in self.imshapes]) + for e, (i, j) in enumerate(self.edges): + im_conf[i] = torch.maximum(im_conf[i], pred1_conf[e]) + im_conf[j] = torch.maximum(im_conf[j], pred2_conf[e]) + return im_conf + + def get_adaptors(self): + adapt = self.pw_adaptors + adapt = torch.cat((adapt[:, 0:1], adapt), dim=-1) # (scale_xy, scale_xy, scale_z) + if self.norm_pw_scale: # normalize so that the product == 1 + adapt = adapt - adapt.mean(dim=1, keepdim=True) + return (adapt / self.pw_break).exp() + + def _get_poses(self, poses): + # normalize rotation + Q = poses[:, :4] + T = signed_expm1(poses[:, 4:7]) + RT = roma.RigidUnitQuat(Q, T).normalize().to_homogeneous() + return RT + + def _set_pose(self, poses, idx, R, T=None, scale=None, force=False): + # all poses == cam-to-world + pose = poses[idx] + if not (pose.requires_grad or force): + return pose + + if R.shape == (4, 4): + assert T is None + T = R[:3, 3] + R = R[:3, :3] + + if R is not None: + pose.data[0:4] = roma.rotmat_to_unitquat(R) + if T is not None: + pose.data[4:7] = signed_log1p(T / (scale or 1)) # translation is function of scale + + if scale is not None: + assert poses.shape[-1] in (8, 13) + pose.data[-1] = np.log(float(scale)) + return pose + + def get_pw_norm_scale_factor(self): + if self.norm_pw_scale: + # normalize scales so that things cannot go south + # we want that exp(scale) ~= self.base_scale + return (np.log(self.base_scale) - self.pw_poses[:, -1].mean()).exp() + else: + return 1 # don't norm scale for known poses + + def get_pw_scale(self): + scale = self.pw_poses[:, -1].exp() # (n_edges,) + scale = scale * self.get_pw_norm_scale_factor() + return scale + + def get_pw_poses(self): # cam to world + RT = self._get_poses(self.pw_poses) + scaled_RT = RT.clone() + scaled_RT[:, :3] *= self.get_pw_scale().view(-1, 1, 1) # scale the rotation AND translation + return scaled_RT + + def get_masks(self): + return [(conf > self.min_conf_thr) for conf in self.im_conf] + + def depth_to_pts3d(self): + raise NotImplementedError() + + def get_pts3d(self, raw=False): + res = self.depth_to_pts3d() + if not raw: + res = [dm[:h*w].view(h, w, 3) for dm, (h, w) in zip(res, self.imshapes)] + return res + + def _set_focal(self, idx, focal, force=False): + raise NotImplementedError() + + def get_focals(self): + raise NotImplementedError() + + def get_known_focal_mask(self): + raise NotImplementedError() + + def get_principal_points(self): + raise NotImplementedError() + + def get_conf(self, mode=None): + trf = self.conf_trf if mode is None else get_conf_trf(mode) + return [trf(c) for c in self.im_conf] + + def get_im_poses(self): + raise NotImplementedError() + + def _set_depthmap(self, idx, depth, force=False): + raise NotImplementedError() + + def get_depthmaps(self, raw=False): + raise NotImplementedError() + + def clean_pointcloud(self, **kw): + cams = inv(self.get_im_poses()) + K = self.get_intrinsics() + depthmaps = self.get_depthmaps() + all_pts3d = self.get_pts3d() + + new_im_confs = clean_pointcloud(self.im_conf, K, cams, depthmaps, all_pts3d, **kw) + + for i, new_conf in enumerate(new_im_confs): + self.im_conf[i].data[:] = new_conf + return self + + def forward(self, ret_details=False): + pw_poses = self.get_pw_poses() # cam-to-world + pw_adapt = self.get_adaptors() + proj_pts3d = self.get_pts3d() + # pre-compute pixel weights + weight_i = {i_j: self.conf_trf(c) for i_j, c in self.conf_i.items()} + weight_j = {i_j: self.conf_trf(c) for i_j, c in self.conf_j.items()} + + loss = 0 + if ret_details: + details = -torch.ones((self.n_imgs, self.n_imgs)) + + for e, (i, j) in enumerate(self.edges): + i_j = edge_str(i, j) + # distance in image i and j + aligned_pred_i = geotrf(pw_poses[e], pw_adapt[e] * self.pred_i[i_j]) + aligned_pred_j = geotrf(pw_poses[e], pw_adapt[e] * self.pred_j[i_j]) + li = self.dist(proj_pts3d[i], aligned_pred_i, weight=weight_i[i_j]).mean() + lj = self.dist(proj_pts3d[j], aligned_pred_j, weight=weight_j[i_j]).mean() + loss = loss + li + lj + + if ret_details: + details[i, j] = li + lj + loss /= self.n_edges # average over all pairs + + if ret_details: + return loss, details + return loss + + @torch.cuda.amp.autocast(enabled=False) + def compute_global_alignment(self, init=None, niter_PnP=10, **kw): + if init is None: + pass + elif init == 'msp' or init == 'mst': + init_fun.init_minimum_spanning_tree(self, niter_PnP=niter_PnP) + elif init == 'known_poses': + init_fun.init_from_known_poses(self, min_conf_thr=self.min_conf_thr, + niter_PnP=niter_PnP) + else: + raise ValueError(f'bad value for {init=}') + + return global_alignment_loop(self, **kw) + + @torch.no_grad() + def mask_sky(self): + res = deepcopy(self) + for i in range(self.n_imgs): + sky = segment_sky(self.imgs[i]) + res.im_conf[i][sky] = 0 + return res + + def show(self, show_pw_cams=False, show_pw_pts3d=False, cam_size=None, **kw): + viz = SceneViz() + if self.imgs is None: + colors = np.random.randint(0, 256, size=(self.n_imgs, 3)) + colors = list(map(tuple, colors.tolist())) + for n in range(self.n_imgs): + viz.add_pointcloud(self.get_pts3d()[n], colors[n], self.get_masks()[n]) + else: + viz.add_pointcloud(self.get_pts3d(), self.imgs, self.get_masks()) + colors = np.random.randint(256, size=(self.n_imgs, 3)) + + # camera poses + im_poses = to_numpy(self.get_im_poses()) + if cam_size is None: + cam_size = auto_cam_size(im_poses) + viz.add_cameras(im_poses, self.get_focals(), colors=colors, + images=self.imgs, imsizes=self.imsizes, cam_size=cam_size) + if show_pw_cams: + pw_poses = self.get_pw_poses() + viz.add_cameras(pw_poses, color=(192, 0, 192), cam_size=cam_size) + + if show_pw_pts3d: + pts = [geotrf(pw_poses[e], self.pred_i[edge_str(i, j)]) for e, (i, j) in enumerate(self.edges)] + viz.add_pointcloud(pts, (128, 0, 128)) + + viz.show(**kw) + return viz + + +def global_alignment_loop(net, lr=0.01, niter=300, schedule='cosine', lr_min=1e-6): + params = [p for p in net.parameters() if p.requires_grad] + if not params: + return net + + verbose = net.verbose + if verbose: + print('Global alignement - optimizing for:') + print([name for name, value in net.named_parameters() if value.requires_grad]) + + lr_base = lr + optimizer = torch.optim.Adam(params, lr=lr, betas=(0.9, 0.9)) + + loss = float('inf') + if verbose: + with tqdm.tqdm(total=niter) as bar: + while bar.n < bar.total: + loss, lr = global_alignment_iter(net, bar.n, niter, lr_base, lr_min, optimizer, schedule) + bar.set_postfix_str(f'{lr=:g} loss={loss:g}') + bar.update() + else: + for n in range(niter): + loss, _ = global_alignment_iter(net, n, niter, lr_base, lr_min, optimizer, schedule) + return loss + + +def global_alignment_iter(net, cur_iter, niter, lr_base, lr_min, optimizer, schedule): + t = cur_iter / niter + if schedule == 'cosine': + lr = cosine_schedule(t, lr_base, lr_min) + elif schedule == 'linear': + lr = linear_schedule(t, lr_base, lr_min) + else: + raise ValueError(f'bad lr {schedule=}') + adjust_learning_rate_by_lr(optimizer, lr) + optimizer.zero_grad() + loss = net() + loss.backward() + optimizer.step() + + return float(loss), lr + + +@torch.no_grad() +def clean_pointcloud( im_confs, K, cams, depthmaps, all_pts3d, + tol=0.001, bad_conf=0, dbg=()): + """ Method: + 1) express all 3d points in each camera coordinate frame + 2) if they're in front of a depthmap --> then lower their confidence + """ + assert len(im_confs) == len(cams) == len(K) == len(depthmaps) == len(all_pts3d) + assert 0 <= tol < 1 + res = [c.clone() for c in im_confs] + + # reshape appropriately + all_pts3d = [p.view(*c.shape,3) for p,c in zip(all_pts3d, im_confs)] + depthmaps = [d.view(*c.shape) for d,c in zip(depthmaps, im_confs)] + + for i, pts3d in enumerate(all_pts3d): + for j in range(len(all_pts3d)): + if i == j: continue + + # project 3dpts in other view + proj = geotrf(cams[j], pts3d) + proj_depth = proj[:,:,2] + u,v = geotrf(K[j], proj, norm=1, ncol=2).round().long().unbind(-1) + + # check which points are actually in the visible cone + H, W = im_confs[j].shape + msk_i = (proj_depth > 0) & (0 <= u) & (u < W) & (0 <= v) & (v < H) + msk_j = v[msk_i], u[msk_i] + + # find bad points = those in front but less confident + bad_points = (proj_depth[msk_i] < (1-tol) * depthmaps[j][msk_j]) & (res[i][msk_i] < res[j][msk_j]) + + bad_msk_i = msk_i.clone() + bad_msk_i[msk_i] = bad_points + res[i][bad_msk_i] = res[i][bad_msk_i].clip_(max=bad_conf) + + return res diff --git a/src/mast3r_src/dust3r/dust3r/cloud_opt/commons.py b/src/mast3r_src/dust3r/dust3r/cloud_opt/commons.py new file mode 100644 index 0000000000000000000000000000000000000000..3be9f855a69ea18c82dcc8e5769e0149a59649bd --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/cloud_opt/commons.py @@ -0,0 +1,90 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utility functions for global alignment +# -------------------------------------------------------- +import torch +import torch.nn as nn +import numpy as np + + +def edge_str(i, j): + return f'{i}_{j}' + + +def i_j_ij(ij): + return edge_str(*ij), ij + + +def edge_conf(conf_i, conf_j, edge): + return float(conf_i[edge].mean() * conf_j[edge].mean()) + + +def compute_edge_scores(edges, conf_i, conf_j): + return {(i, j): edge_conf(conf_i, conf_j, e) for e, (i, j) in edges} + + +def NoGradParamDict(x): + assert isinstance(x, dict) + return nn.ParameterDict(x).requires_grad_(False) + + +def get_imshapes(edges, pred_i, pred_j): + n_imgs = max(max(e) for e in edges) + 1 + imshapes = [None] * n_imgs + for e, (i, j) in enumerate(edges): + shape_i = tuple(pred_i[e].shape[0:2]) + shape_j = tuple(pred_j[e].shape[0:2]) + if imshapes[i]: + assert imshapes[i] == shape_i, f'incorrect shape for image {i}' + if imshapes[j]: + assert imshapes[j] == shape_j, f'incorrect shape for image {j}' + imshapes[i] = shape_i + imshapes[j] = shape_j + return imshapes + + +def get_conf_trf(mode): + if mode == 'log': + def conf_trf(x): return x.log() + elif mode == 'sqrt': + def conf_trf(x): return x.sqrt() + elif mode == 'm1': + def conf_trf(x): return x-1 + elif mode in ('id', 'none'): + def conf_trf(x): return x + else: + raise ValueError(f'bad mode for {mode=}') + return conf_trf + + +def l2_dist(a, b, weight): + return ((a - b).square().sum(dim=-1) * weight) + + +def l1_dist(a, b, weight): + return ((a - b).norm(dim=-1) * weight) + + +ALL_DISTS = dict(l1=l1_dist, l2=l2_dist) + + +def signed_log1p(x): + sign = torch.sign(x) + return sign * torch.log1p(torch.abs(x)) + + +def signed_expm1(x): + sign = torch.sign(x) + return sign * torch.expm1(torch.abs(x)) + + +def cosine_schedule(t, lr_start, lr_end): + assert 0 <= t <= 1 + return lr_end + (lr_start - lr_end) * (1+np.cos(t * np.pi))/2 + + +def linear_schedule(t, lr_start, lr_end): + assert 0 <= t <= 1 + return lr_start + (lr_end - lr_start) * t diff --git a/src/mast3r_src/dust3r/dust3r/cloud_opt/init_im_poses.py b/src/mast3r_src/dust3r/dust3r/cloud_opt/init_im_poses.py new file mode 100644 index 0000000000000000000000000000000000000000..7887c5cde27115273601e704b81ca0b0301f3715 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/cloud_opt/init_im_poses.py @@ -0,0 +1,316 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Initialization functions for global alignment +# -------------------------------------------------------- +from functools import cache + +import numpy as np +import scipy.sparse as sp +import torch +import cv2 +import roma +from tqdm import tqdm + +from dust3r.utils.geometry import geotrf, inv, get_med_dist_between_poses +from dust3r.post_process import estimate_focal_knowing_depth +from dust3r.viz import to_numpy + +from dust3r.cloud_opt.commons import edge_str, i_j_ij, compute_edge_scores + + +@torch.no_grad() +def init_from_known_poses(self, niter_PnP=10, min_conf_thr=3): + device = self.device + + # indices of known poses + nkp, known_poses_msk, known_poses = get_known_poses(self) + assert nkp == self.n_imgs, 'not all poses are known' + + # get all focals + nkf, _, im_focals = get_known_focals(self) + assert nkf == self.n_imgs + im_pp = self.get_principal_points() + + best_depthmaps = {} + # init all pairwise poses + for e, (i, j) in enumerate(tqdm(self.edges, disable=not self.verbose)): + i_j = edge_str(i, j) + + # find relative pose for this pair + P1 = torch.eye(4, device=device) + msk = self.conf_i[i_j] > min(min_conf_thr, self.conf_i[i_j].min() - 0.1) + _, P2 = fast_pnp(self.pred_j[i_j], float(im_focals[i].mean()), + pp=im_pp[i], msk=msk, device=device, niter_PnP=niter_PnP) + + # align the two predicted camera with the two gt cameras + s, R, T = align_multiple_poses(torch.stack((P1, P2)), known_poses[[i, j]]) + # normally we have known_poses[i] ~= sRT_to_4x4(s,R,T,device) @ P1 + # and geotrf(sRT_to_4x4(1,R,T,device), s*P2[:3,3]) + self._set_pose(self.pw_poses, e, R, T, scale=s) + + # remember if this is a good depthmap + score = float(self.conf_i[i_j].mean()) + if score > best_depthmaps.get(i, (0,))[0]: + best_depthmaps[i] = score, i_j, s + + # init all image poses + for n in range(self.n_imgs): + assert known_poses_msk[n] + _, i_j, scale = best_depthmaps[n] + depth = self.pred_i[i_j][:, :, 2] + self._set_depthmap(n, depth * scale) + + +@torch.no_grad() +def init_minimum_spanning_tree(self, **kw): + """ Init all camera poses (image-wise and pairwise poses) given + an initial set of pairwise estimations. + """ + device = self.device + pts3d, _, im_focals, im_poses = minimum_spanning_tree(self.imshapes, self.edges, + self.pred_i, self.pred_j, self.conf_i, self.conf_j, self.im_conf, self.min_conf_thr, + device, has_im_poses=self.has_im_poses, verbose=self.verbose, + **kw) + + return init_from_pts3d(self, pts3d, im_focals, im_poses) + + +def init_from_pts3d(self, pts3d, im_focals, im_poses): + # init poses + nkp, known_poses_msk, known_poses = get_known_poses(self) + if nkp == 1: + raise NotImplementedError("Would be simpler to just align everything afterwards on the single known pose") + elif nkp > 1: + # global rigid SE3 alignment + s, R, T = align_multiple_poses(im_poses[known_poses_msk], known_poses[known_poses_msk]) + trf = sRT_to_4x4(s, R, T, device=known_poses.device) + + # rotate everything + im_poses = trf @ im_poses + im_poses[:, :3, :3] /= s # undo scaling on the rotation part + for img_pts3d in pts3d: + img_pts3d[:] = geotrf(trf, img_pts3d) + + # set all pairwise poses + for e, (i, j) in enumerate(self.edges): + i_j = edge_str(i, j) + # compute transform that goes from cam to world + s, R, T = rigid_points_registration(self.pred_i[i_j], pts3d[i], conf=self.conf_i[i_j]) + self._set_pose(self.pw_poses, e, R, T, scale=s) + + # take into account the scale normalization + s_factor = self.get_pw_norm_scale_factor() + im_poses[:, :3, 3] *= s_factor # apply downscaling factor + for img_pts3d in pts3d: + img_pts3d *= s_factor + + # init all image poses + if self.has_im_poses: + for i in range(self.n_imgs): + cam2world = im_poses[i] + depth = geotrf(inv(cam2world), pts3d[i])[..., 2] + self._set_depthmap(i, depth) + self._set_pose(self.im_poses, i, cam2world) + if im_focals[i] is not None: + self._set_focal(i, im_focals[i]) + + if self.verbose: + print(' init loss =', float(self())) + + +def minimum_spanning_tree(imshapes, edges, pred_i, pred_j, conf_i, conf_j, im_conf, min_conf_thr, + device, has_im_poses=True, niter_PnP=10, verbose=True): + n_imgs = len(imshapes) + sparse_graph = -dict_to_sparse_graph(compute_edge_scores(map(i_j_ij, edges), conf_i, conf_j)) + msp = sp.csgraph.minimum_spanning_tree(sparse_graph).tocoo() + + # temp variable to store 3d points + pts3d = [None] * len(imshapes) + + todo = sorted(zip(-msp.data, msp.row, msp.col)) # sorted edges + im_poses = [None] * n_imgs + im_focals = [None] * n_imgs + + # init with strongest edge + score, i, j = todo.pop() + if verbose: + print(f' init edge ({i}*,{j}*) {score=}') + i_j = edge_str(i, j) + pts3d[i] = pred_i[i_j].clone() + pts3d[j] = pred_j[i_j].clone() + done = {i, j} + if has_im_poses: + im_poses[i] = torch.eye(4, device=device) + im_focals[i] = estimate_focal(pred_i[i_j]) + + # set initial pointcloud based on pairwise graph + msp_edges = [(i, j)] + while todo: + # each time, predict the next one + score, i, j = todo.pop() + + if im_focals[i] is None: + im_focals[i] = estimate_focal(pred_i[i_j]) + + if i in done: + if verbose: + print(f' init edge ({i},{j}*) {score=}') + assert j not in done + # align pred[i] with pts3d[i], and then set j accordingly + i_j = edge_str(i, j) + s, R, T = rigid_points_registration(pred_i[i_j], pts3d[i], conf=conf_i[i_j]) + trf = sRT_to_4x4(s, R, T, device) + pts3d[j] = geotrf(trf, pred_j[i_j]) + done.add(j) + msp_edges.append((i, j)) + + if has_im_poses and im_poses[i] is None: + im_poses[i] = sRT_to_4x4(1, R, T, device) + + elif j in done: + if verbose: + print(f' init edge ({i}*,{j}) {score=}') + assert i not in done + i_j = edge_str(i, j) + s, R, T = rigid_points_registration(pred_j[i_j], pts3d[j], conf=conf_j[i_j]) + trf = sRT_to_4x4(s, R, T, device) + pts3d[i] = geotrf(trf, pred_i[i_j]) + done.add(i) + msp_edges.append((i, j)) + + if has_im_poses and im_poses[i] is None: + im_poses[i] = sRT_to_4x4(1, R, T, device) + else: + # let's try again later + todo.insert(0, (score, i, j)) + + if has_im_poses: + # complete all missing informations + pair_scores = list(sparse_graph.values()) # already negative scores: less is best + edges_from_best_to_worse = np.array(list(sparse_graph.keys()))[np.argsort(pair_scores)] + for i, j in edges_from_best_to_worse.tolist(): + if im_focals[i] is None: + im_focals[i] = estimate_focal(pred_i[edge_str(i, j)]) + + for i in range(n_imgs): + if im_poses[i] is None: + msk = im_conf[i] > min_conf_thr + res = fast_pnp(pts3d[i], im_focals[i], msk=msk, device=device, niter_PnP=niter_PnP) + if res: + im_focals[i], im_poses[i] = res + if im_poses[i] is None: + im_poses[i] = torch.eye(4, device=device) + im_poses = torch.stack(im_poses) + else: + im_poses = im_focals = None + + return pts3d, msp_edges, im_focals, im_poses + + +def dict_to_sparse_graph(dic): + n_imgs = max(max(e) for e in dic) + 1 + res = sp.dok_array((n_imgs, n_imgs)) + for edge, value in dic.items(): + res[edge] = value + return res + + +def rigid_points_registration(pts1, pts2, conf): + R, T, s = roma.rigid_points_registration( + pts1.reshape(-1, 3), pts2.reshape(-1, 3), weights=conf.ravel(), compute_scaling=True) + return s, R, T # return un-scaled (R, T) + + +def sRT_to_4x4(scale, R, T, device): + trf = torch.eye(4, device=device) + trf[:3, :3] = R * scale + trf[:3, 3] = T.ravel() # doesn't need scaling + return trf + + +def estimate_focal(pts3d_i, pp=None): + if pp is None: + H, W, THREE = pts3d_i.shape + assert THREE == 3 + pp = torch.tensor((W/2, H/2), device=pts3d_i.device) + focal = estimate_focal_knowing_depth(pts3d_i.unsqueeze(0), pp.unsqueeze(0), focal_mode='weiszfeld').ravel() + return float(focal) + + +@cache +def pixel_grid(H, W): + return np.mgrid[:W, :H].T.astype(np.float32) + + +def fast_pnp(pts3d, focal, msk, device, pp=None, niter_PnP=10): + # extract camera poses and focals with RANSAC-PnP + if msk.sum() < 4: + return None # we need at least 4 points for PnP + pts3d, msk = map(to_numpy, (pts3d, msk)) + + H, W, THREE = pts3d.shape + assert THREE == 3 + pixels = pixel_grid(H, W) + + if focal is None: + S = max(W, H) + tentative_focals = np.geomspace(S/2, S*3, 21) + else: + tentative_focals = [focal] + + if pp is None: + pp = (W/2, H/2) + else: + pp = to_numpy(pp) + + best = 0, + for focal in tentative_focals: + K = np.float32([(focal, 0, pp[0]), (0, focal, pp[1]), (0, 0, 1)]) + + success, R, T, inliers = cv2.solvePnPRansac(pts3d[msk], pixels[msk], K, None, + iterationsCount=niter_PnP, reprojectionError=5, flags=cv2.SOLVEPNP_SQPNP) + if not success: + continue + + score = len(inliers) + if success and score > best[0]: + best = score, R, T, focal + + if not best[0]: + return None + + _, R, T, best_focal = best + R = cv2.Rodrigues(R)[0] # world to cam + R, T = map(torch.from_numpy, (R, T)) + return best_focal, inv(sRT_to_4x4(1, R, T, device)) # cam to world + + +def get_known_poses(self): + if self.has_im_poses: + known_poses_msk = torch.tensor([not (p.requires_grad) for p in self.im_poses]) + known_poses = self.get_im_poses() + return known_poses_msk.sum(), known_poses_msk, known_poses + else: + return 0, None, None + + +def get_known_focals(self): + if self.has_im_poses: + known_focal_msk = self.get_known_focal_mask() + known_focals = self.get_focals() + return known_focal_msk.sum(), known_focal_msk, known_focals + else: + return 0, None, None + + +def align_multiple_poses(src_poses, target_poses): + N = len(src_poses) + assert src_poses.shape == target_poses.shape == (N, 4, 4) + + def center_and_z(poses): + eps = get_med_dist_between_poses(poses) / 100 + return torch.cat((poses[:, :3, 3], poses[:, :3, 3] + eps*poses[:, :3, 2])) + R, T, s = roma.rigid_points_registration(center_and_z(src_poses), center_and_z(target_poses), compute_scaling=True) + return s, R, T diff --git a/src/mast3r_src/dust3r/dust3r/cloud_opt/modular_optimizer.py b/src/mast3r_src/dust3r/dust3r/cloud_opt/modular_optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..d06464b40276684385c18b9195be1491c6f47f07 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/cloud_opt/modular_optimizer.py @@ -0,0 +1,145 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Slower implementation of the global alignment that allows to freeze partial poses/intrinsics +# -------------------------------------------------------- +import numpy as np +import torch +import torch.nn as nn + +from dust3r.cloud_opt.base_opt import BasePCOptimizer +from dust3r.utils.geometry import geotrf +from dust3r.utils.device import to_cpu, to_numpy +from dust3r.utils.geometry import depthmap_to_pts3d + + +class ModularPointCloudOptimizer (BasePCOptimizer): + """ Optimize a global scene, given a list of pairwise observations. + Unlike PointCloudOptimizer, you can fix parts of the optimization process (partial poses/intrinsics) + Graph node: images + Graph edges: observations = (pred1, pred2) + """ + + def __init__(self, *args, optimize_pp=False, fx_and_fy=False, focal_brake=20, **kwargs): + super().__init__(*args, **kwargs) + self.has_im_poses = True # by definition of this class + self.focal_brake = focal_brake + + # adding thing to optimize + self.im_depthmaps = nn.ParameterList(torch.randn(H, W)/10-3 for H, W in self.imshapes) # log(depth) + self.im_poses = nn.ParameterList(self.rand_pose(self.POSE_DIM) for _ in range(self.n_imgs)) # camera poses + default_focals = [self.focal_brake * np.log(max(H, W)) for H, W in self.imshapes] + self.im_focals = nn.ParameterList(torch.FloatTensor([f, f] if fx_and_fy else [ + f]) for f in default_focals) # camera intrinsics + self.im_pp = nn.ParameterList(torch.zeros((2,)) for _ in range(self.n_imgs)) # camera intrinsics + self.im_pp.requires_grad_(optimize_pp) + + def preset_pose(self, known_poses, pose_msk=None): # cam-to-world + if isinstance(known_poses, torch.Tensor) and known_poses.ndim == 2: + known_poses = [known_poses] + for idx, pose in zip(self._get_msk_indices(pose_msk), known_poses): + if self.verbose: + print(f' (setting pose #{idx} = {pose[:3,3]})') + self._no_grad(self._set_pose(self.im_poses, idx, torch.tensor(pose), force=True)) + + # normalize scale if there's less than 1 known pose + n_known_poses = sum((p.requires_grad is False) for p in self.im_poses) + self.norm_pw_scale = (n_known_poses <= 1) + + def preset_intrinsics(self, known_intrinsics, msk=None): + if isinstance(known_intrinsics, torch.Tensor) and known_intrinsics.ndim == 2: + known_intrinsics = [known_intrinsics] + for K in known_intrinsics: + assert K.shape == (3, 3) + self.preset_focal([K.diagonal()[:2].mean() for K in known_intrinsics], msk) + self.preset_principal_point([K[:2, 2] for K in known_intrinsics], msk) + + def preset_focal(self, known_focals, msk=None): + for idx, focal in zip(self._get_msk_indices(msk), known_focals): + if self.verbose: + print(f' (setting focal #{idx} = {focal})') + self._no_grad(self._set_focal(idx, focal, force=True)) + + def preset_principal_point(self, known_pp, msk=None): + for idx, pp in zip(self._get_msk_indices(msk), known_pp): + if self.verbose: + print(f' (setting principal point #{idx} = {pp})') + self._no_grad(self._set_principal_point(idx, pp, force=True)) + + def _no_grad(self, tensor): + return tensor.requires_grad_(False) + + def _get_msk_indices(self, msk): + if msk is None: + return range(self.n_imgs) + elif isinstance(msk, int): + return [msk] + elif isinstance(msk, (tuple, list)): + return self._get_msk_indices(np.array(msk)) + elif msk.dtype in (bool, torch.bool, np.bool_): + assert len(msk) == self.n_imgs + return np.where(msk)[0] + elif np.issubdtype(msk.dtype, np.integer): + return msk + else: + raise ValueError(f'bad {msk=}') + + def _set_focal(self, idx, focal, force=False): + param = self.im_focals[idx] + if param.requires_grad or force: # can only init a parameter not already initialized + param.data[:] = self.focal_brake * np.log(focal) + return param + + def get_focals(self): + log_focals = torch.stack(list(self.im_focals), dim=0) + return (log_focals / self.focal_brake).exp() + + def _set_principal_point(self, idx, pp, force=False): + param = self.im_pp[idx] + H, W = self.imshapes[idx] + if param.requires_grad or force: # can only init a parameter not already initialized + param.data[:] = to_cpu(to_numpy(pp) - (W/2, H/2)) / 10 + return param + + def get_principal_points(self): + return torch.stack([pp.new((W/2, H/2))+10*pp for pp, (H, W) in zip(self.im_pp, self.imshapes)]) + + def get_intrinsics(self): + K = torch.zeros((self.n_imgs, 3, 3), device=self.device) + focals = self.get_focals().view(self.n_imgs, -1) + K[:, 0, 0] = focals[:, 0] + K[:, 1, 1] = focals[:, -1] + K[:, :2, 2] = self.get_principal_points() + K[:, 2, 2] = 1 + return K + + def get_im_poses(self): # cam to world + cam2world = self._get_poses(torch.stack(list(self.im_poses))) + return cam2world + + def _set_depthmap(self, idx, depth, force=False): + param = self.im_depthmaps[idx] + if param.requires_grad or force: # can only init a parameter not already initialized + param.data[:] = depth.log().nan_to_num(neginf=0) + return param + + def get_depthmaps(self): + return [d.exp() for d in self.im_depthmaps] + + def depth_to_pts3d(self): + # Get depths and projection params if not provided + focals = self.get_focals() + pp = self.get_principal_points() + im_poses = self.get_im_poses() + depth = self.get_depthmaps() + + # convert focal to (1,2,H,W) constant field + def focal_ex(i): return focals[i][..., None, None].expand(1, *focals[i].shape, *self.imshapes[i]) + # get pointmaps in camera frame + rel_ptmaps = [depthmap_to_pts3d(depth[i][None], focal_ex(i), pp=pp[i:i+1])[0] for i in range(im_poses.shape[0])] + # project to world frame + return [geotrf(pose, ptmap) for pose, ptmap in zip(im_poses, rel_ptmaps)] + + def get_pts3d(self): + return self.depth_to_pts3d() diff --git a/src/mast3r_src/dust3r/dust3r/cloud_opt/optimizer.py b/src/mast3r_src/dust3r/dust3r/cloud_opt/optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..42e48613e55faa4ede5a366d1c0bfc4d18ffae4f --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/cloud_opt/optimizer.py @@ -0,0 +1,248 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Main class for the implementation of the global alignment +# -------------------------------------------------------- +import numpy as np +import torch +import torch.nn as nn + +from dust3r.cloud_opt.base_opt import BasePCOptimizer +from dust3r.utils.geometry import xy_grid, geotrf +from dust3r.utils.device import to_cpu, to_numpy + + +class PointCloudOptimizer(BasePCOptimizer): + """ Optimize a global scene, given a list of pairwise observations. + Graph node: images + Graph edges: observations = (pred1, pred2) + """ + + def __init__(self, *args, optimize_pp=False, focal_break=20, **kwargs): + super().__init__(*args, **kwargs) + + self.has_im_poses = True # by definition of this class + self.focal_break = focal_break + + # adding thing to optimize + self.im_depthmaps = nn.ParameterList(torch.randn(H, W)/10-3 for H, W in self.imshapes) # log(depth) + self.im_poses = nn.ParameterList(self.rand_pose(self.POSE_DIM) for _ in range(self.n_imgs)) # camera poses + self.im_focals = nn.ParameterList(torch.FloatTensor( + [self.focal_break*np.log(max(H, W))]) for H, W in self.imshapes) # camera intrinsics + self.im_pp = nn.ParameterList(torch.zeros((2,)) for _ in range(self.n_imgs)) # camera intrinsics + self.im_pp.requires_grad_(optimize_pp) + + self.imshape = self.imshapes[0] + im_areas = [h*w for h, w in self.imshapes] + self.max_area = max(im_areas) + + # adding thing to optimize + self.im_depthmaps = ParameterStack(self.im_depthmaps, is_param=True, fill=self.max_area) + self.im_poses = ParameterStack(self.im_poses, is_param=True) + self.im_focals = ParameterStack(self.im_focals, is_param=True) + self.im_pp = ParameterStack(self.im_pp, is_param=True) + self.register_buffer('_pp', torch.tensor([(w/2, h/2) for h, w in self.imshapes])) + self.register_buffer('_grid', ParameterStack( + [xy_grid(W, H, device=self.device) for H, W in self.imshapes], fill=self.max_area)) + + # pre-compute pixel weights + self.register_buffer('_weight_i', ParameterStack( + [self.conf_trf(self.conf_i[i_j]) for i_j in self.str_edges], fill=self.max_area)) + self.register_buffer('_weight_j', ParameterStack( + [self.conf_trf(self.conf_j[i_j]) for i_j in self.str_edges], fill=self.max_area)) + + # precompute aa + self.register_buffer('_stacked_pred_i', ParameterStack(self.pred_i, self.str_edges, fill=self.max_area)) + self.register_buffer('_stacked_pred_j', ParameterStack(self.pred_j, self.str_edges, fill=self.max_area)) + self.register_buffer('_ei', torch.tensor([i for i, j in self.edges])) + self.register_buffer('_ej', torch.tensor([j for i, j in self.edges])) + self.total_area_i = sum([im_areas[i] for i, j in self.edges]) + self.total_area_j = sum([im_areas[j] for i, j in self.edges]) + + def _check_all_imgs_are_selected(self, msk): + assert np.all(self._get_msk_indices(msk) == np.arange(self.n_imgs)), 'incomplete mask!' + + def preset_pose(self, known_poses, pose_msk=None): # cam-to-world + self._check_all_imgs_are_selected(pose_msk) + + if isinstance(known_poses, torch.Tensor) and known_poses.ndim == 2: + known_poses = [known_poses] + for idx, pose in zip(self._get_msk_indices(pose_msk), known_poses): + if self.verbose: + print(f' (setting pose #{idx} = {pose[:3,3]})') + self._no_grad(self._set_pose(self.im_poses, idx, torch.tensor(pose))) + + # normalize scale if there's less than 1 known pose + n_known_poses = sum((p.requires_grad is False) for p in self.im_poses) + self.norm_pw_scale = (n_known_poses <= 1) + + self.im_poses.requires_grad_(False) + self.norm_pw_scale = False + + def preset_focal(self, known_focals, msk=None): + self._check_all_imgs_are_selected(msk) + + for idx, focal in zip(self._get_msk_indices(msk), known_focals): + if self.verbose: + print(f' (setting focal #{idx} = {focal})') + self._no_grad(self._set_focal(idx, focal)) + + self.im_focals.requires_grad_(False) + + def preset_principal_point(self, known_pp, msk=None): + self._check_all_imgs_are_selected(msk) + + for idx, pp in zip(self._get_msk_indices(msk), known_pp): + if self.verbose: + print(f' (setting principal point #{idx} = {pp})') + self._no_grad(self._set_principal_point(idx, pp)) + + self.im_pp.requires_grad_(False) + + def _get_msk_indices(self, msk): + if msk is None: + return range(self.n_imgs) + elif isinstance(msk, int): + return [msk] + elif isinstance(msk, (tuple, list)): + return self._get_msk_indices(np.array(msk)) + elif msk.dtype in (bool, torch.bool, np.bool_): + assert len(msk) == self.n_imgs + return np.where(msk)[0] + elif np.issubdtype(msk.dtype, np.integer): + return msk + else: + raise ValueError(f'bad {msk=}') + + def _no_grad(self, tensor): + assert tensor.requires_grad, 'it must be True at this point, otherwise no modification occurs' + + def _set_focal(self, idx, focal, force=False): + param = self.im_focals[idx] + if param.requires_grad or force: # can only init a parameter not already initialized + param.data[:] = self.focal_break * np.log(focal) + return param + + def get_focals(self): + log_focals = torch.stack(list(self.im_focals), dim=0) + return (log_focals / self.focal_break).exp() + + def get_known_focal_mask(self): + return torch.tensor([not (p.requires_grad) for p in self.im_focals]) + + def _set_principal_point(self, idx, pp, force=False): + param = self.im_pp[idx] + H, W = self.imshapes[idx] + if param.requires_grad or force: # can only init a parameter not already initialized + param.data[:] = to_cpu(to_numpy(pp) - (W/2, H/2)) / 10 + return param + + def get_principal_points(self): + return self._pp + 10 * self.im_pp + + def get_intrinsics(self): + K = torch.zeros((self.n_imgs, 3, 3), device=self.device) + focals = self.get_focals().flatten() + K[:, 0, 0] = K[:, 1, 1] = focals + K[:, :2, 2] = self.get_principal_points() + K[:, 2, 2] = 1 + return K + + def get_im_poses(self): # cam to world + cam2world = self._get_poses(self.im_poses) + return cam2world + + def _set_depthmap(self, idx, depth, force=False): + depth = _ravel_hw(depth, self.max_area) + + param = self.im_depthmaps[idx] + if param.requires_grad or force: # can only init a parameter not already initialized + param.data[:] = depth.log().nan_to_num(neginf=0) + return param + + def get_depthmaps(self, raw=False): + res = self.im_depthmaps.exp() + if not raw: + res = [dm[:h*w].view(h, w) for dm, (h, w) in zip(res, self.imshapes)] + return res + + def depth_to_pts3d(self): + # Get depths and projection params if not provided + focals = self.get_focals() + pp = self.get_principal_points() + im_poses = self.get_im_poses() + depth = self.get_depthmaps(raw=True) + + # get pointmaps in camera frame + rel_ptmaps = _fast_depthmap_to_pts3d(depth, self._grid, focals, pp=pp) + # project to world frame + return geotrf(im_poses, rel_ptmaps) + + def get_pts3d(self, raw=False): + res = self.depth_to_pts3d() + if not raw: + res = [dm[:h*w].view(h, w, 3) for dm, (h, w) in zip(res, self.imshapes)] + return res + + def forward(self): + pw_poses = self.get_pw_poses() # cam-to-world + pw_adapt = self.get_adaptors().unsqueeze(1) + proj_pts3d = self.get_pts3d(raw=True) + + # rotate pairwise prediction according to pw_poses + aligned_pred_i = geotrf(pw_poses, pw_adapt * self._stacked_pred_i) + aligned_pred_j = geotrf(pw_poses, pw_adapt * self._stacked_pred_j) + + # compute the less + li = self.dist(proj_pts3d[self._ei], aligned_pred_i, weight=self._weight_i).sum() / self.total_area_i + lj = self.dist(proj_pts3d[self._ej], aligned_pred_j, weight=self._weight_j).sum() / self.total_area_j + + return li + lj + + +def _fast_depthmap_to_pts3d(depth, pixel_grid, focal, pp): + pp = pp.unsqueeze(1) + focal = focal.unsqueeze(1) + assert focal.shape == (len(depth), 1, 1) + assert pp.shape == (len(depth), 1, 2) + assert pixel_grid.shape == depth.shape + (2,) + depth = depth.unsqueeze(-1) + return torch.cat((depth * (pixel_grid - pp) / focal, depth), dim=-1) + + +def ParameterStack(params, keys=None, is_param=None, fill=0): + if keys is not None: + params = [params[k] for k in keys] + + if fill > 0: + params = [_ravel_hw(p, fill) for p in params] + + requires_grad = params[0].requires_grad + assert all(p.requires_grad == requires_grad for p in params) + + params = torch.stack(list(params)).float().detach() + if is_param or requires_grad: + params = nn.Parameter(params) + params.requires_grad_(requires_grad) + return params + + +def _ravel_hw(tensor, fill=0): + # ravel H,W + tensor = tensor.view((tensor.shape[0] * tensor.shape[1],) + tensor.shape[2:]) + + if len(tensor) < fill: + tensor = torch.cat((tensor, tensor.new_zeros((fill - len(tensor),)+tensor.shape[1:]))) + return tensor + + +def acceptable_focal_range(H, W, minf=0.5, maxf=3.5): + focal_base = max(H, W) / (2 * np.tan(np.deg2rad(60) / 2)) # size / 1.1547005383792515 + return minf*focal_base, maxf*focal_base + + +def apply_mask(img, msk): + img = img.copy() + img[msk] = 0 + return img diff --git a/src/mast3r_src/dust3r/dust3r/cloud_opt/pair_viewer.py b/src/mast3r_src/dust3r/dust3r/cloud_opt/pair_viewer.py new file mode 100644 index 0000000000000000000000000000000000000000..62ae3b9a5fbca8b96711de051d9d6597830bd488 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/cloud_opt/pair_viewer.py @@ -0,0 +1,127 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dummy optimizer for visualizing pairs +# -------------------------------------------------------- +import numpy as np +import torch +import torch.nn as nn +import cv2 + +from dust3r.cloud_opt.base_opt import BasePCOptimizer +from dust3r.utils.geometry import inv, geotrf, depthmap_to_absolute_camera_coordinates +from dust3r.cloud_opt.commons import edge_str +from dust3r.post_process import estimate_focal_knowing_depth + + +class PairViewer (BasePCOptimizer): + """ + This a Dummy Optimizer. + To use only when the goal is to visualize the results for a pair of images (with is_symmetrized) + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + assert self.is_symmetrized and self.n_edges == 2 + self.has_im_poses = True + + # compute all parameters directly from raw input + self.focals = [] + self.pp = [] + rel_poses = [] + confs = [] + for i in range(self.n_imgs): + conf = float(self.conf_i[edge_str(i, 1-i)].mean() * self.conf_j[edge_str(i, 1-i)].mean()) + if self.verbose: + print(f' - {conf=:.3} for edge {i}-{1-i}') + confs.append(conf) + + H, W = self.imshapes[i] + pts3d = self.pred_i[edge_str(i, 1-i)] + pp = torch.tensor((W/2, H/2)) + focal = float(estimate_focal_knowing_depth(pts3d[None], pp, focal_mode='weiszfeld')) + self.focals.append(focal) + self.pp.append(pp) + + # estimate the pose of pts1 in image 2 + pixels = np.mgrid[:W, :H].T.astype(np.float32) + pts3d = self.pred_j[edge_str(1-i, i)].numpy() + assert pts3d.shape[:2] == (H, W) + msk = self.get_masks()[i].numpy() + K = np.float32([(focal, 0, pp[0]), (0, focal, pp[1]), (0, 0, 1)]) + + try: + res = cv2.solvePnPRansac(pts3d[msk], pixels[msk], K, None, + iterationsCount=100, reprojectionError=5, flags=cv2.SOLVEPNP_SQPNP) + success, R, T, inliers = res + assert success + + R = cv2.Rodrigues(R)[0] # world to cam + pose = inv(np.r_[np.c_[R, T], [(0, 0, 0, 1)]]) # cam to world + except: + pose = np.eye(4) + rel_poses.append(torch.from_numpy(pose.astype(np.float32))) + + # let's use the pair with the most confidence + if confs[0] > confs[1]: + # ptcloud is expressed in camera1 + self.im_poses = [torch.eye(4), rel_poses[1]] # I, cam2-to-cam1 + self.depth = [self.pred_i['0_1'][..., 2], geotrf(inv(rel_poses[1]), self.pred_j['0_1'])[..., 2]] + else: + # ptcloud is expressed in camera2 + self.im_poses = [rel_poses[0], torch.eye(4)] # I, cam1-to-cam2 + self.depth = [geotrf(inv(rel_poses[0]), self.pred_j['1_0'])[..., 2], self.pred_i['1_0'][..., 2]] + + self.im_poses = nn.Parameter(torch.stack(self.im_poses, dim=0), requires_grad=False) + self.focals = nn.Parameter(torch.tensor(self.focals), requires_grad=False) + self.pp = nn.Parameter(torch.stack(self.pp, dim=0), requires_grad=False) + self.depth = nn.ParameterList(self.depth) + for p in self.parameters(): + p.requires_grad = False + + def _set_depthmap(self, idx, depth, force=False): + if self.verbose: + print('_set_depthmap is ignored in PairViewer') + return + + def get_depthmaps(self, raw=False): + depth = [d.to(self.device) for d in self.depth] + return depth + + def _set_focal(self, idx, focal, force=False): + self.focals[idx] = focal + + def get_focals(self): + return self.focals + + def get_known_focal_mask(self): + return torch.tensor([not (p.requires_grad) for p in self.focals]) + + def get_principal_points(self): + return self.pp + + def get_intrinsics(self): + focals = self.get_focals() + pps = self.get_principal_points() + K = torch.zeros((len(focals), 3, 3), device=self.device) + for i in range(len(focals)): + K[i, 0, 0] = K[i, 1, 1] = focals[i] + K[i, :2, 2] = pps[i] + K[i, 2, 2] = 1 + return K + + def get_im_poses(self): + return self.im_poses + + def depth_to_pts3d(self): + pts3d = [] + for d, intrinsics, im_pose in zip(self.depth, self.get_intrinsics(), self.get_im_poses()): + pts, _ = depthmap_to_absolute_camera_coordinates(d.cpu().numpy(), + intrinsics.cpu().numpy(), + im_pose.cpu().numpy()) + pts3d.append(torch.from_numpy(pts).to(device=self.device)) + return pts3d + + def forward(self): + return float('nan') diff --git a/src/mast3r_src/dust3r/dust3r/datasets/__init__.py b/src/mast3r_src/dust3r/dust3r/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2123d09ec2840ab5ee9ca43057c35f93233bde89 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/__init__.py @@ -0,0 +1,50 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +from .utils.transforms import * +from .base.batched_sampler import BatchedRandomSampler # noqa +from .arkitscenes import ARKitScenes # noqa +from .blendedmvs import BlendedMVS # noqa +from .co3d import Co3d # noqa +from .habitat import Habitat # noqa +from .megadepth import MegaDepth # noqa +from .scannetpp import ScanNetpp # noqa +from .staticthings3d import StaticThings3D # noqa +from .waymo import Waymo # noqa +from .wildrgbd import WildRGBD # noqa + + +def get_data_loader(dataset, batch_size, num_workers=8, shuffle=True, drop_last=True, pin_mem=True): + import torch + from croco.utils.misc import get_world_size, get_rank + + # pytorch dataset + if isinstance(dataset, str): + dataset = eval(dataset) + + world_size = get_world_size() + rank = get_rank() + + try: + sampler = dataset.make_sampler(batch_size, shuffle=shuffle, world_size=world_size, + rank=rank, drop_last=drop_last) + except (AttributeError, NotImplementedError): + # not avail for this dataset + if torch.distributed.is_initialized(): + sampler = torch.utils.data.DistributedSampler( + dataset, num_replicas=world_size, rank=rank, shuffle=shuffle, drop_last=drop_last + ) + elif shuffle: + sampler = torch.utils.data.RandomSampler(dataset) + else: + sampler = torch.utils.data.SequentialSampler(dataset) + + data_loader = torch.utils.data.DataLoader( + dataset, + sampler=sampler, + batch_size=batch_size, + num_workers=num_workers, + pin_memory=pin_mem, + drop_last=drop_last, + ) + + return data_loader diff --git a/src/mast3r_src/dust3r/dust3r/datasets/arkitscenes.py b/src/mast3r_src/dust3r/dust3r/datasets/arkitscenes.py new file mode 100644 index 0000000000000000000000000000000000000000..4fad51acdc18b82cd6a4d227de0dac3b25783e33 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/arkitscenes.py @@ -0,0 +1,102 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed arkitscenes +# dataset at https://github.com/apple/ARKitScenes - Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Public License https://github.com/apple/ARKitScenes/tree/main?tab=readme-ov-file#license +# See datasets_preprocess/preprocess_arkitscenes.py +# -------------------------------------------------------- +import os.path as osp +import cv2 +import numpy as np + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +from dust3r.utils.image import imread_cv2 + + +class ARKitScenes(BaseStereoViewDataset): + def __init__(self, *args, split, ROOT, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + if split == "train": + self.split = "Training" + elif split == "test": + self.split = "Test" + else: + raise ValueError("") + + self.loaded_data = self._load_data(self.split) + + def _load_data(self, split): + with np.load(osp.join(self.ROOT, split, 'all_metadata.npz')) as data: + self.scenes = data['scenes'] + self.sceneids = data['sceneids'] + self.images = data['images'] + self.intrinsics = data['intrinsics'].astype(np.float32) + self.trajectories = data['trajectories'].astype(np.float32) + self.pairs = data['pairs'][:, :2].astype(int) + + def __len__(self): + return len(self.pairs) + + def _get_views(self, idx, resolution, rng): + + image_idx1, image_idx2 = self.pairs[idx] + + views = [] + for view_idx in [image_idx1, image_idx2]: + scene_id = self.sceneids[view_idx] + scene_dir = osp.join(self.ROOT, self.split, self.scenes[scene_id]) + + intrinsics = self.intrinsics[view_idx] + camera_pose = self.trajectories[view_idx] + basename = self.images[view_idx] + + # Load RGB image + rgb_image = imread_cv2(osp.join(scene_dir, 'vga_wide', basename.replace('.png', '.jpg'))) + # Load depthmap + depthmap = imread_cv2(osp.join(scene_dir, 'lowres_depth', basename), cv2.IMREAD_UNCHANGED) + depthmap = depthmap.astype(np.float32) / 1000 + depthmap[~np.isfinite(depthmap)] = 0 # invalid + + rgb_image, depthmap, intrinsics = self._crop_resize_if_necessary( + rgb_image, depthmap, intrinsics, resolution, rng=rng, info=view_idx) + + views.append(dict( + img=rgb_image, + depthmap=depthmap.astype(np.float32), + camera_pose=camera_pose.astype(np.float32), + camera_intrinsics=intrinsics.astype(np.float32), + dataset='arkitscenes', + label=self.scenes[scene_id] + '_' + basename, + instance=f'{str(idx)}_{str(view_idx)}', + )) + + return views + + +if __name__ == "__main__": + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = ARKitScenes(split='train', ROOT="data/arkitscenes_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx * 255, (1 - idx) * 255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/src/mast3r_src/dust3r/dust3r/datasets/base/__init__.py b/src/mast3r_src/dust3r/dust3r/datasets/base/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a32692113d830ddc4af4e6ed608f222fbe062e6e --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/base/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). diff --git a/src/mast3r_src/dust3r/dust3r/datasets/base/base_stereo_view_dataset.py b/src/mast3r_src/dust3r/dust3r/datasets/base/base_stereo_view_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..17390ca29d4437fc41f3c946b235888af9e4c888 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/base/base_stereo_view_dataset.py @@ -0,0 +1,220 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# base class for implementing datasets +# -------------------------------------------------------- +import PIL +import numpy as np +import torch + +from dust3r.datasets.base.easy_dataset import EasyDataset +from dust3r.datasets.utils.transforms import ImgNorm +from dust3r.utils.geometry import depthmap_to_absolute_camera_coordinates +import dust3r.datasets.utils.cropping as cropping + + +class BaseStereoViewDataset (EasyDataset): + """ Define all basic options. + + Usage: + class MyDataset (BaseStereoViewDataset): + def _get_views(self, idx, rng): + # overload here + views = [] + views.append(dict(img=, ...)) + return views + """ + + def __init__(self, *, # only keyword arguments + split=None, + resolution=None, # square_size or (width, height) or list of [(width,height), ...] + transform=ImgNorm, + aug_crop=False, + seed=None): + self.num_views = 2 + self.split = split + self._set_resolutions(resolution) + + self.transform = transform + if isinstance(transform, str): + transform = eval(transform) + + self.aug_crop = aug_crop + self.seed = seed + + def __len__(self): + return len(self.scenes) + + def get_stats(self): + return f"{len(self)} pairs" + + def __repr__(self): + resolutions_str = '['+';'.join(f'{w}x{h}' for w, h in self._resolutions)+']' + return f"""{type(self).__name__}({self.get_stats()}, + {self.split=}, + {self.seed=}, + resolutions={resolutions_str}, + {self.transform=})""".replace('self.', '').replace('\n', '').replace(' ', '') + + def _get_views(self, idx, resolution, rng): + raise NotImplementedError() + + def __getitem__(self, idx): + if isinstance(idx, tuple): + # the idx is specifying the aspect-ratio + idx, ar_idx = idx + else: + assert len(self._resolutions) == 1 + ar_idx = 0 + + # set-up the rng + if self.seed: # reseed for each __getitem__ + self._rng = np.random.default_rng(seed=self.seed + idx) + elif not hasattr(self, '_rng'): + seed = torch.initial_seed() # this is different for each dataloader process + self._rng = np.random.default_rng(seed=seed) + + # over-loaded code + resolution = self._resolutions[ar_idx] # DO NOT CHANGE THIS (compatible with BatchedRandomSampler) + views = self._get_views(idx, resolution, self._rng) + assert len(views) == self.num_views + + # check data-types + for v, view in enumerate(views): + assert 'pts3d' not in view, f"pts3d should not be there, they will be computed afterwards based on intrinsics+depthmap for view {view_name(view)}" + view['idx'] = (idx, ar_idx, v) + + # encode the image + width, height = view['img'].size + view['true_shape'] = np.int32((height, width)) + view['img'] = self.transform(view['img']) + + assert 'camera_intrinsics' in view + if 'camera_pose' not in view: + view['camera_pose'] = np.full((4, 4), np.nan, dtype=np.float32) + else: + assert np.isfinite(view['camera_pose']).all(), f'NaN in camera pose for view {view_name(view)}' + assert 'pts3d' not in view + assert 'valid_mask' not in view + assert np.isfinite(view['depthmap']).all(), f'NaN in depthmap for view {view_name(view)}' + pts3d, valid_mask = depthmap_to_absolute_camera_coordinates(**view) + + view['pts3d'] = pts3d + view['valid_mask'] = valid_mask & np.isfinite(pts3d).all(axis=-1) + + # check all datatypes + for key, val in view.items(): + res, err_msg = is_good_type(key, val) + assert res, f"{err_msg} with {key}={val} for view {view_name(view)}" + K = view['camera_intrinsics'] + + # last thing done! + for view in views: + # transpose to make sure all views are the same size + transpose_to_landscape(view) + # this allows to check whether the RNG is is the same state each time + view['rng'] = int.from_bytes(self._rng.bytes(4), 'big') + return views + + def _set_resolutions(self, resolutions): + assert resolutions is not None, 'undefined resolution' + + if not isinstance(resolutions, list): + resolutions = [resolutions] + + self._resolutions = [] + for resolution in resolutions: + if isinstance(resolution, int): + width = height = resolution + else: + width, height = resolution + assert isinstance(width, int), f'Bad type for {width=} {type(width)=}, should be int' + assert isinstance(height, int), f'Bad type for {height=} {type(height)=}, should be int' + assert width >= height + self._resolutions.append((width, height)) + + def _crop_resize_if_necessary(self, image, depthmap, intrinsics, resolution, rng=None, info=None): + """ This function: + - first downsizes the image with LANCZOS inteprolation, + which is better than bilinear interpolation in + """ + if not isinstance(image, PIL.Image.Image): + image = PIL.Image.fromarray(image) + + # downscale with lanczos interpolation so that image.size == resolution + # cropping centered on the principal point + W, H = image.size + cx, cy = intrinsics[:2, 2].round().astype(int) + min_margin_x = min(cx, W-cx) + min_margin_y = min(cy, H-cy) + assert min_margin_x > W/5, f'Bad principal point in view={info}' + assert min_margin_y > H/5, f'Bad principal point in view={info}' + # the new window will be a rectangle of size (2*min_margin_x, 2*min_margin_y) centered on (cx,cy) + l, t = cx - min_margin_x, cy - min_margin_y + r, b = cx + min_margin_x, cy + min_margin_y + crop_bbox = (l, t, r, b) + image, depthmap, intrinsics = cropping.crop_image_depthmap(image, depthmap, intrinsics, crop_bbox) + + # transpose the resolution if necessary + W, H = image.size # new size + assert resolution[0] >= resolution[1] + if H > 1.1*W: + # image is portrait mode + resolution = resolution[::-1] + elif 0.9 < H/W < 1.1 and resolution[0] != resolution[1]: + # image is square, so we chose (portrait, landscape) randomly + if rng.integers(2): + resolution = resolution[::-1] + + # high-quality Lanczos down-scaling + target_resolution = np.array(resolution) + if self.aug_crop > 1: + target_resolution += rng.integers(0, self.aug_crop) + image, depthmap, intrinsics = cropping.rescale_image_depthmap(image, depthmap, intrinsics, target_resolution) + + # actual cropping (if necessary) with bilinear interpolation + intrinsics2 = cropping.camera_matrix_of_crop(intrinsics, image.size, resolution, offset_factor=0.5) + crop_bbox = cropping.bbox_from_intrinsics_in_out(intrinsics, intrinsics2, resolution) + image, depthmap, intrinsics2 = cropping.crop_image_depthmap(image, depthmap, intrinsics, crop_bbox) + + return image, depthmap, intrinsics2 + + +def is_good_type(key, v): + """ returns (is_good, err_msg) + """ + if isinstance(v, (str, int, tuple)): + return True, None + if v.dtype not in (np.float32, torch.float32, bool, np.int32, np.int64, np.uint8): + return False, f"bad {v.dtype=}" + return True, None + + +def view_name(view, batch_index=None): + def sel(x): return x[batch_index] if batch_index not in (None, slice(None)) else x + db = sel(view['dataset']) + label = sel(view['label']) + instance = sel(view['instance']) + return f"{db}/{label}/{instance}" + + +def transpose_to_landscape(view): + height, width = view['true_shape'] + + if width < height: + # rectify portrait to landscape + assert view['img'].shape == (3, height, width) + view['img'] = view['img'].swapaxes(1, 2) + + assert view['valid_mask'].shape == (height, width) + view['valid_mask'] = view['valid_mask'].swapaxes(0, 1) + + assert view['depthmap'].shape == (height, width) + view['depthmap'] = view['depthmap'].swapaxes(0, 1) + + assert view['pts3d'].shape == (height, width, 3) + view['pts3d'] = view['pts3d'].swapaxes(0, 1) + + # transpose x and y pixels + view['camera_intrinsics'] = view['camera_intrinsics'][[1, 0, 2]] diff --git a/src/mast3r_src/dust3r/dust3r/datasets/base/batched_sampler.py b/src/mast3r_src/dust3r/dust3r/datasets/base/batched_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..85f58a65d41bb8101159e032d5b0aac26a7cf1a1 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/base/batched_sampler.py @@ -0,0 +1,74 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Random sampling under a constraint +# -------------------------------------------------------- +import numpy as np +import torch + + +class BatchedRandomSampler: + """ Random sampling under a constraint: each sample in the batch has the same feature, + which is chosen randomly from a known pool of 'features' for each batch. + + For instance, the 'feature' could be the image aspect-ratio. + + The index returned is a tuple (sample_idx, feat_idx). + This sampler ensures that each series of `batch_size` indices has the same `feat_idx`. + """ + + def __init__(self, dataset, batch_size, pool_size, world_size=1, rank=0, drop_last=True): + self.batch_size = batch_size + self.pool_size = pool_size + + self.len_dataset = N = len(dataset) + self.total_size = round_by(N, batch_size*world_size) if drop_last else N + assert world_size == 1 or drop_last, 'must drop the last batch in distributed mode' + + # distributed sampler + self.world_size = world_size + self.rank = rank + self.epoch = None + + def __len__(self): + return self.total_size // self.world_size + + def set_epoch(self, epoch): + self.epoch = epoch + + def __iter__(self): + # prepare RNG + if self.epoch is None: + assert self.world_size == 1 and self.rank == 0, 'use set_epoch() if distributed mode is used' + seed = int(torch.empty((), dtype=torch.int64).random_().item()) + else: + seed = self.epoch + 777 + rng = np.random.default_rng(seed=seed) + + # random indices (will restart from 0 if not drop_last) + sample_idxs = np.arange(self.total_size) + rng.shuffle(sample_idxs) + + # random feat_idxs (same across each batch) + n_batches = (self.total_size+self.batch_size-1) // self.batch_size + feat_idxs = rng.integers(self.pool_size, size=n_batches) + feat_idxs = np.broadcast_to(feat_idxs[:, None], (n_batches, self.batch_size)) + feat_idxs = feat_idxs.ravel()[:self.total_size] + + # put them together + idxs = np.c_[sample_idxs, feat_idxs] # shape = (total_size, 2) + + # Distributed sampler: we select a subset of batches + # make sure the slice for each node is aligned with batch_size + size_per_proc = self.batch_size * ((self.total_size + self.world_size * + self.batch_size-1) // (self.world_size * self.batch_size)) + idxs = idxs[self.rank*size_per_proc: (self.rank+1)*size_per_proc] + + yield from (tuple(idx) for idx in idxs) + + +def round_by(total, multiple, up=False): + if up: + total = total + multiple-1 + return (total//multiple) * multiple diff --git a/src/mast3r_src/dust3r/dust3r/datasets/base/easy_dataset.py b/src/mast3r_src/dust3r/dust3r/datasets/base/easy_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..4939a88f02715a1f80be943ddb6d808e1be84db7 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/base/easy_dataset.py @@ -0,0 +1,157 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# A dataset base class that you can easily resize and combine. +# -------------------------------------------------------- +import numpy as np +from dust3r.datasets.base.batched_sampler import BatchedRandomSampler + + +class EasyDataset: + """ a dataset that you can easily resize and combine. + Examples: + --------- + 2 * dataset ==> duplicate each element 2x + + 10 @ dataset ==> set the size to 10 (random sampling, duplicates if necessary) + + dataset1 + dataset2 ==> concatenate datasets + """ + + def __add__(self, other): + return CatDataset([self, other]) + + def __rmul__(self, factor): + return MulDataset(factor, self) + + def __rmatmul__(self, factor): + return ResizedDataset(factor, self) + + def set_epoch(self, epoch): + pass # nothing to do by default + + def make_sampler(self, batch_size, shuffle=True, world_size=1, rank=0, drop_last=True): + if not (shuffle): + raise NotImplementedError() # cannot deal yet + num_of_aspect_ratios = len(self._resolutions) + return BatchedRandomSampler(self, batch_size, num_of_aspect_ratios, world_size=world_size, rank=rank, drop_last=drop_last) + + +class MulDataset (EasyDataset): + """ Artifically augmenting the size of a dataset. + """ + multiplicator: int + + def __init__(self, multiplicator, dataset): + assert isinstance(multiplicator, int) and multiplicator > 0 + self.multiplicator = multiplicator + self.dataset = dataset + + def __len__(self): + return self.multiplicator * len(self.dataset) + + def __repr__(self): + return f'{self.multiplicator}*{repr(self.dataset)}' + + def __getitem__(self, idx): + if isinstance(idx, tuple): + idx, other = idx + return self.dataset[idx // self.multiplicator, other] + else: + return self.dataset[idx // self.multiplicator] + + @property + def _resolutions(self): + return self.dataset._resolutions + + +class ResizedDataset (EasyDataset): + """ Artifically changing the size of a dataset. + """ + new_size: int + + def __init__(self, new_size, dataset): + assert isinstance(new_size, int) and new_size > 0 + self.new_size = new_size + self.dataset = dataset + + def __len__(self): + return self.new_size + + def __repr__(self): + size_str = str(self.new_size) + for i in range((len(size_str)-1) // 3): + sep = -4*i-3 + size_str = size_str[:sep] + '_' + size_str[sep:] + return f'{size_str} @ {repr(self.dataset)}' + + def set_epoch(self, epoch): + # this random shuffle only depends on the epoch + rng = np.random.default_rng(seed=epoch+777) + + # shuffle all indices + perm = rng.permutation(len(self.dataset)) + + # rotary extension until target size is met + shuffled_idxs = np.concatenate([perm] * (1 + (len(self)-1) // len(self.dataset))) + self._idxs_mapping = shuffled_idxs[:self.new_size] + + assert len(self._idxs_mapping) == self.new_size + + def __getitem__(self, idx): + assert hasattr(self, '_idxs_mapping'), 'You need to call dataset.set_epoch() to use ResizedDataset.__getitem__()' + if isinstance(idx, tuple): + idx, other = idx + return self.dataset[self._idxs_mapping[idx], other] + else: + return self.dataset[self._idxs_mapping[idx]] + + @property + def _resolutions(self): + return self.dataset._resolutions + + +class CatDataset (EasyDataset): + """ Concatenation of several datasets + """ + + def __init__(self, datasets): + for dataset in datasets: + assert isinstance(dataset, EasyDataset) + self.datasets = datasets + self._cum_sizes = np.cumsum([len(dataset) for dataset in datasets]) + + def __len__(self): + return self._cum_sizes[-1] + + def __repr__(self): + # remove uselessly long transform + return ' + '.join(repr(dataset).replace(',transform=Compose( ToTensor() Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)))', '') for dataset in self.datasets) + + def set_epoch(self, epoch): + for dataset in self.datasets: + dataset.set_epoch(epoch) + + def __getitem__(self, idx): + other = None + if isinstance(idx, tuple): + idx, other = idx + + if not (0 <= idx < len(self)): + raise IndexError() + + db_idx = np.searchsorted(self._cum_sizes, idx, 'right') + dataset = self.datasets[db_idx] + new_idx = idx - (self._cum_sizes[db_idx - 1] if db_idx > 0 else 0) + + if other is not None: + new_idx = (new_idx, other) + return dataset[new_idx] + + @property + def _resolutions(self): + resolutions = self.datasets[0]._resolutions + for dataset in self.datasets[1:]: + assert tuple(dataset._resolutions) == tuple(resolutions) + return resolutions diff --git a/src/mast3r_src/dust3r/dust3r/datasets/blendedmvs.py b/src/mast3r_src/dust3r/dust3r/datasets/blendedmvs.py new file mode 100644 index 0000000000000000000000000000000000000000..6459a9c47cd9f0c7d109eebe8a00370bd0005c06 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/blendedmvs.py @@ -0,0 +1,114 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed BlendedMVS +# dataset at https://github.com/YoYo000/BlendedMVS +# See datasets_preprocess/preprocess_blendedmvs.py +# -------------------------------------------------------- +import os.path as osp +import numpy as np + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +from dust3r.utils.image import imread_cv2 + + +class BlendedMVS (BaseStereoViewDataset): + """ Dataset of outdoor street scenes, 5 images each time + """ + + def __init__(self, *args, ROOT, split=None, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + self._load_data(split) + + def _load_data(self, split): + pairs = np.load(osp.join(self.ROOT, 'blendedmvs_pairs.npy')) + if split is None: + selection = slice(None) + if split == 'train': + # select 90% of all scenes + selection = (pairs['seq_low'] % 10) > 0 + if split == 'val': + # select 10% of all scenes + selection = (pairs['seq_low'] % 10) == 0 + self.pairs = pairs[selection] + + # list of all scenes + self.scenes = np.unique(self.pairs['seq_low']) # low is unique enough + + def __len__(self): + return len(self.pairs) + + def get_stats(self): + return f'{len(self)} pairs from {len(self.scenes)} scenes' + + def select_one_scene(self, scene, img1=None, img2=None): + scene_low = int(scene[-16:], 16) + valid = (self.pairs['seq_low'] == scene_low) + if img1: + valid &= (self.pairs['img1'] == int(img1)) + if img2: + valid &= (self.pairs['img2'] == int(img2)) + self.pairs = self.pairs[valid] + self._compute_pair_probas() + + def _get_views(self, pair_idx, resolution, rng): + seqh, seql, img1, img2, score = self.pairs[pair_idx] + + seq = f"{seqh:08x}{seql:016x}" + seq_path = osp.join(self.ROOT, seq) + + views = [] + + for view_index in [img1, img2]: + impath = f"{view_index:08n}" + image = imread_cv2(osp.join(seq_path, impath + ".jpg")) + depthmap = imread_cv2(osp.join(seq_path, impath + ".exr")) + camera_params = np.load(osp.join(seq_path, impath + ".npz")) + + intrinsics = np.float32(camera_params['intrinsics']) + camera_pose = np.eye(4, dtype=np.float32) + camera_pose[:3, :3] = camera_params['R_cam2world'] + camera_pose[:3, 3] = camera_params['t_cam2world'] + + image, depthmap, intrinsics = self._crop_resize_if_necessary( + image, depthmap, intrinsics, resolution, rng, info=(seq_path, impath)) + + views.append(dict( + img=image, + depthmap=depthmap, + camera_pose=camera_pose, # cam2world + camera_intrinsics=intrinsics, + dataset='Waymo', + label=osp.relpath(seq_path, self.ROOT), + instance=impath)) + + return views + + +if __name__ == '__main__': + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = BlendedMVS(split='train', ROOT="data/blendedmvs_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(idx, view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx * 255, (1 - idx) * 255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/src/mast3r_src/dust3r/dust3r/datasets/co3d.py b/src/mast3r_src/dust3r/dust3r/datasets/co3d.py new file mode 100644 index 0000000000000000000000000000000000000000..2ea5c8555d34b776e7a48396dcd0eecece713e34 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/co3d.py @@ -0,0 +1,165 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed Co3d_v2 +# dataset at https://github.com/facebookresearch/co3d - Creative Commons Attribution-NonCommercial 4.0 International +# See datasets_preprocess/preprocess_co3d.py +# -------------------------------------------------------- +import os.path as osp +import json +import itertools +from collections import deque + +import cv2 +import numpy as np + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +from dust3r.utils.image import imread_cv2 + + +class Co3d(BaseStereoViewDataset): + def __init__(self, mask_bg=True, *args, ROOT, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + assert mask_bg in (True, False, 'rand') + self.mask_bg = mask_bg + self.dataset_label = 'Co3d_v2' + + # load all scenes + with open(osp.join(self.ROOT, f'selected_seqs_{self.split}.json'), 'r') as f: + self.scenes = json.load(f) + self.scenes = {k: v for k, v in self.scenes.items() if len(v) > 0} + self.scenes = {(k, k2): v2 for k, v in self.scenes.items() + for k2, v2 in v.items()} + self.scene_list = list(self.scenes.keys()) + + # for each scene, we have 100 images ==> 360 degrees (so 25 frames ~= 90 degrees) + # we prepare all combinations such that i-j = +/- [5, 10, .., 90] degrees + self.combinations = [(i, j) + for i, j in itertools.combinations(range(100), 2) + if 0 < abs(i - j) <= 30 and abs(i - j) % 5 == 0] + + self.invalidate = {scene: {} for scene in self.scene_list} + + def __len__(self): + return len(self.scene_list) * len(self.combinations) + + def _get_metadatapath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'images', f'frame{view_idx:06n}.npz') + + def _get_impath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'images', f'frame{view_idx:06n}.jpg') + + def _get_depthpath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'depths', f'frame{view_idx:06n}.jpg.geometric.png') + + def _get_maskpath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'masks', f'frame{view_idx:06n}.png') + + def _read_depthmap(self, depthpath, input_metadata): + depthmap = imread_cv2(depthpath, cv2.IMREAD_UNCHANGED) + depthmap = (depthmap.astype(np.float32) / 65535) * np.nan_to_num(input_metadata['maximum_depth']) + return depthmap + + def _get_views(self, idx, resolution, rng): + # choose a scene + obj, instance = self.scene_list[idx // len(self.combinations)] + image_pool = self.scenes[obj, instance] + im1_idx, im2_idx = self.combinations[idx % len(self.combinations)] + + # add a bit of randomness + last = len(image_pool) - 1 + + if resolution not in self.invalidate[obj, instance]: # flag invalid images + self.invalidate[obj, instance][resolution] = [False for _ in range(len(image_pool))] + + # decide now if we mask the bg + mask_bg = (self.mask_bg == True) or (self.mask_bg == 'rand' and rng.choice(2)) + + views = [] + imgs_idxs = [max(0, min(im_idx + rng.integers(-4, 5), last)) for im_idx in [im2_idx, im1_idx]] + imgs_idxs = deque(imgs_idxs) + while len(imgs_idxs) > 0: # some images (few) have zero depth + im_idx = imgs_idxs.pop() + + if self.invalidate[obj, instance][resolution][im_idx]: + # search for a valid image + random_direction = 2 * rng.choice(2) - 1 + for offset in range(1, len(image_pool)): + tentative_im_idx = (im_idx + (random_direction * offset)) % len(image_pool) + if not self.invalidate[obj, instance][resolution][tentative_im_idx]: + im_idx = tentative_im_idx + break + + view_idx = image_pool[im_idx] + + impath = self._get_impath(obj, instance, view_idx) + depthpath = self._get_depthpath(obj, instance, view_idx) + + # load camera params + metadata_path = self._get_metadatapath(obj, instance, view_idx) + input_metadata = np.load(metadata_path) + camera_pose = input_metadata['camera_pose'].astype(np.float32) + intrinsics = input_metadata['camera_intrinsics'].astype(np.float32) + + # load image and depth + rgb_image = imread_cv2(impath) + depthmap = self._read_depthmap(depthpath, input_metadata) + + if mask_bg: + # load object mask + maskpath = self._get_maskpath(obj, instance, view_idx) + maskmap = imread_cv2(maskpath, cv2.IMREAD_UNCHANGED).astype(np.float32) + maskmap = (maskmap / 255.0) > 0.1 + + # update the depthmap with mask + depthmap *= maskmap + + rgb_image, depthmap, intrinsics = self._crop_resize_if_necessary( + rgb_image, depthmap, intrinsics, resolution, rng=rng, info=impath) + + num_valid = (depthmap > 0.0).sum() + if num_valid == 0: + # problem, invalidate image and retry + self.invalidate[obj, instance][resolution][im_idx] = True + imgs_idxs.append(im_idx) + continue + + views.append(dict( + img=rgb_image, + depthmap=depthmap, + camera_pose=camera_pose, + camera_intrinsics=intrinsics, + dataset=self.dataset_label, + label=osp.join(obj, instance), + instance=osp.split(impath)[1], + )) + return views + + +if __name__ == "__main__": + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = Co3d(split='train', ROOT="data/co3d_subset_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx * 255, (1 - idx) * 255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/src/mast3r_src/dust3r/dust3r/datasets/habitat.py b/src/mast3r_src/dust3r/dust3r/datasets/habitat.py new file mode 100644 index 0000000000000000000000000000000000000000..11ce8a0ffb2134387d5fb794df89834db3ea8c9f --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/habitat.py @@ -0,0 +1,107 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed habitat +# dataset at https://github.com/facebookresearch/habitat-sim/blob/main/DATASETS.md +# See datasets_preprocess/habitat for more details +# -------------------------------------------------------- +import os.path as osp +import os +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" # noqa +import cv2 # noqa +import numpy as np +from PIL import Image +import json + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset + + +class Habitat(BaseStereoViewDataset): + def __init__(self, size, *args, ROOT, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + assert self.split is not None + # loading list of scenes + with open(osp.join(self.ROOT, f'Habitat_{size}_scenes_{self.split}.txt')) as f: + self.scenes = f.read().splitlines() + self.instances = list(range(1, 5)) + + def filter_scene(self, label, instance=None): + if instance: + subscene, instance = instance.split('_') + label += '/' + subscene + self.instances = [int(instance) - 1] + valid = np.bool_([scene.startswith(label) for scene in self.scenes]) + assert sum(valid), 'no scene was selected for {label=} {instance=}' + self.scenes = [scene for i, scene in enumerate(self.scenes) if valid[i]] + + def _get_views(self, idx, resolution, rng): + scene = self.scenes[idx] + data_path, key = osp.split(osp.join(self.ROOT, scene)) + views = [] + two_random_views = [0, rng.choice(self.instances)] # view 0 is connected with all other views + for view_index in two_random_views: + # load the view (and use the next one if this one's broken) + for ii in range(view_index, view_index + 5): + image, depthmap, intrinsics, camera_pose = self._load_one_view(data_path, key, ii % 5, resolution, rng) + if np.isfinite(camera_pose).all(): + break + views.append(dict( + img=image, + depthmap=depthmap, + camera_pose=camera_pose, # cam2world + camera_intrinsics=intrinsics, + dataset='Habitat', + label=osp.relpath(data_path, self.ROOT), + instance=f"{key}_{view_index}")) + return views + + def _load_one_view(self, data_path, key, view_index, resolution, rng): + view_index += 1 # file indices starts at 1 + impath = osp.join(data_path, f"{key}_{view_index}.jpeg") + image = Image.open(impath) + + depthmap_filename = osp.join(data_path, f"{key}_{view_index}_depth.exr") + depthmap = cv2.imread(depthmap_filename, cv2.IMREAD_GRAYSCALE | cv2.IMREAD_ANYDEPTH) + + camera_params_filename = osp.join(data_path, f"{key}_{view_index}_camera_params.json") + with open(camera_params_filename, 'r') as f: + camera_params = json.load(f) + + intrinsics = np.float32(camera_params['camera_intrinsics']) + camera_pose = np.eye(4, dtype=np.float32) + camera_pose[:3, :3] = camera_params['R_cam2world'] + camera_pose[:3, 3] = camera_params['t_cam2world'] + + image, depthmap, intrinsics = self._crop_resize_if_necessary( + image, depthmap, intrinsics, resolution, rng, info=impath) + return image, depthmap, intrinsics, camera_pose + + +if __name__ == "__main__": + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = Habitat(1_000_000, split='train', ROOT="data/habitat_processed", + resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx * 255, (1 - idx) * 255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/src/mast3r_src/dust3r/dust3r/datasets/megadepth.py b/src/mast3r_src/dust3r/dust3r/datasets/megadepth.py new file mode 100644 index 0000000000000000000000000000000000000000..8131498b76d855e5293fe79b3686fc42bf87eea8 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/megadepth.py @@ -0,0 +1,123 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed MegaDepth +# dataset at https://www.cs.cornell.edu/projects/megadepth/ +# See datasets_preprocess/preprocess_megadepth.py +# -------------------------------------------------------- +import os.path as osp +import numpy as np + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +from dust3r.utils.image import imread_cv2 + + +class MegaDepth(BaseStereoViewDataset): + def __init__(self, *args, split, ROOT, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + self.loaded_data = self._load_data(self.split) + + if self.split is None: + pass + elif self.split == 'train': + self.select_scene(('0015', '0022'), opposite=True) + elif self.split == 'val': + self.select_scene(('0015', '0022')) + else: + raise ValueError(f'bad {self.split=}') + + def _load_data(self, split): + with np.load(osp.join(self.ROOT, 'all_metadata.npz')) as data: + self.all_scenes = data['scenes'] + self.all_images = data['images'] + self.pairs = data['pairs'] + + def __len__(self): + return len(self.pairs) + + def get_stats(self): + return f'{len(self)} pairs from {len(self.all_scenes)} scenes' + + def select_scene(self, scene, *instances, opposite=False): + scenes = (scene,) if isinstance(scene, str) else tuple(scene) + scene_id = [s.startswith(scenes) for s in self.all_scenes] + assert any(scene_id), 'no scene found' + + valid = np.in1d(self.pairs['scene_id'], np.nonzero(scene_id)[0]) + if instances: + image_id = [i.startswith(instances) for i in self.all_images] + image_id = np.nonzero(image_id)[0] + assert len(image_id), 'no instance found' + # both together? + if len(instances) == 2: + valid &= np.in1d(self.pairs['im1_id'], image_id) & np.in1d(self.pairs['im2_id'], image_id) + else: + valid &= np.in1d(self.pairs['im1_id'], image_id) | np.in1d(self.pairs['im2_id'], image_id) + + if opposite: + valid = ~valid + assert valid.any() + self.pairs = self.pairs[valid] + + def _get_views(self, pair_idx, resolution, rng): + scene_id, im1_id, im2_id, score = self.pairs[pair_idx] + + scene, subscene = self.all_scenes[scene_id].split() + seq_path = osp.join(self.ROOT, scene, subscene) + + views = [] + + for im_id in [im1_id, im2_id]: + img = self.all_images[im_id] + try: + image = imread_cv2(osp.join(seq_path, img + '.jpg')) + depthmap = imread_cv2(osp.join(seq_path, img + ".exr")) + camera_params = np.load(osp.join(seq_path, img + ".npz")) + except Exception as e: + raise OSError(f'cannot load {img}, got exception {e}') + + intrinsics = np.float32(camera_params['intrinsics']) + camera_pose = np.float32(camera_params['cam2world']) + + image, depthmap, intrinsics = self._crop_resize_if_necessary( + image, depthmap, intrinsics, resolution, rng, info=(seq_path, img)) + + views.append(dict( + img=image, + depthmap=depthmap, + camera_pose=camera_pose, # cam2world + camera_intrinsics=intrinsics, + dataset='MegaDepth', + label=osp.relpath(seq_path, self.ROOT), + instance=img)) + + return views + + +if __name__ == "__main__": + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = MegaDepth(split='train', ROOT="data/megadepth_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(idx, view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx * 255, (1 - idx) * 255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/src/mast3r_src/dust3r/dust3r/datasets/scannetpp.py b/src/mast3r_src/dust3r/dust3r/datasets/scannetpp.py new file mode 100644 index 0000000000000000000000000000000000000000..520deedd0eb8cba8663af941731d89e0b2e71a80 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/scannetpp.py @@ -0,0 +1,96 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed scannet++ +# dataset at https://github.com/scannetpp/scannetpp - non-commercial research and educational purposes +# https://kaldir.vc.in.tum.de/scannetpp/static/scannetpp-terms-of-use.pdf +# See datasets_preprocess/preprocess_scannetpp.py +# -------------------------------------------------------- +import os.path as osp +import cv2 +import numpy as np + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +from dust3r.utils.image import imread_cv2 + + +class ScanNetpp(BaseStereoViewDataset): + def __init__(self, *args, ROOT, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + assert self.split == 'train' + self.loaded_data = self._load_data() + + def _load_data(self): + with np.load(osp.join(self.ROOT, 'all_metadata.npz')) as data: + self.scenes = data['scenes'] + self.sceneids = data['sceneids'] + self.images = data['images'] + self.intrinsics = data['intrinsics'].astype(np.float32) + self.trajectories = data['trajectories'].astype(np.float32) + self.pairs = data['pairs'][:, :2].astype(int) + + def __len__(self): + return len(self.pairs) + + def _get_views(self, idx, resolution, rng): + + image_idx1, image_idx2 = self.pairs[idx] + + views = [] + for view_idx in [image_idx1, image_idx2]: + scene_id = self.sceneids[view_idx] + scene_dir = osp.join(self.ROOT, self.scenes[scene_id]) + + intrinsics = self.intrinsics[view_idx] + camera_pose = self.trajectories[view_idx] + basename = self.images[view_idx] + + # Load RGB image + rgb_image = imread_cv2(osp.join(scene_dir, 'images', basename + '.jpg')) + # Load depthmap + depthmap = imread_cv2(osp.join(scene_dir, 'depth', basename + '.png'), cv2.IMREAD_UNCHANGED) + depthmap = depthmap.astype(np.float32) / 1000 + depthmap[~np.isfinite(depthmap)] = 0 # invalid + + rgb_image, depthmap, intrinsics = self._crop_resize_if_necessary( + rgb_image, depthmap, intrinsics, resolution, rng=rng, info=view_idx) + + views.append(dict( + img=rgb_image, + depthmap=depthmap.astype(np.float32), + camera_pose=camera_pose.astype(np.float32), + camera_intrinsics=intrinsics.astype(np.float32), + dataset='ScanNet++', + label=self.scenes[scene_id] + '_' + basename, + instance=f'{str(idx)}_{str(view_idx)}', + )) + return views + + +if __name__ == "__main__": + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = ScanNetpp(split='train', ROOT="data/scannetpp_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx*255, (1 - idx)*255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/src/mast3r_src/dust3r/dust3r/datasets/staticthings3d.py b/src/mast3r_src/dust3r/dust3r/datasets/staticthings3d.py new file mode 100644 index 0000000000000000000000000000000000000000..e7f70f0ee7bf8c8ab6bb1702aa2481f3d16df413 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/staticthings3d.py @@ -0,0 +1,96 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed StaticThings3D +# dataset at https://github.com/lmb-freiburg/robustmvd/ +# See datasets_preprocess/preprocess_staticthings3d.py +# -------------------------------------------------------- +import os.path as osp +import numpy as np + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +from dust3r.utils.image import imread_cv2 + + +class StaticThings3D (BaseStereoViewDataset): + """ Dataset of indoor scenes, 5 images each time + """ + def __init__(self, ROOT, *args, mask_bg='rand', **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + + assert mask_bg in (True, False, 'rand') + self.mask_bg = mask_bg + + # loading all pairs + assert self.split is None + self.pairs = np.load(osp.join(ROOT, 'staticthings_pairs.npy')) + + def __len__(self): + return len(self.pairs) + + def get_stats(self): + return f'{len(self)} pairs' + + def _get_views(self, pair_idx, resolution, rng): + scene, seq, cam1, im1, cam2, im2 = self.pairs[pair_idx] + seq_path = osp.join('TRAIN', scene.decode('ascii'), f'{seq:04d}') + + views = [] + + mask_bg = (self.mask_bg == True) or (self.mask_bg == 'rand' and rng.choice(2)) + + CAM = {b'l':'left', b'r':'right'} + for cam, idx in [(CAM[cam1], im1), (CAM[cam2], im2)]: + num = f"{idx:04n}" + img = num+"_clean.jpg" if rng.choice(2) else num+"_final.jpg" + image = imread_cv2(osp.join(self.ROOT, seq_path, cam, img)) + depthmap = imread_cv2(osp.join(self.ROOT, seq_path, cam, num+".exr")) + camera_params = np.load(osp.join(self.ROOT, seq_path, cam, num+".npz")) + + intrinsics = camera_params['intrinsics'] + camera_pose = camera_params['cam2world'] + + if mask_bg: + depthmap[depthmap > 200] = 0 + + image, depthmap, intrinsics = self._crop_resize_if_necessary(image, depthmap, intrinsics, resolution, rng, info=(seq_path,cam,img)) + + views.append(dict( + img = image, + depthmap = depthmap, + camera_pose = camera_pose, # cam2world + camera_intrinsics = intrinsics, + dataset = 'StaticThings3D', + label = seq_path, + instance = cam+'_'+img)) + + return views + + +if __name__ == '__main__': + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = StaticThings3D(ROOT="data/staticthings3d_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(idx, view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx*255, (1 - idx)*255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/src/mast3r_src/dust3r/dust3r/datasets/utils/__init__.py b/src/mast3r_src/dust3r/dust3r/datasets/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a32692113d830ddc4af4e6ed608f222fbe062e6e --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/utils/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). diff --git a/src/mast3r_src/dust3r/dust3r/datasets/utils/cropping.py b/src/mast3r_src/dust3r/dust3r/datasets/utils/cropping.py new file mode 100644 index 0000000000000000000000000000000000000000..bfb4eaa92d21d0ecb8473faa60e5fc13ddf317e3 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/utils/cropping.py @@ -0,0 +1,124 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# croppping utilities +# -------------------------------------------------------- +import PIL.Image +import os +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +import cv2 # noqa +import numpy as np # noqa +from dust3r.utils.geometry import colmap_to_opencv_intrinsics, opencv_to_colmap_intrinsics # noqa +try: + lanczos = PIL.Image.Resampling.LANCZOS + bicubic = PIL.Image.Resampling.BICUBIC +except AttributeError: + lanczos = PIL.Image.LANCZOS + bicubic = PIL.Image.BICUBIC + + +class ImageList: + """ Convenience class to aply the same operation to a whole set of images. + """ + + def __init__(self, images): + if not isinstance(images, (tuple, list, set)): + images = [images] + self.images = [] + for image in images: + if not isinstance(image, PIL.Image.Image): + image = PIL.Image.fromarray(image) + self.images.append(image) + + def __len__(self): + return len(self.images) + + def to_pil(self): + return tuple(self.images) if len(self.images) > 1 else self.images[0] + + @property + def size(self): + sizes = [im.size for im in self.images] + assert all(sizes[0] == s for s in sizes) + return sizes[0] + + def resize(self, *args, **kwargs): + return ImageList(self._dispatch('resize', *args, **kwargs)) + + def crop(self, *args, **kwargs): + return ImageList(self._dispatch('crop', *args, **kwargs)) + + def _dispatch(self, func, *args, **kwargs): + return [getattr(im, func)(*args, **kwargs) for im in self.images] + + +def rescale_image_depthmap(image, depthmap, camera_intrinsics, output_resolution, force=True): + """ Jointly rescale a (image, depthmap) + so that (out_width, out_height) >= output_res + """ + image = ImageList(image) + input_resolution = np.array(image.size) # (W,H) + output_resolution = np.array(output_resolution) + if depthmap is not None: + # can also use this with masks instead of depthmaps + assert tuple(depthmap.shape[:2]) == image.size[::-1] + + # define output resolution + assert output_resolution.shape == (2,) + scale_final = max(output_resolution / image.size) + 1e-8 + if scale_final >= 1 and not force: # image is already smaller than what is asked + return (image.to_pil(), depthmap, camera_intrinsics) + output_resolution = np.floor(input_resolution * scale_final).astype(int) + + # first rescale the image so that it contains the crop + image = image.resize(output_resolution, resample=lanczos if scale_final < 1 else bicubic) + if depthmap is not None: + depthmap = cv2.resize(depthmap, output_resolution, fx=scale_final, + fy=scale_final, interpolation=cv2.INTER_NEAREST) + + # no offset here; simple rescaling + camera_intrinsics = camera_matrix_of_crop( + camera_intrinsics, input_resolution, output_resolution, scaling=scale_final) + + return image.to_pil(), depthmap, camera_intrinsics + + +def camera_matrix_of_crop(input_camera_matrix, input_resolution, output_resolution, scaling=1, offset_factor=0.5, offset=None): + # Margins to offset the origin + margins = np.asarray(input_resolution) * scaling - output_resolution + assert np.all(margins >= 0.0) + if offset is None: + offset = offset_factor * margins + + # Generate new camera parameters + output_camera_matrix_colmap = opencv_to_colmap_intrinsics(input_camera_matrix) + output_camera_matrix_colmap[:2, :] *= scaling + output_camera_matrix_colmap[:2, 2] -= offset + output_camera_matrix = colmap_to_opencv_intrinsics(output_camera_matrix_colmap) + + return output_camera_matrix + + +def crop_image_depthmap(image, depthmap, camera_intrinsics, crop_bbox): + """ + Return a crop of the input view. + """ + image = ImageList(image) + l, t, r, b = crop_bbox + + image = image.crop((l, t, r, b)) + depthmap = depthmap[t:b, l:r] + + camera_intrinsics = camera_intrinsics.copy() + camera_intrinsics[0, 2] -= l + camera_intrinsics[1, 2] -= t + + return image.to_pil(), depthmap, camera_intrinsics + + +def bbox_from_intrinsics_in_out(input_camera_matrix, output_camera_matrix, output_resolution): + out_width, out_height = output_resolution + l, t = np.int32(np.round(input_camera_matrix[:2, 2] - output_camera_matrix[:2, 2])) + crop_bbox = (l, t, l + out_width, t + out_height) + return crop_bbox diff --git a/src/mast3r_src/dust3r/dust3r/datasets/utils/transforms.py b/src/mast3r_src/dust3r/dust3r/datasets/utils/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..eb34f2f01d3f8f829ba71a7e03e181bf18f72c25 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/utils/transforms.py @@ -0,0 +1,11 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# DUST3R default transforms +# -------------------------------------------------------- +import torchvision.transforms as tvf +from dust3r.utils.image import ImgNorm + +# define the standard image transforms +ColorJitter = tvf.Compose([tvf.ColorJitter(0.5, 0.5, 0.5, 0.1), ImgNorm]) diff --git a/src/mast3r_src/dust3r/dust3r/datasets/waymo.py b/src/mast3r_src/dust3r/dust3r/datasets/waymo.py new file mode 100644 index 0000000000000000000000000000000000000000..b9a135152cd8973532405b491450c22942dcd6ca --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/waymo.py @@ -0,0 +1,93 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed WayMo +# dataset at https://github.com/waymo-research/waymo-open-dataset +# See datasets_preprocess/preprocess_waymo.py +# -------------------------------------------------------- +import os.path as osp +import numpy as np + +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset +from dust3r.utils.image import imread_cv2 + + +class Waymo (BaseStereoViewDataset): + """ Dataset of outdoor street scenes, 5 images each time + """ + + def __init__(self, *args, ROOT, **kwargs): + self.ROOT = ROOT + super().__init__(*args, **kwargs) + self._load_data() + + def _load_data(self): + with np.load(osp.join(self.ROOT, 'waymo_pairs.npz')) as data: + self.scenes = data['scenes'] + self.frames = data['frames'] + self.inv_frames = {frame: i for i, frame in enumerate(data['frames'])} + self.pairs = data['pairs'] # (array of (scene_id, img1_id, img2_id) + assert self.pairs[:, 0].max() == len(self.scenes) - 1 + + def __len__(self): + return len(self.pairs) + + def get_stats(self): + return f'{len(self)} pairs from {len(self.scenes)} scenes' + + def _get_views(self, pair_idx, resolution, rng): + seq, img1, img2 = self.pairs[pair_idx] + seq_path = osp.join(self.ROOT, self.scenes[seq]) + + views = [] + + for view_index in [img1, img2]: + impath = self.frames[view_index] + image = imread_cv2(osp.join(seq_path, impath + ".jpg")) + depthmap = imread_cv2(osp.join(seq_path, impath + ".exr")) + camera_params = np.load(osp.join(seq_path, impath + ".npz")) + + intrinsics = np.float32(camera_params['intrinsics']) + camera_pose = np.float32(camera_params['cam2world']) + + image, depthmap, intrinsics = self._crop_resize_if_necessary( + image, depthmap, intrinsics, resolution, rng, info=(seq_path, impath)) + + views.append(dict( + img=image, + depthmap=depthmap, + camera_pose=camera_pose, # cam2world + camera_intrinsics=intrinsics, + dataset='Waymo', + label=osp.relpath(seq_path, self.ROOT), + instance=impath)) + + return views + + +if __name__ == '__main__': + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = Waymo(split='train', ROOT="data/megadepth_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(idx, view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx * 255, (1 - idx) * 255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/src/mast3r_src/dust3r/dust3r/datasets/wildrgbd.py b/src/mast3r_src/dust3r/dust3r/datasets/wildrgbd.py new file mode 100644 index 0000000000000000000000000000000000000000..c41dd0b78402bf8ff1e62c6a50de338aa916e0af --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/datasets/wildrgbd.py @@ -0,0 +1,67 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Dataloader for preprocessed WildRGB-D +# dataset at https://github.com/wildrgbd/wildrgbd/ +# See datasets_preprocess/preprocess_wildrgbd.py +# -------------------------------------------------------- +import os.path as osp + +import cv2 +import numpy as np + +from dust3r.datasets.co3d import Co3d +from dust3r.utils.image import imread_cv2 + + +class WildRGBD(Co3d): + def __init__(self, mask_bg=True, *args, ROOT, **kwargs): + super().__init__(mask_bg, *args, ROOT=ROOT, **kwargs) + self.dataset_label = 'WildRGBD' + + def _get_metadatapath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'metadata', f'{view_idx:0>5d}.npz') + + def _get_impath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'rgb', f'{view_idx:0>5d}.jpg') + + def _get_depthpath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'depth', f'{view_idx:0>5d}.png') + + def _get_maskpath(self, obj, instance, view_idx): + return osp.join(self.ROOT, obj, instance, 'masks', f'{view_idx:0>5d}.png') + + def _read_depthmap(self, depthpath, input_metadata): + # We store depths in the depth scale of 1000. + # That is, when we load depth image and divide by 1000, we could get depth in meters. + depthmap = imread_cv2(depthpath, cv2.IMREAD_UNCHANGED) + depthmap = depthmap.astype(np.float32) / 1000.0 + return depthmap + + +if __name__ == "__main__": + from dust3r.datasets.base.base_stereo_view_dataset import view_name + from dust3r.viz import SceneViz, auto_cam_size + from dust3r.utils.image import rgb + + dataset = WildRGBD(split='train', ROOT="data/wildrgbd_processed", resolution=224, aug_crop=16) + + for idx in np.random.permutation(len(dataset)): + views = dataset[idx] + assert len(views) == 2 + print(view_name(views[0]), view_name(views[1])) + viz = SceneViz() + poses = [views[view_idx]['camera_pose'] for view_idx in [0, 1]] + cam_size = max(auto_cam_size(poses), 0.001) + for view_idx in [0, 1]: + pts3d = views[view_idx]['pts3d'] + valid_mask = views[view_idx]['valid_mask'] + colors = rgb(views[view_idx]['img']) + viz.add_pointcloud(pts3d, colors, valid_mask) + viz.add_camera(pose_c2w=views[view_idx]['camera_pose'], + focal=views[view_idx]['camera_intrinsics'][0, 0], + color=(idx * 255, (1 - idx) * 255, 0), + image=colors, + cam_size=cam_size) + viz.show() diff --git a/src/mast3r_src/dust3r/dust3r/demo.py b/src/mast3r_src/dust3r/dust3r/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..2f2ae673943436816e691b280f874df5273595ba --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/demo.py @@ -0,0 +1,266 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# gradio demo +# -------------------------------------------------------- +import argparse +import math +import gradio +import os +import torch +import numpy as np +import functools +import trimesh +import copy +from scipy.spatial.transform import Rotation + +from dust3r.inference import inference +from dust3r.image_pairs import make_pairs +from dust3r.utils.image import load_images, rgb +from dust3r.utils.device import to_numpy +from dust3r.viz import add_scene_cam, CAM_COLORS, OPENGL, pts3d_to_trimesh, cat_meshes +from dust3r.cloud_opt import global_aligner, GlobalAlignerMode + +import matplotlib.pyplot as pl + + +def get_args_parser(): + parser = argparse.ArgumentParser() + parser_url = parser.add_mutually_exclusive_group() + parser_url.add_argument("--local_network", action='store_true', default=False, + help="make app accessible on local network: address will be set to 0.0.0.0") + parser_url.add_argument("--server_name", type=str, default=None, help="server url, default is 127.0.0.1") + parser.add_argument("--image_size", type=int, default=512, choices=[512, 224], help="image size") + parser.add_argument("--server_port", type=int, help=("will start gradio app on this port (if available). " + "If None, will search for an available port starting at 7860."), + default=None) + parser_weights = parser.add_mutually_exclusive_group(required=True) + parser_weights.add_argument("--weights", type=str, help="path to the model weights", default=None) + parser_weights.add_argument("--model_name", type=str, help="name of the model weights", + choices=["DUSt3R_ViTLarge_BaseDecoder_512_dpt", + "DUSt3R_ViTLarge_BaseDecoder_512_linear", + "DUSt3R_ViTLarge_BaseDecoder_224_linear"]) + parser.add_argument("--device", type=str, default='cuda', help="pytorch device") + parser.add_argument("--tmp_dir", type=str, default=None, help="value for tempfile.tempdir") + parser.add_argument("--silent", action='store_true', default=False, + help="silence logs") + return parser + + +def _convert_scene_output_to_glb(outdir, imgs, pts3d, mask, focals, cams2world, cam_size=0.05, + cam_color=None, as_pointcloud=False, + transparent_cams=False, silent=False): + assert len(pts3d) == len(mask) <= len(imgs) <= len(cams2world) == len(focals) + pts3d = to_numpy(pts3d) + imgs = to_numpy(imgs) + focals = to_numpy(focals) + cams2world = to_numpy(cams2world) + + scene = trimesh.Scene() + + # full pointcloud + if as_pointcloud: + pts = np.concatenate([p[m] for p, m in zip(pts3d, mask)]) + col = np.concatenate([p[m] for p, m in zip(imgs, mask)]) + pct = trimesh.PointCloud(pts.reshape(-1, 3), colors=col.reshape(-1, 3)) + scene.add_geometry(pct) + else: + meshes = [] + for i in range(len(imgs)): + meshes.append(pts3d_to_trimesh(imgs[i], pts3d[i], mask[i])) + mesh = trimesh.Trimesh(**cat_meshes(meshes)) + scene.add_geometry(mesh) + + # add each camera + for i, pose_c2w in enumerate(cams2world): + if isinstance(cam_color, list): + camera_edge_color = cam_color[i] + else: + camera_edge_color = cam_color or CAM_COLORS[i % len(CAM_COLORS)] + add_scene_cam(scene, pose_c2w, camera_edge_color, + None if transparent_cams else imgs[i], focals[i], + imsize=imgs[i].shape[1::-1], screen_width=cam_size) + + rot = np.eye(4) + rot[:3, :3] = Rotation.from_euler('y', np.deg2rad(180)).as_matrix() + scene.apply_transform(np.linalg.inv(cams2world[0] @ OPENGL @ rot)) + outfile = os.path.join(outdir, 'scene.glb') + if not silent: + print('(exporting 3D scene to', outfile, ')') + scene.export(file_obj=outfile) + return outfile + + +def get_3D_model_from_scene(outdir, silent, scene, min_conf_thr=3, as_pointcloud=False, mask_sky=False, + clean_depth=False, transparent_cams=False, cam_size=0.05): + """ + extract 3D_model (glb file) from a reconstructed scene + """ + if scene is None: + return None + # post processes + if clean_depth: + scene = scene.clean_pointcloud() + if mask_sky: + scene = scene.mask_sky() + + # get optimized values from scene + rgbimg = scene.imgs + focals = scene.get_focals().cpu() + cams2world = scene.get_im_poses().cpu() + # 3D pointcloud from depthmap, poses and intrinsics + pts3d = to_numpy(scene.get_pts3d()) + scene.min_conf_thr = float(scene.conf_trf(torch.tensor(min_conf_thr))) + msk = to_numpy(scene.get_masks()) + return _convert_scene_output_to_glb(outdir, rgbimg, pts3d, msk, focals, cams2world, as_pointcloud=as_pointcloud, + transparent_cams=transparent_cams, cam_size=cam_size, silent=silent) + + +def get_reconstructed_scene(outdir, model, device, silent, image_size, filelist, schedule, niter, min_conf_thr, + as_pointcloud, mask_sky, clean_depth, transparent_cams, cam_size, + scenegraph_type, winsize, refid): + """ + from a list of images, run dust3r inference, global aligner. + then run get_3D_model_from_scene + """ + imgs = load_images(filelist, size=image_size, verbose=not silent) + if len(imgs) == 1: + imgs = [imgs[0], copy.deepcopy(imgs[0])] + imgs[1]['idx'] = 1 + if scenegraph_type == "swin": + scenegraph_type = scenegraph_type + "-" + str(winsize) + elif scenegraph_type == "oneref": + scenegraph_type = scenegraph_type + "-" + str(refid) + + pairs = make_pairs(imgs, scene_graph=scenegraph_type, prefilter=None, symmetrize=True) + output = inference(pairs, model, device, batch_size=1, verbose=not silent) + + mode = GlobalAlignerMode.PointCloudOptimizer if len(imgs) > 2 else GlobalAlignerMode.PairViewer + scene = global_aligner(output, device=device, mode=mode, verbose=not silent) + lr = 0.01 + + if mode == GlobalAlignerMode.PointCloudOptimizer: + loss = scene.compute_global_alignment(init='mst', niter=niter, schedule=schedule, lr=lr) + + outfile = get_3D_model_from_scene(outdir, silent, scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size) + + # also return rgb, depth and confidence imgs + # depth is normalized with the max value for all images + # we apply the jet colormap on the confidence maps + rgbimg = scene.imgs + depths = to_numpy(scene.get_depthmaps()) + confs = to_numpy([c for c in scene.im_conf]) + cmap = pl.get_cmap('jet') + depths_max = max([d.max() for d in depths]) + depths = [d / depths_max for d in depths] + confs_max = max([d.max() for d in confs]) + confs = [cmap(d / confs_max) for d in confs] + + imgs = [] + for i in range(len(rgbimg)): + imgs.append(rgbimg[i]) + imgs.append(rgb(depths[i])) + imgs.append(rgb(confs[i])) + + return scene, outfile, imgs + + +def set_scenegraph_options(inputfiles, winsize, refid, scenegraph_type): + num_files = len(inputfiles) if inputfiles is not None else 1 + max_winsize = max(1, math.ceil((num_files - 1) / 2)) + if scenegraph_type == "swin": + winsize = gradio.Slider(label="Scene Graph: Window Size", value=max_winsize, + minimum=1, maximum=max_winsize, step=1, visible=True) + refid = gradio.Slider(label="Scene Graph: Id", value=0, minimum=0, + maximum=num_files - 1, step=1, visible=False) + elif scenegraph_type == "oneref": + winsize = gradio.Slider(label="Scene Graph: Window Size", value=max_winsize, + minimum=1, maximum=max_winsize, step=1, visible=False) + refid = gradio.Slider(label="Scene Graph: Id", value=0, minimum=0, + maximum=num_files - 1, step=1, visible=True) + else: + winsize = gradio.Slider(label="Scene Graph: Window Size", value=max_winsize, + minimum=1, maximum=max_winsize, step=1, visible=False) + refid = gradio.Slider(label="Scene Graph: Id", value=0, minimum=0, + maximum=num_files - 1, step=1, visible=False) + return winsize, refid + + +def main_demo(tmpdirname, model, device, image_size, server_name, server_port, silent=False): + recon_fun = functools.partial(get_reconstructed_scene, tmpdirname, model, device, silent, image_size) + model_from_scene_fun = functools.partial(get_3D_model_from_scene, tmpdirname, silent) + with gradio.Blocks(css=""".gradio-container {margin: 0 !important; min-width: 100%};""", title="DUSt3R Demo") as demo: + # scene state is save so that you can change conf_thr, cam_size... without rerunning the inference + scene = gradio.State(None) + gradio.HTML('

DUSt3R Demo

') + with gradio.Column(): + inputfiles = gradio.File(file_count="multiple") + with gradio.Row(): + schedule = gradio.Dropdown(["linear", "cosine"], + value='linear', label="schedule", info="For global alignment!") + niter = gradio.Number(value=300, precision=0, minimum=0, maximum=5000, + label="num_iterations", info="For global alignment!") + scenegraph_type = gradio.Dropdown(["complete", "swin", "oneref"], + value='complete', label="Scenegraph", + info="Define how to make pairs", + interactive=True) + winsize = gradio.Slider(label="Scene Graph: Window Size", value=1, + minimum=1, maximum=1, step=1, visible=False) + refid = gradio.Slider(label="Scene Graph: Id", value=0, minimum=0, maximum=0, step=1, visible=False) + + run_btn = gradio.Button("Run") + + with gradio.Row(): + # adjust the confidence threshold + min_conf_thr = gradio.Slider(label="min_conf_thr", value=3.0, minimum=1.0, maximum=20, step=0.1) + # adjust the camera size in the output pointcloud + cam_size = gradio.Slider(label="cam_size", value=0.05, minimum=0.001, maximum=0.1, step=0.001) + with gradio.Row(): + as_pointcloud = gradio.Checkbox(value=False, label="As pointcloud") + # two post process implemented + mask_sky = gradio.Checkbox(value=False, label="Mask sky") + clean_depth = gradio.Checkbox(value=True, label="Clean-up depthmaps") + transparent_cams = gradio.Checkbox(value=False, label="Transparent cameras") + + outmodel = gradio.Model3D() + outgallery = gradio.Gallery(label='rgb,depth,confidence', columns=3, height="100%") + + # events + scenegraph_type.change(set_scenegraph_options, + inputs=[inputfiles, winsize, refid, scenegraph_type], + outputs=[winsize, refid]) + inputfiles.change(set_scenegraph_options, + inputs=[inputfiles, winsize, refid, scenegraph_type], + outputs=[winsize, refid]) + run_btn.click(fn=recon_fun, + inputs=[inputfiles, schedule, niter, min_conf_thr, as_pointcloud, + mask_sky, clean_depth, transparent_cams, cam_size, + scenegraph_type, winsize, refid], + outputs=[scene, outmodel, outgallery]) + min_conf_thr.release(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size], + outputs=outmodel) + cam_size.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size], + outputs=outmodel) + as_pointcloud.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size], + outputs=outmodel) + mask_sky.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size], + outputs=outmodel) + clean_depth.change(fn=model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size], + outputs=outmodel) + transparent_cams.change(model_from_scene_fun, + inputs=[scene, min_conf_thr, as_pointcloud, mask_sky, + clean_depth, transparent_cams, cam_size], + outputs=outmodel) + demo.launch(share=False, server_name=server_name, server_port=server_port) diff --git a/src/mast3r_src/dust3r/dust3r/heads/__init__.py b/src/mast3r_src/dust3r/dust3r/heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..53d0aa5610cae95f34f96bdb3ff9e835a2d6208e --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/heads/__init__.py @@ -0,0 +1,19 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# head factory +# -------------------------------------------------------- +from .linear_head import LinearPts3d +from .dpt_head import create_dpt_head + + +def head_factory(head_type, output_mode, net, has_conf=False): + """" build a prediction head for the decoder + """ + if head_type == 'linear' and output_mode == 'pts3d': + return LinearPts3d(net, has_conf) + elif head_type == 'dpt' and output_mode == 'pts3d': + return create_dpt_head(net, has_conf=has_conf) + else: + raise NotImplementedError(f"unexpected {head_type=} and {output_mode=}") diff --git a/src/mast3r_src/dust3r/dust3r/heads/dpt_head.py b/src/mast3r_src/dust3r/dust3r/heads/dpt_head.py new file mode 100644 index 0000000000000000000000000000000000000000..b7bdc9ff587eef3ec8978a22f63659fbf3c277d6 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/heads/dpt_head.py @@ -0,0 +1,115 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# dpt head implementation for DUST3R +# Downstream heads assume inputs of size B x N x C (where N is the number of tokens) ; +# or if it takes as input the output at every layer, the attribute return_all_layers should be set to True +# the forward function also takes as input a dictionnary img_info with key "height" and "width" +# for PixelwiseTask, the output will be of dimension B x num_channels x H x W +# -------------------------------------------------------- +from einops import rearrange +from typing import List +import torch +import torch.nn as nn +from dust3r.heads.postprocess import postprocess +import dust3r.utils.path_to_croco # noqa: F401 +from models.dpt_block import DPTOutputAdapter # noqa + + +class DPTOutputAdapter_fix(DPTOutputAdapter): + """ + Adapt croco's DPTOutputAdapter implementation for dust3r: + remove duplicated weigths, and fix forward for dust3r + """ + + def init(self, dim_tokens_enc=768): + super().init(dim_tokens_enc) + # these are duplicated weights + del self.act_1_postprocess + del self.act_2_postprocess + del self.act_3_postprocess + del self.act_4_postprocess + + def forward(self, encoder_tokens: List[torch.Tensor], image_size=None): + assert self.dim_tokens_enc is not None, 'Need to call init(dim_tokens_enc) function first' + # H, W = input_info['image_size'] + image_size = self.image_size if image_size is None else image_size + H, W = image_size + # Number of patches in height and width + N_H = H // (self.stride_level * self.P_H) + N_W = W // (self.stride_level * self.P_W) + + # Hook decoder onto 4 layers from specified ViT layers + layers = [encoder_tokens[hook] for hook in self.hooks] + + # Extract only task-relevant tokens and ignore global tokens. + layers = [self.adapt_tokens(l) for l in layers] + + # Reshape tokens to spatial representation + layers = [rearrange(l, 'b (nh nw) c -> b c nh nw', nh=N_H, nw=N_W) for l in layers] + + layers = [self.act_postprocess[idx](l) for idx, l in enumerate(layers)] + # Project layers to chosen feature dim + layers = [self.scratch.layer_rn[idx](l) for idx, l in enumerate(layers)] + + # Fuse layers using refinement stages + path_4 = self.scratch.refinenet4(layers[3])[:, :, :layers[2].shape[2], :layers[2].shape[3]] + path_3 = self.scratch.refinenet3(path_4, layers[2]) + path_2 = self.scratch.refinenet2(path_3, layers[1]) + path_1 = self.scratch.refinenet1(path_2, layers[0]) + + # Output head + out = self.head(path_1) + + return out + + +class PixelwiseTaskWithDPT(nn.Module): + """ DPT module for dust3r, can return 3D points + confidence for all pixels""" + + def __init__(self, *, n_cls_token=0, hooks_idx=None, dim_tokens=None, + output_width_ratio=1, num_channels=1, postprocess=None, depth_mode=None, conf_mode=None, **kwargs): + super(PixelwiseTaskWithDPT, self).__init__() + self.return_all_layers = True # backbone needs to return all layers + self.postprocess = postprocess + self.depth_mode = depth_mode + self.conf_mode = conf_mode + + assert n_cls_token == 0, "Not implemented" + dpt_args = dict(output_width_ratio=output_width_ratio, + num_channels=num_channels, + **kwargs) + if hooks_idx is not None: + dpt_args.update(hooks=hooks_idx) + self.dpt = DPTOutputAdapter_fix(**dpt_args) + dpt_init_args = {} if dim_tokens is None else {'dim_tokens_enc': dim_tokens} + self.dpt.init(**dpt_init_args) + + def forward(self, x, img_info): + out = self.dpt(x, image_size=(img_info[0], img_info[1])) + if self.postprocess: + out = self.postprocess(out, self.depth_mode, self.conf_mode) + return out + + +def create_dpt_head(net, has_conf=False): + """ + return PixelwiseTaskWithDPT for given net params + """ + assert net.dec_depth > 9 + l2 = net.dec_depth + feature_dim = 256 + last_dim = feature_dim//2 + out_nchan = 3 + ed = net.enc_embed_dim + dd = net.dec_embed_dim + return PixelwiseTaskWithDPT(num_channels=out_nchan + has_conf, + feature_dim=feature_dim, + last_dim=last_dim, + hooks_idx=[0, l2*2//4, l2*3//4, l2], + dim_tokens=[ed, dd, dd, dd], + postprocess=postprocess, + depth_mode=net.depth_mode, + conf_mode=net.conf_mode, + head_type='regression') diff --git a/src/mast3r_src/dust3r/dust3r/heads/linear_head.py b/src/mast3r_src/dust3r/dust3r/heads/linear_head.py new file mode 100644 index 0000000000000000000000000000000000000000..6b697f29eaa6f43fad0a3e27a8d9b8f1a602a833 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/heads/linear_head.py @@ -0,0 +1,41 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# linear head implementation for DUST3R +# -------------------------------------------------------- +import torch.nn as nn +import torch.nn.functional as F +from dust3r.heads.postprocess import postprocess + + +class LinearPts3d (nn.Module): + """ + Linear head for dust3r + Each token outputs: - 16x16 3D points (+ confidence) + """ + + def __init__(self, net, has_conf=False): + super().__init__() + self.patch_size = net.patch_embed.patch_size[0] + self.depth_mode = net.depth_mode + self.conf_mode = net.conf_mode + self.has_conf = has_conf + + self.proj = nn.Linear(net.dec_embed_dim, (3 + has_conf)*self.patch_size**2) + + def setup(self, croconet): + pass + + def forward(self, decout, img_shape): + H, W = img_shape + tokens = decout[-1] + B, S, D = tokens.shape + + # extract 3D points + feat = self.proj(tokens) # B,S,D + feat = feat.transpose(-1, -2).view(B, -1, H//self.patch_size, W//self.patch_size) + feat = F.pixel_shuffle(feat, self.patch_size) # B,3,H,W + + # permute + norm depth + return postprocess(feat, self.depth_mode, self.conf_mode) diff --git a/src/mast3r_src/dust3r/dust3r/heads/postprocess.py b/src/mast3r_src/dust3r/dust3r/heads/postprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..cd68a90d89b8dcd7d8a4b4ea06ef8b17eb5da093 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/heads/postprocess.py @@ -0,0 +1,58 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# post process function for all heads: extract 3D points/confidence from output +# -------------------------------------------------------- +import torch + + +def postprocess(out, depth_mode, conf_mode): + """ + extract 3D points/confidence from prediction head output + """ + fmap = out.permute(0, 2, 3, 1) # B,H,W,3 + res = dict(pts3d=reg_dense_depth(fmap[:, :, :, 0:3], mode=depth_mode)) + + if conf_mode is not None: + res['conf'] = reg_dense_conf(fmap[:, :, :, 3], mode=conf_mode) + return res + + +def reg_dense_depth(xyz, mode): + """ + extract 3D points from prediction head output + """ + mode, vmin, vmax = mode + + no_bounds = (vmin == -float('inf')) and (vmax == float('inf')) + assert no_bounds + + if mode == 'linear': + if no_bounds: + return xyz # [-inf, +inf] + return xyz.clip(min=vmin, max=vmax) + + # distance to origin + d = xyz.norm(dim=-1, keepdim=True) + xyz = xyz / d.clip(min=1e-8) + + if mode == 'square': + return xyz * d.square() + + if mode == 'exp': + return xyz * torch.expm1(d) + + raise ValueError(f'bad {mode=}') + + +def reg_dense_conf(x, mode): + """ + extract confidence from prediction head output + """ + mode, vmin, vmax = mode + if mode == 'exp': + return vmin + x.exp().clip(max=vmax-vmin) + if mode == 'sigmoid': + return (vmax - vmin) * torch.sigmoid(x) + vmin + raise ValueError(f'bad {mode=}') diff --git a/src/mast3r_src/dust3r/dust3r/image_pairs.py b/src/mast3r_src/dust3r/dust3r/image_pairs.py new file mode 100644 index 0000000000000000000000000000000000000000..ebcf902b4d07b83fe83ffceba3f45ca0d74dfcf7 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/image_pairs.py @@ -0,0 +1,104 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilities needed to load image pairs +# -------------------------------------------------------- +import numpy as np +import torch + + +def make_pairs(imgs, scene_graph='complete', prefilter=None, symmetrize=True): + pairs = [] + if scene_graph == 'complete': # complete graph + for i in range(len(imgs)): + for j in range(i): + pairs.append((imgs[i], imgs[j])) + elif scene_graph.startswith('swin'): + iscyclic = not scene_graph.endswith('noncyclic') + try: + winsize = int(scene_graph.split('-')[1]) + except Exception as e: + winsize = 3 + pairsid = set() + for i in range(len(imgs)): + for j in range(1, winsize + 1): + idx = (i + j) + if iscyclic: + idx = idx % len(imgs) # explicit loop closure + if idx >= len(imgs): + continue + pairsid.add((i, idx) if i < idx else (idx, i)) + for i, j in pairsid: + pairs.append((imgs[i], imgs[j])) + elif scene_graph.startswith('logwin'): + iscyclic = not scene_graph.endswith('noncyclic') + try: + winsize = int(scene_graph.split('-')[1]) + except Exception as e: + winsize = 3 + offsets = [2**i for i in range(winsize)] + pairsid = set() + for i in range(len(imgs)): + ixs_l = [i - off for off in offsets] + ixs_r = [i + off for off in offsets] + for j in ixs_l + ixs_r: + if iscyclic: + j = j % len(imgs) # Explicit loop closure + if j < 0 or j >= len(imgs) or j == i: + continue + pairsid.add((i, j) if i < j else (j, i)) + for i, j in pairsid: + pairs.append((imgs[i], imgs[j])) + elif scene_graph.startswith('oneref'): + refid = int(scene_graph.split('-')[1]) if '-' in scene_graph else 0 + for j in range(len(imgs)): + if j != refid: + pairs.append((imgs[refid], imgs[j])) + if symmetrize: + pairs += [(img2, img1) for img1, img2 in pairs] + + # now, remove edges + if isinstance(prefilter, str) and prefilter.startswith('seq'): + pairs = filter_pairs_seq(pairs, int(prefilter[3:])) + + if isinstance(prefilter, str) and prefilter.startswith('cyc'): + pairs = filter_pairs_seq(pairs, int(prefilter[3:]), cyclic=True) + + return pairs + + +def sel(x, kept): + if isinstance(x, dict): + return {k: sel(v, kept) for k, v in x.items()} + if isinstance(x, (torch.Tensor, np.ndarray)): + return x[kept] + if isinstance(x, (tuple, list)): + return type(x)([x[k] for k in kept]) + + +def _filter_edges_seq(edges, seq_dis_thr, cyclic=False): + # number of images + n = max(max(e) for e in edges) + 1 + + kept = [] + for e, (i, j) in enumerate(edges): + dis = abs(i - j) + if cyclic: + dis = min(dis, abs(i + n - j), abs(i - n - j)) + if dis <= seq_dis_thr: + kept.append(e) + return kept + + +def filter_pairs_seq(pairs, seq_dis_thr, cyclic=False): + edges = [(img1['idx'], img2['idx']) for img1, img2 in pairs] + kept = _filter_edges_seq(edges, seq_dis_thr, cyclic=cyclic) + return [pairs[i] for i in kept] + + +def filter_edges_seq(view1, view2, pred1, pred2, seq_dis_thr, cyclic=False): + edges = [(int(i), int(j)) for i, j in zip(view1['idx'], view2['idx'])] + kept = _filter_edges_seq(edges, seq_dis_thr, cyclic=cyclic) + print(f'>> Filtering edges more than {seq_dis_thr} frames apart: kept {len(kept)}/{len(edges)} edges') + return sel(view1, kept), sel(view2, kept), sel(pred1, kept), sel(pred2, kept) diff --git a/src/mast3r_src/dust3r/dust3r/inference.py b/src/mast3r_src/dust3r/dust3r/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..90540486b077add90ca50f62a5072e082cb2f2d7 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/inference.py @@ -0,0 +1,150 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilities needed for the inference +# -------------------------------------------------------- +import tqdm +import torch +from dust3r.utils.device import to_cpu, collate_with_cat +from dust3r.utils.misc import invalid_to_nans +from dust3r.utils.geometry import depthmap_to_pts3d, geotrf + + +def _interleave_imgs(img1, img2): + res = {} + for key, value1 in img1.items(): + value2 = img2[key] + if isinstance(value1, torch.Tensor): + value = torch.stack((value1, value2), dim=1).flatten(0, 1) + else: + value = [x for pair in zip(value1, value2) for x in pair] + res[key] = value + return res + + +def make_batch_symmetric(batch): + view1, view2 = batch + view1, view2 = (_interleave_imgs(view1, view2), _interleave_imgs(view2, view1)) + return view1, view2 + + +def loss_of_one_batch(batch, model, criterion, device, symmetrize_batch=False, use_amp=False, ret=None): + view1, view2 = batch + ignore_keys = set(['depthmap', 'dataset', 'label', 'instance', 'idx', 'true_shape', 'rng']) + for view in batch: + for name in view.keys(): # pseudo_focal + if name in ignore_keys: + continue + view[name] = view[name].to(device, non_blocking=True) + + if symmetrize_batch: + view1, view2 = make_batch_symmetric(batch) + + with torch.cuda.amp.autocast(enabled=bool(use_amp)): + pred1, pred2 = model(view1, view2) + + # loss is supposed to be symmetric + with torch.cuda.amp.autocast(enabled=False): + loss = criterion(view1, view2, pred1, pred2) if criterion is not None else None + + result = dict(view1=view1, view2=view2, pred1=pred1, pred2=pred2, loss=loss) + return result[ret] if ret else result + + +@torch.no_grad() +def inference(pairs, model, device, batch_size=8, verbose=True): + if verbose: + print(f'>> Inference with model on {len(pairs)} image pairs') + result = [] + + # first, check if all images have the same size + multiple_shapes = not (check_if_same_size(pairs)) + if multiple_shapes: # force bs=1 + batch_size = 1 + + for i in tqdm.trange(0, len(pairs), batch_size, disable=not verbose): + res = loss_of_one_batch(collate_with_cat(pairs[i:i + batch_size]), model, None, device) + result.append(to_cpu(res)) + + result = collate_with_cat(result, lists=multiple_shapes) + + return result + + +def check_if_same_size(pairs): + shapes1 = [img1['img'].shape[-2:] for img1, img2 in pairs] + shapes2 = [img2['img'].shape[-2:] for img1, img2 in pairs] + return all(shapes1[0] == s for s in shapes1) and all(shapes2[0] == s for s in shapes2) + + +def get_pred_pts3d(gt, pred, use_pose=False): + if 'depth' in pred and 'pseudo_focal' in pred: + try: + pp = gt['camera_intrinsics'][..., :2, 2] + except KeyError: + pp = None + pts3d = depthmap_to_pts3d(**pred, pp=pp) + + elif 'pts3d' in pred: + # pts3d from my camera + pts3d = pred['pts3d'] + + elif 'pts3d_in_other_view' in pred: + # pts3d from the other camera, already transformed + assert use_pose is True + return pred['pts3d_in_other_view'] # return! + + if use_pose: + camera_pose = pred.get('camera_pose') + assert camera_pose is not None + pts3d = geotrf(camera_pose, pts3d) + + return pts3d + + +def find_opt_scaling(gt_pts1, gt_pts2, pr_pts1, pr_pts2=None, fit_mode='weiszfeld_stop_grad', valid1=None, valid2=None): + assert gt_pts1.ndim == pr_pts1.ndim == 4 + assert gt_pts1.shape == pr_pts1.shape + if gt_pts2 is not None: + assert gt_pts2.ndim == pr_pts2.ndim == 4 + assert gt_pts2.shape == pr_pts2.shape + + # concat the pointcloud + nan_gt_pts1 = invalid_to_nans(gt_pts1, valid1).flatten(1, 2) + nan_gt_pts2 = invalid_to_nans(gt_pts2, valid2).flatten(1, 2) if gt_pts2 is not None else None + + pr_pts1 = invalid_to_nans(pr_pts1, valid1).flatten(1, 2) + pr_pts2 = invalid_to_nans(pr_pts2, valid2).flatten(1, 2) if pr_pts2 is not None else None + + all_gt = torch.cat((nan_gt_pts1, nan_gt_pts2), dim=1) if gt_pts2 is not None else nan_gt_pts1 + all_pr = torch.cat((pr_pts1, pr_pts2), dim=1) if pr_pts2 is not None else pr_pts1 + + dot_gt_pr = (all_pr * all_gt).sum(dim=-1) + dot_gt_gt = all_gt.square().sum(dim=-1) + + if fit_mode.startswith('avg'): + # scaling = (all_pr / all_gt).view(B, -1).mean(dim=1) + scaling = dot_gt_pr.nanmean(dim=1) / dot_gt_gt.nanmean(dim=1) + elif fit_mode.startswith('median'): + scaling = (dot_gt_pr / dot_gt_gt).nanmedian(dim=1).values + elif fit_mode.startswith('weiszfeld'): + # init scaling with l2 closed form + scaling = dot_gt_pr.nanmean(dim=1) / dot_gt_gt.nanmean(dim=1) + # iterative re-weighted least-squares + for iter in range(10): + # re-weighting by inverse of distance + dis = (all_pr - scaling.view(-1, 1, 1) * all_gt).norm(dim=-1) + # print(dis.nanmean(-1)) + w = dis.clip_(min=1e-8).reciprocal() + # update the scaling with the new weights + scaling = (w * dot_gt_pr).nanmean(dim=1) / (w * dot_gt_gt).nanmean(dim=1) + else: + raise ValueError(f'bad {fit_mode=}') + + if fit_mode.endswith('stop_grad'): + scaling = scaling.detach() + + scaling = scaling.clip(min=1e-3) + # assert scaling.isfinite().all(), bb() + return scaling diff --git a/src/mast3r_src/dust3r/dust3r/losses.py b/src/mast3r_src/dust3r/dust3r/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..5e3aec4619dc4a3574a61e92d35ed35fda2892c4 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/losses.py @@ -0,0 +1,299 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Implementation of DUSt3R training losses +# -------------------------------------------------------- +from copy import copy, deepcopy +import torch +import torch.nn as nn + +from dust3r.inference import get_pred_pts3d, find_opt_scaling +from dust3r.utils.geometry import inv, geotrf, normalize_pointcloud +from dust3r.utils.geometry import get_joint_pointcloud_depth, get_joint_pointcloud_center_scale + + +def Sum(*losses_and_masks): + loss, mask = losses_and_masks[0] + if loss.ndim > 0: + # we are actually returning the loss for every pixels + return losses_and_masks + else: + # we are returning the global loss + for loss2, mask2 in losses_and_masks[1:]: + loss = loss + loss2 + return loss + + +class BaseCriterion(nn.Module): + def __init__(self, reduction='mean'): + super().__init__() + self.reduction = reduction + + +class LLoss (BaseCriterion): + """ L-norm loss + """ + + def forward(self, a, b): + assert a.shape == b.shape and a.ndim >= 2 and 1 <= a.shape[-1] <= 3, f'Bad shape = {a.shape}' + dist = self.distance(a, b) + assert dist.ndim == a.ndim - 1 # one dimension less + if self.reduction == 'none': + return dist + if self.reduction == 'sum': + return dist.sum() + if self.reduction == 'mean': + return dist.mean() if dist.numel() > 0 else dist.new_zeros(()) + raise ValueError(f'bad {self.reduction=} mode') + + def distance(self, a, b): + raise NotImplementedError() + + +class L21Loss (LLoss): + """ Euclidean distance between 3d points """ + + def distance(self, a, b): + return torch.norm(a - b, dim=-1) # normalized L2 distance + + +L21 = L21Loss() + + +class Criterion (nn.Module): + def __init__(self, criterion=None): + super().__init__() + # assert isinstance(criterion, BaseCriterion), f'{criterion} is not a proper criterion!' # @MODIFIED + self.criterion = copy(criterion) + + def get_name(self): + return f'{type(self).__name__}({self.criterion})' + + def with_reduction(self, mode='none'): + res = loss = deepcopy(self) + while loss is not None: + assert isinstance(loss, Criterion) + loss.criterion.reduction = mode # make it return the loss for each sample + loss = loss._loss2 # we assume loss is a Multiloss + return res + + +class MultiLoss (nn.Module): + """ Easily combinable losses (also keep track of individual loss values): + loss = MyLoss1() + 0.1*MyLoss2() + Usage: + Inherit from this class and override get_name() and compute_loss() + """ + + def __init__(self): + super().__init__() + self._alpha = 1 + self._loss2 = None + + def compute_loss(self, *args, **kwargs): + raise NotImplementedError() + + def get_name(self): + raise NotImplementedError() + + def __mul__(self, alpha): + assert isinstance(alpha, (int, float)) + res = copy(self) + res._alpha = alpha + return res + __rmul__ = __mul__ # same + + def __add__(self, loss2): + assert isinstance(loss2, MultiLoss) + res = cur = copy(self) + # find the end of the chain + while cur._loss2 is not None: + cur = cur._loss2 + cur._loss2 = loss2 + return res + + def __repr__(self): + name = self.get_name() + if self._alpha != 1: + name = f'{self._alpha:g}*{name}' + if self._loss2: + name = f'{name} + {self._loss2}' + return name + + def forward(self, *args, **kwargs): + loss = self.compute_loss(*args, **kwargs) + if isinstance(loss, tuple): + loss, details = loss + elif loss.ndim == 0: + details = {self.get_name(): float(loss)} + else: + details = {} + loss = loss * self._alpha + + if self._loss2: + loss2, details2 = self._loss2(*args, **kwargs) + loss = loss + loss2 + details |= details2 + + return loss, details + + +class Regr3D (Criterion, MultiLoss): + """ Ensure that all 3D points are correct. + Asymmetric loss: view1 is supposed to be the anchor. + + P1 = RT1 @ D1 + P2 = RT2 @ D2 + loss1 = (I @ pred_D1) - (RT1^-1 @ RT1 @ D1) + loss2 = (RT21 @ pred_D2) - (RT1^-1 @ P2) + = (RT21 @ pred_D2) - (RT1^-1 @ RT2 @ D2) + """ + + def __init__(self, criterion, norm_mode='avg_dis', gt_scale=False): + super().__init__(criterion) + self.norm_mode = norm_mode + self.gt_scale = gt_scale + + def get_all_pts3d(self, gt1, gt2, pred1, pred2, dist_clip=None): + # everything is normalized w.r.t. camera of view1 + in_camera1 = inv(gt1['camera_pose']) + gt_pts1 = geotrf(in_camera1, gt1['pts3d']) # B,H,W,3 + gt_pts2 = geotrf(in_camera1, gt2['pts3d']) # B,H,W,3 + + valid1 = gt1['valid_mask'].clone() + valid2 = gt2['valid_mask'].clone() + + if dist_clip is not None: + # points that are too far-away == invalid + dis1 = gt_pts1.norm(dim=-1) # (B, H, W) + dis2 = gt_pts2.norm(dim=-1) # (B, H, W) + valid1 = valid1 & (dis1 <= dist_clip) + valid2 = valid2 & (dis2 <= dist_clip) + + pr_pts1 = get_pred_pts3d(gt1, pred1, use_pose=False) + pr_pts2 = get_pred_pts3d(gt2, pred2, use_pose=True) + + # normalize 3d points + if self.norm_mode: + pr_pts1, pr_pts2 = normalize_pointcloud(pr_pts1, pr_pts2, self.norm_mode, valid1, valid2) + if self.norm_mode and not self.gt_scale: + gt_pts1, gt_pts2 = normalize_pointcloud(gt_pts1, gt_pts2, self.norm_mode, valid1, valid2) + + return gt_pts1, gt_pts2, pr_pts1, pr_pts2, valid1, valid2, {} + + def compute_loss(self, gt1, gt2, pred1, pred2, **kw): + gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, monitoring = \ + self.get_all_pts3d(gt1, gt2, pred1, pred2, **kw) + # loss on img1 side + l1 = self.criterion(pred_pts1[mask1], gt_pts1[mask1]) + # loss on gt2 side + l2 = self.criterion(pred_pts2[mask2], gt_pts2[mask2]) + self_name = type(self).__name__ + details = {self_name + '_pts3d_1': float(l1.mean()), self_name + '_pts3d_2': float(l2.mean())} + return Sum((l1, mask1), (l2, mask2)), (details | monitoring) + + +class ConfLoss (MultiLoss): + """ Weighted regression by learned confidence. + Assuming the input pixel_loss is a pixel-level regression loss. + + Principle: + high-confidence means high conf = 0.1 ==> conf_loss = x / 10 + alpha*log(10) + low confidence means low conf = 10 ==> conf_loss = x * 10 - alpha*log(10) + + alpha: hyperparameter + """ + + def __init__(self, pixel_loss, alpha=1): + super().__init__() + assert alpha > 0 + self.alpha = alpha + self.pixel_loss = pixel_loss.with_reduction('none') + + def get_name(self): + return f'ConfLoss({self.pixel_loss})' + + def get_conf_log(self, x): + return x, torch.log(x) + + def compute_loss(self, gt1, gt2, pred1, pred2, **kw): + # compute per-pixel loss + ((loss1, msk1), (loss2, msk2)), details = self.pixel_loss(gt1, gt2, pred1, pred2, **kw) + if loss1.numel() == 0: + print('NO VALID POINTS in img1', force=True) + if loss2.numel() == 0: + print('NO VALID POINTS in img2', force=True) + + # weight by confidence + conf1, log_conf1 = self.get_conf_log(pred1['conf'][msk1]) + conf2, log_conf2 = self.get_conf_log(pred2['conf'][msk2]) + conf_loss1 = loss1 * conf1 - self.alpha * log_conf1 + conf_loss2 = loss2 * conf2 - self.alpha * log_conf2 + + # average + nan protection (in case of no valid pixels at all) + conf_loss1 = conf_loss1.mean() if conf_loss1.numel() > 0 else 0 + conf_loss2 = conf_loss2.mean() if conf_loss2.numel() > 0 else 0 + + return conf_loss1 + conf_loss2, dict(conf_loss_1=float(conf_loss1), conf_loss2=float(conf_loss2), **details) + + +class Regr3D_ShiftInv (Regr3D): + """ Same than Regr3D but invariant to depth shift. + """ + + def get_all_pts3d(self, gt1, gt2, pred1, pred2): + # compute unnormalized points + gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, monitoring = \ + super().get_all_pts3d(gt1, gt2, pred1, pred2) + + # compute median depth + gt_z1, gt_z2 = gt_pts1[..., 2], gt_pts2[..., 2] + pred_z1, pred_z2 = pred_pts1[..., 2], pred_pts2[..., 2] + gt_shift_z = get_joint_pointcloud_depth(gt_z1, gt_z2, mask1, mask2)[:, None, None] + pred_shift_z = get_joint_pointcloud_depth(pred_z1, pred_z2, mask1, mask2)[:, None, None] + + # subtract the median depth + gt_z1 -= gt_shift_z + gt_z2 -= gt_shift_z + pred_z1 -= pred_shift_z + pred_z2 -= pred_shift_z + + # monitoring = dict(monitoring, gt_shift_z=gt_shift_z.mean().detach(), pred_shift_z=pred_shift_z.mean().detach()) + return gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, monitoring + + +class Regr3D_ScaleInv (Regr3D): + """ Same than Regr3D but invariant to depth shift. + if gt_scale == True: enforce the prediction to take the same scale than GT + """ + + def get_all_pts3d(self, gt1, gt2, pred1, pred2): + # compute depth-normalized points + gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, monitoring = super().get_all_pts3d(gt1, gt2, pred1, pred2) + + # measure scene scale + _, gt_scale = get_joint_pointcloud_center_scale(gt_pts1, gt_pts2, mask1, mask2) + _, pred_scale = get_joint_pointcloud_center_scale(pred_pts1, pred_pts2, mask1, mask2) + + # prevent predictions to be in a ridiculous range + pred_scale = pred_scale.clip(min=1e-3, max=1e3) + + # subtract the median depth + if self.gt_scale: + pred_pts1 *= gt_scale / pred_scale + pred_pts2 *= gt_scale / pred_scale + # monitoring = dict(monitoring, pred_scale=(pred_scale/gt_scale).mean()) + else: + gt_pts1 /= gt_scale + gt_pts2 /= gt_scale + pred_pts1 /= pred_scale + pred_pts2 /= pred_scale + # monitoring = dict(monitoring, gt_scale=gt_scale.mean(), pred_scale=pred_scale.mean().detach()) + + return gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, monitoring + + +class Regr3D_ScaleShiftInv (Regr3D_ScaleInv, Regr3D_ShiftInv): + # calls Regr3D_ShiftInv first, then Regr3D_ScaleInv + pass diff --git a/src/mast3r_src/dust3r/dust3r/model.py b/src/mast3r_src/dust3r/dust3r/model.py new file mode 100644 index 0000000000000000000000000000000000000000..e0a8d67bbb48b224125d67799fe89e17ba086281 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/model.py @@ -0,0 +1,207 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# DUSt3R model class +# -------------------------------------------------------- +from copy import deepcopy +import torch +import os +from packaging import version +import huggingface_hub + +from .utils.misc import fill_default_args, freeze_all_params, is_symmetrized, interleave, transpose_to_landscape +from .heads import head_factory +from dust3r.patch_embed import get_patch_embed + +import dust3r.utils.path_to_croco # noqa: F401 +from models.croco import CroCoNet # noqa + +inf = float('inf') + +hf_version_number = huggingface_hub.__version__ +assert version.parse(hf_version_number) >= version.parse("0.22.0"), "Outdated huggingface_hub version, please reinstall requirements.txt" + +def load_model(model_path, device, verbose=True): + if verbose: + print('... loading model from', model_path) + ckpt = torch.load(model_path, map_location='cpu') + args = ckpt['args'].model.replace("ManyAR_PatchEmbed", "PatchEmbedDust3R") + if 'landscape_only' not in args: + args = args[:-1] + ', landscape_only=False)' + else: + args = args.replace(" ", "").replace('landscape_only=True', 'landscape_only=False') + assert "landscape_only=False" in args + if verbose: + print(f"instantiating : {args}") + net = eval(args) + s = net.load_state_dict(ckpt['model'], strict=False) + if verbose: + print(s) + return net.to(device) + + +class AsymmetricCroCo3DStereo ( + CroCoNet, + huggingface_hub.PyTorchModelHubMixin, + library_name="dust3r", + repo_url="https://github.com/naver/dust3r", + tags=["image-to-3d"], +): + """ Two siamese encoders, followed by two decoders. + The goal is to output 3d points directly, both images in view1's frame + (hence the asymmetry). + """ + + def __init__(self, + output_mode='pts3d', + head_type='linear', + depth_mode=('exp', -inf, inf), + conf_mode=('exp', 1, inf), + freeze='none', + landscape_only=True, + patch_embed_cls='PatchEmbedDust3R', # PatchEmbedDust3R or ManyAR_PatchEmbed + **croco_kwargs): + self.patch_embed_cls = patch_embed_cls + self.croco_args = fill_default_args(croco_kwargs, super().__init__) + super().__init__(**croco_kwargs) + + # dust3r specific initialization + self.dec_blocks2 = deepcopy(self.dec_blocks) + self.set_downstream_head(output_mode, head_type, landscape_only, depth_mode, conf_mode, **croco_kwargs) + self.set_freeze(freeze) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, **kw): + if os.path.isfile(pretrained_model_name_or_path): + return load_model(pretrained_model_name_or_path, device='cpu') + else: + return super(AsymmetricCroCo3DStereo, cls).from_pretrained(pretrained_model_name_or_path, **kw) + + def _set_patch_embed(self, img_size=224, patch_size=16, enc_embed_dim=768): + self.patch_embed = get_patch_embed(self.patch_embed_cls, img_size, patch_size, enc_embed_dim) + + def load_state_dict(self, ckpt, **kw): + # duplicate all weights for the second decoder if not present + new_ckpt = dict(ckpt) + if not any(k.startswith('dec_blocks2') for k in ckpt): + for key, value in ckpt.items(): + if key.startswith('dec_blocks'): + new_ckpt[key.replace('dec_blocks', 'dec_blocks2')] = value + return super().load_state_dict(new_ckpt, **kw) + + def set_freeze(self, freeze): # this is for use by downstream models + self.freeze = freeze + to_be_frozen = { + 'none': [], + 'mask': [self.mask_token], + 'encoder': [self.mask_token, self.patch_embed, self.enc_blocks], + } + freeze_all_params(to_be_frozen[freeze]) + + def _set_prediction_head(self, *args, **kwargs): + """ No prediction head """ + return + + def set_downstream_head(self, output_mode, head_type, landscape_only, depth_mode, conf_mode, patch_size, img_size, + **kw): + assert img_size[0] % patch_size == 0 and img_size[1] % patch_size == 0, \ + f'{img_size=} must be multiple of {patch_size=}' + self.output_mode = output_mode + self.head_type = head_type + self.depth_mode = depth_mode + self.conf_mode = conf_mode + # allocate heads + self.downstream_head1 = head_factory(head_type, output_mode, self, has_conf=bool(conf_mode)) + self.downstream_head2 = head_factory(head_type, output_mode, self, has_conf=bool(conf_mode)) + # magic wrapper + self.head1 = transpose_to_landscape(self.downstream_head1, activate=landscape_only) + self.head2 = transpose_to_landscape(self.downstream_head2, activate=landscape_only) + + def _encode_image(self, image, true_shape): + # embed the image into patches (x has size B x Npatches x C) + # print("image.shape", image.shape) + # print("true_shape", true_shape) + # print("true_shape.shape", true_shape.shape) + x, pos = self.patch_embed(image, true_shape=true_shape) + + # add positional embedding without cls token + assert self.enc_pos_embed is None + + # now apply the transformer encoder and normalization + for blk in self.enc_blocks: + x = blk(x, pos) + + x = self.enc_norm(x) + return x, pos, None + + def _encode_image_pairs(self, img1, img2, true_shape1, true_shape2): + if img1.shape[-2:] == img2.shape[-2:]: + out, pos, _ = self._encode_image(torch.cat((img1, img2), dim=0), + torch.cat((true_shape1, true_shape2), dim=0)) + out, out2 = out.chunk(2, dim=0) + pos, pos2 = pos.chunk(2, dim=0) + else: + out, pos, _ = self._encode_image(img1, true_shape1) + out2, pos2, _ = self._encode_image(img2, true_shape2) + return out, out2, pos, pos2 + + def _encode_symmetrized(self, view1, view2): + img1 = view1['img'] + img2 = view2['img'] + B = img1.shape[0] + # Recover true_shape when available, otherwise assume that the img shape is the true one + shape1 = view1.get('true_shape', torch.tensor(img1.shape[-2:])[None].repeat(B, 1)) + shape2 = view2.get('true_shape', torch.tensor(img2.shape[-2:])[None].repeat(B, 1)) + # warning! maybe the images have different portrait/landscape orientations + + if is_symmetrized(view1, view2): + # computing half of forward pass!' + feat1, feat2, pos1, pos2 = self._encode_image_pairs(img1[::2], img2[::2], shape1[::2], shape2[::2]) + feat1, feat2 = interleave(feat1, feat2) + pos1, pos2 = interleave(pos1, pos2) + else: + feat1, feat2, pos1, pos2 = self._encode_image_pairs(img1, img2, shape1, shape2) + + return (shape1, shape2), (feat1, feat2), (pos1, pos2) + + def _decoder(self, f1, pos1, f2, pos2): + final_output = [(f1, f2)] # before projection + + # project to decoder dim + f1 = self.decoder_embed(f1) + f2 = self.decoder_embed(f2) + + final_output.append((f1, f2)) + for blk1, blk2 in zip(self.dec_blocks, self.dec_blocks2): + # img1 side + f1, _ = blk1(*final_output[-1][::+1], pos1, pos2) + # img2 side + f2, _ = blk2(*final_output[-1][::-1], pos2, pos1) + # store the result + final_output.append((f1, f2)) + + # normalize last output + del final_output[1] # duplicate with final_output[0] + final_output[-1] = tuple(map(self.dec_norm, final_output[-1])) + return zip(*final_output) + + def _downstream_head(self, head_num, decout, img_shape): + B, S, D = decout[-1].shape + # img_shape = tuple(map(int, img_shape)) + head = getattr(self, f'head{head_num}') + return head(decout, img_shape) + + def forward(self, view1, view2): + # encode the two images --> B,S,D + (shape1, shape2), (feat1, feat2), (pos1, pos2) = self._encode_symmetrized(view1, view2) + + # combine all ref images into object-centric representation + dec1, dec2 = self._decoder(feat1, pos1, feat2, pos2) + + with torch.cuda.amp.autocast(enabled=False): + res1 = self._downstream_head(1, [tok.float() for tok in dec1], shape1) + res2 = self._downstream_head(2, [tok.float() for tok in dec2], shape2) + + res2['pts3d_in_other_view'] = res2.pop('pts3d') # predict view2's pts3d in view1's frame + return res1, res2 diff --git a/src/mast3r_src/dust3r/dust3r/optim_factory.py b/src/mast3r_src/dust3r/dust3r/optim_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..9b9c16e0e0fda3fd03c3def61abc1f354f75c584 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/optim_factory.py @@ -0,0 +1,14 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# optimization functions +# -------------------------------------------------------- + + +def adjust_learning_rate_by_lr(optimizer, lr): + for param_group in optimizer.param_groups: + if "lr_scale" in param_group: + param_group["lr"] = lr * param_group["lr_scale"] + else: + param_group["lr"] = lr diff --git a/src/mast3r_src/dust3r/dust3r/patch_embed.py b/src/mast3r_src/dust3r/dust3r/patch_embed.py new file mode 100644 index 0000000000000000000000000000000000000000..07bb184bccb9d16657581576779904065d2dc857 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/patch_embed.py @@ -0,0 +1,70 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# PatchEmbed implementation for DUST3R, +# in particular ManyAR_PatchEmbed that Handle images with non-square aspect ratio +# -------------------------------------------------------- +import torch +import dust3r.utils.path_to_croco # noqa: F401 +from models.blocks import PatchEmbed # noqa + + +def get_patch_embed(patch_embed_cls, img_size, patch_size, enc_embed_dim): + assert patch_embed_cls in ['PatchEmbedDust3R', 'ManyAR_PatchEmbed'] + patch_embed = eval(patch_embed_cls)(img_size, patch_size, 3, enc_embed_dim) + return patch_embed + + +class PatchEmbedDust3R(PatchEmbed): + def forward(self, x, **kw): + B, C, H, W = x.shape + assert H % self.patch_size[0] == 0, f"Input image height ({H}) is not a multiple of patch size ({self.patch_size[0]})." + assert W % self.patch_size[1] == 0, f"Input image width ({W}) is not a multiple of patch size ({self.patch_size[1]})." + x = self.proj(x) + pos = self.position_getter(B, x.size(2), x.size(3), x.device) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + x = self.norm(x) + return x, pos + + +class ManyAR_PatchEmbed (PatchEmbed): + """ Handle images with non-square aspect ratio. + All images in the same batch have the same aspect ratio. + true_shape = [(height, width) ...] indicates the actual shape of each image. + """ + + def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True): + self.embed_dim = embed_dim + super().__init__(img_size, patch_size, in_chans, embed_dim, norm_layer, flatten) + + def forward(self, img, true_shape): + B, C, H, W = img.shape + assert W >= H, f'img should be in landscape mode, but got {W=} {H=}' + assert H % self.patch_size[0] == 0, f"Input image height ({H}) is not a multiple of patch size ({self.patch_size[0]})." + assert W % self.patch_size[1] == 0, f"Input image width ({W}) is not a multiple of patch size ({self.patch_size[1]})." + assert true_shape.shape == (B, 2), f"true_shape has the wrong shape={true_shape.shape}" + + # size expressed in tokens + W //= self.patch_size[0] + H //= self.patch_size[1] + n_tokens = H * W + + height, width = true_shape.T + is_landscape = (width >= height) + is_portrait = ~is_landscape + + # allocate result + x = img.new_zeros((B, n_tokens, self.embed_dim)) + pos = img.new_zeros((B, n_tokens, 2), dtype=torch.int64) + + # linear projection, transposed if necessary + x[is_landscape] = self.proj(img[is_landscape]).permute(0, 2, 3, 1).flatten(1, 2).float() + x[is_portrait] = self.proj(img[is_portrait].swapaxes(-1, -2)).permute(0, 2, 3, 1).flatten(1, 2).float() + + pos[is_landscape] = self.position_getter(1, H, W, pos.device) + pos[is_portrait] = self.position_getter(1, W, H, pos.device) + + x = self.norm(x) + return x, pos diff --git a/src/mast3r_src/dust3r/dust3r/post_process.py b/src/mast3r_src/dust3r/dust3r/post_process.py new file mode 100644 index 0000000000000000000000000000000000000000..550a9b41025ad003228ef16f97d045fc238746e4 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/post_process.py @@ -0,0 +1,60 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilities for interpreting the DUST3R output +# -------------------------------------------------------- +import numpy as np +import torch +from dust3r.utils.geometry import xy_grid + + +def estimate_focal_knowing_depth(pts3d, pp, focal_mode='median', min_focal=0., max_focal=np.inf): + """ Reprojection method, for when the absolute depth is known: + 1) estimate the camera focal using a robust estimator + 2) reproject points onto true rays, minimizing a certain error + """ + B, H, W, THREE = pts3d.shape + assert THREE == 3 + + # centered pixel grid + pixels = xy_grid(W, H, device=pts3d.device).view(1, -1, 2) - pp.view(-1, 1, 2) # B,HW,2 + pts3d = pts3d.flatten(1, 2) # (B, HW, 3) + + if focal_mode == 'median': + with torch.no_grad(): + # direct estimation of focal + u, v = pixels.unbind(dim=-1) + x, y, z = pts3d.unbind(dim=-1) + fx_votes = (u * z) / x + fy_votes = (v * z) / y + + # assume square pixels, hence same focal for X and Y + f_votes = torch.cat((fx_votes.view(B, -1), fy_votes.view(B, -1)), dim=-1) + focal = torch.nanmedian(f_votes, dim=-1).values + + elif focal_mode == 'weiszfeld': + # init focal with l2 closed form + # we try to find focal = argmin Sum | pixel - focal * (x,y)/z| + xy_over_z = (pts3d[..., :2] / pts3d[..., 2:3]).nan_to_num(posinf=0, neginf=0) # homogeneous (x,y,1) + + dot_xy_px = (xy_over_z * pixels).sum(dim=-1) + dot_xy_xy = xy_over_z.square().sum(dim=-1) + + focal = dot_xy_px.mean(dim=1) / dot_xy_xy.mean(dim=1) + + # iterative re-weighted least-squares + for iter in range(10): + # re-weighting by inverse of distance + dis = (pixels - focal.view(-1, 1, 1) * xy_over_z).norm(dim=-1) + # print(dis.nanmean(-1)) + w = dis.clip(min=1e-8).reciprocal() + # update the scaling with the new weights + focal = (w * dot_xy_px).mean(dim=1) / (w * dot_xy_xy).mean(dim=1) + else: + raise ValueError(f'bad {focal_mode=}') + + focal_base = max(H, W) / (2 * np.tan(np.deg2rad(60) / 2)) # size / 1.1547005383792515 + focal = focal.clip(min=min_focal*focal_base, max=max_focal*focal_base) + # print(focal) + return focal diff --git a/src/mast3r_src/dust3r/dust3r/training.py b/src/mast3r_src/dust3r/dust3r/training.py new file mode 100644 index 0000000000000000000000000000000000000000..972212331b769c4fea467681404cab400a2c6edd --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/training.py @@ -0,0 +1,376 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# training code for DUSt3R +# -------------------------------------------------------- +# References: +# MAE: https://github.com/facebookresearch/mae +# DeiT: https://github.com/facebookresearch/deit +# BEiT: https://github.com/microsoft/unilm/tree/master/beit +# -------------------------------------------------------- +import argparse +import datetime +import json +import numpy as np +import os +import sys +import time +import math +from collections import defaultdict +from pathlib import Path +from typing import Sized + +import torch +import torch.backends.cudnn as cudnn +from torch.utils.tensorboard import SummaryWriter +torch.backends.cuda.matmul.allow_tf32 = True # for gpu >= Ampere and pytorch >= 1.12 + +from dust3r.model import AsymmetricCroCo3DStereo, inf # noqa: F401, needed when loading the model +from dust3r.datasets import get_data_loader # noqa +from dust3r.losses import * # noqa: F401, needed when loading the model +from dust3r.inference import loss_of_one_batch # noqa + +import dust3r.utils.path_to_croco # noqa: F401 +import croco.utils.misc as misc # noqa +from croco.utils.misc import NativeScalerWithGradNormCount as NativeScaler # noqa + + +def get_args_parser(): + parser = argparse.ArgumentParser('DUST3R training', add_help=False) + # model and criterion + parser.add_argument('--model', default="AsymmetricCroCo3DStereo(patch_embed_cls='ManyAR_PatchEmbed')", + type=str, help="string containing the model to build") + parser.add_argument('--pretrained', default=None, help='path of a starting checkpoint') + parser.add_argument('--train_criterion', default="ConfLoss(Regr3D(L21, norm_mode='avg_dis'), alpha=0.2)", + type=str, help="train criterion") + parser.add_argument('--test_criterion', default=None, type=str, help="test criterion") + + # dataset + parser.add_argument('--train_dataset', required=True, type=str, help="training set") + parser.add_argument('--test_dataset', default='[None]', type=str, help="testing set") + + # training + parser.add_argument('--seed', default=0, type=int, help="Random seed") + parser.add_argument('--batch_size', default=64, type=int, + help="Batch size per GPU (effective batch size is batch_size * accum_iter * # gpus") + parser.add_argument('--accum_iter', default=1, type=int, + help="Accumulate gradient iterations (for increasing the effective batch size under memory constraints)") + parser.add_argument('--epochs', default=800, type=int, help="Maximum number of epochs for the scheduler") + + parser.add_argument('--weight_decay', type=float, default=0.05, help="weight decay (default: 0.05)") + parser.add_argument('--lr', type=float, default=None, metavar='LR', help='learning rate (absolute lr)') + parser.add_argument('--blr', type=float, default=1.5e-4, metavar='LR', + help='base learning rate: absolute_lr = base_lr * total_batch_size / 256') + parser.add_argument('--min_lr', type=float, default=0., metavar='LR', + help='lower lr bound for cyclic schedulers that hit 0') + parser.add_argument('--warmup_epochs', type=int, default=40, metavar='N', help='epochs to warmup LR') + + parser.add_argument('--amp', type=int, default=0, + choices=[0, 1], help="Use Automatic Mixed Precision for pretraining") + + # others + parser.add_argument('--num_workers', default=8, type=int) + parser.add_argument('--world_size', default=1, type=int, help='number of distributed processes') + parser.add_argument('--local_rank', default=-1, type=int) + parser.add_argument('--dist_url', default='env://', help='url used to set up distributed training') + + parser.add_argument('--eval_freq', type=int, default=1, help='Test loss evaluation frequency') + parser.add_argument('--save_freq', default=1, type=int, + help='frequence (number of epochs) to save checkpoint in checkpoint-last.pth') + parser.add_argument('--keep_freq', default=20, type=int, + help='frequence (number of epochs) to save checkpoint in checkpoint-%d.pth') + parser.add_argument('--print_freq', default=20, type=int, + help='frequence (number of iterations) to print infos while training') + + # output dir + parser.add_argument('--output_dir', default='./output/', type=str, help="path where to save the output") + return parser + + +def train(args): + misc.init_distributed_mode(args) + global_rank = misc.get_rank() + world_size = misc.get_world_size() + + print("output_dir: " + args.output_dir) + if args.output_dir: + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + + # auto resume + last_ckpt_fname = os.path.join(args.output_dir, f'checkpoint-last.pth') + args.resume = last_ckpt_fname if os.path.isfile(last_ckpt_fname) else None + + print('job dir: {}'.format(os.path.dirname(os.path.realpath(__file__)))) + print("{}".format(args).replace(', ', ',\n')) + + device = "cuda" if torch.cuda.is_available() else "cpu" + device = torch.device(device) + + # fix the seed + seed = args.seed + misc.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + + cudnn.benchmark = True + + # training dataset and loader + print('Building train dataset {:s}'.format(args.train_dataset)) + # dataset and loader + data_loader_train = build_dataset(args.train_dataset, args.batch_size, args.num_workers, test=False) + print('Building test dataset {:s}'.format(args.train_dataset)) + data_loader_test = {dataset.split('(')[0]: build_dataset(dataset, args.batch_size, args.num_workers, test=True) + for dataset in args.test_dataset.split('+')} + + # model + print('Loading model: {:s}'.format(args.model)) + model = eval(args.model) + print(f'>> Creating train criterion = {args.train_criterion}') + train_criterion = eval(args.train_criterion).to(device) + print(f'>> Creating test criterion = {args.test_criterion or args.train_criterion}') + test_criterion = eval(args.test_criterion or args.criterion).to(device) + + model.to(device) + model_without_ddp = model + print("Model = %s" % str(model_without_ddp)) + + if args.pretrained and not args.resume: + print('Loading pretrained: ', args.pretrained) + ckpt = torch.load(args.pretrained, map_location=device) + print(model.load_state_dict(ckpt['model'], strict=False)) + del ckpt # in case it occupies memory + + eff_batch_size = args.batch_size * args.accum_iter * misc.get_world_size() + if args.lr is None: # only base_lr is specified + args.lr = args.blr * eff_batch_size / 256 + print("base lr: %.2e" % (args.lr * 256 / eff_batch_size)) + print("actual lr: %.2e" % args.lr) + print("accumulate grad iterations: %d" % args.accum_iter) + print("effective batch size: %d" % eff_batch_size) + + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel( + model, device_ids=[args.gpu], find_unused_parameters=True, static_graph=True) + model_without_ddp = model.module + + # following timm: set wd as 0 for bias and norm layers + param_groups = misc.get_parameter_groups(model_without_ddp, args.weight_decay) + optimizer = torch.optim.AdamW(param_groups, lr=args.lr, betas=(0.9, 0.95)) + print(optimizer) + loss_scaler = NativeScaler() + + def write_log_stats(epoch, train_stats, test_stats): + if misc.is_main_process(): + if log_writer is not None: + log_writer.flush() + + log_stats = dict(epoch=epoch, **{f'train_{k}': v for k, v in train_stats.items()}) + for test_name in data_loader_test: + if test_name not in test_stats: + continue + log_stats.update({test_name + '_' + k: v for k, v in test_stats[test_name].items()}) + + with open(os.path.join(args.output_dir, "log.txt"), mode="a", encoding="utf-8") as f: + f.write(json.dumps(log_stats) + "\n") + + def save_model(epoch, fname, best_so_far): + misc.save_model(args=args, model_without_ddp=model_without_ddp, optimizer=optimizer, + loss_scaler=loss_scaler, epoch=epoch, fname=fname, best_so_far=best_so_far) + + best_so_far = misc.load_model(args=args, model_without_ddp=model_without_ddp, + optimizer=optimizer, loss_scaler=loss_scaler) + if best_so_far is None: + best_so_far = float('inf') + if global_rank == 0 and args.output_dir is not None: + log_writer = SummaryWriter(log_dir=args.output_dir) + else: + log_writer = None + + print(f"Start training for {args.epochs} epochs") + start_time = time.time() + train_stats = test_stats = {} + for epoch in range(args.start_epoch, args.epochs + 1): + + # Save immediately the last checkpoint + if epoch > args.start_epoch: + if args.save_freq and epoch % args.save_freq == 0 or epoch == args.epochs: + save_model(epoch - 1, 'last', best_so_far) + + # Test on multiple datasets + new_best = False + if (epoch > 0 and args.eval_freq > 0 and epoch % args.eval_freq == 0): + test_stats = {} + for test_name, testset in data_loader_test.items(): + stats = test_one_epoch(model, test_criterion, testset, + device, epoch, log_writer=log_writer, args=args, prefix=test_name) + test_stats[test_name] = stats + + # Save best of all + if stats['loss_med'] < best_so_far: + best_so_far = stats['loss_med'] + new_best = True + + # Save more stuff + write_log_stats(epoch, train_stats, test_stats) + + if epoch > args.start_epoch: + if args.keep_freq and epoch % args.keep_freq == 0: + save_model(epoch - 1, str(epoch), best_so_far) + if new_best: + save_model(epoch - 1, 'best', best_so_far) + if epoch >= args.epochs: + break # exit after writing last test to disk + + # Train + train_stats = train_one_epoch( + model, train_criterion, data_loader_train, + optimizer, device, epoch, loss_scaler, + log_writer=log_writer, + args=args) + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + + save_final_model(args, args.epochs, model_without_ddp, best_so_far=best_so_far) + + +def save_final_model(args, epoch, model_without_ddp, best_so_far=None): + output_dir = Path(args.output_dir) + checkpoint_path = output_dir / 'checkpoint-final.pth' + to_save = { + 'args': args, + 'model': model_without_ddp if isinstance(model_without_ddp, dict) else model_without_ddp.cpu().state_dict(), + 'epoch': epoch + } + if best_so_far is not None: + to_save['best_so_far'] = best_so_far + print(f'>> Saving model to {checkpoint_path} ...') + misc.save_on_master(to_save, checkpoint_path) + + +def build_dataset(dataset, batch_size, num_workers, test=False): + split = ['Train', 'Test'][test] + print(f'Building {split} Data loader for dataset: ', dataset) + loader = get_data_loader(dataset, + batch_size=batch_size, + num_workers=num_workers, + pin_mem=True, + shuffle=not (test), + drop_last=not (test)) + + print(f"{split} dataset length: ", len(loader)) + return loader + + +def train_one_epoch(model: torch.nn.Module, criterion: torch.nn.Module, + data_loader: Sized, optimizer: torch.optim.Optimizer, + device: torch.device, epoch: int, loss_scaler, + args, + log_writer=None): + assert torch.backends.cuda.matmul.allow_tf32 == True + + model.train(True) + metric_logger = misc.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', misc.SmoothedValue(window_size=1, fmt='{value:.6f}')) + header = 'Epoch: [{}]'.format(epoch) + accum_iter = args.accum_iter + + if log_writer is not None: + print('log_dir: {}'.format(log_writer.log_dir)) + + if hasattr(data_loader, 'dataset') and hasattr(data_loader.dataset, 'set_epoch'): + data_loader.dataset.set_epoch(epoch) + if hasattr(data_loader, 'sampler') and hasattr(data_loader.sampler, 'set_epoch'): + data_loader.sampler.set_epoch(epoch) + + optimizer.zero_grad() + + for data_iter_step, batch in enumerate(metric_logger.log_every(data_loader, args.print_freq, header)): + epoch_f = epoch + data_iter_step / len(data_loader) + + # we use a per iteration (instead of per epoch) lr scheduler + if data_iter_step % accum_iter == 0: + misc.adjust_learning_rate(optimizer, epoch_f, args) + + loss_tuple = loss_of_one_batch(batch, model, criterion, device, + symmetrize_batch=True, + use_amp=bool(args.amp), ret='loss') + loss, loss_details = loss_tuple # criterion returns two values + loss_value = float(loss) + + if not math.isfinite(loss_value): + print("Loss is {}, stopping training".format(loss_value), force=True) + sys.exit(1) + + loss /= accum_iter + loss_scaler(loss, optimizer, parameters=model.parameters(), + update_grad=(data_iter_step + 1) % accum_iter == 0) + if (data_iter_step + 1) % accum_iter == 0: + optimizer.zero_grad() + + del loss + del batch + + lr = optimizer.param_groups[0]["lr"] + metric_logger.update(epoch=epoch_f) + metric_logger.update(lr=lr) + metric_logger.update(loss=loss_value, **loss_details) + + if (data_iter_step + 1) % accum_iter == 0 and ((data_iter_step + 1) % (accum_iter * args.print_freq)) == 0: + loss_value_reduce = misc.all_reduce_mean(loss_value) # MUST BE EXECUTED BY ALL NODES + if log_writer is None: + continue + """ We use epoch_1000x as the x-axis in tensorboard. + This calibrates different curves when batch size changes. + """ + epoch_1000x = int(epoch_f * 1000) + log_writer.add_scalar('train_loss', loss_value_reduce, epoch_1000x) + log_writer.add_scalar('train_lr', lr, epoch_1000x) + log_writer.add_scalar('train_iter', epoch_1000x, epoch_1000x) + for name, val in loss_details.items(): + log_writer.add_scalar('train_' + name, val, epoch_1000x) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger) + return {k: meter.global_avg for k, meter in metric_logger.meters.items()} + + +@torch.no_grad() +def test_one_epoch(model: torch.nn.Module, criterion: torch.nn.Module, + data_loader: Sized, device: torch.device, epoch: int, + args, log_writer=None, prefix='test'): + + model.eval() + metric_logger = misc.MetricLogger(delimiter=" ") + metric_logger.meters = defaultdict(lambda: misc.SmoothedValue(window_size=9**9)) + header = 'Test Epoch: [{}]'.format(epoch) + + if log_writer is not None: + print('log_dir: {}'.format(log_writer.log_dir)) + + if hasattr(data_loader, 'dataset') and hasattr(data_loader.dataset, 'set_epoch'): + data_loader.dataset.set_epoch(epoch) + if hasattr(data_loader, 'sampler') and hasattr(data_loader.sampler, 'set_epoch'): + data_loader.sampler.set_epoch(epoch) + + for _, batch in enumerate(metric_logger.log_every(data_loader, args.print_freq, header)): + loss_tuple = loss_of_one_batch(batch, model, criterion, device, + symmetrize_batch=True, + use_amp=bool(args.amp), ret='loss') + loss_value, loss_details = loss_tuple # criterion returns two values + metric_logger.update(loss=float(loss_value), **loss_details) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger) + + aggs = [('avg', 'global_avg'), ('med', 'median')] + results = {f'{k}_{tag}': getattr(meter, attr) for k, meter in metric_logger.meters.items() for tag, attr in aggs} + + if log_writer is not None: + for name, val in results.items(): + log_writer.add_scalar(prefix + '_' + name, val, 1000 * epoch) + + return results diff --git a/src/mast3r_src/dust3r/dust3r/utils/__init__.py b/src/mast3r_src/dust3r/dust3r/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a32692113d830ddc4af4e6ed608f222fbe062e6e --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/utils/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). diff --git a/src/mast3r_src/dust3r/dust3r/utils/device.py b/src/mast3r_src/dust3r/dust3r/utils/device.py new file mode 100644 index 0000000000000000000000000000000000000000..e3b6a74dac05a2e1ba3a2b2f0faa8cea08ece745 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/utils/device.py @@ -0,0 +1,76 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilitary functions for DUSt3R +# -------------------------------------------------------- +import numpy as np +import torch + + +def todevice(batch, device, callback=None, non_blocking=False): + ''' Transfer some variables to another device (i.e. GPU, CPU:torch, CPU:numpy). + + batch: list, tuple, dict of tensors or other things + device: pytorch device or 'numpy' + callback: function that would be called on every sub-elements. + ''' + if callback: + batch = callback(batch) + + if isinstance(batch, dict): + return {k: todevice(v, device) for k, v in batch.items()} + + if isinstance(batch, (tuple, list)): + return type(batch)(todevice(x, device) for x in batch) + + x = batch + if device == 'numpy': + if isinstance(x, torch.Tensor): + x = x.detach().cpu().numpy() + elif x is not None: + if isinstance(x, np.ndarray): + x = torch.from_numpy(x) + if torch.is_tensor(x): + x = x.to(device, non_blocking=non_blocking) + return x + + +to_device = todevice # alias + + +def to_numpy(x): return todevice(x, 'numpy') +def to_cpu(x): return todevice(x, 'cpu') +def to_cuda(x): return todevice(x, 'cuda') + + +def collate_with_cat(whatever, lists=False): + if isinstance(whatever, dict): + return {k: collate_with_cat(vals, lists=lists) for k, vals in whatever.items()} + + elif isinstance(whatever, (tuple, list)): + if len(whatever) == 0: + return whatever + elem = whatever[0] + T = type(whatever) + + if elem is None: + return None + if isinstance(elem, (bool, float, int, str)): + return whatever + if isinstance(elem, tuple): + return T(collate_with_cat(x, lists=lists) for x in zip(*whatever)) + if isinstance(elem, dict): + return {k: collate_with_cat([e[k] for e in whatever], lists=lists) for k in elem} + + if isinstance(elem, torch.Tensor): + return listify(whatever) if lists else torch.cat(whatever) + if isinstance(elem, np.ndarray): + return listify(whatever) if lists else torch.cat([torch.from_numpy(x) for x in whatever]) + + # otherwise, we just chain lists + return sum(whatever, T()) + + +def listify(elems): + return [x for e in elems for x in e] diff --git a/src/mast3r_src/dust3r/dust3r/utils/geometry.py b/src/mast3r_src/dust3r/dust3r/utils/geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..ce365faf2acb97ffaafa1b80cb8ee0c28de0b6d6 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/utils/geometry.py @@ -0,0 +1,366 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# geometry utilitary functions +# -------------------------------------------------------- +import torch +import numpy as np +from scipy.spatial import cKDTree as KDTree + +from dust3r.utils.misc import invalid_to_zeros, invalid_to_nans +from dust3r.utils.device import to_numpy + + +def xy_grid(W, H, device=None, origin=(0, 0), unsqueeze=None, cat_dim=-1, homogeneous=False, **arange_kw): + """ Output a (H,W,2) array of int32 + with output[j,i,0] = i + origin[0] + output[j,i,1] = j + origin[1] + """ + if device is None: + # numpy + arange, meshgrid, stack, ones = np.arange, np.meshgrid, np.stack, np.ones + else: + # torch + arange = lambda *a, **kw: torch.arange(*a, device=device, **kw) + meshgrid, stack = torch.meshgrid, torch.stack + ones = lambda *a: torch.ones(*a, device=device) + + tw, th = [arange(o, o + s, **arange_kw) for s, o in zip((W, H), origin)] + grid = meshgrid(tw, th, indexing='xy') + if homogeneous: + grid = grid + (ones((H, W)),) + if unsqueeze is not None: + grid = (grid[0].unsqueeze(unsqueeze), grid[1].unsqueeze(unsqueeze)) + if cat_dim is not None: + grid = stack(grid, cat_dim) + return grid + + +def geotrf(Trf, pts, ncol=None, norm=False): + """ Apply a geometric transformation to a list of 3-D points. + + H: 3x3 or 4x4 projection matrix (typically a Homography) + p: numpy/torch/tuple of coordinates. Shape must be (...,2) or (...,3) + + ncol: int. number of columns of the result (2 or 3) + norm: float. if != 0, the resut is projected on the z=norm plane. + + Returns an array of projected 2d points. + """ + assert Trf.ndim >= 2 + if isinstance(Trf, np.ndarray): + pts = np.asarray(pts) + elif isinstance(Trf, torch.Tensor): + pts = torch.as_tensor(pts, dtype=Trf.dtype) + + # adapt shape if necessary + output_reshape = pts.shape[:-1] + ncol = ncol or pts.shape[-1] + + # optimized code + if (isinstance(Trf, torch.Tensor) and isinstance(pts, torch.Tensor) and + Trf.ndim == 3 and pts.ndim == 4): + d = pts.shape[3] + if Trf.shape[-1] == d: + pts = torch.einsum("bij, bhwj -> bhwi", Trf, pts) + elif Trf.shape[-1] == d + 1: + pts = torch.einsum("bij, bhwj -> bhwi", Trf[:, :d, :d], pts) + Trf[:, None, None, :d, d] + else: + raise ValueError(f'bad shape, not ending with 3 or 4, for {pts.shape=}') + else: + if Trf.ndim >= 3: + n = Trf.ndim - 2 + assert Trf.shape[:n] == pts.shape[:n], 'batch size does not match' + Trf = Trf.reshape(-1, Trf.shape[-2], Trf.shape[-1]) + + if pts.ndim > Trf.ndim: + # Trf == (B,d,d) & pts == (B,H,W,d) --> (B, H*W, d) + pts = pts.reshape(Trf.shape[0], -1, pts.shape[-1]) + elif pts.ndim == 2: + # Trf == (B,d,d) & pts == (B,d) --> (B, 1, d) + pts = pts[:, None, :] + + if pts.shape[-1] + 1 == Trf.shape[-1]: + Trf = Trf.swapaxes(-1, -2) # transpose Trf + pts = pts @ Trf[..., :-1, :] + Trf[..., -1:, :] + elif pts.shape[-1] == Trf.shape[-1]: + Trf = Trf.swapaxes(-1, -2) # transpose Trf + pts = pts @ Trf + else: + pts = Trf @ pts.T + if pts.ndim >= 2: + pts = pts.swapaxes(-1, -2) + + if norm: + pts = pts / pts[..., -1:] # DONT DO /= BECAUSE OF WEIRD PYTORCH BUG + if norm != 1: + pts *= norm + + res = pts[..., :ncol].reshape(*output_reshape, ncol) + return res + + +def inv(mat): + """ Invert a torch or numpy matrix + """ + if isinstance(mat, torch.Tensor): + return torch.linalg.inv(mat) + if isinstance(mat, np.ndarray): + return np.linalg.inv(mat) + raise ValueError(f'bad matrix type = {type(mat)}') + + +def depthmap_to_pts3d(depth, pseudo_focal, pp=None, **_): + """ + Args: + - depthmap (BxHxW array): + - pseudo_focal: [B,H,W] ; [B,2,H,W] or [B,1,H,W] + Returns: + pointmap of absolute coordinates (BxHxWx3 array) + """ + + if len(depth.shape) == 4: + B, H, W, n = depth.shape + else: + B, H, W = depth.shape + n = None + + if len(pseudo_focal.shape) == 3: # [B,H,W] + pseudo_focalx = pseudo_focaly = pseudo_focal + elif len(pseudo_focal.shape) == 4: # [B,2,H,W] or [B,1,H,W] + pseudo_focalx = pseudo_focal[:, 0] + if pseudo_focal.shape[1] == 2: + pseudo_focaly = pseudo_focal[:, 1] + else: + pseudo_focaly = pseudo_focalx + else: + raise NotImplementedError("Error, unknown input focal shape format.") + + assert pseudo_focalx.shape == depth.shape[:3] + assert pseudo_focaly.shape == depth.shape[:3] + grid_x, grid_y = xy_grid(W, H, cat_dim=0, device=depth.device)[:, None] + + # set principal point + if pp is None: + grid_x = grid_x - (W - 1) / 2 + grid_y = grid_y - (H - 1) / 2 + else: + grid_x = grid_x.expand(B, -1, -1) - pp[:, 0, None, None] + grid_y = grid_y.expand(B, -1, -1) - pp[:, 1, None, None] + + if n is None: + pts3d = torch.empty((B, H, W, 3), device=depth.device) + pts3d[..., 0] = depth * grid_x / pseudo_focalx + pts3d[..., 1] = depth * grid_y / pseudo_focaly + pts3d[..., 2] = depth + else: + pts3d = torch.empty((B, H, W, 3, n), device=depth.device) + pts3d[..., 0, :] = depth * (grid_x / pseudo_focalx)[..., None] + pts3d[..., 1, :] = depth * (grid_y / pseudo_focaly)[..., None] + pts3d[..., 2, :] = depth + return pts3d + + +def depthmap_to_camera_coordinates(depthmap, camera_intrinsics, pseudo_focal=None): + """ + Args: + - depthmap (HxW array): + - camera_intrinsics: a 3x3 matrix + Returns: + pointmap of absolute coordinates (HxWx3 array), and a mask specifying valid pixels. + """ + camera_intrinsics = np.float32(camera_intrinsics) + H, W = depthmap.shape + + # Compute 3D ray associated with each pixel + # Strong assumption: there are no skew terms + assert camera_intrinsics[0, 1] == 0.0 + assert camera_intrinsics[1, 0] == 0.0 + if pseudo_focal is None: + fu = camera_intrinsics[0, 0] + fv = camera_intrinsics[1, 1] + else: + assert pseudo_focal.shape == (H, W) + fu = fv = pseudo_focal + cu = camera_intrinsics[0, 2] + cv = camera_intrinsics[1, 2] + + u, v = np.meshgrid(np.arange(W), np.arange(H)) + z_cam = depthmap + x_cam = (u - cu) * z_cam / fu + y_cam = (v - cv) * z_cam / fv + X_cam = np.stack((x_cam, y_cam, z_cam), axis=-1).astype(np.float32) + + # Mask for valid coordinates + valid_mask = (depthmap > 0.0) + return X_cam, valid_mask + + +def depthmap_to_absolute_camera_coordinates(depthmap, camera_intrinsics, camera_pose, **kw): + """ + Args: + - depthmap (HxW array): + - camera_intrinsics: a 3x3 matrix + - camera_pose: a 4x3 or 4x4 cam2world matrix + Returns: + pointmap of absolute coordinates (HxWx3 array), and a mask specifying valid pixels.""" + X_cam, valid_mask = depthmap_to_camera_coordinates(depthmap, camera_intrinsics) + + X_world = X_cam # default + if camera_pose is not None: + # R_cam2world = np.float32(camera_params["R_cam2world"]) + # t_cam2world = np.float32(camera_params["t_cam2world"]).squeeze() + R_cam2world = camera_pose[:3, :3] + t_cam2world = camera_pose[:3, 3] + + # Express in absolute coordinates (invalid depth values) + X_world = np.einsum("ik, vuk -> vui", R_cam2world, X_cam) + t_cam2world[None, None, :] + + return X_world, valid_mask + + +def colmap_to_opencv_intrinsics(K): + """ + Modify camera intrinsics to follow a different convention. + Coordinates of the center of the top-left pixels are by default: + - (0.5, 0.5) in Colmap + - (0,0) in OpenCV + """ + K = K.copy() + K[0, 2] -= 0.5 + K[1, 2] -= 0.5 + return K + + +def opencv_to_colmap_intrinsics(K): + """ + Modify camera intrinsics to follow a different convention. + Coordinates of the center of the top-left pixels are by default: + - (0.5, 0.5) in Colmap + - (0,0) in OpenCV + """ + K = K.copy() + K[0, 2] += 0.5 + K[1, 2] += 0.5 + return K + + +def normalize_pointcloud(pts1, pts2, norm_mode='avg_dis', valid1=None, valid2=None, ret_factor=False): + """ renorm pointmaps pts1, pts2 with norm_mode + """ + assert pts1.ndim >= 3 and pts1.shape[-1] == 3 + assert pts2 is None or (pts2.ndim >= 3 and pts2.shape[-1] == 3) + norm_mode, dis_mode = norm_mode.split('_') + + if norm_mode == 'avg': + # gather all points together (joint normalization) + nan_pts1, nnz1 = invalid_to_zeros(pts1, valid1, ndim=3) + nan_pts2, nnz2 = invalid_to_zeros(pts2, valid2, ndim=3) if pts2 is not None else (None, 0) + all_pts = torch.cat((nan_pts1, nan_pts2), dim=1) if pts2 is not None else nan_pts1 + + # compute distance to origin + all_dis = all_pts.norm(dim=-1) + if dis_mode == 'dis': + pass # do nothing + elif dis_mode == 'log1p': + all_dis = torch.log1p(all_dis) + elif dis_mode == 'warp-log1p': + # actually warp input points before normalizing them + log_dis = torch.log1p(all_dis) + warp_factor = log_dis / all_dis.clip(min=1e-8) + H1, W1 = pts1.shape[1:-1] + pts1 = pts1 * warp_factor[:, :W1 * H1].view(-1, H1, W1, 1) + if pts2 is not None: + H2, W2 = pts2.shape[1:-1] + pts2 = pts2 * warp_factor[:, W1 * H1:].view(-1, H2, W2, 1) + all_dis = log_dis # this is their true distance afterwards + else: + raise ValueError(f'bad {dis_mode=}') + + norm_factor = all_dis.sum(dim=1) / (nnz1 + nnz2 + 1e-8) + else: + # gather all points together (joint normalization) + nan_pts1 = invalid_to_nans(pts1, valid1, ndim=3) + nan_pts2 = invalid_to_nans(pts2, valid2, ndim=3) if pts2 is not None else None + all_pts = torch.cat((nan_pts1, nan_pts2), dim=1) if pts2 is not None else nan_pts1 + + # compute distance to origin + all_dis = all_pts.norm(dim=-1) + + if norm_mode == 'avg': + norm_factor = all_dis.nanmean(dim=1) + elif norm_mode == 'median': + norm_factor = all_dis.nanmedian(dim=1).values.detach() + elif norm_mode == 'sqrt': + norm_factor = all_dis.sqrt().nanmean(dim=1)**2 + else: + raise ValueError(f'bad {norm_mode=}') + + norm_factor = norm_factor.clip(min=1e-8) + while norm_factor.ndim < pts1.ndim: + norm_factor.unsqueeze_(-1) + + res = pts1 / norm_factor + if pts2 is not None: + res = (res, pts2 / norm_factor) + if ret_factor: + res = res + (norm_factor,) + return res + + +@torch.no_grad() +def get_joint_pointcloud_depth(z1, z2, valid_mask1, valid_mask2=None, quantile=0.5): + # set invalid points to NaN + _z1 = invalid_to_nans(z1, valid_mask1).reshape(len(z1), -1) + _z2 = invalid_to_nans(z2, valid_mask2).reshape(len(z2), -1) if z2 is not None else None + _z = torch.cat((_z1, _z2), dim=-1) if z2 is not None else _z1 + + # compute median depth overall (ignoring nans) + if quantile == 0.5: + shift_z = torch.nanmedian(_z, dim=-1).values + else: + shift_z = torch.nanquantile(_z, quantile, dim=-1) + return shift_z # (B,) + + +@torch.no_grad() +def get_joint_pointcloud_center_scale(pts1, pts2, valid_mask1=None, valid_mask2=None, z_only=False, center=True): + # set invalid points to NaN + _pts1 = invalid_to_nans(pts1, valid_mask1).reshape(len(pts1), -1, 3) + _pts2 = invalid_to_nans(pts2, valid_mask2).reshape(len(pts2), -1, 3) if pts2 is not None else None + _pts = torch.cat((_pts1, _pts2), dim=1) if pts2 is not None else _pts1 + + # compute median center + _center = torch.nanmedian(_pts, dim=1, keepdim=True).values # (B,1,3) + if z_only: + _center[..., :2] = 0 # do not center X and Y + + # compute median norm + _norm = ((_pts - _center) if center else _pts).norm(dim=-1) + scale = torch.nanmedian(_norm, dim=1).values + return _center[:, None, :, :], scale[:, None, None, None] + + +def find_reciprocal_matches(P1, P2): + """ + returns 3 values: + 1 - reciprocal_in_P2: a boolean array of size P2.shape[0], a "True" value indicates a match + 2 - nn2_in_P1: a int array of size P2.shape[0], it contains the indexes of the closest points in P1 + 3 - reciprocal_in_P2.sum(): the number of matches + """ + tree1 = KDTree(P1) + tree2 = KDTree(P2) + + _, nn1_in_P2 = tree2.query(P1, workers=8) + _, nn2_in_P1 = tree1.query(P2, workers=8) + + reciprocal_in_P1 = (nn2_in_P1[nn1_in_P2] == np.arange(len(nn1_in_P2))) + reciprocal_in_P2 = (nn1_in_P2[nn2_in_P1] == np.arange(len(nn2_in_P1))) + assert reciprocal_in_P1.sum() == reciprocal_in_P2.sum() + return reciprocal_in_P2, nn2_in_P1, reciprocal_in_P2.sum() + + +def get_med_dist_between_poses(poses): + from scipy.spatial.distance import pdist + return np.median(pdist([to_numpy(p[:3, 3]) for p in poses])) diff --git a/src/mast3r_src/dust3r/dust3r/utils/image.py b/src/mast3r_src/dust3r/dust3r/utils/image.py new file mode 100644 index 0000000000000000000000000000000000000000..356bc9ae4fd3f72b3163bd1833a42b00dc5e7688 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/utils/image.py @@ -0,0 +1,146 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilitary functions about images (loading/converting...) +# -------------------------------------------------------- +import os +import torch +import numpy as np +import PIL.Image +from PIL.ImageOps import exif_transpose +import torchvision.transforms as tvf +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +import cv2 # noqa + +try: + from pillow_heif import register_heif_opener # noqa + register_heif_opener() + heif_support_enabled = True +except ImportError: + heif_support_enabled = False + +ImgNorm = tvf.Compose([tvf.ToTensor(), tvf.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) +original_transform = tvf.Compose([tvf.ToTensor()]) + + +def img_to_arr( img ): + if isinstance(img, str): + img = imread_cv2(img) + return img + +def imread_cv2(path, options=cv2.IMREAD_COLOR): + """ Open an image or a depthmap with opencv-python. + """ + if path.endswith(('.exr', 'EXR')): + options = cv2.IMREAD_ANYDEPTH + img = cv2.imread(path, options) + if img is None: + raise IOError(f'Could not load image={path} with {options=}') + if img.ndim == 3: + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + return img + + +def rgb(ftensor, true_shape=None): + if isinstance(ftensor, list): + return [rgb(x, true_shape=true_shape) for x in ftensor] + if isinstance(ftensor, torch.Tensor): + ftensor = ftensor.detach().cpu().numpy() # H,W,3 + if ftensor.ndim == 3 and ftensor.shape[0] == 3: + ftensor = ftensor.transpose(1, 2, 0) + elif ftensor.ndim == 4 and ftensor.shape[1] == 3: + ftensor = ftensor.transpose(0, 2, 3, 1) + if true_shape is not None: + H, W = true_shape + ftensor = ftensor[:H, :W] + if ftensor.dtype == np.uint8: + img = np.float32(ftensor) / 255 + else: + img = (ftensor * 0.5) + 0.5 + return img.clip(min=0, max=1) + + +def _resize_pil_image(img, long_edge_size): + S = max(img.size) + if S > long_edge_size: + interp = PIL.Image.LANCZOS + elif S <= long_edge_size: + interp = PIL.Image.BICUBIC + new_size = tuple(int(round(x*long_edge_size/S)) for x in img.size) + return img.resize(new_size, interp) + + +def load_images(folder_or_list, size, square_ok=False, verbose=True): + """ open and convert all images in a list or folder to proper input format for DUSt3R + """ + if isinstance(folder_or_list, str): + if verbose: + print(f'>> Loading images from {folder_or_list}') + root, folder_content = folder_or_list, sorted(os.listdir(folder_or_list)) + + elif isinstance(folder_or_list, list): + if verbose: + print(f'>> Loading a list of {len(folder_or_list)} images') + root, folder_content = '', folder_or_list + + else: + raise ValueError(f'bad {folder_or_list=} ({type(folder_or_list)})') + + supported_images_extensions = ['.jpg', '.jpeg', '.png'] + if heif_support_enabled: + supported_images_extensions += ['.heic', '.heif'] + supported_images_extensions = tuple(supported_images_extensions) + + imgs = [] + for path in folder_content: + if not path.lower().endswith(supported_images_extensions): + continue + img = exif_transpose(PIL.Image.open(os.path.join(root, path))).convert('RGB') + # @MODIFIED + our_method = True + if our_method: + # Resize short side to 512 + W1, H1 = img.size + img = _resize_pil_image(img, round(size * max(W1/H1, H1/W1))) + W, H = img.size + cx, cy = W//2, H//2 + half = min(cx, cy) + img = img.crop((cx-half, cy-half, cx+half, cy+half)) + else: + W1, H1 = img.size + if size == 224: + # resize short side to 224 (then crop) + img = _resize_pil_image(img, round(size * max(W1/H1, H1/W1))) + else: + # resize long side to 512 + img = _resize_pil_image(img, size) + W, H = img.size + cx, cy = W//2, H//2 + if size == 224: + half = min(cx, cy) + img = img.crop((cx-half, cy-half, cx+half, cy+half)) + else: + halfw, halfh = ((2*cx)//16)*8, ((2*cy)//16)*8 + if not (square_ok) and W == H: + halfh = 3*halfw/4 + img = img.crop((cx-halfw, cy-halfh, cx+halfw, cy+halfh)) + + W2, H2 = img.size + if verbose: + print(f' - adding {path} with resolution {W1}x{H1} --> {W2}x{H2}') + # @MODIFIED + imgs.append( + { + 'img': ImgNorm(img)[None], + 'true_shape': np.int32([img.size[::-1]]), + 'idx': len(imgs), + 'instance': str(len(imgs)), + 'original_img': original_transform(img)[None] + } + ) + + assert imgs, 'no images foud at '+root + if verbose: + print(f' (Found {len(imgs)} images)') + return imgs diff --git a/src/mast3r_src/dust3r/dust3r/utils/misc.py b/src/mast3r_src/dust3r/dust3r/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..88c4d2dab6d5c14021ed9ed6646c3159a3a4637b --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/utils/misc.py @@ -0,0 +1,121 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilitary functions for DUSt3R +# -------------------------------------------------------- +import torch + + +def fill_default_args(kwargs, func): + import inspect # a bit hacky but it works reliably + signature = inspect.signature(func) + + for k, v in signature.parameters.items(): + if v.default is inspect.Parameter.empty: + continue + kwargs.setdefault(k, v.default) + + return kwargs + + +def freeze_all_params(modules): + for module in modules: + try: + for n, param in module.named_parameters(): + param.requires_grad = False + except AttributeError: + # module is directly a parameter + module.requires_grad = False + + +def is_symmetrized(gt1, gt2): + x = gt1['instance'] + y = gt2['instance'] + if len(x) == len(y) and len(x) == 1: + return False # special case of batchsize 1 + ok = True + for i in range(0, len(x), 2): + ok = ok and (x[i] == y[i + 1]) and (x[i + 1] == y[i]) + return ok + + +def flip(tensor): + """ flip so that tensor[0::2] <=> tensor[1::2] """ + return torch.stack((tensor[1::2], tensor[0::2]), dim=1).flatten(0, 1) + + +def interleave(tensor1, tensor2): + res1 = torch.stack((tensor1, tensor2), dim=1).flatten(0, 1) + res2 = torch.stack((tensor2, tensor1), dim=1).flatten(0, 1) + return res1, res2 + + +def transpose_to_landscape(head, activate=True): + """ Predict in the correct aspect-ratio, + then transpose the result in landscape + and stack everything back together. + """ + def wrapper_no(decout, true_shape): + B = len(true_shape) + assert true_shape[0:1].allclose(true_shape), 'true_shape must be all identical' + H, W = true_shape[0].cpu().tolist() + res = head(decout, (H, W)) + return res + + def wrapper_yes(decout, true_shape): + B = len(true_shape) + # by definition, the batch is in landscape mode so W >= H + H, W = int(true_shape.min()), int(true_shape.max()) + + height, width = true_shape.T + is_landscape = (width >= height) + is_portrait = ~is_landscape + + # true_shape = true_shape.cpu() + if is_landscape.all(): + return head(decout, (H, W)) + if is_portrait.all(): + return transposed(head(decout, (W, H))) + + # batch is a mix of both portraint & landscape + def selout(ar): return [d[ar] for d in decout] + l_result = head(selout(is_landscape), (H, W)) + p_result = transposed(head(selout(is_portrait), (W, H))) + + # allocate full result + result = {} + for k in l_result | p_result: + x = l_result[k].new(B, *l_result[k].shape[1:]) + x[is_landscape] = l_result[k] + x[is_portrait] = p_result[k] + result[k] = x + + return result + + return wrapper_yes if activate else wrapper_no + + +def transposed(dic): + return {k: v.swapaxes(1, 2) for k, v in dic.items()} + + +def invalid_to_nans(arr, valid_mask, ndim=999): + if valid_mask is not None: + arr = arr.clone() + arr[~valid_mask] = float('nan') + if arr.ndim > ndim: + arr = arr.flatten(-2 - (arr.ndim - ndim), -2) + return arr + + +def invalid_to_zeros(arr, valid_mask, ndim=999): + if valid_mask is not None: + arr = arr.clone() + arr[~valid_mask] = 0 + nnz = valid_mask.view(len(valid_mask), -1).sum(1) + else: + nnz = arr.numel() // len(arr) if len(arr) else 0 # number of point per image + if arr.ndim > ndim: + arr = arr.flatten(-2 - (arr.ndim - ndim), -2) + return arr, nnz diff --git a/src/mast3r_src/dust3r/dust3r/utils/parallel.py b/src/mast3r_src/dust3r/dust3r/utils/parallel.py new file mode 100644 index 0000000000000000000000000000000000000000..06ae7fefdb9d2298929f0cbc20dfbc57eb7d7f7b --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/utils/parallel.py @@ -0,0 +1,79 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilitary functions for multiprocessing +# -------------------------------------------------------- +from tqdm import tqdm +from multiprocessing.dummy import Pool as ThreadPool +from multiprocessing import cpu_count + + +def parallel_threads(function, args, workers=0, star_args=False, kw_args=False, front_num=1, Pool=ThreadPool, **tqdm_kw): + """ tqdm but with parallel execution. + + Will essentially return + res = [ function(arg) # default + function(*arg) # if star_args is True + function(**arg) # if kw_args is True + for arg in args] + + Note: + the first elements of args will not be parallelized. + This can be useful for debugging. + """ + while workers <= 0: + workers += cpu_count() + if workers == 1: + front_num = float('inf') + + # convert into an iterable + try: + n_args_parallel = len(args) - front_num + except TypeError: + n_args_parallel = None + args = iter(args) + + # sequential execution first + front = [] + while len(front) < front_num: + try: + a = next(args) + except StopIteration: + return front # end of the iterable + front.append(function(*a) if star_args else function(**a) if kw_args else function(a)) + + # then parallel execution + out = [] + with Pool(workers) as pool: + # Pass the elements of args into function + if star_args: + futures = pool.imap(starcall, [(function, a) for a in args]) + elif kw_args: + futures = pool.imap(starstarcall, [(function, a) for a in args]) + else: + futures = pool.imap(function, args) + # Print out the progress as tasks complete + for f in tqdm(futures, total=n_args_parallel, **tqdm_kw): + out.append(f) + return front + out + + +def parallel_processes(*args, **kwargs): + """ Same as parallel_threads, with processes + """ + import multiprocessing as mp + kwargs['Pool'] = mp.Pool + return parallel_threads(*args, **kwargs) + + +def starcall(args): + """ convenient wrapper for Process.Pool """ + function, args = args + return function(*args) + + +def starstarcall(args): + """ convenient wrapper for Process.Pool """ + function, args = args + return function(**args) diff --git a/src/mast3r_src/dust3r/dust3r/utils/path_to_croco.py b/src/mast3r_src/dust3r/dust3r/utils/path_to_croco.py new file mode 100644 index 0000000000000000000000000000000000000000..39226ce6bc0e1993ba98a22096de32cb6fa916b4 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/utils/path_to_croco.py @@ -0,0 +1,19 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# CroCo submodule import +# -------------------------------------------------------- + +import sys +import os.path as path +HERE_PATH = path.normpath(path.dirname(__file__)) +CROCO_REPO_PATH = path.normpath(path.join(HERE_PATH, '../../croco')) +CROCO_MODELS_PATH = path.join(CROCO_REPO_PATH, 'models') +# check the presence of models directory in repo to be sure its cloned +if path.isdir(CROCO_MODELS_PATH): + # workaround for sibling import + sys.path.insert(0, CROCO_REPO_PATH) +else: + raise ImportError(f"croco is not initialized, could not find: {CROCO_MODELS_PATH}.\n " + "Did you forget to run 'git submodule update --init --recursive' ?") diff --git a/src/mast3r_src/dust3r/dust3r/viz.py b/src/mast3r_src/dust3r/dust3r/viz.py new file mode 100644 index 0000000000000000000000000000000000000000..9150e8b850d9f1e6bf9ddf6e865d34fc743e276a --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r/viz.py @@ -0,0 +1,381 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Visualization utilities using trimesh +# -------------------------------------------------------- +import PIL.Image +import numpy as np +from scipy.spatial.transform import Rotation +import torch + +from dust3r.utils.geometry import geotrf, get_med_dist_between_poses, depthmap_to_absolute_camera_coordinates +from dust3r.utils.device import to_numpy +from dust3r.utils.image import rgb, img_to_arr + +try: + import trimesh +except ImportError: + print('/!\\ module trimesh is not installed, cannot visualize results /!\\') + + + +def cat_3d(vecs): + if isinstance(vecs, (np.ndarray, torch.Tensor)): + vecs = [vecs] + return np.concatenate([p.reshape(-1, 3) for p in to_numpy(vecs)]) + + +def show_raw_pointcloud(pts3d, colors, point_size=2): + scene = trimesh.Scene() + + pct = trimesh.PointCloud(cat_3d(pts3d), colors=cat_3d(colors)) + scene.add_geometry(pct) + + scene.show(line_settings={'point_size': point_size}) + + +def pts3d_to_trimesh(img, pts3d, valid=None): + H, W, THREE = img.shape + assert THREE == 3 + assert img.shape == pts3d.shape + + vertices = pts3d.reshape(-1, 3) + + # make squares: each pixel == 2 triangles + idx = np.arange(len(vertices)).reshape(H, W) + idx1 = idx[:-1, :-1].ravel() # top-left corner + idx2 = idx[:-1, +1:].ravel() # right-left corner + idx3 = idx[+1:, :-1].ravel() # bottom-left corner + idx4 = idx[+1:, +1:].ravel() # bottom-right corner + faces = np.concatenate(( + np.c_[idx1, idx2, idx3], + np.c_[idx3, idx2, idx1], # same triangle, but backward (cheap solution to cancel face culling) + np.c_[idx2, idx3, idx4], + np.c_[idx4, idx3, idx2], # same triangle, but backward (cheap solution to cancel face culling) + ), axis=0) + + # prepare triangle colors + face_colors = np.concatenate(( + img[:-1, :-1].reshape(-1, 3), + img[:-1, :-1].reshape(-1, 3), + img[+1:, +1:].reshape(-1, 3), + img[+1:, +1:].reshape(-1, 3) + ), axis=0) + + # remove invalid faces + if valid is not None: + assert valid.shape == (H, W) + valid_idxs = valid.ravel() + valid_faces = valid_idxs[faces].all(axis=-1) + faces = faces[valid_faces] + face_colors = face_colors[valid_faces] + + assert len(faces) == len(face_colors) + return dict(vertices=vertices, face_colors=face_colors, faces=faces) + + +def cat_meshes(meshes): + vertices, faces, colors = zip(*[(m['vertices'], m['faces'], m['face_colors']) for m in meshes]) + n_vertices = np.cumsum([0]+[len(v) for v in vertices]) + for i in range(len(faces)): + faces[i][:] += n_vertices[i] + + vertices = np.concatenate(vertices) + colors = np.concatenate(colors) + faces = np.concatenate(faces) + return dict(vertices=vertices, face_colors=colors, faces=faces) + + +def show_duster_pairs(view1, view2, pred1, pred2): + import matplotlib.pyplot as pl + pl.ion() + + for e in range(len(view1['instance'])): + i = view1['idx'][e] + j = view2['idx'][e] + img1 = rgb(view1['img'][e]) + img2 = rgb(view2['img'][e]) + conf1 = pred1['conf'][e].squeeze() + conf2 = pred2['conf'][e].squeeze() + score = conf1.mean()*conf2.mean() + print(f">> Showing pair #{e} {i}-{j} {score=:g}") + pl.clf() + pl.subplot(221).imshow(img1) + pl.subplot(223).imshow(img2) + pl.subplot(222).imshow(conf1, vmin=1, vmax=30) + pl.subplot(224).imshow(conf2, vmin=1, vmax=30) + pts1 = pred1['pts3d'][e] + pts2 = pred2['pts3d_in_other_view'][e] + pl.subplots_adjust(0, 0, 1, 1, 0, 0) + if input('show pointcloud? (y/n) ') == 'y': + show_raw_pointcloud(cat(pts1, pts2), cat(img1, img2), point_size=5) + + +def auto_cam_size(im_poses): + return 0.1 * get_med_dist_between_poses(im_poses) + + +class SceneViz: + def __init__(self): + self.scene = trimesh.Scene() + + def add_rgbd(self, image, depth, intrinsics=None, cam2world=None, zfar=np.inf, mask=None): + image = img_to_arr(image) + + # make up some intrinsics + if intrinsics is None: + H, W, THREE = image.shape + focal = max(H, W) + intrinsics = np.float32([[focal, 0, W/2], [0, focal, H/2], [0, 0, 1]]) + + # compute 3d points + pts3d = depthmap_to_pts3d(depth, intrinsics, cam2world=cam2world) + + return self.add_pointcloud(pts3d, image, mask=(depth 150) + mask |= (hsv[:, :, 1] < 30) & (hsv[:, :, 2] > 180) + mask |= (hsv[:, :, 1] < 50) & (hsv[:, :, 2] > 220) + + # Morphological operations + kernel = np.ones((5, 5), np.uint8) + mask2 = ndimage.binary_opening(mask, structure=kernel) + + # keep only largest CC + _, labels, stats, _ = cv2.connectedComponentsWithStats(mask2.view(np.uint8), connectivity=8) + cc_sizes = stats[1:, cv2.CC_STAT_AREA] + order = cc_sizes.argsort()[::-1] # bigger first + i = 0 + selection = [] + while i < len(order) and cc_sizes[order[i]] > cc_sizes[order[0]] / 2: + selection.append(1 + order[i]) + i += 1 + mask3 = np.in1d(labels, selection).reshape(labels.shape) + + # Apply mask + return torch.from_numpy(mask3) diff --git a/src/mast3r_src/dust3r/dust3r_visloc/README.md b/src/mast3r_src/dust3r/dust3r_visloc/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6d0512ac1516ba2655a10d4ae3d10b51346e233c --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r_visloc/README.md @@ -0,0 +1,93 @@ +# Visual Localization with DUSt3R + +## Dataset preparation + +### CambridgeLandmarks + +Each subscene should look like this: + +``` +Cambridge_Landmarks +├─ mapping +│ ├─ GreatCourt +│ │ └─ colmap/reconstruction +│ │ ├─ cameras.txt +│ │ ├─ images.txt +│ │ └─ points3D.txt +├─ kapture +│ ├─ GreatCourt +│ │ └─ query # https://github.com/naver/kapture/blob/main/doc/datasets.adoc#cambridge-landmarks +│ ... +├─ GreatCourt +│ ├─ pairsfile/query +│ │ └─ AP-GeM-LM18_top50.txt # https://github.com/naver/deep-image-retrieval/blob/master/dirtorch/extract_kapture.py followed by https://github.com/naver/kapture-localization/blob/main/tools/kapture_compute_image_pairs.py +│ ├─ seq1 +│ ... +... +``` + +### 7Scenes +Each subscene should look like this: + +``` +7-scenes +├─ chess +│ ├─ mapping/ # https://github.com/naver/kapture/blob/main/doc/datasets.adoc#1-7-scenes +│ ├─ query/ # https://github.com/naver/kapture/blob/main/doc/datasets.adoc#1-7-scenes +│ └─ pairsfile/query/ +│ └─ APGeM-LM18_top20.txt # https://github.com/naver/deep-image-retrieval/blob/master/dirtorch/extract_kapture.py followed by https://github.com/naver/kapture-localization/blob/main/tools/kapture_compute_image_pairs.py +... +``` + +### Aachen-Day-Night + +``` +Aachen-Day-Night-v1.1 +├─ mapping +│ ├─ colmap/reconstruction +│ │ ├─ cameras.txt +│ │ ├─ images.txt +│ │ └─ points3D.txt +├─ kapture +│ └─ query # https://github.com/naver/kapture/blob/main/doc/datasets.adoc#2-aachen-day-night-v11 +├─ images +│ ├─ db +│ ├─ query +│ └─ sequences +└─ pairsfile/query + └─ fire_top50.txt # https://github.com/naver/fire/blob/main/kapture_compute_pairs.py +``` + +### InLoc + +``` +InLoc +├─ mapping # https://github.com/naver/kapture/blob/main/doc/datasets.adoc#6-inloc +├─ query # https://github.com/naver/kapture/blob/main/doc/datasets.adoc#6-inloc +└─ pairsfile/query + └─ pairs-query-netvlad40-temporal.txt # https://github.com/cvg/Hierarchical-Localization/blob/master/pairs/inloc/pairs-query-netvlad40-temporal.txt +``` + +## Example Commands + +With `visloc.py` you can run our visual localization experiments on Aachen-Day-Night, InLoc, Cambridge Landmarks and 7 Scenes. + +```bash +# Aachen-Day-Night-v1.1: +# scene in 'day' 'night' +# scene can also be 'all' +python3 visloc.py --model_name DUSt3R_ViTLarge_BaseDecoder_512_dpt --dataset "VislocAachenDayNight('/path/to/prepared/Aachen-Day-Night-v1.1/', subscene='${scene}', pairsfile='fire_top50', topk=20)" --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/Aachen-Day-Night-v1.1/${scene}/loc + +# InLoc +python3 visloc.py --model_name DUSt3R_ViTLarge_BaseDecoder_512_dpt --dataset "VislocInLoc('/path/to/prepared/InLoc/', pairsfile='pairs-query-netvlad40-temporal', topk=20)" --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/InLoc/loc + + +# 7-scenes: +# scene in 'chess' 'fire' 'heads' 'office' 'pumpkin' 'redkitchen' 'stairs' +python3 visloc.py --model_name MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt --dataset "VislocSevenScenes('/path/to/prepared/7-scenes/', subscene='${scene}', pairsfile='APGeM-LM18_top20', topk=1)" --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/7-scenes/${scene}/loc + +# Cambridge Landmarks: +# scene in 'ShopFacade' 'GreatCourt' 'KingsCollege' 'OldHospital' 'StMarysChurch' +python3 visloc.py --model_name DUSt3R_ViTLarge_BaseDecoder_512_dpt --dataset "VislocCambridgeLandmarks('/path/to/prepared/Cambridge_Landmarks/', subscene='${scene}', pairsfile='APGeM-LM18_top20', topk=1)" --pnp_mode poselib --reprojection_error_diag_ratio 0.008 --output_dir /path/to/output/Cambridge_Landmarks/${scene}/loc + +``` \ No newline at end of file diff --git a/src/mast3r_src/dust3r/dust3r_visloc/__init__.py b/src/mast3r_src/dust3r/dust3r_visloc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a32692113d830ddc4af4e6ed608f222fbe062e6e --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r_visloc/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). diff --git a/src/mast3r_src/dust3r/dust3r_visloc/datasets/__init__.py b/src/mast3r_src/dust3r/dust3r_visloc/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..566926b1e248e4b64fc5182031af634435bb8601 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r_visloc/datasets/__init__.py @@ -0,0 +1,6 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +from .sevenscenes import VislocSevenScenes +from .cambridge_landmarks import VislocCambridgeLandmarks +from .aachen_day_night import VislocAachenDayNight +from .inloc import VislocInLoc diff --git a/src/mast3r_src/dust3r/dust3r_visloc/datasets/aachen_day_night.py b/src/mast3r_src/dust3r/dust3r_visloc/datasets/aachen_day_night.py new file mode 100644 index 0000000000000000000000000000000000000000..159548e8b51a1b5872a2392cd9107ff96e40e801 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r_visloc/datasets/aachen_day_night.py @@ -0,0 +1,24 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# AachenDayNight dataloader +# -------------------------------------------------------- +import os +from dust3r_visloc.datasets.base_colmap import BaseVislocColmapDataset + + +class VislocAachenDayNight(BaseVislocColmapDataset): + def __init__(self, root, subscene, pairsfile, topk=1, cache_sfm=False): + assert subscene in [None, '', 'day', 'night', 'all'] + self.subscene = subscene + image_path = os.path.join(root, 'images') + map_path = os.path.join(root, 'mapping/colmap/reconstruction') + query_path = os.path.join(root, 'kapture', 'query') + pairsfile_path = os.path.join(root, 'pairsfile/query', pairsfile + '.txt') + super().__init__(image_path=image_path, map_path=map_path, + query_path=query_path, pairsfile_path=pairsfile_path, + topk=topk, cache_sfm=cache_sfm) + self.scenes = [filename for filename in self.scenes if filename in self.pairs] + if self.subscene == 'day' or self.subscene == 'night': + self.scenes = [filename for filename in self.scenes if self.subscene in filename] diff --git a/src/mast3r_src/dust3r/dust3r_visloc/datasets/base_colmap.py b/src/mast3r_src/dust3r/dust3r_visloc/datasets/base_colmap.py new file mode 100644 index 0000000000000000000000000000000000000000..5dc2d64f69fb0954a148f0f4170508fe2045e046 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r_visloc/datasets/base_colmap.py @@ -0,0 +1,282 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Base class for colmap / kapture +# -------------------------------------------------------- +import os +import numpy as np +from tqdm import tqdm +import collections +import pickle +import PIL.Image +import torch +from scipy.spatial.transform import Rotation +import torchvision.transforms as tvf + +from kapture.core import CameraType +from kapture.io.csv import kapture_from_dir +from kapture_localization.utils.pairsfile import get_ordered_pairs_from_file + +from dust3r_visloc.datasets.utils import cam_to_world_from_kapture, get_resize_function, rescale_points3d +from dust3r_visloc.datasets.base_dataset import BaseVislocDataset +from dust3r.datasets.utils.transforms import ImgNorm +from dust3r.utils.geometry import colmap_to_opencv_intrinsics + +KaptureSensor = collections.namedtuple('Sensor', 'sensor_params camera_params') + + +def kapture_to_opencv_intrinsics(sensor): + """ + Convert from Kapture to OpenCV parameters. + Warning: we assume that the camera and pixel coordinates follow Colmap conventions here. + Args: + sensor: Kapture sensor + """ + sensor_type = sensor.sensor_params[0] + if sensor_type == "SIMPLE_PINHOLE": + # Simple pinhole model. + # We still call OpenCV undistorsion however for code simplicity. + w, h, f, cx, cy = sensor.camera_params + k1 = 0 + k2 = 0 + p1 = 0 + p2 = 0 + fx = fy = f + elif sensor_type == "PINHOLE": + w, h, fx, fy, cx, cy = sensor.camera_params + k1 = 0 + k2 = 0 + p1 = 0 + p2 = 0 + elif sensor_type == "SIMPLE_RADIAL": + w, h, f, cx, cy, k1 = sensor.camera_params + k2 = 0 + p1 = 0 + p2 = 0 + fx = fy = f + elif sensor_type == "RADIAL": + w, h, f, cx, cy, k1, k2 = sensor.camera_params + p1 = 0 + p2 = 0 + fx = fy = f + elif sensor_type == "OPENCV": + w, h, fx, fy, cx, cy, k1, k2, p1, p2 = sensor.camera_params + else: + raise NotImplementedError(f"Sensor type {sensor_type} is not supported yet.") + + cameraMatrix = np.asarray([[fx, 0, cx], + [0, fy, cy], + [0, 0, 1]], dtype=np.float32) + + # We assume that Kapture data comes from Colmap: the origin is different. + cameraMatrix = colmap_to_opencv_intrinsics(cameraMatrix) + + distCoeffs = np.asarray([k1, k2, p1, p2], dtype=np.float32) + return cameraMatrix, distCoeffs, (w, h) + + +def K_from_colmap(elems): + sensor = KaptureSensor(elems, tuple(map(float, elems[1:]))) + cameraMatrix, distCoeffs, (w, h) = kapture_to_opencv_intrinsics(sensor) + res = dict(resolution=(w, h), + intrinsics=cameraMatrix, + distortion=distCoeffs) + return res + + +def pose_from_qwxyz_txyz(elems): + qw, qx, qy, qz, tx, ty, tz = map(float, elems) + pose = np.eye(4) + pose[:3, :3] = Rotation.from_quat((qx, qy, qz, qw)).as_matrix() + pose[:3, 3] = (tx, ty, tz) + return np.linalg.inv(pose) # returns cam2world + + +class BaseVislocColmapDataset(BaseVislocDataset): + def __init__(self, image_path, map_path, query_path, pairsfile_path, topk=1, cache_sfm=False): + super().__init__() + self.topk = topk + self.num_views = self.topk + 1 + self.image_path = image_path + self.cache_sfm = cache_sfm + + self._load_sfm(map_path) + + kdata_query = kapture_from_dir(query_path) + assert kdata_query.records_camera is not None and kdata_query.trajectories is not None + + kdata_query_searchindex = {kdata_query.records_camera[(timestamp, sensor_id)]: (timestamp, sensor_id) + for timestamp, sensor_id in kdata_query.records_camera.key_pairs()} + self.query_data = {'kdata': kdata_query, 'searchindex': kdata_query_searchindex} + + self.pairs = get_ordered_pairs_from_file(pairsfile_path) + self.scenes = kdata_query.records_camera.data_list() + + def _load_sfm(self, sfm_dir): + sfm_cache_path = os.path.join(sfm_dir, 'dust3r_cache.pkl') + if os.path.isfile(sfm_cache_path) and self.cache_sfm: + with open(sfm_cache_path, "rb") as f: + data = pickle.load(f) + self.img_infos = data['img_infos'] + self.points3D = data['points3D'] + return + + # load cameras + with open(os.path.join(sfm_dir, 'cameras.txt'), 'r') as f: + raw = f.read().splitlines()[3:] # skip header + + intrinsics = {} + for camera in tqdm(raw): + camera = camera.split(' ') + intrinsics[int(camera[0])] = K_from_colmap(camera[1:]) + + # load images + with open(os.path.join(sfm_dir, 'images.txt'), 'r') as f: + raw = f.read().splitlines() + raw = [line for line in raw if not line.startswith('#')] # skip header + + self.img_infos = {} + for image, points in tqdm(zip(raw[0::2], raw[1::2]), total=len(raw) // 2): + image = image.split(' ') + points = points.split(' ') + + img_name = image[-1] + current_points2D = {int(i): (float(x), float(y)) + for i, x, y in zip(points[2::3], points[0::3], points[1::3]) if i != '-1'} + self.img_infos[img_name] = dict(intrinsics[int(image[-2])], + path=img_name, + camera_pose=pose_from_qwxyz_txyz(image[1: -2]), + sparse_pts2d=current_points2D) + + # load 3D points + with open(os.path.join(sfm_dir, 'points3D.txt'), 'r') as f: + raw = f.read().splitlines() + raw = [line for line in raw if not line.startswith('#')] # skip header + + self.points3D = {} + for point in tqdm(raw): + point = point.split() + self.points3D[int(point[0])] = tuple(map(float, point[1:4])) + + if self.cache_sfm: + to_save = \ + { + 'img_infos': self.img_infos, + 'points3D': self.points3D + } + with open(sfm_cache_path, "wb") as f: + pickle.dump(to_save, f) + + def __len__(self): + return len(self.scenes) + + def _get_view_query(self, imgname): + kdata, searchindex = map(self.query_data.get, ['kdata', 'searchindex']) + + timestamp, camera_id = searchindex[imgname] + + camera_params = kdata.sensors[camera_id].camera_params + if kdata.sensors[camera_id].camera_type == CameraType.SIMPLE_PINHOLE: + W, H, f, cx, cy = camera_params + k1 = 0 + fx = fy = f + elif kdata.sensors[camera_id].camera_type == CameraType.SIMPLE_RADIAL: + W, H, f, cx, cy, k1 = camera_params + fx = fy = f + else: + raise NotImplementedError('not implemented') + + W, H = int(W), int(H) + intrinsics = np.float32([(fx, 0, cx), + (0, fy, cy), + (0, 0, 1)]) + intrinsics = colmap_to_opencv_intrinsics(intrinsics) + distortion = [k1, 0, 0, 0] + + if kdata.trajectories is not None and (timestamp, camera_id) in kdata.trajectories: + cam_to_world = cam_to_world_from_kapture(kdata, timestamp, camera_id) + else: + cam_to_world = np.eye(4, dtype=np.float32) + + # Load RGB image + rgb_image = PIL.Image.open(os.path.join(self.image_path, imgname)).convert('RGB') + rgb_image.load() + resize_func, _, to_orig = get_resize_function(self.maxdim, self.patch_size, H, W) + rgb_tensor = resize_func(ImgNorm(rgb_image)) + + view = { + 'intrinsics': intrinsics, + 'distortion': distortion, + 'cam_to_world': cam_to_world, + 'rgb': rgb_image, + 'rgb_rescaled': rgb_tensor, + 'to_orig': to_orig, + 'idx': 0, + 'image_name': imgname + } + return view + + def _get_view_map(self, imgname, idx): + infos = self.img_infos[imgname] + + rgb_image = PIL.Image.open(os.path.join(self.image_path, infos['path'])).convert('RGB') + rgb_image.load() + W, H = rgb_image.size + intrinsics = infos['intrinsics'] + intrinsics = colmap_to_opencv_intrinsics(intrinsics) + distortion_coefs = infos['distortion'] + + pts2d = infos['sparse_pts2d'] + sparse_pos2d = np.float32(list(pts2d.values())) # pts2d from colmap + sparse_pts3d = np.float32([self.points3D[i] for i in pts2d]) + + # store full resolution 2D->3D + sparse_pos2d_cv2 = sparse_pos2d.copy() + sparse_pos2d_cv2[:, 0] -= 0.5 + sparse_pos2d_cv2[:, 1] -= 0.5 + sparse_pos2d_int = sparse_pos2d_cv2.round().astype(np.int64) + valid = (sparse_pos2d_int[:, 0] >= 0) & (sparse_pos2d_int[:, 0] < W) & ( + sparse_pos2d_int[:, 1] >= 0) & (sparse_pos2d_int[:, 1] < H) + sparse_pos2d_int = sparse_pos2d_int[valid] + # nan => invalid + pts3d = np.full((H, W, 3), np.nan, dtype=np.float32) + pts3d[sparse_pos2d_int[:, 1], sparse_pos2d_int[:, 0]] = sparse_pts3d[valid] + pts3d = torch.from_numpy(pts3d) + + cam_to_world = infos['camera_pose'] # cam2world + + # also store resized resolution 2D->3D + resize_func, to_resize, to_orig = get_resize_function(self.maxdim, self.patch_size, H, W) + rgb_tensor = resize_func(ImgNorm(rgb_image)) + + HR, WR = rgb_tensor.shape[1:] + _, _, pts3d_rescaled, valid_rescaled = rescale_points3d(sparse_pos2d_cv2, sparse_pts3d, to_resize, HR, WR) + pts3d_rescaled = torch.from_numpy(pts3d_rescaled) + valid_rescaled = torch.from_numpy(valid_rescaled) + + view = { + 'intrinsics': intrinsics, + 'distortion': distortion_coefs, + 'cam_to_world': cam_to_world, + 'rgb': rgb_image, + "pts3d": pts3d, + "valid": pts3d.sum(dim=-1).isfinite(), + 'rgb_rescaled': rgb_tensor, + "pts3d_rescaled": pts3d_rescaled, + "valid_rescaled": valid_rescaled, + 'to_orig': to_orig, + 'idx': idx, + 'image_name': imgname + } + return view + + def __getitem__(self, idx): + assert self.maxdim is not None and self.patch_size is not None + query_image = self.scenes[idx] + map_images = [p[0] for p in self.pairs[query_image][:self.topk]] + views = [] + views.append(self._get_view_query(query_image)) + for idx, map_image in enumerate(map_images): + views.append(self._get_view_map(map_image, idx + 1)) + return views diff --git a/src/mast3r_src/dust3r/dust3r_visloc/datasets/base_dataset.py b/src/mast3r_src/dust3r/dust3r_visloc/datasets/base_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..cda3774c5ab5b668be5eecf89681abc96df5fe17 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r_visloc/datasets/base_dataset.py @@ -0,0 +1,19 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Base class +# -------------------------------------------------------- +class BaseVislocDataset: + def __init__(self): + pass + + def set_resolution(self, model): + self.maxdim = max(model.patch_embed.img_size) + self.patch_size = model.patch_embed.patch_size + + def __len__(self): + raise NotImplementedError() + + def __getitem__(self, idx): + raise NotImplementedError() \ No newline at end of file diff --git a/src/mast3r_src/dust3r/dust3r_visloc/datasets/cambridge_landmarks.py b/src/mast3r_src/dust3r/dust3r_visloc/datasets/cambridge_landmarks.py new file mode 100644 index 0000000000000000000000000000000000000000..ca3e131941bf444d86a709d23e518e7b93d3d0f6 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r_visloc/datasets/cambridge_landmarks.py @@ -0,0 +1,19 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Cambridge Landmarks dataloader +# -------------------------------------------------------- +import os +from dust3r_visloc.datasets.base_colmap import BaseVislocColmapDataset + + +class VislocCambridgeLandmarks (BaseVislocColmapDataset): + def __init__(self, root, subscene, pairsfile, topk=1, cache_sfm=False): + image_path = os.path.join(root, subscene) + map_path = os.path.join(root, 'mapping', subscene, 'colmap/reconstruction') + query_path = os.path.join(root, 'kapture', subscene, 'query') + pairsfile_path = os.path.join(root, subscene, 'pairsfile/query', pairsfile + '.txt') + super().__init__(image_path=image_path, map_path=map_path, + query_path=query_path, pairsfile_path=pairsfile_path, + topk=topk, cache_sfm=cache_sfm) \ No newline at end of file diff --git a/src/mast3r_src/dust3r/dust3r_visloc/datasets/inloc.py b/src/mast3r_src/dust3r/dust3r_visloc/datasets/inloc.py new file mode 100644 index 0000000000000000000000000000000000000000..99ed11f554203d353d0559d0589f40ec1ffbf66e --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r_visloc/datasets/inloc.py @@ -0,0 +1,167 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# InLoc dataloader +# -------------------------------------------------------- +import os +import numpy as np +import torch +import PIL.Image +import scipy.io + +import kapture +from kapture.io.csv import kapture_from_dir +from kapture_localization.utils.pairsfile import get_ordered_pairs_from_file + +from dust3r_visloc.datasets.utils import cam_to_world_from_kapture, get_resize_function, rescale_points3d +from dust3r_visloc.datasets.base_dataset import BaseVislocDataset +from dust3r.datasets.utils.transforms import ImgNorm +from dust3r.utils.geometry import xy_grid, geotrf + + +def read_alignments(path_to_alignment): + aligns = {} + with open(path_to_alignment, "r") as fid: + while True: + line = fid.readline() + if not line: + break + if len(line) == 4: + trans_nr = line[:-1] + while line != 'After general icp:\n': + line = fid.readline() + line = fid.readline() + p = [] + for i in range(4): + elems = line.split(' ') + line = fid.readline() + for e in elems: + if len(e) != 0: + p.append(float(e)) + P = np.array(p).reshape(4, 4) + aligns[trans_nr] = P + return aligns + + +class VislocInLoc(BaseVislocDataset): + def __init__(self, root, pairsfile, topk=1): + super().__init__() + self.root = root + self.topk = topk + self.num_views = self.topk + 1 + self.maxdim = None + self.patch_size = None + + query_path = os.path.join(self.root, 'query') + kdata_query = kapture_from_dir(query_path) + assert kdata_query.records_camera is not None + kdata_query_searchindex = {kdata_query.records_camera[(timestamp, sensor_id)]: (timestamp, sensor_id) + for timestamp, sensor_id in kdata_query.records_camera.key_pairs()} + self.query_data = {'path': query_path, 'kdata': kdata_query, 'searchindex': kdata_query_searchindex} + + map_path = os.path.join(self.root, 'mapping') + kdata_map = kapture_from_dir(map_path) + assert kdata_map.records_camera is not None and kdata_map.trajectories is not None + kdata_map_searchindex = {kdata_map.records_camera[(timestamp, sensor_id)]: (timestamp, sensor_id) + for timestamp, sensor_id in kdata_map.records_camera.key_pairs()} + self.map_data = {'path': map_path, 'kdata': kdata_map, 'searchindex': kdata_map_searchindex} + + try: + self.pairs = get_ordered_pairs_from_file(os.path.join(self.root, 'pairfiles/query', pairsfile + '.txt')) + except Exception as e: + # if using pairs from hloc + self.pairs = {} + with open(os.path.join(self.root, 'pairfiles/query', pairsfile + '.txt'), 'r') as fid: + lines = fid.readlines() + for line in lines: + splits = line.rstrip("\n\r").split(" ") + self.pairs.setdefault(splits[0].replace('query/', ''), []).append( + (splits[1].replace('database/cutouts/', ''), 1.0) + ) + + self.scenes = kdata_query.records_camera.data_list() + + self.aligns_DUC1 = read_alignments(os.path.join(self.root, 'mapping/DUC1_alignment/all_transformations.txt')) + self.aligns_DUC2 = read_alignments(os.path.join(self.root, 'mapping/DUC2_alignment/all_transformations.txt')) + + def __len__(self): + return len(self.scenes) + + def __getitem__(self, idx): + assert self.maxdim is not None and self.patch_size is not None + query_image = self.scenes[idx] + map_images = [p[0] for p in self.pairs[query_image][:self.topk]] + views = [] + dataarray = [(query_image, self.query_data, False)] + [(map_image, self.map_data, True) + for map_image in map_images] + for idx, (imgname, data, should_load_depth) in enumerate(dataarray): + imgpath, kdata, searchindex = map(data.get, ['path', 'kdata', 'searchindex']) + + timestamp, camera_id = searchindex[imgname] + + # for InLoc, SIMPLE_PINHOLE + camera_params = kdata.sensors[camera_id].camera_params + W, H, f, cx, cy = camera_params + distortion = [0, 0, 0, 0] + intrinsics = np.float32([(f, 0, cx), + (0, f, cy), + (0, 0, 1)]) + + if kdata.trajectories is not None and (timestamp, camera_id) in kdata.trajectories: + cam_to_world = cam_to_world_from_kapture(kdata, timestamp, camera_id) + else: + cam_to_world = np.eye(4, dtype=np.float32) + + # Load RGB image + rgb_image = PIL.Image.open(os.path.join(imgpath, 'sensors/records_data', imgname)).convert('RGB') + rgb_image.load() + + W, H = rgb_image.size + resize_func, to_resize, to_orig = get_resize_function(self.maxdim, self.patch_size, H, W) + + rgb_tensor = resize_func(ImgNorm(rgb_image)) + + view = { + 'intrinsics': intrinsics, + 'distortion': distortion, + 'cam_to_world': cam_to_world, + 'rgb': rgb_image, + 'rgb_rescaled': rgb_tensor, + 'to_orig': to_orig, + 'idx': idx, + 'image_name': imgname + } + + # Load depthmap + if should_load_depth: + depthmap_filename = os.path.join(imgpath, 'sensors/records_data', imgname + '.mat') + depthmap = scipy.io.loadmat(depthmap_filename) + + pt3d_cut = depthmap['XYZcut'] + scene_id = imgname.replace('\\', '/').split('/')[1] + if imgname.startswith('DUC1'): + pts3d_full = geotrf(self.aligns_DUC1[scene_id], pt3d_cut) + else: + pts3d_full = geotrf(self.aligns_DUC2[scene_id], pt3d_cut) + + pts3d_valid = np.isfinite(pts3d_full.sum(axis=-1)) + + pts3d = pts3d_full[pts3d_valid] + pts2d_int = xy_grid(W, H)[pts3d_valid] + pts2d = pts2d_int.astype(np.float64) + + # nan => invalid + pts3d_full[~pts3d_valid] = np.nan + pts3d_full = torch.from_numpy(pts3d_full) + view['pts3d'] = pts3d_full + view["valid"] = pts3d_full.sum(dim=-1).isfinite() + + HR, WR = rgb_tensor.shape[1:] + _, _, pts3d_rescaled, valid_rescaled = rescale_points3d(pts2d, pts3d, to_resize, HR, WR) + pts3d_rescaled = torch.from_numpy(pts3d_rescaled) + valid_rescaled = torch.from_numpy(valid_rescaled) + view['pts3d_rescaled'] = pts3d_rescaled + view["valid_rescaled"] = valid_rescaled + views.append(view) + return views diff --git a/src/mast3r_src/dust3r/dust3r_visloc/datasets/sevenscenes.py b/src/mast3r_src/dust3r/dust3r_visloc/datasets/sevenscenes.py new file mode 100644 index 0000000000000000000000000000000000000000..c15e851d262f0d7ba7071c933d8fe8f0a6b1c49d --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r_visloc/datasets/sevenscenes.py @@ -0,0 +1,123 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# 7 Scenes dataloader +# -------------------------------------------------------- +import os +import numpy as np +import torch +import PIL.Image + +import kapture +from kapture.io.csv import kapture_from_dir +from kapture_localization.utils.pairsfile import get_ordered_pairs_from_file +from kapture.io.records import depth_map_from_file + +from dust3r_visloc.datasets.utils import cam_to_world_from_kapture, get_resize_function, rescale_points3d +from dust3r_visloc.datasets.base_dataset import BaseVislocDataset +from dust3r.datasets.utils.transforms import ImgNorm +from dust3r.utils.geometry import depthmap_to_absolute_camera_coordinates, xy_grid, geotrf + + +class VislocSevenScenes(BaseVislocDataset): + def __init__(self, root, subscene, pairsfile, topk=1): + super().__init__() + self.root = root + self.subscene = subscene + self.topk = topk + self.num_views = self.topk + 1 + self.maxdim = None + self.patch_size = None + + query_path = os.path.join(self.root, subscene, 'query') + kdata_query = kapture_from_dir(query_path) + assert kdata_query.records_camera is not None and kdata_query.trajectories is not None and kdata_query.rigs is not None + kapture.rigs_remove_inplace(kdata_query.trajectories, kdata_query.rigs) + kdata_query_searchindex = {kdata_query.records_camera[(timestamp, sensor_id)]: (timestamp, sensor_id) + for timestamp, sensor_id in kdata_query.records_camera.key_pairs()} + self.query_data = {'path': query_path, 'kdata': kdata_query, 'searchindex': kdata_query_searchindex} + + map_path = os.path.join(self.root, subscene, 'mapping') + kdata_map = kapture_from_dir(map_path) + assert kdata_map.records_camera is not None and kdata_map.trajectories is not None and kdata_map.rigs is not None + kapture.rigs_remove_inplace(kdata_map.trajectories, kdata_map.rigs) + kdata_map_searchindex = {kdata_map.records_camera[(timestamp, sensor_id)]: (timestamp, sensor_id) + for timestamp, sensor_id in kdata_map.records_camera.key_pairs()} + self.map_data = {'path': map_path, 'kdata': kdata_map, 'searchindex': kdata_map_searchindex} + + self.pairs = get_ordered_pairs_from_file(os.path.join(self.root, subscene, + 'pairfiles/query', + pairsfile + '.txt')) + self.scenes = kdata_query.records_camera.data_list() + + def __len__(self): + return len(self.scenes) + + def __getitem__(self, idx): + assert self.maxdim is not None and self.patch_size is not None + query_image = self.scenes[idx] + map_images = [p[0] for p in self.pairs[query_image][:self.topk]] + views = [] + dataarray = [(query_image, self.query_data, False)] + [(map_image, self.map_data, True) + for map_image in map_images] + for idx, (imgname, data, should_load_depth) in enumerate(dataarray): + imgpath, kdata, searchindex = map(data.get, ['path', 'kdata', 'searchindex']) + + timestamp, camera_id = searchindex[imgname] + + # for 7scenes, SIMPLE_PINHOLE + camera_params = kdata.sensors[camera_id].camera_params + W, H, f, cx, cy = camera_params + distortion = [0, 0, 0, 0] + intrinsics = np.float32([(f, 0, cx), + (0, f, cy), + (0, 0, 1)]) + + cam_to_world = cam_to_world_from_kapture(kdata, timestamp, camera_id) + + # Load RGB image + rgb_image = PIL.Image.open(os.path.join(imgpath, 'sensors/records_data', imgname)).convert('RGB') + rgb_image.load() + + W, H = rgb_image.size + resize_func, to_resize, to_orig = get_resize_function(self.maxdim, self.patch_size, H, W) + + rgb_tensor = resize_func(ImgNorm(rgb_image)) + + view = { + 'intrinsics': intrinsics, + 'distortion': distortion, + 'cam_to_world': cam_to_world, + 'rgb': rgb_image, + 'rgb_rescaled': rgb_tensor, + 'to_orig': to_orig, + 'idx': idx, + 'image_name': imgname + } + + # Load depthmap + if should_load_depth: + depthmap_filename = os.path.join(imgpath, 'sensors/records_data', + imgname.replace('color.png', 'depth.reg')) + depthmap = depth_map_from_file(depthmap_filename, (int(W), int(H))).astype(np.float32) + pts3d_full, pts3d_valid = depthmap_to_absolute_camera_coordinates(depthmap, intrinsics, cam_to_world) + + pts3d = pts3d_full[pts3d_valid] + pts2d_int = xy_grid(W, H)[pts3d_valid] + pts2d = pts2d_int.astype(np.float64) + + # nan => invalid + pts3d_full[~pts3d_valid] = np.nan + pts3d_full = torch.from_numpy(pts3d_full) + view['pts3d'] = pts3d_full + view["valid"] = pts3d_full.sum(dim=-1).isfinite() + + HR, WR = rgb_tensor.shape[1:] + _, _, pts3d_rescaled, valid_rescaled = rescale_points3d(pts2d, pts3d, to_resize, HR, WR) + pts3d_rescaled = torch.from_numpy(pts3d_rescaled) + valid_rescaled = torch.from_numpy(valid_rescaled) + view['pts3d_rescaled'] = pts3d_rescaled + view["valid_rescaled"] = valid_rescaled + views.append(view) + return views diff --git a/src/mast3r_src/dust3r/dust3r_visloc/datasets/utils.py b/src/mast3r_src/dust3r/dust3r_visloc/datasets/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6053ae2e5ba6c0b0f5f014161b666623d6e0f3f5 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r_visloc/datasets/utils.py @@ -0,0 +1,118 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# dataset utilities +# -------------------------------------------------------- +import numpy as np +import quaternion +import torchvision.transforms as tvf +from dust3r.utils.geometry import geotrf + + +def cam_to_world_from_kapture(kdata, timestamp, camera_id): + camera_to_world = kdata.trajectories[timestamp, camera_id].inverse() + camera_pose = np.eye(4, dtype=np.float32) + camera_pose[:3, :3] = quaternion.as_rotation_matrix(camera_to_world.r) + camera_pose[:3, 3] = camera_to_world.t_raw + return camera_pose + + +ratios_resolutions = { + 224: {1.0: [224, 224]}, + 512: {4 / 3: [512, 384], 32 / 21: [512, 336], 16 / 9: [512, 288], 2 / 1: [512, 256], 16 / 5: [512, 160]} +} + + +def get_HW_resolution(H, W, maxdim, patchsize=16): + assert maxdim in ratios_resolutions, "Error, maxdim can only be 224 or 512 for now. Other maxdims not implemented yet." + ratios_resolutions_maxdim = ratios_resolutions[maxdim] + mindims = set([min(res) for res in ratios_resolutions_maxdim.values()]) + ratio = W / H + ref_ratios = np.array([*(ratios_resolutions_maxdim.keys())]) + islandscape = (W >= H) + if islandscape: + diff = np.abs(ratio - ref_ratios) + else: + diff = np.abs(ratio - (1 / ref_ratios)) + selkey = ref_ratios[np.argmin(diff)] + res = ratios_resolutions_maxdim[selkey] + # check patchsize and make sure output resolution is a multiple of patchsize + if isinstance(patchsize, tuple): + assert len(patchsize) == 2 and isinstance(patchsize[0], int) and isinstance( + patchsize[1], int), "What is your patchsize format? Expected a single int or a tuple of two ints." + assert patchsize[0] == patchsize[1], "Error, non square patches not managed" + patchsize = patchsize[0] + assert max(res) == maxdim + assert min(res) in mindims + return res[::-1] if islandscape else res # return HW + + +def get_resize_function(maxdim, patch_size, H, W, is_mask=False): + if [max(H, W), min(H, W)] in ratios_resolutions[maxdim].values(): + return lambda x: x, np.eye(3), np.eye(3) + else: + target_HW = get_HW_resolution(H, W, maxdim=maxdim, patchsize=patch_size) + + ratio = W / H + target_ratio = target_HW[1] / target_HW[0] + to_orig_crop = np.eye(3) + to_rescaled_crop = np.eye(3) + if abs(ratio - target_ratio) < np.finfo(np.float32).eps: + crop_W = W + crop_H = H + elif ratio - target_ratio < 0: + crop_W = W + crop_H = int(W / target_ratio) + to_orig_crop[1, 2] = (H - crop_H) / 2.0 + to_rescaled_crop[1, 2] = -(H - crop_H) / 2.0 + else: + crop_W = int(H * target_ratio) + crop_H = H + to_orig_crop[0, 2] = (W - crop_W) / 2.0 + to_rescaled_crop[0, 2] = - (W - crop_W) / 2.0 + + crop_op = tvf.CenterCrop([crop_H, crop_W]) + + if is_mask: + resize_op = tvf.Resize(size=target_HW, interpolation=tvf.InterpolationMode.NEAREST_EXACT) + else: + resize_op = tvf.Resize(size=target_HW) + to_orig_resize = np.array([[crop_W / target_HW[1], 0, 0], + [0, crop_H / target_HW[0], 0], + [0, 0, 1]]) + to_rescaled_resize = np.array([[target_HW[1] / crop_W, 0, 0], + [0, target_HW[0] / crop_H, 0], + [0, 0, 1]]) + + op = tvf.Compose([crop_op, resize_op]) + + return op, to_rescaled_resize @ to_rescaled_crop, to_orig_crop @ to_orig_resize + + +def rescale_points3d(pts2d, pts3d, to_resize, HR, WR): + # rescale pts2d as floats + # to colmap, so that the image is in [0, D] -> [0, NewD] + pts2d = pts2d.copy() + pts2d[:, 0] += 0.5 + pts2d[:, 1] += 0.5 + + pts2d_rescaled = geotrf(to_resize, pts2d, norm=True) + + pts2d_rescaled_int = pts2d_rescaled.copy() + # convert back to cv2 before round [-0.5, 0.5] -> pixel 0 + pts2d_rescaled_int[:, 0] -= 0.5 + pts2d_rescaled_int[:, 1] -= 0.5 + pts2d_rescaled_int = pts2d_rescaled_int.round().astype(np.int64) + + # update valid (remove cropped regions) + valid_rescaled = (pts2d_rescaled_int[:, 0] >= 0) & (pts2d_rescaled_int[:, 0] < WR) & ( + pts2d_rescaled_int[:, 1] >= 0) & (pts2d_rescaled_int[:, 1] < HR) + + pts2d_rescaled_int = pts2d_rescaled_int[valid_rescaled] + + # rebuild pts3d from rescaled ps2d poses + pts3d_rescaled = np.full((HR, WR, 3), np.nan, dtype=np.float32) # pts3d in 512 x something + pts3d_rescaled[pts2d_rescaled_int[:, 1], pts2d_rescaled_int[:, 0]] = pts3d[valid_rescaled] + + return pts2d_rescaled, pts2d_rescaled_int, pts3d_rescaled, np.isfinite(pts3d_rescaled.sum(axis=-1)) diff --git a/src/mast3r_src/dust3r/dust3r_visloc/evaluation.py b/src/mast3r_src/dust3r/dust3r_visloc/evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..027179f2b1007db558f57d3d67f48a6d7aa1ab9d --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r_visloc/evaluation.py @@ -0,0 +1,65 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# evaluation utilities +# -------------------------------------------------------- +import numpy as np +import quaternion +import torch +import roma +import collections +import os + + +def aggregate_stats(info_str, pose_errors, angular_errors): + stats = collections.Counter() + median_pos_error = np.median(pose_errors) + median_angular_error = np.median(angular_errors) + out_str = f'{info_str}: {len(pose_errors)} images - {median_pos_error=}, {median_angular_error=}' + + for trl_thr, ang_thr in [(0.1, 1), (0.25, 2), (0.5, 5), (5, 10)]: + for pose_error, angular_error in zip(pose_errors, angular_errors): + correct_for_this_threshold = (pose_error < trl_thr) and (angular_error < ang_thr) + stats[trl_thr, ang_thr] += correct_for_this_threshold + stats = {f'acc@{key[0]:g}m,{key[1]}deg': 100 * val / len(pose_errors) for key, val in stats.items()} + for metric, perf in stats.items(): + out_str += f' - {metric:12s}={float(perf):.3f}' + return out_str + + +def get_pose_error(pr_camtoworld, gt_cam_to_world): + abs_transl_error = torch.linalg.norm(torch.tensor(pr_camtoworld[:3, 3]) - torch.tensor(gt_cam_to_world[:3, 3])) + abs_angular_error = roma.rotmat_geodesic_distance(torch.tensor(pr_camtoworld[:3, :3]), + torch.tensor(gt_cam_to_world[:3, :3])) * 180 / np.pi + return abs_transl_error, abs_angular_error + + +def export_results(output_dir, xp_label, query_names, poses_pred): + if output_dir is not None: + os.makedirs(output_dir, exist_ok=True) + + lines = "" + lines_ltvl = "" + for query_name, pr_querycam_to_world in zip(query_names, poses_pred): + if pr_querycam_to_world is None: + pr_world_to_querycam = np.eye(4) + else: + pr_world_to_querycam = np.linalg.inv(pr_querycam_to_world) + query_shortname = os.path.basename(query_name) + pr_world_to_querycam_q = quaternion.from_rotation_matrix(pr_world_to_querycam[:3, :3]) + pr_world_to_querycam_t = pr_world_to_querycam[:3, 3] + + line_pose = quaternion.as_float_array(pr_world_to_querycam_q).tolist() + \ + pr_world_to_querycam_t.flatten().tolist() + + line_content = [query_name] + line_pose + lines += ' '.join(str(v) for v in line_content) + '\n' + + line_content_ltvl = [query_shortname] + line_pose + lines_ltvl += ' '.join(str(v) for v in line_content_ltvl) + '\n' + + with open(os.path.join(output_dir, xp_label + '_results.txt'), 'wt') as f: + f.write(lines) + with open(os.path.join(output_dir, xp_label + '_ltvl.txt'), 'wt') as f: + f.write(lines_ltvl) diff --git a/src/mast3r_src/dust3r/dust3r_visloc/localization.py b/src/mast3r_src/dust3r/dust3r_visloc/localization.py new file mode 100644 index 0000000000000000000000000000000000000000..ac8ae198dc3479f12a976bab0bda692328880710 --- /dev/null +++ b/src/mast3r_src/dust3r/dust3r_visloc/localization.py @@ -0,0 +1,140 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# main pnp code +# -------------------------------------------------------- +import numpy as np +import quaternion +import cv2 +from packaging import version + +from dust3r.utils.geometry import opencv_to_colmap_intrinsics + +try: + import poselib # noqa + HAS_POSELIB = True +except Exception as e: + HAS_POSELIB = False + +try: + import pycolmap # noqa + version_number = pycolmap.__version__ + if version.parse(version_number) < version.parse("0.5.0"): + HAS_PYCOLMAP = False + else: + HAS_PYCOLMAP = True +except Exception as e: + HAS_PYCOLMAP = False + +def run_pnp(pts2D, pts3D, K, distortion = None, mode='cv2', reprojectionError=5, img_size = None): + """ + use OPENCV model for distortion (4 values) + """ + assert mode in ['cv2', 'poselib', 'pycolmap'] + try: + if len(pts2D) > 4 and mode == "cv2": + confidence = 0.9999 + iterationsCount = 10_000 + if distortion is not None: + cv2_pts2ds = np.copy(pts2D) + cv2_pts2ds = cv2.undistortPoints(cv2_pts2ds, K, np.array(distortion), R=None, P=K) + pts2D = cv2_pts2ds.reshape((-1, 2)) + + success, r_pose, t_pose, _ = cv2.solvePnPRansac(pts3D, pts2D, K, None, flags=cv2.SOLVEPNP_SQPNP, + iterationsCount=iterationsCount, + reprojectionError=reprojectionError, + confidence=confidence) + if not success: + return False, None + r_pose = cv2.Rodrigues(r_pose)[0] # world2cam == world2cam2 + RT = np.r_[np.c_[r_pose, t_pose], [(0,0,0,1)]] # world2cam2 + return True, np.linalg.inv(RT) # cam2toworld + elif len(pts2D) > 4 and mode == "poselib": + assert HAS_POSELIB + confidence = 0.9999 + iterationsCount = 10_000 + # NOTE: `Camera` struct currently contains `width`/`height` fields, + # however these are not used anywhere in the code-base and are provided simply to be consistent with COLMAP. + # so we put garbage in there + colmap_intrinsics = opencv_to_colmap_intrinsics(K) + fx = colmap_intrinsics[0, 0] + fy = colmap_intrinsics[1, 1] + cx = colmap_intrinsics[0, 2] + cy = colmap_intrinsics[1, 2] + width = img_size[0] if img_size is not None else int(cx*2) + height = img_size[1] if img_size is not None else int(cy*2) + + if distortion is None: + camera = {'model': 'PINHOLE', 'width': width, 'height': height, 'params': [fx, fy, cx, cy]} + else: + camera = {'model': 'OPENCV', 'width': width, 'height': height, + 'params': [fx, fy, cx, cy] + distortion} + + pts2D = np.copy(pts2D) + pts2D[:, 0] += 0.5 + pts2D[:, 1] += 0.5 + pose, _ = poselib.estimate_absolute_pose(pts2D, pts3D, camera, + {'max_reproj_error': reprojectionError, + 'max_iterations': iterationsCount, + 'success_prob': confidence}, {}) + if pose is None: + return False, None + RT = pose.Rt # (3x4) + RT = np.r_[RT, [(0,0,0,1)]] # world2cam + return True, np.linalg.inv(RT) # cam2toworld + elif len(pts2D) > 4 and mode == "pycolmap": + assert HAS_PYCOLMAP + assert img_size is not None + + pts2D = np.copy(pts2D) + pts2D[:, 0] += 0.5 + pts2D[:, 1] += 0.5 + colmap_intrinsics = opencv_to_colmap_intrinsics(K) + fx = colmap_intrinsics[0, 0] + fy = colmap_intrinsics[1, 1] + cx = colmap_intrinsics[0, 2] + cy = colmap_intrinsics[1, 2] + width = img_size[0] + height = img_size[1] + if distortion is None: + camera_dict = {'model': 'PINHOLE', 'width': width, 'height': height, 'params': [fx, fy, cx, cy]} + else: + camera_dict = {'model': 'OPENCV', 'width': width, 'height': height, + 'params': [fx, fy, cx, cy] + distortion} + + pycolmap_camera = pycolmap.Camera( + model=camera_dict['model'], width=camera_dict['width'], height=camera_dict['height'], + params=camera_dict['params']) + + pycolmap_estimation_options = dict(ransac=dict(max_error=reprojectionError, min_inlier_ratio=0.01, + min_num_trials=1000, max_num_trials=100000, + confidence=0.9999)) + pycolmap_refinement_options=dict(refine_focal_length=False, refine_extra_params=False) + ret = pycolmap.absolute_pose_estimation(pts2D, pts3D, pycolmap_camera, + estimation_options=pycolmap_estimation_options, + refinement_options=pycolmap_refinement_options) + if ret is None: + ret = {'success': False} + else: + ret['success'] = True + if callable(ret['cam_from_world'].matrix): + retmat = ret['cam_from_world'].matrix() + else: + retmat = ret['cam_from_world'].matrix + ret['qvec'] = quaternion.from_rotation_matrix(retmat[:3, :3]) + ret['tvec'] = retmat[:3, 3] + + if not (ret['success'] and ret['num_inliers'] > 0): + success = False + pose = None + else: + success = True + pr_world_to_querycam = np.r_[ret['cam_from_world'].matrix(), [(0,0,0,1)]] + pose = np.linalg.inv(pr_world_to_querycam) + return success, pose + else: + return False, None + except Exception as e: + print(f'error during pnp: {e}') + return False, None \ No newline at end of file diff --git a/src/mast3r_src/dust3r/requirements.txt b/src/mast3r_src/dust3r/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..d2bf20ed439b43b0604f12985288d8b8d6b55f8f --- /dev/null +++ b/src/mast3r_src/dust3r/requirements.txt @@ -0,0 +1,13 @@ +torch +torchvision +roma +gradio +matplotlib +tqdm +opencv-python +scipy +einops +trimesh +tensorboard +pyglet<2 +huggingface-hub[torch]>=0.22 \ No newline at end of file diff --git a/src/mast3r_src/dust3r/requirements_optional.txt b/src/mast3r_src/dust3r/requirements_optional.txt new file mode 100644 index 0000000000000000000000000000000000000000..d42662c0e87c6ce4ac990f2afedecc96cdea7f06 --- /dev/null +++ b/src/mast3r_src/dust3r/requirements_optional.txt @@ -0,0 +1,7 @@ +pillow-heif # add heif/heic image support +pyrender # for rendering depths in scannetpp +kapture # for visloc data loading +kapture-localization +numpy-quaternion +pycolmap # for pnp +poselib # for pnp diff --git a/src/mast3r_src/dust3r/train.py b/src/mast3r_src/dust3r/train.py new file mode 100644 index 0000000000000000000000000000000000000000..503e63572376c259e6b259850e19c3f6036aa535 --- /dev/null +++ b/src/mast3r_src/dust3r/train.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# training executable for DUSt3R +# -------------------------------------------------------- +from dust3r.training import get_args_parser, train + +if __name__ == '__main__': + args = get_args_parser() + args = args.parse_args() + train(args) diff --git a/src/mast3r_src/dust3r/visloc.py b/src/mast3r_src/dust3r/visloc.py new file mode 100644 index 0000000000000000000000000000000000000000..6411b3eaf96dea961f9524e887a12d92f2012c6b --- /dev/null +++ b/src/mast3r_src/dust3r/visloc.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Simple visloc script +# -------------------------------------------------------- +import numpy as np +import random +import argparse +from tqdm import tqdm +import math + +from dust3r.inference import inference +from dust3r.model import AsymmetricCroCo3DStereo +from dust3r.utils.geometry import find_reciprocal_matches, xy_grid, geotrf + +from dust3r_visloc.datasets import * +from dust3r_visloc.localization import run_pnp +from dust3r_visloc.evaluation import get_pose_error, aggregate_stats, export_results + + +def get_args_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", type=str, required=True, help="visloc dataset to eval") + parser_weights = parser.add_mutually_exclusive_group(required=True) + parser_weights.add_argument("--weights", type=str, help="path to the model weights", default=None) + parser_weights.add_argument("--model_name", type=str, help="name of the model weights", + choices=["DUSt3R_ViTLarge_BaseDecoder_512_dpt", + "DUSt3R_ViTLarge_BaseDecoder_512_linear", + "DUSt3R_ViTLarge_BaseDecoder_224_linear"]) + parser.add_argument("--confidence_threshold", type=float, default=3.0, + help="confidence values higher than threshold are invalid") + parser.add_argument("--device", type=str, default='cuda', help="pytorch device") + parser.add_argument("--pnp_mode", type=str, default="cv2", choices=['cv2', 'poselib', 'pycolmap'], + help="pnp lib to use") + parser_reproj = parser.add_mutually_exclusive_group() + parser_reproj.add_argument("--reprojection_error", type=float, default=5.0, help="pnp reprojection error") + parser_reproj.add_argument("--reprojection_error_diag_ratio", type=float, default=None, + help="pnp reprojection error as a ratio of the diagonal of the image") + + parser.add_argument("--pnp_max_points", type=int, default=100_000, help="pnp maximum number of points kept") + parser.add_argument("--viz_matches", type=int, default=0, help="debug matches") + + parser.add_argument("--output_dir", type=str, default=None, help="output path") + parser.add_argument("--output_label", type=str, default='', help="prefix for results files") + return parser + + +if __name__ == '__main__': + parser = get_args_parser() + args = parser.parse_args() + conf_thr = args.confidence_threshold + device = args.device + pnp_mode = args.pnp_mode + reprojection_error = args.reprojection_error + reprojection_error_diag_ratio = args.reprojection_error_diag_ratio + pnp_max_points = args.pnp_max_points + viz_matches = args.viz_matches + + if args.weights is not None: + weights_path = args.weights + else: + weights_path = "naver/" + args.model_name + model = AsymmetricCroCo3DStereo.from_pretrained(weights_path).to(args.device) + + dataset = eval(args.dataset) + dataset.set_resolution(model) + + query_names = [] + poses_pred = [] + pose_errors = [] + angular_errors = [] + for idx in tqdm(range(len(dataset))): + views = dataset[(idx)] # 0 is the query + query_view = views[0] + map_views = views[1:] + query_names.append(query_view['image_name']) + + query_pts2d = [] + query_pts3d = [] + for map_view in map_views: + # prepare batch + imgs = [] + for idx, img in enumerate([query_view['rgb_rescaled'], map_view['rgb_rescaled']]): + imgs.append(dict(img=img.unsqueeze(0), true_shape=np.int32([img.shape[1:]]), + idx=idx, instance=str(idx))) + output = inference([tuple(imgs)], model, device, batch_size=1, verbose=False) + pred1, pred2 = output['pred1'], output['pred2'] + confidence_masks = [pred1['conf'].squeeze(0) >= conf_thr, + (pred2['conf'].squeeze(0) >= conf_thr) & map_view['valid_rescaled']] + pts3d = [pred1['pts3d'].squeeze(0), pred2['pts3d_in_other_view'].squeeze(0)] + + # find 2D-2D matches between the two images + pts2d_list, pts3d_list = [], [] + for i in range(2): + conf_i = confidence_masks[i].cpu().numpy() + true_shape_i = imgs[i]['true_shape'][0] + pts2d_list.append(xy_grid(true_shape_i[1], true_shape_i[0])[conf_i]) + pts3d_list.append(pts3d[i].detach().cpu().numpy()[conf_i]) + + PQ, PM = pts3d_list[0], pts3d_list[1] + if len(PQ) == 0 or len(PM) == 0: + continue + reciprocal_in_PM, nnM_in_PQ, num_matches = find_reciprocal_matches(PQ, PM) + if viz_matches > 0: + print(f'found {num_matches} matches') + matches_im1 = pts2d_list[1][reciprocal_in_PM] + matches_im0 = pts2d_list[0][nnM_in_PQ][reciprocal_in_PM] + valid_pts3d = map_view['pts3d_rescaled'][matches_im1[:, 1], matches_im1[:, 0]] + + # from cv2 to colmap + matches_im0 = matches_im0.astype(np.float64) + matches_im1 = matches_im1.astype(np.float64) + matches_im0[:, 0] += 0.5 + matches_im0[:, 1] += 0.5 + matches_im1[:, 0] += 0.5 + matches_im1[:, 1] += 0.5 + # rescale coordinates + matches_im0 = geotrf(query_view['to_orig'], matches_im0, norm=True) + matches_im1 = geotrf(query_view['to_orig'], matches_im1, norm=True) + # from colmap back to cv2 + matches_im0[:, 0] -= 0.5 + matches_im0[:, 1] -= 0.5 + matches_im1[:, 0] -= 0.5 + matches_im1[:, 1] -= 0.5 + + # visualize a few matches + if viz_matches > 0: + viz_imgs = [np.array(query_view['rgb']), np.array(map_view['rgb'])] + from matplotlib import pyplot as pl + n_viz = viz_matches + match_idx_to_viz = np.round(np.linspace(0, num_matches - 1, n_viz)).astype(int) + viz_matches_im0, viz_matches_im1 = matches_im0[match_idx_to_viz], matches_im1[match_idx_to_viz] + + H0, W0, H1, W1 = *viz_imgs[0].shape[:2], *viz_imgs[1].shape[:2] + img0 = np.pad(viz_imgs[0], ((0, max(H1 - H0, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img1 = np.pad(viz_imgs[1], ((0, max(H0 - H1, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img = np.concatenate((img0, img1), axis=1) + pl.figure() + pl.imshow(img) + cmap = pl.get_cmap('jet') + for i in range(n_viz): + (x0, y0), (x1, y1) = viz_matches_im0[i].T, viz_matches_im1[i].T + pl.plot([x0, x1 + W0], [y0, y1], '-+', color=cmap(i / (n_viz - 1)), scalex=False, scaley=False) + pl.show(block=True) + + if len(valid_pts3d) == 0: + pass + else: + query_pts3d.append(valid_pts3d.cpu().numpy()) + query_pts2d.append(matches_im0) + + if len(query_pts2d) == 0: + success = False + pr_querycam_to_world = None + else: + query_pts2d = np.concatenate(query_pts2d, axis=0).astype(np.float32) + query_pts3d = np.concatenate(query_pts3d, axis=0) + if len(query_pts2d) > pnp_max_points: + idxs = random.sample(range(len(query_pts2d)), pnp_max_points) + query_pts3d = query_pts3d[idxs] + query_pts2d = query_pts2d[idxs] + + W, H = query_view['rgb'].size + if reprojection_error_diag_ratio is not None: + reprojection_error_img = reprojection_error_diag_ratio * math.sqrt(W**2 + H**2) + else: + reprojection_error_img = reprojection_error + success, pr_querycam_to_world = run_pnp(query_pts2d, query_pts3d, + query_view['intrinsics'], query_view['distortion'], + pnp_mode, reprojection_error_img, img_size=[W, H]) + + if not success: + abs_transl_error = float('inf') + abs_angular_error = float('inf') + else: + abs_transl_error, abs_angular_error = get_pose_error(pr_querycam_to_world, query_view['cam_to_world']) + + pose_errors.append(abs_transl_error) + angular_errors.append(abs_angular_error) + poses_pred.append(pr_querycam_to_world) + + xp_label = f'tol_conf_{conf_thr}' + if args.output_label: + xp_label = args.output_label + '_' + xp_label + if reprojection_error_diag_ratio is not None: + xp_label = xp_label + f'_reproj_diag_{reprojection_error_diag_ratio}' + else: + xp_label = xp_label + f'_reproj_err_{reprojection_error}' + export_results(args.output_dir, xp_label, query_names, poses_pred) + out_string = aggregate_stats(f'{args.dataset}', pose_errors, angular_errors) + print(out_string) diff --git a/src/mast3r_src/mast3r/__init__.py b/src/mast3r_src/mast3r/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d7dd877d649ce4dbd749dd7195a8b34c0f91d4f0 --- /dev/null +++ b/src/mast3r_src/mast3r/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). \ No newline at end of file diff --git a/src/mast3r_src/mast3r/catmlp_dpt_head.py b/src/mast3r_src/mast3r/catmlp_dpt_head.py new file mode 100644 index 0000000000000000000000000000000000000000..9ad780559c5676b2e6caa2487a1aa12af631a51d --- /dev/null +++ b/src/mast3r_src/mast3r/catmlp_dpt_head.py @@ -0,0 +1,327 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# MASt3R heads +# -------------------------------------------------------- +import torch +import torch.nn.functional as F +from einops import rearrange + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.heads.postprocess import reg_dense_depth, reg_dense_conf # noqa +from dust3r.heads.dpt_head import PixelwiseTaskWithDPT # noqa +import dust3r.utils.path_to_croco # noqa +from models.blocks import Mlp # noqa + + +def reg_desc(desc, mode): + if 'norm' in mode: + desc = desc / desc.norm(dim=-1, keepdim=True) + else: + raise ValueError(f"Unknown desc mode {mode}") + return desc + + +def postprocess(out, depth_mode, conf_mode, desc_dim=None, desc_mode='norm', two_confs=False, desc_conf_mode=None): + if desc_conf_mode is None: + desc_conf_mode = conf_mode + fmap = out.permute(0, 2, 3, 1) # B,H,W,D + res = dict(pts3d=reg_dense_depth(fmap[..., 0:3], mode=depth_mode)) + if conf_mode is not None: + res['conf'] = reg_dense_conf(fmap[..., 3], mode=conf_mode) + if desc_dim is not None: + start = 3 + int(conf_mode is not None) + res['desc'] = reg_desc(fmap[..., start:start + desc_dim], mode=desc_mode) + if two_confs: + res['desc_conf'] = reg_dense_conf(fmap[..., start + desc_dim], mode=desc_conf_mode) + else: + res['desc_conf'] = res['conf'].clone() + return res + + +class Cat_MLP_LocalFeatures_DPT_Pts3d(PixelwiseTaskWithDPT): + """ Mixture between MLP and DPT head that outputs 3d points and local features (with MLP). + The input for both heads is a concatenation of Encoder and Decoder outputs + """ + + def __init__(self, net, has_conf=False, local_feat_dim=16, hidden_dim_factor=4., hooks_idx=None, dim_tokens=None, + num_channels=1, postprocess=None, feature_dim=256, last_dim=32, depth_mode=None, conf_mode=None, head_type="regression", **kwargs): + super().__init__(num_channels=num_channels, feature_dim=feature_dim, last_dim=last_dim, hooks_idx=hooks_idx, + dim_tokens=dim_tokens, depth_mode=depth_mode, postprocess=postprocess, conf_mode=conf_mode, head_type=head_type) + self.local_feat_dim = local_feat_dim + + patch_size = net.patch_embed.patch_size + if isinstance(patch_size, tuple): + assert len(patch_size) == 2 and isinstance(patch_size[0], int) and isinstance( + patch_size[1], int), "What is your patchsize format? Expected a single int or a tuple of two ints." + assert patch_size[0] == patch_size[1], "Error, non square patches not managed" + patch_size = patch_size[0] + self.patch_size = patch_size + + self.desc_mode = net.desc_mode + self.has_conf = has_conf + self.two_confs = net.two_confs # independent confs for 3D regr and descs + self.desc_conf_mode = net.desc_conf_mode + idim = net.enc_embed_dim + net.dec_embed_dim + + self.head_local_features = Mlp(in_features=idim, + hidden_features=int(hidden_dim_factor * idim), + out_features=(self.local_feat_dim + self.two_confs) * self.patch_size**2) + + def forward(self, decout, img_shape): + # pass through the heads + pts3d = self.dpt(decout, image_size=(img_shape[0], img_shape[1])) + + # recover encoder and decoder outputs + enc_output, dec_output = decout[0], decout[-1] + cat_output = torch.cat([enc_output, dec_output], dim=-1) # concatenate + H, W = img_shape + B, S, D = cat_output.shape + + # extract local_features + local_features = self.head_local_features(cat_output) # B,S,D + local_features = local_features.transpose(-1, -2).view(B, -1, H // self.patch_size, W // self.patch_size) + local_features = F.pixel_shuffle(local_features, self.patch_size) # B,d,H,W + + # post process 3D pts, descriptors and confidences + out = torch.cat([pts3d, local_features], dim=1) + if self.postprocess: + out = self.postprocess(out, + depth_mode=self.depth_mode, + conf_mode=self.conf_mode, + desc_dim=self.local_feat_dim, + desc_mode=self.desc_mode, + two_confs=self.two_confs, + desc_conf_mode=self.desc_conf_mode) + return out + +# @MODIFIED +def reg_dense_offsets(xyz, shift=6.0): + """ + Apply an activation function to the offsets so that they are small at initialization + """ + d = xyz.norm(dim=-1, keepdim=True) + xyz = xyz / d.clip(min=1e-8) + offsets = xyz * (torch.exp(d - shift) - torch.exp(torch.zeros_like(d) - shift)) + return offsets + +# @MODIFIED +def reg_dense_scales(scales): + """ + Apply an activation function to the offsets so that they are small at initialization + """ + scales = scales.exp() + return scales + +# @MODIFIED +def reg_dense_rotation(rotations, eps=1e-8): + """ + Apply PixelSplat's rotation normalization + """ + return rotations / (rotations.norm(dim=-1, keepdim=True) + eps) + +# @MODIFIED +def reg_dense_sh(sh): + """ + Apply PixelSplat's spherical harmonic postprocessing + """ + sh = rearrange(sh, '... (xyz d_sh) -> ... xyz d_sh', xyz=3) + return sh + +# @MODIFIED +def reg_dense_opacities(opacities): + """ + Apply PixelSplat's opacity postprocessing + """ + return opacities.sigmoid() + +# @MODIFIED +def gaussian_postprocess(out, depth_mode, conf_mode, desc_dim=None, desc_mode='norm', two_confs=False, desc_conf_mode=None, use_offsets=False, sh_degree=1): + + if desc_conf_mode is None: + desc_conf_mode = conf_mode + + fmap = out.permute(0, 2, 3, 1) # B,H,W,D + assert conf_mode is not None, "Confidence mode must be provided for Gaussian head" + assert desc_dim is not None, "Descriptor dimension must be provided for Gaussian head" + assert two_confs, "Two confidences must be provided for Gaussian head" + + pts3d, conf, desc, desc_conf, offset, scales, rotations, sh, opacities = torch.split(fmap, [3, 1, desc_dim, 1, 3, 3, 4, 3 * sh_degree, 1], dim=-1) + + pts3d = reg_dense_depth(pts3d, mode=depth_mode) + conf = reg_dense_conf(conf.squeeze(-1), mode=conf_mode) + desc = reg_desc(desc, mode=desc_mode) + desc_conf = reg_dense_conf(desc_conf.squeeze(-1), mode=desc_conf_mode) + offset = reg_dense_offsets(offset) + scales = reg_dense_scales(scales) + rotations = reg_dense_rotation(rotations) + sh = reg_dense_sh(sh) + opacities = reg_dense_opacities(opacities) + + res = { + 'pts3d': pts3d, + 'conf': conf, + 'desc': desc, + 'desc_conf': desc_conf, + 'scales': scales, + 'rotations': rotations, + 'sh': sh, + 'opacities': opacities + } + + if use_offsets: + res['means'] = pts3d.detach() + offset + else: + res['means'] = pts3d.detach() + + return res + + +# @MODIFIED +class GaussianHead(PixelwiseTaskWithDPT): + """Version of the above, modified to also output Gaussian parameters""" + + def __init__(self, net, has_conf=False, local_feat_dim=16, hidden_dim_factor=4., hooks_idx=None, dim_tokens=None, + num_channels=1, postprocess=None, feature_dim=256, last_dim=32, depth_mode=None, conf_mode=None, head_type="regression", use_offsets=False, sh_degree=1, **kwargs): + super().__init__(num_channels=num_channels, feature_dim=feature_dim, last_dim=last_dim, hooks_idx=hooks_idx, + dim_tokens=dim_tokens, depth_mode=depth_mode, postprocess=postprocess, conf_mode=conf_mode, head_type=head_type) + self.local_feat_dim = local_feat_dim + + patch_size = net.patch_embed.patch_size + if isinstance(patch_size, tuple): + assert len(patch_size) == 2 and isinstance(patch_size[0], int) and isinstance( + patch_size[1], int), "What is your patchsize format? Expected a single int or a tuple of two ints." + assert patch_size[0] == patch_size[1], "Error, non square patches not managed" + patch_size = patch_size[0] + self.patch_size = patch_size + + self.desc_mode = net.desc_mode + self.has_conf = has_conf + self.two_confs = net.two_confs # independent confs for 3D regr and descs + self.desc_conf_mode = net.desc_conf_mode + idim = net.enc_embed_dim + net.dec_embed_dim + + self.head_local_features = Mlp(in_features=idim, + hidden_features=int(hidden_dim_factor * idim), + out_features=(self.local_feat_dim + self.two_confs) * self.patch_size**2) + + # Gaussian Num Channels = + # 3D mean offsets (3) + + # Scales (3) + + # Rotations (4) + + # Spherical Harmonics (3 * sh_degree) + + # Opacity (1) + gaussian_num_channels = 3 + 3 + 4 + 3 * sh_degree + 1 + self.gaussian_dpt = PixelwiseTaskWithDPT( + num_channels=gaussian_num_channels, feature_dim=feature_dim, last_dim=last_dim, hooks_idx=hooks_idx, + dim_tokens=dim_tokens, depth_mode=depth_mode, postprocess=postprocess, conf_mode=conf_mode, head_type=head_type + ) + + final_conv_layer = self.gaussian_dpt.dpt.head[-1] + splits_and_inits = [ + (3, 0.001, 0.001), # 3D mean offsets + (3, 0.00003, -7.0), # Scales + (4, 1.0, 0.0), # Rotations + (3 * sh_degree, 1.0, 0.0), # Spherical Harmonics + (1, 1.0, -2.0) # Opacity + ] + start_channels = 0 + for out_channel, s, b in splits_and_inits: + torch.nn.init.xavier_uniform_( + final_conv_layer.weight[start_channels:start_channels+out_channel, :, :, :], + s + ) + torch.nn.init.constant_( + final_conv_layer.bias[start_channels:start_channels+out_channel], + b + ) + start_channels += out_channel + + self.use_offsets = use_offsets + self.sh_degree = sh_degree + + + def forward(self, decout, img_shape): + # pass through the heads + pts3d = self.dpt(decout, image_size=(img_shape[0], img_shape[1])) + + # recover encoder and decoder outputs + enc_output, dec_output = decout[0], decout[-1] + cat_output = torch.cat([enc_output, dec_output], dim=-1) # concatenate + H, W = img_shape + B, S, D = cat_output.shape + + # extract local_features + local_features = self.head_local_features(cat_output) # B,S,D + local_features = local_features.transpose(-1, -2).view(B, -1, H // self.patch_size, W // self.patch_size) + local_features = F.pixel_shuffle(local_features, self.patch_size) # B,d,H,W + + # extract gaussian_features + gaussian_features = self.gaussian_dpt.dpt(decout, image_size=(img_shape[0], img_shape[1])) + # gaussian_features = self.gaussian_local_features(cat_output) # B,S,D + # gaussian_features = gaussian_features.transpose(-1, -2).view(B, -1, H // self.patch_size, W // self.patch_size) + # gaussian_features = F.pixel_shuffle(gaussian_features, self.patch_size) # B,d,H,W + + # post process 3D pts, descriptors and confidences + out = torch.cat([pts3d, local_features, gaussian_features], dim=1) + if self.postprocess: + out = gaussian_postprocess(out, + depth_mode=self.depth_mode, + conf_mode=self.conf_mode, + desc_dim=self.local_feat_dim, + desc_mode=self.desc_mode, + two_confs=self.two_confs, + desc_conf_mode=self.desc_conf_mode, + use_offsets=self.use_offsets, + sh_degree=self.sh_degree) + return out + + +def mast3r_head_factory(head_type, output_mode, net, has_conf=False, use_offsets=False, sh_degree=1): + """" build a prediction head for the decoder + """ + if head_type == 'catmlp+dpt' and output_mode.startswith('pts3d+desc'): + local_feat_dim = int(output_mode[10:]) + assert net.dec_depth > 9 + l2 = net.dec_depth + feature_dim = 256 + last_dim = feature_dim // 2 + out_nchan = 3 + ed = net.enc_embed_dim + dd = net.dec_embed_dim + return Cat_MLP_LocalFeatures_DPT_Pts3d(net, local_feat_dim=local_feat_dim, has_conf=has_conf, + num_channels=out_nchan + has_conf, + feature_dim=feature_dim, + last_dim=last_dim, + hooks_idx=[0, l2 * 2 // 4, l2 * 3 // 4, l2], + dim_tokens=[ed, dd, dd, dd], + postprocess=postprocess, + depth_mode=net.depth_mode, + conf_mode=net.conf_mode, + head_type='regression') + # @MODIFIED + elif head_type == 'gaussian_head' and output_mode.startswith('pts3d+gaussian+desc'): + local_feat_dim = int(output_mode[19:]) + assert net.dec_depth > 9 + l2 = net.dec_depth + feature_dim = 256 + last_dim = feature_dim // 2 + out_nchan = 3 + ed = net.enc_embed_dim + dd = net.dec_embed_dim + return GaussianHead(net, local_feat_dim=local_feat_dim, has_conf=has_conf, + num_channels=out_nchan + has_conf, + feature_dim=feature_dim, + last_dim=last_dim, + hooks_idx=[0, l2 * 2 // 4, l2 * 3 // 4, l2], + dim_tokens=[ed, dd, dd, dd], + postprocess=postprocess, + depth_mode=net.depth_mode, + conf_mode=net.conf_mode, + head_type='regression', + use_offsets=use_offsets, + sh_degree=sh_degree) + else: + raise NotImplementedError( + f"unexpected {head_type=} and {output_mode=}") diff --git a/src/mast3r_src/mast3r/cloud_opt/__init__.py b/src/mast3r_src/mast3r/cloud_opt/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d7dd877d649ce4dbd749dd7195a8b34c0f91d4f0 --- /dev/null +++ b/src/mast3r_src/mast3r/cloud_opt/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). \ No newline at end of file diff --git a/src/mast3r_src/mast3r/cloud_opt/sparse_ga.py b/src/mast3r_src/mast3r/cloud_opt/sparse_ga.py new file mode 100644 index 0000000000000000000000000000000000000000..a54305cbf93bfe7f371db13da47f3de93474cd24 --- /dev/null +++ b/src/mast3r_src/mast3r/cloud_opt/sparse_ga.py @@ -0,0 +1,1047 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# MASt3R Sparse Global Alignement +# -------------------------------------------------------- +from tqdm import tqdm +import roma +import torch +import torch.nn as nn +import torch.nn.functional as F +import numpy as np +import os +from collections import namedtuple +from functools import lru_cache +from scipy import sparse as sp + +from mast3r.utils.misc import mkdir_for, hash_md5 +from mast3r.cloud_opt.utils.losses import gamma_loss +from mast3r.cloud_opt.utils.schedules import linear_schedule, cosine_schedule +from mast3r.fast_nn import fast_reciprocal_NNs, merge_corres + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.utils.geometry import inv, geotrf # noqa +from dust3r.utils.device import to_cpu, to_numpy, todevice # noqa +from dust3r.post_process import estimate_focal_knowing_depth # noqa +from dust3r.optim_factory import adjust_learning_rate_by_lr # noqa +from dust3r.cloud_opt.base_opt import clean_pointcloud +from dust3r.viz import SceneViz + + +class SparseGA(): + def __init__(self, img_paths, pairs_in, res_fine, anchors, canonical_paths=None): + def fetch_img(im): + def torgb(x): return (x[0].permute(1, 2, 0).numpy() * .5 + .5).clip(min=0., max=1.) + for im1, im2 in pairs_in: + if im1['instance'] == im: + return torgb(im1['img']) + if im2['instance'] == im: + return torgb(im2['img']) + self.canonical_paths = canonical_paths + self.img_paths = img_paths + self.imgs = [fetch_img(img) for img in img_paths] + self.intrinsics = res_fine['intrinsics'] + self.cam2w = res_fine['cam2w'] + self.depthmaps = res_fine['depthmaps'] + self.pts3d = res_fine['pts3d'] + self.pts3d_colors = [] + self.working_device = self.cam2w.device + for i in range(len(self.imgs)): + im = self.imgs[i] + x, y = anchors[i][0][..., :2].detach().cpu().numpy().T + self.pts3d_colors.append(im[y, x]) + assert self.pts3d_colors[-1].shape == self.pts3d[i].shape + self.n_imgs = len(self.imgs) + + def get_focals(self): + return torch.tensor([ff[0, 0] for ff in self.intrinsics]).to(self.working_device) + + def get_principal_points(self): + return torch.stack([ff[:2, -1] for ff in self.intrinsics]).to(self.working_device) + + def get_im_poses(self): + return self.cam2w + + def get_sparse_pts3d(self): + return self.pts3d + + def get_dense_pts3d(self, clean_depth=True, subsample=8): + assert self.canonical_paths, 'cache_path is required for dense 3d points' + device = self.cam2w.device + confs = [] + base_focals = [] + anchors = {} + for i, canon_path in enumerate(self.canonical_paths): + (canon, canon2, conf), focal = torch.load(canon_path, map_location=device) + confs.append(conf) + base_focals.append(focal) + + H, W = conf.shape + pixels = torch.from_numpy(np.mgrid[:W, :H].T.reshape(-1, 2)).float().to(device) + idxs, offsets = anchor_depth_offsets(canon2, {i: (pixels, None)}, subsample=subsample) + anchors[i] = (pixels, idxs[i], offsets[i]) + + # densify sparse depthmaps + pts3d, depthmaps = make_pts3d(anchors, self.intrinsics, self.cam2w, [ + d.ravel() for d in self.depthmaps], base_focals=base_focals, ret_depth=True) + + if clean_depth: + confs = clean_pointcloud(confs, self.intrinsics, inv(self.cam2w), depthmaps, pts3d) + + return pts3d, depthmaps, confs + + def get_pts3d_colors(self): + return self.pts3d_colors + + def get_depthmaps(self): + return self.depthmaps + + def get_masks(self): + return [slice(None, None) for _ in range(len(self.imgs))] + + def show(self, show_cams=True): + pts3d, _, confs = self.get_dense_pts3d() + show_reconstruction(self.imgs, self.intrinsics if show_cams else None, self.cam2w, + [p.clip(min=-50, max=50) for p in pts3d], + masks=[c > 1 for c in confs]) + + +def convert_dust3r_pairs_naming(imgs, pairs_in): + for pair_id in range(len(pairs_in)): + for i in range(2): + pairs_in[pair_id][i]['instance'] = imgs[pairs_in[pair_id][i]['idx']] + return pairs_in + + +def sparse_global_alignment(imgs, pairs_in, cache_path, model, subsample=8, desc_conf='desc_conf', + device='cuda', dtype=torch.float32, shared_intrinsics=False, **kw): + """ Sparse alignment with MASt3R + imgs: list of image paths + cache_path: path where to dump temporary files (str) + + lr1, niter1: learning rate and #iterations for coarse global alignment (3D matching) + lr2, niter2: learning rate and #iterations for refinement (2D reproj error) + + lora_depth: smart dimensionality reduction with depthmaps + """ + # Convert pair naming convention from dust3r to mast3r + pairs_in = convert_dust3r_pairs_naming(imgs, pairs_in) + # forward pass + pairs, cache_path = forward_mast3r(pairs_in, model, + cache_path=cache_path, subsample=subsample, + desc_conf=desc_conf, device=device) + + # extract canonical pointmaps + tmp_pairs, pairwise_scores, canonical_views, canonical_paths, preds_21 = \ + prepare_canonical_data(imgs, pairs, subsample, cache_path=cache_path, mode='avg-angle', device=device) + + # compute minimal spanning tree + mst = compute_min_spanning_tree(pairwise_scores) + + # remove all edges not in the spanning tree? + # min_spanning_tree = {(imgs[i],imgs[j]) for i,j in mst[1]} + # tmp_pairs = {(a,b):v for (a,b),v in tmp_pairs.items() if {(a,b),(b,a)} & min_spanning_tree} + + # smartly combine all usefull data + imsizes, pps, base_focals, core_depth, anchors, corres, corres2d, preds_21 = \ + condense_data(imgs, tmp_pairs, canonical_views, preds_21, dtype) + + imgs, res_coarse, res_fine = sparse_scene_optimizer( + imgs, subsample, imsizes, pps, base_focals, core_depth, anchors, corres, corres2d, preds_21, canonical_paths, mst, + shared_intrinsics=shared_intrinsics, cache_path=cache_path, device=device, dtype=dtype, **kw) + + return SparseGA(imgs, pairs_in, res_fine or res_coarse, anchors, canonical_paths) + + +def sparse_scene_optimizer(imgs, subsample, imsizes, pps, base_focals, core_depth, anchors, corres, corres2d, + preds_21, canonical_paths, mst, cache_path, + lr1=0.2, niter1=500, loss1=gamma_loss(1.1), + lr2=0.02, niter2=500, loss2=gamma_loss(0.4), + lossd=gamma_loss(1.1), + opt_pp=True, opt_depth=True, + schedule=cosine_schedule, depth_mode='add', exp_depth=False, + lora_depth=False, # dict(k=96, gamma=15, min_norm=.5), + shared_intrinsics=False, + init={}, device='cuda', dtype=torch.float32, + matching_conf_thr=5., loss_dust3r_w=0.01, + verbose=True, dbg=()): + + # extrinsic parameters + vec0001 = torch.tensor((0, 0, 0, 1), dtype=dtype, device=device) + quats = [nn.Parameter(vec0001.clone()) for _ in range(len(imgs))] + trans = [nn.Parameter(torch.zeros(3, device=device, dtype=dtype)) for _ in range(len(imgs))] + + # intialize + ones = torch.ones((len(imgs), 1), device=device, dtype=dtype) + median_depths = torch.ones(len(imgs), device=device, dtype=dtype) + for img in imgs: + idx = imgs.index(img) + init_values = init.setdefault(img, {}) + if verbose and init_values: + print(f' >> initializing img=...{img[-25:]} [{idx}] for {set(init_values)}') + + K = init_values.get('intrinsics') + if K is not None: + K = K.detach() + focal = K[:2, :2].diag().mean() + pp = K[:2, 2] + base_focals[idx] = focal + pps[idx] = pp + pps[idx] /= imsizes[idx] # default principal_point would be (0.5, 0.5) + + depth = init_values.get('depthmap') + if depth is not None: + core_depth[idx] = depth.detach() + + median_depths[idx] = med_depth = core_depth[idx].median() + core_depth[idx] /= med_depth + + cam2w = init_values.get('cam2w') + if cam2w is not None: + rot = cam2w[:3, :3].detach() + cam_center = cam2w[:3, 3].detach() + quats[idx].data[:] = roma.rotmat_to_unitquat(rot) + trans_offset = med_depth * torch.cat((imsizes[idx] / base_focals[idx] * (0.5 - pps[idx]), ones[:1, 0])) + trans[idx].data[:] = cam_center + rot @ trans_offset + del rot + assert False, 'inverse kinematic chain not yet implemented' + + # intrinsics parameters + if shared_intrinsics: + # Optimize a single set of intrinsics for all cameras. Use averages as init. + confs = torch.stack([torch.load(pth)[0][2].mean() for pth in canonical_paths]).to(pps) + weighting = confs / confs.sum() + pp = nn.Parameter((weighting @ pps).to(dtype)) + pps = [pp for _ in range(len(imgs))] + focal_m = weighting @ base_focals + log_focal = nn.Parameter(focal_m.view(1).log().to(dtype)) + log_focals = [log_focal for _ in range(len(imgs))] + else: + pps = [nn.Parameter(pp.to(dtype)) for pp in pps] + log_focals = [nn.Parameter(f.view(1).log().to(dtype)) for f in base_focals] + + diags = imsizes.float().norm(dim=1) + min_focals = 0.25 * diags # diag = 1.2~1.4*max(W,H) => beta >= 1/(2*1.2*tan(fov/2)) ~= 0.26 + max_focals = 10 * diags + + assert len(mst[1]) == len(pps) - 1 + + def make_K_cam_depth(log_focals, pps, trans, quats, log_sizes, core_depth): + # make intrinsics + focals = torch.cat(log_focals).exp().clip(min=min_focals, max=max_focals) + pps = torch.stack(pps) + K = torch.eye(3, dtype=dtype, device=device)[None].expand(len(imgs), 3, 3).clone() + K[:, 0, 0] = K[:, 1, 1] = focals + K[:, 0:2, 2] = pps * imsizes + if trans is None: + return K + + # security! optimization is always trying to crush the scale down + sizes = torch.cat(log_sizes).exp() + global_scaling = 1 / sizes.min() + + # compute distance of camera to focal plane + # tan(fov) = W/2 / focal + z_cameras = sizes * median_depths * focals / base_focals + + # make extrinsic + rel_cam2cam = torch.eye(4, dtype=dtype, device=device)[None].expand(len(imgs), 4, 4).clone() + rel_cam2cam[:, :3, :3] = roma.unitquat_to_rotmat(F.normalize(torch.stack(quats), dim=1)) + rel_cam2cam[:, :3, 3] = torch.stack(trans) + + # camera are defined as a kinematic chain + tmp_cam2w = [None] * len(K) + tmp_cam2w[mst[0]] = rel_cam2cam[mst[0]] + for i, j in mst[1]: + # i is the cam_i_to_world reference, j is the relative pose = cam_j_to_cam_i + tmp_cam2w[j] = tmp_cam2w[i] @ rel_cam2cam[j] + tmp_cam2w = torch.stack(tmp_cam2w) + + # smart reparameterizaton of cameras + trans_offset = z_cameras.unsqueeze(1) * torch.cat((imsizes / focals.unsqueeze(1) * (0.5 - pps), ones), dim=-1) + new_trans = global_scaling * (tmp_cam2w[:, :3, 3:4] - tmp_cam2w[:, :3, :3] @ trans_offset.unsqueeze(-1)) + cam2w = torch.cat((torch.cat((tmp_cam2w[:, :3, :3], new_trans), dim=2), + vec0001.view(1, 1, 4).expand(len(K), 1, 4)), dim=1) + + depthmaps = [] + for i in range(len(imgs)): + core_depth_img = core_depth[i] + if exp_depth: + core_depth_img = core_depth_img.exp() + if lora_depth: # compute core_depth as a low-rank decomposition of 3d points + core_depth_img = lora_depth_proj[i] @ core_depth_img + if depth_mode == 'add': + core_depth_img = z_cameras[i] + (core_depth_img - 1) * (median_depths[i] * sizes[i]) + elif depth_mode == 'mul': + core_depth_img = z_cameras[i] * core_depth_img + else: + raise ValueError(f'Bad {depth_mode=}') + depthmaps.append(global_scaling * core_depth_img) + + return K, (inv(cam2w), cam2w), depthmaps + + K = make_K_cam_depth(log_focals, pps, None, None, None, None) + + if shared_intrinsics: + print('init focal (shared) = ', to_numpy(K[0, 0, 0]).round(2)) + else: + print('init focals =', to_numpy(K[:, 0, 0])) + + # spectral low-rank projection of depthmaps + if lora_depth: + core_depth, lora_depth_proj = spectral_projection_of_depthmaps( + imgs, K, core_depth, subsample, cache_path=cache_path, **lora_depth) + if exp_depth: + core_depth = [d.clip(min=1e-4).log() for d in core_depth] + core_depth = [nn.Parameter(d.ravel().to(dtype)) for d in core_depth] + log_sizes = [nn.Parameter(torch.zeros(1, dtype=dtype, device=device)) for _ in range(len(imgs))] + + # Fetch img slices + _, confs_sum, imgs_slices = corres + + # Define which pairs are fine to use with matching + def matching_check(x): return x.max() > matching_conf_thr + is_matching_ok = {} + for s in imgs_slices: + is_matching_ok[s.img1, s.img2] = matching_check(s.confs) + + # Subsample preds_21 + subsamp_preds_21 = {} + for imk, imv in preds_21.items(): + subsamp_preds_21[imk] = {} + for im2k, (pred, conf) in preds_21[imk].items(): + idxs = anchors[imgs.index(im2k)][1] + subsamp_preds_21[imk][im2k] = (pred[idxs], conf[idxs]) # anchors subsample + + # Prepare slices and corres for losses + dust3r_slices = [s for s in imgs_slices if not is_matching_ok[s.img1, s.img2]] + loss3d_slices = [s for s in imgs_slices if is_matching_ok[s.img1, s.img2]] + cleaned_corres2d = [] + for cci, (img1, pix1, confs, confsum, imgs_slices) in enumerate(corres2d): + cf_sum = 0 + pix1_filtered = [] + confs_filtered = [] + curstep = 0 + cleaned_slices = [] + for img2, slice2 in imgs_slices: + if is_matching_ok[img1, img2]: + tslice = slice(curstep, curstep + slice2.stop - slice2.start, slice2.step) + pix1_filtered.append(pix1[tslice]) + confs_filtered.append(confs[tslice]) + cleaned_slices.append((img2, slice2)) + curstep += slice2.stop - slice2.start + if pix1_filtered != []: + pix1_filtered = torch.cat(pix1_filtered) + confs_filtered = torch.cat(confs_filtered) + cf_sum = confs_filtered.sum() + cleaned_corres2d.append((img1, pix1_filtered, confs_filtered, cf_sum, cleaned_slices)) + + def loss_dust3r(cam2w, pts3d, pix_loss): + # In the case no correspondence could be established, fallback to DUSt3R GA regression loss formulation (sparsified) + loss = 0. + cf_sum = 0. + for s in dust3r_slices: + if init[imgs[s.img1]].get('freeze') and init[imgs[s.img2]].get('freeze'): + continue + # fallback to dust3r regression + tgt_pts, tgt_confs = preds_21[imgs[s.img2]][imgs[s.img1]] + tgt_pts = geotrf(cam2w[s.img2], tgt_pts) + cf_sum += tgt_confs.sum() + loss += tgt_confs @ pix_loss(pts3d[s.img1], tgt_pts) + return loss / cf_sum if cf_sum != 0. else 0. + + def loss_3d(K, w2cam, pts3d, pix_loss): + # For each correspondence, we have two 3D points (one for each image of the pair). + # For each 3D point, we have 2 reproj errors + if any(v.get('freeze') for v in init.values()): + pts3d_1 = [] + pts3d_2 = [] + confs = [] + for s in loss3d_slices: + if init[imgs[s.img1]].get('freeze') and init[imgs[s.img2]].get('freeze'): + continue + pts3d_1.append(pts3d[s.img1][s.slice1]) + pts3d_2.append(pts3d[s.img2][s.slice2]) + confs.append(s.confs) + else: + pts3d_1 = [pts3d[s.img1][s.slice1] for s in loss3d_slices] + pts3d_2 = [pts3d[s.img2][s.slice2] for s in loss3d_slices] + confs = [s.confs for s in loss3d_slices] + + if pts3d_1 != []: + confs = torch.cat(confs) + pts3d_1 = torch.cat(pts3d_1) + pts3d_2 = torch.cat(pts3d_2) + loss = confs @ pix_loss(pts3d_1, pts3d_2) + cf_sum = confs.sum() + else: + loss = 0. + cf_sum = 1. + + return loss / cf_sum + + def loss_2d(K, w2cam, pts3d, pix_loss): + # For each correspondence, we have two 3D points (one for each image of the pair). + # For each 3D point, we have 2 reproj errors + proj_matrix = K @ w2cam[:, :3] + loss = npix = 0 + for img1, pix1_filtered, confs_filtered, cf_sum, cleaned_slices in cleaned_corres2d: + if init[imgs[img1]].get('freeze', 0) >= 1: + continue # no need + pts3d_in_img1 = [pts3d[img2][slice2] for img2, slice2 in cleaned_slices] + if pts3d_in_img1 != []: + pts3d_in_img1 = torch.cat(pts3d_in_img1) + loss += confs_filtered @ pix_loss(pix1_filtered, reproj2d(proj_matrix[img1], pts3d_in_img1)) + npix += confs_filtered.sum() + + return loss / npix if npix != 0 else 0. + + def optimize_loop(loss_func, lr_base, niter, pix_loss, lr_end=0): + # create optimizer + params = pps + log_focals + quats + trans + log_sizes + core_depth + optimizer = torch.optim.Adam(params, lr=1, weight_decay=0, betas=(0.9, 0.9)) + ploss = pix_loss if 'meta' in repr(pix_loss) else (lambda a: pix_loss) + + with tqdm(total=niter) as bar: + for iter in range(niter or 1): + K, (w2cam, cam2w), depthmaps = make_K_cam_depth(log_focals, pps, trans, quats, log_sizes, core_depth) + pts3d = make_pts3d(anchors, K, cam2w, depthmaps, base_focals=base_focals) + if niter == 0: + break + + alpha = (iter / niter) + lr = schedule(alpha, lr_base, lr_end) + adjust_learning_rate_by_lr(optimizer, lr) + pix_loss = ploss(1 - alpha) + optimizer.zero_grad() + loss = loss_func(K, w2cam, pts3d, pix_loss) + loss_dust3r_w * loss_dust3r(cam2w, pts3d, lossd) + loss.backward() + optimizer.step() + + # make sure the pose remains well optimizable + for i in range(len(imgs)): + quats[i].data[:] /= quats[i].data.norm() + + loss = float(loss) + if loss != loss: + break # NaN loss + bar.set_postfix_str(f'{lr=:.4f}, {loss=:.3f}') + bar.update(1) + + if niter: + print(f'>> final loss = {loss}') + return dict(intrinsics=K.detach(), cam2w=cam2w.detach(), + depthmaps=[d.detach() for d in depthmaps], pts3d=[p.detach() for p in pts3d]) + + # at start, don't optimize 3d points + for i, img in enumerate(imgs): + trainable = not (init[img].get('freeze')) + pps[i].requires_grad_(False) + log_focals[i].requires_grad_(False) + quats[i].requires_grad_(trainable) + trans[i].requires_grad_(trainable) + log_sizes[i].requires_grad_(trainable) + core_depth[i].requires_grad_(False) + + res_coarse = optimize_loop(loss_3d, lr_base=lr1, niter=niter1, pix_loss=loss1) + + res_fine = None + if niter2: + # now we can optimize 3d points + for i, img in enumerate(imgs): + if init[img].get('freeze', 0) >= 1: + continue + pps[i].requires_grad_(bool(opt_pp)) + log_focals[i].requires_grad_(True) + core_depth[i].requires_grad_(opt_depth) + + # refinement with 2d reproj + res_fine = optimize_loop(loss_2d, lr_base=lr2, niter=niter2, pix_loss=loss2) + + K = make_K_cam_depth(log_focals, pps, None, None, None, None) + if shared_intrinsics: + print('Final focal (shared) = ', to_numpy(K[0, 0, 0]).round(2)) + else: + print('Final focals =', to_numpy(K[:, 0, 0])) + + return imgs, res_coarse, res_fine + + +@lru_cache +def mask110(device, dtype): + return torch.tensor((1, 1, 0), device=device, dtype=dtype) + + +def proj3d(inv_K, pixels, z): + if pixels.shape[-1] == 2: + pixels = torch.cat((pixels, torch.ones_like(pixels[..., :1])), dim=-1) + return z.unsqueeze(-1) * (pixels * inv_K.diag() + inv_K[:, 2] * mask110(z.device, z.dtype)) + + +def make_pts3d(anchors, K, cam2w, depthmaps, base_focals=None, ret_depth=False): + focals = K[:, 0, 0] + invK = inv(K) + all_pts3d = [] + depth_out = [] + + for img, (pixels, idxs, offsets) in anchors.items(): + # from depthmaps to 3d points + if base_focals is None: + pass + else: + # compensate for focal + # depth + depth * (offset - 1) * base_focal / focal + # = depth * (1 + (offset - 1) * (base_focal / focal)) + offsets = 1 + (offsets - 1) * (base_focals[img] / focals[img]) + + pts3d = proj3d(invK[img], pixels, depthmaps[img][idxs] * offsets) + if ret_depth: + depth_out.append(pts3d[..., 2]) # before camera rotation + + # rotate to world coordinate + pts3d = geotrf(cam2w[img], pts3d) + all_pts3d.append(pts3d) + + if ret_depth: + return all_pts3d, depth_out + return all_pts3d + + +def make_dense_pts3d(intrinsics, cam2w, depthmaps, canonical_paths, subsample, device='cuda'): + base_focals = [] + anchors = {} + confs = [] + for i, canon_path in enumerate(canonical_paths): + (canon, canon2, conf), focal = torch.load(canon_path, map_location=device) + confs.append(conf) + base_focals.append(focal) + H, W = conf.shape + pixels = torch.from_numpy(np.mgrid[:W, :H].T.reshape(-1, 2)).float().to(device) + idxs, offsets = anchor_depth_offsets(canon2, {i: (pixels, None)}, subsample=subsample) + anchors[i] = (pixels, idxs[i], offsets[i]) + + # densify sparse depthmaps + pts3d, depthmaps_out = make_pts3d(anchors, intrinsics, cam2w, [ + d.ravel() for d in depthmaps], base_focals=base_focals, ret_depth=True) + + return pts3d, depthmaps_out, confs + + +@torch.no_grad() +def forward_mast3r(pairs, model, cache_path, desc_conf='desc_conf', + device='cuda', subsample=8, **matching_kw): + res_paths = {} + + for img1, img2 in tqdm(pairs): + idx1 = hash_md5(img1['instance']) + idx2 = hash_md5(img2['instance']) + + path1 = cache_path + f'/forward/{idx1}/{idx2}.pth' + path2 = cache_path + f'/forward/{idx2}/{idx1}.pth' + path_corres = cache_path + f'/corres_conf={desc_conf}_{subsample=}/{idx1}-{idx2}.pth' + path_corres2 = cache_path + f'/corres_conf={desc_conf}_{subsample=}/{idx2}-{idx1}.pth' + + if os.path.isfile(path_corres2) and not os.path.isfile(path_corres): + score, (xy1, xy2, confs) = torch.load(path_corres2) + torch.save((score, (xy2, xy1, confs)), path_corres) + + if not all(os.path.isfile(p) for p in (path1, path2, path_corres)): + if model is None: + continue + res = symmetric_inference(model, img1, img2, device=device) + X11, X21, X22, X12 = [r['pts3d'][0] for r in res] + C11, C21, C22, C12 = [r['conf'][0] for r in res] + descs = [r['desc'][0] for r in res] + qonfs = [r[desc_conf][0] for r in res] + + # save + torch.save(to_cpu((X11, C11, X21, C21)), mkdir_for(path1)) + torch.save(to_cpu((X22, C22, X12, C12)), mkdir_for(path2)) + + # perform reciprocal matching + corres = extract_correspondences(descs, qonfs, device=device, subsample=subsample) + + conf_score = (C11.mean() * C12.mean() * C21.mean() * C22.mean()).sqrt().sqrt() + matching_score = (float(conf_score), float(corres[2].sum()), len(corres[2])) + if cache_path is not None: + torch.save((matching_score, corres), mkdir_for(path_corres)) + + res_paths[img1['instance'], img2['instance']] = (path1, path2), path_corres + + del model + torch.cuda.empty_cache() + + return res_paths, cache_path + + +def symmetric_inference(model, img1, img2, device): + shape1 = torch.from_numpy(img1['true_shape']).to(device, non_blocking=True) + shape2 = torch.from_numpy(img2['true_shape']).to(device, non_blocking=True) + img1 = img1['img'].to(device, non_blocking=True) + img2 = img2['img'].to(device, non_blocking=True) + + # compute encoder only once + feat1, feat2, pos1, pos2 = model._encode_image_pairs(img1, img2, shape1, shape2) + + def decoder(feat1, feat2, pos1, pos2, shape1, shape2): + dec1, dec2 = model._decoder(feat1, pos1, feat2, pos2) + with torch.cuda.amp.autocast(enabled=False): + res1 = model._downstream_head(1, [tok.float() for tok in dec1], shape1) + res2 = model._downstream_head(2, [tok.float() for tok in dec2], shape2) + return res1, res2 + + # decoder 1-2 + res11, res21 = decoder(feat1, feat2, pos1, pos2, shape1, shape2) + # decoder 2-1 + res22, res12 = decoder(feat2, feat1, pos2, pos1, shape2, shape1) + + return (res11, res21, res22, res12) + + +def extract_correspondences(feats, qonfs, subsample=8, device=None, ptmap_key='pred_desc'): + feat11, feat21, feat22, feat12 = feats + qonf11, qonf21, qonf22, qonf12 = qonfs + + assert feat11.shape[:2] == feat12.shape[:2] == qonf11.shape == qonf12.shape, (feat11.shape, feat12.shape, qonf11.shape, qonf12.shape) + assert feat21.shape[:2] == feat22.shape[:2] == qonf21.shape == qonf22.shape, (feat21.shape, feat22.shape, qonf21.shape, qonf22.shape) + + if '3d' in ptmap_key: + opt = dict(device='cpu', workers=32) + else: + opt = dict(device=device, dist='dot', block_size=2**13) + + # matching the two pairs + idx1 = [] + idx2 = [] + qonf1 = [] + qonf2 = [] + # TODO add non symmetric / pixel_tol options + for A, B, QA, QB in [(feat11, feat21, qonf11.cpu(), qonf21.cpu()), + (feat12, feat22, qonf12.cpu(), qonf22.cpu())]: + nn1to2 = fast_reciprocal_NNs(A, B, subsample_or_initxy1=subsample, ret_xy=False, **opt) + nn2to1 = fast_reciprocal_NNs(B, A, subsample_or_initxy1=subsample, ret_xy=False, **opt) + + idx1.append(np.r_[nn1to2[0], nn2to1[1]]) + idx2.append(np.r_[nn1to2[1], nn2to1[0]]) + qonf1.append(QA.ravel()[idx1[-1]]) + qonf2.append(QB.ravel()[idx2[-1]]) + + # merge corres from opposite pairs + H1, W1 = feat11.shape[:2] + H2, W2 = feat22.shape[:2] + cat = np.concatenate + + xy1, xy2, idx = merge_corres(cat(idx1), cat(idx2), (H1, W1), (H2, W2), ret_xy=True, ret_index=True) + corres = (xy1.copy(), xy2.copy(), np.sqrt(cat(qonf1)[idx] * cat(qonf2)[idx])) + + return todevice(corres, device) + + +@torch.no_grad() +def prepare_canonical_data(imgs, tmp_pairs, subsample, order_imgs=False, min_conf_thr=0, + cache_path=None, device='cuda', **kw): + canonical_views = {} + pairwise_scores = torch.zeros((len(imgs), len(imgs)), device=device) + canonical_paths = [] + preds_21 = {} + + for img in tqdm(imgs): + if cache_path: + cache = os.path.join(cache_path, 'canon_views', hash_md5(img) + f'_{subsample=}_{kw=}.pth') + canonical_paths.append(cache) + try: + (canon, canon2, cconf), focal = torch.load(cache, map_location=device) + except IOError: + # cache does not exist yet, we create it! + canon = focal = None + + # collect all pred1 + n_pairs = sum((img in pair) for pair in tmp_pairs) + + ptmaps11 = None + pixels = {} + n = 0 + for (img1, img2), ((path1, path2), path_corres) in tmp_pairs.items(): + score = None + if img == img1: + X, C, X2, C2 = torch.load(path1, map_location=device) + score, (xy1, xy2, confs) = load_corres(path_corres, device, min_conf_thr) + pixels[img2] = xy1, confs + if img not in preds_21: + preds_21[img] = {} + # Subsample preds_21 + preds_21[img][img2] = X2[::subsample, ::subsample].reshape(-1, 3), C2[::subsample, ::subsample].ravel() + + if img == img2: + X, C, X2, C2 = torch.load(path2, map_location=device) + score, (xy1, xy2, confs) = load_corres(path_corres, device, min_conf_thr) + pixels[img1] = xy2, confs + if img not in preds_21: + preds_21[img] = {} + preds_21[img][img1] = X2[::subsample, ::subsample].reshape(-1, 3), C2[::subsample, ::subsample].ravel() + + if score is not None: + i, j = imgs.index(img1), imgs.index(img2) + # score = score[0] + # score = np.log1p(score[2]) + score = score[2] + pairwise_scores[i, j] = score + pairwise_scores[j, i] = score + + if canon is not None: + continue + if ptmaps11 is None: + H, W = C.shape + ptmaps11 = torch.empty((n_pairs, H, W, 3), device=device) + confs11 = torch.empty((n_pairs, H, W), device=device) + + ptmaps11[n] = X + confs11[n] = C + n += 1 + + if canon is None: + canon, canon2, cconf = canonical_view(ptmaps11, confs11, subsample, **kw) + del ptmaps11 + del confs11 + + # compute focals + H, W = canon.shape[:2] + pp = torch.tensor([W / 2, H / 2], device=device) + if focal is None: + focal = estimate_focal_knowing_depth(canon[None], pp, focal_mode='weiszfeld', min_focal=0.5, max_focal=3.5) + if cache: + torch.save(to_cpu(((canon, canon2, cconf), focal)), mkdir_for(cache)) + + # extract depth offsets with correspondences + core_depth = canon[subsample // 2::subsample, subsample // 2::subsample, 2] + idxs, offsets = anchor_depth_offsets(canon2, pixels, subsample=subsample) + + canonical_views[img] = (pp, (H, W), focal.view(1), core_depth, pixels, idxs, offsets) + + return tmp_pairs, pairwise_scores, canonical_views, canonical_paths, preds_21 + + +def load_corres(path_corres, device, min_conf_thr): + score, (xy1, xy2, confs) = torch.load(path_corres, map_location=device) + valid = confs > min_conf_thr if min_conf_thr else slice(None) + # valid = (xy1 > 0).all(dim=1) & (xy2 > 0).all(dim=1) & (xy1 < 512).all(dim=1) & (xy2 < 512).all(dim=1) + # print(f'keeping {valid.sum()} / {len(valid)} correspondences') + return score, (xy1[valid], xy2[valid], confs[valid]) + + +PairOfSlices = namedtuple( + 'ImgPair', 'img1, slice1, pix1, anchor_idxs1, img2, slice2, pix2, anchor_idxs2, confs, confs_sum') + + +def condense_data(imgs, tmp_paths, canonical_views, preds_21, dtype=torch.float32): + # aggregate all data properly + set_imgs = set(imgs) + + principal_points = [] + shapes = [] + focals = [] + core_depth = [] + img_anchors = {} + tmp_pixels = {} + + for idx1, img1 in enumerate(imgs): + # load stuff + pp, shape, focal, anchors, pixels_confs, idxs, offsets = canonical_views[img1] + + principal_points.append(pp) + shapes.append(shape) + focals.append(focal) + core_depth.append(anchors) + + img_uv1 = [] + img_idxs = [] + img_offs = [] + cur_n = [0] + + for img2, (pixels, match_confs) in pixels_confs.items(): + if img2 not in set_imgs: + continue + assert len(pixels) == len(idxs[img2]) == len(offsets[img2]) + img_uv1.append(torch.cat((pixels, torch.ones_like(pixels[:, :1])), dim=-1)) + img_idxs.append(idxs[img2]) + img_offs.append(offsets[img2]) + cur_n.append(cur_n[-1] + len(pixels)) + # store the position of 3d points + tmp_pixels[img1, img2] = pixels.to(dtype), match_confs.to(dtype), slice(*cur_n[-2:]) + img_anchors[idx1] = (torch.cat(img_uv1), torch.cat(img_idxs), torch.cat(img_offs)) + + all_confs = [] + imgs_slices = [] + corres2d = {img: [] for img in range(len(imgs))} + + for img1, img2 in tmp_paths: + try: + pix1, confs1, slice1 = tmp_pixels[img1, img2] + pix2, confs2, slice2 = tmp_pixels[img2, img1] + except KeyError: + continue + img1 = imgs.index(img1) + img2 = imgs.index(img2) + confs = (confs1 * confs2).sqrt() + + # prepare for loss_3d + all_confs.append(confs) + anchor_idxs1 = canonical_views[imgs[img1]][5][imgs[img2]] + anchor_idxs2 = canonical_views[imgs[img2]][5][imgs[img1]] + imgs_slices.append(PairOfSlices(img1, slice1, pix1, anchor_idxs1, + img2, slice2, pix2, anchor_idxs2, + confs, float(confs.sum()))) + + # prepare for loss_2d + corres2d[img1].append((pix1, confs, img2, slice2)) + corres2d[img2].append((pix2, confs, img1, slice1)) + + all_confs = torch.cat(all_confs) + corres = (all_confs, float(all_confs.sum()), imgs_slices) + + def aggreg_matches(img1, list_matches): + pix1, confs, img2, slice2 = zip(*list_matches) + all_pix1 = torch.cat(pix1).to(dtype) + all_confs = torch.cat(confs).to(dtype) + return img1, all_pix1, all_confs, float(all_confs.sum()), [(j, sl2) for j, sl2 in zip(img2, slice2)] + corres2d = [aggreg_matches(img, m) for img, m in corres2d.items()] + + imsizes = torch.tensor([(W, H) for H, W in shapes], device=pp.device) # (W,H) + principal_points = torch.stack(principal_points) + focals = torch.cat(focals) + + # Subsample preds_21 + subsamp_preds_21 = {} + for imk, imv in preds_21.items(): + subsamp_preds_21[imk] = {} + for im2k, (pred, conf) in preds_21[imk].items(): + idxs = img_anchors[imgs.index(im2k)][1] + subsamp_preds_21[imk][im2k] = (pred[idxs], conf[idxs]) # anchors subsample + + return imsizes, principal_points, focals, core_depth, img_anchors, corres, corres2d, subsamp_preds_21 + + +def canonical_view(ptmaps11, confs11, subsample, mode='avg-angle'): + assert len(ptmaps11) == len(confs11) > 0, 'not a single view1 for img={i}' + + # canonical pointmap is just a weighted average + confs11 = confs11.unsqueeze(-1) - 0.999 + canon = (confs11 * ptmaps11).sum(0) / confs11.sum(0) + + canon_depth = ptmaps11[..., 2].unsqueeze(1) + S = slice(subsample // 2, None, subsample) + center_depth = canon_depth[:, :, S, S] + assert (center_depth > 0).all() + stacked_depth = F.pixel_unshuffle(canon_depth, subsample) + stacked_confs = F.pixel_unshuffle(confs11[:, None, :, :, 0], subsample) + + if mode == 'avg-reldepth': + rel_depth = stacked_depth / center_depth + stacked_canon = (stacked_confs * rel_depth).sum(dim=0) / stacked_confs.sum(dim=0) + canon2 = F.pixel_shuffle(stacked_canon.unsqueeze(0), subsample).squeeze() + + elif mode == 'avg-angle': + xy = ptmaps11[..., 0:2].permute(0, 3, 1, 2) + stacked_xy = F.pixel_unshuffle(xy, subsample) + B, _, H, W = stacked_xy.shape + stacked_radius = (stacked_xy.view(B, 2, -1, H, W) - xy[:, :, None, S, S]).norm(dim=1) + stacked_radius.clip_(min=1e-8) + + stacked_angle = torch.arctan((stacked_depth - center_depth) / stacked_radius) + avg_angle = (stacked_confs * stacked_angle).sum(dim=0) / stacked_confs.sum(dim=0) + + # back to depth + stacked_depth = stacked_radius.mean(dim=0) * torch.tan(avg_angle) + + canon2 = F.pixel_shuffle((1 + stacked_depth / canon[S, S, 2]).unsqueeze(0), subsample).squeeze() + else: + raise ValueError(f'bad {mode=}') + + confs = (confs11.square().sum(dim=0) / confs11.sum(dim=0)).squeeze() + return canon, canon2, confs + + +def anchor_depth_offsets(canon_depth, pixels, subsample=8): + device = canon_depth.device + + # create a 2D grid of anchor 3D points + H1, W1 = canon_depth.shape + yx = np.mgrid[subsample // 2:H1:subsample, subsample // 2:W1:subsample] + H2, W2 = yx.shape[1:] + cy, cx = yx.reshape(2, -1) + core_depth = canon_depth[cy, cx] + assert (core_depth > 0).all() + + # slave 3d points (attached to core 3d points) + core_idxs = {} # core_idxs[img2] = {corr_idx:core_idx} + core_offs = {} # core_offs[img2] = {corr_idx:3d_offset} + + for img2, (xy1, _confs) in pixels.items(): + px, py = xy1.long().T + + # find nearest anchor == block quantization + core_idx = (py // subsample) * W2 + (px // subsample) + core_idxs[img2] = core_idx.to(device) + + # compute relative depth offsets w.r.t. anchors + ref_z = core_depth[core_idx] + pts_z = canon_depth[py, px] + offset = pts_z / ref_z + core_offs[img2] = offset.detach().to(device) + + return core_idxs, core_offs + + +def spectral_clustering(graph, k=None, normalized_cuts=False): + graph.fill_diagonal_(0) + + # graph laplacian + degrees = graph.sum(dim=-1) + laplacian = torch.diag(degrees) - graph + if normalized_cuts: + i_inv = torch.diag(degrees.sqrt().reciprocal()) + laplacian = i_inv @ laplacian @ i_inv + + # compute eigenvectors! + eigval, eigvec = torch.linalg.eigh(laplacian) + return eigval[:k], eigvec[:, :k] + + +def sim_func(p1, p2, gamma): + diff = (p1 - p2).norm(dim=-1) + avg_depth = (p1[:, :, 2] + p2[:, :, 2]) + rel_distance = diff / avg_depth + sim = torch.exp(-gamma * rel_distance.square()) + return sim + + +def backproj(K, depthmap, subsample): + H, W = depthmap.shape + uv = np.mgrid[subsample // 2:subsample * W:subsample, subsample // 2:subsample * H:subsample].T.reshape(H, W, 2) + xyz = depthmap.unsqueeze(-1) * geotrf(inv(K), todevice(uv, K.device), ncol=3) + return xyz + + +def spectral_projection_depth(K, depthmap, subsample, k=64, cache_path='', + normalized_cuts=True, gamma=7, min_norm=5): + try: + if cache_path: + cache_path = cache_path + f'_{k=}_norm={normalized_cuts}_{gamma=}.pth' + lora_proj = torch.load(cache_path, map_location=K.device) + + except IOError: + # reconstruct 3d points in camera coordinates + xyz = backproj(K, depthmap, subsample) + + # compute all distances + xyz = xyz.reshape(-1, 3) + graph = sim_func(xyz[:, None], xyz[None, :], gamma=gamma) + _, lora_proj = spectral_clustering(graph, k, normalized_cuts=normalized_cuts) + + if cache_path: + torch.save(lora_proj.cpu(), mkdir_for(cache_path)) + + lora_proj, coeffs = lora_encode_normed(lora_proj, depthmap.ravel(), min_norm=min_norm) + + # depthmap ~= lora_proj @ coeffs + return coeffs, lora_proj + + +def lora_encode_normed(lora_proj, x, min_norm, global_norm=False): + # encode the pointmap + coeffs = torch.linalg.pinv(lora_proj) @ x + + # rectify the norm of basis vector to be ~ equal + if coeffs.ndim == 1: + coeffs = coeffs[:, None] + if global_norm: + lora_proj *= coeffs[1:].norm() * min_norm / coeffs.shape[1] + elif min_norm: + lora_proj *= coeffs.norm(dim=1).clip(min=min_norm) + # can have rounding errors here! + coeffs = (torch.linalg.pinv(lora_proj.double()) @ x.double()).float() + + return lora_proj.detach(), coeffs.detach() + + +@torch.no_grad() +def spectral_projection_of_depthmaps(imgs, intrinsics, depthmaps, subsample, cache_path=None, **kw): + # recover 3d points + core_depth = [] + lora_proj = [] + + for i, img in enumerate(tqdm(imgs)): + cache = os.path.join(cache_path, 'lora_depth', hash_md5(img)) if cache_path else None + depth, proj = spectral_projection_depth(intrinsics[i], depthmaps[i], subsample, + cache_path=cache, **kw) + core_depth.append(depth) + lora_proj.append(proj) + + return core_depth, lora_proj + + +def reproj2d(Trf, pts3d): + res = (pts3d @ Trf[:3, :3].transpose(-1, -2)) + Trf[:3, 3] + clipped_z = res[:, 2:3].clip(min=1e-3) # make sure we don't have nans! + uv = res[:, 0:2] / clipped_z + return uv.clip(min=-1000, max=2000) + + +def bfs(tree, start_node): + order, predecessors = sp.csgraph.breadth_first_order(tree, start_node, directed=False) + ranks = np.arange(len(order)) + ranks[order] = ranks.copy() + return ranks, predecessors + + +def compute_min_spanning_tree(pws): + sparse_graph = sp.dok_array(pws.shape) + for i, j in pws.nonzero().cpu().tolist(): + sparse_graph[i, j] = -float(pws[i, j]) + msp = sp.csgraph.minimum_spanning_tree(sparse_graph) + + # now reorder the oriented edges, starting from the central point + ranks1, _ = bfs(msp, 0) + ranks2, _ = bfs(msp, ranks1.argmax()) + ranks1, _ = bfs(msp, ranks2.argmax()) + # this is the point farther from any leaf + root = np.minimum(ranks1, ranks2).argmax() + + # find the ordered list of edges that describe the tree + order, predecessors = sp.csgraph.breadth_first_order(msp, root, directed=False) + order = order[1:] # root not do not have a predecessor + edges = [(predecessors[i], i) for i in order] + + return root, edges + + +def show_reconstruction(shapes_or_imgs, K, cam2w, pts3d, gt_cam2w=None, gt_K=None, cam_size=None, masks=None, **kw): + viz = SceneViz() + + cc = cam2w[:, :3, 3] + cs = cam_size or float(torch.cdist(cc, cc).fill_diagonal_(np.inf).min(dim=0).values.median()) + colors = 64 + np.random.randint(255 - 64, size=(len(cam2w), 3)) + + if isinstance(shapes_or_imgs, np.ndarray) and shapes_or_imgs.ndim == 2: + cam_kws = dict(imsizes=shapes_or_imgs[:, ::-1], cam_size=cs) + else: + imgs = shapes_or_imgs + cam_kws = dict(images=imgs, cam_size=cs) + if K is not None: + viz.add_cameras(to_numpy(cam2w), to_numpy(K), colors=colors, **cam_kws) + + if gt_cam2w is not None: + if gt_K is None: + gt_K = K + viz.add_cameras(to_numpy(gt_cam2w), to_numpy(gt_K), colors=colors, marker='o', **cam_kws) + + if pts3d is not None: + for i, p in enumerate(pts3d): + if not len(p): + continue + if masks is None: + viz.add_pointcloud(to_numpy(p), color=tuple(colors[i].tolist())) + else: + viz.add_pointcloud(to_numpy(p), mask=masks[i], color=imgs[i]) + viz.show(**kw) diff --git a/src/mast3r_src/mast3r/cloud_opt/triangulation.py b/src/mast3r_src/mast3r/cloud_opt/triangulation.py new file mode 100644 index 0000000000000000000000000000000000000000..2af88df37bfd360161b4e96b93b0fd28a0ecf183 --- /dev/null +++ b/src/mast3r_src/mast3r/cloud_opt/triangulation.py @@ -0,0 +1,80 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Matches Triangulation Utils +# -------------------------------------------------------- + +import numpy as np +import torch + +# Batched Matches Triangulation +def batched_triangulate(pts2d, # [B, Ncams, Npts, 2] + proj_mats): # [B, Ncams, 3, 4] I@E projection matrix + B, Ncams, Npts, two = pts2d.shape + assert two==2 + assert proj_mats.shape == (B, Ncams, 3, 4) + # P - xP + x = proj_mats[...,0,:][...,None,:] - torch.einsum('bij,bik->bijk', pts2d[...,0], proj_mats[...,2,:]) # [B, Ncams, Npts, 4] + y = proj_mats[...,1,:][...,None,:] - torch.einsum('bij,bik->bijk', pts2d[...,1], proj_mats[...,2,:]) # [B, Ncams, Npts, 4] + eq = torch.cat([x, y], dim=1).transpose(1, 2) # [B, Npts, 2xNcams, 4] + return torch.linalg.lstsq(eq[...,:3], -eq[...,3]).solution + +def matches_to_depths(intrinsics, # input camera intrinsics [B, Ncams, 3, 3] + extrinsics, # input camera extrinsics [B, Ncams, 3, 4] + matches, # input correspondences [B, Ncams, Npts, 2] + batchsize=16, # bs for batched processing + min_num_valids_ratio=.3 # at least this ratio of image pairs need to predict a match for a given pixel of img1 + ): + B, Nv, H, W, five = matches.shape + min_num_valids = np.floor(Nv*min_num_valids_ratio) + out_aggregated_points, out_depths, out_confs = [], [], [] + for b in range(B//batchsize+1): # batched processing + start, stop = b*batchsize,min(B,(b+1)*batchsize) + sub_batch=slice(start,stop) + sub_batchsize = stop-start + if sub_batchsize==0:continue + points1, points2, confs = matches[sub_batch, ..., :2], matches[sub_batch, ..., 2:4], matches[sub_batch, ..., -1] + allpoints = torch.cat([points1.view([sub_batchsize*Nv,1,H*W,2]), points2.view([sub_batchsize*Nv,1,H*W,2])],dim=1) # [BxNv, 2, HxW, 2] + + allcam_Ps = intrinsics[sub_batch] @ extrinsics[sub_batch,:,:3,:] + cam_Ps1, cam_Ps2 = allcam_Ps[:,[0]].repeat([1,Nv,1,1]), allcam_Ps[:,1:] # [B, Nv, 3, 4] + formatted_camPs = torch.cat([cam_Ps1.reshape([sub_batchsize*Nv,1,3,4]), cam_Ps2.reshape([sub_batchsize*Nv,1,3,4])],dim=1) # [BxNv, 2, 3, 4] + + # Triangulate matches to 3D + points_3d_world = batched_triangulate(allpoints, formatted_camPs) # [BxNv, HxW, three] + + # Aggregate pairwise predictions + points_3d_world = points_3d_world.view([sub_batchsize,Nv,H,W,3]) + valids = points_3d_world.isfinite() + valids_sum = valids.sum(dim=-1) + validsuni=valids_sum.unique() + assert torch.all(torch.logical_or(validsuni == 0 , validsuni == 3)), "Error, can only be nan for none or all XYZ values, not a subset" + confs[valids_sum==0] = 0. + points_3d_world = points_3d_world*confs[...,None] + + # Take care of NaNs + normalization = confs.sum(dim=1)[:,None].repeat(1,Nv,1,1) + normalization[normalization <= 1e-5] = 1. + points_3d_world[valids] /= normalization[valids_sum==3][:,None].repeat(1,3).view(-1) + points_3d_world[~valids] = 0. + aggregated_points = points_3d_world.sum(dim=1) # weighted average (by confidence value) ignoring nans + + # Reset invalid values to nans, with a min visibility threshold + aggregated_points[valids_sum.sum(dim=1)/3 <= min_num_valids] = torch.nan + + # From 3D to depths + refcamE = extrinsics[sub_batch, 0] + points_3d_camera = (refcamE[:,:3, :3] @ aggregated_points.view(sub_batchsize,-1,3).transpose(-2,-1) + refcamE[:,:3,[3]]).transpose(-2,-1) # [B,HxW,3] + depths = points_3d_camera.view(sub_batchsize,H,W,3)[..., 2] # [B,H,W] + + # Cat results + out_aggregated_points.append(aggregated_points.cpu()) + out_depths.append(depths.cpu()) + out_confs.append(confs.sum(dim=1).cpu()) + + out_aggregated_points = torch.cat(out_aggregated_points,dim=0) + out_depths = torch.cat(out_depths,dim=0) + out_confs = torch.cat(out_confs,dim=0) + + return out_aggregated_points, out_depths, out_confs diff --git a/src/mast3r_src/mast3r/cloud_opt/tsdf_optimizer.py b/src/mast3r_src/mast3r/cloud_opt/tsdf_optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..69f138c0301e4ad3cd4804d265f241b923e1b2b8 --- /dev/null +++ b/src/mast3r_src/mast3r/cloud_opt/tsdf_optimizer.py @@ -0,0 +1,273 @@ +import torch +from torch import nn +import numpy as np +from tqdm import tqdm +from matplotlib import pyplot as pl + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.utils.geometry import depthmap_to_pts3d, geotrf, inv +from dust3r.cloud_opt.base_opt import clean_pointcloud + + +class TSDFPostProcess: + """ Optimizes a signed distance-function to improve depthmaps. + """ + + def __init__(self, optimizer, subsample=8, TSDF_thresh=0., TSDF_batchsize=int(1e7)): + self.TSDF_thresh = TSDF_thresh # None -> no TSDF + self.TSDF_batchsize = TSDF_batchsize + self.optimizer = optimizer + + pts3d, depthmaps, confs = optimizer.get_dense_pts3d(clean_depth=False, subsample=subsample) + pts3d, depthmaps = self._TSDF_postprocess_or_not(pts3d, depthmaps, confs) + self.pts3d = pts3d + self.depthmaps = depthmaps + self.confs = confs + + def _get_depthmaps(self, TSDF_filtering_thresh=None): + if TSDF_filtering_thresh: + self._refine_depths_with_TSDF(self.optimizer, TSDF_filtering_thresh) # compute refined depths if needed + dms = self.TSDF_im_depthmaps if TSDF_filtering_thresh else self.im_depthmaps + return [d.exp() for d in dms] + + @torch.no_grad() + def _refine_depths_with_TSDF(self, TSDF_filtering_thresh, niter=1, nsamples=1000): + """ + Leverage TSDF to post-process estimated depths + for each pixel, find zero level of TSDF along ray (or closest to 0) + """ + print("Post-Processing Depths with TSDF fusion.") + self.TSDF_im_depthmaps = [] + alldepths, allposes, allfocals, allpps, allimshapes = self._get_depthmaps(), self.optimizer.get_im_poses( + ), self.optimizer.get_focals(), self.optimizer.get_principal_points(), self.imshapes + for vi in tqdm(range(self.optimizer.n_imgs)): + dm, pose, focal, pp, imshape = alldepths[vi], allposes[vi], allfocals[vi], allpps[vi], allimshapes[vi] + minvals = torch.full(dm.shape, 1e20) + + for it in range(niter): + H, W = dm.shape + curthresh = (niter - it) * TSDF_filtering_thresh + dm_offsets = (torch.randn(H, W, nsamples).to(dm) - 1.) * \ + curthresh # decreasing search std along with iterations + newdm = dm[..., None] + dm_offsets # [H,W,Nsamp] + curproj = self._backproj_pts3d(in_depths=[newdm], in_im_poses=pose[None], in_focals=focal[None], in_pps=pp[None], in_imshapes=[ + imshape])[0] # [H,W,Nsamp,3] + # Batched TSDF eval + curproj = curproj.view(-1, 3) + tsdf_vals = [] + valids = [] + for batch in range(0, len(curproj), self.TSDF_batchsize): + values, valid = self._TSDF_query( + curproj[batch:min(batch + self.TSDF_batchsize, len(curproj))], curthresh) + tsdf_vals.append(values) + valids.append(valid) + tsdf_vals = torch.cat(tsdf_vals, dim=0) + valids = torch.cat(valids, dim=0) + + tsdf_vals = tsdf_vals.view([H, W, nsamples]) + valids = valids.view([H, W, nsamples]) + + # keep depth value that got us the closest to 0 + tsdf_vals[~valids] = torch.inf # ignore invalid values + tsdf_vals = tsdf_vals.abs() + mins = torch.argmin(tsdf_vals, dim=-1, keepdim=True) + # when all samples live on a very flat zone, do nothing + allbad = (tsdf_vals == curthresh).sum(dim=-1) == nsamples + dm[~allbad] = torch.gather(newdm, -1, mins)[..., 0][~allbad] + + # Save refined depth map + self.TSDF_im_depthmaps.append(dm.log()) + + def _TSDF_query(self, qpoints, TSDF_filtering_thresh, weighted=True): + """ + TSDF query call: returns the weighted TSDF value for each query point [N, 3] + """ + N, three = qpoints.shape + assert three == 3 + qpoints = qpoints[None].repeat(self.optimizer.n_imgs, 1, 1) # [B,N,3] + # get projection coordinates and depths onto images + coords_and_depth = self._proj_pts3d(pts3d=qpoints, cam2worlds=self.optimizer.get_im_poses( + ), focals=self.optimizer.get_focals(), pps=self.optimizer.get_principal_points()) + image_coords = coords_and_depth[..., :2].round().to(int) # for now, there's no interpolation... + proj_depths = coords_and_depth[..., -1] + # recover depth values after scene optim + pred_depths, pred_confs, valids = self._get_pixel_depths(image_coords) + # Gather TSDF scores + all_SDF_scores = pred_depths - proj_depths # SDF + unseen = all_SDF_scores < -TSDF_filtering_thresh # handle visibility + # all_TSDF_scores = all_SDF_scores.clip(-TSDF_filtering_thresh,TSDF_filtering_thresh) # SDF -> TSDF + all_TSDF_scores = all_SDF_scores.clip(-TSDF_filtering_thresh, 1e20) # SDF -> TSDF + # Gather TSDF confidences and ignore points that are unseen, either OOB during reproj or too far behind seen depth + all_TSDF_weights = (~unseen).float() * valids.float() + if weighted: + all_TSDF_weights = pred_confs.exp() * all_TSDF_weights + # Aggregate all votes, ignoring zeros + TSDF_weights = all_TSDF_weights.sum(dim=0) + valids = TSDF_weights != 0. + TSDF_wsum = (all_TSDF_weights * all_TSDF_scores).sum(dim=0) + TSDF_wsum[valids] /= TSDF_weights[valids] + return TSDF_wsum, valids + + def _get_pixel_depths(self, image_coords, TSDF_filtering_thresh=None, with_normals_conf=False): + """ Recover depth value for each input pixel coordinate, along with OOB validity mask + """ + B, N, two = image_coords.shape + assert B == self.optimizer.n_imgs and two == 2 + depths = torch.zeros([B, N], device=image_coords.device) + valids = torch.zeros([B, N], dtype=bool, device=image_coords.device) + confs = torch.zeros([B, N], device=image_coords.device) + curconfs = self._get_confs_with_normals() if with_normals_conf else self.im_conf + for ni, (imc, depth, conf) in enumerate(zip(image_coords, self._get_depthmaps(TSDF_filtering_thresh), curconfs)): + H, W = depth.shape + valids[ni] = torch.logical_and(0 <= imc[:, 1], imc[:, 1] < + H) & torch.logical_and(0 <= imc[:, 0], imc[:, 0] < W) + imc[~valids[ni]] = 0 + depths[ni] = depth[imc[:, 1], imc[:, 0]] + confs[ni] = conf.cuda()[imc[:, 1], imc[:, 0]] + return depths, confs, valids + + def _get_confs_with_normals(self): + outconfs = [] + # Confidence basedf on depth gradient + + class Sobel(nn.Module): + def __init__(self): + super().__init__() + self.filter = nn.Conv2d(in_channels=1, out_channels=2, kernel_size=3, stride=1, padding=1, bias=False) + Gx = torch.tensor([[2.0, 0.0, -2.0], [4.0, 0.0, -4.0], [2.0, 0.0, -2.0]]) + Gy = torch.tensor([[2.0, 4.0, 2.0], [0.0, 0.0, 0.0], [-2.0, -4.0, -2.0]]) + G = torch.cat([Gx.unsqueeze(0), Gy.unsqueeze(0)], 0) + G = G.unsqueeze(1) + self.filter.weight = nn.Parameter(G, requires_grad=False) + + def forward(self, img): + x = self.filter(img) + x = torch.mul(x, x) + x = torch.sum(x, dim=1, keepdim=True) + x = torch.sqrt(x) + return x + + grad_op = Sobel().to(self.im_depthmaps[0].device) + for conf, depth in zip(self.im_conf, self.im_depthmaps): + grad_confs = (1. - grad_op(depth[None, None])[0, 0]).clip(0) + if not 'dbg show': + pl.imshow(grad_confs.cpu()) + pl.show() + outconfs.append(conf * grad_confs.to(conf)) + return outconfs + + def _proj_pts3d(self, pts3d, cam2worlds, focals, pps): + """ + Projection operation: from 3D points to 2D coordinates + depths + """ + B = pts3d.shape[0] + assert pts3d.shape[0] == cam2worlds.shape[0] + # prepare Extrinsincs + R, t = cam2worlds[:, :3, :3], cam2worlds[:, :3, -1] + Rinv = R.transpose(-2, -1) + tinv = -Rinv @ t[..., None] + + # prepare intrinsics + intrinsics = torch.eye(3).to(cam2worlds)[None].repeat(focals.shape[0], 1, 1) + if len(focals.shape) == 1: + focals = torch.stack([focals, focals], dim=-1) + intrinsics[:, 0, 0] = focals[:, 0] + intrinsics[:, 1, 1] = focals[:, 1] + intrinsics[:, :2, -1] = pps + # Project + projpts = intrinsics @ (Rinv @ pts3d.transpose(-2, -1) + tinv) # I(RX+t) : [B,3,N] + projpts = projpts.transpose(-2, -1) # [B,N,3] + projpts[..., :2] /= projpts[..., [-1]] # [B,N,3] (X/Z , Y/Z, Z) + return projpts + + def _backproj_pts3d(self, in_depths=None, in_im_poses=None, + in_focals=None, in_pps=None, in_imshapes=None): + """ + Backprojection operation: from image depths to 3D points + """ + # Get depths and projection params if not provided + focals = self.optimizer.get_focals() if in_focals is None else in_focals + im_poses = self.optimizer.get_im_poses() if in_im_poses is None else in_im_poses + depth = self._get_depthmaps() if in_depths is None else in_depths + pp = self.optimizer.get_principal_points() if in_pps is None else in_pps + imshapes = self.imshapes if in_imshapes is None else in_imshapes + def focal_ex(i): return focals[i][..., None, None].expand(1, *focals[i].shape, *imshapes[i]) + dm_to_3d = [depthmap_to_pts3d(depth[i][None], focal_ex(i), pp=pp[[i]]) for i in range(im_poses.shape[0])] + + def autoprocess(x): + x = x[0] + return x.transpose(-2, -1) if len(x.shape) == 4 else x + return [geotrf(pose, autoprocess(pt)) for pose, pt in zip(im_poses, dm_to_3d)] + + def _pts3d_to_depth(self, pts3d, cam2worlds, focals, pps): + """ + Projection operation: from 3D points to 2D coordinates + depths + """ + B = pts3d.shape[0] + assert pts3d.shape[0] == cam2worlds.shape[0] + # prepare Extrinsincs + R, t = cam2worlds[:, :3, :3], cam2worlds[:, :3, -1] + Rinv = R.transpose(-2, -1) + tinv = -Rinv @ t[..., None] + + # prepare intrinsics + intrinsics = torch.eye(3).to(cam2worlds)[None].repeat(self.optimizer.n_imgs, 1, 1) + if len(focals.shape) == 1: + focals = torch.stack([focals, focals], dim=-1) + intrinsics[:, 0, 0] = focals[:, 0] + intrinsics[:, 1, 1] = focals[:, 1] + intrinsics[:, :2, -1] = pps + # Project + projpts = intrinsics @ (Rinv @ pts3d.transpose(-2, -1) + tinv) # I(RX+t) : [B,3,N] + projpts = projpts.transpose(-2, -1) # [B,N,3] + projpts[..., :2] /= projpts[..., [-1]] # [B,N,3] (X/Z , Y/Z, Z) + return projpts + + def _depth_to_pts3d(self, in_depths=None, in_im_poses=None, in_focals=None, in_pps=None, in_imshapes=None): + """ + Backprojection operation: from image depths to 3D points + """ + # Get depths and projection params if not provided + focals = self.optimizer.get_focals() if in_focals is None else in_focals + im_poses = self.optimizer.get_im_poses() if in_im_poses is None else in_im_poses + depth = self._get_depthmaps() if in_depths is None else in_depths + pp = self.optimizer.get_principal_points() if in_pps is None else in_pps + imshapes = self.imshapes if in_imshapes is None else in_imshapes + + def focal_ex(i): return focals[i][..., None, None].expand(1, *focals[i].shape, *imshapes[i]) + + dm_to_3d = [depthmap_to_pts3d(depth[i][None], focal_ex(i), pp=pp[i:i + 1]) for i in range(im_poses.shape[0])] + + def autoprocess(x): + x = x[0] + H, W, three = x.shape[:3] + return x.transpose(-2, -1) if len(x.shape) == 4 else x + return [geotrf(pp, autoprocess(pt)) for pp, pt in zip(im_poses, dm_to_3d)] + + def _get_pts3d(self, TSDF_filtering_thresh=None, **kw): + """ + return 3D points (possibly filtering depths with TSDF) + """ + return self._backproj_pts3d(in_depths=self._get_depthmaps(TSDF_filtering_thresh=TSDF_filtering_thresh), **kw) + + def _TSDF_postprocess_or_not(self, pts3d, depthmaps, confs, niter=1): + # Setup inner variables + self.imshapes = [im.shape[:2] for im in self.optimizer.imgs] + self.im_depthmaps = [dd.log().view(imshape) for dd, imshape in zip(depthmaps, self.imshapes)] + self.im_conf = confs + + if self.TSDF_thresh > 0.: + # Create or update self.TSDF_im_depthmaps that contain logdepths filtered with TSDF + self._refine_depths_with_TSDF(self.TSDF_thresh, niter=niter) + depthmaps = [dd.exp() for dd in self.TSDF_im_depthmaps] + # Turn them into 3D points + pts3d = self._backproj_pts3d(in_depths=depthmaps) + depthmaps = [dd.flatten() for dd in depthmaps] + pts3d = [pp.view(-1, 3) for pp in pts3d] + return pts3d, depthmaps + + def get_dense_pts3d(self, clean_depth=True): + if clean_depth: + confs = clean_pointcloud(self.confs, self.optimizer.intrinsics, inv(self.optimizer.cam2w), + self.depthmaps, self.pts3d) + return self.pts3d, self.depthmaps, confs diff --git a/src/mast3r_src/mast3r/cloud_opt/utils/__init__.py b/src/mast3r_src/mast3r/cloud_opt/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d7dd877d649ce4dbd749dd7195a8b34c0f91d4f0 --- /dev/null +++ b/src/mast3r_src/mast3r/cloud_opt/utils/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). \ No newline at end of file diff --git a/src/mast3r_src/mast3r/cloud_opt/utils/losses.py b/src/mast3r_src/mast3r/cloud_opt/utils/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..e1dd36afd6862592b8d00c499988136a972bd6e6 --- /dev/null +++ b/src/mast3r_src/mast3r/cloud_opt/utils/losses.py @@ -0,0 +1,32 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# losses for sparse ga +# -------------------------------------------------------- +import torch +import numpy as np + + +def l05_loss(x, y): + return torch.linalg.norm(x - y, dim=-1).sqrt() + + +def l1_loss(x, y): + return torch.linalg.norm(x - y, dim=-1) + + +def gamma_loss(gamma, mul=1, offset=None, clip=np.inf): + if offset is None: + if gamma == 1: + return l1_loss + # d(x**p)/dx = 1 ==> p * x**(p-1) == 1 ==> x = (1/p)**(1/(p-1)) + offset = (1 / gamma)**(1 / (gamma - 1)) + + def loss_func(x, y): + return (mul * l1_loss(x, y).clip(max=clip) + offset) ** gamma - offset ** gamma + return loss_func + + +def meta_gamma_loss(): + return lambda alpha: gamma_loss(alpha) diff --git a/src/mast3r_src/mast3r/cloud_opt/utils/schedules.py b/src/mast3r_src/mast3r/cloud_opt/utils/schedules.py new file mode 100644 index 0000000000000000000000000000000000000000..d96253b4348d2f089c10142c5991e5afb8a9b683 --- /dev/null +++ b/src/mast3r_src/mast3r/cloud_opt/utils/schedules.py @@ -0,0 +1,17 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# lr schedules for sparse ga +# -------------------------------------------------------- +import numpy as np + + +def linear_schedule(alpha, lr_base, lr_end=0): + lr = (1 - alpha) * lr_base + alpha * lr_end + return lr + + +def cosine_schedule(alpha, lr_base, lr_end=0): + lr = lr_end + (lr_base - lr_end) * (1 + np.cos(alpha * np.pi)) / 2 + return lr diff --git a/src/mast3r_src/mast3r/colmap/__init__.py b/src/mast3r_src/mast3r/colmap/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d7dd877d649ce4dbd749dd7195a8b34c0f91d4f0 --- /dev/null +++ b/src/mast3r_src/mast3r/colmap/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). \ No newline at end of file diff --git a/src/mast3r_src/mast3r/colmap/database.py b/src/mast3r_src/mast3r/colmap/database.py new file mode 100644 index 0000000000000000000000000000000000000000..4220b378ec3ffeacf02ad9cb8cefbd8e30b26bed --- /dev/null +++ b/src/mast3r_src/mast3r/colmap/database.py @@ -0,0 +1,383 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# MASt3R to colmap export functions +# -------------------------------------------------------- +import os +import torch +import copy +import numpy as np +import torchvision +import numpy as np +from tqdm import tqdm +from scipy.cluster.hierarchy import DisjointSet +from scipy.spatial.transform import Rotation as R + +from mast3r.utils.misc import hash_md5 + +from mast3r.fast_nn import extract_correspondences_nonsym, bruteforce_reciprocal_nns + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.utils.geometry import find_reciprocal_matches, xy_grid # noqa + + +def convert_im_matches_pairs(img0, img1, image_to_colmap, im_keypoints, matches_im0, matches_im1, viz): + if viz: + from matplotlib import pyplot as pl + + image_mean = torch.as_tensor( + [0.5, 0.5, 0.5], device='cpu').reshape(1, 3, 1, 1) + image_std = torch.as_tensor( + [0.5, 0.5, 0.5], device='cpu').reshape(1, 3, 1, 1) + rgb0 = img0['img'] * image_std + image_mean + rgb0 = torchvision.transforms.functional.to_pil_image(rgb0[0]) + rgb0 = np.array(rgb0) + + rgb1 = img1['img'] * image_std + image_mean + rgb1 = torchvision.transforms.functional.to_pil_image(rgb1[0]) + rgb1 = np.array(rgb1) + + imgs = [rgb0, rgb1] + # visualize a few matches + n_viz = 100 + num_matches = matches_im0.shape[0] + match_idx_to_viz = np.round(np.linspace( + 0, num_matches - 1, n_viz)).astype(int) + viz_matches_im0, viz_matches_im1 = matches_im0[match_idx_to_viz], matches_im1[match_idx_to_viz] + + H0, W0, H1, W1 = *imgs[0].shape[:2], *imgs[1].shape[:2] + rgb0 = np.pad(imgs[0], ((0, max(H1 - H0, 0)), + (0, 0), (0, 0)), 'constant', constant_values=0) + rgb1 = np.pad(imgs[1], ((0, max(H0 - H1, 0)), + (0, 0), (0, 0)), 'constant', constant_values=0) + img = np.concatenate((rgb0, rgb1), axis=1) + pl.figure() + pl.imshow(img) + cmap = pl.get_cmap('jet') + for ii in range(n_viz): + (x0, y0), (x1, + y1) = viz_matches_im0[ii].T, viz_matches_im1[ii].T + pl.plot([x0, x1 + W0], [y0, y1], '-+', color=cmap(ii / + (n_viz - 1)), scalex=False, scaley=False) + pl.show(block=True) + + matches = [matches_im0.astype(np.float64), matches_im1.astype(np.float64)] + imgs = [img0, img1] + imidx0 = img0['idx'] + imidx1 = img1['idx'] + ravel_matches = [] + for j in range(2): + H, W = imgs[j]['true_shape'][0] + with np.errstate(invalid='ignore'): + qx, qy = matches[j].round().astype(np.int32).T + ravel_matches_j = qx.clip(min=0, max=W - 1, out=qx) + W * qy.clip(min=0, max=H - 1, out=qy) + ravel_matches.append(ravel_matches_j) + imidxj = imgs[j]['idx'] + for m in ravel_matches_j: + if m not in im_keypoints[imidxj]: + im_keypoints[imidxj][m] = 0 + im_keypoints[imidxj][m] += 1 + imid0 = copy.deepcopy(image_to_colmap[imidx0]['colmap_imid']) + imid1 = copy.deepcopy(image_to_colmap[imidx1]['colmap_imid']) + if imid0 > imid1: + colmap_matches = np.stack([ravel_matches[1], ravel_matches[0]], axis=-1) + imid0, imid1 = imid1, imid0 + imidx0, imidx1 = imidx1, imidx0 + else: + colmap_matches = np.stack([ravel_matches[0], ravel_matches[1]], axis=-1) + colmap_matches = np.unique(colmap_matches, axis=0) + return imidx0, imidx1, colmap_matches + + +def get_im_matches(pred1, pred2, pairs, image_to_colmap, im_keypoints, conf_thr, + is_sparse=True, subsample=8, pixel_tol=0, viz=False, device='cuda'): + im_matches = {} + for i in range(len(pred1['pts3d'])): + imidx0 = pairs[i][0]['idx'] + imidx1 = pairs[i][1]['idx'] + if 'desc' in pred1: # mast3r + descs = [pred1['desc'][i], pred2['desc'][i]] + confidences = [pred1['desc_conf'][i], pred2['desc_conf'][i]] + desc_dim = descs[0].shape[-1] + + if is_sparse: + corres = extract_correspondences_nonsym(descs[0], descs[1], confidences[0], confidences[1], + device=device, subsample=subsample, pixel_tol=pixel_tol) + conf = corres[2] + mask = conf >= conf_thr + matches_im0 = corres[0][mask].cpu().numpy() + matches_im1 = corres[1][mask].cpu().numpy() + else: + confidence_masks = [confidences[0] >= + conf_thr, confidences[1] >= conf_thr] + pts2d_list, desc_list = [], [] + for j in range(2): + conf_j = confidence_masks[j].cpu().numpy().flatten() + true_shape_j = pairs[i][j]['true_shape'][0] + pts2d_j = xy_grid( + true_shape_j[1], true_shape_j[0]).reshape(-1, 2)[conf_j] + desc_j = descs[j].detach().cpu( + ).numpy().reshape(-1, desc_dim)[conf_j] + pts2d_list.append(pts2d_j) + desc_list.append(desc_j) + if len(desc_list[0]) == 0 or len(desc_list[1]) == 0: + continue + + nn0, nn1 = bruteforce_reciprocal_nns(desc_list[0], desc_list[1], + device=device, dist='dot', block_size=2**13) + reciprocal_in_P0 = (nn1[nn0] == np.arange(len(nn0))) + + matches_im1 = pts2d_list[1][nn0][reciprocal_in_P0] + matches_im0 = pts2d_list[0][reciprocal_in_P0] + else: + pts3d = [pred1['pts3d'][i], pred2['pts3d_in_other_view'][i]] + confidences = [pred1['conf'][i], pred2['conf'][i]] + + if is_sparse: + corres = extract_correspondences_nonsym(pts3d[0], pts3d[1], confidences[0], confidences[1], + device=device, subsample=subsample, pixel_tol=pixel_tol, + ptmap_key='3d') + conf = corres[2] + mask = conf >= conf_thr + matches_im0 = corres[0][mask].cpu().numpy() + matches_im1 = corres[1][mask].cpu().numpy() + else: + confidence_masks = [confidences[0] >= + conf_thr, confidences[1] >= conf_thr] + # find 2D-2D matches between the two images + pts2d_list, pts3d_list = [], [] + for j in range(2): + conf_j = confidence_masks[j].cpu().numpy().flatten() + true_shape_j = pairs[i][j]['true_shape'][0] + pts2d_j = xy_grid(true_shape_j[1], true_shape_j[0]).reshape(-1, 2)[conf_j] + pts3d_j = pts3d[j].detach().cpu().numpy().reshape(-1, 3)[conf_j] + pts2d_list.append(pts2d_j) + pts3d_list.append(pts3d_j) + + PQ, PM = pts3d_list[0], pts3d_list[1] + if len(PQ) == 0 or len(PM) == 0: + continue + reciprocal_in_PM, nnM_in_PQ, num_matches = find_reciprocal_matches( + PQ, PM) + + matches_im1 = pts2d_list[1][reciprocal_in_PM] + matches_im0 = pts2d_list[0][nnM_in_PQ][reciprocal_in_PM] + + if len(matches_im0) == 0: + continue + imidx0, imidx1, colmap_matches = convert_im_matches_pairs(pairs[i][0], pairs[i][1], + image_to_colmap, im_keypoints, + matches_im0, matches_im1, viz) + im_matches[(imidx0, imidx1)] = colmap_matches + return im_matches + + +def get_im_matches_from_cache(pairs, cache_path, desc_conf, subsample, + image_to_colmap, im_keypoints, conf_thr, + viz=False, device='cuda'): + im_matches = {} + for i in range(len(pairs)): + imidx0 = pairs[i][0]['idx'] + imidx1 = pairs[i][1]['idx'] + + corres_idx1 = hash_md5(pairs[i][0]['instance']) + corres_idx2 = hash_md5(pairs[i][1]['instance']) + + path_corres = cache_path + f'/corres_conf={desc_conf}_{subsample=}/{corres_idx1}-{corres_idx2}.pth' + if os.path.isfile(path_corres): + score, (xy1, xy2, confs) = torch.load(path_corres, map_location=device) + else: + path_corres = cache_path + f'/corres_conf={desc_conf}_{subsample=}/{corres_idx2}-{corres_idx1}.pth' + score, (xy2, xy1, confs) = torch.load(path_corres, map_location=device) + mask = confs >= conf_thr + matches_im0 = xy1[mask].cpu().numpy() + matches_im1 = xy2[mask].cpu().numpy() + + if len(matches_im0) == 0: + continue + imidx0, imidx1, colmap_matches = convert_im_matches_pairs(pairs[i][0], pairs[i][1], + image_to_colmap, im_keypoints, + matches_im0, matches_im1, viz) + im_matches[(imidx0, imidx1)] = colmap_matches + return im_matches + + +def export_images(db, images, image_paths, focals, ga_world_to_cam, camera_model): + # add cameras/images to the db + # with the output of ga as prior + image_to_colmap = {} + im_keypoints = {} + for idx in range(len(image_paths)): + im_keypoints[idx] = {} + H, W = images[idx]["orig_shape"] + if focals is None: + focal_x = focal_y = 1.2 * max(W, H) + prior_focal_length = False + cx = W / 2.0 + cy = H / 2.0 + elif isinstance(focals[idx], np.ndarray) and len(focals[idx].shape) == 2: + # intrinsics + focal_x = focals[idx][0, 0] + focal_y = focals[idx][1, 1] + cx = focals[idx][0, 2] * images[idx]["to_orig"][0, 0] + cy = focals[idx][1, 2] * images[idx]["to_orig"][1, 1] + prior_focal_length = True + else: + focal_x = focal_y = float(focals[idx]) + prior_focal_length = True + cx = W / 2.0 + cy = H / 2.0 + focal_x = focal_x * images[idx]["to_orig"][0, 0] + focal_y = focal_y * images[idx]["to_orig"][1, 1] + + if camera_model == "SIMPLE_PINHOLE": + model_id = 0 + focal = (focal_x + focal_y) / 2.0 + params = np.asarray([focal, cx, cy], np.float64) + elif camera_model == "PINHOLE": + model_id = 1 + params = np.asarray([focal_x, focal_y, cx, cy], np.float64) + elif camera_model == "SIMPLE_RADIAL": + model_id = 2 + focal = (focal_x + focal_y) / 2.0 + params = np.asarray([focal, cx, cy, 0.0], np.float64) + elif camera_model == "OPENCV": + model_id = 4 + params = np.asarray([focal_x, focal_y, cx, cy, 0.0, 0.0, 0.0, 0.0], np.float64) + else: + raise ValueError(f"invalid camera model {camera_model}") + + H, W = int(H), int(W) + # OPENCV camera model + camid = db.add_camera( + model_id, W, H, params, prior_focal_length=prior_focal_length) + if ga_world_to_cam is None: + prior_t = np.zeros(3) + prior_q = np.zeros(4) + else: + q = R.from_matrix(ga_world_to_cam[idx][:3, :3]).as_quat() + prior_t = ga_world_to_cam[idx][:3, 3] + prior_q = np.array([q[-1], q[0], q[1], q[2]]) + imid = db.add_image( + image_paths[idx], camid, prior_q=prior_q, prior_t=prior_t) + image_to_colmap[idx] = { + 'colmap_imid': imid, + 'colmap_camid': camid + } + return image_to_colmap, im_keypoints + + +def export_matches(db, images, image_to_colmap, im_keypoints, im_matches, min_len_track, skip_geometric_verification): + colmap_image_pairs = [] + # 2D-2D are quite dense + # we want to remove the very small tracks + # and export only kpt for which we have values + # build tracks + print("building tracks") + keypoints_to_track_id = {} + track_id_to_kpt_list = [] + to_merge = [] + for (imidx0, imidx1), colmap_matches in tqdm(im_matches.items()): + if imidx0 not in keypoints_to_track_id: + keypoints_to_track_id[imidx0] = {} + if imidx1 not in keypoints_to_track_id: + keypoints_to_track_id[imidx1] = {} + + for m in colmap_matches: + if m[0] not in keypoints_to_track_id[imidx0] and m[1] not in keypoints_to_track_id[imidx1]: + # new pair of kpts never seen before + track_idx = len(track_id_to_kpt_list) + keypoints_to_track_id[imidx0][m[0]] = track_idx + keypoints_to_track_id[imidx1][m[1]] = track_idx + track_id_to_kpt_list.append( + [(imidx0, m[0]), (imidx1, m[1])]) + elif m[1] not in keypoints_to_track_id[imidx1]: + # 0 has a track, not 1 + track_idx = keypoints_to_track_id[imidx0][m[0]] + keypoints_to_track_id[imidx1][m[1]] = track_idx + track_id_to_kpt_list[track_idx].append((imidx1, m[1])) + elif m[0] not in keypoints_to_track_id[imidx0]: + # 1 has a track, not 0 + track_idx = keypoints_to_track_id[imidx1][m[1]] + keypoints_to_track_id[imidx0][m[0]] = track_idx + track_id_to_kpt_list[track_idx].append((imidx0, m[0])) + else: + # both have tracks, merge them + track_idx0 = keypoints_to_track_id[imidx0][m[0]] + track_idx1 = keypoints_to_track_id[imidx1][m[1]] + if track_idx0 != track_idx1: + # let's deal with them later + to_merge.append((track_idx0, track_idx1)) + + # regroup merge targets + print("merging tracks") + unique = np.unique(to_merge) + tree = DisjointSet(unique) + for track_idx0, track_idx1 in tqdm(to_merge): + tree.merge(track_idx0, track_idx1) + + subsets = tree.subsets() + print("applying merge") + for setvals in tqdm(subsets): + new_trackid = len(track_id_to_kpt_list) + kpt_list = [] + for track_idx in setvals: + kpt_list.extend(track_id_to_kpt_list[track_idx]) + for imidx, kpid in track_id_to_kpt_list[track_idx]: + keypoints_to_track_id[imidx][kpid] = new_trackid + track_id_to_kpt_list.append(kpt_list) + + # binc = np.bincount([len(v) for v in track_id_to_kpt_list]) + # nonzero = np.nonzero(binc) + # nonzerobinc = binc[nonzero[0]] + # print(nonzero[0].tolist()) + # print(nonzerobinc) + num_valid_tracks = sum( + [1 for v in track_id_to_kpt_list if len(v) >= min_len_track]) + + keypoints_to_idx = {} + print(f"squashing keypoints - {num_valid_tracks} valid tracks") + for imidx, keypoints_imid in tqdm(im_keypoints.items()): + imid = image_to_colmap[imidx]['colmap_imid'] + keypoints_kept = [] + keypoints_to_idx[imidx] = {} + for kp in keypoints_imid.keys(): + if kp not in keypoints_to_track_id[imidx]: + continue + track_idx = keypoints_to_track_id[imidx][kp] + track_length = len(track_id_to_kpt_list[track_idx]) + if track_length < min_len_track: + continue + keypoints_to_idx[imidx][kp] = len(keypoints_kept) + keypoints_kept.append(kp) + if len(keypoints_kept) == 0: + continue + keypoints_kept = np.array(keypoints_kept) + keypoints_kept = np.unravel_index(keypoints_kept, images[imidx]['true_shape'][0])[ + 0].base[:, ::-1].copy().astype(np.float32) + # rescale coordinates + keypoints_kept[:, 0] += 0.5 + keypoints_kept[:, 1] += 0.5 + keypoints_kept = geotrf(images[imidx]['to_orig'], keypoints_kept, norm=True) + + H, W = images[imidx]['orig_shape'] + keypoints_kept[:, 0] = keypoints_kept[:, 0].clip(min=0, max=W - 0.01) + keypoints_kept[:, 1] = keypoints_kept[:, 1].clip(min=0, max=H - 0.01) + + db.add_keypoints(imid, keypoints_kept) + + print("exporting im_matches") + for (imidx0, imidx1), colmap_matches in im_matches.items(): + imid0, imid1 = image_to_colmap[imidx0]['colmap_imid'], image_to_colmap[imidx1]['colmap_imid'] + assert imid0 < imid1 + final_matches = np.array([[keypoints_to_idx[imidx0][m[0]], keypoints_to_idx[imidx1][m[1]]] + for m in colmap_matches + if m[0] in keypoints_to_idx[imidx0] and m[1] in keypoints_to_idx[imidx1]]) + if len(final_matches) > 0: + colmap_image_pairs.append( + (images[imidx0]['instance'], images[imidx1]['instance'])) + db.add_matches(imid0, imid1, final_matches) + if skip_geometric_verification: + db.add_two_view_geometry(imid0, imid1, final_matches) + return colmap_image_pairs diff --git a/src/mast3r_src/mast3r/datasets/__init__.py b/src/mast3r_src/mast3r/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c625aca0a773c105ed229ff87364721b4755bc8d --- /dev/null +++ b/src/mast3r_src/mast3r/datasets/__init__.py @@ -0,0 +1,62 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). + +from .base.mast3r_base_stereo_view_dataset import MASt3RBaseStereoViewDataset + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.datasets.arkitscenes import ARKitScenes as DUSt3R_ARKitScenes # noqa +from dust3r.datasets.blendedmvs import BlendedMVS as DUSt3R_BlendedMVS # noqa +from dust3r.datasets.co3d import Co3d as DUSt3R_Co3d # noqa +from dust3r.datasets.megadepth import MegaDepth as DUSt3R_MegaDepth # noqa +from dust3r.datasets.scannetpp import ScanNetpp as DUSt3R_ScanNetpp # noqa +from dust3r.datasets.staticthings3d import StaticThings3D as DUSt3R_StaticThings3D # noqa +from dust3r.datasets.waymo import Waymo as DUSt3R_Waymo # noqa +from dust3r.datasets.wildrgbd import WildRGBD as DUSt3R_WildRGBD # noqa + + +class ARKitScenes(DUSt3R_ARKitScenes, MASt3RBaseStereoViewDataset): + def __init__(self, *args, split, ROOT, **kwargs): + super().__init__(*args, split=split, ROOT=ROOT, **kwargs) + self.is_metric_scale = True + + +class BlendedMVS(DUSt3R_BlendedMVS, MASt3RBaseStereoViewDataset): + def __init__(self, *args, ROOT, split=None, **kwargs): + super().__init__(*args, ROOT=ROOT, split=split, **kwargs) + self.is_metric_scale = False + + +class Co3d(DUSt3R_Co3d, MASt3RBaseStereoViewDataset): + def __init__(self, mask_bg=True, *args, ROOT, **kwargs): + super().__init__(mask_bg, *args, ROOT=ROOT, **kwargs) + self.is_metric_scale = False + + +class MegaDepth(DUSt3R_MegaDepth, MASt3RBaseStereoViewDataset): + def __init__(self, *args, split, ROOT, **kwargs): + super().__init__(*args, split=split, ROOT=ROOT, **kwargs) + self.is_metric_scale = False + + +class ScanNetpp(DUSt3R_ScanNetpp, MASt3RBaseStereoViewDataset): + def __init__(self, *args, ROOT, **kwargs): + super().__init__(*args, ROOT=ROOT, **kwargs) + self.is_metric_scale = True + + +class StaticThings3D(DUSt3R_StaticThings3D, MASt3RBaseStereoViewDataset): + def __init__(self, ROOT, *args, mask_bg='rand', **kwargs): + super().__init__(ROOT, *args, mask_bg=mask_bg, **kwargs) + self.is_metric_scale = False + + +class Waymo(DUSt3R_Waymo, MASt3RBaseStereoViewDataset): + def __init__(self, *args, ROOT, **kwargs): + super().__init__(*args, ROOT=ROOT, **kwargs) + self.is_metric_scale = True + + +class WildRGBD(DUSt3R_WildRGBD, MASt3RBaseStereoViewDataset): + def __init__(self, mask_bg=True, *args, ROOT, **kwargs): + super().__init__(mask_bg, *args, ROOT=ROOT, **kwargs) + self.is_metric_scale = True diff --git a/src/mast3r_src/mast3r/datasets/base/__init__.py b/src/mast3r_src/mast3r/datasets/base/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d7dd877d649ce4dbd749dd7195a8b34c0f91d4f0 --- /dev/null +++ b/src/mast3r_src/mast3r/datasets/base/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). \ No newline at end of file diff --git a/src/mast3r_src/mast3r/datasets/base/mast3r_base_stereo_view_dataset.py b/src/mast3r_src/mast3r/datasets/base/mast3r_base_stereo_view_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..3ced0ef0dc6b1d6225781af55d3e924e133fdeaf --- /dev/null +++ b/src/mast3r_src/mast3r/datasets/base/mast3r_base_stereo_view_dataset.py @@ -0,0 +1,355 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# base class for implementing datasets +# -------------------------------------------------------- +import PIL.Image +import PIL.Image as Image +import numpy as np +import torch +import copy + +from mast3r.datasets.utils.cropping import (extract_correspondences_from_pts3d, + gen_random_crops, in2d_rect, crop_to_homography) + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.datasets.base.base_stereo_view_dataset import BaseStereoViewDataset, view_name, is_good_type # noqa +from dust3r.datasets.utils.transforms import ImgNorm +from dust3r.utils.geometry import depthmap_to_absolute_camera_coordinates, geotrf, depthmap_to_camera_coordinates +import dust3r.datasets.utils.cropping as cropping + + +class MASt3RBaseStereoViewDataset(BaseStereoViewDataset): + def __init__(self, *, # only keyword arguments + split=None, + resolution=None, # square_size or (width, height) or list of [(width,height), ...] + transform=ImgNorm, + aug_crop=False, + aug_swap=False, + aug_monocular=False, + aug_portrait_or_landscape=True, # automatic choice between landscape/portrait when possible + aug_rot90=False, + n_corres=0, + nneg=0, + n_tentative_crops=4, + seed=None): + super().__init__(split=split, resolution=resolution, transform=transform, aug_crop=aug_crop, seed=seed) + self.is_metric_scale = False # by default a dataset is not metric scale, subclasses can overwrite this + + self.aug_swap = aug_swap + self.aug_monocular = aug_monocular + self.aug_portrait_or_landscape = aug_portrait_or_landscape + self.aug_rot90 = aug_rot90 + + self.n_corres = n_corres + self.nneg = nneg + assert self.n_corres == 'all' or isinstance(self.n_corres, int) or (isinstance(self.n_corres, list) and len( + self.n_corres) == self.num_views), f"Error, n_corres should either be 'all', a single integer or a list of length {self.num_views}" + assert self.nneg == 0 or self.n_corres != 'all' + self.n_tentative_crops = n_tentative_crops + + def _swap_view_aug(self, views): + if self._rng.random() < 0.5: + views.reverse() + + def _crop_resize_if_necessary(self, image, depthmap, intrinsics, resolution, rng=None, info=None): + """ This function: + - first downsizes the image with LANCZOS inteprolation, + which is better than bilinear interpolation in + """ + if not isinstance(image, PIL.Image.Image): + image = PIL.Image.fromarray(image) + + # transpose the resolution if necessary + W, H = image.size # new size + assert resolution[0] >= resolution[1] + if H > 1.1 * W: + # image is portrait mode + resolution = resolution[::-1] + elif 0.9 < H / W < 1.1 and resolution[0] != resolution[1]: + # image is square, so we chose (portrait, landscape) randomly + if rng.integers(2) and self.aug_portrait_or_landscape: + resolution = resolution[::-1] + + # high-quality Lanczos down-scaling + target_resolution = np.array(resolution) + image, depthmap, intrinsics = cropping.rescale_image_depthmap(image, depthmap, intrinsics, target_resolution) + + # actual cropping (if necessary) with bilinear interpolation + offset_factor = 0.5 + intrinsics2 = cropping.camera_matrix_of_crop(intrinsics, image.size, resolution, offset_factor=offset_factor) + crop_bbox = cropping.bbox_from_intrinsics_in_out(intrinsics, intrinsics2, resolution) + image, depthmap, intrinsics2 = cropping.crop_image_depthmap(image, depthmap, intrinsics, crop_bbox) + + return image, depthmap, intrinsics2 + + def generate_crops_from_pair(self, view1, view2, resolution, aug_crop_arg, n_crops=4, rng=np.random): + views = [view1, view2] + + if aug_crop_arg is False: + # compatibility + for i in range(2): + view = views[i] + view['img'], view['depthmap'], view['camera_intrinsics'] = self._crop_resize_if_necessary(view['img'], + view['depthmap'], + view['camera_intrinsics'], + resolution, + rng=rng) + view['pts3d'], view['valid_mask'] = depthmap_to_absolute_camera_coordinates(view['depthmap'], + view['camera_intrinsics'], + view['camera_pose']) + return + + # extract correspondences + corres = extract_correspondences_from_pts3d(*views, target_n_corres=None, rng=rng) + + # generate 4 random crops in each view + view_crops = [] + crops_resolution = [] + corres_msks = [] + for i in range(2): + + if aug_crop_arg == 'auto': + S = min(views[i]['img'].size) + R = min(resolution) + aug_crop = S * (S - R) // R + aug_crop = max(.1 * S, aug_crop) # for cropping: augment scale of at least 10%, and more if possible + else: + aug_crop = aug_crop_arg + + # tranpose the target resolution if necessary + assert resolution[0] >= resolution[1] + W, H = imsize = views[i]['img'].size + crop_resolution = resolution + if H > 1.1 * W: + # image is portrait mode + crop_resolution = resolution[::-1] + elif 0.9 < H / W < 1.1 and resolution[0] != resolution[1]: + # image is square, so we chose (portrait, landscape) randomly + if rng.integers(2): + crop_resolution = resolution[::-1] + + crops = gen_random_crops(imsize, n_crops, crop_resolution, aug_crop=aug_crop, rng=rng) + view_crops.append(crops) + crops_resolution.append(crop_resolution) + + # compute correspondences + corres_msks.append(in2d_rect(corres[i], crops)) + + # compute IoU for each + intersection = np.float32(corres_msks[0]).T @ np.float32(corres_msks[1]) + # select best pair of crops + best = np.unravel_index(intersection.argmax(), (n_crops, n_crops)) + crops = [view_crops[i][c] for i, c in enumerate(best)] + + # crop with the homography + for i in range(2): + view = views[i] + imsize, K_new, R, H = crop_to_homography(view['camera_intrinsics'], crops[i], crops_resolution[i]) + # imsize, K_new, H = upscale_homography(imsize, resolution, K_new, H) + + # update camera params + K_old = view['camera_intrinsics'] + view['camera_intrinsics'] = K_new + view['camera_pose'] = view['camera_pose'].copy() + view['camera_pose'][:3, :3] = view['camera_pose'][:3, :3] @ R + + # apply homography to image and depthmap + homo8 = (H / H[2, 2]).ravel().tolist()[:8] + view['img'] = view['img'].transform(imsize, Image.Transform.PERSPECTIVE, + homo8, + resample=Image.Resampling.BICUBIC) + + depthmap2 = depthmap_to_camera_coordinates(view['depthmap'], K_old)[0] @ R[:, 2] + view['depthmap'] = np.array(Image.fromarray(depthmap2).transform( + imsize, Image.Transform.PERSPECTIVE, homo8)) + + if 'track_labels' in view: + # convert from uint64 --> uint32, because PIL.Image cannot handle uint64 + mapping, track_labels = np.unique(view['track_labels'], return_inverse=True) + track_labels = track_labels.astype(np.uint32).reshape(view['track_labels'].shape) + + # homography transformation + res = np.array(Image.fromarray(track_labels).transform(imsize, Image.Transform.PERSPECTIVE, homo8)) + view['track_labels'] = mapping[res] # mapping back to uint64 + + # recompute 3d points from scratch + view['pts3d'], view['valid_mask'] = depthmap_to_absolute_camera_coordinates(view['depthmap'], + view['camera_intrinsics'], + view['camera_pose']) + + def __getitem__(self, idx): + if isinstance(idx, tuple): + # the idx is specifying the aspect-ratio + idx, ar_idx = idx + else: + assert len(self._resolutions) == 1 + ar_idx = 0 + + # set-up the rng + if self.seed: # reseed for each __getitem__ + self._rng = np.random.default_rng(seed=self.seed + idx) + elif not hasattr(self, '_rng'): + seed = torch.initial_seed() # this is different for each dataloader process + self._rng = np.random.default_rng(seed=seed) + + # over-loaded code + resolution = self._resolutions[ar_idx] # DO NOT CHANGE THIS (compatible with BatchedRandomSampler) + views = self._get_views(idx, resolution, self._rng) + assert len(views) == self.num_views + + for v, view in enumerate(views): + assert 'pts3d' not in view, f"pts3d should not be there, they will be computed afterwards based on intrinsics+depthmap for view {view_name(view)}" + view['idx'] = (idx, ar_idx, v) + view['is_metric_scale'] = self.is_metric_scale + + assert 'camera_intrinsics' in view + if 'camera_pose' not in view: + view['camera_pose'] = np.full((4, 4), np.nan, dtype=np.float32) + else: + assert np.isfinite(view['camera_pose']).all(), f'NaN in camera pose for view {view_name(view)}' + assert 'pts3d' not in view + assert 'valid_mask' not in view + assert np.isfinite(view['depthmap']).all(), f'NaN in depthmap for view {view_name(view)}' + + pts3d, valid_mask = depthmap_to_absolute_camera_coordinates(**view) + + view['pts3d'] = pts3d + view['valid_mask'] = valid_mask & np.isfinite(pts3d).all(axis=-1) + + self.generate_crops_from_pair(views[0], views[1], resolution=resolution, + aug_crop_arg=self.aug_crop, + n_crops=self.n_tentative_crops, + rng=self._rng) + for v, view in enumerate(views): + # encode the image + width, height = view['img'].size + view['true_shape'] = np.int32((height, width)) + view['img'] = self.transform(view['img']) + # Pixels for which depth is fundamentally undefined + view['sky_mask'] = (view['depthmap'] < 0) + + if self.aug_swap: + self._swap_view_aug(views) + + if self.aug_monocular: + if self._rng.random() < self.aug_monocular: + views = [copy.deepcopy(views[0]) for _ in range(len(views))] + + # automatic extraction of correspondences from pts3d + pose + if self.n_corres > 0 and ('corres' not in view): + corres1, corres2, valid = extract_correspondences_from_pts3d(*views, self.n_corres, + self._rng, nneg=self.nneg) + views[0]['corres'] = corres1 + views[1]['corres'] = corres2 + views[0]['valid_corres'] = valid + views[1]['valid_corres'] = valid + + if self.aug_rot90 is False: + pass + elif self.aug_rot90 == 'same': + rotate_90(views, k=self._rng.choice(4)) + elif self.aug_rot90 == 'diff': + rotate_90(views[:1], k=self._rng.choice(4)) + rotate_90(views[1:], k=self._rng.choice(4)) + else: + raise ValueError(f'Bad value for {self.aug_rot90=}') + + # check data-types metric_scale + for v, view in enumerate(views): + if 'corres' not in view: + view['corres'] = np.full((self.n_corres, 2), np.nan, dtype=np.float32) + + # check all datatypes + for key, val in view.items(): + res, err_msg = is_good_type(key, val) + assert res, f"{err_msg} with {key}={val} for view {view_name(view)}" + K = view['camera_intrinsics'] + + # check shapes + assert view['depthmap'].shape == view['img'].shape[1:] + assert view['depthmap'].shape == view['pts3d'].shape[:2] + assert view['depthmap'].shape == view['valid_mask'].shape + + # last thing done! + for view in views: + # transpose to make sure all views are the same size + transpose_to_landscape(view) + # this allows to check whether the RNG is is the same state each time + view['rng'] = int.from_bytes(self._rng.bytes(4), 'big') + + return views + + +def transpose_to_landscape(view, revert=False): + height, width = view['true_shape'] + + if width < height: + if revert: + height, width = width, height + + # rectify portrait to landscape + assert view['img'].shape == (3, height, width) + view['img'] = view['img'].swapaxes(1, 2) + + assert view['valid_mask'].shape == (height, width) + view['valid_mask'] = view['valid_mask'].swapaxes(0, 1) + + assert view['sky_mask'].shape == (height, width) + view['sky_mask'] = view['sky_mask'].swapaxes(0, 1) + + assert view['depthmap'].shape == (height, width) + view['depthmap'] = view['depthmap'].swapaxes(0, 1) + + assert view['pts3d'].shape == (height, width, 3) + view['pts3d'] = view['pts3d'].swapaxes(0, 1) + + # transpose x and y pixels + view['camera_intrinsics'] = view['camera_intrinsics'][[1, 0, 2]] + + # transpose correspondences x and y + view['corres'] = view['corres'][:, [1, 0]] + + +def rotate_90(views, k=1): + from scipy.spatial.transform import Rotation + # print('rotation =', k) + + RT = np.eye(4, dtype=np.float32) + RT[:3, :3] = Rotation.from_euler('z', 90 * k, degrees=True).as_matrix() + + for view in views: + view['img'] = torch.rot90(view['img'], k=k, dims=(-2, -1)) # WARNING!! dims=(-1,-2) != dims=(-2,-1) + view['depthmap'] = np.rot90(view['depthmap'], k=k).copy() + view['camera_pose'] = view['camera_pose'] @ RT + + RT2 = np.eye(3, dtype=np.float32) + RT2[:2, :2] = RT[:2, :2] * ((1, -1), (-1, 1)) + H, W = view['depthmap'].shape + if k % 4 == 0: + pass + elif k % 4 == 1: + # top-left (0,0) pixel becomes (0,H-1) + RT2[:2, 2] = (0, H - 1) + elif k % 4 == 2: + # top-left (0,0) pixel becomes (W-1,H-1) + RT2[:2, 2] = (W - 1, H - 1) + elif k % 4 == 3: + # top-left (0,0) pixel becomes (W-1,0) + RT2[:2, 2] = (W - 1, 0) + else: + raise ValueError(f'Bad value for {k=}') + + view['camera_intrinsics'][:2, 2] = geotrf(RT2, view['camera_intrinsics'][:2, 2]) + if k % 2 == 1: + K = view['camera_intrinsics'] + np.fill_diagonal(K, K.diagonal()[[1, 0, 2]]) + + pts3d, valid_mask = depthmap_to_absolute_camera_coordinates(**view) + view['pts3d'] = pts3d + view['valid_mask'] = np.rot90(view['valid_mask'], k=k).copy() + view['sky_mask'] = np.rot90(view['sky_mask'], k=k).copy() + + view['corres'] = geotrf(RT2, view['corres']).round().astype(view['corres'].dtype) + view['true_shape'] = np.int32((H, W)) diff --git a/src/mast3r_src/mast3r/datasets/utils/__init__.py b/src/mast3r_src/mast3r/datasets/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a32692113d830ddc4af4e6ed608f222fbe062e6e --- /dev/null +++ b/src/mast3r_src/mast3r/datasets/utils/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). diff --git a/src/mast3r_src/mast3r/datasets/utils/cropping.py b/src/mast3r_src/mast3r/datasets/utils/cropping.py new file mode 100644 index 0000000000000000000000000000000000000000..57f4d84b019eaac9cf0c308a94f2cb8e2ec1a6ba --- /dev/null +++ b/src/mast3r_src/mast3r/datasets/utils/cropping.py @@ -0,0 +1,219 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# cropping/match extraction +# -------------------------------------------------------- +import numpy as np +import mast3r.utils.path_to_dust3r # noqa +from dust3r.utils.device import to_numpy +from dust3r.utils.geometry import inv, geotrf + + +def reciprocal_1d(corres_1_to_2, corres_2_to_1, ret_recip=False): + is_reciprocal1 = (corres_2_to_1[corres_1_to_2] == np.arange(len(corres_1_to_2))) + pos1 = is_reciprocal1.nonzero()[0] + pos2 = corres_1_to_2[pos1] + if ret_recip: + return is_reciprocal1, pos1, pos2 + return pos1, pos2 + + +def extract_correspondences_from_pts3d(view1, view2, target_n_corres, rng=np.random, ret_xy=True, nneg=0): + view1, view2 = to_numpy((view1, view2)) + # project pixels from image1 --> 3d points --> image2 pixels + shape1, corres1_to_2 = reproject_view(view1['pts3d'], view2) + shape2, corres2_to_1 = reproject_view(view2['pts3d'], view1) + + # compute reciprocal correspondences: + # pos1 == valid pixels (correspondences) in image1 + is_reciprocal1, pos1, pos2 = reciprocal_1d(corres1_to_2, corres2_to_1, ret_recip=True) + is_reciprocal2 = (corres1_to_2[corres2_to_1] == np.arange(len(corres2_to_1))) + + if target_n_corres is None: + if ret_xy: + pos1 = unravel_xy(pos1, shape1) + pos2 = unravel_xy(pos2, shape2) + return pos1, pos2 + + available_negatives = min((~is_reciprocal1).sum(), (~is_reciprocal2).sum()) + target_n_positives = int(target_n_corres * (1 - nneg)) + n_positives = min(len(pos1), target_n_positives) + n_negatives = min(target_n_corres - n_positives, available_negatives) + + if n_negatives + n_positives != target_n_corres: + # should be really rare => when there are not enough negatives + # in that case, break nneg and add a few more positives ? + n_positives = target_n_corres - n_negatives + assert n_positives <= len(pos1) + + assert n_positives <= len(pos1) + assert n_positives <= len(pos2) + assert n_negatives <= (~is_reciprocal1).sum() + assert n_negatives <= (~is_reciprocal2).sum() + assert n_positives + n_negatives == target_n_corres + + valid = np.ones(n_positives, dtype=bool) + if n_positives < len(pos1): + # random sub-sampling of valid correspondences + perm = rng.permutation(len(pos1))[:n_positives] + pos1 = pos1[perm] + pos2 = pos2[perm] + + if n_negatives > 0: + # add false correspondences if not enough + def norm(p): return p / p.sum() + pos1 = np.r_[pos1, rng.choice(shape1[0] * shape1[1], size=n_negatives, replace=False, p=norm(~is_reciprocal1))] + pos2 = np.r_[pos2, rng.choice(shape2[0] * shape2[1], size=n_negatives, replace=False, p=norm(~is_reciprocal2))] + valid = np.r_[valid, np.zeros(n_negatives, dtype=bool)] + + # convert (x+W*y) back to 2d (x,y) coordinates + if ret_xy: + pos1 = unravel_xy(pos1, shape1) + pos2 = unravel_xy(pos2, shape2) + return pos1, pos2, valid + + +def reproject_view(pts3d, view2): + shape = view2['pts3d'].shape[:2] + return reproject(pts3d, view2['camera_intrinsics'], inv(view2['camera_pose']), shape) + + +def reproject(pts3d, K, world2cam, shape): + H, W, THREE = pts3d.shape + assert THREE == 3 + + # reproject in camera2 space + with np.errstate(divide='ignore', invalid='ignore'): + pos = geotrf(K @ world2cam[:3], pts3d, norm=1, ncol=2) + + # quantize to pixel positions + return (H, W), ravel_xy(pos, shape) + + +def ravel_xy(pos, shape): + H, W = shape + with np.errstate(invalid='ignore'): + qx, qy = pos.reshape(-1, 2).round().astype(np.int32).T + quantized_pos = qx.clip(min=0, max=W - 1, out=qx) + W * qy.clip(min=0, max=H - 1, out=qy) + return quantized_pos + + +def unravel_xy(pos, shape): + # convert (x+W*y) back to 2d (x,y) coordinates + return np.unravel_index(pos, shape)[0].base[:, ::-1].copy() + + +def _rotation_origin_to_pt(target): + """ Align the origin (0,0,1) with the target point (x,y,1) in projective space. + Method: rotate z to put target on (x'+,0,1), then rotate on Y to get (0,0,1) and un-rotate z. + """ + from scipy.spatial.transform import Rotation + x, y = target + rot_z = np.arctan2(y, x) + rot_y = np.arctan(np.linalg.norm(target)) + R = Rotation.from_euler('ZYZ', [rot_z, rot_y, -rot_z]).as_matrix() + return R + + +def _dotmv(Trf, pts, ncol=None, norm=False): + assert Trf.ndim >= 2 + ncol = ncol or pts.shape[-1] + + # adapt shape if necessary + output_reshape = pts.shape[:-1] + if Trf.ndim >= 3: + n = Trf.ndim - 2 + assert Trf.shape[:n] == pts.shape[:n], 'batch size does not match' + Trf = Trf.reshape(-1, Trf.shape[-2], Trf.shape[-1]) + + if pts.ndim > Trf.ndim: + # Trf == (B,d,d) & pts == (B,H,W,d) --> (B, H*W, d) + pts = pts.reshape(Trf.shape[0], -1, pts.shape[-1]) + elif pts.ndim == 2: + # Trf == (B,d,d) & pts == (B,d) --> (B, 1, d) + pts = pts[:, None, :] + + if pts.shape[-1] + 1 == Trf.shape[-1]: + Trf = Trf.swapaxes(-1, -2) # transpose Trf + pts = pts @ Trf[..., :-1, :] + Trf[..., -1:, :] + + elif pts.shape[-1] == Trf.shape[-1]: + Trf = Trf.swapaxes(-1, -2) # transpose Trf + pts = pts @ Trf + else: + pts = Trf @ pts.T + if pts.ndim >= 2: + pts = pts.swapaxes(-1, -2) + + if norm: + pts = pts / pts[..., -1:] # DONT DO /= BECAUSE OF WEIRD PYTORCH BUG + if norm != 1: + pts *= norm + + res = pts[..., :ncol].reshape(*output_reshape, ncol) + return res + + +def crop_to_homography(K, crop, target_size=None): + """ Given an image and its intrinsics, + we want to replicate a rectangular crop with an homography, + so that the principal point of the new 'crop' is centered. + """ + # build intrinsics for the crop + crop = np.round(crop) + crop_size = crop[2:] - crop[:2] + K2 = K.copy() # same focal + K2[:2, 2] = crop_size / 2 # new principal point is perfectly centered + + # find which corner is the most far-away from current principal point + # so that the final homography does not go over the image borders + corners = crop.reshape(-1, 2) + corner_idx = np.abs(corners - K[:2, 2]).argmax(0) + corner = corners[corner_idx, [0, 1]] + # align with the corresponding corner from the target view + corner2 = np.c_[[0, 0], crop_size][[0, 1], corner_idx] + + old_pt = _dotmv(np.linalg.inv(K), corner, norm=1) + new_pt = _dotmv(np.linalg.inv(K2), corner2, norm=1) + R = _rotation_origin_to_pt(old_pt) @ np.linalg.inv(_rotation_origin_to_pt(new_pt)) + + if target_size is not None: + imsize = target_size + target_size = np.asarray(target_size) + scaling = min(target_size / crop_size) + K2[:2] *= scaling + K2[:2, 2] = target_size / 2 + else: + imsize = tuple(np.int32(crop_size).tolist()) + + return imsize, K2, R, K @ R @ np.linalg.inv(K2) + + +def gen_random_crops(imsize, n_crops, resolution, aug_crop, rng=np.random): + """ Generate random crops of size=resolution, + for an input image upscaled to (imsize + randint(0 , aug_crop)) + """ + resolution_crop = np.array(resolution) * min(np.array(imsize) / resolution) + + # (virtually) upscale the input image + # scaling = rng.uniform(1, 1+(aug_crop+1)/min(imsize)) + scaling = np.exp(rng.uniform(0, np.log(1 + aug_crop / min(imsize)))) + imsize2 = np.int32(np.array(imsize) * scaling) + + # generate some random crops + topleft = rng.random((n_crops, 2)) * (imsize2 - resolution_crop) + crops = np.c_[topleft, topleft + resolution_crop] + # print(f"{scaling=}, {topleft=}") + # reduce the resolution to come back to original size + crops /= scaling + return crops + + +def in2d_rect(corres, crops): + # corres = (N,2) + # crops = (M,4) + # output = (N, M) + is_sup = (corres[:, None] >= crops[None, :, 0:2]) + is_inf = (corres[:, None] < crops[None, :, 2:4]) + return (is_sup & is_inf).all(axis=-1) diff --git a/src/mast3r_src/mast3r/fast_nn.py b/src/mast3r_src/mast3r/fast_nn.py new file mode 100644 index 0000000000000000000000000000000000000000..05537f43c1be10b3733e80def8295c2ff5b5b8c0 --- /dev/null +++ b/src/mast3r_src/mast3r/fast_nn.py @@ -0,0 +1,223 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# MASt3R Fast Nearest Neighbor +# -------------------------------------------------------- +import torch +import numpy as np +import math +from scipy.spatial import KDTree + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.utils.device import to_numpy, todevice # noqa + + +@torch.no_grad() +def bruteforce_reciprocal_nns(A, B, device='cuda', block_size=None, dist='l2'): + if isinstance(A, np.ndarray): + A = torch.from_numpy(A).to(device) + if isinstance(B, np.ndarray): + B = torch.from_numpy(B).to(device) + + A = A.to(device) + B = B.to(device) + + if dist == 'l2': + dist_func = torch.cdist + argmin = torch.min + elif dist == 'dot': + def dist_func(A, B): + return A @ B.T + + def argmin(X, dim): + sim, nn = torch.max(X, dim=dim) + return sim.neg_(), nn + else: + raise ValueError(f'Unknown {dist=}') + + if block_size is None or len(A) * len(B) <= block_size**2: + dists = dist_func(A, B) + _, nn_A = argmin(dists, dim=1) + _, nn_B = argmin(dists, dim=0) + else: + dis_A = torch.full((A.shape[0],), float('inf'), device=device, dtype=A.dtype) + dis_B = torch.full((B.shape[0],), float('inf'), device=device, dtype=B.dtype) + nn_A = torch.full((A.shape[0],), -1, device=device, dtype=torch.int64) + nn_B = torch.full((B.shape[0],), -1, device=device, dtype=torch.int64) + number_of_iteration_A = math.ceil(A.shape[0] / block_size) + number_of_iteration_B = math.ceil(B.shape[0] / block_size) + + for i in range(number_of_iteration_A): + A_i = A[i * block_size:(i + 1) * block_size] + for j in range(number_of_iteration_B): + B_j = B[j * block_size:(j + 1) * block_size] + dists_blk = dist_func(A_i, B_j) # A, B, 1 + # dists_blk = dists[i * block_size:(i+1)*block_size, j * block_size:(j+1)*block_size] + min_A_i, argmin_A_i = argmin(dists_blk, dim=1) + min_B_j, argmin_B_j = argmin(dists_blk, dim=0) + + col_mask = min_A_i < dis_A[i * block_size:(i + 1) * block_size] + line_mask = min_B_j < dis_B[j * block_size:(j + 1) * block_size] + + dis_A[i * block_size:(i + 1) * block_size][col_mask] = min_A_i[col_mask] + dis_B[j * block_size:(j + 1) * block_size][line_mask] = min_B_j[line_mask] + + nn_A[i * block_size:(i + 1) * block_size][col_mask] = argmin_A_i[col_mask] + (j * block_size) + nn_B[j * block_size:(j + 1) * block_size][line_mask] = argmin_B_j[line_mask] + (i * block_size) + nn_A = nn_A.cpu().numpy() + nn_B = nn_B.cpu().numpy() + return nn_A, nn_B + + +class cdistMatcher: + def __init__(self, db_pts, device='cuda'): + self.db_pts = db_pts.to(device) + self.device = device + + def query(self, queries, k=1, **kw): + assert k == 1 + if queries.numel() == 0: + return None, [] + nnA, nnB = bruteforce_reciprocal_nns(queries, self.db_pts, device=self.device, **kw) + dis = None + return dis, nnA + + +def merge_corres(idx1, idx2, shape1=None, shape2=None, ret_xy=True, ret_index=False): + assert idx1.dtype == idx2.dtype == np.int32 + + # unique and sort along idx1 + corres = np.unique(np.c_[idx2, idx1].view(np.int64), return_index=ret_index) + if ret_index: + corres, indices = corres + xy2, xy1 = corres[:, None].view(np.int32).T + + if ret_xy: + assert shape1 and shape2 + xy1 = np.unravel_index(xy1, shape1) + xy2 = np.unravel_index(xy2, shape2) + if ret_xy != 'y_x': + xy1 = xy1[0].base[:, ::-1] + xy2 = xy2[0].base[:, ::-1] + + if ret_index: + return xy1, xy2, indices + return xy1, xy2 + + +def fast_reciprocal_NNs(pts1, pts2, subsample_or_initxy1=8, ret_xy=True, pixel_tol=0, ret_basin=False, + device='cuda', **matcher_kw): + H1, W1, DIM1 = pts1.shape + H2, W2, DIM2 = pts2.shape + assert DIM1 == DIM2 + + pts1 = pts1.reshape(-1, DIM1) + pts2 = pts2.reshape(-1, DIM2) + + if isinstance(subsample_or_initxy1, int) and pixel_tol == 0: + S = subsample_or_initxy1 + y1, x1 = np.mgrid[S // 2:H1:S, S // 2:W1:S].reshape(2, -1) + max_iter = 10 + else: + x1, y1 = subsample_or_initxy1 + if isinstance(x1, torch.Tensor): + x1 = x1.cpu().numpy() + if isinstance(y1, torch.Tensor): + y1 = y1.cpu().numpy() + max_iter = 1 + + xy1 = np.int32(np.unique(x1 + W1 * y1)) # make sure there's no doublons + xy2 = np.full_like(xy1, -1) + old_xy1 = xy1.copy() + old_xy2 = xy2.copy() + + if 'dist' in matcher_kw or 'block_size' in matcher_kw \ + or (isinstance(device, str) and device.startswith('cuda')) \ + or (isinstance(device, torch.device) and device.type.startswith('cuda')): + pts1 = pts1.to(device) + pts2 = pts2.to(device) + tree1 = cdistMatcher(pts1, device=device) + tree2 = cdistMatcher(pts2, device=device) + else: + pts1, pts2 = to_numpy((pts1, pts2)) + tree1 = KDTree(pts1) + tree2 = KDTree(pts2) + + notyet = np.ones(len(xy1), dtype=bool) + basin = np.full((H1 * W1 + 1,), -1, dtype=np.int32) if ret_basin else None + + niter = 0 + # n_notyet = [len(notyet)] + while notyet.any(): + _, xy2[notyet] = to_numpy(tree2.query(pts1[xy1[notyet]], **matcher_kw)) + if not ret_basin: + notyet &= (old_xy2 != xy2) # remove points that have converged + + _, xy1[notyet] = to_numpy(tree1.query(pts2[xy2[notyet]], **matcher_kw)) + if ret_basin: + basin[old_xy1[notyet]] = xy1[notyet] + notyet &= (old_xy1 != xy1) # remove points that have converged + + # n_notyet.append(notyet.sum()) + niter += 1 + if niter >= max_iter: + break + + old_xy2[:] = xy2 + old_xy1[:] = xy1 + + # print('notyet_stats:', ' '.join(map(str, (n_notyet+[0]*10)[:max_iter]))) + + if pixel_tol > 0: + # in case we only want to match some specific points + # and still have some way of checking reciprocity + old_yx1 = np.unravel_index(old_xy1, (H1, W1))[0].base + new_yx1 = np.unravel_index(xy1, (H1, W1))[0].base + dis = np.linalg.norm(old_yx1 - new_yx1, axis=-1) + converged = dis < pixel_tol + if not isinstance(subsample_or_initxy1, int): + xy1 = old_xy1 # replace new points by old ones + else: + converged = ~notyet # converged correspondences + + # keep only unique correspondences, and sort on xy1 + xy1, xy2 = merge_corres(xy1[converged], xy2[converged], (H1, W1), (H2, W2), ret_xy=ret_xy) + if ret_basin: + return xy1, xy2, basin + return xy1, xy2 + + +def extract_correspondences_nonsym(A, B, confA, confB, subsample=8, device=None, ptmap_key='pred_desc', pixel_tol=0): + if '3d' in ptmap_key: + opt = dict(device='cpu', workers=32) + else: + opt = dict(device=device, dist='dot', block_size=2**13) + + # matching the two pairs + idx1 = [] + idx2 = [] + # merge corres from opposite pairs + HA, WA = A.shape[:2] + HB, WB = B.shape[:2] + if pixel_tol == 0: + nn1to2 = fast_reciprocal_NNs(A, B, subsample_or_initxy1=subsample, ret_xy=False, **opt) + nn2to1 = fast_reciprocal_NNs(B, A, subsample_or_initxy1=subsample, ret_xy=False, **opt) + else: + S = subsample + yA, xA = np.mgrid[S // 2:HA:S, S // 2:WA:S].reshape(2, -1) + yB, xB = np.mgrid[S // 2:HB:S, S // 2:WB:S].reshape(2, -1) + + nn1to2 = fast_reciprocal_NNs(A, B, subsample_or_initxy1=(xA, yA), ret_xy=False, pixel_tol=pixel_tol, **opt) + nn2to1 = fast_reciprocal_NNs(B, A, subsample_or_initxy1=(xB, yB), ret_xy=False, pixel_tol=pixel_tol, **opt) + + idx1 = np.r_[nn1to2[0], nn2to1[1]] + idx2 = np.r_[nn1to2[1], nn2to1[0]] + + c1 = confA.ravel()[idx1] + c2 = confB.ravel()[idx2] + + xy1, xy2, idx = merge_corres(idx1, idx2, (HA, WA), (HB, WB), ret_xy=True, ret_index=True) + conf = np.minimum(c1[idx], c2[idx]) + corres = (xy1.copy(), xy2.copy(), conf) + return todevice(corres, device) diff --git a/src/mast3r_src/mast3r/losses.py b/src/mast3r_src/mast3r/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..5e13530b7cedecf467342a4291186a2e173d60ec --- /dev/null +++ b/src/mast3r_src/mast3r/losses.py @@ -0,0 +1,514 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Implementation of MASt3R training losses +# -------------------------------------------------------- +import torch +import torch.nn as nn +import numpy as np +from sklearn.metrics import average_precision_score + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.losses import BaseCriterion, Criterion, MultiLoss, Sum, ConfLoss +from dust3r.losses import Regr3D as Regr3D_dust3r +from dust3r.utils.geometry import (geotrf, inv, normalize_pointcloud) +from dust3r.inference import get_pred_pts3d +from dust3r.utils.geometry import get_joint_pointcloud_depth, get_joint_pointcloud_center_scale + + +def apply_log_to_norm(xyz): + d = xyz.norm(dim=-1, keepdim=True) + xyz = xyz / d.clip(min=1e-8) + xyz = xyz * torch.log1p(d) + return xyz + + +class Regr3D (Regr3D_dust3r): + def __init__(self, criterion, norm_mode='avg_dis', gt_scale=False, opt_fit_gt=False, + sky_loss_value=2, max_metric_scale=False, loss_in_log=False): + self.loss_in_log = loss_in_log + if norm_mode.startswith('?'): + # do no norm pts from metric scale datasets + self.norm_all = False + self.norm_mode = norm_mode[1:] + else: + self.norm_all = True + self.norm_mode = norm_mode + super().__init__(criterion, self.norm_mode, gt_scale) + + self.sky_loss_value = sky_loss_value + self.max_metric_scale = max_metric_scale + + def get_all_pts3d(self, gt1, gt2, pred1, pred2, dist_clip=None): + # everything is normalized w.r.t. camera of view1 + in_camera1 = inv(gt1['camera_pose']) + gt_pts1 = geotrf(in_camera1, gt1['pts3d']) # B,H,W,3 + gt_pts2 = geotrf(in_camera1, gt2['pts3d']) # B,H,W,3 + + valid1 = gt1['valid_mask'].clone() + valid2 = gt2['valid_mask'].clone() + + if dist_clip is not None: + # points that are too far-away == invalid + dis1 = gt_pts1.norm(dim=-1) # (B, H, W) + dis2 = gt_pts2.norm(dim=-1) # (B, H, W) + valid1 = valid1 & (dis1 <= dist_clip) + valid2 = valid2 & (dis2 <= dist_clip) + + if self.loss_in_log == 'before': + # this only make sense when depth_mode == 'linear' + gt_pts1 = apply_log_to_norm(gt_pts1) + gt_pts2 = apply_log_to_norm(gt_pts2) + + pr_pts1 = get_pred_pts3d(gt1, pred1, use_pose=False).clone() + pr_pts2 = get_pred_pts3d(gt2, pred2, use_pose=True).clone() + + if not self.norm_all: + if self.max_metric_scale: + B = valid1.shape[0] + # valid1: B, H, W + # torch.linalg.norm(gt_pts1, dim=-1) -> B, H, W + # dist1_to_cam1 -> reshape to B, H*W + dist1_to_cam1 = torch.where(valid1, torch.linalg.norm(gt_pts1, dim=-1), 0).view(B, -1) + dist2_to_cam1 = torch.where(valid2, torch.linalg.norm(gt_pts2, dim=-1), 0).view(B, -1) + + # is_metric_scale: B + # dist1_to_cam1.max(dim=-1).values -> B + gt1['is_metric_scale'] = gt1['is_metric_scale'] \ + & (dist1_to_cam1.max(dim=-1).values < self.max_metric_scale) \ + & (dist2_to_cam1.max(dim=-1).values < self.max_metric_scale) + gt2['is_metric_scale'] = gt1['is_metric_scale'] + + mask = ~gt1['is_metric_scale'] + else: + mask = torch.ones_like(gt1['is_metric_scale']) + # normalize 3d points + if self.norm_mode and mask.any(): + pr_pts1[mask], pr_pts2[mask] = normalize_pointcloud(pr_pts1[mask], pr_pts2[mask], self.norm_mode, + valid1[mask], valid2[mask]) + + if self.norm_mode and not self.gt_scale: + gt_pts1, gt_pts2, norm_factor = normalize_pointcloud(gt_pts1, gt_pts2, self.norm_mode, + valid1, valid2, ret_factor=True) + # apply the same normalization to prediction + pr_pts1[~mask] = pr_pts1[~mask] / norm_factor[~mask] + pr_pts2[~mask] = pr_pts2[~mask] / norm_factor[~mask] + + # return sky segmentation, making sure they don't include any labelled 3d points + sky1 = gt1['sky_mask'] & (~valid1) + sky2 = gt2['sky_mask'] & (~valid2) + return gt_pts1, gt_pts2, pr_pts1, pr_pts2, valid1, valid2, sky1, sky2, {} + + def compute_loss(self, gt1, gt2, pred1, pred2, **kw): + gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, sky1, sky2, monitoring = \ + self.get_all_pts3d(gt1, gt2, pred1, pred2, **kw) + + if self.sky_loss_value > 0: + assert self.criterion.reduction == 'none', 'sky_loss_value should be 0 if no conf loss' + # add the sky pixel as "valid" pixels... + mask1 = mask1 | sky1 + mask2 = mask2 | sky2 + + # loss on img1 side + pred_pts1 = pred_pts1[mask1] + gt_pts1 = gt_pts1[mask1] + if self.loss_in_log and self.loss_in_log != 'before': + # this only make sense when depth_mode == 'exp' + pred_pts1 = apply_log_to_norm(pred_pts1) + gt_pts1 = apply_log_to_norm(gt_pts1) + l1 = self.criterion(pred_pts1, gt_pts1) + + # loss on gt2 side + pred_pts2 = pred_pts2[mask2] + gt_pts2 = gt_pts2[mask2] + if self.loss_in_log and self.loss_in_log != 'before': + pred_pts2 = apply_log_to_norm(pred_pts2) + gt_pts2 = apply_log_to_norm(gt_pts2) + l2 = self.criterion(pred_pts2, gt_pts2) + + if self.sky_loss_value > 0: + assert self.criterion.reduction == 'none', 'sky_loss_value should be 0 if no conf loss' + # ... but force the loss to be high there + l1 = torch.where(sky1[mask1], self.sky_loss_value, l1) + l2 = torch.where(sky2[mask2], self.sky_loss_value, l2) + self_name = type(self).__name__ + details = {self_name + '_pts3d_1': float(l1.mean()), self_name + '_pts3d_2': float(l2.mean())} + return Sum((l1, mask1), (l2, mask2)), (details | monitoring) + + +class Regr3D_ShiftInv (Regr3D): + """ Same than Regr3D but invariant to depth shift. + """ + + def get_all_pts3d(self, gt1, gt2, pred1, pred2): + # compute unnormalized points + gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, sky1, sky2, monitoring = \ + super().get_all_pts3d(gt1, gt2, pred1, pred2) + + # compute median depth + gt_z1, gt_z2 = gt_pts1[..., 2], gt_pts2[..., 2] + pred_z1, pred_z2 = pred_pts1[..., 2], pred_pts2[..., 2] + gt_shift_z = get_joint_pointcloud_depth(gt_z1, gt_z2, mask1, mask2)[:, None, None] + pred_shift_z = get_joint_pointcloud_depth(pred_z1, pred_z2, mask1, mask2)[:, None, None] + + # subtract the median depth + gt_z1 -= gt_shift_z + gt_z2 -= gt_shift_z + pred_z1 -= pred_shift_z + pred_z2 -= pred_shift_z + + # monitoring = dict(monitoring, gt_shift_z=gt_shift_z.mean().detach(), pred_shift_z=pred_shift_z.mean().detach()) + return gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, sky1, sky2, monitoring + + +class Regr3D_ScaleInv (Regr3D): + """ Same than Regr3D but invariant to depth scale. + if gt_scale == True: enforce the prediction to take the same scale than GT + """ + + def get_all_pts3d(self, gt1, gt2, pred1, pred2): + # compute depth-normalized points + gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, sky1, sky2, monitoring = \ + super().get_all_pts3d(gt1, gt2, pred1, pred2) + + # measure scene scale + _, gt_scale = get_joint_pointcloud_center_scale(gt_pts1, gt_pts2, mask1, mask2) + _, pred_scale = get_joint_pointcloud_center_scale(pred_pts1, pred_pts2, mask1, mask2) + + # prevent predictions to be in a ridiculous range + pred_scale = pred_scale.clip(min=1e-3, max=1e3) + + # subtract the median depth + if self.gt_scale: + pred_pts1 *= gt_scale / pred_scale + pred_pts2 *= gt_scale / pred_scale + # monitoring = dict(monitoring, pred_scale=(pred_scale/gt_scale).mean()) + else: + gt_pts1 /= gt_scale + gt_pts2 /= gt_scale + pred_pts1 /= pred_scale + pred_pts2 /= pred_scale + # monitoring = dict(monitoring, gt_scale=gt_scale.mean(), pred_scale=pred_scale.mean().detach()) + + return gt_pts1, gt_pts2, pred_pts1, pred_pts2, mask1, mask2, sky1, sky2, monitoring + + +class Regr3D_ScaleShiftInv (Regr3D_ScaleInv, Regr3D_ShiftInv): + # calls Regr3D_ShiftInv first, then Regr3D_ScaleInv + pass + + +def get_similarities(desc1, desc2, euc=False): + if euc: # euclidean distance in same range than similarities + dists = (desc1[:, :, None] - desc2[:, None]).norm(dim=-1) + sim = 1 / (1 + dists) + else: + # Compute similarities + sim = desc1 @ desc2.transpose(-2, -1) + return sim + + +class MatchingCriterion(BaseCriterion): + def __init__(self, reduction='mean', fp=torch.float32): + super().__init__(reduction) + self.fp = fp + + def forward(self, a, b, valid_matches=None, euc=False): + assert a.ndim >= 2 and 1 <= a.shape[-1], f'Bad shape = {a.shape}' + dist = self.loss(a.to(self.fp), b.to(self.fp), valid_matches, euc=euc) + # one dimension less or reduction to single value + assert (valid_matches is None and dist.ndim == a.ndim - + 1) or self.reduction in ['mean', 'sum', '1-mean', 'none'] + if self.reduction == 'none': + return dist + if self.reduction == 'sum': + return dist.sum() + if self.reduction == 'mean': + return dist.mean() if dist.numel() > 0 else dist.new_zeros(()) + if self.reduction == '1-mean': + return 1. - dist.mean() if dist.numel() > 0 else dist.new_ones(()) + raise ValueError(f'bad {self.reduction=} mode') + + def loss(self, a, b, valid_matches=None): + raise NotImplementedError + + +class InfoNCE(MatchingCriterion): + def __init__(self, temperature=0.07, eps=1e-8, mode='all', **kwargs): + super().__init__(**kwargs) + self.temperature = temperature + self.eps = eps + assert mode in ['all', 'proper', 'dual'] + self.mode = mode + + def loss(self, desc1, desc2, valid_matches=None, euc=False): + # valid positives are along diagonals + B, N, D = desc1.shape + B2, N2, D2 = desc2.shape + assert B == B2 and D == D2 + if valid_matches is None: + valid_matches = torch.ones([B, N], dtype=bool) + # torch.all(valid_matches.sum(dim=-1) > 0) some pairs have no matches???? + assert valid_matches.shape == torch.Size([B, N]) and valid_matches.sum() > 0 + + # Tempered similarities + sim = get_similarities(desc1, desc2, euc) / self.temperature + sim[sim.isnan()] = -torch.inf # ignore nans + # Softmax of positives with temperature + sim = sim.exp_() # save peak memory + positives = sim.diagonal(dim1=-2, dim2=-1) + + # Loss + if self.mode == 'all': # Previous InfoNCE + loss = -torch.log((positives / sim.sum(dim=-1).sum(dim=-1, keepdim=True)).clip(self.eps)) + elif self.mode == 'proper': # Proper InfoNCE + loss = -(torch.log((positives / sim.sum(dim=-2)).clip(self.eps)) + + torch.log((positives / sim.sum(dim=-1)).clip(self.eps))) + elif self.mode == 'dual': # Dual Softmax + loss = -(torch.log((positives**2 / sim.sum(dim=-1) / sim.sum(dim=-2)).clip(self.eps))) + else: + raise ValueError("This should not happen...") + return loss[valid_matches] + + +class APLoss (MatchingCriterion): + """ AP loss. + + Input: (N, M) values in [min, max] + label: (N, M) values in {0, 1} + + Returns: 1 - mAP (mean AP for each n in {1..N}) + Note: typically, this is what you wanna minimize + """ + + def __init__(self, nq='torch', min=0, max=1, euc=False, **kw): + super().__init__(**kw) + # Exact/True AP loss (not differentiable) + if nq == 0: + nq = 'sklearn' # special case + try: + self.compute_AP = eval('self.compute_true_AP_' + nq) + except: + raise ValueError("Unknown mode %s for AP loss" % nq) + + @staticmethod + def compute_true_AP_sklearn(scores, labels): + def compute_AP(label, score): + return average_precision_score(label, score) + + aps = scores.new_zeros((scores.shape[0], scores.shape[1])) + label_np = labels.cpu().numpy().astype(bool) + scores_np = scores.cpu().numpy() + for bi in range(scores_np.shape[0]): + for i in range(scores_np.shape[1]): + labels = label_np[bi, i, :] + if labels.sum() < 1: + continue + aps[bi, i] = compute_AP(labels, scores_np[bi, i, :]) + return aps + + @staticmethod + def compute_true_AP_torch(scores, labels): + assert scores.shape == labels.shape + B, N, M = labels.shape + dev = labels.device + with torch.no_grad(): + # sort scores + _, order = scores.sort(dim=-1, descending=True) + # sort labels accordingly + labels = labels[torch.arange(B, device=dev)[:, None, None].expand(order.shape), + torch.arange(N, device=dev)[None, :, None].expand(order.shape), + order] + # compute number of positives per query + npos = labels.sum(dim=-1) + assert torch.all(torch.isclose(npos, npos[0, 0]) + ), "only implemented for constant number of positives per query" + npos = int(npos[0, 0]) + # compute precision at each recall point + posrank = labels.nonzero()[:, -1].view(B, N, npos) + recall = torch.arange(1, 1 + npos, dtype=torch.float32, device=dev)[None, None, :].expand(B, N, npos) + precision = recall / (1 + posrank).float() + # average precision values at all recall points + aps = precision.mean(dim=-1) + + return aps + + def loss(self, desc1, desc2, valid_matches=None, euc=False): # if matches is None, positives are the diagonal + B, N1, D = desc1.shape + B2, N2, D2 = desc2.shape + assert B == B2 and D == D2 + + scores = get_similarities(desc1, desc2, euc) + + labels = torch.zeros([B, N1, N2], dtype=scores.dtype, device=scores.device) + + # allow all diagonal positives and only mask afterwards + labels.diagonal(dim1=-2, dim2=-1)[...] = 1. + apscore = self.compute_AP(scores, labels) + if valid_matches is not None: + apscore = apscore[valid_matches] + return apscore + + +class MatchingLoss (Criterion, MultiLoss): + """ + Matching loss per image + only compare pixels inside an image but not in the whole batch as what would be done usually + """ + + def __init__(self, criterion, withconf=False, use_pts3d=False, negatives_padding=0, blocksize=4096): + super().__init__(criterion) + self.negatives_padding = negatives_padding + self.use_pts3d = use_pts3d + self.blocksize = blocksize + self.withconf = withconf + + def add_negatives(self, outdesc2, desc2, batchid, x2, y2): + if self.negatives_padding: + B, H, W, D = desc2.shape + negatives = torch.ones([B, H, W], device=desc2.device, dtype=bool) + negatives[batchid, y2, x2] = False + sel = negatives & (negatives.view([B, -1]).cumsum(dim=-1).view(B, H, W) + <= self.negatives_padding) # take the N-first negatives + outdesc2 = torch.cat([outdesc2, desc2[sel].view([B, -1, D])], dim=1) + return outdesc2 + + def get_confs(self, pred1, pred2, sel1, sel2): + if self.withconf: + if self.use_pts3d: + outconfs1 = pred1['conf'][sel1] + outconfs2 = pred2['conf'][sel2] + else: + outconfs1 = pred1['desc_conf'][sel1] + outconfs2 = pred2['desc_conf'][sel2] + else: + outconfs1 = outconfs2 = None + return outconfs1, outconfs2 + + def get_descs(self, pred1, pred2): + if self.use_pts3d: + desc1, desc2 = pred1['pts3d'], pred2['pts3d_in_other_view'] + else: + desc1, desc2 = pred1['desc'], pred2['desc'] + return desc1, desc2 + + def get_matching_descs(self, gt1, gt2, pred1, pred2, **kw): + outdesc1 = outdesc2 = outconfs1 = outconfs2 = None + # Recover descs, GT corres and valid mask + desc1, desc2 = self.get_descs(pred1, pred2) + + (x1, y1), (x2, y2) = gt1['corres'].unbind(-1), gt2['corres'].unbind(-1) + valid_matches = gt1['valid_corres'] + + # Select descs that have GT matches + B, N = x1.shape + batchid = torch.arange(B)[:, None].repeat(1, N) # B, N + outdesc1, outdesc2 = desc1[batchid, y1, x1], desc2[batchid, y2, x2] # B, N, D + + # Padd with unused negatives + outdesc2 = self.add_negatives(outdesc2, desc2, batchid, x2, y2) + + # Gather confs if needed + sel1 = batchid, y1, x1 + sel2 = batchid, y2, x2 + outconfs1, outconfs2 = self.get_confs(pred1, pred2, sel1, sel2) + + return outdesc1, outdesc2, outconfs1, outconfs2, valid_matches, {'use_euclidean_dist': self.use_pts3d} + + def blockwise_criterion(self, descs1, descs2, confs1, confs2, valid_matches, euc, rng=np.random, shuffle=True): + loss = None + details = {} + B, N, D = descs1.shape + + if N <= self.blocksize: # Blocks are larger than provided descs, compute regular loss + loss = self.criterion(descs1, descs2, valid_matches, euc=euc) + else: # Compute criterion on the blockdiagonal only, after shuffling + # Shuffle if necessary + matches_perm = slice(None) + if shuffle: + matches_perm = np.stack([rng.choice(range(N), size=N, replace=False) for _ in range(B)]) + batchid = torch.tile(torch.arange(B), (N, 1)).T + matches_perm = batchid, matches_perm + + descs1 = descs1[matches_perm] + descs2 = descs2[matches_perm] + valid_matches = valid_matches[matches_perm] + + assert N % self.blocksize == 0, "Error, can't chunk block-diagonal, please check blocksize" + n_chunks = N // self.blocksize + descs1 = descs1.reshape([B * n_chunks, self.blocksize, D]) # [B*(N//blocksize), blocksize, D] + descs2 = descs2.reshape([B * n_chunks, self.blocksize, D]) # [B*(N//blocksize), blocksize, D] + valid_matches = valid_matches.view([B * n_chunks, self.blocksize]) + loss = self.criterion(descs1, descs2, valid_matches, euc=euc) + if self.withconf: + confs1, confs2 = map(lambda x: x[matches_perm], (confs1, confs2)) # apply perm to confidences if needed + + if self.withconf: + # split confidences between positives/negatives for loss computation + details['conf_pos'] = map(lambda x: x[valid_matches.view(B, -1)], (confs1, confs2)) + details['conf_neg'] = map(lambda x: x[~valid_matches.view(B, -1)], (confs1, confs2)) + details['Conf1_std'] = confs1.std() + details['Conf2_std'] = confs2.std() + + return loss, details + + def compute_loss(self, gt1, gt2, pred1, pred2, **kw): + # Gather preds and GT + descs1, descs2, confs1, confs2, valid_matches, monitoring = self.get_matching_descs( + gt1, gt2, pred1, pred2, **kw) + + # loss on matches + loss, details = self.blockwise_criterion(descs1, descs2, confs1, confs2, + valid_matches, euc=monitoring.pop('use_euclidean_dist', False)) + + details[type(self).__name__] = float(loss.mean()) + return loss, (details | monitoring) + + +class ConfMatchingLoss(ConfLoss): + """ Weight matching by learned confidence. Same as ConfLoss but for a matching criterion + Assuming the input matching_loss is a match-level loss. + """ + + def __init__(self, pixel_loss, alpha=1., confmode='prod', neg_conf_loss_quantile=False): + super().__init__(pixel_loss, alpha) + self.pixel_loss.withconf = True + self.confmode = confmode + self.neg_conf_loss_quantile = neg_conf_loss_quantile + + def aggregate_confs(self, confs1, confs2): # get the confidences resulting from the two view predictions + if self.confmode == 'prod': + confs = confs1 * confs2 if confs1 is not None and confs2 is not None else 1. + elif self.confmode == 'mean': + confs = .5 * (confs1 + confs2) if confs1 is not None and confs2 is not None else 1. + else: + raise ValueError(f"Unknown conf mode {self.confmode}") + return confs + + def compute_loss(self, gt1, gt2, pred1, pred2, **kw): + # compute per-pixel loss + loss, details = self.pixel_loss(gt1, gt2, pred1, pred2, **kw) + # Recover confidences for positive and negative samples + conf1_pos, conf2_pos = details.pop('conf_pos') + conf1_neg, conf2_neg = details.pop('conf_neg') + conf_pos = self.aggregate_confs(conf1_pos, conf2_pos) + + # weight Matching loss by confidence on positives + conf_pos, log_conf_pos = self.get_conf_log(conf_pos) + conf_loss = loss * conf_pos - self.alpha * log_conf_pos + # average + nan protection (in case of no valid pixels at all) + conf_loss = conf_loss.mean() if conf_loss.numel() > 0 else 0 + # Add negative confs loss to give some supervision signal to confidences for pixels that are not matched in GT + if self.neg_conf_loss_quantile: + conf_neg = torch.cat([conf1_neg, conf2_neg]) + conf_neg, log_conf_neg = self.get_conf_log(conf_neg) + + # recover quantile that will be used for negatives loss value assignment + neg_loss_value = torch.quantile(loss, self.neg_conf_loss_quantile).detach() + neg_loss = neg_loss_value * conf_neg - self.alpha * log_conf_neg + + neg_loss = neg_loss.mean() if neg_loss.numel() > 0 else 0 + conf_loss = conf_loss + neg_loss + + return conf_loss, dict(matching_conf_loss=float(conf_loss), **details) diff --git a/src/mast3r_src/mast3r/model.py b/src/mast3r_src/mast3r/model.py new file mode 100644 index 0000000000000000000000000000000000000000..6d63d8a98d0471a5bb61701df93866010d3c1a5a --- /dev/null +++ b/src/mast3r_src/mast3r/model.py @@ -0,0 +1,70 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# MASt3R model class +# -------------------------------------------------------- +import torch +import torch.nn.functional as F +import os + +from mast3r.catmlp_dpt_head import mast3r_head_factory + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.model import AsymmetricCroCo3DStereo # noqa +from dust3r.utils.misc import transpose_to_landscape # noqa + + +inf = float('inf') + + +def load_model(model_path, device, verbose=True): + if verbose: + print('... loading model from', model_path) + ckpt = torch.load(model_path, map_location='cpu') + args = ckpt['args'].model.replace("ManyAR_PatchEmbed", "PatchEmbedDust3R") + if 'landscape_only' not in args: + args = args[:-1] + ', landscape_only=False)' + else: + args = args.replace(" ", "").replace('landscape_only=True', 'landscape_only=False') + assert "landscape_only=False" in args + if verbose: + print(f"instantiating : {args}") + net = eval(args) + s = net.load_state_dict(ckpt['model'], strict=False) + if verbose: + print(s) + return net.to(device) + + +class AsymmetricMASt3R(AsymmetricCroCo3DStereo): + def __init__(self, desc_mode=('norm'), two_confs=False, desc_conf_mode=None, use_offsets=False, sh_degree=1, **kwargs): + self.desc_mode = desc_mode + self.two_confs = two_confs + self.desc_conf_mode = desc_conf_mode + self.use_offsets = use_offsets + self.sh_degree = sh_degree + super().__init__(**kwargs) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, **kw): + if os.path.isfile(pretrained_model_name_or_path): + return load_model(pretrained_model_name_or_path, device='cpu') + else: + return super(AsymmetricMASt3R, cls).from_pretrained(pretrained_model_name_or_path, **kw) + + def set_downstream_head(self, output_mode, head_type, landscape_only, depth_mode, conf_mode, patch_size, img_size, **kw): + assert img_size[0] % patch_size == 0 and img_size[ + 1] % patch_size == 0, f'{img_size=} must be multiple of {patch_size=}' + self.output_mode = output_mode + self.head_type = head_type + self.depth_mode = depth_mode + self.conf_mode = conf_mode + if self.desc_conf_mode is None: + self.desc_conf_mode = conf_mode + # allocate heads + self.downstream_head1 = mast3r_head_factory(head_type, output_mode, self, has_conf=bool(conf_mode), use_offsets=self.use_offsets, sh_degree=self.sh_degree) + self.downstream_head2 = mast3r_head_factory(head_type, output_mode, self, has_conf=bool(conf_mode), use_offsets=self.use_offsets, sh_degree=self.sh_degree) + # magic wrapper + self.head1 = transpose_to_landscape(self.downstream_head1, activate=landscape_only) + self.head2 = transpose_to_landscape(self.downstream_head2, activate=landscape_only) diff --git a/src/mast3r_src/mast3r/utils/__init__.py b/src/mast3r_src/mast3r/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d7dd877d649ce4dbd749dd7195a8b34c0f91d4f0 --- /dev/null +++ b/src/mast3r_src/mast3r/utils/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). \ No newline at end of file diff --git a/src/mast3r_src/mast3r/utils/coarse_to_fine.py b/src/mast3r_src/mast3r/utils/coarse_to_fine.py new file mode 100644 index 0000000000000000000000000000000000000000..c062e8608f82c608f2d605d69a95a7e0f301b3cf --- /dev/null +++ b/src/mast3r_src/mast3r/utils/coarse_to_fine.py @@ -0,0 +1,214 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# coarse to fine utilities +# -------------------------------------------------------- +import numpy as np + + +def crop_tag(cell): + return f'[{cell[1]}:{cell[3]},{cell[0]}:{cell[2]}]' + + +def crop_slice(cell): + return slice(cell[1], cell[3]), slice(cell[0], cell[2]) + + +def _start_pos(total_size, win_size, overlap): + # we must have AT LEAST overlap between segments + # first segment starts at 0, last segment starts at total_size-win_size + assert 0 <= overlap < 1 + assert total_size >= win_size + spacing = win_size * (1 - overlap) + last_pt = total_size - win_size + n_windows = 2 + int((last_pt - 1) // spacing) + return np.linspace(0, last_pt, n_windows).round().astype(int) + + +def multiple_of_16(x): + return (x // 16) * 16 + + +def _make_overlapping_grid(H, W, size, overlap): + H_win = multiple_of_16(H * size // max(H, W)) + W_win = multiple_of_16(W * size // max(H, W)) + x = _start_pos(W, W_win, overlap) + y = _start_pos(H, H_win, overlap) + grid = np.stack(np.meshgrid(x, y, indexing='xy'), axis=-1) + grid = np.concatenate((grid, grid + (W_win, H_win)), axis=-1) + return grid.reshape(-1, 4) + + +def _cell_size(cell2): + width, height = cell2[:, 2] - cell2[:, 0], cell2[:, 3] - cell2[:, 1] + assert width.min() >= 0 + assert height.min() >= 0 + return width, height + + +def _norm_windows(cell2, H2, W2, forced_resolution=None): + # make sure the window aspect ratio is 3/4, or the output resolution is forced_resolution if defined + outcell = cell2.copy() + width, height = _cell_size(cell2) + width2, height2 = width.clip(max=W2), height.clip(max=H2) + if forced_resolution is None: + width2[width < height] = (height2[width < height] * 3.01 / 4).clip(max=W2) + height2[width >= height] = (width2[width >= height] * 3.01 / 4).clip(max=H2) + else: + forced_H, forced_W = forced_resolution + width2[:] = forced_W + height2[:] = forced_H + + half = (width2 - width) / 2 + outcell[:, 0] -= half + outcell[:, 2] += half + half = (height2 - height) / 2 + outcell[:, 1] -= half + outcell[:, 3] += half + + # proj to integers + outcell = np.floor(outcell).astype(int) + # Take care of flooring errors + tmpw, tmph = _cell_size(outcell) + outcell[:, 0] += tmpw.astype(tmpw.dtype) - width2.astype(tmpw.dtype) + outcell[:, 1] += tmph.astype(tmpw.dtype) - height2.astype(tmpw.dtype) + + # make sure 0 <= x < W2 and 0 <= y < H2 + outcell[:, 0::2] -= outcell[:, [0]].clip(max=0) + outcell[:, 1::2] -= outcell[:, [1]].clip(max=0) + outcell[:, 0::2] -= outcell[:, [2]].clip(min=W2) - W2 + outcell[:, 1::2] -= outcell[:, [3]].clip(min=H2) - H2 + + width, height = _cell_size(outcell) + assert np.all(width == width2.astype(width.dtype)) and np.all( + height == height2.astype(height.dtype)), "Error, output is not of the expected shape." + assert np.all(width <= W2) + assert np.all(height <= H2) + return outcell + + +def _weight_pixels(cell, pix, assigned, gauss_var=2): + center = cell.reshape(-1, 2, 2).mean(axis=1) + width, height = _cell_size(cell) + + # square distance between each cell center and each point + dist = (center[:, None] - pix[None]) / np.c_[width, height][:, None] + dist2 = np.square(dist).sum(axis=-1) + + assert assigned.shape == dist2.shape + res = np.where(assigned, np.exp(-gauss_var * dist2), 0) + return res + + +def pos2d_in_rect(p1, cell1): + x, y = p1.T + l, t, r, b = cell1 + assigned = (l <= x) & (x < r) & (t <= y) & (y < b) + return assigned + + +def _score_cell(cell1, H2, W2, p1, p2, min_corres=10, forced_resolution=None): + assert p1.shape == p2.shape + + # compute keypoint assignment + assigned = pos2d_in_rect(p1, cell1[None].T) + assert assigned.shape == (len(cell1), len(p1)) + + # remove cells without correspondences + valid_cells = assigned.sum(axis=1) >= min_corres + cell1 = cell1[valid_cells] + assigned = assigned[valid_cells] + if not valid_cells.any(): + return cell1, cell1, assigned + + # fill-in the assigned points in both image + assigned_p1 = np.empty((len(cell1), len(p1), 2), dtype=np.float32) + assigned_p2 = np.empty((len(cell1), len(p2), 2), dtype=np.float32) + assigned_p1[:] = p1[None] + assigned_p2[:] = p2[None] + assigned_p1[~assigned] = np.nan + assigned_p2[~assigned] = np.nan + + # find the median center and scale of assigned points in each cell + # cell_center1 = np.nanmean(assigned_p1, axis=1) + cell_center2 = np.nanmean(assigned_p2, axis=1) + im1_q25, im1_q75 = np.nanquantile(assigned_p1, (0.1, 0.9), axis=1) + im2_q25, im2_q75 = np.nanquantile(assigned_p2, (0.1, 0.9), axis=1) + + robust_std1 = (im1_q75 - im1_q25).clip(20.) + robust_std2 = (im2_q75 - im2_q25).clip(20.) + + cell_size1 = (cell1[:, 2:4] - cell1[:, 0:2]) + cell_size2 = cell_size1 * robust_std2 / robust_std1 + cell2 = np.c_[cell_center2 - cell_size2 / 2, cell_center2 + cell_size2 / 2] + + # make sure cell bounds are valid + cell2 = _norm_windows(cell2, H2, W2, forced_resolution=forced_resolution) + + # compute correspondence weights + corres_weights = _weight_pixels(cell1, p1, assigned) * _weight_pixels(cell2, p2, assigned) + + # return a list of window pairs and assigned correspondences + return cell1, cell2, corres_weights + + +def greedy_selection(corres_weights, target=0.9): + # corres_weight = (n_cell_pair, n_corres) matrix. + # If corres_weight[c,p]>0, means that correspondence p is visible in cell pair p + assert 0 < target <= 1 + corres_weights = corres_weights.copy() + + total = corres_weights.max(axis=0).sum() + target *= total + + # init = empty + res = [] + cur = np.zeros(corres_weights.shape[1]) # current selection + + while cur.sum() < target: + # pick the nex best cell pair + best = corres_weights.sum(axis=1).argmax() + res.append(best) + + # update current + cur += corres_weights[best] + # print('appending', best, 'with score', corres_weights[best].sum(), '-->', cur.sum()) + + # remove from all other views + corres_weights = (corres_weights - corres_weights[best]).clip(min=0) + + return res + + +def select_pairs_of_crops(img_q, img_b, pos2d_in_query, pos2d_in_ref, maxdim=512, overlap=.5, forced_resolution=None): + # prepare the overlapping cells + grid_q = _make_overlapping_grid(*img_q.shape[:2], maxdim, overlap) + grid_b = _make_overlapping_grid(*img_b.shape[:2], maxdim, overlap) + + assert forced_resolution is None or len(forced_resolution) == 2 + if isinstance(forced_resolution[0], int) or not len(forced_resolution[0]) == 2: + forced_resolution1 = forced_resolution2 = forced_resolution + else: + assert len(forced_resolution[1]) == 2 + forced_resolution1 = forced_resolution[0] + forced_resolution2 = forced_resolution[1] + + # Make sure crops respect constraints + grid_q = _norm_windows(grid_q.astype(float), *img_q.shape[:2], forced_resolution=forced_resolution1) + grid_b = _norm_windows(grid_b.astype(float), *img_b.shape[:2], forced_resolution=forced_resolution2) + + # score cells + pairs_q = _score_cell(grid_q, *img_b.shape[:2], pos2d_in_query, pos2d_in_ref, forced_resolution=forced_resolution2) + pairs_b = _score_cell(grid_b, *img_q.shape[:2], pos2d_in_ref, pos2d_in_query, forced_resolution=forced_resolution1) + pairs_b = pairs_b[1], pairs_b[0], pairs_b[2] # cellq, cellb, corres_weights + + # greedy selection until all correspondences are generated + cell1, cell2, corres_weights = map(np.concatenate, zip(pairs_q, pairs_b)) + if len(corres_weights) == 0: + return # tolerated for empty generators + order = greedy_selection(corres_weights, target=0.9) + + for i in order: + def pair_tag(qi, bi): return (str(qi) + crop_tag(cell1[i]), str(bi) + crop_tag(cell2[i])) + yield cell1[i], cell2[i], pair_tag diff --git a/src/mast3r_src/mast3r/utils/collate.py b/src/mast3r_src/mast3r/utils/collate.py new file mode 100644 index 0000000000000000000000000000000000000000..72ee3a437b87ef7049dcd03b93e594a8325b780c --- /dev/null +++ b/src/mast3r_src/mast3r/utils/collate.py @@ -0,0 +1,62 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# Collate extensions +# -------------------------------------------------------- + +import torch +import collections +from torch.utils.data._utils.collate import default_collate_fn_map, default_collate_err_msg_format +from typing import Callable, Dict, Optional, Tuple, Type, Union, List + + +def cat_collate_tensor_fn(batch, *, collate_fn_map): + return torch.cat(batch, dim=0) + + +def cat_collate_list_fn(batch, *, collate_fn_map: Optional[Dict[Union[Type, Tuple[Type, ...]], Callable]] = None): + return [item for bb in batch for item in bb] # concatenate all lists + + +cat_collate_fn_map = default_collate_fn_map.copy() +cat_collate_fn_map[torch.Tensor] = cat_collate_tensor_fn +cat_collate_fn_map[List] = cat_collate_list_fn +cat_collate_fn_map[type(None)] = lambda _, **kw: None # When some Nones, simply return a single None + + +def cat_collate(batch, *, collate_fn_map: Optional[Dict[Union[Type, Tuple[Type, ...]], Callable]] = None): + r"""Custom collate function that concatenates stuff instead of stacking them, and handles NoneTypes """ + elem = batch[0] + elem_type = type(elem) + + if collate_fn_map is not None: + if elem_type in collate_fn_map: + return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map) + + for collate_type in collate_fn_map: + if isinstance(elem, collate_type): + return collate_fn_map[collate_type](batch, collate_fn_map=collate_fn_map) + + if isinstance(elem, collections.abc.Mapping): + try: + return elem_type({key: cat_collate([d[key] for d in batch], collate_fn_map=collate_fn_map) for key in elem}) + except TypeError: + # The mapping type may not support `__init__(iterable)`. + return {key: cat_collate([d[key] for d in batch], collate_fn_map=collate_fn_map) for key in elem} + elif isinstance(elem, tuple) and hasattr(elem, '_fields'): # namedtuple + return elem_type(*(cat_collate(samples, collate_fn_map=collate_fn_map) for samples in zip(*batch))) + elif isinstance(elem, collections.abc.Sequence): + transposed = list(zip(*batch)) # It may be accessed twice, so we use a list. + + if isinstance(elem, tuple): + # Backwards compatibility. + return [cat_collate(samples, collate_fn_map=collate_fn_map) for samples in transposed] + else: + try: + return elem_type([cat_collate(samples, collate_fn_map=collate_fn_map) for samples in transposed]) + except TypeError: + # The sequence type may not support `__init__(iterable)` (e.g., `range`). + return [cat_collate(samples, collate_fn_map=collate_fn_map) for samples in transposed] + + raise TypeError(default_collate_err_msg_format.format(elem_type)) diff --git a/src/mast3r_src/mast3r/utils/misc.py b/src/mast3r_src/mast3r/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..1a5403c67116f5156e47537df8fbcfcacb7bb474 --- /dev/null +++ b/src/mast3r_src/mast3r/utils/misc.py @@ -0,0 +1,17 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# utilitary functions for MASt3R +# -------------------------------------------------------- +import os +import hashlib + + +def mkdir_for(f): + os.makedirs(os.path.dirname(f), exist_ok=True) + return f + + +def hash_md5(s): + return hashlib.md5(s.encode('utf-8')).hexdigest() diff --git a/src/mast3r_src/mast3r/utils/path_to_dust3r.py b/src/mast3r_src/mast3r/utils/path_to_dust3r.py new file mode 100644 index 0000000000000000000000000000000000000000..ebfd78cb432ef45ee30bb893f7f90fff08474b93 --- /dev/null +++ b/src/mast3r_src/mast3r/utils/path_to_dust3r.py @@ -0,0 +1,19 @@ +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# dust3r submodule import +# -------------------------------------------------------- + +import sys +import os.path as path +HERE_PATH = path.normpath(path.dirname(__file__)) +DUSt3R_REPO_PATH = path.normpath(path.join(HERE_PATH, '../../dust3r')) +DUSt3R_LIB_PATH = path.join(DUSt3R_REPO_PATH, 'dust3r') +# check the presence of models directory in repo to be sure its cloned +if path.isdir(DUSt3R_LIB_PATH): + # workaround for sibling import + sys.path.insert(0, DUSt3R_REPO_PATH) +else: + raise ImportError(f"dust3r is not initialized, could not find: {DUSt3R_LIB_PATH}.\n " + "Did you forget to run 'git submodule update --init --recursive' ?") diff --git a/src/mast3r_src/requirements.txt b/src/mast3r_src/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ff88936c77d98f85c1a4d5ae4bfcf374e332f4b3 --- /dev/null +++ b/src/mast3r_src/requirements.txt @@ -0,0 +1 @@ +scikit-learn \ No newline at end of file diff --git a/src/mast3r_src/train.py b/src/mast3r_src/train.py new file mode 100644 index 0000000000000000000000000000000000000000..57a7689c408f63701cf4287ada50802f1a5cef6a --- /dev/null +++ b/src/mast3r_src/train.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# training executable for MASt3R +# -------------------------------------------------------- +from mast3r.model import AsymmetricMASt3R +from mast3r.losses import ConfMatchingLoss, MatchingLoss, APLoss, Regr3D, InfoNCE, Regr3D_ScaleShiftInv +from mast3r.datasets import ARKitScenes, BlendedMVS, Co3d, MegaDepth, ScanNetpp, StaticThings3D, Waymo, WildRGBD + +import mast3r.utils.path_to_dust3r # noqa +# add mast3r classes to dust3r imports +import dust3r.training +dust3r.training.AsymmetricMASt3R = AsymmetricMASt3R +dust3r.training.Regr3D = Regr3D +dust3r.training.Regr3D_ScaleShiftInv = Regr3D_ScaleShiftInv +dust3r.training.MatchingLoss = MatchingLoss +dust3r.training.ConfMatchingLoss = ConfMatchingLoss +dust3r.training.InfoNCE = InfoNCE +dust3r.training.APLoss = APLoss + +import dust3r.datasets +dust3r.datasets.ARKitScenes = ARKitScenes +dust3r.datasets.BlendedMVS = BlendedMVS +dust3r.datasets.Co3d = Co3d +dust3r.datasets.MegaDepth = MegaDepth +dust3r.datasets.ScanNetpp = ScanNetpp +dust3r.datasets.StaticThings3D = StaticThings3D +dust3r.datasets.Waymo = Waymo +dust3r.datasets.WildRGBD = WildRGBD + +from dust3r.training import get_args_parser as dust3r_get_args_parser # noqa +from dust3r.training import train # noqa + + +def get_args_parser(): + parser = dust3r_get_args_parser() + # change defaults + parser.prog = 'MASt3R training' + parser.set_defaults(model="AsymmetricMASt3R(patch_embed_cls='ManyAR_PatchEmbed')") + return parser + + +if __name__ == '__main__': + args = get_args_parser() + args = args.parse_args() + train(args) diff --git a/src/mast3r_src/visloc.py b/src/mast3r_src/visloc.py new file mode 100644 index 0000000000000000000000000000000000000000..8e460169068d3d82308538aef6c14c756e9848ce --- /dev/null +++ b/src/mast3r_src/visloc.py @@ -0,0 +1,538 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024-present Naver Corporation. All rights reserved. +# Licensed under CC BY-NC-SA 4.0 (non-commercial use only). +# +# -------------------------------------------------------- +# visloc script with support for coarse to fine +# -------------------------------------------------------- +import os +import numpy as np +import random +import torch +import torchvision.transforms as tvf +import argparse +from tqdm import tqdm +from PIL import Image +import math + +from mast3r.model import AsymmetricMASt3R +from mast3r.fast_nn import fast_reciprocal_NNs +from mast3r.utils.coarse_to_fine import select_pairs_of_crops, crop_slice +from mast3r.utils.collate import cat_collate, cat_collate_fn_map +from mast3r.utils.misc import mkdir_for +from mast3r.datasets.utils.cropping import crop_to_homography + +import mast3r.utils.path_to_dust3r # noqa +from dust3r.inference import inference, loss_of_one_batch +from dust3r.utils.geometry import geotrf, colmap_to_opencv_intrinsics, opencv_to_colmap_intrinsics +from dust3r.datasets.utils.transforms import ImgNorm +from dust3r_visloc.datasets import * +from dust3r_visloc.localization import run_pnp +from dust3r_visloc.evaluation import get_pose_error, aggregate_stats, export_results +from dust3r_visloc.datasets.utils import get_HW_resolution, rescale_points3d + + +def get_args_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", type=str, required=True, help="visloc dataset to eval") + parser_weights = parser.add_mutually_exclusive_group(required=True) + parser_weights.add_argument("--weights", type=str, help="path to the model weights", default=None) + parser_weights.add_argument("--model_name", type=str, help="name of the model weights", + choices=["MASt3R_ViTLarge_BaseDecoder_512_catmlpdpt_metric"]) + + parser.add_argument("--confidence_threshold", type=float, default=1.001, + help="confidence values higher than threshold are invalid") + parser.add_argument('--pixel_tol', default=5, type=int) + + parser.add_argument("--coarse_to_fine", action='store_true', default=False, + help="do the matching from coarse to fine") + parser.add_argument("--max_image_size", type=int, default=None, + help="max image size for the fine resolution") + parser.add_argument("--c2f_crop_with_homography", action='store_true', default=False, + help="when using coarse to fine, crop with homographies to keep cx, cy centered") + + parser.add_argument("--device", type=str, default='cuda', help="pytorch device") + parser.add_argument("--pnp_mode", type=str, default="cv2", choices=['cv2', 'poselib', 'pycolmap'], + help="pnp lib to use") + parser_reproj = parser.add_mutually_exclusive_group() + parser_reproj.add_argument("--reprojection_error", type=float, default=5.0, help="pnp reprojection error") + parser_reproj.add_argument("--reprojection_error_diag_ratio", type=float, default=None, + help="pnp reprojection error as a ratio of the diagonal of the image") + + parser.add_argument("--max_batch_size", type=int, default=48, + help="max batch size for inference on crops when using coarse to fine") + parser.add_argument("--pnp_max_points", type=int, default=100_000, help="pnp maximum number of points kept") + parser.add_argument("--viz_matches", type=int, default=0, help="debug matches") + + parser.add_argument("--output_dir", type=str, default=None, help="output path") + parser.add_argument("--output_label", type=str, default='', help="prefix for results files") + return parser + + +@torch.no_grad() +def coarse_matching(query_view, map_view, model, device, pixel_tol, fast_nn_params): + # prepare batch + imgs = [] + for idx, img in enumerate([query_view['rgb_rescaled'], map_view['rgb_rescaled']]): + imgs.append(dict(img=img.unsqueeze(0), true_shape=np.int32([img.shape[1:]]), + idx=idx, instance=str(idx))) + output = inference([tuple(imgs)], model, device, batch_size=1, verbose=False) + pred1, pred2 = output['pred1'], output['pred2'] + conf_list = [pred1['desc_conf'].squeeze(0).cpu().numpy(), pred2['desc_conf'].squeeze(0).cpu().numpy()] + desc_list = [pred1['desc'].squeeze(0).detach(), pred2['desc'].squeeze(0).detach()] + + # find 2D-2D matches between the two images + PQ, PM = desc_list[0], desc_list[1] + if len(PQ) == 0 or len(PM) == 0: + return [], [], [], [] + + if pixel_tol == 0: + matches_im_map, matches_im_query = fast_reciprocal_NNs(PM, PQ, subsample_or_initxy1=8, **fast_nn_params) + HM, WM = map_view['rgb_rescaled'].shape[1:] + HQ, WQ = query_view['rgb_rescaled'].shape[1:] + # ignore small border around the edge + valid_matches_map = (matches_im_map[:, 0] >= 3) & (matches_im_map[:, 0] < WM - 3) & ( + matches_im_map[:, 1] >= 3) & (matches_im_map[:, 1] < HM - 3) + valid_matches_query = (matches_im_query[:, 0] >= 3) & (matches_im_query[:, 0] < WQ - 3) & ( + matches_im_query[:, 1] >= 3) & (matches_im_query[:, 1] < HQ - 3) + valid_matches = valid_matches_map & valid_matches_query + matches_im_map = matches_im_map[valid_matches] + matches_im_query = matches_im_query[valid_matches] + valid_pts3d = [] + matches_confs = [] + else: + yM, xM = torch.where(map_view['valid_rescaled']) + matches_im_map, matches_im_query = fast_reciprocal_NNs(PM, PQ, (xM, yM), pixel_tol=pixel_tol, **fast_nn_params) + valid_pts3d = map_view['pts3d_rescaled'].cpu().numpy()[matches_im_map[:, 1], matches_im_map[:, 0]] + matches_confs = np.minimum( + conf_list[1][matches_im_map[:, 1], matches_im_map[:, 0]], + conf_list[0][matches_im_query[:, 1], matches_im_query[:, 0]] + ) + # from cv2 to colmap + matches_im_query = matches_im_query.astype(np.float64) + matches_im_map = matches_im_map.astype(np.float64) + matches_im_query[:, 0] += 0.5 + matches_im_query[:, 1] += 0.5 + matches_im_map[:, 0] += 0.5 + matches_im_map[:, 1] += 0.5 + # rescale coordinates + matches_im_query = geotrf(query_view['to_orig'], matches_im_query, norm=True) + matches_im_map = geotrf(map_view['to_orig'], matches_im_map, norm=True) + # from colmap back to cv2 + matches_im_query[:, 0] -= 0.5 + matches_im_query[:, 1] -= 0.5 + matches_im_map[:, 0] -= 0.5 + matches_im_map[:, 1] -= 0.5 + return valid_pts3d, matches_im_query, matches_im_map, matches_confs + + +@torch.no_grad() +def crops_inference(pairs, model, device, batch_size=48, verbose=True): + assert len(pairs) == 2, "Error, data should be a tuple of dicts containing the batch of image pairs" + # Forward a possibly big bunch of data, by blocks of batch_size + B = pairs[0]['img'].shape[0] + if B < batch_size: + return loss_of_one_batch(pairs, model, None, device=device, symmetrize_batch=False) + preds = [] + for ii in range(0, B, batch_size): + sel = slice(ii, ii + min(B - ii, batch_size)) + temp_data = [{}, {}] + for di in [0, 1]: + temp_data[di] = {kk: pairs[di][kk][sel] + for kk in pairs[di].keys() if pairs[di][kk] is not None} # copy chunk for forward + preds.append(loss_of_one_batch(temp_data, model, + None, device=device, symmetrize_batch=False)) # sequential forward + # Merge all preds + return cat_collate(preds, collate_fn_map=cat_collate_fn_map) + + +@torch.no_grad() +def fine_matching(query_views, map_views, model, device, max_batch_size, pixel_tol, fast_nn_params): + assert pixel_tol > 0 + output = crops_inference([query_views, map_views], + model, device, batch_size=max_batch_size, verbose=False) + pred1, pred2 = output['pred1'], output['pred2'] + descs1 = pred1['desc'].clone() + descs2 = pred2['desc'].clone() + confs1 = pred1['desc_conf'].clone() + confs2 = pred2['desc_conf'].clone() + + # Compute matches + valid_pts3d, matches_im_map, matches_im_query, matches_confs = [], [], [], [] + for ppi, (pp1, pp2, cc11, cc21) in enumerate(zip(descs1, descs2, confs1, confs2)): + valid_ppi = map_views['valid'][ppi] + pts3d_ppi = map_views['pts3d'][ppi].cpu().numpy() + conf_list_ppi = [cc11.cpu().numpy(), cc21.cpu().numpy()] + + y_ppi, x_ppi = torch.where(valid_ppi) + matches_im_map_ppi, matches_im_query_ppi = fast_reciprocal_NNs(pp2, pp1, (x_ppi, y_ppi), + pixel_tol=pixel_tol, **fast_nn_params) + + valid_pts3d_ppi = pts3d_ppi[matches_im_map_ppi[:, 1], matches_im_map_ppi[:, 0]] + matches_confs_ppi = np.minimum( + conf_list_ppi[1][matches_im_map_ppi[:, 1], matches_im_map_ppi[:, 0]], + conf_list_ppi[0][matches_im_query_ppi[:, 1], matches_im_query_ppi[:, 0]] + ) + # inverse operation where we uncrop pixel coordinates + matches_im_map_ppi = geotrf(map_views['to_orig'][ppi].cpu().numpy(), matches_im_map_ppi.copy(), norm=True) + matches_im_query_ppi = geotrf(query_views['to_orig'][ppi].cpu().numpy(), matches_im_query_ppi.copy(), norm=True) + + matches_im_map.append(matches_im_map_ppi) + matches_im_query.append(matches_im_query_ppi) + valid_pts3d.append(valid_pts3d_ppi) + matches_confs.append(matches_confs_ppi) + + if len(valid_pts3d) == 0: + return [], [], [], [] + + matches_im_map = np.concatenate(matches_im_map, axis=0) + matches_im_query = np.concatenate(matches_im_query, axis=0) + valid_pts3d = np.concatenate(valid_pts3d, axis=0) + matches_confs = np.concatenate(matches_confs, axis=0) + return valid_pts3d, matches_im_query, matches_im_map, matches_confs + + +def crop(img, mask, pts3d, crop, intrinsics=None): + out_cropped_img = img.clone() + if mask is not None: + out_cropped_mask = mask.clone() + else: + out_cropped_mask = None + if pts3d is not None: + out_cropped_pts3d = pts3d.clone() + else: + out_cropped_pts3d = None + to_orig = torch.eye(3, device=img.device) + + # If intrinsics available, crop and apply rectifying homography. Otherwise, just crop + if intrinsics is not None: + K_old = intrinsics + imsize, K_new, R, H = crop_to_homography(K_old, crop) + # apply homography to image + H /= H[2, 2] + homo8 = H.ravel().tolist()[:8] + # From float tensor to uint8 PIL Image + pilim = Image.fromarray((255 * (img + 1.) / 2).to(torch.uint8).numpy()) + pilout_cropped_img = pilim.transform(imsize, Image.Transform.PERSPECTIVE, + homo8, resample=Image.Resampling.BICUBIC) + + # From uint8 PIL Image to float tensor + out_cropped_img = 2. * torch.tensor(np.array(pilout_cropped_img)).to(img) / 255. - 1. + if out_cropped_mask is not None: + pilmask = Image.fromarray((255 * out_cropped_mask).to(torch.uint8).numpy()) + pilout_cropped_mask = pilmask.transform( + imsize, Image.Transform.PERSPECTIVE, homo8, resample=Image.Resampling.NEAREST) + out_cropped_mask = torch.from_numpy(np.array(pilout_cropped_mask) > 0).to(out_cropped_mask.dtype) + if out_cropped_pts3d is not None: + out_cropped_pts3d = out_cropped_pts3d.numpy() + out_cropped_X = np.array(Image.fromarray(out_cropped_pts3d[:, :, 0]).transform(imsize, + Image.Transform.PERSPECTIVE, + homo8, + resample=Image.Resampling.NEAREST)) + out_cropped_Y = np.array(Image.fromarray(out_cropped_pts3d[:, :, 1]).transform(imsize, + Image.Transform.PERSPECTIVE, + homo8, + resample=Image.Resampling.NEAREST)) + out_cropped_Z = np.array(Image.fromarray(out_cropped_pts3d[:, :, 2]).transform(imsize, + Image.Transform.PERSPECTIVE, + homo8, + resample=Image.Resampling.NEAREST)) + + out_cropped_pts3d = torch.from_numpy(np.stack([out_cropped_X, out_cropped_Y, out_cropped_Z], axis=-1)) + + to_orig = torch.tensor(H, device=img.device) + else: + out_cropped_img = img[crop_slice(crop)] + if out_cropped_mask is not None: + out_cropped_mask = out_cropped_mask[crop_slice(crop)] + if out_cropped_pts3d is not None: + out_cropped_pts3d = out_cropped_pts3d[crop_slice(crop)] + to_orig[:2, -1] = torch.tensor(crop[:2]) + + return out_cropped_img, out_cropped_mask, out_cropped_pts3d, to_orig + + +def resize_image_to_max(max_image_size, rgb, K): + W, H = rgb.size + if max_image_size and max(W, H) > max_image_size: + islandscape = (W >= H) + if islandscape: + WMax = max_image_size + HMax = int(H * (WMax / W)) + else: + HMax = max_image_size + WMax = int(W * (HMax / H)) + resize_op = tvf.Compose([ImgNorm, tvf.Resize(size=[HMax, WMax])]) + rgb_tensor = resize_op(rgb).permute(1, 2, 0) + to_orig_max = np.array([[W / WMax, 0, 0], + [0, H / HMax, 0], + [0, 0, 1]]) + to_resize_max = np.array([[WMax / W, 0, 0], + [0, HMax / H, 0], + [0, 0, 1]]) + + # Generate new camera parameters + new_K = opencv_to_colmap_intrinsics(K) + new_K[0, :] *= WMax / W + new_K[1, :] *= HMax / H + new_K = colmap_to_opencv_intrinsics(new_K) + else: + rgb_tensor = ImgNorm(rgb).permute(1, 2, 0) + to_orig_max = np.eye(3) + to_resize_max = np.eye(3) + HMax, WMax = H, W + new_K = K + return rgb_tensor, new_K, to_orig_max, to_resize_max, (HMax, WMax) + + +if __name__ == '__main__': + parser = get_args_parser() + args = parser.parse_args() + conf_thr = args.confidence_threshold + device = args.device + pnp_mode = args.pnp_mode + assert args.pixel_tol > 0 + reprojection_error = args.reprojection_error + reprojection_error_diag_ratio = args.reprojection_error_diag_ratio + pnp_max_points = args.pnp_max_points + viz_matches = args.viz_matches + + if args.weights is not None: + weights_path = args.weights + else: + weights_path = "naver/" + args.model_name + model = AsymmetricMASt3R.from_pretrained(weights_path).to(args.device) + fast_nn_params = dict(device=device, dist='dot', block_size=2**13) + dataset = eval(args.dataset) + dataset.set_resolution(model) + + query_names = [] + poses_pred = [] + pose_errors = [] + angular_errors = [] + params_str = f'tol_{args.pixel_tol}' + ("_c2f" if args.coarse_to_fine else '') + if args.max_image_size is not None: + params_str = params_str + f'_{args.max_image_size}' + if args.coarse_to_fine and args.c2f_crop_with_homography: + params_str = params_str + '_with_homography' + for idx in tqdm(range(len(dataset))): + views = dataset[(idx)] # 0 is the query + query_view = views[0] + map_views = views[1:] + query_names.append(query_view['image_name']) + + query_pts2d = [] + query_pts3d = [] + maxdim = max(model.patch_embed.img_size) + query_rgb_tensor, query_K, query_to_orig_max, query_to_resize_max, (HQ, WQ) = resize_image_to_max( + args.max_image_size, query_view['rgb'], query_view['intrinsics']) + + # pairs of crops have the same resolution + query_resolution = get_HW_resolution(HQ, WQ, maxdim=maxdim, patchsize=model.patch_embed.patch_size) + for map_view in map_views: + if args.output_dir is not None: + cache_file = os.path.join(args.output_dir, 'matches', params_str, + query_view['image_name'], map_view['image_name'] + '.npz') + else: + cache_file = None + + if cache_file is not None and os.path.isfile(cache_file): + matches = np.load(cache_file) + valid_pts3d = matches['valid_pts3d'] + matches_im_query = matches['matches_im_query'] + matches_im_map = matches['matches_im_map'] + matches_conf = matches['matches_conf'] + else: + # coarse matching + if args.coarse_to_fine and (maxdim < max(WQ, HQ)): + # use all points + _, coarse_matches_im0, coarse_matches_im1, _ = coarse_matching(query_view, map_view, model, device, + 0, fast_nn_params) + + # visualize a few matches + if viz_matches > 0: + num_matches = coarse_matches_im1.shape[0] + print(f'found {num_matches} matches') + + viz_imgs = [np.array(query_view['rgb']), np.array(map_view['rgb'])] + from matplotlib import pyplot as pl + n_viz = viz_matches + match_idx_to_viz = np.round(np.linspace(0, num_matches - 1, n_viz)).astype(int) + viz_matches_im_query = coarse_matches_im0[match_idx_to_viz] + viz_matches_im_map = coarse_matches_im1[match_idx_to_viz] + + H0, W0, H1, W1 = *viz_imgs[0].shape[:2], *viz_imgs[1].shape[:2] + img0 = np.pad(viz_imgs[0], ((0, max(H1 - H0, 0)), (0, 0), (0, 0)), + 'constant', constant_values=0) + img1 = np.pad(viz_imgs[1], ((0, max(H0 - H1, 0)), (0, 0), (0, 0)), + 'constant', constant_values=0) + img = np.concatenate((img0, img1), axis=1) + pl.figure() + pl.imshow(img) + cmap = pl.get_cmap('jet') + for i in range(n_viz): + (x0, y0), (x1, y1) = viz_matches_im_query[i].T, viz_matches_im_map[i].T + pl.plot([x0, x1 + W0], [y0, y1], '-+', + color=cmap(i / (n_viz - 1)), scalex=False, scaley=False) + pl.show(block=True) + + valid_all = map_view['valid'] + pts3d = map_view['pts3d'] + + WM_full, HM_full = map_view['rgb'].size + map_rgb_tensor, map_K, map_to_orig_max, map_to_resize_max, (HM, WM) = resize_image_to_max( + args.max_image_size, map_view['rgb'], map_view['intrinsics']) + if WM_full != WM or HM_full != HM: + y_full, x_full = torch.where(valid_all) + pos2d_cv2 = torch.stack([x_full, y_full], dim=-1).cpu().numpy().astype(np.float64) + sparse_pts3d = pts3d[y_full, x_full].cpu().numpy() + _, _, pts3d_max, valid_max = rescale_points3d( + pos2d_cv2, sparse_pts3d, map_to_resize_max, HM, WM) + pts3d = torch.from_numpy(pts3d_max) + valid_all = torch.from_numpy(valid_max) + + coarse_matches_im0 = geotrf(query_to_resize_max, coarse_matches_im0, norm=True) + coarse_matches_im1 = geotrf(map_to_resize_max, coarse_matches_im1, norm=True) + + crops1, crops2 = [], [] + crops_v1, crops_p1 = [], [] + to_orig1, to_orig2 = [], [] + map_resolution = get_HW_resolution(HM, WM, maxdim=maxdim, patchsize=model.patch_embed.patch_size) + + for crop_q, crop_b, pair_tag in select_pairs_of_crops(map_rgb_tensor, + query_rgb_tensor, + coarse_matches_im1, + coarse_matches_im0, + maxdim=maxdim, + overlap=.5, + forced_resolution=[map_resolution, + query_resolution]): + # Per crop processing + if not args.c2f_crop_with_homography: + map_K = None + query_K = None + + c1, v1, p1, trf1 = crop(map_rgb_tensor, valid_all, pts3d, crop_q, map_K) + c2, _, _, trf2 = crop(query_rgb_tensor, None, None, crop_b, query_K) + crops1.append(c1) + crops2.append(c2) + crops_v1.append(v1) + crops_p1.append(p1) + to_orig1.append(trf1) + to_orig2.append(trf2) + + if len(crops1) == 0 or len(crops2) == 0: + valid_pts3d, matches_im_query, matches_im_map, matches_conf = [], [], [], [] + else: + crops1, crops2 = torch.stack(crops1), torch.stack(crops2) + if len(crops1.shape) == 3: + crops1, crops2 = crops1[None], crops2[None] + crops_v1 = torch.stack(crops_v1) + crops_p1 = torch.stack(crops_p1) + to_orig1, to_orig2 = torch.stack(to_orig1), torch.stack(to_orig2) + map_crop_view = dict(img=crops1.permute(0, 3, 1, 2), + instance=['1' for _ in range(crops1.shape[0])], + valid=crops_v1, pts3d=crops_p1, + to_orig=to_orig1) + query_crop_view = dict(img=crops2.permute(0, 3, 1, 2), + instance=['2' for _ in range(crops2.shape[0])], + to_orig=to_orig2) + + # Inference and Matching + valid_pts3d, matches_im_query, matches_im_map, matches_conf = fine_matching(query_crop_view, + map_crop_view, + model, device, + args.max_batch_size, + args.pixel_tol, + fast_nn_params) + matches_im_query = geotrf(query_to_orig_max, matches_im_query, norm=True) + matches_im_map = geotrf(map_to_orig_max, matches_im_map, norm=True) + else: + # use only valid 2d points + valid_pts3d, matches_im_query, matches_im_map, matches_conf = coarse_matching(query_view, map_view, + model, device, + args.pixel_tol, + fast_nn_params) + if cache_file is not None: + mkdir_for(cache_file) + np.savez(cache_file, valid_pts3d=valid_pts3d, matches_im_query=matches_im_query, + matches_im_map=matches_im_map, matches_conf=matches_conf) + + # apply conf + if len(matches_conf) > 0: + mask = matches_conf >= conf_thr + valid_pts3d = valid_pts3d[mask] + matches_im_query = matches_im_query[mask] + matches_im_map = matches_im_map[mask] + matches_conf = matches_conf[mask] + + # visualize a few matches + if viz_matches > 0: + num_matches = matches_im_map.shape[0] + print(f'found {num_matches} matches') + + viz_imgs = [np.array(query_view['rgb']), np.array(map_view['rgb'])] + from matplotlib import pyplot as pl + n_viz = viz_matches + match_idx_to_viz = np.round(np.linspace(0, num_matches - 1, n_viz)).astype(int) + viz_matches_im_query = matches_im_query[match_idx_to_viz] + viz_matches_im_map = matches_im_map[match_idx_to_viz] + + H0, W0, H1, W1 = *viz_imgs[0].shape[:2], *viz_imgs[1].shape[:2] + img0 = np.pad(viz_imgs[0], ((0, max(H1 - H0, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img1 = np.pad(viz_imgs[1], ((0, max(H0 - H1, 0)), (0, 0), (0, 0)), 'constant', constant_values=0) + img = np.concatenate((img0, img1), axis=1) + pl.figure() + pl.imshow(img) + cmap = pl.get_cmap('jet') + for i in range(n_viz): + (x0, y0), (x1, y1) = viz_matches_im_query[i].T, viz_matches_im_map[i].T + pl.plot([x0, x1 + W0], [y0, y1], '-+', color=cmap(i / (n_viz - 1)), scalex=False, scaley=False) + pl.show(block=True) + + if len(valid_pts3d) == 0: + pass + else: + query_pts3d.append(valid_pts3d) + query_pts2d.append(matches_im_query) + + if len(query_pts2d) == 0: + success = False + pr_querycam_to_world = None + else: + query_pts2d = np.concatenate(query_pts2d, axis=0).astype(np.float32) + query_pts3d = np.concatenate(query_pts3d, axis=0) + if len(query_pts2d) > pnp_max_points: + idxs = random.sample(range(len(query_pts2d)), pnp_max_points) + query_pts3d = query_pts3d[idxs] + query_pts2d = query_pts2d[idxs] + + W, H = query_view['rgb'].size + if reprojection_error_diag_ratio is not None: + reprojection_error_img = reprojection_error_diag_ratio * math.sqrt(W**2 + H**2) + else: + reprojection_error_img = reprojection_error + success, pr_querycam_to_world = run_pnp(query_pts2d, query_pts3d, + query_view['intrinsics'], query_view['distortion'], + pnp_mode, reprojection_error_img, img_size=[W, H]) + + if not success: + abs_transl_error = float('inf') + abs_angular_error = float('inf') + else: + abs_transl_error, abs_angular_error = get_pose_error(pr_querycam_to_world, query_view['cam_to_world']) + + pose_errors.append(abs_transl_error) + angular_errors.append(abs_angular_error) + poses_pred.append(pr_querycam_to_world) + + xp_label = params_str + f'_conf_{conf_thr}' + if args.output_label: + xp_label = args.output_label + "_" + xp_label + if reprojection_error_diag_ratio is not None: + xp_label = xp_label + f'_reproj_diag_{reprojection_error_diag_ratio}' + else: + xp_label = xp_label + f'_reproj_err_{reprojection_error}' + export_results(args.output_dir, xp_label, query_names, poses_pred) + out_string = aggregate_stats(f'{args.dataset}', pose_errors, angular_errors) + print(out_string) diff --git a/src/pixelsplat_src/benchmarker.py b/src/pixelsplat_src/benchmarker.py new file mode 100644 index 0000000000000000000000000000000000000000..67a4722fa8150042fcdb5379c87e17028223cbf8 --- /dev/null +++ b/src/pixelsplat_src/benchmarker.py @@ -0,0 +1,42 @@ +import json +from collections import defaultdict +from contextlib import contextmanager +from pathlib import Path +from time import time + +import numpy as np +import torch +import os + + +class Benchmarker: + def __init__(self): + self.execution_times = defaultdict(list) + + @contextmanager + def time(self, tag: str, num_calls: int = 1): + try: + start_time = time() + yield + finally: + end_time = time() + for _ in range(num_calls): + self.execution_times[tag].append((end_time - start_time) / num_calls) + + def dump(self, path: str) -> None: + parent = os.path.abspath(os.path.join(path, os.pardir)) + os.makedirs(parent, exist_ok=True) + # path.parent.mkdir(exist_ok=True, parents=True) + with open(path, "w") as f: + json.dump(dict(self.execution_times), f) + # with path.open("w") as f: + # json.dump(dict(self.execution_times), f) + + def dump_memory(self, path: Path) -> None: + path.parent.mkdir(exist_ok=True, parents=True) + with path.open("w") as f: + json.dump(torch.cuda.memory_stats()["allocated_bytes.all.peak"], f) + + def summarize(self) -> None: + for tag, times in self.execution_times.items(): + print(f"{tag}: {len(times)} calls, avg. {np.mean(times)} seconds per call") diff --git a/src/pixelsplat_src/cuda_splatting.py b/src/pixelsplat_src/cuda_splatting.py new file mode 100644 index 0000000000000000000000000000000000000000..c9887a8391becf783347989906cfbf1b9f46e2d2 --- /dev/null +++ b/src/pixelsplat_src/cuda_splatting.py @@ -0,0 +1,266 @@ +from math import isqrt +from typing import Literal + +import torch +from diff_gaussian_rasterization import ( + GaussianRasterizationSettings, + GaussianRasterizer, +) +from einops import einsum, rearrange, repeat +from torch import Tensor + +from .projection import get_fov, homogenize_points + + +def get_projection_matrix( + near, + far, + fov_x, + fov_y, +): + """Maps points in the viewing frustum to (-1, 1) on the X/Y axes and (0, 1) on the Z + axis. Differs from the OpenGL version in that Z doesn't have range (-1, 1) after + transformation and that Z is flipped. + """ + tan_fov_x = (0.5 * fov_x).tan() + tan_fov_y = (0.5 * fov_y).tan() + + top = tan_fov_y * near + bottom = -top + right = tan_fov_x * near + left = -right + + (b,) = near.shape + result = torch.zeros((b, 4, 4), dtype=torch.float32, device=near.device) + result[:, 0, 0] = 2 * near / (right - left) + result[:, 1, 1] = 2 * near / (top - bottom) + result[:, 0, 2] = (right + left) / (right - left) + result[:, 1, 2] = (top + bottom) / (top - bottom) + result[:, 3, 2] = 1 + result[:, 2, 2] = far / (far - near) + result[:, 2, 3] = -(far * near) / (far - near) + return result + + +def render_cuda( + extrinsics, + intrinsics, + near, + far, + image_shape: tuple[int, int], + background_color, + gaussian_means, + gaussian_covariances, + gaussian_sh_coefficients, + gaussian_opacities, + scale_invariant: bool = True, + use_sh: bool = True, +): + assert use_sh or gaussian_sh_coefficients.shape[-1] == 1 + + # Make sure everything is in a range where numerical issues don't appear. + if scale_invariant: + scale = 1 / near + extrinsics = extrinsics.clone() + extrinsics[..., :3, 3] = extrinsics[..., :3, 3] * scale[:, None] + gaussian_covariances = gaussian_covariances * (scale[:, None, None, None] ** 2) + gaussian_means = gaussian_means * scale[:, None, None] + near = near * scale + far = far * scale + + _, _, _, n = gaussian_sh_coefficients.shape + degree = isqrt(n) - 1 + shs = rearrange(gaussian_sh_coefficients, "b g xyz n -> b g n xyz").contiguous() + + b, _, _ = extrinsics.shape + h, w = image_shape + + fov_x, fov_y = get_fov(intrinsics).unbind(dim=-1) + tan_fov_x = (0.5 * fov_x).tan() + tan_fov_y = (0.5 * fov_y).tan() + + projection_matrix = get_projection_matrix(near, far, fov_x, fov_y) + projection_matrix = rearrange(projection_matrix, "b i j -> b j i") + view_matrix = rearrange(extrinsics.inverse(), "b i j -> b j i") + full_projection = view_matrix @ projection_matrix + + all_images = [] + all_radii = [] + for i in range(b): + # Set up a tensor for the gradients of the screen-space means. + mean_gradients = torch.zeros_like(gaussian_means[i], requires_grad=True) + try: + mean_gradients.retain_grad() + except Exception: + pass + + settings = GaussianRasterizationSettings( + image_height=h, + image_width=w, + tanfovx=tan_fov_x[i].item(), + tanfovy=tan_fov_y[i].item(), + bg=background_color[i], + scale_modifier=1.0, + viewmatrix=view_matrix[i], + projmatrix=full_projection[i], + sh_degree=degree, + campos=extrinsics[i, :3, 3], + prefiltered=False, # This matches the original usage. + debug=False, + ) + rasterizer = GaussianRasterizer(settings) + + row, col = torch.triu_indices(3, 3) + + image, radii = rasterizer( + means3D=gaussian_means[i], + means2D=mean_gradients, + shs=shs[i] if use_sh else None, + colors_precomp=None if use_sh else shs[i, :, 0, :], + opacities=gaussian_opacities[i, ..., None], + cov3D_precomp=gaussian_covariances[i, :, row, col], + ) + all_images.append(image) + all_radii.append(radii) + return torch.stack(all_images) + + +def render_cuda_orthographic( + extrinsics, + width, + height, + near, + far, + image_shape: tuple[int, int], + background_color, + gaussian_means, + gaussian_covariances, + gaussian_sh_coefficients, + gaussian_opacities, + fov_degrees, + use_sh: bool = True, + dump: dict | None = None, +): + b, _, _ = extrinsics.shape + h, w = image_shape + assert use_sh or gaussian_sh_coefficients.shape[-1] == 1 + + _, _, _, n = gaussian_sh_coefficients.shape + degree = isqrt(n) - 1 + shs = rearrange(gaussian_sh_coefficients, "b g xyz n -> b g n xyz").contiguous() + + # Create fake "orthographic" projection by moving the camera back and picking a + # small field of view. + fov_x = torch.tensor(fov_degrees, device=extrinsics.device).deg2rad() + tan_fov_x = (0.5 * fov_x).tan() + distance_to_near = (0.5 * width) / tan_fov_x + tan_fov_y = 0.5 * height / distance_to_near + fov_y = (2 * tan_fov_y).atan() + near = near + distance_to_near + far = far + distance_to_near + move_back = torch.eye(4, dtype=torch.float32, device=extrinsics.device) + move_back[2, 3] = -distance_to_near + extrinsics = extrinsics @ move_back + + # Escape hatch for visualization/figures. + if dump is not None: + dump["extrinsics"] = extrinsics + dump["fov_x"] = fov_x + dump["fov_y"] = fov_y + dump["near"] = near + dump["far"] = far + + projection_matrix = get_projection_matrix( + near, far, repeat(fov_x, "-> b", b=b), fov_y + ) + projection_matrix = rearrange(projection_matrix, "b i j -> b j i") + view_matrix = rearrange(extrinsics.inverse(), "b i j -> b j i") + full_projection = view_matrix @ projection_matrix + + all_images = [] + all_radii = [] + for i in range(b): + # Set up a tensor for the gradients of the screen-space means. + mean_gradients = torch.zeros_like(gaussian_means[i], requires_grad=True) + try: + mean_gradients.retain_grad() + except Exception: + pass + + settings = GaussianRasterizationSettings( + image_height=h, + image_width=w, + tanfovx=tan_fov_x, + tanfovy=tan_fov_y, + bg=background_color[i], + scale_modifier=1.0, + viewmatrix=view_matrix[i], + projmatrix=full_projection[i], + sh_degree=degree, + campos=extrinsics[i, :3, 3], + prefiltered=False, # This matches the original usage. + debug=False, + ) + rasterizer = GaussianRasterizer(settings) + + row, col = torch.triu_indices(3, 3) + + image, radii = rasterizer( + means3D=gaussian_means[i], + means2D=mean_gradients, + shs=shs[i] if use_sh else None, + colors_precomp=None if use_sh else shs[i, :, 0, :], + opacities=gaussian_opacities[i, ..., None], + cov3D_precomp=gaussian_covariances[i, :, row, col], + ) + all_images.append(image) + all_radii.append(radii) + return torch.stack(all_images) + + +DepthRenderingMode = Literal["depth", "disparity", "relative_disparity", "log"] + + +def render_depth_cuda( + extrinsics, + intrinsics, + near, + far, + image_shape: tuple[int, int], + gaussian_means, + gaussian_covariances, + gaussian_opacities, + scale_invariant: bool = True, + mode: DepthRenderingMode = "depth", +): + # Specify colors according to Gaussian depths. + camera_space_gaussians = einsum( + extrinsics.inverse(), homogenize_points(gaussian_means), "b i j, b g j -> b g i" + ) + fake_color = camera_space_gaussians[..., 2] + + if mode == "disparity": + fake_color = 1 / fake_color + elif mode == "relative_disparity": + fake_color = depth_to_relative_disparity( + fake_color, near[:, None], far[:, None] + ) + elif mode == "log": + fake_color = fake_color.minimum(near[:, None]).maximum(far[:, None]).log() + + # Render using depth as color. + b, _ = fake_color.shape + result = render_cuda( + extrinsics, + intrinsics, + near, + far, + image_shape, + torch.zeros((b, 3), dtype=fake_color.dtype, device=fake_color.device), + gaussian_means, + gaussian_covariances, + repeat(fake_color, "b g -> b g c ()", c=3), + gaussian_opacities, + scale_invariant=scale_invariant, + ) + return result.mean(dim=1) diff --git a/src/pixelsplat_src/decoder_splatting_cuda.py b/src/pixelsplat_src/decoder_splatting_cuda.py new file mode 100644 index 0000000000000000000000000000000000000000..a0a6859305338d68071ab7e549593662e8107bab --- /dev/null +++ b/src/pixelsplat_src/decoder_splatting_cuda.py @@ -0,0 +1,53 @@ +import torch +from einops import rearrange, repeat + +from .cuda_splatting import render_cuda +from utils.geometry import normalize_intrinsics + + +class DecoderSplattingCUDA(torch.nn.Module): + + def __init__(self, background_color): + super().__init__() + self.register_buffer( + "background_color", + torch.tensor(background_color, dtype=torch.float32), + persistent=False, + ) + + def forward(self, batch, pred1, pred2, image_shape): + + base_pose = batch['context'][0]['camera_pose'] # [b, 4, 4] + inv_base_pose = torch.inverse(base_pose) + + extrinsics = torch.stack([target_view['camera_pose'] for target_view in batch['target']], dim=1) + intrinsics = torch.stack([target_view['camera_intrinsics'] for target_view in batch['target']], dim=1) + intrinsics = normalize_intrinsics(intrinsics, image_shape)[..., :3, :3] + + # Rotate the ground truth extrinsics into the coordinate system used by MAST3R + # --i.e. in the coordinate system of the first context view, normalized by the scene scale + extrinsics = inv_base_pose[:, None, :, :] @ extrinsics + + means = torch.stack([pred1["means"], pred2["means_in_other_view"]], dim=1) + covariances = torch.stack([pred1["covariances"], pred2["covariances"]], dim=1) + harmonics = torch.stack([pred1["sh"], pred2["sh"]], dim=1) + opacities = torch.stack([pred1["opacities"], pred2["opacities"]], dim=1) + + b, v, _, _ = extrinsics.shape + near = torch.full((b, v), 0.1, device=means.device) + far = torch.full((b, v), 1000.0, device=means.device) + + color = render_cuda( + rearrange(extrinsics, "b v i j -> (b v) i j"), + rearrange(intrinsics, "b v i j -> (b v) i j"), + rearrange(near, "b v -> (b v)"), + rearrange(far, "b v -> (b v)"), + image_shape, + repeat(self.background_color, "c -> (b v) c", b=b, v=v), + repeat(rearrange(means, "b v h w xyz -> b (v h w) xyz"), "b g xyz -> (b v) g xyz", v=v), + repeat(rearrange(covariances, "b v h w i j -> b (v h w) i j"), "b g i j -> (b v) g i j", v=v), + repeat(rearrange(harmonics, "b v h w c d_sh -> b (v h w) c d_sh"), "b g c d_sh -> (b v) g c d_sh", v=v), + repeat(rearrange(opacities, "b v h w 1 -> b (v h w)"), "b g -> (b v) g", v=v), + ) + color = rearrange(color, "(b v) c h w -> b v c h w", b=b, v=v) + return color, None \ No newline at end of file diff --git a/src/pixelsplat_src/projection.py b/src/pixelsplat_src/projection.py new file mode 100644 index 0000000000000000000000000000000000000000..464fe2cf2eb4809b9cd9551b36f3099ce98f0d33 --- /dev/null +++ b/src/pixelsplat_src/projection.py @@ -0,0 +1,233 @@ +from math import prod + +import torch +from einops import einsum, rearrange, reduce, repeat +from torch import Tensor + + +def homogenize_points( + points, +): + """Convert batched points (xyz) to (xyz1).""" + return torch.cat([points, torch.ones_like(points[..., :1])], dim=-1) + + +def homogenize_vectors( + vectors, +): + """Convert batched vectors (xyz) to (xyz0).""" + return torch.cat([vectors, torch.zeros_like(vectors[..., :1])], dim=-1) + + +def transform_rigid( + homogeneous_coordinates, + transformation, +): + """Apply a rigid-body transformation to points or vectors.""" + return einsum(transformation, homogeneous_coordinates, "... i j, ... j -> ... i") + + +def transform_cam2world( + homogeneous_coordinates, + extrinsics, +): + """Transform points from 3D camera coordinates to 3D world coordinates.""" + return transform_rigid(homogeneous_coordinates, extrinsics) + + +def transform_world2cam( + homogeneous_coordinates, + extrinsics, +): + """Transform points from 3D world coordinates to 3D camera coordinates.""" + return transform_rigid(homogeneous_coordinates, extrinsics.inverse()) + + +def project_camera_space( + points, + intrinsics, + epsilon, + infinity, +): + points = points / (points[..., -1:] + epsilon) + points = points.nan_to_num(posinf=infinity, neginf=-infinity) + points = einsum(intrinsics, points, "... i j, ... j -> ... i") + return points[..., :-1] + + +def project( + points, + extrinsics, + intrinsics, + epsilon, +): + points = homogenize_points(points) + points = transform_world2cam(points, extrinsics)[..., :-1] + in_front_of_camera = points[..., -1] >= 0 + return project_camera_space(points, intrinsics, epsilon=epsilon), in_front_of_camera + + +def unproject( + coordinates, + z, + intrinsics, +): + """Unproject 2D camera coordinates with the given Z values.""" + + # Apply the inverse intrinsics to the coordinates. + coordinates = homogenize_points(coordinates) + ray_directions = einsum( + intrinsics.inverse(), coordinates, "... i j, ... j -> ... i" + ) + + # Apply the supplied depth values. + return ray_directions * z[..., None] + + +def get_world_rays( + coordinates, + extrinsics, + intrinsics, +): + # Get camera-space ray directions. + directions = unproject( + coordinates, + torch.ones_like(coordinates[..., 0]), + intrinsics, + ) + directions = directions / directions.norm(dim=-1, keepdim=True) + + # Transform ray directions to world coordinates. + directions = homogenize_vectors(directions) + directions = transform_cam2world(directions, extrinsics)[..., :-1] + + # Tile the ray origins to have the same shape as the ray directions. + origins = extrinsics[..., :-1, -1].broadcast_to(directions.shape) + + return origins, directions + + +def sample_image_grid( + shape: tuple[int, ...], + device: torch.device = torch.device("cpu"), +): + """Get normalized (range 0 to 1) coordinates and integer indices for an image.""" + + # Each entry is a pixel-wise integer coordinate. In the 2D case, each entry is a + # (row, col) coordinate. + indices = [torch.arange(length, device=device) for length in shape] + stacked_indices = torch.stack(torch.meshgrid(*indices, indexing="ij"), dim=-1) + + # Each entry is a floating-point coordinate in the range (0, 1). In the 2D case, + # each entry is an (x, y) coordinate. + coordinates = [(idx + 0.5) / length for idx, length in zip(indices, shape)] + coordinates = reversed(coordinates) + coordinates = torch.stack(torch.meshgrid(*coordinates, indexing="xy"), dim=-1) + + return coordinates, stacked_indices + + +def sample_training_rays( + image, + intrinsics, + extrinsics, + num_rays: int, +): + device = extrinsics.device + b, v, _, *grid_shape = image.shape + + # Generate all possible target rays. + xy, _ = sample_image_grid(tuple(grid_shape), device) + origins, directions = get_world_rays( + rearrange(xy, "... d -> ... () () d"), + extrinsics, + intrinsics, + ) + origins = rearrange(origins, "... b v xy -> b (v ...) xy", b=b, v=v) + directions = rearrange(directions, "... b v xy -> b (v ...) xy", b=b, v=v) + pixels = rearrange(image, "b v c ... -> b (v ...) c") + + # Sample random rays. + num_possible_rays = v * prod(grid_shape) + ray_indices = torch.randint(num_possible_rays, (b, num_rays), device=device) + batch_indices = repeat(torch.arange(b, device=device), "b -> b n", n=num_rays) + + return ( + origins[batch_indices, ray_indices], + directions[batch_indices, ray_indices], + pixels[batch_indices, ray_indices], + ) + + +def intersect_rays( + origins_x, + directions_x, + origins_y, + directions_y, + eps, + inf, +): + """Compute the least-squares intersection of rays. Uses the math from here: + https://math.stackexchange.com/a/1762491/286022 + """ + + # Broadcast the rays so their shapes match. + shape = torch.broadcast_shapes( + origins_x.shape, + directions_x.shape, + origins_y.shape, + directions_y.shape, + ) + origins_x = origins_x.broadcast_to(shape) + directions_x = directions_x.broadcast_to(shape) + origins_y = origins_y.broadcast_to(shape) + directions_y = directions_y.broadcast_to(shape) + + # Detect and remove batch elements where the directions are parallel. + parallel = einsum(directions_x, directions_y, "... xyz, ... xyz -> ...") > 1 - eps + origins_x = origins_x[~parallel] + directions_x = directions_x[~parallel] + origins_y = origins_y[~parallel] + directions_y = directions_y[~parallel] + + # Stack the rays into (2, *shape). + origins = torch.stack([origins_x, origins_y], dim=0) + directions = torch.stack([directions_x, directions_y], dim=0) + dtype = origins.dtype + device = origins.device + + # Compute n_i * n_i^T - eye(3) from the equation. + n = einsum(directions, directions, "r b i, r b j -> r b i j") + n = n - torch.eye(3, dtype=dtype, device=device).broadcast_to((2, 1, 3, 3)) + + # Compute the left-hand side of the equation. + lhs = reduce(n, "r b i j -> b i j", "sum") + + # Compute the right-hand side of the equation. + rhs = einsum(n, origins, "r b i j, r b j -> r b i") + rhs = reduce(rhs, "r b i -> b i", "sum") + + # Left-matrix-multiply both sides by the pseudo-inverse of lhs to find p. + result = torch.linalg.lstsq(lhs, rhs).solution + + # Handle the case of parallel lines by setting depth to infinity. + result_all = torch.ones(shape, dtype=dtype, device=device) * inf + result_all[~parallel] = result + return result_all + + +def get_fov(intrinsics): + intrinsics_inv = intrinsics.inverse() + + def process_vector(vector): + vector = torch.tensor(vector, dtype=torch.float32, device=intrinsics.device) + vector = einsum(intrinsics_inv, vector, "b i j, j -> b i") + return vector / vector.norm(dim=-1, keepdim=True) + + left = process_vector([0, 0.5, 1]) + right = process_vector([1, 0.5, 1]) + top = process_vector([0.5, 0, 1]) + bottom = process_vector([0.5, 1, 1]) + fov_x = (left * right).sum(dim=-1).acos() + fov_y = (top * bottom).sum(dim=-1).acos() + return torch.stack((fov_x, fov_y), dim=-1) diff --git a/utils/compute_ssim.py b/utils/compute_ssim.py new file mode 100644 index 0000000000000000000000000000000000000000..1e57fd5ccf49c80171e45a1f3b4ddf37dda6060f --- /dev/null +++ b/utils/compute_ssim.py @@ -0,0 +1,26 @@ +import torch +from skimage.metrics import structural_similarity +import numpy as np + +@torch.no_grad() +def compute_ssim(ground_truth, predicted, full=True): + # The arguments to `structural_similarity` have been chosen to match + # PixelSplat (apart from `full = full`) + ssim = [ + structural_similarity( + gt.detach().cpu().numpy(), + hat.detach().cpu().numpy(), + win_size=11, + gaussian_weights=True, + channel_axis=0, + data_range=1.0, + full=full, + ) + for gt, hat in zip(ground_truth, predicted) + ] + if full: + ssim = [spatial for _, spatial in ssim] + ssim = np.array(ssim) + ssim = torch.tensor(ssim, dtype=predicted.dtype, device=predicted.device) + assert not torch.isnan(ssim).any(), "SSIM has NaNs" + return ssim diff --git a/utils/export.py b/utils/export.py new file mode 100644 index 0000000000000000000000000000000000000000..661d168350437a5f4e54a7be9f73eacf66edd353 --- /dev/null +++ b/utils/export.py @@ -0,0 +1,213 @@ +import os + +from plyfile import PlyData, PlyElement +from scipy.spatial.transform import Rotation +import einops +import numpy as np +import torch +import torchvision +import trimesh +import lightning as L + +import utils.loss_mask as loss_mask +from src.mast3r_src.dust3r.dust3r.viz import OPENGL, pts3d_to_trimesh, cat_meshes + + +class SaveBatchData(L.Callback): + '''A Lightning callback that occasionally saves batch inputs and outputs to disk. + It is not critical to the training process, and can be disabled if unwanted.''' + + def __init__(self, save_dir, train_save_interval=100, val_save_interval=100, test_save_interval=100): + self.save_dir = save_dir + self.train_save_interval = train_save_interval + self.val_save_interval = val_save_interval + self.test_save_interval = test_save_interval + + def on_train_batch_end(self, trainer, pl_module, outputs, batch, batch_idx): + if batch_idx % self.train_save_interval == 0 and trainer.global_rank == 0: + self.save_batch_data('train', trainer, pl_module, batch, batch_idx) + + def on_validation_batch_end(self, trainer, pl_module, outputs, batch, batch_idx): + if batch_idx % self.val_save_interval == 0 and trainer.global_rank == 0: + self.save_batch_data('val', trainer, pl_module, batch, batch_idx) + + def on_test_batch_end(self, trainer, pl_module, outputs, batch, batch_idx): + if batch_idx % self.test_save_interval == 0 and trainer.global_rank == 0: + self.save_batch_data('test', trainer, pl_module, batch, batch_idx) + + def save_batch_data(self, prefix, trainer, pl_module, batch, batch_idx): + + print(f'Saving {prefix} data at epoch {trainer.current_epoch} and batch {batch_idx}') + + # Run the batch through the model again + _, _, h, w = batch["context"][0]["img"].shape + view1, view2 = batch['context'] + pred1, pred2 = pl_module.forward(view1, view2) + color, depth = pl_module.decoder(batch, pred1, pred2, (h, w)) + mask = loss_mask.calculate_loss_mask(batch) + + # Save the data + save_dir = os.path.join( + self.save_dir, + f"{prefix}_epoch_{trainer.current_epoch}_batch_{batch_idx}" + ) + log_batch_files(batch, color, depth, mask, view1, view2, pred1, pred2, save_dir) + + +def save_as_ply(pred1, pred2, save_path): + """Save the 3D Gaussians as a point cloud in the PLY format. + Adapted loosely from PixelSplat""" + + def construct_list_of_attributes(num_rest: int) -> list[str]: + '''Construct a list of attributes for the PLY file format. This + corresponds to the attributes used by online readers, such as + https://niujinshuchong.github.io/mip-splatting-demo/index.html''' + attributes = ["x", "y", "z", "nx", "ny", "nz"] + for i in range(3): + attributes.append(f"f_dc_{i}") + for i in range(num_rest): + attributes.append(f"f_rest_{i}") + attributes.append("opacity") + for i in range(3): + attributes.append(f"scale_{i}") + for i in range(4): + attributes.append(f"rot_{i}") + return attributes + + def covariance_to_quaternion_and_scale(covariance): + '''Convert the covariance matrix to a four dimensional quaternion and + a three dimensional scale vector''' + + # Perform singular value decomposition + U, S, V = torch.linalg.svd(covariance) + + # The scale factors are the square roots of the eigenvalues + scale = torch.sqrt(S) + scale = scale.detach().cpu().numpy() + + # The rotation matrix is U*Vt + rotation_matrix = torch.bmm(U, V.transpose(-2, -1)) + rotation_matrix_np = rotation_matrix.detach().cpu().numpy() + + # Use scipy to convert the rotation matrix to a quaternion + rotation = Rotation.from_matrix(rotation_matrix_np) + quaternion = rotation.as_quat() + + return quaternion, scale + + # Collect the Gaussian parameters + means = torch.stack([pred1["pts3d"], pred2["pts3d_in_other_view"]], dim=1) + covariances = torch.stack([pred1["covariances"], pred2["covariances"]], dim=1) + harmonics = torch.stack([pred1["sh"], pred2["sh"]], dim=1)[..., 0] # Only use the first harmonic + opacities = torch.stack([pred1["opacities"], pred2["opacities"]], dim=1) + + # Rearrange the tensors to the correct shape + means = einops.rearrange(means[0], "view h w xyz -> (view h w) xyz").detach().cpu().numpy() + covariances = einops.rearrange(covariances[0], "v h w i j -> (v h w) i j") + harmonics = einops.rearrange(harmonics[0], "view h w xyz -> (view h w) xyz").detach().cpu().numpy() + opacities = einops.rearrange(opacities[0], "view h w xyz -> (view h w) xyz").detach().cpu().numpy() + + # Convert the covariance matrices to quaternions and scales + rotations, scales = covariance_to_quaternion_and_scale(covariances) + + # Construct the attributes + rest = np.zeros_like(means) + attributes = np.concatenate((means, rest, harmonics, opacities, np.log(scales), rotations), axis=-1) + dtype_full = [(attribute, "f4") for attribute in construct_list_of_attributes(0)] + elements = np.empty(attributes.shape[0], dtype=dtype_full) + elements[:] = list(map(tuple, attributes)) + + # Save the point cloud + point_cloud = PlyElement.describe(elements, "vertex") + scene = PlyData([point_cloud]) + scene.write(save_path) + + +def save_3d(view1, view2, pred1, pred2, save_dir, as_pointcloud=True, all_points=True): + """Save the 3D points as a point cloud or as a mesh. Adapted from DUSt3R""" + + os.makedirs(save_dir, exist_ok=True) + batch_size = pred1["pts3d"].shape[0] + views = [view1, view2] + + for b in range(batch_size): + + pts3d = [pred1["pts3d"][b].cpu().numpy()] + [pred2["pts3d_in_other_view"][b].cpu().numpy()] + imgs = [einops.rearrange(view["original_img"][b], "c h w -> h w c").cpu().numpy() for view in views] + mask = [view["valid_mask"][b].cpu().numpy() for view in views] + + # Treat all pixels as valid, because we want to render the entire viewpoint + if all_points: + mask = [np.ones_like(m) for m in mask] + + # Construct the scene from the 3D points as a point cloud or as a mesh + scene = trimesh.Scene() + if as_pointcloud: + pts = np.concatenate([p[m] for p, m in zip(pts3d, mask)]) + col = np.concatenate([p[m] for p, m in zip(imgs, mask)]) + pct = trimesh.PointCloud(pts.reshape(-1, 3), colors=col.reshape(-1, 3)) + scene.add_geometry(pct) + save_path = os.path.join(save_dir, f"{b}.ply") + else: + meshes = [] + for i in range(len(imgs)): + meshes.append(pts3d_to_trimesh(imgs[i], pts3d[i], mask[i])) + mesh = trimesh.Trimesh(**cat_meshes(meshes)) + scene.add_geometry(mesh) + save_path = os.path.join(save_dir, f"{b}.glb") + + # Save the scene + scene.export(file_obj=save_path) + + +@torch.no_grad() +def log_batch_files(batch, color, depth, mask, view1, view2, pred1, pred2, save_dir, should_save_3d=False): + '''Save all the relevant debug files for a batch''' + + os.makedirs(save_dir, exist_ok=True) + + # Save the 3D Gaussians as a .ply file + save_as_ply(pred1, pred2, os.path.join(save_dir, f"gaussians.ply")) + + # Save the 3D points as a point cloud and as a mesh (disabled) + if should_save_3d: + save_3d(view1, view2, pred1, pred2, os.path.join(save_dir, "3d_mesh"), as_pointcloud=False) + save_3d(view1, view2, pred1, pred2, os.path.join(save_dir, "3d_pointcloud"), as_pointcloud=True) + + # Save the color, depth and valid masks for the input context images + context_images = torch.stack([view["img"] for view in batch["context"]], dim=1) + context_original_images = torch.stack([view["original_img"] for view in batch["context"]], dim=1) + context_depthmaps = torch.stack([view["depthmap"] for view in batch["context"]], dim=1) + context_valid_masks = torch.stack([view["valid_mask"] for view in batch["context"]], dim=1) + for b in range(min(context_images.shape[0], 4)): + torchvision.utils.save_image(context_images[b], os.path.join(save_dir, f"sample_{b}_img_context.jpg")) + torchvision.utils.save_image(context_original_images[b], os.path.join(save_dir, f"sample_{b}_original_img_context.jpg")) + torchvision.utils.save_image(context_depthmaps[b, :, None, ...], os.path.join(save_dir, f"sample_{b}_depthmap.jpg"), normalize=True) + torchvision.utils.save_image(context_valid_masks[b, :, None, ...].float(), os.path.join(save_dir, f"sample_{b}_valid_mask_context.jpg"), normalize=True) + + # Save the color and depth images for the target images + target_original_images = torch.stack([view["original_img"] for view in batch["target"]], dim=1) + target_depthmaps = torch.stack([view["depthmap"] for view in batch["target"]], dim=1) + context_valid_masks = torch.stack([view["valid_mask"] for view in batch["context"]], dim=1) + for b in range(min(target_original_images.shape[0], 4)): + torchvision.utils.save_image(target_original_images[b], os.path.join(save_dir, f"sample_{b}_original_img_target.jpg")) + torchvision.utils.save_image(target_depthmaps[b, :, None, ...], os.path.join(save_dir, f"sample_{b}_depthmap_target.jpg"), normalize=True) + + # Save the rendered images and depths + for b in range(min(color.shape[0], 4)): + torchvision.utils.save_image(color[b, ...], os.path.join(save_dir, f"sample_{b}_rendered_color.jpg")) + if depth is not None: + for b in range(min(color.shape[0], 4)): + torchvision.utils.save_image(depth[b, :, None, ...], os.path.join(save_dir, f"sample_{b}_rendered_depth.jpg"), normalize=True) + + # Save the loss masks + for b in range(min(mask.shape[0], 4)): + torchvision.utils.save_image(mask[b, :, None, ...].float(), os.path.join(save_dir, f"sample_{b}_loss_mask.jpg"), normalize=True) + + # Save the masked target and rendered images + target_original_images = torch.stack([view["original_img"] for view in batch["target"]], dim=1) + masked_target_original_images = target_original_images * mask[..., None, :, :] + masked_predictions = color * mask[..., None, :, :] + for b in range(min(target_original_images.shape[0], 4)): + torchvision.utils.save_image(masked_target_original_images[b], os.path.join(save_dir, f"sample_{b}_masked_original_img_target.jpg")) + torchvision.utils.save_image(masked_predictions[b], os.path.join(save_dir, f"sample_{b}_masked_rendered_color.jpg")) \ No newline at end of file diff --git a/utils/geometry.py b/utils/geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..3b2f7fde4b15a24ee73207308b4c8728b511de3f --- /dev/null +++ b/utils/geometry.py @@ -0,0 +1,171 @@ +import einops +import torch + +# --- Intrinsics Transformations --- + +def normalize_intrinsics(intrinsics, image_shape): + '''Normalize an intrinsics matrix given the image shape''' + intrinsics = intrinsics.clone() + intrinsics[..., 0, :] /= image_shape[1] + intrinsics[..., 1, :] /= image_shape[0] + return intrinsics + + +def unnormalize_intrinsics(intrinsics, image_shape): + '''Unnormalize an intrinsics matrix given the image shape''' + intrinsics = intrinsics.clone() + intrinsics[..., 0, :] *= image_shape[1] + intrinsics[..., 1, :] *= image_shape[0] + return intrinsics + + +# --- Quaternions, Rotations and Scales --- + +def quaternion_to_matrix(quaternions, eps: float = 1e-8): + ''' + Convert the 4-dimensional quaternions to 3x3 rotation matrices. + This is adapted from Pytorch3D: + https://github.com/facebookresearch/pytorch3d/blob/main/pytorch3d/transforms/rotation_conversions.py + ''' + + # Order changed to match scipy format! + i, j, k, r = torch.unbind(quaternions, dim=-1) + two_s = 2 / ((quaternions * quaternions).sum(dim=-1) + eps) + + o = torch.stack( + ( + 1 - two_s * (j * j + k * k), + two_s * (i * j - k * r), + two_s * (i * k + j * r), + two_s * (i * j + k * r), + 1 - two_s * (i * i + k * k), + two_s * (j * k - i * r), + two_s * (i * k - j * r), + two_s * (j * k + i * r), + 1 - two_s * (i * i + j * j), + ), + -1, + ) + return einops.rearrange(o, "... (i j) -> ... i j", i=3, j=3) + + +def build_covariance(scale, rotation_xyzw): + '''Build the 3x3 covariance matrix from the three dimensional scale and the + four dimension quaternion''' + scale = scale.diag_embed() + rotation = quaternion_to_matrix(rotation_xyzw) + return ( + rotation + @ scale + @ einops.rearrange(scale, "... i j -> ... j i") + @ einops.rearrange(rotation, "... i j -> ... j i") + ) + + +# --- Projections --- + +def homogenize_points(points): + """Append a '1' along the final dimension of the tensor (i.e. convert xyz->xyz1)""" + return torch.cat([points, torch.ones_like(points[..., :1])], dim=-1) + + +def normalize_homogenous_points(points): + """Normalize the point vectors""" + return points / points[..., -1:] + + +def pixel_space_to_camera_space(pixel_space_points, depth, intrinsics): + """ + Convert pixel space points to camera space points. + + Args: + pixel_space_points (torch.Tensor): Pixel space points with shape (h, w, 2) + depth (torch.Tensor): Depth map with shape (b, v, h, w, 1) + intrinsics (torch.Tensor): Camera intrinsics with shape (b, v, 3, 3) + + Returns: + torch.Tensor: Camera space points with shape (b, v, h, w, 3). + """ + pixel_space_points = homogenize_points(pixel_space_points) + camera_space_points = torch.einsum('b v i j , h w j -> b v h w i', intrinsics.inverse(), pixel_space_points) + camera_space_points = camera_space_points * depth + return camera_space_points + + +def camera_space_to_world_space(camera_space_points, c2w): + """ + Convert camera space points to world space points. + + Args: + camera_space_points (torch.Tensor): Camera space points with shape (b, v, h, w, 3) + c2w (torch.Tensor): Camera to world extrinsics matrix with shape (b, v, 4, 4) + + Returns: + torch.Tensor: World space points with shape (b, v, h, w, 3). + """ + camera_space_points = homogenize_points(camera_space_points) + world_space_points = torch.einsum('b v i j , b v h w j -> b v h w i', c2w, camera_space_points) + return world_space_points[..., :3] + + +def camera_space_to_pixel_space(camera_space_points, intrinsics): + """ + Convert camera space points to pixel space points. + + Args: + camera_space_points (torch.Tensor): Camera space points with shape (b, v1, v2, h, w, 3) + c2w (torch.Tensor): Camera to world extrinsics matrix with shape (b, v2, 3, 3) + + Returns: + torch.Tensor: World space points with shape (b, v1, v2, h, w, 2). + """ + camera_space_points = normalize_homogenous_points(camera_space_points) + pixel_space_points = torch.einsum('b u i j , b v u h w j -> b v u h w i', intrinsics, camera_space_points) + return pixel_space_points[..., :2] + + +def world_space_to_camera_space(world_space_points, c2w): + """ + Convert world space points to pixel space points. + + Args: + world_space_points (torch.Tensor): World space points with shape (b, v1, h, w, 3) + c2w (torch.Tensor): Camera to world extrinsics matrix with shape (b, v2, 4, 4) + + Returns: + torch.Tensor: Camera space points with shape (b, v1, v2, h, w, 3). + """ + world_space_points = homogenize_points(world_space_points) + camera_space_points = torch.einsum('b u i j , b v h w j -> b v u h w i', c2w.inverse(), world_space_points) + return camera_space_points[..., :3] + + +def unproject_depth(depth, intrinsics, c2w): + """ + Turn the depth map into a 3D point cloud in world space + + Args: + depth: (b, v, h, w, 1) + intrinsics: (b, v, 3, 3) + c2w: (b, v, 4, 4) + + Returns: + torch.Tensor: World space points with shape (b, v, h, w, 3). + """ + + # Compute indices of pixels + h, w = depth.shape[-3], depth.shape[-2] + x_grid, y_grid = torch.meshgrid( + torch.arange(w, device=depth.device, dtype=torch.float32), + torch.arange(h, device=depth.device, dtype=torch.float32), + indexing='xy' + ) # (h, w), (h, w) + + # Compute coordinates of pixels in camera space + pixel_space_points = torch.stack((x_grid, y_grid), dim=-1) # (..., h, w, 2) + camera_points = pixel_space_to_camera_space(pixel_space_points, depth, intrinsics) # (..., h, w, 3) + + # Convert points to world space + world_points = camera_space_to_world_space(camera_points, c2w) # (..., h, w, 3) + + return world_points diff --git a/utils/loss_mask.py b/utils/loss_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..06e51dd3b48910692a325681d12404637cead13d --- /dev/null +++ b/utils/loss_mask.py @@ -0,0 +1,92 @@ +import einops +import torch + +from utils.geometry import unproject_depth, world_space_to_camera_space, camera_space_to_pixel_space + + +@torch.no_grad() +def calculate_in_frustum_mask(depth_1, intrinsics_1, c2w_1, depth_2, intrinsics_2, c2w_2): + """ + A function that takes in the depth, intrinsics and c2w matrices of two sets + of views, and then works out which of the pixels in the first set of views + has a direct corresponding pixel in any of views in the second set + + Args: + depth_1: (b, v1, h, w) + intrinsics_1: (b, v1, 3, 3) + c2w_1: (b, v1, 4, 4) + depth_2: (b, v2, h, w) + intrinsics_2: (b, v2, 3, 3) + c2w_2: (b, v2, 4, 4) + + Returns: + torch.Tensor: Camera space points with shape (b, v1, v2, h, w, 3). + """ + + _, v1, h, w = depth_1.shape + _, v2, _, _ = depth_2.shape + + # Unproject the depth to get the 3D points in world space + points_3d = unproject_depth(depth_1[..., None], intrinsics_1, c2w_1) # (b, v1, h, w, 3) + + # Project the 3D points into the pixel space of all the second views simultaneously + camera_points = world_space_to_camera_space(points_3d, c2w_2) # (b, v1, v2, h, w, 3) + points_2d = camera_space_to_pixel_space(camera_points, intrinsics_2) # (b, v1, v2, h, w, 2) + + # Calculate the depth of each point + rendered_depth = camera_points[..., 2] # (b, v1, v2, h, w) + + # We use three conditions to determine if a point should be masked + + # Condition 1: Check if the points are in the frustum of any of the v2 views + in_frustum_mask = ( + (points_2d[..., 0] > 0) & + (points_2d[..., 0] < w) & + (points_2d[..., 1] > 0) & + (points_2d[..., 1] < h) + ) # (b, v1, v2, h, w) + in_frustum_mask = in_frustum_mask.any(dim=-3) # (b, v1, h, w) + + # Condition 2: Check if the points have non-zero (i.e. valid) depth in the input view + non_zero_depth = depth_1 > 1e-6 + + # Condition 3: Check if the points have matching depth to any of the v2 + # views torch.nn.functional.grid_sample expects the input coordinates to + # be normalized to the range [-1, 1], so we normalize first + points_2d[..., 0] /= w + points_2d[..., 1] /= h + points_2d = points_2d * 2 - 1 + matching_depth = torch.ones_like(rendered_depth, dtype=torch.bool) + for b in range(depth_1.shape[0]): + for i in range(v1): + for j in range(v2): + depth = einops.rearrange(depth_2[b, j], 'h w -> 1 1 h w') + coords = einops.rearrange(points_2d[b, i, j], 'h w c -> 1 h w c') + sampled_depths = torch.nn.functional.grid_sample(depth, coords, align_corners=False)[0, 0] + matching_depth[b, i, j] = torch.isclose(rendered_depth[b, i, j], sampled_depths, atol=1e-1) + + matching_depth = matching_depth.any(dim=-3) # (..., v1, h, w) + + mask = in_frustum_mask & non_zero_depth & matching_depth + return mask + + +@torch.no_grad() +def calculate_loss_mask(batch): + '''Calcuate the loss mask for the target views in the batch''' + + target_depth = torch.stack([target_view['depthmap'] for target_view in batch['target']], dim=1) + target_intrinsics = torch.stack([target_view['camera_intrinsics'] for target_view in batch['target']], dim=1) + target_c2w = torch.stack([target_view['camera_pose'] for target_view in batch['target']], dim=1) + context_depth = torch.stack([context_view['depthmap'] for context_view in batch['context']], dim=1) + context_intrinsics = torch.stack([context_view['camera_intrinsics'] for context_view in batch['context']], dim=1) + context_c2w = torch.stack([context_view['camera_pose'] for context_view in batch['context']], dim=1) + + target_intrinsics = target_intrinsics[..., :3, :3] + context_intrinsics = context_intrinsics[..., :3, :3] + + mask = calculate_in_frustum_mask( + target_depth, target_intrinsics, target_c2w, + context_depth, context_intrinsics, context_c2w + ) + return mask diff --git a/utils/sh_utils.py b/utils/sh_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..bbca7d192aa3a7edf8c5b2d24dee535eac765785 --- /dev/null +++ b/utils/sh_utils.py @@ -0,0 +1,118 @@ +# Copyright 2021 The PlenOctree Authors. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import torch + +C0 = 0.28209479177387814 +C1 = 0.4886025119029199 +C2 = [ + 1.0925484305920792, + -1.0925484305920792, + 0.31539156525252005, + -1.0925484305920792, + 0.5462742152960396 +] +C3 = [ + -0.5900435899266435, + 2.890611442640554, + -0.4570457994644658, + 0.3731763325901154, + -0.4570457994644658, + 1.445305721320277, + -0.5900435899266435 +] +C4 = [ + 2.5033429417967046, + -1.7701307697799304, + 0.9461746957575601, + -0.6690465435572892, + 0.10578554691520431, + -0.6690465435572892, + 0.47308734787878004, + -1.7701307697799304, + 0.6258357354491761, +] + + +def eval_sh(deg, sh, dirs): + """ + Evaluate spherical harmonics at unit directions + using hardcoded SH polynomials. + Works with torch/np/jnp. + ... Can be 0 or more batch dimensions. + Args: + deg: int SH deg. Currently, 0-3 supported + sh: jnp.ndarray SH coeffs [..., C, (deg + 1) ** 2] + dirs: jnp.ndarray unit directions [..., 3] + Returns: + [..., C] + """ + assert deg <= 4 and deg >= 0 + coeff = (deg + 1) ** 2 + assert sh.shape[-1] >= coeff + + result = C0 * sh[..., 0] + if deg > 0: + x, y, z = dirs[..., 0:1], dirs[..., 1:2], dirs[..., 2:3] + result = (result - + C1 * y * sh[..., 1] + + C1 * z * sh[..., 2] - + C1 * x * sh[..., 3]) + + if deg > 1: + xx, yy, zz = x * x, y * y, z * z + xy, yz, xz = x * y, y * z, x * z + result = (result + + C2[0] * xy * sh[..., 4] + + C2[1] * yz * sh[..., 5] + + C2[2] * (2.0 * zz - xx - yy) * sh[..., 6] + + C2[3] * xz * sh[..., 7] + + C2[4] * (xx - yy) * sh[..., 8]) + + if deg > 2: + result = (result + + C3[0] * y * (3 * xx - yy) * sh[..., 9] + + C3[1] * xy * z * sh[..., 10] + + C3[2] * y * (4 * zz - xx - yy)* sh[..., 11] + + C3[3] * z * (2 * zz - 3 * xx - 3 * yy) * sh[..., 12] + + C3[4] * x * (4 * zz - xx - yy) * sh[..., 13] + + C3[5] * z * (xx - yy) * sh[..., 14] + + C3[6] * x * (xx - 3 * yy) * sh[..., 15]) + + if deg > 3: + result = (result + C4[0] * xy * (xx - yy) * sh[..., 16] + + C4[1] * yz * (3 * xx - yy) * sh[..., 17] + + C4[2] * xy * (7 * zz - 1) * sh[..., 18] + + C4[3] * yz * (7 * zz - 3) * sh[..., 19] + + C4[4] * (zz * (35 * zz - 30) + 3) * sh[..., 20] + + C4[5] * xz * (7 * zz - 3) * sh[..., 21] + + C4[6] * (xx - yy) * (7 * zz - 1) * sh[..., 22] + + C4[7] * xz * (xx - 3 * yy) * sh[..., 23] + + C4[8] * (xx * (xx - 3 * yy) - yy * (3 * xx - yy)) * sh[..., 24]) + return result + +def RGB2SH(rgb): + return (rgb - 0.5) / C0 + +def SH2RGB(sh): + return sh * C0 + 0.5 \ No newline at end of file diff --git a/workspace.py b/workspace.py new file mode 100644 index 0000000000000000000000000000000000000000..6fbc71734b5cd95aee1fe732a16809c9e1093f5a --- /dev/null +++ b/workspace.py @@ -0,0 +1,83 @@ +import logging +import os +import time + +import git +import omegaconf + +logger = logging.getLogger(__name__) + + +def load_config(config_path, command_line_args=None): + """Loads the config file using OmegaConf, performing merges with base configs and the command line arguments""" + + logger.info(f"Loading from: {config_path}") + + # Load the config using OmegaConf + config = omegaconf.OmegaConf.load(config_path) + + # Load all the base configs to include, and merge them with the current config, giving precedence to the current config + if hasattr(config, "include"): + base_config_paths = [os.path.join(os.path.dirname(config_path), include_path) for include_path in config.include] + base_configs = [load_config(base_config_path) for base_config_path in base_config_paths] + config = omegaconf.OmegaConf.merge(*base_configs, config) + + # Load the command line arguments, and merge them with the current config, giving precedence to the command line + if command_line_args is not None: + command_line_config = omegaconf.OmegaConf.from_dotlist(command_line_args) + config = omegaconf.OmegaConf.merge(config, command_line_config) + + return config + + +def save_git_commit_info(save_path): + """Use gitpython to save info about the current git commit to a file""" + + repo = git.Repo(search_parent_directories=True) + head_commit = repo.head.commit + git_commit_info = { + "hexsha": head_commit.hexsha, + "authored": { + "author": head_commit.author.name, + "authored_time": head_commit.authored_date, + }, + "committed": { + "commit": head_commit.committer.name, + "committed_time": head_commit.committed_date, + }, + "message": head_commit.message.strip(), + } + + git_commit_info = omegaconf.OmegaConf.create(git_commit_info) + omegaconf.OmegaConf.save(git_commit_info, save_path) + return git_commit_info + + +def create_workspace(config): + """Create a results folder in the target directory""" + + # Treat the name as a time.strftime format string (so that every experiment is named after when it was run) + config.name = time.strftime(config.name, time.localtime()) + + # Create the results directory + os.makedirs(config.save_dir) + + # Save the config to the results directory + omegaconf.OmegaConf.save(config, os.path.join(config.save_dir, "config.yaml")) + save_git_commit_info(os.path.join(config.save_dir, "git.yaml")) + + # Set up the print loggers by removing all handlers associated with the root logger object, + # then setting up the logger to print messages *and* save them to a file + for handler in logging.root.handlers: + logging.root.removeHandler(handler) + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO, + handlers=[ + logging.FileHandler(os.path.join(config.save_dir, "output.log")), + logging.StreamHandler(), + ], + ) + + return config