import gradio as gr import numpy as np from torchvision import transforms import torch from helpers import * import sys import csv from monoscene.monoscene import MonoScene csv.field_size_limit(sys.maxsize) torch.set_grad_enabled(False) # pipeline = pipeline(model="anhquancao/monoscene_kitti") # model = AutoModel.from_pretrained( # "anhquancao/monoscene_kitti", trust_remote_code=True, revision='bf033f87c2a86b60903ab811b790a1532c1ae313' # )#.cuda() model = MonoScene.load_from_checkpoint( "monoscene_nyu.ckpt", dataset="NYU", feature=200, project_scale=1, full_scene_size=(60, 36, 60), ) img_W, img_H = 640, 480 def predict(img): img = np.array(img, dtype=np.float32, copy=False) / 255.0 normalize_rgb = transforms.Compose( [ transforms.ToTensor(), transforms.Normalize( mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] ), ] ) img = normalize_rgb(img) batch = get_projections(img_W, img_H) batch["img"] = img for k in batch: batch[k] = batch[k].unsqueeze(0) # .cuda() pred = model(batch).squeeze() y_pred = torch.softmax(pred["ssc_logit"], dim=1).detach().cpu().numpy() cam_pose = np.asarray([[ 9.6699458e-01, 4.2662762e-02, 2.5120059e-01, 0.0000000e+00], [-2.5147417e-01, 1.0867463e-03, 9.6786356e-01, 0.0000000e+00], [ 4.1018680e-02, -9.9908894e-01, 1.1779292e-02, 1.1794727e+00], [ 0.0000000e+00, 0.0000000e+00, 0.0000000e+00, 1.0000000e+00]]) vox_origin = np.array([-1.54591799, 0.8907361 , -0.05 ]) fig = draw(y_pred.squeeze(),cam_pose, vox_origin) return fig description = """ MonoScene Demo on SemanticKITTI Validation Set (Sequence 08), which uses the camera parameters of Sequence 08. Due to the CPU-only inference, it might take up to 20s to predict a scene. \n The output is downsampled by 2 for faster rendering. Darker colors represent the scenery outside the Field of View, i.e. not visible on the image.