import os import gradio as gr from PIL import Image import torch import matplotlib.pyplot as plt import imageio import numpy as np import argparse from point_e.diffusion.configs import DIFFUSION_CONFIGS, diffusion_from_config from point_e.diffusion.sampler import PointCloudSampler from point_e.models.download import load_checkpoint from point_e.models.configs import MODEL_CONFIGS, model_from_config from point_e.util.plotting import plot_point_cloud from point_e.util.pc_to_mesh import marching_cubes_mesh from diffusers import StableDiffusionPipeline import trimesh state = "" device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') css = ''' .instruction{position: absolute; top: 0;right: 0;margin-top: 0px !important} .arrow{position: absolute;top: 0;right: -110px;margin-top: -8px !important} #component-4, #component-3, #component-10{min-height: 0} .duplicate-button img{margin: 0} ''' def set_state(s): print(s) global state state = s def get_state(): return state def load_img2mesh_model(model_name): set_state(f'Creating img2mesh model {model_name}...') i2m_name = model_name i2m_model = model_from_config(MODEL_CONFIGS[i2m_name], device) i2m_model.eval() base_diffusion_i2m = diffusion_from_config(DIFFUSION_CONFIGS[i2m_name]) set_state(f'Downloading img2mesh checkpoint {model_name}...') i2m_model.load_state_dict(load_checkpoint(i2m_name, device)) return i2m_model, base_diffusion_i2m def get_sampler(model_name, txt2obj, guidance_scale): if txt2obj: set_state('Creating txt2mesh model...') t2m_name = 'base40M-textvec' t2m_model = model_from_config(MODEL_CONFIGS[t2m_name], device) t2m_model.eval() base_diffusion_t2m = diffusion_from_config(DIFFUSION_CONFIGS[t2m_name]) set_state('Downloading txt2mesh checkpoint...') t2m_model.load_state_dict(load_checkpoint(t2m_name, device)) else: i2m_model, base_diffusion_i2m = load_img2mesh_model(model_name) set_state('Creating upsample model...') upsampler_model = model_from_config(MODEL_CONFIGS['upsample'], device) upsampler_model.eval() upsampler_diffusion = diffusion_from_config(DIFFUSION_CONFIGS['upsample']) set_state('Downloading upsampler checkpoint...') upsampler_model.load_state_dict(load_checkpoint('upsample', device)) return PointCloudSampler( device=device, models=[t2m_model if txt2obj else i2m_model, upsampler_model], diffusions=[base_diffusion_t2m if txt2obj else base_diffusion_i2m, upsampler_diffusion], num_points=[1024, 4096 - 1024], aux_channels=['R', 'G', 'B'], guidance_scale=[guidance_scale, 0.0 if txt2obj else guidance_scale], model_kwargs_key_filter=('texts', '') if txt2obj else ("*",) ) def generate_txt2img(prompt): pipe = StableDiffusionPipeline.from_pretrained("point_e_model_cache/stable-diffusion-2-1", torch_dtype=torch.float16) pipe = pipe.to("cuda") image = pipe(prompt).images[0] return image def generate_3D(input, model_name='base1B', guidance_scale=3.0, grid_size=128): set_state('Entered generate function...') # try: # input = Image.fromarray(input) # except: # img = generate_txt2img(input) # img.save('/tmp/img.png') # input = Image.open('/tmp/img.png') if isinstance(input, Image.Image): input = prepare_img(input) # if input is a string, it's a text prompt sampler = get_sampler(model_name, txt2obj=True if isinstance(input, str) else False, guidance_scale=guidance_scale) # Produce a sample from the model. set_state('Sampling...') samples = None kw_args = dict(texts=[input]) if isinstance(input, str) else dict(images=[input]) for x in sampler.sample_batch_progressive(batch_size=1, model_kwargs=kw_args): samples = x set_state('Converting to point cloud...') pc = sampler.output_to_point_clouds(samples)[0] set_state('Converting to mesh...') save_ply(pc, '/tmp/mesh.ply', grid_size) set_state('') return ply_to_glb('/tmp/mesh.ply', '/tmp/mesh.glb'), create_gif(pc), gr.update(value=['/tmp/mesh.glb', '/tmp/mesh.ply'], visible=True) def prepare_img(img): w, h = img.size if w > h: img = img.crop((w - h) / 2, 0, w - (w - h) / 2, h) else: img = img.crop((0, (h - w) / 2, w, h - (h - w) / 2)) # resize to 256x256 img = img.resize((256, 256)) return img def ply_to_glb(ply_file, glb_file): mesh = trimesh.load(ply_file) # Save the mesh as a glb file using Trimesh mesh.export(glb_file, file_type='glb') return glb_file def save_ply(pc, file_name, grid_size): set_state('Creating SDF model...') sdf_name = 'sdf' sdf_model = model_from_config(MODEL_CONFIGS[sdf_name], device) sdf_model.eval() set_state('Loading SDF model...') sdf_model.load_state_dict(load_checkpoint(sdf_name, device)) # Produce a mesh (with vertex colors) mesh = marching_cubes_mesh( pc=pc, model=sdf_model, batch_size=4096, grid_size=grid_size, # increase to 128 for resolution used in evals progress=True, ) # Write the mesh to a PLY file to import into some other program. with open(file_name, 'wb') as f: mesh.write_ply(f) def create_gif(pc): fig = plt.figure(facecolor='black', figsize=(4, 4)) ax = fig.add_subplot(111, projection='3d', facecolor='black') fixed_bounds=((-0.75, -0.75, -0.75),(0.75, 0.75, 0.75)) # Create an empty list to store the frames frames = [] # Create a loop to generate the frames for the GIF for angle in range(0, 360, 4): # Clear the plot and plot the point cloud ax.clear() color_args = np.stack( [pc.channels["R"], pc.channels["G"], pc.channels["B"]], axis=-1 ) c = pc.coords ax.scatter(c[:, 0], c[:, 1], c[:, 2], c=color_args) # Set the viewpoint for the plot ax.view_init(elev=10, azim=angle) # Turn off the axis labels and ticks ax.axis('off') ax.set_xlim3d(fixed_bounds[0][0], fixed_bounds[1][0]) ax.set_ylim3d(fixed_bounds[0][1], fixed_bounds[1][1]) ax.set_zlim3d(fixed_bounds[0][2], fixed_bounds[1][2]) # Draw the figure to update the image data fig.canvas.draw() # Save the plot as a frame for the GIF frame = np.array(fig.canvas.renderer.buffer_rgba()) w, h = frame.shape[0], frame.shape[1] i = int(round((h - int(h*0.6)) / 2.)) frame = frame[i:i + int(h*0.6),i:i + int(h*0.6)] frames.append(frame) # Save the GIF using imageio imageio.mimsave('/tmp/pointcloud.mp4', frames, fps=30) return '/tmp/pointcloud.mp4' block = gr.Blocks().queue(max_size=250, concurrency_count=6) with block: with gr.Box(): if(not torch.cuda.is_available()): top_description = gr.HTML(f''' ''') else: top_description = gr.HTML(f''' ''') with gr.Row(): with gr.Column(): with gr.Tab("Image to 3D"): gr.Markdown("Best results with images of objects on an empty background.") input_image = gr.Image(label="Image") img_button = gr.Button(label="Generate") with gr.Tab("Text to 3D"): gr.Markdown("Uses Stable Diffusion to create an image from the prompt.") prompt = gr.Textbox(label="Prompt", placeholder="A HD photo of a Corgi") text_button = gr.Button(label="Generate") with gr.Accordion("Advanced options", open=False): model = gr.Radio(["base40M", "base300M", "base1B"], label="Model", value="base1B") scale = gr.Slider( label="Guidance Scale", minimum=1.0, maximum=10.0, value=3.0, step=0.1 ) with gr.Column(): model_gif = gr.Video(label="3D Model GIF") # btn_pc_to_obj = gr.Button(value="Convert to OBJ", visible=False) model_3d = gr.Model3D(value=None) file_out = gr.File(label="Files", visible=False) if torch.cuda.is_available(): gr.Examples( examples=[ ["images/pumpkin.png"], ["images/fantasy_world.png"], ], inputs=[input_image], outputs=[model_3d, model_gif, file_out], fn=generate_3D, cache_examples=True ) img_button.click(fn=generate_3D, inputs=[input_image, model, scale], outputs=[model_3d, model_gif, file_out]) text_button.click(fn=generate_3D, inputs=[prompt, model, scale], outputs=[model_3d, model_gif, file_out]) block.launch(show_api=False)