Update scripts/utils.py

scripts/utils.py

@@ -1,19 +1,18 @@
 import torch
 import numpy as np
 from PIL import Image
-import pymeshlab as ml
+import pymeshlab
+import trimesh
 from pytorch3d.renderer import TexturesVertex
 from pytorch3d.structures import Meshes
 from rembg import new_session, remove
-import trimesh
-from typing import List, Tuple
 import torch.nn.functional as F
+from typing import List, Tuple
+from pygltflib import GLTF2, Material, PbrMetallicRoughness
 
 # Constants
-NEG_PROMPT = "sketch, sculpture, hand drawing, outline, single color, NSFW, lowres, bad anatomy, bad hands, text, error, missing fingers, yellow sleeves, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry, (worst quality:1.4), (low quality:1.4)"
 
-# CUDA Configuration
-CUDA_PROVIDERS = [
+providers = [
     ('CUDAExecutionProvider', {
         'device_id': 0,
         'arena_extend_strategy': 'kSameAsRequested',
@@ -22,21 +21,22 @@ CUDA_PROVIDERS = [
     })
 ]
 
-# Initialize rembg session
-rembg_session = new_session(providers=CUDA_PROVIDERS)
+session = new_session(providers=providers)
+
+NEG_PROMPT = "sketch, sculpture, hand drawing, outline, single color, NSFW, lowres, bad anatomy,bad hands, text, error, missing fingers, yellow sleeves, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry,(worst quality:1.4),(low quality:1.4)"
+
+# Helper functions
+
 
-# Mesh Loading and Conversion Functions
 def load_mesh_with_trimesh(file_name, file_type=None):
     mesh = trimesh.load(file_name, file_type=file_type)
     if isinstance(mesh, trimesh.Scene):
         mesh = _process_trimesh_scene(mesh)
-    
-    vertices = torch.from_numpy(mesh.vertices).T
-    faces = torch.from_numpy(mesh.faces).T
-    colors = _get_mesh_colors(mesh)
-    
+
+    vertices, faces, colors = _extract_mesh_data(mesh)
     return vertices, faces, colors
 
+
 def _process_trimesh_scene(mesh):
     from io import BytesIO
     with BytesIO() as f:
@@ -46,31 +46,63 @@ def _process_trimesh_scene(mesh):
     if isinstance(mesh, trimesh.Scene):
         mesh = trimesh.util.concatenate(
             tuple(trimesh.Trimesh(vertices=g.vertices, faces=g.faces)
-                for g in mesh.geometry.values()))
+                  for g in mesh.geometry.values()))
     return mesh
 
+
+def _extract_mesh_data(mesh):
+    vertices = torch.from_numpy(mesh.vertices).T
+    faces = torch.from_numpy(mesh.faces).T
+    colors = _get_mesh_colors(mesh)
+    return vertices, faces, colors
+
+
 def _get_mesh_colors(mesh):
     if mesh.visual is not None and hasattr(mesh.visual, 'vertex_colors'):
         return torch.from_numpy(mesh.visual.vertex_colors)[..., :3].T / 255.
-    return torch.ones_like(mesh.vertices.T) * 0.5
+    return torch.ones_like(mesh.vertices).T * 0.5
+
 
-# Mesh Conversion Functions
-def meshlab_mesh_to_py3dmesh(mesh: ml.Mesh) -> Meshes:
+def meshlab_mesh_to_py3dmesh(mesh: pymeshlab.Mesh) -> Meshes:
     verts = torch.from_numpy(mesh.vertex_matrix()).float()
     faces = torch.from_numpy(mesh.face_matrix()).long()
     colors = torch.from_numpy(mesh.vertex_color_matrix()[..., :3]).float()
     textures = TexturesVertex(verts_features=[colors])
     return Meshes(verts=[verts], faces=[faces], textures=textures)
 
-def py3dmesh_to_meshlab_mesh(meshes: Meshes) -> ml.Mesh:
-    colors_in = F.pad(meshes.textures.verts_features_packed().cpu().float(), [0,1], value=1).numpy().astype(np.float64)
-    return ml.Mesh(
+
+def py3dmesh_to_meshlab_mesh(meshes: Meshes) -> pymeshlab.Mesh:
+    colors_in = F.pad(meshes.textures.verts_features_packed().cpu().float(), [
+                      0, 1], value=1).numpy().astype(np.float64)
+    return pymeshlab.Mesh(
         vertex_matrix=meshes.verts_packed().cpu().float().numpy().astype(np.float64),
         face_matrix=meshes.faces_packed().cpu().long().numpy().astype(np.int32),
-        v_normals_matrix=meshes.verts_normals_packed().cpu().float().numpy().astype(np.float64),
+        v_normals_matrix=meshes.verts_normals_packed(
+        ).cpu().float().numpy().astype(np.float64),
         v_color_matrix=colors_in)
 
-# Normal Map Rotation Functions
+
+def to_pyml_mesh(vertices, faces):
+    return pymeshlab.Mesh(
+        vertex_matrix=vertices.cpu().float().numpy().astype(np.float64),
+        face_matrix=faces.cpu().long().numpy().astype(np.int32),
+    )
+
+
+def to_py3d_mesh(vertices, faces, normals=None):
+    mesh = Meshes(verts=[vertices], faces=[faces], textures=None)
+    if normals is None:
+        normals = mesh.verts_normals_packed()
+    mesh.textures = TexturesVertex(verts_features=[normals / 2 + 0.5])
+    return mesh
+
+
+def from_py3d_mesh(mesh):
+    return mesh.verts_list()[0], mesh.faces_list()[0], mesh.textures.verts_features_packed()
+
+# Normal map rotation functions
+
+
 def rotate_normalmap_by_angle(normal_map: np.ndarray, angle: float):
     angle_rad = np.radians(angle)
     R = np.array([
@@ -80,105 +112,164 @@ def rotate_normalmap_by_angle(normal_map: np.ndarray, angle: float):
     ])
     return np.dot(normal_map.reshape(-1, 3), R.T).reshape(normal_map.shape)
 
-def rotate_normals(normal_pils, return_types='np', rotate_direction=1):
+
+def rotate_normals(normal_pils, return_types='np', rotate_direction=1) -> np.ndarray:
     n_views = len(normal_pils)
     ret = []
     for idx, rgba_normal in enumerate(normal_pils):
-        normal_np = _process_normal_map(rgba_normal, idx, n_views, rotate_direction)
-        ret.append(_format_output(normal_np, return_types))
+        normal_np, alpha_np = _process_normal_image(rgba_normal)
+        normal_np = rotate_normalmap_by_angle(
+            normal_np, rotate_direction * idx * (360 / n_views))
+        rgba_normal_np = _combine_normal_and_alpha(normal_np, alpha_np)
+        ret.append(_format_output(rgba_normal_np, return_types))
     return ret
 
-def _process_normal_map(rgba_normal, idx, n_views, rotate_direction):
+
+def _process_normal_image(rgba_normal):
     normal_np = np.array(rgba_normal)[:, :, :3] / 255 * 2 - 1
     alpha_np = np.array(rgba_normal)[:, :, 3] / 255
-    normal_np = rotate_normalmap_by_angle(normal_np, rotate_direction * idx * (360 / n_views))
-    normal_np = (normal_np + 1) / 2 * alpha_np[..., None]
+    return normal_np, alpha_np
+
+
+def _combine_normal_and_alpha(normal_np, alpha_np):
+    normal_np = (normal_np + 1) / 2
+    normal_np = normal_np * alpha_np[..., None]
     return np.concatenate([normal_np * 255, alpha_np[:, :, None] * 255], axis=-1)
 
-def _format_output(normal_np, return_types):
+
+def _format_output(rgba_normal_np, return_types):
     if return_types == 'np':
-        return normal_np
+        return rgba_normal_np
     elif return_types == 'pil':
-        return Image.fromarray(normal_np.astype(np.uint8))
+        return Image.fromarray(rgba_normal_np.astype(np.uint8))
     else:
-        raise ValueError(f"return_types should be 'np' or 'pil', but got {return_types}")
+        raise ValueError(
+            f"return_types should be 'np' or 'pil', but got {return_types}")
+
+
+def rotate_normalmap_by_angle_torch(normal_map, angle):
+    angle_rad = torch.tensor(np.radians(angle)).to(normal_map)
+    R = torch.tensor([
+        [torch.cos(angle_rad), 0, torch.sin(angle_rad)],
+        [0, 1, 0],
+        [-torch.sin(angle_rad), 0, torch.cos(angle_rad)]
+    ]).to(normal_map)
+    return torch.matmul(normal_map.view(-1, 3), R.T).view(normal_map.shape)
+
+
+def do_rotate(rgba_normal, angle):
+    rgba_normal = torch.from_numpy(rgba_normal).float().cuda() / 255
+    rotated_normal_tensor = rotate_normalmap_by_angle_torch(
+        rgba_normal[..., :3] * 2 - 1, angle)
+    rotated_normal_tensor = (rotated_normal_tensor + 1) / 2
+    rotated_normal_tensor = rotated_normal_tensor * rgba_normal[:, :, [3]]
+    return torch.cat([rotated_normal_tensor * 255, rgba_normal[:, :, [3]] * 255], dim=-1).cpu().numpy()
+
+
+def rotate_normals_torch(normal_pils, return_types='np', rotate_direction=1):
+    n_views = len(normal_pils)
+    ret = []
+    for idx, rgba_normal in enumerate(normal_pils):
+        angle = rotate_direction * idx * (360 / n_views)
+        rgba_normal_np = do_rotate(np.array(rgba_normal), angle)
+        ret.append(_format_output(rgba_normal_np, return_types))
+    return ret
+
+# Background change functions
+
 
-# Background Change Functions
 def change_bkgd(img_pils, new_bkgd=(0., 0., 0.)):
     new_bkgd = np.array(new_bkgd).reshape(1, 1, 3)
-    return [_process_image(rgba_img, new_bkgd) for rgba_img in img_pils]
+    return [_change_single_image_bkgd(rgba_img, new_bkgd) for rgba_img in img_pils]
+
 
-def _process_image(rgba_img, new_bkgd):
-    img_np = np.array(rgba_img)[:, :, :3] / 255
-    alpha_np = np.array(rgba_img)[:, :, 3] / 255
+def _change_single_image_bkgd(rgba_img, new_bkgd):
+    img_np, alpha_np = np.array(
+        rgba_img)[:, :, :3] / 255, np.array(rgba_img)[:, :, 3] / 255
     ori_bkgd = img_np[:1, :1]
     alpha_np_clamp = np.clip(alpha_np, 1e-6, 1)
-    ori_img_np = (img_np - ori_bkgd * (1 - alpha_np[..., None])) / alpha_np_clamp[..., None]
-    img_np = np.where(alpha_np[..., None] > 0.05, ori_img_np * alpha_np[..., None] + new_bkgd * (1 - alpha_np[..., None]), new_bkgd)
-    rgba_img_np = np.concatenate([img_np * 255, alpha_np[..., None] * 255], axis=-1)
+    ori_img_np = (img_np - ori_bkgd *
+                  (1 - alpha_np[..., None])) / alpha_np_clamp[..., None]
+    img_np = np.where(alpha_np[..., None] > 0.05, ori_img_np *
+                      alpha_np[..., None] + new_bkgd * (1 - alpha_np[..., None]), new_bkgd)
+    rgba_img_np = np.concatenate(
+        [img_np * 255, alpha_np[..., None] * 255], axis=-1)
     return Image.fromarray(rgba_img_np.astype(np.uint8))
 
-# Mesh Cleaning Function
-def simple_clean_mesh(pyml_mesh: ml.Mesh, apply_smooth=True, stepsmoothnum=1, apply_sub_divide=False, sub_divide_threshold=0.25):
-    ms = ml.MeshSet()
-    ms.add_mesh(pyml_mesh, "cube_mesh")
-    
-    if apply_smooth:
-        ms.apply_filter("apply_coord_laplacian_smoothing", stepsmoothnum=stepsmoothnum, cotangentweight=False)
-    if apply_sub_divide:
-        ms.apply_filter("meshing_repair_non_manifold_vertices")
-        ms.apply_filter("meshing_repair_non_manifold_edges", method='Remove Faces')
-        ms.apply_filter("meshing_surface_subdivision_loop", iterations=2, threshold=ml.PercentageValue(sub_divide_threshold))
-    return meshlab_mesh_to_py3dmesh(ms.current_mesh())
 
-# Image Processing Functions
-def expand2square(pil_img, background_color):
-    width, height = pil_img.size
-    if width == height:
-        return pil_img
-    new_size = max(width, height)
-    result = Image.new(pil_img.mode, (new_size, new_size), background_color)
-    offset = ((new_size - width) // 2, (new_size - height) // 2)
-    result.paste(pil_img, offset)
-    return result
+def change_bkgd_to_normal(normal_pils) -> List[Image.Image]:
+    n_views = len(normal_pils)
+    return [_change_single_normal_bkgd(rgba_normal, idx, n_views) for idx, rgba_normal in enumerate(normal_pils)]
 
-def simple_preprocess(input_image, rembg_session=rembg_session, background_color=255):
-    RES = 2048
-    input_image.thumbnail([RES, RES], Image.Resampling.LANCZOS)
-    if input_image.mode != 'RGBA':
-        image_rem = input_image.convert('RGBA')
-        input_image = remove(image_rem, alpha_matting=False, session=rembg_session)
 
-    arr = np.asarray(input_image)
-    alpha = arr[:, :, -1]
-    x_nonzero, y_nonzero = np.nonzero(alpha > 60)
-    x_min, x_max = x_nonzero.min(), x_nonzero.max()
-    y_min, y_max = y_nonzero.min(), y_nonzero.max()
-    arr = arr[x_min:x_max+1, y_min:y_max+1]
-    input_image = Image.fromarray(arr)
-    return expand2square(input_image, (background_color, background_color, background_color, 0))
+def _change_single_normal_bkgd(rgba_normal, idx, n_views):
+    target_bkgd = rotate_normalmap_by_angle(
+        np.array([[[0., 0., 1.]]]), idx * (360 / n_views))
+    normal_np, alpha_np = np.array(
+        rgba_normal)[:, :, :3] / 255 * 2 - 1, np.array(rgba_normal)[:, :, 3] / 255
+    old_bkgd = normal_np[:1, :1]
+    normal_np[alpha_np > 0.05] = (normal_np[alpha_np > 0.05] - old_bkgd * (
+        1 - alpha_np[alpha_np > 0.05][..., None])) / alpha_np[alpha_np > 0.05][..., None]
+    normal_np = normal_np * alpha_np[..., None] + \
+        target_bkgd * (1 - alpha_np[..., None])
+    normal_np = (normal_np + 1) / 2
+    rgba_normal_np = np.concatenate(
+        [normal_np * 255, alpha_np[..., None] * 255], axis=-1)
+    return Image.fromarray(rgba_normal_np.astype(np.uint8))
+
+# Mesh and GLB handling functions
+
+
+def fix_vert_color_glb(mesh_path):
+    obj1 = GLTF2().load(mesh_path)
+    obj1.meshes[0].primitives[0].material = 0
+    obj1.materials.append(Material(
+        pbrMetallicRoughness=PbrMetallicRoughness(
+            baseColorFactor=[1.0, 1.0, 1.0, 1.0],
+            metallicFactor=0.,
+            roughnessFactor=1.0,
+        ),
+        emissiveFactor=[0.0, 0.0, 0.0],
+        doubleSided=True,
+    ))
+    obj1.save(mesh_path)
+
+
+def srgb_to_linear(c_srgb):
+    return np.where(c_srgb <= 0.04045, c_srgb / 12.92, ((c_srgb + 0.055) / 1.055) ** 2.4).clip(0, 1.)
+
 
-# Mesh Saving Functions
 def save_py3dmesh_with_trimesh_fast(meshes: Meshes, save_glb_path, apply_sRGB_to_LinearRGB=True):
-    vertices = meshes.verts_packed().cpu().float().numpy()
-    triangles = meshes.faces_packed().cpu().long().numpy()
-    np_color = meshes.textures.verts_features_packed().cpu().float().numpy()
-    
+    vertices, triangles, np_color = _extract_mesh_data_for_trimesh(meshes)
+
     if save_glb_path.endswith(".glb"):
         vertices[:, [0, 2]] = -vertices[:, [0, 2]]
 
     if apply_sRGB_to_LinearRGB:
         np_color = srgb_to_linear(np_color)
-    
-    mesh = trimesh.Trimesh(vertices=vertices, faces=triangles, vertex_colors=np_color)
+
+    mesh = trimesh.Trimesh(
+        vertices=vertices, faces=triangles, vertex_colors=np_color)
     mesh.remove_unreferenced_vertices()
     mesh.export(save_glb_path)
-    
+
     if save_glb_path.endswith(".glb"):
         fix_vert_color_glb(save_glb_path)
-    print(f"Saved to {save_glb_path}")
+    print(f"saving to {save_glb_path}")
 
-def save_glb_and_video(save_mesh_prefix: str, meshes: Meshes, with_timestamp=True, **kwargs) -> Tuple[str, str]:
+
+def _extract_mesh_data_for_trimesh(meshes):
+    vertices = meshes.verts_packed().cpu().float().numpy()
+    triangles = meshes.faces_packed().cpu().long().numpy()
+    np_color = meshes.textures.verts_features_packed().cpu().float().numpy()
+    assert vertices.shape[0] == np_color.shape[0]
+    assert np_color.shape[1] == 3
+    assert 0 <= np_color.min() and np_color.max(
+    ) <= 1, f"min={np_color.min()}, max={np_color.max()}"
+    return vertices, triangles, np_color
+
+
+def save_glb_and_video(save_mesh_prefix: str, meshes: Meshes, with_timestamp=True, dist=3.5, azim_offset=180, resolution=512, fov_in_degrees=1 / 1.15, cam_type="ortho", view_padding=60, export_video=True) -> Tuple[str, str]:
     import time
     if '.' in save_mesh_prefix:
         save_mesh_prefix = ".".join(save_mesh_prefix.split('.')[:-1])
@@ -188,38 +279,89 @@ def save_glb_and_video(save_mesh_prefix: str, meshes: Meshes, with_timestamp=Tru
     save_py3dmesh_with_trimesh_fast(meshes, ret_mesh)
     return ret_mesh, None
 
-# Utility Functions
-def srgb_to_linear(c_srgb):
-    return np.where(c_srgb <= 0.04045, c_srgb / 12.92, ((c_srgb + 0.055) / 1.055) ** 2.4).clip(0, 1.)
+# Mesh cleaning and preprocessing functions (continued)
+
+
+def simple_clean_mesh(pyml_mesh: pymeshlab.Mesh, apply_smooth=True, stepsmoothnum=1, apply_sub_divide=False, sub_divide_threshold=0.25):
+    ms = pymeshlab.MeshSet()
+    ms.add_mesh(pyml_mesh, "cube_mesh")
+
+    if apply_smooth:
+        ms.apply_filter("apply_coord_laplacian_smoothing",
+                        stepsmoothnum=stepsmoothnum, cotangentweight=False)
+
+    if apply_sub_divide:
+        ms.apply_filter("meshing_repair_non_manifold_vertices")
+        ms.apply_filter("meshing_repair_non_manifold_edges",
+                        method='Remove Faces')
+        ms.apply_filter("meshing_surface_subdivision_loop", iterations=2,
+                        threshold=pymeshlab.PercentageValue(sub_divide_threshold))
+
+    return meshlab_mesh_to_py3dmesh(ms.current_mesh())
+
+
+def expand2square(pil_img, background_color):
+    width, height = pil_img.size
+    if width == height:
+        return pil_img
+
+    new_size = max(width, height)
+    result = Image.new(pil_img.mode, (new_size, new_size), background_color)
+
+    if width > height:
+        result.paste(pil_img, (0, (width - height) // 2))
+    else:
+        result.paste(pil_img, ((height - width) // 2, 0))
+
+    return result
+
+
+def simple_preprocess(input_image, rembg_session=session, background_color=255):
+    RES = 2048
+    input_image.thumbnail([RES, RES], Image.Resampling.LANCZOS)
+
+    if input_image.mode != 'RGBA':
+        image_rem = input_image.convert('RGBA')
+        input_image = remove(
+            image_rem, alpha_matting=False, session=rembg_session)
+
+    arr = np.asarray(input_image)
+    alpha = arr[:, :, -1]
+
+    x_nonzero, y_nonzero = (alpha > 60).sum(axis=1).nonzero()[
+        0], (alpha > 60).sum(axis=0).nonzero()[0]
+    x_min, x_max = int(x_nonzero.min()), int(x_nonzero.max())
+    y_min, y_max = int(y_nonzero.min()), int(y_nonzero.max())
+
+    arr = arr[x_min:x_max, y_min:y_max]
+    input_image = Image.fromarray(arr)
+    return expand2square(input_image, (background_color, background_color, background_color, 0))
 
-def fix_vert_color_glb(mesh_path):
-    from pygltflib import GLTF2, Material, PbrMetallicRoughness
-    obj1 = GLTF2().load(mesh_path)
-    obj1.meshes[0].primitives[0].material = 0
-    obj1.materials.append(Material(
-        pbrMetallicRoughness = PbrMetallicRoughness(
-            baseColorFactor = [1.0, 1.0, 1.0, 1.0],
-            metallicFactor = 0.,
-            roughnessFactor = 1.0,
-        ),
-        emissiveFactor = [0.0, 0.0, 0.0],
-        doubleSided = True,
-    ))
-    obj1.save(mesh_path)
 
 def init_target(img_pils, new_bkgd=(0., 0., 0.), device="cuda"):
-    new_bkgd = torch.tensor(new_bkgd, dtype=torch.float32).view(1, 1, 3).to(device)
-    imgs = torch.stack([torch.from_numpy(np.array(img, dtype=np.float32)) for img in img_pils]).to(device) / 255
-    img_nps, alpha_nps = imgs[..., :3], imgs[..., 3]
+    new_bkgd = torch.tensor(
+        new_bkgd, dtype=torch.float32).view(1, 1, 3).to(device)
+    imgs = torch.stack([torch.from_numpy(np.array(img, dtype=np.float32))
+                       for img in img_pils]).to(device) / 255
+
+    img_nps = imgs[..., :3]
+    alpha_nps = imgs[..., 3]
     ori_bkgds = img_nps[:, :1, :1]
-    
+
     alpha_nps_clamp = torch.clamp(alpha_nps, 1e-6, 1)
-    ori_img_nps = (img_nps - ori_bkgds * (1 - alpha_nps.unsqueeze(-1))) / alpha_nps_clamp.unsqueeze(-1)
+    ori_img_nps = (img_nps - ori_bkgds * (1 - alpha_nps.unsqueeze(-1))
+                   ) / alpha_nps_clamp.unsqueeze(-1)
     ori_img_nps = torch.clamp(ori_img_nps, 0, 1)
-    img_nps = torch.where(alpha_nps.unsqueeze(-1) > 0.05, ori_img_nps * alpha_nps.unsqueeze(-1) + new_bkgd * (1 - alpha_nps.unsqueeze(-1)), new_bkgd)
+
+    img_nps = torch.where(alpha_nps.unsqueeze(-1) > 0.05,
+                          ori_img_nps *
+                          alpha_nps.unsqueeze(-1) + new_bkgd *
+                          (1 - alpha_nps.unsqueeze(-1)),
+                          new_bkgd)
 
     return torch.cat([img_nps, alpha_nps.unsqueeze(-1)], dim=-1)
 
+
 def save_obj_and_video(save_mesh_prefix: str, meshes: Meshes, with_timestamp=True, **kwargs) -> Tuple[str, str]:
     if '.' in save_mesh_prefix:
         save_mesh_prefix = ".".join(save_mesh_prefix.split('.')[:-1])