utils/utils.py

import torch
import numpy as np
from einops import rearrange
from kornia.geometry.transform.crop2d import warp_affine

from utils.matlab_cp2tform import get_similarity_transform_for_cv2
from torchvision.transforms import Pad

REFERNCE_FACIAL_POINTS_RELATIVE = np.array([[38.29459953, 51.69630051],
                                            [72.53179932, 51.50139999],
                                            [56.02519989, 71.73660278],
                                            [41.54930115, 92.3655014],
                                            [70.72990036, 92.20410156]
                                            ]) / 112 # Original points are 112 * 96 added 8 to the x axis to make it 112 * 112


def verify_load(missing_keys, unexpected_keys):
    if len(unexpected_keys) > 0:
        raise RuntimeError(f"Found unexpected keys in state dict while loading the encoder:\n{unexpected_keys}")
    
    filtered_missing = [key for key in missing_keys if not "extract_kv" in key]
    if len(filtered_missing) > 0:
        raise RuntimeError(f"Missing keys in state dict while loading the encoder:\n{filtered_missing}")


@torch.no_grad()
def detect_face(images: torch.Tensor, mtcnn: torch.nn.Module) -> torch.Tensor:
    """
    Detect faces in the images using MTCNN. If no face is detected, use the whole image.
    """
    images = rearrange(images, "b c h w -> b h w c")
    if images.dtype != torch.uint8:
        images = ((images * 0.5 + 0.5) * 255).type(torch.uint8)  # Unnormalize
        
    _, _, landmarks = mtcnn(images, landmarks=True)

    return landmarks


def extract_faces_and_landmarks(images: torch.Tensor, output_size=112, mtcnn: torch.nn.Module = None, refernce_points=REFERNCE_FACIAL_POINTS_RELATIVE):
    """
    detect faces in the images and crop them (in a differentiable way) to 112x112 using MTCNN.
    """
    images = Pad(200)(images)
    landmarks_batched = detect_face(images, mtcnn=mtcnn)
    affine_transformations = []
    invalid_indices = []
    for i, landmarks in enumerate(landmarks_batched):
        if landmarks is None:
            invalid_indices.append(i)
            affine_transformations.append(np.eye(2, 3).astype(np.float32))
        else:
            affine_transformations.append(get_similarity_transform_for_cv2(landmarks[0].astype(np.float32),
                                                                           refernce_points.astype(np.float32) * output_size))
    affine_transformations = torch.from_numpy(np.stack(affine_transformations).astype(np.float32)).to(device=images.device, dtype=torch.float32)

    invalid_indices = torch.tensor(invalid_indices).to(device=images.device)

    fp_images = images.to(torch.float32)
    return  warp_affine(fp_images, affine_transformations, dsize=(output_size, output_size)).to(dtype=images.dtype), invalid_indices