handler.py

# from typing import List, Any
# import torch
# from diffusers import StableCascadePriorPipeline, StableCascadeDecoderPipeline

# # Configurar el dispositivo para ejecutar el modelo
# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# if device.type != 'cuda':
#     raise ValueError("Se requiere ejecutar en GPU")

# # Configurar el tipo de dato mixto basado en la capacidad de la GPU
# dtype = torch.bfloat16 if torch.cuda.get_device_capability(device.index)[0] >= 8 else torch.float16

# start_test
import cv2
import numpy as np

import diffusers
from diffusers.models import ControlNetModel
from diffusers.pipelines.controlnet.multicontrolnet import MultiControlNetModel
from diffusers.utils import load_image

import torch
import torch.nn.functional as F
from torchvision.transforms import Compose

import PIL
from PIL import Image

from depth_anything.dpt import DepthAnything
from depth_anything.util.transform import Resize, NormalizeImage, PrepareForNet

from insightface.app import FaceAnalysis
from pipeline_stable_diffusion_xl_instantid_full import StableDiffusionXLInstantIDPipeline, draw_kps
from controlnet_aux import OpenposeDetector
from huggingface_hub import hf_hub_download


# end_test

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if device.type != 'cuda':
    raise ValueError("Se requiere ejecutar en GPU")

dtype = torch.float16 if str(device).__contains__("cuda") else torch.float32

class EndpointHandler():
    def __init__(self, model_dir):
        hf_hub_download(repo_id="InstantX/InstantID", filename="ControlNetModel/config.json", local_dir="./checkpoints")
        hf_hub_download(
            repo_id="InstantX/InstantID",
            filename="ControlNetModel/diffusion_pytorch_model.safetensors",
            local_dir="./checkpoints",
        )
        hf_hub_download(repo_id="InstantX/InstantID", filename="ip-adapter.bin", local_dir="./checkpoints")

        print("Model dir: ", model_dir)
        face_adapter = f"./checkpoints/ip-adapter.bin"
        controlnet_path = f"./checkpoints/ControlNetModel"

        # transform = Compose([
        #     Resize(
        #         width=518,
        #         height=518,
        #         resize_target=False,
        #         keep_aspect_ratio=True,
        #         ensure_multiple_of=14,
        #         resize_method='lower_bound',
        #         image_interpolation_method=cv2.INTER_CUBIC,
        #     ),
        #     NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
        #     PrepareForNet(),
        # ])

        self.controlnet_identitynet = ControlNetModel.from_pretrained(
            controlnet_path, torch_dtype=dtype
        )

        pretrained_model_name_or_path = "wangqixun/YamerMIX_v8"

        self.pipe = StableDiffusionXLInstantIDPipeline.from_pretrained(
        pretrained_model_name_or_path,
        controlnet=[self.controlnet_identitynet],
        torch_dtype=dtype,
        safety_checker=None,
        feature_extractor=None,
        ).to(device)


        self.pipe.scheduler = diffusers.EulerDiscreteScheduler.from_config(
            self.pipe.scheduler.config
        )

        # load and disable LCM
        self.pipe.load_lora_weights("latent-consistency/lcm-lora-sdxl")
        self.pipe.disable_lora()

        self.pipe.cuda()
        self.pipe.load_ip_adapter_instantid(face_adapter)
        self.pipe.image_proj_model.to("cuda")
        self.pipe.unet.to("cuda")


        # controlnet-pose/canny/depth
        controlnet_pose_model = "thibaud/controlnet-openpose-sdxl-1.0"
        # controlnet_canny_model = "diffusers/controlnet-canny-sdxl-1.0"
        # controlnet_depth_model = "diffusers/controlnet-depth-sdxl-1.0-small"

        controlnet_pose = ControlNetModel.from_pretrained(
            controlnet_pose_model, torch_dtype=dtype
        ).to(device)
        # controlnet_canny = ControlNetModel.from_pretrained(
        #     controlnet_canny_model, torch_dtype=dtype
        # ).to(device)
        # controlnet_depth = ControlNetModel.from_pretrained(
        #     controlnet_depth_model, torch_dtype=dtype
        # ).to(device)

        # def get_canny_image(image, t1=100, t2=200):
        #     image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
        #     edges = cv2.Canny(image, t1, t2)
        #     return Image.fromarray(edges, "L")
        
        # def get_depth_map(image):
        
        #     image = np.array(image) / 255.0

        #     h, w = image.shape[:2]

        #     image = transform({'image': image})['image']
        #     image = torch.from_numpy(image).unsqueeze(0).to("cuda")

        #     with torch.no_grad():
        #         depth = depth_anything(image)

        #     depth = F.interpolate(depth[None], (h, w), mode='bilinear', align_corners=False)[0, 0]
        #     depth = (depth - depth.min()) / (depth.max() - depth.min()) * 255.0

        #     depth = depth.cpu().numpy().astype(np.uint8)

        #     depth_image = Image.fromarray(depth)

        #     return depth_image
        
        self.controlnet_map = {
            "pose": controlnet_pose,
        #     "canny": controlnet_canny,
        #     "depth": controlnet_depth,
        }

        openpose = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
        # depth_anything = DepthAnything.from_pretrained('LiheYoung/depth_anything_vitl14').to(device).eval()


        self.controlnet_map_fn = {
            "pose": openpose,
        #     "canny": get_canny_image,
        #     "depth": get_depth_map,
        }
        
        self.app = FaceAnalysis(name="buffalo_l", root="./", providers=["CPUExecutionProvider"])
        self.app.prepare(ctx_id=0, det_size=(640, 640))
        
    def __call__(self, param):
        self.pipe.scheduler = diffusers.LCMScheduler.from_config(self.pipe.scheduler.config)
        self.pipe.enable_lora()
        
        adapter_strength_ratio = 0.8
        identitynet_strength_ratio = 0.8
        pose_strength = 0.4
        # canny_strength = 0.3
        # depth_strength = 0.5
        controlnet_selection = ["pose"]
        # controlnet_selection = ["pose", "canny", "depth"]
        
        face_image_path = "https://i.ibb.co/SKg69dD/kaifu-resize.png"
        pose_image_path = "https://i.ibb.co/ZSrQ8ZJ/pose.jpg"
        
        def convert_from_cv2_to_image(img: np.ndarray) -> Image:
            return Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))

        def convert_from_image_to_cv2(img: Image) -> np.ndarray:
            return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)

        def resize_img(
            input_image,
            max_side=1280,
            min_side=1024,
            size=None,
            pad_to_max_side=False,
            mode=PIL.Image.BILINEAR,
            base_pixel_number=64,
        ):
            w, h = input_image.size
            if size is not None:
                w_resize_new, h_resize_new = size
            else:
                ratio = min_side / min(h, w)
                w, h = round(ratio * w), round(ratio * h)
                ratio = max_side / max(h, w)
                input_image = input_image.resize([round(ratio * w), round(ratio * h)], mode)
                w_resize_new = (round(ratio * w) // base_pixel_number) * base_pixel_number
                h_resize_new = (round(ratio * h) // base_pixel_number) * base_pixel_number
            input_image = input_image.resize([w_resize_new, h_resize_new], mode)

            if pad_to_max_side:
                res = np.ones([max_side, max_side, 3], dtype=np.uint8) * 255
                offset_x = (max_side - w_resize_new) // 2
                offset_y = (max_side - h_resize_new) // 2
                res[
                    offset_y : offset_y + h_resize_new, offset_x : offset_x + w_resize_new
                ] = np.array(input_image)
                input_image = Image.fromarray(res)
            return input_image        

        # check if the input is valid
        # if face_image_path is None:
        #     raise gr.Error(
        #         f"Cannot find any input face image! Please upload the face image"
        #     )
        #  check the prompt
        # if prompt is None:
        prompt = "a person"
        negative_prompt=""
        
        # apply the style template
        # prompt, negative_prompt = apply_style(style_name, prompt, negative_prompt)

        face_image = load_image(face_image_path)
        face_image = resize_img(face_image, max_side=1024)
        face_image_cv2 = convert_from_image_to_cv2(face_image)
        height, width, _ = face_image_cv2.shape

        # Extract face features
        face_info = self.app.get(face_image_cv2)

                
        print(len(face_info))
        print("error si no hay face")
        # if len(face_info) == 0:
        #     raise gr.Error(
        #         f"Unable to detect a face in the image. Please upload a different photo with a clear face."
        #     )

        face_info = sorted(
            face_info,
            key=lambda x: (x["bbox"][2] - x["bbox"][0]) * x["bbox"][3] - x["bbox"][1],
        )[
            -1
        ]  # only use the maximum face
        
        
        face_emb = face_info["embedding"]
        face_kps = draw_kps(convert_from_cv2_to_image(face_image_cv2), face_info["kps"])
        img_controlnet = face_image
        if pose_image_path is not None:
            pose_image = load_image(pose_image_path)
            pose_image = resize_img(pose_image, max_side=1024)
            img_controlnet = pose_image
            pose_image_cv2 = convert_from_image_to_cv2(pose_image)

            face_info = self.app.get(pose_image_cv2)

            # get error if no face is detected 
            # if len(face_info) == 0:
            #     raise gr.Error(
            #         f"Cannot find any face in the reference image! Please upload another person image"
            #     )

            face_info = face_info[-1]
            face_kps = draw_kps(pose_image, face_info["kps"])

            width, height = face_kps.size

        control_mask = np.zeros([height, width, 3])
        x1, y1, x2, y2 = face_info["bbox"]
        x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
        control_mask[y1:y2, x1:x2] = 255
        control_mask = Image.fromarray(control_mask.astype(np.uint8))

        if len(controlnet_selection) > 0:
            controlnet_scales = {
                "pose": pose_strength,
                # "canny": canny_strength,
                # "depth": depth_strength,
            }
            self.pipe.controlnet = MultiControlNetModel(
                [self.controlnet_identitynet]
                + [self.controlnet_map[s] for s in controlnet_selection]
            )
            control_scales = [float(identitynet_strength_ratio)] + [
                controlnet_scales[s] for s in controlnet_selection
            ]
            control_images = [face_kps] + [
                self.controlnet_map_fn[s](img_controlnet).resize((width, height))
                for s in controlnet_selection
            ]
        else:
            self.pipe.controlnet = self.controlnet_identitynet
            control_scales = float(identitynet_strength_ratio)
            control_images = face_kps

        generator = torch.Generator(device=device.type).manual_seed(3)

        print("Start inference...")

        self.pipe.set_ip_adapter_scale(adapter_strength_ratio)
        images = self.pipe(
            prompt=prompt,
            negative_prompt=negative_prompt,
            image_embeds=face_emb,
            image=control_images,
            control_mask=control_mask,
            controlnet_conditioning_scale=control_scales,
            num_inference_steps=30,
            guidance_scale=7.5,
            height=height,
            width=width,
            generator=generator,
        ).images
        
        print("Inference done!")
        
        return images[0]