handler.py

from typing import Any, Dict

import torch
from transformers import AutoModel, AutoProcessor


class EndpointHandler:
    def __init__(self, path=""):
        # load model and processor from path
        self.processor = AutoProcessor.from_pretrained("suno/bark-small")
        self.model = AutoModel.from_pretrained(
            "suno/bark-small",
        ).to("cuda")

    def __call__(self, data: Dict[str, Any]) -> Dict[str, str]:
        """
        Args:
            data (:dict:):
                The payload with the text prompt and generation parameters.
        """
        # process input
        text = data.pop("inputs", data)
        parameters = data.get("parameters", None)
        if parameters is None:
            parameters = {}
        # preprocess
        inputs = self.processor(
            text=[text],
            return_tensors="pt",
            voice_preset=parameters.get("voice_preset", None),
        ).to("cuda")

        with torch.autocast("cuda"):
            outputs = self.model.generate(**inputs)

        # postprocess the prediction
        prediction = outputs.cpu().numpy().tolist()

        return [{"generated_audio": prediction}]