utils/prompt_preparer.py

# Copyright (c) 2023 Amphion.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import torch


class PromptPreparer:
    def prepare_prompts(self, y, y_lens, codes, nar_stage, y_prompts_codes):
        if self.prefix_mode == 0:
            y_emb, prefix_len = self._handle_prefix_mode_0(y, codes, nar_stage)
        elif self.prefix_mode == 1:
            y_emb, prefix_len = self._handle_prefix_mode_1(y, y_lens, codes, nar_stage)
        elif self.prefix_mode in [2, 4]:
            y_emb, prefix_len = self._handle_prefix_mode_2_4(
                y, y_lens, codes, nar_stage, y_prompts_codes
            )
        else:
            raise ValueError("Invalid prefix mode")

        return y_emb, prefix_len

    def _handle_prefix_mode_0(self, y, codes, nar_stage):
        prefix_len = 0
        y_emb = self.nar_audio_embeddings[0](y)
        for j in range(1, nar_stage):
            y_emb = y_emb + self.nar_audio_embeddings[j](codes[..., j])
        return y_emb, 0

    def _handle_prefix_mode_1(self, y, y_lens, codes, nar_stage):
        int_low = (0.25 * y_lens.min()).type(torch.int64).item()
        prefix_len = torch.randint(int_low, int_low * 2, size=()).item()
        prefix_len = min(prefix_len, 225)

        y_prompts = self.nar_audio_embeddings[0](y[:, :prefix_len])
        y_emb = self.nar_audio_embeddings[0](y[:, prefix_len:])
        for j in range(1, self.num_quantizers):
            y_prompts += self.nar_audio_embeddings[j](codes[:, :prefix_len, j])
            if j < nar_stage:
                y_emb += self.nar_audio_embeddings[j](codes[:, prefix_len:, j])
        y_emb = torch.concat([y_prompts, y_emb], axis=1)
        return y_emb, prefix_len

    def _handle_prefix_mode_2_4(self, y, y_lens, codes, nar_stage, y_prompts_codes):
        if self.prefix_mode == 2:
            prefix_len = min(225, int(0.25 * y_lens.min().item()))

            y_prompts_codes = []
            for b in range(codes.shape[0]):
                start = self.rng.randint(0, y_lens[b].item() - prefix_len)
                y_prompts_codes.append(
                    torch.clone(codes[b, start : start + prefix_len])
                )
                codes[b, start : start + prefix_len, nar_stage] = self.audio_token_num
            y_prompts_codes = torch.stack(y_prompts_codes, dim=0)
        else:
            prefix_len = y_prompts_codes.shape[1]

        y_prompts = self.nar_audio_embeddings[0](y_prompts_codes[..., 0])
        y_emb = self.nar_audio_embeddings[0](y)
        for j in range(1, self.num_quantizers):
            y_prompts += self.nar_audio_embeddings[j](y_prompts_codes[..., j])
            if j < nar_stage:
                y_emb += self.nar_audio_embeddings[j](codes[..., j])
        y_emb = torch.concat([y_prompts, y_emb], axis=1)

        return y_emb, prefix_len