Spaces:
Paused
Paused
| import torch | |
| import torch.nn.functional as F | |
| from torch import nn | |
| # adapted from https://github.com/cvqluu/GE2E-Loss | |
| class GE2ELoss(nn.Module): | |
| def __init__(self, init_w=10.0, init_b=-5.0, loss_method="softmax"): | |
| """ | |
| Implementation of the Generalized End-to-End loss defined in https://arxiv.org/abs/1710.10467 [1] | |
| Accepts an input of size (N, M, D) | |
| where N is the number of speakers in the batch, | |
| M is the number of utterances per speaker, | |
| and D is the dimensionality of the embedding vector (e.g. d-vector) | |
| Args: | |
| - init_w (float): defines the initial value of w in Equation (5) of [1] | |
| - init_b (float): definies the initial value of b in Equation (5) of [1] | |
| """ | |
| super().__init__() | |
| # pylint: disable=E1102 | |
| self.w = nn.Parameter(torch.tensor(init_w)) | |
| # pylint: disable=E1102 | |
| self.b = nn.Parameter(torch.tensor(init_b)) | |
| self.loss_method = loss_method | |
| print(" > Initialized Generalized End-to-End loss") | |
| assert self.loss_method in ["softmax", "contrast"] | |
| if self.loss_method == "softmax": | |
| self.embed_loss = self.embed_loss_softmax | |
| if self.loss_method == "contrast": | |
| self.embed_loss = self.embed_loss_contrast | |
| # pylint: disable=R0201 | |
| def calc_new_centroids(self, dvecs, centroids, spkr, utt): | |
| """ | |
| Calculates the new centroids excluding the reference utterance | |
| """ | |
| excl = torch.cat((dvecs[spkr, :utt], dvecs[spkr, utt + 1 :])) | |
| excl = torch.mean(excl, 0) | |
| new_centroids = [] | |
| for i, centroid in enumerate(centroids): | |
| if i == spkr: | |
| new_centroids.append(excl) | |
| else: | |
| new_centroids.append(centroid) | |
| return torch.stack(new_centroids) | |
| def calc_cosine_sim(self, dvecs, centroids): | |
| """ | |
| Make the cosine similarity matrix with dims (N,M,N) | |
| """ | |
| cos_sim_matrix = [] | |
| for spkr_idx, speaker in enumerate(dvecs): | |
| cs_row = [] | |
| for utt_idx, utterance in enumerate(speaker): | |
| new_centroids = self.calc_new_centroids(dvecs, centroids, spkr_idx, utt_idx) | |
| # vector based cosine similarity for speed | |
| cs_row.append( | |
| torch.clamp( | |
| torch.mm( | |
| utterance.unsqueeze(1).transpose(0, 1), | |
| new_centroids.transpose(0, 1), | |
| ) | |
| / (torch.norm(utterance) * torch.norm(new_centroids, dim=1)), | |
| 1e-6, | |
| ) | |
| ) | |
| cs_row = torch.cat(cs_row, dim=0) | |
| cos_sim_matrix.append(cs_row) | |
| return torch.stack(cos_sim_matrix) | |
| # pylint: disable=R0201 | |
| def embed_loss_softmax(self, dvecs, cos_sim_matrix): | |
| """ | |
| Calculates the loss on each embedding $L(e_{ji})$ by taking softmax | |
| """ | |
| N, M, _ = dvecs.shape | |
| L = [] | |
| for j in range(N): | |
| L_row = [] | |
| for i in range(M): | |
| L_row.append(-F.log_softmax(cos_sim_matrix[j, i], 0)[j]) | |
| L_row = torch.stack(L_row) | |
| L.append(L_row) | |
| return torch.stack(L) | |
| # pylint: disable=R0201 | |
| def embed_loss_contrast(self, dvecs, cos_sim_matrix): | |
| """ | |
| Calculates the loss on each embedding $L(e_{ji})$ by contrast loss with closest centroid | |
| """ | |
| N, M, _ = dvecs.shape | |
| L = [] | |
| for j in range(N): | |
| L_row = [] | |
| for i in range(M): | |
| centroids_sigmoids = torch.sigmoid(cos_sim_matrix[j, i]) | |
| excl_centroids_sigmoids = torch.cat((centroids_sigmoids[:j], centroids_sigmoids[j + 1 :])) | |
| L_row.append(1.0 - torch.sigmoid(cos_sim_matrix[j, i, j]) + torch.max(excl_centroids_sigmoids)) | |
| L_row = torch.stack(L_row) | |
| L.append(L_row) | |
| return torch.stack(L) | |
| def forward(self, x, _label=None): | |
| """ | |
| Calculates the GE2E loss for an input of dimensions (num_speakers, num_utts_per_speaker, dvec_feats) | |
| """ | |
| assert x.size()[1] >= 2 | |
| centroids = torch.mean(x, 1) | |
| cos_sim_matrix = self.calc_cosine_sim(x, centroids) | |
| torch.clamp(self.w, 1e-6) | |
| cos_sim_matrix = self.w * cos_sim_matrix + self.b | |
| L = self.embed_loss(x, cos_sim_matrix) | |
| return L.mean() | |
| # adapted from https://github.com/clovaai/voxceleb_trainer/blob/master/loss/angleproto.py | |
| class AngleProtoLoss(nn.Module): | |
| """ | |
| Implementation of the Angular Prototypical loss defined in https://arxiv.org/abs/2003.11982 | |
| Accepts an input of size (N, M, D) | |
| where N is the number of speakers in the batch, | |
| M is the number of utterances per speaker, | |
| and D is the dimensionality of the embedding vector | |
| Args: | |
| - init_w (float): defines the initial value of w | |
| - init_b (float): definies the initial value of b | |
| """ | |
| def __init__(self, init_w=10.0, init_b=-5.0): | |
| super().__init__() | |
| # pylint: disable=E1102 | |
| self.w = nn.Parameter(torch.tensor(init_w)) | |
| # pylint: disable=E1102 | |
| self.b = nn.Parameter(torch.tensor(init_b)) | |
| self.criterion = torch.nn.CrossEntropyLoss() | |
| print(" > Initialized Angular Prototypical loss") | |
| def forward(self, x, _label=None): | |
| """ | |
| Calculates the AngleProto loss for an input of dimensions (num_speakers, num_utts_per_speaker, dvec_feats) | |
| """ | |
| assert x.size()[1] >= 2 | |
| out_anchor = torch.mean(x[:, 1:, :], 1) | |
| out_positive = x[:, 0, :] | |
| num_speakers = out_anchor.size()[0] | |
| cos_sim_matrix = F.cosine_similarity( | |
| out_positive.unsqueeze(-1).expand(-1, -1, num_speakers), | |
| out_anchor.unsqueeze(-1).expand(-1, -1, num_speakers).transpose(0, 2), | |
| ) | |
| torch.clamp(self.w, 1e-6) | |
| cos_sim_matrix = cos_sim_matrix * self.w + self.b | |
| label = torch.arange(num_speakers).to(cos_sim_matrix.device) | |
| L = self.criterion(cos_sim_matrix, label) | |
| return L | |
| class SoftmaxLoss(nn.Module): | |
| """ | |
| Implementation of the Softmax loss as defined in https://arxiv.org/abs/2003.11982 | |
| Args: | |
| - embedding_dim (float): speaker embedding dim | |
| - n_speakers (float): number of speakers | |
| """ | |
| def __init__(self, embedding_dim, n_speakers): | |
| super().__init__() | |
| self.criterion = torch.nn.CrossEntropyLoss() | |
| self.fc = nn.Linear(embedding_dim, n_speakers) | |
| print("Initialised Softmax Loss") | |
| def forward(self, x, label=None): | |
| # reshape for compatibility | |
| x = x.reshape(-1, x.size()[-1]) | |
| label = label.reshape(-1) | |
| x = self.fc(x) | |
| L = self.criterion(x, label) | |
| return L | |
| def inference(self, embedding): | |
| x = self.fc(embedding) | |
| activations = torch.nn.functional.softmax(x, dim=1).squeeze(0) | |
| class_id = torch.argmax(activations) | |
| return class_id | |
| class SoftmaxAngleProtoLoss(nn.Module): | |
| """ | |
| Implementation of the Softmax AnglePrototypical loss as defined in https://arxiv.org/abs/2009.14153 | |
| Args: | |
| - embedding_dim (float): speaker embedding dim | |
| - n_speakers (float): number of speakers | |
| - init_w (float): defines the initial value of w | |
| - init_b (float): definies the initial value of b | |
| """ | |
| def __init__(self, embedding_dim, n_speakers, init_w=10.0, init_b=-5.0): | |
| super().__init__() | |
| self.softmax = SoftmaxLoss(embedding_dim, n_speakers) | |
| self.angleproto = AngleProtoLoss(init_w, init_b) | |
| print("Initialised SoftmaxAnglePrototypical Loss") | |
| def forward(self, x, label=None): | |
| """ | |
| Calculates the SoftmaxAnglePrototypical loss for an input of dimensions (num_speakers, num_utts_per_speaker, dvec_feats) | |
| """ | |
| Lp = self.angleproto(x) | |
| Ls = self.softmax(x, label) | |
| return Ls + Lp | |