from collections import OrderedDict from typing import Tuple, Union import math # import torchvision import torch import numpy as np import torch from torch import nn # from torch.nn.modules.utils import _pair from torch.nn import Dropout from functools import reduce from operator import mul # from vpt.src.utils import logging from .ca import Cross_Attention # logger = logging.get_logger("visual_prompt") class Bottleneck(nn.Module): expansion = 4 def __init__(self, inplanes, planes, stride=1): super().__init__() # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1 self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False) self.bn1 = nn.BatchNorm2d(planes) self.relu1 = nn.ReLU(inplace=True) self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False) self.bn2 = nn.BatchNorm2d(planes) self.relu2 = nn.ReLU(inplace=True) self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity() self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False) self.bn3 = nn.BatchNorm2d(planes * self.expansion) self.relu3 = nn.ReLU(inplace=True) self.downsample = None self.stride = stride if stride > 1 or inplanes != planes * Bottleneck.expansion: # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1 self.downsample = nn.Sequential(OrderedDict([ ("-1", nn.AvgPool2d(stride)), ("0", nn.Conv2d(inplanes, planes * self.expansion, 1, stride=1, bias=False)), ("1", nn.BatchNorm2d(planes * self.expansion)) ])) def forward(self, x: torch.Tensor): identity = x out = self.relu1(self.bn1(self.conv1(x))) out = self.relu2(self.bn2(self.conv2(out))) out = self.avgpool(out) out = self.bn3(self.conv3(out)) if self.downsample is not None: identity = self.downsample(x) out += identity out = self.relu3(out) return out # implement attention module for v-v self-attention class Attention(nn.Module): def __init__(self, out_dim, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., settings=''): super().__init__() self.num_heads = num_heads head_dim = dim // num_heads self.scale = qk_scale or head_dim ** -0.5 self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) self.attn_drop = nn.Dropout(attn_drop) self.proj = nn.Linear(out_dim, dim) self.proj_drop = nn.Dropout(proj_drop) self.settings = settings def forward(self, x): B, N, C = x.shape qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) q, k, v = qkv[0], qkv[1], qkv[2] # original self-attention for the original path attn_ori = (q @ k.transpose(-2, -1)) * self.scale attn_ori = attn_ori.softmax(dim=-1) attn_ori = self.attn_drop(attn_ori) # replace k & q by v k = v q = k # resnets have only one self-attention, norm and larger scale perform better if self.settings == 'resnet': k = k / (k.norm(p=2, dim=-1, keepdim=True) + 1e-6) q = k scale = self.scale * 8 else: scale = self.scale # self-attention, higher temperate for resnets performs better attn = (q @ k.transpose(-2, -1)) * scale attn = (attn).softmax(dim=-1) attn = self.attn_drop(attn) x_ori = (attn_ori @ v).transpose(1, 2).reshape(B, N, C) x = (attn @ v).transpose(1, 2).reshape(B, N, C) # clip_surgery #x = v.transpose(1, 2).reshape(B, N, C) # mask_clip x = self.proj_drop(self.proj(x)) x_ori = self.proj_drop(self.proj(x_ori)) return [x, x_ori] class AttentionPool2d(nn.Module): def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None): super().__init__() self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5) self.k_proj = nn.Linear(embed_dim, embed_dim) self.q_proj = nn.Linear(embed_dim, embed_dim) self.v_proj = nn.Linear(embed_dim, embed_dim) self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) self.num_heads = num_heads self.attn = None self.embed_dim = embed_dim self.num_heads = num_heads self.output_dim = output_dim def forward(self, x): # reform transformer layer after init and load weights, using v only if self.attn == None: self.attn = Attention(self.output_dim, self.embed_dim, self.num_heads, True) self.attn.qkv.weight = torch.nn.Parameter(torch.cat([self.v_proj.weight, self.v_proj.weight, self.v_proj.weight], 0)) self.attn.qkv.bias = torch.nn.Parameter(torch.cat([self.v_proj.bias, self.v_proj.bias, self.v_proj.bias])) self.attn.proj.weight = self.c_proj.weight self.attn.proj.bias = self.c_proj.bias x = x.reshape(x.shape[0], x.shape[1], x.shape[2] * x.shape[3]).permute(2, 0, 1) # NCHW -> (HW)NC x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC side = int((self.positional_embedding.shape[0] - 1) ** 0.5) new_side = int((x.shape[0] - 1) ** 0.5) # update the position embedding during inference for varied input size if side != new_side: new_pos = self.positional_embedding[1:, :].reshape(-1, side, side, x.shape[-1]).permute(0, 3, 1, 2) new_pos = torch.nn.functional.interpolate(new_pos, (new_side, new_side), mode='bilinear') new_pos = new_pos.reshape(-1, x.shape[-1], new_side * new_side).transpose(1, 2) self.positional_embedding.data = torch.cat([self.positional_embedding[:1, :], new_pos[0]], 0) x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC x, x_ori = self.attn(x.transpose(0, 1)) # cls token from the original path, and img tokens from the new path x[:, 0, :] = x_ori[:, 0, :] return x class ModifiedResNet(nn.Module): """ A ResNet class that is similar to torchvision's but contains the following changes: - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool. - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1 - The final pooling layer is a QKV attention instead of an average pool """ def __init__(self, layers, output_dim, heads, input_resolution=224, width=64): super().__init__() self.output_dim = output_dim self.input_resolution = input_resolution # the 3-layer stem self.conv1 = nn.Conv2d(3, width // 2, kernel_size=3, stride=2, padding=1, bias=False) self.bn1 = nn.BatchNorm2d(width // 2) self.relu1 = nn.ReLU(inplace=True) self.conv2 = nn.Conv2d(width // 2, width // 2, kernel_size=3, padding=1, bias=False) self.bn2 = nn.BatchNorm2d(width // 2) self.relu2 = nn.ReLU(inplace=True) self.conv3 = nn.Conv2d(width // 2, width, kernel_size=3, padding=1, bias=False) self.bn3 = nn.BatchNorm2d(width) self.relu3 = nn.ReLU(inplace=True) self.avgpool = nn.AvgPool2d(2) # residual layers self._inplanes = width # this is a *mutable* variable used during construction self.layer1 = self._make_layer(width, layers[0]) self.layer2 = self._make_layer(width * 2, layers[1], stride=2) self.layer3 = self._make_layer(width * 4, layers[2], stride=2) self.layer4 = self._make_layer(width * 8, layers[3], stride=2) embed_dim = width * 32 # the ResNet feature dimension self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, heads, output_dim) def _make_layer(self, planes, blocks, stride=1): layers = [Bottleneck(self._inplanes, planes, stride)] self._inplanes = planes * Bottleneck.expansion for _ in range(1, blocks): layers.append(Bottleneck(self._inplanes, planes)) return nn.Sequential(*layers) def forward(self, x): def stem(x): x = self.relu1(self.bn1(self.conv1(x))) x = self.relu2(self.bn2(self.conv2(x))) x = self.relu3(self.bn3(self.conv3(x))) x = self.avgpool(x) return x x = x.type(self.conv1.weight.dtype) x = stem(x) x = self.layer1(x) x = self.layer2(x) x = self.layer3(x) x = self.layer4(x) x = self.attnpool(x) # shape BNC return x class LayerNorm(nn.LayerNorm): """Subclass torch's LayerNorm to handle fp16.""" def forward(self, x: torch.Tensor): orig_type = x.dtype ret = super().forward(x.clone().type(torch.float32)) return ret.type(orig_type) class QuickGELU(nn.Module): def forward(self, x: torch.Tensor): return x * torch.sigmoid(1.702 * x) class ResidualAttentionBlock(nn.Module): def __init__(self, d_model: int, n_head: int, attn_mask: torch.Tensor = None): super().__init__() self.attn = nn.MultiheadAttention(d_model, n_head) self.ln_1 = LayerNorm(d_model) self.mlp = nn.Sequential(OrderedDict([ ("c_fc", nn.Linear(d_model, d_model * 4)), ("gelu", QuickGELU()), ("c_proj", nn.Linear(d_model * 4, d_model)) ])) self.ln_2 = LayerNorm(d_model) self.attn_mask = attn_mask self.attn_probs = None self.attn_grad = None self.attn_keys = None def set_attn_probs(self, attn_probs): self.attn_probs = attn_probs def set_attn_keys(self, attn_keys): self.attn_keys = attn_keys def set_attn_grad(self, attn_grad): self.attn_grad = attn_grad def attention(self, x: torch.Tensor, attn_mask: torch.Tensor = None, mode="train"): self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None if isinstance(self.attn, Attention): x = x.transpose(0, 1) x, x_ori = self.attn(x) return [x.transpose(0, 1), x_ori.transpose(0, 1)] else: return self.attn(x, x, x, need_weights=False, attn_mask=self.attn_mask)[0] def forward(self, x, attn_mask: torch.Tensor = None, mode="train"): # dual paths for blocks deeper than "d" if isinstance(self.attn, Attention): if isinstance(x, list): x, x_ori = x x_res = self.attention(self.ln_1(x_ori)) x_res, x_ori_res = x_res x_ori += x_ori_res x_ori = x_ori + self.mlp(self.ln_2(x_ori)) x += x_res # skip ffn for the new path return [x, x_ori] # start of dual path else: x_res = self.attention(self.ln_1(x)) if isinstance(x_res, list): x_res, x_ori_res = x_res x_ori = x + x_ori_res x_ori = x_ori + self.mlp(self.ln_2(x_ori)) x += x_res return [x, x_ori] # single path before "d" else: x = x + self.attention(self.ln_1(x)) x = x + self.mlp(self.ln_2(x)) return x class Transformer(nn.Module): def __init__(self, width: int, layers: int, heads: int, attn_mask: torch.Tensor = None, need_weights: bool = False): super().__init__() self.width = width self.layers = layers self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, attn_mask) for i in range(layers)]) self.ca = Cross_Attention(d_model=768) def forward(self, x: torch.Tensor,layers=12,text_bool=False,text_features=None,mode="train"): for idx,l in enumerate(self.resblocks): x=l(x) if idx+1 == layers: if text_bool: return x # implement cross attention between image tokens and text tokens x_l = x[0] x_ori_l = x[1] text_features = text_features.unsqueeze(0).repeat(x_l.shape[0], 1, 1) x_l = x_l.permute(1, 0, 2) text_features = text_features.permute(1, 0, 2) if mode == "test": x_l = x_l.repeat(text_features.shape[0], 1, 1) x_l_ca = self.ca(x_l, text_features) x_l_ca = x_l_ca.permute(1, 0, 2) x_ori_l = x_ori_l.permute(1, 0, 2) if mode == "test": x_ori_l = x_ori_l.repeat(text_features.shape[0], 1, 1) x_ori_l_ca = self.ca(x_ori_l, text_features) x_ori_l_ca = x_ori_l_ca.permute(1, 0, 2) return [x_l_ca, x_ori_l_ca] class PromptedVisionTransformer(nn.Module): def __init__(self, input_resolution: int, patch_size: int, width: int, layers: int, heads: int, output_dim: int,prompt_config:dict,train_bool:bool): super().__init__() self.train_bool = train_bool self.patch_size = patch_size self.input_resolution = input_resolution self.output_dim = output_dim self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False) scale = width ** -0.5 self.class_embedding = nn.Parameter(scale * torch.randn(width)) self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width)) self.ln_pre = LayerNorm(width) self.transformer = Transformer(width, layers, heads, need_weights=True) self.attn = None self.embed_dim = width self.num_heads = heads self.ln_post = LayerNorm(width) self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) self.prompt_config = prompt_config self.prompt_dropout = Dropout(self.prompt_config.DROPOUT) num_tokens = self.prompt_config.NUM_TOKENS self.num_tokens = num_tokens # number of prompted tokens # if project the prompt embeddings if self.prompt_config.PROJECT > -1: # only for prepend / add prompt_dim = self.prompt_config.PROJECT self.prompt_proj = nn.Linear( prompt_dim, 768) nn.init.kaiming_normal_( self.prompt_proj.weight, a=0, mode='fan_out') else: prompt_dim = 768 self.prompt_proj = nn.Identity() # initiate prompt: if self.prompt_config.INITIATION == "random": val = math.sqrt(6. / float(3 * reduce(mul, (patch_size,patch_size), 1) + prompt_dim)) # noqa self.prompt_embeddings = nn.Parameter(torch.zeros( 1, num_tokens, prompt_dim)) # xavier_uniform initialization nn.init.uniform_(self.prompt_embeddings.data, -val, val) if self.prompt_config.DEEP: # noqa total_d_layer = 12-1 #config.transformer["num_layers"]-1 self.deep_prompt_embeddings = nn.Parameter(torch.zeros( total_d_layer, num_tokens, prompt_dim)) # xavier_uniform initialization nn.init.uniform_(self.deep_prompt_embeddings.data, -val, val) else: raise ValueError("Other initiation scheme is not supported") if not self.train_bool: if self.attn == None: # apply architecture surgery on the last 6 blocks for i in range(1, 7): # surgery 7, maskclip 2 self.attn = Attention(self.embed_dim, self.embed_dim, self.num_heads, True) self.attn.qkv.weight.data = self.transformer.resblocks[-i].attn.in_proj_weight.clone() self.attn.qkv.bias.data = self.transformer.resblocks[-i].attn.in_proj_bias.clone() self.attn.proj.weight.data = self.transformer.resblocks[-i].attn.out_proj.weight.clone() self.attn.proj.bias.data = self.transformer.resblocks[-i].attn.out_proj.bias.clone() self.transformer.resblocks[-i].attn = self.attn # @torch.no_grad() def forward(self, x: torch.Tensor,layers: int = 12,text_features:torch.Tensor = None,mode:str = "test"): if self.attn == None: # apply architecture surgery on the last 6 blocks for i in range(1, 7): # surgery 7, maskclip 2 self.attn = Attention(self.embed_dim, self.embed_dim, self.num_heads, True) self.attn.qkv.weight.data = self.transformer.resblocks[-i].attn.in_proj_weight.clone() self.attn.qkv.bias.data = self.transformer.resblocks[-i].attn.in_proj_bias.clone() self.attn.proj.weight.data = self.transformer.resblocks[-i].attn.out_proj.weight.clone() self.attn.proj.bias.data = self.transformer.resblocks[-i].attn.out_proj.bias.clone() self.transformer.resblocks[-i].attn = self.attn B = x.shape[0] x = self.conv1(x) # shape = [*, width, grid, grid] x = x.reshape(x.shape[0], x.shape[1], -1) # shape = [*, width, grid ** 2] x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] ,, torch.Size([B, 196, 768]) x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1) # shape = [*, grid ** 2 + 1, width] side = int((self.positional_embedding.shape[0] - 1) ** 0.5) new_side = int((x.shape[1] - 1) ** 0.5) # update the position embedding during inference for varied input size if side != new_side: new_pos = self.positional_embedding[1:, :].reshape(-1, side, side, x.shape[-1]).permute(0, 3, 1, 2) new_pos = torch.nn.functional.interpolate(new_pos, (new_side, new_side), mode='bilinear') new_pos = new_pos.reshape(-1, x.shape[-1], new_side * new_side).transpose(1, 2) self.positional_embedding.data = torch.cat([self.positional_embedding[:1, :], new_pos[0]], 0) pos = self.positional_embedding.to(x.dtype) x = x + pos # add positional embedding torch.Size([B, 197, 768]) # ADD VISUAL PROMPTS HERE if self.num_tokens > 0: x = torch.cat(( x[:, :1, :], self.prompt_dropout(self.prompt_proj(self.prompt_embeddings).expand(B, -1, -1)), x[:, 1:, :] ), dim=1) # (batch_size, cls_token + n_prompt + n_patches, hidden_dim) x = self.ln_pre(x) # layer norm x = x.permute(1, 0, 2) # NLD -> LND if mode == "train": x_multi = torch.zeros(len(layers),x.shape[1],x.shape[0],512).to(x.device) elif mode == "test": x_multi = torch.zeros(len(layers),text_features.shape[0],x.shape[0],512).to(x.device) for d,layer in enumerate(layers): x_l, x_ori_l = self.transformer(x,layers=layer,text_bool=False, text_features=text_features,mode = mode) x_l[0, :, :] = x_ori_l[0, :, :] # clip_surgery x_l = x_l.permute(1, 0, 2) # LND -> NLD x_l = self.ln_post(x_l) # layer norm x_l = x_l @ self.proj x_multi[d] = x_l return x_multi class ModifiedCLIPSurgery(nn.Module): def __init__(self, embed_dim: int, # vision image_resolution: int, vision_layers: Union[Tuple[int, int, int, int], int], vision_width: int, vision_patch_size: int, # text context_length: int, vocab_size: int, transformer_width: int, transformer_heads: int, transformer_layers: int, cfg:dict, train_bool:bool, ): super().__init__() if "prompt" in cfg.MODEL.TRANSFER_TYPE: prompt_cfg = cfg.MODEL.PROMPT else: prompt_cfg = None self.prompt_config = prompt_cfg self.context_length = context_length if isinstance(vision_layers, (tuple, list)): vision_heads = vision_width * 32 // 64 self.visual = ModifiedResNet( layers=vision_layers, output_dim=embed_dim, heads=vision_heads, input_resolution=image_resolution, width=vision_width ) else: vision_heads = vision_width // 64 self.visual = PromptedVisionTransformer( input_resolution=image_resolution, patch_size=vision_patch_size, width=vision_width, layers=vision_layers, heads=vision_heads, output_dim=embed_dim, prompt_config=self.prompt_config, train_bool=train_bool, ) self.transformer = Transformer( width=transformer_width, layers=transformer_layers, heads=transformer_heads, attn_mask=self.build_attention_mask() ) self.vocab_size = vocab_size self.token_embedding = nn.Embedding(vocab_size, transformer_width) self.positional_embedding = nn.Parameter(torch.empty(self.context_length, transformer_width)) self.ln_final = LayerNorm(transformer_width) self.text_projection = nn.Parameter(torch.empty(transformer_width, embed_dim)) self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) self.initialize_parameters() def initialize_parameters(self): nn.init.normal_(self.token_embedding.weight, std=0.02) nn.init.normal_(self.positional_embedding, std=0.01) # skipped because self.visual is PromptedVisionTransformer if isinstance(self.visual, ModifiedResNet): if self.visual.attnpool is not None: std = self.visual.attnpool.c_proj.in_features ** -0.5 nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std) nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std) nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std) nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std) for resnet_block in [self.visual.layer1, self.visual.layer2, self.visual.layer3, self.visual.layer4]: for name, param in resnet_block.named_parameters(): if name.endswith("bn3.weight"): nn.init.zeros_(param) proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5) attn_std = self.transformer.width ** -0.5 fc_std = (2 * self.transformer.width) ** -0.5 for block in self.transformer.resblocks: nn.init.normal_(block.attn.in_proj_weight, std=attn_std) nn.init.normal_(block.attn.out_proj.weight, std=proj_std) nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) if self.text_projection is not None: nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5) def build_attention_mask(self): # lazily create causal attention mask, with full attention between the vision tokens # pytorch uses additive attention mask; fill with -inf mask = torch.empty(self.context_length, self.context_length) mask.fill_(float("-inf")) mask.triu_(1) # zero out the lower diagonal return mask @property def dtype(self): return self.visual.conv1.weight.dtype def encode_image(self, image,layers:int=12,text_features=None,mode="test"): return self.visual(image.type(self.dtype),layers=layers,text_features=text_features,mode=mode) def encode_text(self, text): text_bool=True x = self.token_embedding(text).type(self.dtype) # [batch_size, n_ctx, d_model] x = x + self.positional_embedding.type(self.dtype) x = x.permute(1, 0, 2) # NLD -> LND x = self.transformer(x,layers=12,text_bool=text_bool,text_features=None) # always get the last layer features for text x = x.permute(1, 0, 2) # LND -> NLD x = self.ln_final(x).type(self.dtype) # x.shape = [batch_size, n_ctx, transformer.width] x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection return x def forward(self, image, text,layer_num=12,return_logits=False,mode="train"): text_features = self.encode_text(text) patch_features = self.encode_image(image,layers=layer_num,text_features=text_features,mode=mode).squeeze(0) # normalized features patch_features = patch_features / patch_features.norm(dim=1, keepdim=True) text_features = text_features / text_features.norm(dim=1, keepdim=True) if return_logits: logit_scale = self.logit_scale.exp() sketch_features = patch_features[:,0,:] logits_sketch = logit_scale * sketch_features @ text_features.t() logits_text = logits_sketch.t() return logits_sketch,logits_text else: return patch_features,text_features