tools/model_converters/clip2mmseg.py

# Copyright (c) OpenMMLab. All rights reserved.
import argparse
import os.path as osp
from collections import OrderedDict

import mmengine
import torch
from mmengine.runner import CheckpointLoader


def convert_vitlayer(paras):
    new_para_name = ''
    if paras[0] == 'ln_1':
        new_para_name = '.'.join(['ln1'] + paras[1:])
    elif paras[0] == 'attn':
        new_para_name = '.'.join(['attn.attn'] + paras[1:])
    elif paras[0] == 'ln_2':
        new_para_name = '.'.join(['ln2'] + paras[1:])
    elif paras[0] == 'mlp':
        if paras[1] == 'c_fc':
            new_para_name = '.'.join(['ffn.layers.0.0'] + paras[-1:])
        else:
            new_para_name = '.'.join(['ffn.layers.1'] + paras[-1:])
    else:
        print(f'Wrong for {paras}')
    return new_para_name


def convert_translayer(paras):
    new_para_name = ''
    if paras[0] == 'attn':
        new_para_name = '.'.join(['attentions.0.attn'] + paras[1:])
    elif paras[0] == 'ln_1':
        new_para_name = '.'.join(['norms.0'] + paras[1:])
    elif paras[0] == 'ln_2':
        new_para_name = '.'.join(['norms.1'] + paras[1:])
    elif paras[0] == 'mlp':
        if paras[1] == 'c_fc':
            new_para_name = '.'.join(['ffns.0.layers.0.0'] + paras[2:])
        elif paras[1] == 'c_proj':
            new_para_name = '.'.join(['ffns.0.layers.1'] + paras[2:])
        else:
            print(f'Wrong for {paras}')
    else:
        print(f'Wrong for {paras}')
    return new_para_name


def convert_key_name(ckpt, visual_split):
    new_ckpt = OrderedDict()
    for k, v in ckpt.items():
        key_list = k.split('.')
        if key_list[0] == 'visual':
            new_transform_name = 'image_encoder'
            if key_list[1] == 'class_embedding':
                new_name = '.'.join([new_transform_name, 'cls_token'])
            elif key_list[1] == 'positional_embedding':
                new_name = '.'.join([new_transform_name, 'pos_embed'])
            elif key_list[1] == 'conv1':
                new_name = '.'.join([
                    new_transform_name, 'patch_embed.projection', key_list[2]
                ])
            elif key_list[1] == 'ln_pre':
                new_name = '.'.join(
                    [new_transform_name, key_list[1], key_list[2]])
            elif key_list[1] == 'transformer':
                new_layer_name = 'layers'
                layer_index = key_list[3]
                paras = key_list[4:]
                if int(layer_index) < visual_split:
                    new_para_name = convert_vitlayer(paras)
                    new_name = '.'.join([
                        new_transform_name, new_layer_name, layer_index,
                        new_para_name
                    ])
                else:
                    new_para_name = convert_translayer(paras)
                    new_transform_name = 'decode_head.rec_with_attnbias'
                    new_layer_name = 'layers'
                    layer_index = str(int(layer_index) - visual_split)
                    new_name = '.'.join([
                        new_transform_name, new_layer_name, layer_index,
                        new_para_name
                    ])
            elif key_list[1] == 'proj':
                new_name = 'decode_head.rec_with_attnbias.proj.weight'
            elif key_list[1] == 'ln_post':
                new_name = k.replace('visual', 'decode_head.rec_with_attnbias')
            else:
                print(f'pop parameter: {k}')
                continue
        else:
            text_encoder_name = 'text_encoder'
            if key_list[0] == 'transformer':
                layer_name = 'transformer'
                layer_index = key_list[2]
                paras = key_list[3:]
                new_para_name = convert_translayer(paras)
                new_name = '.'.join([
                    text_encoder_name, layer_name, layer_index, new_para_name
                ])
            elif key_list[0] in [
                    'positional_embedding', 'text_projection', 'bg_embed',
                    'attn_mask', 'logit_scale', 'token_embedding', 'ln_final'
            ]:
                new_name = 'text_encoder.' + k
            else:
                print(f'pop parameter: {k}')
                continue
        new_ckpt[new_name] = v

    return new_ckpt


def convert_tensor(ckpt):
    cls_token = ckpt['image_encoder.cls_token']
    new_cls_token = cls_token.unsqueeze(0).unsqueeze(0)
    ckpt['image_encoder.cls_token'] = new_cls_token
    pos_embed = ckpt['image_encoder.pos_embed']
    new_pos_embed = pos_embed.unsqueeze(0)
    ckpt['image_encoder.pos_embed'] = new_pos_embed
    proj_weight = ckpt['decode_head.rec_with_attnbias.proj.weight']
    new_proj_weight = proj_weight.transpose(1, 0)
    ckpt['decode_head.rec_with_attnbias.proj.weight'] = new_proj_weight
    return ckpt


def main():
    parser = argparse.ArgumentParser(
        description='Convert keys in timm pretrained vit models to '
        'MMSegmentation style.')
    parser.add_argument('src', help='src model path or url')
    # The dst path must be a full path of the new checkpoint.
    parser.add_argument('dst', help='save path')
    args = parser.parse_args()

    if any([s in args.src for s in ['B-16', 'b16', 'base_patch16']]):
        visual_split = 9
    elif any([s in args.src for s in ['L-14', 'l14', 'large_patch14']]):
        visual_split = 18
    else:
        print('Make sure the clip model is ViT-B/16 or ViT-L/14!')
        visual_split = -1
    checkpoint = CheckpointLoader.load_checkpoint(args.src, map_location='cpu')
    if isinstance(checkpoint, torch.jit.RecursiveScriptModule):
        state_dict = checkpoint.state_dict()
    else:
        if 'state_dict' in checkpoint:
            # timm checkpoint
            state_dict = checkpoint['state_dict']
        elif 'model' in checkpoint:
            # deit checkpoint
            state_dict = checkpoint['model']
        else:
            state_dict = checkpoint
    weight = convert_key_name(state_dict, visual_split)
    weight = convert_tensor(weight)
    mmengine.mkdir_or_exist(osp.dirname(args.dst))
    torch.save(weight, args.dst)


if __name__ == '__main__':
    main()