Upload convert-hf-vision-to-gguf.py

convert-hf-vision-to-gguf.py

@@ -0,0 +1,182 @@
+import argparse
+
+import numpy as np
+from gguf import *
+from safetensors import safe_open
+from transformers import AutoTokenizer
+
+
+def k(raw_key: str, arch: str) -> str:
+    return raw_key.format(arch=arch)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--model", type=str, default="nanoLLaVA/model.safetensors")
+parser.add_argument("--tokenizer", type=str, default="nanoLLaVA")
+args = parser.parse_args()
+
+tensors = safe_open(args.model, framework="np", device="cpu")
+
+### Vision encoder
+
+ftype = 1  # fp16
+
+fname_middle = "mmproj-"
+has_text_encoder = False
+has_llava_projector = True
+
+fname_out = "nanollava-mmproj-f16.gguf"
+fout = GGUFWriter(fname_out, arch="clip")
+
+fout.add_bool("clip.has_text_encoder", False)
+fout.add_bool("clip.has_vision_encoder", True)
+fout.add_bool("clip.has_llava_projector", True)
+fout.add_file_type(ftype)  # fp16
+
+model_name = "qnguyen3/nanoLLaVA"
+fout.add_name(model_name)
+fout.add_description("image encoder for " + model_name)
+fout.add_string("clip.projector_type", "mlp")
+
+# vision model hparams
+VISION = "clip.vision"
+fout.add_uint32("clip.vision.image_size", 378)
+fout.add_uint32("clip.vision.patch_size", 14)
+fout.add_uint32(k(KEY_EMBEDDING_LENGTH, VISION), 1152)
+fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, VISION), 4304)
+fout.add_uint32("clip.vision.projection_dim", 2048)
+fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, VISION), 16)
+fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, VISION), 1e-6)
+fout.add_uint32(k(KEY_BLOCK_COUNT, VISION), 27 + 1)
+
+fout.add_array("clip.vision.image_mean", [0.5, 0.5, 0.5])
+fout.add_array("clip.vision.image_std", [0.5, 0.5, 0.5])
+fout.add_bool("clip.use_gelu", True)  # using regular GELU instead of quick
+
+# vision projection
+fout.add_tensor(
+    "mm.0.weight",
+    tensors.get_tensor("model.mm_projector.0.weight").astype(
+        np.float16
+    ),
+)
+fout.add_tensor(
+    "mm.0.bias",
+    tensors.get_tensor("model.mm_projector.0.bias").astype(np.float32),
+)
+fout.add_tensor(
+    "mm.2.weight",
+    tensors.get_tensor("model.mm_projector.2.weight").astype(
+        np.float16
+    ),
+)
+fout.add_tensor(
+    "mm.2.bias",
+    tensors.get_tensor("model.mm_projector.2.bias").astype(np.float32),
+)
+
+# encoder (siglip)
+fout.add_tensor(
+    "v.position_embd.weight",
+    tensors.get_tensor("model.vision_tower.vision_tower.vision_model.embeddings.position_embedding.weight").astype(
+        np.float16
+    ),
+)
+fout.add_tensor(
+    "v.patch_embd.weight",
+    tensors.get_tensor(
+        "model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.weight"
+    )
+    .reshape(1152, 3, 14, 14)
+    .astype(np.float16),
+)
+fout.add_tensor(
+    "v.patch_embd.bias",
+    tensors.get_tensor(
+        "model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.bias"
+    ).astype(np.float32),
+)
+
+fout.add_tensor(
+    "v.post_ln.weight",
+    tensors.get_tensor("model.vision_tower.vision_tower.vision_model.post_layernorm.weight").astype(
+        np.float32
+    ),
+)
+fout.add_tensor(
+    "v.post_ln.bias",
+    tensors.get_tensor("model.vision_tower.vision_tower.vision_model.post_layernorm.bias").astype(
+        np.float32
+    ),
+)
+
+def blk_tensor(i, name):
+    return tensors.get_tensor(
+        rf"model.vision_tower.vision_tower.vision_model.encoder.layers.{i}.{name}"
+    )
+
+def add_tensor(blk_id, gguf_id=None):
+    if gguf_id is None:
+        gguf_id = blk_id
+
+    fout.add_tensor(f"v.blk.{gguf_id}.attn_q.weight", blk_tensor(blk_id, "self_attn.q_proj.weight").astype(np.float16))
+    fout.add_tensor(f"v.blk.{gguf_id}.attn_q.bias", blk_tensor(blk_id, "self_attn.q_proj.bias").astype(np.float32))
+    fout.add_tensor(f"v.blk.{gguf_id}.attn_k.weight", blk_tensor(blk_id, "self_attn.k_proj.weight").astype(np.float16))
+    fout.add_tensor(f"v.blk.{gguf_id}.attn_k.bias", blk_tensor(blk_id, "self_attn.k_proj.bias").astype(np.float32))
+    fout.add_tensor(f"v.blk.{gguf_id}.attn_v.weight", blk_tensor(blk_id, "self_attn.v_proj.weight").astype(np.float16))
+    fout.add_tensor(f"v.blk.{gguf_id}.attn_v.bias", blk_tensor(blk_id, "self_attn.v_proj.bias").astype(np.float32))
+
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.attn_out.weight",
+        blk_tensor(blk_id, "self_attn.out_proj.weight").astype(np.float16),
+    )
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.attn_out.bias",
+        blk_tensor(blk_id, "self_attn.out_proj.bias").astype(np.float32),
+    )
+
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ln1.weight",
+        blk_tensor(blk_id, "layer_norm1.weight").astype(np.float32),
+    )
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ln1.bias",
+        blk_tensor(blk_id, "layer_norm1.bias").astype(np.float32),
+    )
+
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ffn_down.weight",
+        blk_tensor(blk_id, "mlp.fc1.weight").astype(np.float16),
+    )
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ffn_down.bias",
+        blk_tensor(blk_id, "mlp.fc1.bias").astype(np.float32),
+    )
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ffn_up.weight",
+        blk_tensor(blk_id, "mlp.fc2.weight").astype(np.float16),
+    )
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ffn_up.bias",
+        blk_tensor(blk_id, "mlp.fc2.bias").astype(np.float32),
+    )
+
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ln2.weight",
+        blk_tensor(blk_id, "layer_norm2.weight").astype(np.float32),
+    )
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ln2.bias",
+        blk_tensor(blk_id, "layer_norm2.bias").astype(np.float32),
+    )
+
+for i in range(27):
+    add_tensor(i)
+
+# Duplicate the last block (llava-cli skips over this)
+add_tensor(26, 27)
+
+fout.write_header_to_file()
+fout.write_kv_data_to_file()
+fout.write_tensors_to_file()
+fout.close()