leon-se/Idefics3-8B-Llama3-FP8-Dynamic

FP8-Dynamic quant for Idefics2-8B-Llama3, requires about ~12 GB

vLLM usage

vllm serve leon-se/Idefics3-8B-Llama3-FP8-Dynamic --trust-remote-code

Code used for quantization

from transformers import AutoProcessor, AutoModelForVision2Seq
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.transformers import oneshot

model_name = "HuggingfaceM4/Idefics3-8B-Llama3"

processor = AutoProcessor.from_pretrained(model_name)
model = AutoModelForVision2Seq.from_pretrained(model_name, device_map="cuda", torch_dtype="auto", trust_remote_code=True)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="FP8_DYNAMIC",
    ignore=["re:.*lm_head", "re:model.vision_model.*", "re:model.connector.*"],
)

SAVE_DIR = model_name.split("/")[1] + "-FP8-Dynamic"
oneshot(model=model, recipe=recipe, output_dir=SAVE_DIR)
processor.save_pretrained(SAVE_DIR)