diffusers

module/ip_adapter/utils.py

@@ -123,7 +123,12 @@ def load_adapter_to_pipe(
         attn_procs = init_attn_proc(unet, adapter_tokens, use_lcm, use_adaln)
         unet.set_attn_processor(attn_procs)
         adapter_modules = torch.nn.ModuleList(unet.attn_processors.values())
-        missing, _ = adapter_modules.load_state_dict(state_dict["ip_adapter"], strict=False)
+        
+        # Filter out LoRA-related keys from the state dict
+        filtered_state_dict = {k: v for k, v in state_dict["ip_adapter"].items() 
+                              if not any(x in k for x in ['lora_A', 'lora_B'])}
+        
+        missing, _ = adapter_modules.load_state_dict(filtered_state_dict, strict=False)
         if len(missing) > 0:
             raise ValueError(f"Missing keys in adapter_modules: {missing}")