Fix PEFT integration with new weight loader (#42701)

simplify

Author: Cyrilvallez

src/transformers/conversion_mapping.py

@@ -228,7 +228,7 @@ def get_model_conversion_mapping(
     """
     weight_conversions = []
 
-    # Load models with key mapping
+    # Load models with explicit, user-provided key mapping
     if key_mapping is not None:
         weight_conversions = [WeightRenaming(source_patterns=k, target_patterns=v) for k, v in key_mapping.items()]
     elif any(

src/transformers/integrations/peft.py

@@ -17,6 +17,7 @@
 import os
 from typing import Any, Literal
 
+from ..conversion_mapping import get_model_conversion_mapping
 from ..core_model_loading import WeightRenaming, rename_source_key
 from ..utils import (
     CONFIG_NAME,
@@ -46,26 +47,6 @@
 logger = logging.get_logger(__name__)
 
 
-# DO NOT MODIFY, KEPT FOR BC ONLY
-VLMS = [
-    "aria",
-    "ayavision",
-    "emu3",
-    "fuyu",
-    "gotocr2",
-    "gemma3",
-    "internvl",
-    "llava",  # all llava prefixed models fall under this check
-    "mistral3",
-    "mllama",
-    "paligemma",
-    "qwen2vl",
-    "qwen2_5_vl",
-    "videollava",
-    "vipllava",
-]
-
-
 class PeftAdapterMixin:
     """
     A class containing all functions for loading and using adapters weights that are supported in PEFT library. For
@@ -211,11 +192,10 @@ def load_adapter(
             if any(conf.peft_type != PeftType.LORA for conf in self.peft_config.values()):
                 raise ValueError("Hotswapping is currently only supported for LoRA, please set `hotswap=False`.")
 
+        key_mapping = adapter_kwargs.pop("key_mapping", None) if adapter_kwargs is not None else None
+        weight_conversions = get_model_conversion_mapping(self, key_mapping=key_mapping)
         # peft only supports low_cpu_mem_usage starting from v0.13.0
         peft_load_kwargs = {}
-        key_mapping = adapter_kwargs.pop("key_mapping", None) if adapter_kwargs is not None else None
-        if key_mapping is None and any(allowed_name in self.__class__.__name__.lower() for allowed_name in VLMS):
-            key_mapping = self._checkpoint_conversion_mapping
         peft_load_kwargs["low_cpu_mem_usage"] = low_cpu_mem_usage
 
         adapter_name = adapter_name if adapter_name is not None else "default"
@@ -292,8 +272,8 @@ def load_adapter(
 
         # We need to pre-process the state dict to remove unneeded prefixes - for backward compatibility
         renamings = []
-        if key_mapping:
-            renamings = [entry for entry in key_mapping if isinstance(entry, WeightRenaming)]
+        if weight_conversions:
+            renamings = [entry for entry in weight_conversions if isinstance(entry, WeightRenaming)]
         processed_adapter_state_dict = {}
         prefix = "base_model.model."
         state_dict = self.state_dict()

src/transformers/modeling_utils.py

@@ -4046,7 +4046,7 @@ def from_pretrained(
             hf_quantizer.postprocess_model(model, config=config)  # usually a no-op but sometimes needed
 
         if _adapter_model_path is not None:
-            adapter_kwargs["key_mapping"] = weight_conversions  # TODO: Dynamic weight loader for adapters
+            adapter_kwargs["key_mapping"] = key_mapping
             model.load_adapter(
                 _adapter_model_path,
                 adapter_name=adapter_name,