Skip to content

Commit 03b91f7

Browse files
[Bugfix] Fix compressed-tensors models failing to load with transformers backend (#30287)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
1 parent f6227c2 commit 03b91f7

File tree

1 file changed

+30
-9
lines changed

1 file changed

+30
-9
lines changed

vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -116,16 +116,37 @@ def get_name(self) -> QuantizationMethods:
116116
return "compressed-tensors"
117117

118118
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
119-
self.target_scheme_map = hf_to_vllm_mapper.apply_dict(self.target_scheme_map)
120-
self.ignore = hf_to_vllm_mapper.apply_list(self.ignore)
121-
self.sparsity_scheme_map = hf_to_vllm_mapper.apply_dict(
122-
self.sparsity_scheme_map
123-
)
124-
self.sparsity_ignore_list = hf_to_vllm_mapper.apply_list(
125-
self.sparsity_ignore_list
126-
)
119+
"""
120+
Transform layer paths in config targets to match vLLM's naming.
121+
122+
The WeightsMapper is designed for weight paths, but some backends
123+
(e.g. transformers) use broad prefix mappings like "" -> "model."
124+
which would incorrectly transform non-path targets.
125+
126+
compressed-tensors targets can be:
127+
- Layer paths: "layers.0.self_attn.q_proj" -> transformed
128+
- Module class names: "Linear" -> preserved (no ".")
129+
- Regex patterns: "re:.*proj" -> preserved (starts with "re:")
130+
"""
131+
132+
def _map_target(target: str) -> str | None:
133+
is_layer_path = "." in target and not target.startswith("re:")
134+
if is_layer_path:
135+
return hf_to_vllm_mapper._map_name(target)
136+
return target
137+
138+
def _apply_dict(d: dict) -> dict:
139+
return {k: v for t, v in d.items() if (k := _map_target(t)) is not None}
140+
141+
def _apply_list(lst: list) -> list:
142+
return [t for x in lst if (t := _map_target(x)) is not None]
143+
144+
self.target_scheme_map = _apply_dict(self.target_scheme_map)
145+
self.ignore = _apply_list(self.ignore)
146+
self.sparsity_scheme_map = _apply_dict(self.sparsity_scheme_map)
147+
self.sparsity_ignore_list = _apply_list(self.sparsity_ignore_list)
127148
if self.kv_cache_scheme is not None:
128-
self.kv_cache_scheme = hf_to_vllm_mapper.apply_dict(self.kv_cache_scheme)
149+
self.kv_cache_scheme = _apply_dict(self.kv_cache_scheme)
129150

130151
def get_quant_method(
131152
self,

0 commit comments

Comments
 (0)