diff --git a/modules/sdnq/common.py b/modules/sdnq/common.py index 30d8a82e7..537c890e8 100644 --- a/modules/sdnq/common.py +++ b/modules/sdnq/common.py @@ -357,9 +357,8 @@ if use_torch_compile: kwargs["fullgraph"] = True if kwargs.get("dynamic", None) is None: kwargs["dynamic"] = False - if torch_version[0] > 2 or (torch_version[0] == 2 and torch_version[1] >= 12): - if kwargs.get("recompile_limit", None) is None: - kwargs["recompile_limit"] = max(8192, getattr(torch._dynamo.config, "recompile_limit", 0)) + if (torch_version[0] > 2 or (torch_version[0] == 2 and torch_version[1] >= 12)) and kwargs.get("recompile_limit", None) is None: + kwargs["recompile_limit"] = max(8192, getattr(torch._dynamo.config, "recompile_limit", 0)) if os.environ.get("SDNQ_COMPILE_KWARGS", None) is not None: for key, value in json.loads(os.environ.get("SDNQ_COMPILE_KWARGS")).items(): kwargs[key] = value diff --git a/modules/sdnq/file_loader.py b/modules/sdnq/file_loader.py index d5fe53574..7ea8e1fbe 100644 --- a/modules/sdnq/file_loader.py +++ b/modules/sdnq/file_loader.py @@ -19,7 +19,7 @@ def load_safetensors(files: list[str], state_dict: dict | None = None, key_mappi state_dict = {} for fn in files: with safe_open(fn, framework="pt", device=str(device)) as f: - for key in f.keys(): + for key in f: state_dict[map_keys(key, key_mapping)] = f.get_tensor(key) diff --git a/modules/sdnq/layers/__init__.py b/modules/sdnq/layers/__init__.py index 6835d059e..2e66f7718 100644 --- a/modules/sdnq/layers/__init__.py +++ b/modules/sdnq/layers/__init__.py @@ -30,7 +30,7 @@ class SDNQLayer(torch.nn.Module): return self.forward_func(self, *args, **kwargs) def __repr__(self) -> str: - return f"{self.__class__.__name__}(original_class={self.original_class} forward_func={self.forward_func} sdnq_dequantizer={repr(getattr(self, 'sdnq_dequantizer', None))})" + return f"{self.__class__.__name__}(original_class={self.original_class} forward_func={self.forward_func} sdnq_dequantizer={getattr(self, 'sdnq_dequantizer', None)})" class SDNQLinear(SDNQLayer, torch.nn.Linear): diff --git a/modules/sdnq/loader.py b/modules/sdnq/loader.py index 276c609cd..60235af69 100644 --- a/modules/sdnq/loader.py +++ b/modules/sdnq/loader.py @@ -168,9 +168,8 @@ def load_sdnq_model( # older transformers case, handle known models manually if model.__class__.__name__ in {"T5EncoderModel", "UMT5EncoderModel"} and "encoder.embed_tokens.weight" not in state_dict: state_dict["encoder.embed_tokens.weight"] = state_dict["shared.weight"] - elif model.__class__.__name__ in {"Qwen3ForCausalLM"} and "lm_head.weight" not in state_dict: - if "model.embed_tokens.weight" in state_dict: - state_dict["lm_head.weight"] = state_dict["model.embed_tokens.weight"] + elif model.__class__.__name__ in {"Qwen3ForCausalLM"} and "lm_head.weight" not in state_dict and "model.embed_tokens.weight" in state_dict: + state_dict["lm_head.weight"] = state_dict["model.embed_tokens.weight"] model.load_state_dict(state_dict, assign=True) del state_dict diff --git a/modules/sdnq/quant_utils.py b/modules/sdnq/quant_utils.py index 4e52295bd..a711cfe64 100644 --- a/modules/sdnq/quant_utils.py +++ b/modules/sdnq/quant_utils.py @@ -130,6 +130,8 @@ def rotate_hadamard(weight: torch.Tensor, group_size: int = 256, hadamard: torch hadamard = get_hadamard(group_size, dtype=weight.dtype, device=weight.device) else: group_size = hadamard.shape[-1] + if hadamard.dtype != weight.dtype: + hadamard = hadamard.to(dtype=weight.dtype) if is_conv: weight_shape = list(weight.shape)[1:] weight = weight.flatten(1,-1) diff --git a/modules/sdnq/quantizer.py b/modules/sdnq/quantizer.py index 15b6e8043..0b323eec8 100644 --- a/modules/sdnq/quantizer.py +++ b/modules/sdnq/quantizer.py @@ -375,7 +375,7 @@ def sdnq_quantize_layer_weight_dynamic( if quantization_loss <= dynamic_loss_threshold: del original_weight_fp32 if quantization_config is not None: - if sdnq_dequantizer.weights_dtype not in quantization_config.modules_dtype_dict.keys(): + if sdnq_dequantizer.weights_dtype not in quantization_config.modules_dtype_dict: quantization_config.modules_dtype_dict[sdnq_dequantizer.weights_dtype] = [param_name] else: quantization_config.modules_dtype_dict[sdnq_dequantizer.weights_dtype].append(param_name) @@ -570,7 +570,7 @@ class SDNQQuantize: missing_keys: list[str] | None = None, **kwargs, ) -> dict[str, torch.Tensor]: - _module_name, value = tuple(input_dict.items())[0] + _module_name, value = next(iter(input_dict.items())) value = value[0] self.hf_quantizer.create_quantized_param(model, value, full_layer_name, value.device) param, name = get_module_from_name(model, full_layer_name) @@ -1001,7 +1001,7 @@ class SDNQConfig(QuantizationConfigMixin): value = list(value) self.modules_dtype_dict[key] = value if not isinstance(key, str) or not isinstance(value, list): - raise ValueError(f"modules_dtype_dict must be a dictionary of strings and lists but got {type(key)} and {type(value)}") + raise TypeError(f"modules_dtype_dict must be a dictionary of strings and lists but got {type(key)} and {type(value)}") if self.modules_quant_config is None: self.modules_quant_config = {} diff --git a/modules/sdnq/utils.py b/modules/sdnq/utils.py index cc423432f..e79e420ce 100644 --- a/modules/sdnq/utils.py +++ b/modules/sdnq/utils.py @@ -80,7 +80,7 @@ def get_quant_args_from_config(quantization_config: dict) -> dict: quantization_config_dict.pop("is_training", None) quantization_config_dict.pop("sdnq_version", None) if quantization_config_dict.get("modules_quant_config", None) is not None: - for key in quantization_config_dict["modules_quant_config"].keys(): + for key in quantization_config_dict["modules_quant_config"]: quantization_config_dict["modules_quant_config"][key] = get_quant_args_from_config(quantization_config_dict["modules_quant_config"][key]) return quantization_config_dict @@ -90,7 +90,7 @@ def get_minimum_dtype(weights_dtype: str, param_name: str, modules_dtype_dict: d for key, value in modules_dtype_dict.items(): if check_param_name_in(param_name, value) is not None: key = key.lower() - if key.startswith("minimum") or key.endswith("bit") or key.endswith("bits"): + if key.startswith("minimum") or key.endswith(("bit", "bits")): minimum_bits_str = key.removeprefix("minimum").removeprefix("-").removeprefix("_").removesuffix("bits").removesuffix("bit").removesuffix("-").removesuffix("_") if minimum_bits_str.startswith("uint"): is_unsigned = True @@ -189,7 +189,7 @@ def add_module_skip_keys(model: torch.nn.Module, quantization_config): if skip_key_list is not None: quantization_config.modules_to_not_convert.extend(skip_key_list[0]) for key, value in skip_key_list[1].items(): - if key in quantization_config.modules_dtype_dict.keys(): + if key in quantization_config.modules_dtype_dict: quantization_config.modules_dtype_dict[key].extend(value) else: quantization_config.modules_dtype_dict[key] = value