diff --git a/modules/sdnq/quantizer.py b/modules/sdnq/quantizer.py
index 9a8283a1b..09f10abe0 100644
--- a/modules/sdnq/quantizer.py
+++ b/modules/sdnq/quantizer.py
@@ -201,11 +201,11 @@ def update_modules_quant_config(quant_kwargs: dict, modules_quant_config: dict[s
         and quant_kwargs["use_dynamic_quantization"] and quant_kwargs[use_quantized_matmul_key]
         and quant_kwargs["quantized_matmul_dtype"] is None and not is_fp8_mm_supported
         and not dtype_dict[layer.sdnq_dequantizer.weights_dtype]["is_integer"] and dtype_dict[layer.sdnq_dequantizer.weights_dtype]["num_bits"] < 16
+        and not layer.sdnq_dequantizer.use_quantized_matmul
     ):
-        if not layer.sdnq_dequantizer.use_quantized_matmul:
-            if quant_kwargs["param_name"] not in modules_quant_config.keys():
-                modules_quant_config[quant_kwargs["param_name"]] = {}
-            modules_quant_config[quant_kwargs["param_name"]][use_quantized_matmul_key] = layer.sdnq_dequantizer.use_quantized_matmul
+        if quant_kwargs["param_name"] not in modules_quant_config.keys():
+            modules_quant_config[quant_kwargs["param_name"]] = {}
+        modules_quant_config[quant_kwargs["param_name"]][use_quantized_matmul_key] = False
     return modules_quant_config
 
 
@@ -582,7 +582,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", quantized_matmul_dtype=None
 def apply_sdnq_to_module(model, weights_dtype="int8", quantized_matmul_dtype=None, torch_dtype=None, group_size=0, svd_rank=32, svd_steps=8, dynamic_loss_threshold=None, use_svd=False, quant_conv=False, quant_embedding=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, use_dynamic_quantization=False, use_stochastic_rounding=False, dequantize_fp32=True, non_blocking=False, modules_to_not_convert: list[str] | None = None, modules_dtype_dict: dict[str, list[str]] | None = None, modules_quant_config: dict[str, dict] | None = None, quantization_device=None, return_device=None, full_param_name=""): # pylint: disable=unused-argument
     has_children = list(model.children())
     if not has_children:
-        return model, modules_to_not_convert, modules_dtype_dict
+        return model, (modules_to_not_convert, modules_dtype_dict, modules_quant_config)
     if modules_to_not_convert is None:
         modules_to_not_convert = []
     if modules_dtype_dict is None: