Update and refactor NNCF and add more quant options

installer.py

@@ -747,7 +747,7 @@ def install_openvino(torch_command):
         torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.6.0+cpu torchvision==0.21.0+cpu --index-url https://download.pytorch.org/whl/cpu')
 
     install(os.environ.get('OPENVINO_COMMAND', 'openvino==2025.0.0'), 'openvino')
-    install(os.environ.get('NNCF_COMMAND', 'nncf==2.15.0'), 'nncf')
+    install(os.environ.get('NNCF_COMMAND', 'nncf==2.16.0'), 'nncf')
     os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX')
     if os.environ.get("NEOReadDebugKeys", None) is None:
         os.environ.setdefault('NEOReadDebugKeys', '1')
@@ -779,7 +779,7 @@ def install_torch_addons():
     if opts.get('cuda_compile_backend', '') == 'olive-ai':
         install('olive-ai')
     if opts.get('nncf_compress_weights', False) and not args.use_openvino:
-        install('nncf==2.7.0', 'nncf')
+        install('nncf==2.16.0', 'nncf')
     if opts.get('optimum_quanto_weights', False):
         install('optimum-quanto==0.2.7', 'optimum-quanto')
     if opts.get('torchao_quantization', False):
@@ -1149,7 +1149,7 @@ def install_optional():
     install('albumentations==1.4.3', ignore=True)
     install('pydantic==1.10.21', ignore=True)
     reload('pydantic')
-    install('nncf==2.7.0', ignore=True, no_deps=True) # requires older pandas
+    install('nncf==2.16.0', ignore=True, no_deps=True) # requires older pandas
     # install('flash-attn', ignore=True) # requires cuda and nvcc to be installed
     install('gguf', ignore=True)
     try:

modules/control/units/controlnet.py

@@ -289,8 +289,6 @@ class ControlNet():
                 if "ControlNet" in opts.nncf_compress_weights:
                     try:
                         log.debug(f'Control {what} model NNCF Compress: id="{model_id}"')
-                        from installer import install
-                        install('nncf==2.7.0', quiet=True)
                         from modules.model_quant import nncf_compress_model
                         self.model = nncf_compress_model(self.model)
                     except Exception as e:

modules/model_quant.py

@@ -261,7 +261,7 @@ def load_nncf(msg='', silent=False):
     if intel_nncf is not None:
         return intel_nncf
     if not installed('nncf'):
-        install('nncf==2.7.0', quiet=True)
+        install('nncf==2.16.0', quiet=True)
         log.warning('Quantization: nncf installed please restart')
     try:
         import nncf
@@ -320,19 +320,17 @@ def apply_layerwise(sd_model, quiet:bool=False):
             log.error(f'Quantization: type=layerwise {e}')
 
 
-def nncf_send_to_device(model, device):
-    for child in model.children():
-        if child.__class__.__name__ == "WeightsDecompressor":
-            child.scale = child.scale.to(device)
-            child.zero_point = child.zero_point.to(device)
-        nncf_send_to_device(child, device)
-
-
 def nncf_compress_model(model, op=None, sd_model=None, send_to_device=True, do_gc=True):
-    from modules import devices, shared
     global quant_last_model_name, quant_last_model_device # pylint: disable=global-statement
-    nncf = load_nncf('Quantize model: type=NNCF')
+    from modules import devices, shared
+    from accelerate import init_empty_weights
+
+    load_nncf('Quantize model: type=NNCF')
+    from modules.model_quant_nncf import apply_nncf_to_module, nncf_send_to_device
+    from nncf.torch.nncf_module_replacement import replace_modules_by_nncf_modules # get around lazy import
+
     model.eval()
+
     if model.__class__.__name__ in {"T5EncoderModel", "UMT5EncoderModel"}:
         import torch
         from modules.model_quant_nncf import NNCF_T5DenseGatedActDense # T5DenseGatedActDense uses fp32
@@ -341,14 +339,23 @@ def nncf_compress_model(model, op=None, sd_model=None, send_to_device=True, do_g
                 model.encoder.block[i].layer[1].DenseReluDense,
                 dtype=torch.float32 if devices.dtype != torch.bfloat16 else torch.bfloat16
             )
+
     backup_embeddings = None
     if hasattr(model, "get_input_embeddings"):
         backup_embeddings = copy.deepcopy(model.get_input_embeddings())
-    model = nncf.compress_weights(model)
+
+    with init_empty_weights():
+        model, _ = replace_modules_by_nncf_modules(model)
+
+    num_bits = 8 if shared.opts.nncf_compress_weights_mode in {"INT8", "INT8_SYM", "INT8_ASYM"} else 4
+    is_asym_mode = shared.opts.nncf_compress_weights_mode in {"INT8", "INT4", "INT8_ASYM", "INT4_ASYM"}
+    model = apply_nncf_to_module(model, num_bits, is_asym_mode, quant_conv=shared.opts.nncf_quantize_conv_layers)
     if send_to_device:
         nncf_send_to_device(model, devices.device)
+
     if hasattr(model, "set_input_embeddings") and backup_embeddings is not None:
         model.set_input_embeddings(backup_embeddings)
+
     if op is not None and shared.opts.nncf_quantize_shuffle_weights:
         if quant_last_model_name is not None:
             if "." in quant_last_model_name:
@@ -371,7 +378,8 @@ def nncf_compress_model(model, op=None, sd_model=None, send_to_device=True, do_g
 
 
 def nncf_compress_weights(sd_model):
-    try:
+    #try:
+    if True:
         t0 = time.time()
         from modules import shared, devices, sd_models
         log.info(f"Quantization: type=NNCF modules={shared.opts.nncf_compress_weights}")
@@ -390,8 +398,8 @@ def nncf_compress_weights(sd_model):
 
         t1 = time.time()
         log.info(f"Quantization: type=NNCF time={t1-t0:.2f}")
-    except Exception as e:
-        log.warning(f"Quantization: type=NNCF {e}")
+    #except Exception as e:
+    #    log.warning(f"Quantization: type=NNCF {e}")
     return sd_model
 
 

modules/model_quant_nncf.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 from dataclasses import dataclass
 from enum import Enum
 
@@ -6,35 +6,154 @@ import torch
 from diffusers.quantizers.base import DiffusersQuantizer
 from diffusers.quantizers.quantization_config import QuantizationConfigMixin
 from diffusers.utils import get_module_from_name
+
 from accelerate import init_empty_weights
+from accelerate.utils import CustomDtype
 
 from modules import devices
 
 
-class NNCF_T5DenseGatedActDense(torch.nn.Module): # forward can't find what self is without creating a class
-    def __init__(self, T5DenseGatedActDense, dtype):
-        super().__init__()
-        self.wi_0 = T5DenseGatedActDense.wi_0
-        self.wi_1 = T5DenseGatedActDense.wi_1
-        self.wo = T5DenseGatedActDense.wo
-        self.dropout = T5DenseGatedActDense.dropout
-        self.act = T5DenseGatedActDense.act
-        self.torch_dtype = dtype
+torch_dtype_dict = {
+    "int8": torch.int8,
+    "uint8": torch.uint8,
+    "int4": CustomDtype.INT4,
+    "uint4": CustomDtype.INT4,
+}
 
-    def forward(self, hidden_states):
-        hidden_gelu = self.act(self.wi_0(hidden_states))
-        hidden_linear = self.wi_1(hidden_states)
-        hidden_states = hidden_gelu * hidden_linear
-        hidden_states = self.dropout(hidden_states)
-        hidden_states = hidden_states.to(self.torch_dtype) # this line needs to be forced
-        hidden_states = self.wo(hidden_states)
-        return hidden_states
+weights_dtype_dict = {
+    "int8_asym": "uint8",
+    "int8_sym": "int8",
+    "int4_asym": "uint4",
+    "int4_sym": "int4",
+    "int8": "uint8",
+    "int4": "uint4",
+}
 
+linear_types = ["NNCFLinear", "Linear"]
+conv_types = ["NNCFConv1d", "NNCFConv2d", "NNCFConv3d", "Conv1d", "Conv2d", "Conv3d"]
+conv_transpose_types = ["NNCFConvTranspose1d", "NNCFConvTranspose2d", "NNCFConvTranspose3d", "ConvTranspose1d", "ConvTranspose2d", "ConvTranspose3d"]
+
+allowed_types = []
+allowed_types.extend(linear_types)
+allowed_types.extend(conv_types)
+allowed_types.extend(conv_transpose_types)
 
 class QuantizationMethod(str, Enum):
     NNCF = "nncf"
 
 
+# de-abstracted and modified slghtly from the actual quant functions of nncf 2.16.0:
+def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_conv=True, param_name=None):
+    if layer.__class__.__name__ in allowed_types:
+        if torch_dtype is None:
+            torch_dtype = devices.dtype
+        layer.weight.data = layer.weight.data.float()
+
+        if layer.__class__.__name__ in conv_types:
+            if is_asym_mode or not quant_conv: # don't quant convs with asym mode
+                return layer
+            reduction_axes = [i for i in range(layer.weight.ndim) if i != 0]
+        if layer.__class__.__name__ in conv_transpose_types:
+            if is_asym_mode or not quant_conv: # don't quant convs with asym mode
+                return layer
+            reduction_axes = [i for i in range(layer.weight.ndim) if i != 1]
+        else:
+            reduction_axes = [layer.weight.ndim - 1]
+
+        if is_asym_mode:
+            level_low = 0
+            level_high = 2**num_bits - 1
+            min_values = torch.amin(layer.weight, dim=reduction_axes, keepdims=True)  # [a1, r, a2] -> [a1, 1, a2]
+            max_values = torch.amax(layer.weight, dim=reduction_axes, keepdims=True)  # [a1, r, a2] -> [a1, 1, a2]
+
+            levels = level_high - level_low + 1
+            scale = ((max_values - min_values) / (levels - 1)).type(torch.float32)
+            eps = torch.finfo(scale.dtype).eps
+
+            scale = torch.where(torch.abs(scale) < eps, eps, scale)
+            zero_point = level_low - torch.round(min_values / scale)
+            zero_point = torch.clip(zero_point.type(torch.int32), level_low, level_high)
+        else:
+            factor = 2 ** (num_bits - 1)
+
+            w_abs_min = torch.abs(torch.amin(layer.weight, dim=reduction_axes, keepdims=True))
+            w_max = torch.amax(layer.weight, dim=reduction_axes, keepdims=True)
+
+            scale = torch.where(w_abs_min >= w_max, w_abs_min, -w_max)
+            scale /= factor
+
+            eps = torch.finfo(scale.dtype).eps
+            scale = torch.where(torch.abs(scale) < eps, eps, scale)
+            zero_point = None
+
+        dtype = torch.uint8 if is_asym_mode else torch.int8
+        level_low = 0 if is_asym_mode else -(2 ** (num_bits - 1))
+        level_high = 2**num_bits - 1 if is_asym_mode else 2 ** (num_bits - 1) - 1
+
+        compressed_weight = layer.weight.data / scale
+        if zero_point is not None:
+            compressed_weight += zero_point.type(layer.weight.dtype)
+
+        compressed_weight = torch.round(compressed_weight)
+        compressed_weight = torch.clip(compressed_weight, level_low, level_high).to(dtype)
+
+        if num_bits == 4:
+            if is_asym_mode:
+                decompressor = INT4AsymmetricWeightsDecompressor(
+                    scale=scale.data,
+                    zero_point=zero_point.data,
+                    compressed_weight_shape=compressed_weight.shape,
+                    result_shape=layer.weight.shape,
+                    result_dtype=torch_dtype
+                )
+            else:
+                decompressor = INT4SymmetricWeightsDecompressor(
+                    scale=scale.data,
+                    compressed_weight_shape=compressed_weight.shape,
+                    result_shape=layer.weight.shape,
+                    result_dtype=torch_dtype
+                )
+        else:
+            if is_asym_mode:
+                decompressor = INT8AsymmetricWeightsDecompressor(
+                    scale=scale.data,
+                    zero_point=zero_point.data,
+                    result_dtype=torch_dtype
+                )
+            else:
+                decompressor = INT8SymmetricWeightsDecompressor(
+                    scale=scale.data,
+                    result_dtype=torch_dtype
+                )
+
+        layer.register_pre_forward_operation(decompressor)
+        compressed_weight = decompressor.pack_weight(compressed_weight)
+
+        layer.weight.requires_grad = False
+        layer.weight.data = compressed_weight
+    return layer
+
+
+def apply_nncf_to_module(model, num_bits, is_asym_mode, quant_conv=True):
+    has_children = list(model.children())
+    if not has_children:
+        return model
+    for param_name, module in model.named_children():
+        if module.__class__.__name__.startswith("NNCF") and hasattr(module, "weight") and module.weight is not None:
+            module = nncf_compress_layer(module, num_bits, is_asym_mode, torch_dtype=devices.dtype, quant_conv=quant_conv, param_name=param_name)
+        module = apply_nncf_to_module(module, num_bits, is_asym_mode, quant_conv=quant_conv)
+    return model
+
+
+def nncf_send_to_device(model, device):
+    for child in model.children():
+        if "WeightsDecompressor" in child.__class__.__name__:
+            child.scale = child.scale.to(device)
+            if hasattr(child, "zero_point"):
+                child.zero_point = child.zero_point.to(device)
+        nncf_send_to_device(child, device)
+
+
 class NNCFQuantizer(DiffusersQuantizer):
     r"""
     Diffusers Quantizer for NNCF
@@ -81,33 +200,26 @@ class NNCFQuantizer(DiffusersQuantizer):
 
         split_param_name = param_name.split(".")
         if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert):
-            from nncf.torch.quantization.quantize_functions import get_scale_zp_from_input_low_input_high
-            from nncf.torch.quantization.weights_compression import WeightsDecompressor
-            from nncf.torch.layers import NNCFEmbedding
-
-            if not isinstance(layer, torch.nn.Embedding) and not isinstance(layer, NNCFEmbedding):
-                target_dim = layer.target_weight_dim_for_compression
-                stat_dim = (target_dim + 1) % 2
-                input_low = torch.min(layer.weight, dim=stat_dim).values.detach()
-                input_high = torch.max(layer.weight, dim=stat_dim).values.detach()
-                scale, zero_point = get_scale_zp_from_input_low_input_high(0, 255, input_low, input_high)
-
-                scale = scale.unsqueeze(stat_dim)
-                zero_point = zero_point.unsqueeze(stat_dim)
-                layer.register_pre_forward_operation(WeightsDecompressor(zero_point, scale))
-
-                compressed_weight = layer.weight.data / scale + zero_point
-                compressed_weight = torch.clamp(torch.round(compressed_weight), 0, 255)
-
-                layer.weight.requires_grad = False
-                layer.weight.data = compressed_weight.type(dtype=torch.uint8)
+            layer = nncf_compress_layer(
+                layer,
+                self.quantization_config.num_bits,
+                self.quantization_config.is_asym_mode,
+                torch_dtype=self.torch_dtype,
+                param_name=param_name
+            )
 
     def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]:
-        max_memory = {key: val * 0.80 for key, val in max_memory.items()}
+        max_memory = {key: val * 0.70 for key, val in max_memory.items()}
         return max_memory
 
     def adjust_target_dtype(self, target_dtype: "torch.dtype") -> "torch.dtype":
-        return torch.uint8
+        return torch_dtype_dict[self.quantization_config.weights_dtype]
+
+    def update_torch_dtype(self, torch_dtype: "torch.dtype" = None) -> "torch.dtype":
+        if torch_dtype is None:
+            torch_dtype = devices.dtype
+        self.torch_dtype = torch_dtype
+        return torch_dtype
 
     def _process_model_before_weight_loading(
         self,
@@ -137,7 +249,6 @@ class NNCFQuantizer(DiffusersQuantizer):
             model, _ = replace_modules_by_nncf_modules(model)
 
     def _process_model_after_weight_loading(self, model, **kwargs):
-        from modules.model_quant import nncf_send_to_device
         nncf_send_to_device(model, devices.device)
         return model
 
@@ -182,7 +293,7 @@ class NNCFConfig(QuantizationConfigMixin):
 
     Args:
         weights_dtype (`str`, *optional*, defaults to `"int8"`):
-            The target dtype for the weights after quantization. Supported values are ("int8")
+            The target dtype for the weights after quantization. Supported values are ("int8", "int8_sym", "int4", "int4_sym")
        modules_to_not_convert (`list`, *optional*, default to `None`):
             The list of modules to not quantize, useful for quantizing models that explicitly require to have some
             modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers).
@@ -190,21 +301,212 @@ class NNCFConfig(QuantizationConfigMixin):
 
     def __init__(
         self,
-        weights_dtype: str = "int8",
+        weights_dtype: str = "int8_sym",
         modules_to_not_convert: Optional[List[str]] = None,
         **kwargs,
     ):
         self.quant_method = QuantizationMethod.NNCF
-        self.weights_dtype = weights_dtype
+        self.weights_dtype = weights_dtype_dict[weights_dtype.lower()]
         self.modules_to_not_convert = modules_to_not_convert
 
         self.post_init()
 
+        self.num_bits = 8 if self.weights_dtype in {"int8", "uint8"} else 4
+        self.is_asym_mode = self.weights_dtype in {"uint8", "uint4"}
+        self.is_integer = True
+        self.group_size = -1
+
     def post_init(self):
         r"""
         Safety checker that arguments are correct
         """
-        accepted_weights = ["int8", "uint8"]
+        accepted_weights = ["int8", "uint8", "int4", "uint4"]
         if self.weights_dtype not in accepted_weights:
             raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights_dtype}")
 
+
+class NNCF_T5DenseGatedActDense(torch.nn.Module): # forward can't find what self is without creating a class
+    def __init__(self, T5DenseGatedActDense, dtype):
+        super().__init__()
+        self.wi_0 = T5DenseGatedActDense.wi_0
+        self.wi_1 = T5DenseGatedActDense.wi_1
+        self.wo = T5DenseGatedActDense.wo
+        self.dropout = T5DenseGatedActDense.dropout
+        self.act = T5DenseGatedActDense.act
+        self.torch_dtype = dtype
+
+    def forward(self, hidden_states):
+        hidden_gelu = self.act(self.wi_0(hidden_states))
+        hidden_linear = self.wi_1(hidden_states)
+        hidden_states = hidden_gelu * hidden_linear
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = hidden_states.to(self.torch_dtype) # this line needs to be forced
+        hidden_states = self.wo(hidden_states)
+        return hidden_states
+
+
+# WeightsDecompressor classes and functions are modified from NNCF 2.16.0
+
+def decompress_asymmetric(input: torch.Tensor, scale: torch.Tensor, zero_point: torch.Tensor) -> torch.Tensor:
+    input = input.type(scale.dtype)
+    zero_point = zero_point.type(scale.dtype)
+    decompressed_input = (input - zero_point) * scale
+    return decompressed_input
+
+
+def decompress_symmetric(input: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
+    input = input.type(scale.dtype)
+    decompressed_input = input * scale
+    return decompressed_input
+
+
+def unpack_uint4(packed_tensor: torch.Tensor) -> torch.Tensor:
+    return torch.stack((torch.bitwise_and(packed_tensor, 15), torch.bitwise_right_shift(packed_tensor, 4)), dim=-1)
+
+
+def unpack_int4(packed_tensor: torch.Tensor) -> torch.Tensor:
+    t = unpack_uint4(packed_tensor)
+    return t.type(torch.int8) - 8
+
+
+def pack_uint4(tensor: torch.Tensor) -> torch.Tensor:
+    if tensor.dtype != torch.uint8:
+        msg = f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported."
+        raise RuntimeError(msg)
+    packed_tensor = tensor.contiguous()
+    packed_tensor = packed_tensor.reshape(-1, 2)
+    packed_tensor = torch.bitwise_and(packed_tensor[..., ::2], 15) | packed_tensor[..., 1::2] << 4
+    return packed_tensor
+
+
+def pack_int4(tensor: torch.Tensor) -> torch.Tensor:
+    if tensor.dtype != torch.int8:
+        msg = f"Invalid tensor dtype {tensor.type}. torch.int8 type is supported."
+        raise RuntimeError(msg)
+    tensor = tensor + 8
+    return pack_uint4(tensor.type(torch.uint8))
+
+
+class INT8AsymmetricWeightsDecompressor(torch.nn.Module):
+    def __init__(self, scale: torch.Tensor, zero_point: torch.Tensor, result_dtype: Optional[torch.dtype] = None):
+        super().__init__()
+        self.scale = scale
+        self.zero_point = self.pack_weight(zero_point)
+        self.result_dtype = result_dtype
+
+    @property
+    def quantization_mode(self):
+        return"asymmetric"
+
+    def pack_weight(self, weight: torch.Tensor) -> torch.Tensor:
+        if torch.is_floating_point(weight):
+            msg = f"Invalid weight dtype {weight.type}. Integer types are supported."
+            raise ValueError(msg)
+        if torch.any((weight < 0) | (weight > 255)):
+            msg = "Weight values are not in [0, 255]."
+            raise ValueError(msg)
+        return weight.type(torch.uint8)
+
+    def forward(self, x, *args):
+        result = decompress_asymmetric(x.weight, self.scale, self.zero_point)
+        x.weight = result.type(self.result_dtype)
+
+
+class INT8SymmetricWeightsDecompressor(torch.nn.Module):
+    def __init__(self, scale: torch.Tensor, result_dtype: Optional[torch.dtype] = None):
+        super().__init__()
+        self.scale = scale
+        self.result_dtype = result_dtype
+
+    @property
+    def quantization_mode(self):
+        return"symmetric"
+
+    def pack_weight(self, weight: torch.Tensor) -> torch.Tensor:
+        if torch.any((weight < -128) | (weight > 127)):
+            msg = "Weight values are not in [-128, 127]."
+            raise ValueError(msg)
+        return weight.type(torch.int8)
+
+    def forward(self, x, *args):
+        result = decompress_symmetric(x.weight, self.scale)
+        x.weight = result.type(self.result_dtype)
+
+
+class INT4AsymmetricWeightsDecompressor(torch.nn.Module):
+    def __init__(
+        self,
+        scale: torch.Tensor,
+        zero_point: torch.Tensor,
+        compressed_weight_shape: Tuple[int, ...],
+        result_shape: Optional[Tuple[int, ...]] = None,
+        result_dtype: Optional[torch.dtype] = None,
+    ):
+        super().__init__()
+        self.scale = scale
+
+        self.zero_point_shape = zero_point.shape
+        self.zero_point = self.pack_weight(zero_point)
+
+        self.compressed_weight_shape = compressed_weight_shape
+        self.result_shape = result_shape
+        self.result_dtype = result_dtype
+
+    @property
+    def quantization_mode(self):
+        return"asymmetric"
+
+    def pack_weight(self, weight: torch.Tensor) -> torch.Tensor:
+        if torch.any((weight < 0) | (weight > 15)):
+            msg = "Weight values are not in [0, 15]."
+            raise ValueError(msg)
+        return pack_uint4(weight.type(torch.uint8))
+
+    def forward(self, x, *args):
+        result = unpack_uint4(x.weight)
+        result = result.reshape(self.compressed_weight_shape)
+
+        zero_point = unpack_uint4(self.zero_point)
+        zero_point = zero_point.reshape(self.zero_point_shape)
+
+        result = decompress_asymmetric(result, self.scale, zero_point)
+        result = result.reshape(self.result_shape) if self.result_shape is not None else result
+        x.weight = result.type(self.result_dtype)
+
+
+class INT4SymmetricWeightsDecompressor(torch.nn.Module):
+    def __init__(
+        self,
+        scale: torch.Tensor,
+        compressed_weight_shape: Tuple[int, ...],
+        result_shape: Optional[Tuple[int, ...]] = None,
+        result_dtype: Optional[torch.dtype] = None,
+    ):
+        super().__init__()
+        self.scale = scale
+
+        self.compressed_weight_shape = compressed_weight_shape
+        self.result_shape = result_shape
+        self.result_dtype = result_dtype
+
+    @property
+    def quantization_mode(self):
+        return"symmetric"
+
+    def pack_weight(self, weight: torch.Tensor) -> torch.Tensor:
+        if torch.is_floating_point(weight):
+            msg = f"Invalid weight dtype {weight.type}. Integer types are supported."
+            raise ValueError(msg)
+        if torch.any((weight < -8) | (weight > 7)):
+            msg = "Tensor values are not in [-8, 7]."
+            raise ValueError(msg)
+        return pack_int4(weight.type(torch.int8))
+
+    def forward(self, x, *args):
+        result = unpack_int4(x.weight)
+        result = result.reshape(self.compressed_weight_shape)
+
+        result = decompress_symmetric(result, self.scale)
+        result = result.reshape(self.result_shape) if self.result_shape is not None else result
+        x.weight = result.type(self.result_dtype)
+

modules/shared.py

@@ -538,11 +538,12 @@ options_templates.update(options_section(('quantization', "Quantization Settings
     "nncf_compress_sep": OptionInfo("<h2>NNCF: Neural Network Compression Framework</h2>", "", gr.HTML),
     "nncf_compress_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}),
     "nncf_compress_mode": OptionInfo("post", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native}),
-    "nncf_compress_weights_mode": OptionInfo("INT8", "Quantization type", gr.Dropdown, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'] if cmd_opts.use_openvino else ['INT8']}),
+    "nncf_compress_weights_mode": OptionInfo("INT8_SYM", "Quantization type", gr.Dropdown, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'] if cmd_opts.use_openvino else ['INT8', 'INT8_SYM', 'INT4', 'INT4_SYM']}),
     "nncf_compress_weights_raito": OptionInfo(0, "Compress ratio", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}),
     "nncf_compress_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 512, "step": 1, "visible": cmd_opts.use_openvino}),
     "nncf_quantize": OptionInfo([], "OpenVINO enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "TE"], "visible": cmd_opts.use_openvino}),
     "nncf_quantize_mode": OptionInfo("INT8", "OpenVINO mode", gr.Dropdown, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}),
+    "nncf_quantize_conv_layers": OptionInfo(True, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}),
     "nncf_quantize_shuffle_weights": OptionInfo(False, "Shuffle weights", gr.Checkbox, {"visible": native}),
 
     "layerwise_quantization_sep": OptionInfo("<h2>Layerwise Casting</h2>", "", gr.HTML),