diff --git a/installer.py b/installer.py
index 687a07bd4..99555a9f2 100644
--- a/installer.py
+++ b/installer.py
@@ -747,7 +747,7 @@ def install_openvino(torch_command):
torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.6.0+cpu torchvision==0.21.0+cpu --index-url https://download.pytorch.org/whl/cpu')
install(os.environ.get('OPENVINO_COMMAND', 'openvino==2025.0.0'), 'openvino')
- install(os.environ.get('NNCF_COMMAND', 'nncf==2.15.0'), 'nncf')
+ install(os.environ.get('NNCF_COMMAND', 'nncf==2.16.0'), 'nncf')
os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX')
if os.environ.get("NEOReadDebugKeys", None) is None:
os.environ.setdefault('NEOReadDebugKeys', '1')
@@ -779,7 +779,7 @@ def install_torch_addons():
if opts.get('cuda_compile_backend', '') == 'olive-ai':
install('olive-ai')
if opts.get('nncf_compress_weights', False) and not args.use_openvino:
- install('nncf==2.7.0', 'nncf')
+ install('nncf==2.16.0', 'nncf')
if opts.get('optimum_quanto_weights', False):
install('optimum-quanto==0.2.7', 'optimum-quanto')
if opts.get('torchao_quantization', False):
@@ -1149,7 +1149,7 @@ def install_optional():
install('albumentations==1.4.3', ignore=True)
install('pydantic==1.10.21', ignore=True)
reload('pydantic')
- install('nncf==2.7.0', ignore=True, no_deps=True) # requires older pandas
+ install('nncf==2.16.0', ignore=True, no_deps=True) # requires older pandas
# install('flash-attn', ignore=True) # requires cuda and nvcc to be installed
install('gguf', ignore=True)
try:
diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py
index a68207389..7a971c340 100644
--- a/modules/control/units/controlnet.py
+++ b/modules/control/units/controlnet.py
@@ -289,8 +289,6 @@ class ControlNet():
if "ControlNet" in opts.nncf_compress_weights:
try:
log.debug(f'Control {what} model NNCF Compress: id="{model_id}"')
- from installer import install
- install('nncf==2.7.0', quiet=True)
from modules.model_quant import nncf_compress_model
self.model = nncf_compress_model(self.model)
except Exception as e:
diff --git a/modules/model_quant.py b/modules/model_quant.py
index 7561da7a6..596a369b5 100644
--- a/modules/model_quant.py
+++ b/modules/model_quant.py
@@ -261,7 +261,7 @@ def load_nncf(msg='', silent=False):
if intel_nncf is not None:
return intel_nncf
if not installed('nncf'):
- install('nncf==2.7.0', quiet=True)
+ install('nncf==2.16.0', quiet=True)
log.warning('Quantization: nncf installed please restart')
try:
import nncf
@@ -320,19 +320,17 @@ def apply_layerwise(sd_model, quiet:bool=False):
log.error(f'Quantization: type=layerwise {e}')
-def nncf_send_to_device(model, device):
- for child in model.children():
- if child.__class__.__name__ == "WeightsDecompressor":
- child.scale = child.scale.to(device)
- child.zero_point = child.zero_point.to(device)
- nncf_send_to_device(child, device)
-
-
def nncf_compress_model(model, op=None, sd_model=None, send_to_device=True, do_gc=True):
- from modules import devices, shared
global quant_last_model_name, quant_last_model_device # pylint: disable=global-statement
- nncf = load_nncf('Quantize model: type=NNCF')
+ from modules import devices, shared
+ from accelerate import init_empty_weights
+
+ load_nncf('Quantize model: type=NNCF')
+ from modules.model_quant_nncf import apply_nncf_to_module, nncf_send_to_device
+ from nncf.torch.nncf_module_replacement import replace_modules_by_nncf_modules # get around lazy import
+
model.eval()
+
if model.__class__.__name__ in {"T5EncoderModel", "UMT5EncoderModel"}:
import torch
from modules.model_quant_nncf import NNCF_T5DenseGatedActDense # T5DenseGatedActDense uses fp32
@@ -341,14 +339,23 @@ def nncf_compress_model(model, op=None, sd_model=None, send_to_device=True, do_g
model.encoder.block[i].layer[1].DenseReluDense,
dtype=torch.float32 if devices.dtype != torch.bfloat16 else torch.bfloat16
)
+
backup_embeddings = None
if hasattr(model, "get_input_embeddings"):
backup_embeddings = copy.deepcopy(model.get_input_embeddings())
- model = nncf.compress_weights(model)
+
+ with init_empty_weights():
+ model, _ = replace_modules_by_nncf_modules(model)
+
+ num_bits = 8 if shared.opts.nncf_compress_weights_mode in {"INT8", "INT8_SYM", "INT8_ASYM"} else 4
+ is_asym_mode = shared.opts.nncf_compress_weights_mode in {"INT8", "INT4", "INT8_ASYM", "INT4_ASYM"}
+ model = apply_nncf_to_module(model, num_bits, is_asym_mode, quant_conv=shared.opts.nncf_quantize_conv_layers)
if send_to_device:
nncf_send_to_device(model, devices.device)
+
if hasattr(model, "set_input_embeddings") and backup_embeddings is not None:
model.set_input_embeddings(backup_embeddings)
+
if op is not None and shared.opts.nncf_quantize_shuffle_weights:
if quant_last_model_name is not None:
if "." in quant_last_model_name:
@@ -371,7 +378,8 @@ def nncf_compress_model(model, op=None, sd_model=None, send_to_device=True, do_g
def nncf_compress_weights(sd_model):
- try:
+ #try:
+ if True:
t0 = time.time()
from modules import shared, devices, sd_models
log.info(f"Quantization: type=NNCF modules={shared.opts.nncf_compress_weights}")
@@ -390,8 +398,8 @@ def nncf_compress_weights(sd_model):
t1 = time.time()
log.info(f"Quantization: type=NNCF time={t1-t0:.2f}")
- except Exception as e:
- log.warning(f"Quantization: type=NNCF {e}")
+ #except Exception as e:
+ # log.warning(f"Quantization: type=NNCF {e}")
return sd_model
diff --git a/modules/model_quant_nncf.py b/modules/model_quant_nncf.py
index 2b9129b07..ad5454da3 100644
--- a/modules/model_quant_nncf.py
+++ b/modules/model_quant_nncf.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
from dataclasses import dataclass
from enum import Enum
@@ -6,35 +6,154 @@ import torch
from diffusers.quantizers.base import DiffusersQuantizer
from diffusers.quantizers.quantization_config import QuantizationConfigMixin
from diffusers.utils import get_module_from_name
+
from accelerate import init_empty_weights
+from accelerate.utils import CustomDtype
from modules import devices
-class NNCF_T5DenseGatedActDense(torch.nn.Module): # forward can't find what self is without creating a class
- def __init__(self, T5DenseGatedActDense, dtype):
- super().__init__()
- self.wi_0 = T5DenseGatedActDense.wi_0
- self.wi_1 = T5DenseGatedActDense.wi_1
- self.wo = T5DenseGatedActDense.wo
- self.dropout = T5DenseGatedActDense.dropout
- self.act = T5DenseGatedActDense.act
- self.torch_dtype = dtype
+torch_dtype_dict = {
+ "int8": torch.int8,
+ "uint8": torch.uint8,
+ "int4": CustomDtype.INT4,
+ "uint4": CustomDtype.INT4,
+}
- def forward(self, hidden_states):
- hidden_gelu = self.act(self.wi_0(hidden_states))
- hidden_linear = self.wi_1(hidden_states)
- hidden_states = hidden_gelu * hidden_linear
- hidden_states = self.dropout(hidden_states)
- hidden_states = hidden_states.to(self.torch_dtype) # this line needs to be forced
- hidden_states = self.wo(hidden_states)
- return hidden_states
+weights_dtype_dict = {
+ "int8_asym": "uint8",
+ "int8_sym": "int8",
+ "int4_asym": "uint4",
+ "int4_sym": "int4",
+ "int8": "uint8",
+ "int4": "uint4",
+}
+linear_types = ["NNCFLinear", "Linear"]
+conv_types = ["NNCFConv1d", "NNCFConv2d", "NNCFConv3d", "Conv1d", "Conv2d", "Conv3d"]
+conv_transpose_types = ["NNCFConvTranspose1d", "NNCFConvTranspose2d", "NNCFConvTranspose3d", "ConvTranspose1d", "ConvTranspose2d", "ConvTranspose3d"]
+
+allowed_types = []
+allowed_types.extend(linear_types)
+allowed_types.extend(conv_types)
+allowed_types.extend(conv_transpose_types)
class QuantizationMethod(str, Enum):
NNCF = "nncf"
+# de-abstracted and modified slghtly from the actual quant functions of nncf 2.16.0:
+def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_conv=True, param_name=None):
+ if layer.__class__.__name__ in allowed_types:
+ if torch_dtype is None:
+ torch_dtype = devices.dtype
+ layer.weight.data = layer.weight.data.float()
+
+ if layer.__class__.__name__ in conv_types:
+ if is_asym_mode or not quant_conv: # don't quant convs with asym mode
+ return layer
+ reduction_axes = [i for i in range(layer.weight.ndim) if i != 0]
+ if layer.__class__.__name__ in conv_transpose_types:
+ if is_asym_mode or not quant_conv: # don't quant convs with asym mode
+ return layer
+ reduction_axes = [i for i in range(layer.weight.ndim) if i != 1]
+ else:
+ reduction_axes = [layer.weight.ndim - 1]
+
+ if is_asym_mode:
+ level_low = 0
+ level_high = 2**num_bits - 1
+ min_values = torch.amin(layer.weight, dim=reduction_axes, keepdims=True) # [a1, r, a2] -> [a1, 1, a2]
+ max_values = torch.amax(layer.weight, dim=reduction_axes, keepdims=True) # [a1, r, a2] -> [a1, 1, a2]
+
+ levels = level_high - level_low + 1
+ scale = ((max_values - min_values) / (levels - 1)).type(torch.float32)
+ eps = torch.finfo(scale.dtype).eps
+
+ scale = torch.where(torch.abs(scale) < eps, eps, scale)
+ zero_point = level_low - torch.round(min_values / scale)
+ zero_point = torch.clip(zero_point.type(torch.int32), level_low, level_high)
+ else:
+ factor = 2 ** (num_bits - 1)
+
+ w_abs_min = torch.abs(torch.amin(layer.weight, dim=reduction_axes, keepdims=True))
+ w_max = torch.amax(layer.weight, dim=reduction_axes, keepdims=True)
+
+ scale = torch.where(w_abs_min >= w_max, w_abs_min, -w_max)
+ scale /= factor
+
+ eps = torch.finfo(scale.dtype).eps
+ scale = torch.where(torch.abs(scale) < eps, eps, scale)
+ zero_point = None
+
+ dtype = torch.uint8 if is_asym_mode else torch.int8
+ level_low = 0 if is_asym_mode else -(2 ** (num_bits - 1))
+ level_high = 2**num_bits - 1 if is_asym_mode else 2 ** (num_bits - 1) - 1
+
+ compressed_weight = layer.weight.data / scale
+ if zero_point is not None:
+ compressed_weight += zero_point.type(layer.weight.dtype)
+
+ compressed_weight = torch.round(compressed_weight)
+ compressed_weight = torch.clip(compressed_weight, level_low, level_high).to(dtype)
+
+ if num_bits == 4:
+ if is_asym_mode:
+ decompressor = INT4AsymmetricWeightsDecompressor(
+ scale=scale.data,
+ zero_point=zero_point.data,
+ compressed_weight_shape=compressed_weight.shape,
+ result_shape=layer.weight.shape,
+ result_dtype=torch_dtype
+ )
+ else:
+ decompressor = INT4SymmetricWeightsDecompressor(
+ scale=scale.data,
+ compressed_weight_shape=compressed_weight.shape,
+ result_shape=layer.weight.shape,
+ result_dtype=torch_dtype
+ )
+ else:
+ if is_asym_mode:
+ decompressor = INT8AsymmetricWeightsDecompressor(
+ scale=scale.data,
+ zero_point=zero_point.data,
+ result_dtype=torch_dtype
+ )
+ else:
+ decompressor = INT8SymmetricWeightsDecompressor(
+ scale=scale.data,
+ result_dtype=torch_dtype
+ )
+
+ layer.register_pre_forward_operation(decompressor)
+ compressed_weight = decompressor.pack_weight(compressed_weight)
+
+ layer.weight.requires_grad = False
+ layer.weight.data = compressed_weight
+ return layer
+
+
+def apply_nncf_to_module(model, num_bits, is_asym_mode, quant_conv=True):
+ has_children = list(model.children())
+ if not has_children:
+ return model
+ for param_name, module in model.named_children():
+ if module.__class__.__name__.startswith("NNCF") and hasattr(module, "weight") and module.weight is not None:
+ module = nncf_compress_layer(module, num_bits, is_asym_mode, torch_dtype=devices.dtype, quant_conv=quant_conv, param_name=param_name)
+ module = apply_nncf_to_module(module, num_bits, is_asym_mode, quant_conv=quant_conv)
+ return model
+
+
+def nncf_send_to_device(model, device):
+ for child in model.children():
+ if "WeightsDecompressor" in child.__class__.__name__:
+ child.scale = child.scale.to(device)
+ if hasattr(child, "zero_point"):
+ child.zero_point = child.zero_point.to(device)
+ nncf_send_to_device(child, device)
+
+
class NNCFQuantizer(DiffusersQuantizer):
r"""
Diffusers Quantizer for NNCF
@@ -81,33 +200,26 @@ class NNCFQuantizer(DiffusersQuantizer):
split_param_name = param_name.split(".")
if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert):
- from nncf.torch.quantization.quantize_functions import get_scale_zp_from_input_low_input_high
- from nncf.torch.quantization.weights_compression import WeightsDecompressor
- from nncf.torch.layers import NNCFEmbedding
-
- if not isinstance(layer, torch.nn.Embedding) and not isinstance(layer, NNCFEmbedding):
- target_dim = layer.target_weight_dim_for_compression
- stat_dim = (target_dim + 1) % 2
- input_low = torch.min(layer.weight, dim=stat_dim).values.detach()
- input_high = torch.max(layer.weight, dim=stat_dim).values.detach()
- scale, zero_point = get_scale_zp_from_input_low_input_high(0, 255, input_low, input_high)
-
- scale = scale.unsqueeze(stat_dim)
- zero_point = zero_point.unsqueeze(stat_dim)
- layer.register_pre_forward_operation(WeightsDecompressor(zero_point, scale))
-
- compressed_weight = layer.weight.data / scale + zero_point
- compressed_weight = torch.clamp(torch.round(compressed_weight), 0, 255)
-
- layer.weight.requires_grad = False
- layer.weight.data = compressed_weight.type(dtype=torch.uint8)
+ layer = nncf_compress_layer(
+ layer,
+ self.quantization_config.num_bits,
+ self.quantization_config.is_asym_mode,
+ torch_dtype=self.torch_dtype,
+ param_name=param_name
+ )
def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]:
- max_memory = {key: val * 0.80 for key, val in max_memory.items()}
+ max_memory = {key: val * 0.70 for key, val in max_memory.items()}
return max_memory
def adjust_target_dtype(self, target_dtype: "torch.dtype") -> "torch.dtype":
- return torch.uint8
+ return torch_dtype_dict[self.quantization_config.weights_dtype]
+
+ def update_torch_dtype(self, torch_dtype: "torch.dtype" = None) -> "torch.dtype":
+ if torch_dtype is None:
+ torch_dtype = devices.dtype
+ self.torch_dtype = torch_dtype
+ return torch_dtype
def _process_model_before_weight_loading(
self,
@@ -137,7 +249,6 @@ class NNCFQuantizer(DiffusersQuantizer):
model, _ = replace_modules_by_nncf_modules(model)
def _process_model_after_weight_loading(self, model, **kwargs):
- from modules.model_quant import nncf_send_to_device
nncf_send_to_device(model, devices.device)
return model
@@ -182,7 +293,7 @@ class NNCFConfig(QuantizationConfigMixin):
Args:
weights_dtype (`str`, *optional*, defaults to `"int8"`):
- The target dtype for the weights after quantization. Supported values are ("int8")
+ The target dtype for the weights after quantization. Supported values are ("int8", "int8_sym", "int4", "int4_sym")
modules_to_not_convert (`list`, *optional*, default to `None`):
The list of modules to not quantize, useful for quantizing models that explicitly require to have some
modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers).
@@ -190,21 +301,212 @@ class NNCFConfig(QuantizationConfigMixin):
def __init__(
self,
- weights_dtype: str = "int8",
+ weights_dtype: str = "int8_sym",
modules_to_not_convert: Optional[List[str]] = None,
**kwargs,
):
self.quant_method = QuantizationMethod.NNCF
- self.weights_dtype = weights_dtype
+ self.weights_dtype = weights_dtype_dict[weights_dtype.lower()]
self.modules_to_not_convert = modules_to_not_convert
self.post_init()
+ self.num_bits = 8 if self.weights_dtype in {"int8", "uint8"} else 4
+ self.is_asym_mode = self.weights_dtype in {"uint8", "uint4"}
+ self.is_integer = True
+ self.group_size = -1
+
def post_init(self):
r"""
Safety checker that arguments are correct
"""
- accepted_weights = ["int8", "uint8"]
+ accepted_weights = ["int8", "uint8", "int4", "uint4"]
if self.weights_dtype not in accepted_weights:
raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights_dtype}")
+
+class NNCF_T5DenseGatedActDense(torch.nn.Module): # forward can't find what self is without creating a class
+ def __init__(self, T5DenseGatedActDense, dtype):
+ super().__init__()
+ self.wi_0 = T5DenseGatedActDense.wi_0
+ self.wi_1 = T5DenseGatedActDense.wi_1
+ self.wo = T5DenseGatedActDense.wo
+ self.dropout = T5DenseGatedActDense.dropout
+ self.act = T5DenseGatedActDense.act
+ self.torch_dtype = dtype
+
+ def forward(self, hidden_states):
+ hidden_gelu = self.act(self.wi_0(hidden_states))
+ hidden_linear = self.wi_1(hidden_states)
+ hidden_states = hidden_gelu * hidden_linear
+ hidden_states = self.dropout(hidden_states)
+ hidden_states = hidden_states.to(self.torch_dtype) # this line needs to be forced
+ hidden_states = self.wo(hidden_states)
+ return hidden_states
+
+
+# WeightsDecompressor classes and functions are modified from NNCF 2.16.0
+
+def decompress_asymmetric(input: torch.Tensor, scale: torch.Tensor, zero_point: torch.Tensor) -> torch.Tensor:
+ input = input.type(scale.dtype)
+ zero_point = zero_point.type(scale.dtype)
+ decompressed_input = (input - zero_point) * scale
+ return decompressed_input
+
+
+def decompress_symmetric(input: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
+ input = input.type(scale.dtype)
+ decompressed_input = input * scale
+ return decompressed_input
+
+
+def unpack_uint4(packed_tensor: torch.Tensor) -> torch.Tensor:
+ return torch.stack((torch.bitwise_and(packed_tensor, 15), torch.bitwise_right_shift(packed_tensor, 4)), dim=-1)
+
+
+def unpack_int4(packed_tensor: torch.Tensor) -> torch.Tensor:
+ t = unpack_uint4(packed_tensor)
+ return t.type(torch.int8) - 8
+
+
+def pack_uint4(tensor: torch.Tensor) -> torch.Tensor:
+ if tensor.dtype != torch.uint8:
+ msg = f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported."
+ raise RuntimeError(msg)
+ packed_tensor = tensor.contiguous()
+ packed_tensor = packed_tensor.reshape(-1, 2)
+ packed_tensor = torch.bitwise_and(packed_tensor[..., ::2], 15) | packed_tensor[..., 1::2] << 4
+ return packed_tensor
+
+
+def pack_int4(tensor: torch.Tensor) -> torch.Tensor:
+ if tensor.dtype != torch.int8:
+ msg = f"Invalid tensor dtype {tensor.type}. torch.int8 type is supported."
+ raise RuntimeError(msg)
+ tensor = tensor + 8
+ return pack_uint4(tensor.type(torch.uint8))
+
+
+class INT8AsymmetricWeightsDecompressor(torch.nn.Module):
+ def __init__(self, scale: torch.Tensor, zero_point: torch.Tensor, result_dtype: Optional[torch.dtype] = None):
+ super().__init__()
+ self.scale = scale
+ self.zero_point = self.pack_weight(zero_point)
+ self.result_dtype = result_dtype
+
+ @property
+ def quantization_mode(self):
+ return"asymmetric"
+
+ def pack_weight(self, weight: torch.Tensor) -> torch.Tensor:
+ if torch.is_floating_point(weight):
+ msg = f"Invalid weight dtype {weight.type}. Integer types are supported."
+ raise ValueError(msg)
+ if torch.any((weight < 0) | (weight > 255)):
+ msg = "Weight values are not in [0, 255]."
+ raise ValueError(msg)
+ return weight.type(torch.uint8)
+
+ def forward(self, x, *args):
+ result = decompress_asymmetric(x.weight, self.scale, self.zero_point)
+ x.weight = result.type(self.result_dtype)
+
+
+class INT8SymmetricWeightsDecompressor(torch.nn.Module):
+ def __init__(self, scale: torch.Tensor, result_dtype: Optional[torch.dtype] = None):
+ super().__init__()
+ self.scale = scale
+ self.result_dtype = result_dtype
+
+ @property
+ def quantization_mode(self):
+ return"symmetric"
+
+ def pack_weight(self, weight: torch.Tensor) -> torch.Tensor:
+ if torch.any((weight < -128) | (weight > 127)):
+ msg = "Weight values are not in [-128, 127]."
+ raise ValueError(msg)
+ return weight.type(torch.int8)
+
+ def forward(self, x, *args):
+ result = decompress_symmetric(x.weight, self.scale)
+ x.weight = result.type(self.result_dtype)
+
+
+class INT4AsymmetricWeightsDecompressor(torch.nn.Module):
+ def __init__(
+ self,
+ scale: torch.Tensor,
+ zero_point: torch.Tensor,
+ compressed_weight_shape: Tuple[int, ...],
+ result_shape: Optional[Tuple[int, ...]] = None,
+ result_dtype: Optional[torch.dtype] = None,
+ ):
+ super().__init__()
+ self.scale = scale
+
+ self.zero_point_shape = zero_point.shape
+ self.zero_point = self.pack_weight(zero_point)
+
+ self.compressed_weight_shape = compressed_weight_shape
+ self.result_shape = result_shape
+ self.result_dtype = result_dtype
+
+ @property
+ def quantization_mode(self):
+ return"asymmetric"
+
+ def pack_weight(self, weight: torch.Tensor) -> torch.Tensor:
+ if torch.any((weight < 0) | (weight > 15)):
+ msg = "Weight values are not in [0, 15]."
+ raise ValueError(msg)
+ return pack_uint4(weight.type(torch.uint8))
+
+ def forward(self, x, *args):
+ result = unpack_uint4(x.weight)
+ result = result.reshape(self.compressed_weight_shape)
+
+ zero_point = unpack_uint4(self.zero_point)
+ zero_point = zero_point.reshape(self.zero_point_shape)
+
+ result = decompress_asymmetric(result, self.scale, zero_point)
+ result = result.reshape(self.result_shape) if self.result_shape is not None else result
+ x.weight = result.type(self.result_dtype)
+
+
+class INT4SymmetricWeightsDecompressor(torch.nn.Module):
+ def __init__(
+ self,
+ scale: torch.Tensor,
+ compressed_weight_shape: Tuple[int, ...],
+ result_shape: Optional[Tuple[int, ...]] = None,
+ result_dtype: Optional[torch.dtype] = None,
+ ):
+ super().__init__()
+ self.scale = scale
+
+ self.compressed_weight_shape = compressed_weight_shape
+ self.result_shape = result_shape
+ self.result_dtype = result_dtype
+
+ @property
+ def quantization_mode(self):
+ return"symmetric"
+
+ def pack_weight(self, weight: torch.Tensor) -> torch.Tensor:
+ if torch.is_floating_point(weight):
+ msg = f"Invalid weight dtype {weight.type}. Integer types are supported."
+ raise ValueError(msg)
+ if torch.any((weight < -8) | (weight > 7)):
+ msg = "Tensor values are not in [-8, 7]."
+ raise ValueError(msg)
+ return pack_int4(weight.type(torch.int8))
+
+ def forward(self, x, *args):
+ result = unpack_int4(x.weight)
+ result = result.reshape(self.compressed_weight_shape)
+
+ result = decompress_symmetric(result, self.scale)
+ result = result.reshape(self.result_shape) if self.result_shape is not None else result
+ x.weight = result.type(self.result_dtype)
+
diff --git a/modules/shared.py b/modules/shared.py
index 5ba516c62..df0d5fcea 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -538,11 +538,12 @@ options_templates.update(options_section(('quantization', "Quantization Settings
"nncf_compress_sep": OptionInfo("NNCF: Neural Network Compression Framework
", "", gr.HTML),
"nncf_compress_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}),
"nncf_compress_mode": OptionInfo("post", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native}),
- "nncf_compress_weights_mode": OptionInfo("INT8", "Quantization type", gr.Dropdown, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'] if cmd_opts.use_openvino else ['INT8']}),
+ "nncf_compress_weights_mode": OptionInfo("INT8_SYM", "Quantization type", gr.Dropdown, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'] if cmd_opts.use_openvino else ['INT8', 'INT8_SYM', 'INT4', 'INT4_SYM']}),
"nncf_compress_weights_raito": OptionInfo(0, "Compress ratio", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}),
"nncf_compress_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 512, "step": 1, "visible": cmd_opts.use_openvino}),
"nncf_quantize": OptionInfo([], "OpenVINO enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "TE"], "visible": cmd_opts.use_openvino}),
"nncf_quantize_mode": OptionInfo("INT8", "OpenVINO mode", gr.Dropdown, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}),
+ "nncf_quantize_conv_layers": OptionInfo(True, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}),
"nncf_quantize_shuffle_weights": OptionInfo(False, "Shuffle weights", gr.Checkbox, {"visible": native}),
"layerwise_quantization_sep": OptionInfo("Layerwise Casting
", "", gr.HTML),