diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py
index 7a971c340..d2e59348c 100644
--- a/modules/control/units/controlnet.py
+++ b/modules/control/units/controlnet.py
@@ -286,13 +286,13 @@ class ControlNet():
return
if self.dtype is not None:
self.model.to(self.dtype)
- if "ControlNet" in opts.nncf_compress_weights:
+ if "ControlNet" in opts.sdnq_quantize_weights:
try:
- log.debug(f'Control {what} model NNCF Compress: id="{model_id}"')
- from modules.model_quant import nncf_compress_model
- self.model = nncf_compress_model(self.model)
+ log.debug(f'Control {what} model SDNQ Compress: id="{model_id}"')
+ from modules.model_quant import sdnq_quantize_model
+ self.model = sdnq_quantize_model(self.model)
except Exception as e:
- log.error(f'Control {what} model NNCF Compression failed: id="{model_id}" {e}')
+ log.error(f'Control {what} model SDNQ Compression failed: id="{model_id}" {e}')
elif "ControlNet" in opts.optimum_quanto_weights:
try:
log.debug(f'Control {what} model Optimum Quanto: id="{model_id}"')
diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py
index 7b00ee389..062413288 100644
--- a/modules/lora/lora_apply.py
+++ b/modules/lora/lora_apply.py
@@ -45,8 +45,8 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n
self.network_weights_backup = True
else:
self.network_weights_backup = weight.clone().to(devices.cpu)
- if hasattr(self, "nncf_decompressor"):
- self.nncf_decompressor_backup = self.nncf_decompressor.to(devices.cpu)
+ if hasattr(self, "sdnq_decompressor"):
+ self.sdnq_decompressor_backup = self.sdnq_decompressor.to(devices.cpu)
if bias_backup is None:
if getattr(self, 'bias', None) is not None:
@@ -79,10 +79,10 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.
continue
try:
t0 = time.time()
- if hasattr(self, "nncf_decompressor"):
+ if hasattr(self, "sdnq_decompressor"):
return_device = self.weight.data.device
self.weight.data = self.weight.data.to(devices.device)
- weight = self.nncf_decompressor.to(devices.device)(self, return_decompressed_only=True)
+ weight = self.sdnq_decompressor.to(devices.device)(self, return_decompressed_only=True)
self.weight.data = self.weight.data.to(return_device)
else:
weight = self.weight.to(devices.device) # must perform calc on gpu due to performance
@@ -139,23 +139,23 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G
# weight._quantize(devices.device) / weight.to(device=device)
except Exception as e:
shared.log.error(f'Network load: type=LoRA quant=bnb cls={self.__class__.__name__} type={self.quant_type} blocksize={self.blocksize} state={vars(self.quant_state)} weight={self.weight} bias={lora_weights} {e}')
- elif not bias and hasattr(self, "nncf_decompressor"):
+ elif not bias and hasattr(self, "sdnq_decompressor"):
num_bits = None
is_asym_mode = None
try:
- from modules.model_quant_nncf import nncf_compress_layer
- num_bits = self.nncf_decompressor.num_bits
- is_asym_mode = self.nncf_decompressor.quantization_mode == "asymmetric"
+ from modules.model_quant_sdnq import sdnq_quantize_layer
+ num_bits = self.sdnq_decompressor.num_bits
+ is_asym_mode = self.sdnq_decompressor.quantization_mode == "asymmetric"
self.weight = torch.nn.Parameter(model_weights.to(devices.device), requires_grad=False)
- dequant_weight = self.nncf_decompressor(self, return_decompressed_only=True)
+ dequant_weight = self.sdnq_decompressor(self, return_decompressed_only=True)
new_weight = dequant_weight.to(devices.device, dtype=torch.float32) + lora_weights.to(devices.device, dtype=torch.float32)
self.weight = torch.nn.Parameter(new_weight, requires_grad=False)
- self.nncf_decompressor = None
- self = nncf_compress_layer(self, num_bits, is_asym_mode, torch_dtype=devices.dtype, quant_conv=shared.opts.nncf_quantize_conv_layers, group_size=shared.opts.nncf_compress_weights_group_size, use_int8_matmul=shared.opts.nncf_decompress_int8_matmul)
+ self.sdnq_decompressor = None
+ self = sdnq_quantize_layer(self, num_bits, is_asym_mode, torch_dtype=devices.dtype, quant_conv=shared.opts.sdnq_quantize_conv_layers, group_size=shared.opts.sdnq_quantize_weights_group_size, use_int8_matmul=shared.opts.sdnq_decompress_int8_matmul)
self = self.to(device)
del dequant_weight
except Exception as e:
- shared.log.error(f'Network load: type=LoRA quant=nncf cls={self.__class__.__name__} bits={num_bits} is_asym_mode={is_asym_mode} weight={self.weight} lora_weights={lora_weights} {e}')
+ shared.log.error(f'Network load: type=LoRA quant=sdnq cls={self.__class__.__name__} bits={num_bits} is_asym_mode={is_asym_mode} weight={self.weight} lora_weights={lora_weights} {e}')
else:
try:
new_weight = model_weights.to(devices.device) + lora_weights.to(devices.device)
@@ -217,8 +217,8 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn
network_add_weights(self, model_weights=weights_backup, lora_weights=updown, deactivate=deactivate, device=device, bias=False)
else:
self.weight = torch.nn.Parameter(weights_backup.to(device), requires_grad=False)
- if hasattr(self, "nncf_decompressor_backup"):
- self.nncf_decompressor = self.nncf_decompressor_backup.to(device)
+ if hasattr(self, "sdnq_decompressor_backup"):
+ self.sdnq_decompressor = self.sdnq_decompressor_backup.to(device)
if bias_backup is not None:
self.bias = None
diff --git a/modules/model_quant.py b/modules/model_quant.py
index 3b453691d..c0cd6fb9f 100644
--- a/modules/model_quant.py
+++ b/modules/model_quant.py
@@ -103,35 +103,35 @@ def create_quanto_config(kwargs = None, allow_quanto: bool = True, module: str =
return kwargs
-def create_nncf_config(kwargs = None, allow_nncf: bool = True, module: str = 'Model'):
+def create_sdnq_config(kwargs = None, allow_sdnq: bool = True, module: str = 'Model', weights_dtype: str = None):
from modules import shared
- if len(shared.opts.nncf_compress_weights) > 0 and (shared.opts.nncf_compress_mode == 'pre') and allow_nncf:
- if 'Model' in shared.opts.nncf_compress_weights or (module is not None and module in shared.opts.nncf_compress_weights) or module == 'any':
- from modules.model_quant_nncf import NNCFQuantizer, NNCFConfig
- diffusers.quantizers.auto.AUTO_QUANTIZER_MAPPING["nncf"] = NNCFQuantizer
- transformers.quantizers.auto.AUTO_QUANTIZER_MAPPING["nncf"] = NNCFQuantizer
- diffusers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["nncf"] = NNCFConfig
- transformers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["nncf"] = NNCFConfig
+ if len(shared.opts.sdnq_quantize_weights) > 0 and (shared.opts.sdnq_quantize_mode == 'pre') and allow_sdnq:
+ if 'Model' in shared.opts.sdnq_quantize_weights or (module is not None and module in shared.opts.sdnq_quantize_weights) or module == 'any':
+ from modules.model_quant_sdnq import SDNQQuantizer, SDNQConfig
+ diffusers.quantizers.auto.AUTO_QUANTIZER_MAPPING["sdnq"] = SDNQQuantizer
+ transformers.quantizers.auto.AUTO_QUANTIZER_MAPPING["sdnq"] = SDNQQuantizer
+ diffusers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["sdnq"] = SDNQConfig
+ transformers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["sdnq"] = SDNQConfig
- nncf_config = NNCFConfig(
- weights_dtype=shared.opts.nncf_compress_weights_mode.lower(),
- group_size=shared.opts.nncf_compress_weights_group_size,
- use_int8_matmul=shared.opts.nncf_decompress_int8_matmul,
+ sdnq_config = SDNQConfig(
+ weights_dtype=weights_dtype if weights_dtype is not None else shared.opts.sdnq_quantize_weights_mode,
+ group_size=shared.opts.sdnq_quantize_weights_group_size,
+ use_int8_matmul=shared.opts.sdnq_decompress_int8_matmul,
)
- log.debug(f'Quantization: module="{module}" type=nncf dtype={shared.opts.nncf_compress_weights_mode}')
+ log.debug(f'Quantization: module="{module}" type=sdnq dtype={shared.opts.sdnq_quantize_weights_mode}')
if kwargs is None:
- return nncf_config
+ return sdnq_config
else:
- kwargs['quantization_config'] = nncf_config
+ kwargs['quantization_config'] = sdnq_config
return kwargs
return kwargs
def check_quant(module: str = ''):
from modules import shared
- if 'Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization or 'Model' in shared.opts.quanto_quantization or 'Model' in shared.opts.nncf_compress_weights:
+ if 'Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization or 'Model' in shared.opts.quanto_quantization or 'Model' in shared.opts.sdnq_quantize_weights:
return True
- if module in shared.opts.bnb_quantization or module in shared.opts.torchao_quantization or module in shared.opts.quanto_quantization or module in shared.opts.nncf_compress_weights:
+ if module in shared.opts.bnb_quantization or module in shared.opts.torchao_quantization or module in shared.opts.quanto_quantization or module in shared.opts.sdnq_quantize_weights:
return True
return False
@@ -165,10 +165,10 @@ def create_config(kwargs = None, allow: bool = True, module: str = 'Model'):
if debug:
log.trace(f'Quantization: type=quanto config={kwargs.get("quantization_config", None)}')
return kwargs
- kwargs = create_nncf_config(kwargs, allow_nncf=allow, module=module)
+ kwargs = create_sdnq_config(kwargs, allow_sdnq=allow, module=module)
if kwargs is not None and 'quantization_config' in kwargs:
if debug:
- log.trace(f'Quantization: type=nncf config={kwargs.get("quantization_config", None)}')
+ log.trace(f'Quantization: type=sdnq config={kwargs.get("quantization_config", None)}')
return kwargs
return kwargs
@@ -298,18 +298,18 @@ def apply_layerwise(sd_model, quiet:bool=False):
log.error(f'Quantization: type=layerwise {e}')
-def nncf_compress_model(model, op=None, sd_model=None, do_gc=True):
+def sdnq_quantize_model(model, op=None, sd_model=None, do_gc=True):
global quant_last_model_name, quant_last_model_device # pylint: disable=global-statement
from modules import devices, shared
- from modules.model_quant_nncf import apply_nncf_to_module
+ from modules.model_quant_sdnq import apply_sdnq_to_module
model.eval()
if model.__class__.__name__ in {"T5EncoderModel", "UMT5EncoderModel"}:
import torch
- from modules.model_quant_nncf import NNCF_T5DenseGatedActDense # T5DenseGatedActDense uses fp32
+ from modules.model_quant_sdnq import SDNQ_T5DenseGatedActDense # T5DenseGatedActDense uses fp32
for i in range(len(model.encoder.block)):
- model.encoder.block[i].layer[1].DenseReluDense = NNCF_T5DenseGatedActDense(
+ model.encoder.block[i].layer[1].DenseReluDense = SDNQ_T5DenseGatedActDense(
model.encoder.block[i].layer[1].DenseReluDense,
dtype=torch.float32 if devices.dtype != torch.bfloat16 else torch.bfloat16
)
@@ -318,15 +318,15 @@ def nncf_compress_model(model, op=None, sd_model=None, do_gc=True):
if hasattr(model, "get_input_embeddings"):
backup_embeddings = copy.deepcopy(model.get_input_embeddings())
- num_bits = 8 if shared.opts.nncf_compress_weights_mode in {"INT8", "INT8_SYM", "INT8_ASYM"} else 4
- is_asym_mode = shared.opts.nncf_compress_weights_mode in {"INT8", "INT4", "INT8_ASYM", "INT4_ASYM"}
- model = apply_nncf_to_module(model, num_bits, is_asym_mode, quant_conv=shared.opts.nncf_quantize_conv_layers)
- model.quantization_method = 'NNCF'
+ num_bits = 8 if shared.opts.sdnq_quantize_weights_mode in {"int8", "uint8"} else 4
+ is_asym_mode = shared.opts.sdnq_quantize_weights_mode in {"uint8", "uint4"}
+ model = apply_sdnq_to_module(model, num_bits, is_asym_mode, quant_conv=shared.opts.sdnq_quantize_conv_layers)
+ model.quantization_method = 'SDNQ'
if hasattr(model, "set_input_embeddings") and backup_embeddings is not None:
model.set_input_embeddings(backup_embeddings)
- if op is not None and shared.opts.nncf_quantize_shuffle_weights:
+ if op is not None and shared.opts.sdnq_quantize_shuffle_weights:
if quant_last_model_name is not None:
if "." in quant_last_model_name:
last_model_names = quant_last_model_name.split(".")
@@ -347,14 +347,14 @@ def nncf_compress_model(model, op=None, sd_model=None, do_gc=True):
return model
-def nncf_compress_weights(sd_model):
+def sdnq_quantize_weights(sd_model):
try:
t0 = time.time()
from modules import shared, devices, sd_models
- log.info(f"Quantization: type=NNCF modules={shared.opts.nncf_compress_weights}")
+ log.info(f"Quantization: type=SDNQ modules={shared.opts.sdnq_quantize_weights}")
global quant_last_model_name, quant_last_model_device # pylint: disable=global-statement
- sd_model = sd_models.apply_function_to_model(sd_model, nncf_compress_model, shared.opts.nncf_compress_weights, op="nncf")
+ sd_model = sd_models.apply_function_to_model(sd_model, sdnq_quantize_model, shared.opts.sdnq_quantize_weights, op="sdnq")
if quant_last_model_name is not None:
if "." in quant_last_model_name:
last_model_names = quant_last_model_name.split(".")
@@ -366,9 +366,9 @@ def nncf_compress_weights(sd_model):
quant_last_model_device = None
t1 = time.time()
- log.info(f"Quantization: type=NNCF time={t1-t0:.2f}")
+ log.info(f"Quantization: type=SDNQ time={t1-t0:.2f}")
except Exception as e:
- log.warning(f"Quantization: type=NNCF {e}")
+ log.warning(f"Quantization: type=SDNQ {e}")
return sd_model
@@ -529,8 +529,8 @@ def get_dit_args(load_config:dict={}, module:str=None, device_map:bool=False, al
def do_post_load_quant(sd_model):
from modules import shared
- if shared.opts.nncf_compress_weights and shared.opts.nncf_compress_mode == 'post' and not (shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx"):
- sd_model = nncf_compress_weights(sd_model)
+ if shared.opts.sdnq_quantize_weights and shared.opts.sdnq_quantize_mode == 'post':
+ sd_model = sdnq_quantize_weights(sd_model)
if shared.opts.optimum_quanto_weights:
sd_model = optimum_quanto_weights(sd_model)
if shared.opts.torchao_quantization and shared.opts.torchao_quantization_mode == 'post':
diff --git a/modules/model_quant_nncf.py b/modules/model_quant_sdnq.py
similarity index 92%
rename from modules/model_quant_nncf.py
rename to modules/model_quant_sdnq.py
index 8efe64cbb..9ed00916b 100644
--- a/modules/model_quant_nncf.py
+++ b/modules/model_quant_sdnq.py
@@ -19,17 +19,10 @@ torch_dtype_dict = {
"int4": CustomDtype.INT4,
"uint4": CustomDtype.INT4,
}
-weights_dtype_dict = {
- "int8_asym": "uint8",
- "int8_sym": "int8",
- "int4_asym": "uint4",
- "int4_sym": "int4",
- "int8": "uint8",
- "int4": "uint4",
-}
-linear_types = ["NNCFLinear", "Linear"]
-conv_types = ["NNCFConv1d", "NNCFConv2d", "NNCFConv3d", "Conv1d", "Conv2d", "Conv3d"]
-conv_transpose_types = ["NNCFConvTranspose1d", "NNCFConvTranspose2d", "NNCFConvTranspose3d", "ConvTranspose1d", "ConvTranspose2d", "ConvTranspose3d"]
+
+linear_types = ["Linear"]
+conv_types = ["Conv1d", "Conv2d", "Conv3d"]
+conv_transpose_types = ["ConvTranspose1d", "ConvTranspose2d", "ConvTranspose3d"]
allowed_types = []
allowed_types.extend(linear_types)
allowed_types.extend(conv_types)
@@ -37,10 +30,10 @@ allowed_types.extend(conv_transpose_types)
class QuantizationMethod(str, Enum):
- NNCF = "nncf"
+ SDNQ = "sdnq"
-def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_conv=False, group_size=0, use_int8_matmul=False, param_name=None): # pylint: disable=unused-argument
+def sdnq_quantize_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_conv=False, group_size=0, use_int8_matmul=False, param_name=None): # pylint: disable=unused-argument
layer_class_name = layer.__class__.__name__
if layer_class_name in allowed_types:
is_conv_type = False
@@ -106,7 +99,7 @@ def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c
zero_point = None
compressed_weight = quantize_int(layer.weight, scale, zero_point, is_asym_mode, num_bits)
- if not shared.opts.nncf_decompress_fp32:
+ if not shared.opts.sdnq_decompress_fp32:
scale = scale.to(torch_dtype)
if zero_point is not None:
zero_point = zero_point.to(torch_dtype)
@@ -152,7 +145,7 @@ def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c
layer.weight.requires_grad = False
layer.weight.data = compressed_weight
- layer.nncf_decompressor = decompressor
+ layer.sdnq_decompressor = decompressor
if is_linear_type:
if use_int8_matmul:
@@ -172,38 +165,29 @@ def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c
return layer
-def apply_nncf_to_module(model, num_bits, is_asym_mode, quant_conv=False):
+def apply_sdnq_to_module(model, num_bits, is_asym_mode, quant_conv=False):
has_children = list(model.children())
if not has_children:
return model
for param_name, module in model.named_children():
if hasattr(module, "weight") and module.weight is not None:
- module = nncf_compress_layer(
+ module = sdnq_quantize_layer(
module,
num_bits,
is_asym_mode,
torch_dtype=devices.dtype,
quant_conv=quant_conv,
- group_size=shared.opts.nncf_compress_weights_group_size,
- use_int8_matmul=shared.opts.nncf_decompress_int8_matmul,
+ group_size=shared.opts.sdnq_quantize_weights_group_size,
+ use_int8_matmul=shared.opts.sdnq_decompress_int8_matmul,
param_name=param_name,
)
- module = apply_nncf_to_module(module, num_bits, is_asym_mode, quant_conv=quant_conv)
+ module = apply_sdnq_to_module(module, num_bits, is_asym_mode, quant_conv=quant_conv)
return model
-def nncf_send_to_device(model, device):
- for child in model.children():
- if "WeightsDecompressor" in child.__class__.__name__:
- child.scale = child.scale.to(device)
- if hasattr(child, "zero_point"):
- child.zero_point = child.zero_point.to(device)
- nncf_send_to_device(child, device)
-
-
-class NNCFQuantizer(DiffusersQuantizer):
+class SDNQQuantizer(DiffusersQuantizer):
r"""
- Diffusers Quantizer for NNCF
+ Diffusers Quantizer for SDNQ
"""
requires_parameters_quantization = True
@@ -247,7 +231,7 @@ class NNCFQuantizer(DiffusersQuantizer):
split_param_name = param_name.split(".")
if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert):
- layer = nncf_compress_layer(
+ layer = sdnq_quantize_layer(
layer,
self.quantization_config.num_bits,
self.quantization_config.is_asym_mode,
@@ -321,14 +305,14 @@ class NNCFQuantizer(DiffusersQuantizer):
@dataclass
-class NNCFConfig(QuantizationConfigMixin):
+class SDNQConfig(QuantizationConfigMixin):
"""
This is a wrapper class about all possible attributes and features that you can play with a model that has been
- loaded using `nncf`.
+ loaded using `sdnq`.
Args:
weights_dtype (`str`, *optional*, defaults to `"int8"`):
- The target dtype for the weights after quantization. Supported values are ("int8", "int8_sym", "int4", "int4_sym")
+ The target dtype for the weights after quantization. Supported values are ("int8", "uint8", "int4", "uint4")
modules_to_not_convert (`list`, *optional*, default to `None`):
The list of modules to not quantize, useful for quantizing models that explicitly require to have some
modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers).
@@ -336,23 +320,21 @@ class NNCFConfig(QuantizationConfigMixin):
def __init__( # pylint: disable=super-init-not-called
self,
- weights_dtype: str = "int8_sym",
+ weights_dtype: str = "int8",
group_size: int = 0,
use_int8_matmul: bool = False,
modules_to_not_convert: Optional[List[str]] = None,
**kwargs, # pylint: disable=unused-argument
):
- self.quant_method = QuantizationMethod.NNCF
- self.weights_dtype = weights_dtype_dict[weights_dtype.lower()]
+ self.weights_dtype = weights_dtype
+ self.quant_method = QuantizationMethod.SDNQ
self.group_size = group_size
self.use_int8_matmul = use_int8_matmul
self.modules_to_not_convert = modules_to_not_convert
-
- self.post_init()
-
self.num_bits = 8 if self.weights_dtype in {"int8", "uint8"} else 4
self.is_asym_mode = self.weights_dtype in {"uint8", "uint4"}
self.is_integer = True
+ self.post_init()
def post_init(self):
r"""
@@ -363,7 +345,7 @@ class NNCFConfig(QuantizationConfigMixin):
raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights_dtype}")
-class NNCF_T5DenseGatedActDense(torch.nn.Module): # forward can't find what self is without creating a class
+class SDNQ_T5DenseGatedActDense(torch.nn.Module): # forward can't find what self is without creating a class
def __init__(self, T5DenseGatedActDense, dtype):
super().__init__()
self.wi_0 = T5DenseGatedActDense.wi_0
@@ -496,31 +478,31 @@ def int8_matmul(
def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor:
if torch.numel(input[0]) / input[0].shape[-1] < 32:
- return torch.nn.functional.linear(input, self.nncf_decompressor(self, return_decompressed_only=True, skip_int8_matmul=True), self.bias)
- return int8_matmul(input, self.weight, self.bias, self.nncf_decompressor.scale, getattr(self.nncf_decompressor, "compressed_weight_shape", None))
+ return torch.nn.functional.linear(input, self.sdnq_decompressor(self, return_decompressed_only=True, skip_int8_matmul=True), self.bias)
+ return int8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale, getattr(self.sdnq_decompressor, "compressed_weight_shape", None))
def quantized_linear_forward(self, input: torch.FloatTensor) -> torch.FloatTensor:
- return torch.nn.functional.linear(input, self.nncf_decompressor(self, return_decompressed_only=True), self.bias)
+ return torch.nn.functional.linear(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias)
def quantized_conv_forward(self, input) -> torch.FloatTensor:
- return self._conv_forward(input, self.nncf_decompressor(self, return_decompressed_only=True), self.bias)
+ return self._conv_forward(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias)
def quantized_conv_transpose_1d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor:
output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 1, self.dilation)
- return torch.nn.functional.conv_transpose1d(input, self.nncf_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation)
+ return torch.nn.functional.conv_transpose1d(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation)
def quantized_conv_transpose_2d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor:
output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 2, self.dilation)
- return torch.nn.functional.conv_transpose2d(input, self.nncf_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation)
+ return torch.nn.functional.conv_transpose2d(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation)
def quantized_conv_transpose_3d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor:
output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 3, self.dilation)
- return torch.nn.functional.conv_transpose3d(input, self.nncf_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation)
+ return torch.nn.functional.conv_transpose3d(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation)
class INT8AsymmetricWeightsDecompressor(torch.nn.Module):
@@ -647,7 +629,7 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module):
x.weight = result
-if shared.opts.nncf_decompress_compile:
+if shared.opts.sdnq_decompress_compile:
try:
torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit) # pylint: disable=protected-access
decompress_asymmetric_compiled = torch.compile(decompress_asymmetric, fullgraph=True)
@@ -662,7 +644,7 @@ if shared.opts.nncf_decompress_compile:
quantize_int8_matmul_input_compiled = torch.compile(quantize_int8_matmul_input, fullgraph=True)
unpack_int4_compiled = torch.compile(unpack_int4, fullgraph=True)
except Exception as e:
- shared.log.warning(f"Quantization: type=nncf Decompress using torch.compile is not available: {e}")
+ shared.log.warning(f"Quantization: type=sdnq Decompress using torch.compile is not available: {e}")
decompress_asymmetric_compiled = decompress_asymmetric
decompress_symmetric_compiled = decompress_symmetric
decompress_int4_asymmetric_compiled = decompress_int4_asymmetric
diff --git a/modules/model_te.py b/modules/model_te.py
index d853eaf99..6472721ee 100644
--- a/modules/model_te.py
+++ b/modules/model_te.py
@@ -70,8 +70,13 @@ def load_t5(name=None, cache_dir=None):
t5 = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder='text_encoder_3', quantization_config=quantization_config, cache_dir=cache_dir, torch_dtype=devices.dtype)
elif 'int8' in name.lower():
- from modules.model_quant import create_nncf_config
- quantization_config = create_nncf_config(kwargs=None, allow_nncf=True, module="any")
+ from modules.model_quant import create_sdnq_config
+ quantization_config = create_sdnq_config(kwargs=None, allow_sdnq=True, module='any', weights_dtype='int8')
+ t5 = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder='text_encoder_3', quantization_config=quantization_config, cache_dir=cache_dir, torch_dtype=devices.dtype)
+
+ elif 'uint4' in name.lower():
+ from modules.model_quant import create_sdnq_config
+ quantization_config = create_sdnq_config(kwargs=None, allow_sdnq=True, module='any', weights_dtype='uint4')
t5 = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder='text_encoder_3', quantization_config=quantization_config, cache_dir=cache_dir, torch_dtype=devices.dtype)
elif 'qint4' in name.lower():
diff --git a/modules/sd_models_utils.py b/modules/sd_models_utils.py
index 2f10ad074..0e1b72f4c 100644
--- a/modules/sd_models_utils.py
+++ b/modules/sd_models_utils.py
@@ -161,10 +161,10 @@ def apply_function_to_model(sd_model, function, options, op=None):
sd_model.decoder = None
sd_model.decoder = sd_model.decoder_pipe.decoder = function(sd_model.decoder_pipe.decoder, op="decoder_pipe.decoder", sd_model=sd_model)
if hasattr(sd_model, 'prior_pipe') and hasattr(sd_model.prior_pipe, 'prior'):
- if op == "nncf" and "StableCascade" in sd_model.__class__.__name__: # fixes dtype errors
+ if op == "sdnq" and "StableCascade" in sd_model.__class__.__name__: # fixes dtype errors
backup_clip_txt_pooled_mapper = copy.deepcopy(sd_model.prior_pipe.prior.clip_txt_pooled_mapper)
sd_model.prior_pipe.prior = function(sd_model.prior_pipe.prior, op="prior_pipe.prior", sd_model=sd_model)
- if op == "nncf" and "StableCascade" in sd_model.__class__.__name__:
+ if op == "sdnq" and "StableCascade" in sd_model.__class__.__name__:
sd_model.prior_pipe.prior.clip_txt_pooled_mapper = backup_clip_txt_pooled_mapper
if "TE" in options:
if hasattr(sd_model, 'text_encoder') and hasattr(sd_model.text_encoder, 'config'):
diff --git a/modules/shared.py b/modules/shared.py
index cba39b41d..91def4d6e 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -527,19 +527,16 @@ options_templates.update(options_section(('quantization', "Quantization Settings
"bnb_quantization_type": OptionInfo("nf4", "Quantization type", gr.Dropdown, {"choices": ['nf4', 'fp8', 'fp4'], "visible": native}),
"bnb_quantization_storage": OptionInfo("uint8", "Backend storage", gr.Dropdown, {"choices": ["float16", "float32", "int8", "uint8", "float64", "bfloat16"], "visible": native}),
- "nncf_compress_sep": OptionInfo("NNCF: Neural Network Compression Framework
", "", gr.HTML),
- "nncf_compress_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}),
- "nncf_compress_mode": OptionInfo("post", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native and not cmd_opts.use_openvino}),
- "nncf_compress_weights_mode": OptionInfo("INT8_SYM", "Quantization type", gr.Dropdown, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'] if cmd_opts.use_openvino else ['INT8', 'INT8_SYM', 'INT4', 'INT4_SYM']}),
- "nncf_compress_weights_raito": OptionInfo(0, "Compress ratio", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}),
- "nncf_compress_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}),
- "nncf_quantize": OptionInfo([], "OpenVINO enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "TE"], "visible": cmd_opts.use_openvino}),
- "nncf_quantize_mode": OptionInfo("INT8", "OpenVINO activations mode", gr.Dropdown, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}),
- "nncf_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native and not cmd_opts.use_openvino}),
- "nncf_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native and not cmd_opts.use_openvino}),
- "nncf_decompress_compile": OptionInfo(devices.has_triton(), "Decompress using torch.compile", gr.Checkbox, {"visible": native and not cmd_opts.use_openvino}),
- "nncf_decompress_int8_matmul": OptionInfo(False, "Use direct INT8 MatMul", gr.Checkbox, {"visible": native and not cmd_opts.use_openvino}),
- "nncf_quantize_shuffle_weights": OptionInfo(False, "Shuffle weights in post mode", gr.Checkbox, {"visible": native and not cmd_opts.use_openvino}),
+ "sdnq_quantize_sep": OptionInfo("SDNQ: SDNext Quantization
", "", gr.HTML),
+ "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}),
+ "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native}),
+ "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ['int8', 'uint8', 'int4', 'uint4'], "visible": native}),
+ "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}),
+ "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}),
+ "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}),
+ "sdnq_decompress_compile": OptionInfo(devices.has_triton(), "Decompress using torch.compile", gr.Checkbox, {"visible": native}),
+ "sdnq_decompress_int8_matmul": OptionInfo(False, "Use direct INT8 MatMul", gr.Checkbox, {"visible": native}),
+ "sdnq_quantize_shuffle_weights": OptionInfo(False, "Shuffle weights in post mode", gr.Checkbox, {"visible": native}),
"quanto_quantization_sep": OptionInfo("Optimum Quanto
", "", gr.HTML),
"quanto_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}),
@@ -565,6 +562,14 @@ options_templates.update(options_section(('quantization', "Quantization Settings
"nunchaku_quantization": OptionInfo([], "SVDQuant enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}),
"nunchaku_attention": OptionInfo(False, "Nunchaku attention", gr.Checkbox, {"visible": native}),
"nunchaku_offload": OptionInfo(False, "Nunchaku offloading", gr.Checkbox, {"visible": native}),
+
+ "nncf_compress_sep": OptionInfo("NNCF: Neural Network Compression Framework
", "", gr.HTML, {"visible": cmd_opts.use_openvino}),
+ "nncf_compress_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": cmd_opts.use_openvino}),
+ "nncf_compress_weights_mode": OptionInfo("INT8_SYM", "Quantization type", gr.Dropdown, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'], "visible": cmd_opts.use_openvino}),
+ "nncf_compress_weights_raito": OptionInfo(0, "Compress ratio", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}),
+ "nncf_compress_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": cmd_opts.use_openvino}),
+ "nncf_quantize": OptionInfo([], "Static Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "TE"], "visible": cmd_opts.use_openvino}),
+ "nncf_quantize_mode": OptionInfo("INT8", "OpenVINO activations mode", gr.Dropdown, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}),
}))
options_templates.update(options_section(('advanced', "Pipeline Modifiers"), {