From a009e17d2b3b8eea134fb6d05840a36a7d0770e8 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 17 May 2025 19:46:47 +0300 Subject: [PATCH 01/94] NNCF use per token input quantization with int8 matmul --- modules/model_quant_nncf.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/model_quant_nncf.py b/modules/model_quant_nncf.py index 9e8818330..3d8fb4624 100644 --- a/modules/model_quant_nncf.py +++ b/modules/model_quant_nncf.py @@ -461,9 +461,11 @@ def unpack_int4(packed_tensor: torch.Tensor, shape: torch.Size, dtype: Optional[ def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.ByteTensor, torch.FloatTensor]: - input_scale = torch.div(input.abs().max(), 127) - input = torch.div(input, input_scale).round_().clamp_(-128, 127).to(torch.int8).flatten(0,-2) - scale = torch.mul(input_scale, scale) + input_scale = torch.div(input.abs().max(dim=-1).values, 127).unsqueeze(-1) + input = torch.div(input, input_scale).round_().clamp_(-128, 127).to(torch.int8).flatten(0,-2).contiguous() + scale = torch.mul(input_scale, scale).flatten(0,-2).contiguous() + if scale.dtype == torch.float16: # fp16 will overflow + scale = scale.to(dtype=torch.float32) return input, scale @@ -485,7 +487,7 @@ def int8_matmul( class linear_forward_int8_matmul(): def __func__(self, input) -> torch.FloatTensor: if self.pre_ops["0"].skip_int8_matmul: - return torch.nn.Linear.forward(self, input) + return torch.nn.functional.linear(input, self.weight, self.bias) result = int8_matmul(input, self.weight, self.pre_ops["0"].scale, getattr(self.pre_ops["0"], "compressed_weight_shape", None)) if self.bias is not None: result.add_(self.bias) From d0e6f01286ec8468c997a88cf7646c4edaa2f3c0 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 18 May 2025 04:39:48 +0300 Subject: [PATCH 02/94] IPEX remove GradScaler and use torch.amp instead --- modules/intel/ipex/__init__.py | 26 +---- modules/intel/ipex/gradscaler.py | 183 ------------------------------- modules/intel/ipex/hijacks.py | 56 +++++++++- 3 files changed, 55 insertions(+), 210 deletions(-) delete mode 100644 modules/intel/ipex/gradscaler.py diff --git a/modules/intel/ipex/__init__.py b/modules/intel/ipex/__init__.py index 2d82830c1..b0fcb655e 100644 --- a/modules/intel/ipex/__init__.py +++ b/modules/intel/ipex/__init__.py @@ -1,6 +1,5 @@ import os import sys -import contextlib import torch try: import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import @@ -21,7 +20,7 @@ def ipex_init(): # pylint: disable=too-many-statements try: # force xpu device on torch compile and triton # import inductor utils to get around lazy import - from torch._inductor import utils as torch_inductor_utils # pylint: disable=import-error, unused-import + from torch._inductor import utils as torch_inductor_utils # pylint: disable=import-error, unused-import # noqa: F401 torch._inductor.utils.GPU_TYPES = ["xpu"] torch._inductor.utils.get_gpu_type = lambda *args, **kwargs: "xpu" from triton import backends as triton_backends # pylint: disable=import-error @@ -161,29 +160,6 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.seed_all = torch.xpu.seed_all torch.cuda.initial_seed = torch.xpu.initial_seed - # AMP: - if has_ipex: - torch.xpu.amp.custom_fwd = torch.cuda.amp.custom_fwd - torch.xpu.amp.custom_bwd = torch.cuda.amp.custom_bwd - torch.cuda.amp = torch.xpu.amp - if torch_version < 2.3: - torch.is_autocast_enabled = torch.xpu.is_autocast_xpu_enabled - torch.get_autocast_gpu_dtype = torch.xpu.get_autocast_xpu_dtype - - if not hasattr(torch.cuda.amp, "common"): - torch.cuda.amp.common = contextlib.nullcontext() - torch.cuda.amp.common.amp_definitely_not_available = lambda: False - - try: - torch.cuda.amp.GradScaler = torch.xpu.amp.GradScaler - except Exception: # pylint: disable=broad-exception-caught - try: - from .gradscaler import gradscaler_init # pylint: disable=import-outside-toplevel, import-error - gradscaler_init() - torch.cuda.amp.GradScaler = torch.xpu.amp.GradScaler - except Exception: # pylint: disable=broad-exception-caught - torch.cuda.amp.GradScaler = ipex.cpu.autocast._grad_scaler.GradScaler - # C if torch_version < 2.3: torch._C._cuda_getCurrentRawStream = ipex._C._getCurrentRawStream diff --git a/modules/intel/ipex/gradscaler.py b/modules/intel/ipex/gradscaler.py deleted file mode 100644 index 0a8610095..000000000 --- a/modules/intel/ipex/gradscaler.py +++ /dev/null @@ -1,183 +0,0 @@ -from collections import defaultdict -import torch -import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import -import intel_extension_for_pytorch._C as core # pylint: disable=import-error, unused-import - -# pylint: disable=protected-access, missing-function-docstring, line-too-long - -device_supports_fp64 = torch.xpu.has_fp64_dtype() if hasattr(torch.xpu, "has_fp64_dtype") else torch.xpu.get_device_properties("xpu").has_fp64 -OptState = ipex.cpu.autocast._grad_scaler.OptState -_MultiDeviceReplicator = ipex.cpu.autocast._grad_scaler._MultiDeviceReplicator -_refresh_per_optimizer_state = ipex.cpu.autocast._grad_scaler._refresh_per_optimizer_state - -def _unscale_grads_(self, optimizer, inv_scale, found_inf, allow_fp16): # pylint: disable=unused-argument - per_device_inv_scale = _MultiDeviceReplicator(inv_scale) - per_device_found_inf = _MultiDeviceReplicator(found_inf) - - # To set up _amp_foreach_non_finite_check_and_unscale_, split grads by device and dtype. - # There could be hundreds of grads, so we'd like to iterate through them just once. - # However, we don't know their devices or dtypes in advance. - - # https://stackoverflow.com/questions/5029934/defaultdict-of-defaultdict - # Google says mypy struggles with defaultdicts type annotations. - per_device_and_dtype_grads = defaultdict(lambda: defaultdict(list)) # type: ignore[var-annotated] - # sync grad to master weight - if hasattr(optimizer, "sync_grad"): - optimizer.sync_grad() - with torch.no_grad(): - for group in optimizer.param_groups: - for param in group["params"]: - if param.grad is None: - continue - if (not allow_fp16) and param.grad.dtype == torch.float16: - raise ValueError("Attempting to unscale FP16 gradients.") - if param.grad.is_sparse: - # is_coalesced() == False means the sparse grad has values with duplicate indices. - # coalesce() deduplicates indices and adds all values that have the same index. - # For scaled fp16 values, there's a good chance coalescing will cause overflow, - # so we should check the coalesced _values(). - if param.grad.dtype is torch.float16: - param.grad = param.grad.coalesce() - to_unscale = param.grad._values() - else: - to_unscale = param.grad - - # -: is there a way to split by device and dtype without appending in the inner loop? - to_unscale = to_unscale.to("cpu") - per_device_and_dtype_grads[to_unscale.device][ - to_unscale.dtype - ].append(to_unscale) - - for _, per_dtype_grads in per_device_and_dtype_grads.items(): - for grads in per_dtype_grads.values(): - core._amp_foreach_non_finite_check_and_unscale_( - grads, - per_device_found_inf.get("cpu"), - per_device_inv_scale.get("cpu"), - ) - - return per_device_found_inf._per_device_tensors - -def unscale_(self, optimizer): - """ - Divides ("unscales") the optimizer's gradient tensors by the scale factor. - :meth:`unscale_` is optional, serving cases where you need to - :ref:`modify or inspect gradients` - between the backward pass(es) and :meth:`step`. - If :meth:`unscale_` is not called explicitly, gradients will be unscaled automatically during :meth:`step`. - Simple example, using :meth:`unscale_` to enable clipping of unscaled gradients:: - ... - scaler.scale(loss).backward() - scaler.unscale_(optimizer) - torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm) - scaler.step(optimizer) - scaler.update() - Args: - optimizer (torch.optim.Optimizer): Optimizer that owns the gradients to be unscaled. - .. warning:: - :meth:`unscale_` should only be called once per optimizer per :meth:`step` call, - and only after all gradients for that optimizer's assigned parameters have been accumulated. - Calling :meth:`unscale_` twice for a given optimizer between each :meth:`step` triggers a RuntimeError. - .. warning:: - :meth:`unscale_` may unscale sparse gradients out of place, replacing the ``.grad`` attribute. - """ - if not self._enabled: - return - - self._check_scale_growth_tracker("unscale_") - - optimizer_state = self._per_optimizer_states[id(optimizer)] - - if optimizer_state["stage"] is OptState.UNSCALED: # pylint: disable=no-else-raise - raise RuntimeError( - "unscale_() has already been called on this optimizer since the last update()." - ) - elif optimizer_state["stage"] is OptState.STEPPED: - raise RuntimeError("unscale_() is being called after step().") - - # FP32 division can be imprecise for certain compile options, so we carry out the reciprocal in FP64. - assert self._scale is not None - if device_supports_fp64: - inv_scale = self._scale.double().reciprocal().float() - else: - inv_scale = self._scale.to("cpu").double().reciprocal().float().to(self._scale.device) - found_inf = torch.full( - (1,), 0.0, dtype=torch.float32, device=self._scale.device - ) - - optimizer_state["found_inf_per_device"] = self._unscale_grads_( - optimizer, inv_scale, found_inf, False - ) - optimizer_state["stage"] = OptState.UNSCALED - -def update(self, new_scale=None): - """ - Updates the scale factor. - If any optimizer steps were skipped the scale is multiplied by ``backoff_factor`` - to reduce it. If ``growth_interval`` unskipped iterations occurred consecutively, - the scale is multiplied by ``growth_factor`` to increase it. - Passing ``new_scale`` sets the new scale value manually. (``new_scale`` is not - used directly, it's used to fill GradScaler's internal scale tensor. So if - ``new_scale`` was a tensor, later in-place changes to that tensor will not further - affect the scale GradScaler uses internally.) - Args: - new_scale (float or :class:`torch.FloatTensor`, optional, default=None): New scale factor. - .. warning:: - :meth:`update` should only be called at the end of the iteration, after ``scaler.step(optimizer)`` has - been invoked for all optimizers used this iteration. - """ - if not self._enabled: - return - - _scale, _growth_tracker = self._check_scale_growth_tracker("update") - - if new_scale is not None: - # Accept a new user-defined scale. - if isinstance(new_scale, float): - self._scale.fill_(new_scale) # type: ignore[union-attr] - else: - reason = "new_scale should be a float or a 1-element torch.FloatTensor with requires_grad=False." - assert isinstance(new_scale, torch.FloatTensor), reason # type: ignore[attr-defined] - assert new_scale.numel() == 1, reason - assert new_scale.requires_grad is False, reason - self._scale.copy_(new_scale) # type: ignore[union-attr] - else: - # Consume shared inf/nan data collected from optimizers to update the scale. - # If all found_inf tensors are on the same device as self._scale, this operation is asynchronous. - found_infs = [ - found_inf.to(device="cpu", non_blocking=True) - for state in self._per_optimizer_states.values() - for found_inf in state["found_inf_per_device"].values() - ] - - assert len(found_infs) > 0, "No inf checks were recorded prior to update." - - found_inf_combined = found_infs[0] - if len(found_infs) > 1: - for i in range(1, len(found_infs)): - found_inf_combined += found_infs[i] - - to_device = _scale.device - _scale = _scale.to("cpu") - _growth_tracker = _growth_tracker.to("cpu") - - core._amp_update_scale_( - _scale, - _growth_tracker, - found_inf_combined, - self._growth_factor, - self._backoff_factor, - self._growth_interval, - ) - - _scale = _scale.to(to_device) - _growth_tracker = _growth_tracker.to(to_device) - # To prepare for next iteration, clear the data collected from optimizers this iteration. - self._per_optimizer_states = defaultdict(_refresh_per_optimizer_state) - -def gradscaler_init(): - torch.xpu.amp.GradScaler = ipex.cpu.autocast._grad_scaler.GradScaler - torch.xpu.amp.GradScaler._unscale_grads_ = _unscale_grads_ - torch.xpu.amp.GradScaler.unscale_ = unscale_ - torch.xpu.amp.GradScaler.update = update - return torch.xpu.amp.GradScaler diff --git a/modules/intel/ipex/hijacks.py b/modules/intel/ipex/hijacks.py index e439365fb..db4d0e6f3 100644 --- a/modules/intel/ipex/hijacks.py +++ b/modules/intel/ipex/hijacks.py @@ -51,14 +51,42 @@ def return_xpu(device): # keep the device instance type, aka return string if th # Autocast original_autocast_init = torch.amp.autocast_mode.autocast.__init__ @wraps(torch.amp.autocast_mode.autocast.__init__) -def autocast_init(self, device_type, dtype=None, enabled=True, cache_enabled=None): - if device_type == "cuda" or device_type == "xpu": +def autocast_init(self, device_type=None, dtype=None, enabled=True, cache_enabled=None): + if device_type is None or check_cuda(device_type) or check_device_type(device_type, "xpu"): if dtype is None: dtype = devices.dtype return original_autocast_init(self, device_type="xpu", dtype=dtype, enabled=enabled, cache_enabled=cache_enabled) else: return original_autocast_init(self, device_type=device_type, dtype=dtype, enabled=enabled, cache_enabled=cache_enabled) + +original_grad_scaler_init = torch.amp.grad_scaler.GradScaler.__init__ +@wraps(torch.amp.grad_scaler.GradScaler.__init__) +def GradScaler_init(self, device: str = None, init_scale: float = 2.0**16, growth_factor: float = 2.0, backoff_factor: float = 0.5, growth_interval: int = 2000, enabled: bool = True): + if device is None or check_cuda(device): + return original_grad_scaler_init(self, device=return_xpu(device), init_scale=init_scale, growth_factor=growth_factor, backoff_factor=backoff_factor, growth_interval=growth_interval, enabled=enabled) + else: + return original_grad_scaler_init(self, device=device, init_scale=init_scale, growth_factor=growth_factor, backoff_factor=backoff_factor, growth_interval=growth_interval, enabled=enabled) + + +original_is_autocast_enabled = torch.is_autocast_enabled +@wraps(torch.is_autocast_enabled) +def torch_is_autocast_enabled(device_type=None): + if device_type is None or check_cuda(device_type): + return original_is_autocast_enabled(return_xpu(device_type)) + else: + return original_is_autocast_enabled(device_type) + + +original_get_autocast_dtype = torch.get_autocast_dtype +@wraps(torch.get_autocast_dtype) +def torch_get_autocast_dtype(device_type=None): + if device_type is None or check_cuda(device_type) or check_device_type(device_type, "xpu"): + return torch.bfloat16 + else: + return original_get_autocast_dtype(device_type) + + # Latent Antialias CPU Offload: # IPEX 2.5 and above has partial support but doesn't really work most of the time. original_interpolate = torch.nn.functional.interpolate @@ -404,4 +432,28 @@ def ipex_hijacks(): if not device_supports_fp64: torch.from_numpy = from_numpy torch.as_tensor = as_tensor + + # AMP: + torch.amp.grad_scaler.GradScaler.__init__ = GradScaler_init + torch.is_autocast_enabled = torch_is_autocast_enabled + torch.get_autocast_gpu_dtype = torch_get_autocast_dtype + torch.get_autocast_dtype = torch_get_autocast_dtype + + if hasattr(torch.xpu, "amp"): + if not hasattr(torch.xpu.amp, "custom_fwd"): + torch.xpu.amp.custom_fwd = torch.cuda.amp.custom_fwd + torch.xpu.amp.custom_bwd = torch.cuda.amp.custom_bwd + if not hasattr(torch.xpu.amp, "GradScaler"): + torch.xpu.amp.GradScaler = torch.amp.grad_scaler.GradScaler + torch.cuda.amp = torch.xpu.amp + else: + if not hasattr(torch.amp, "custom_fwd"): + torch.amp.custom_fwd = torch.cuda.amp.custom_fwd + torch.amp.custom_bwd = torch.cuda.amp.custom_bwd + torch.cuda.amp = torch.amp + + if not hasattr(torch.cuda.amp, "common"): + torch.cuda.amp.common = nullcontext() + torch.cuda.amp.common.amp_definitely_not_available = lambda: False + return device_supports_fp64, can_allocate_plus_4gb From 3d8390de9b8280c03679d87fcf9766e10f964198 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 18 May 2025 04:44:51 +0300 Subject: [PATCH 03/94] IPEX return devices.dtype instead of bf16 --- modules/intel/ipex/hijacks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/intel/ipex/hijacks.py b/modules/intel/ipex/hijacks.py index db4d0e6f3..8c308f921 100644 --- a/modules/intel/ipex/hijacks.py +++ b/modules/intel/ipex/hijacks.py @@ -82,7 +82,7 @@ original_get_autocast_dtype = torch.get_autocast_dtype @wraps(torch.get_autocast_dtype) def torch_get_autocast_dtype(device_type=None): if device_type is None or check_cuda(device_type) or check_device_type(device_type, "xpu"): - return torch.bfloat16 + return devices.dtype else: return original_get_autocast_dtype(device_type) From b23162a36b8d0703d3495fe6cca16bd0506d614e Mon Sep 17 00:00:00 2001 From: hypercryptoman <110080854+hypercryptoman@users.noreply.github.com> Date: Sun, 18 May 2025 23:35:55 +1000 Subject: [PATCH 04/94] Fix: Correct arguments for prompt_enhance.py apply method --- scripts/prompt_enhance.py | 70 +++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index 8b11dfcb5..fe2ea0851 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -1,3 +1,4 @@ + from dataclasses import dataclass import io import os @@ -18,7 +19,7 @@ debug_log = shared.log.trace if debug_enabled else lambda *args, **kwargs: None def b64(image): if image is None: return '' - if isinstance(image, gr.Image): + if isinstance(image, gr.Image): # should not happen return None with io.BytesIO() as stream: image.convert('RGB').save(stream, 'JPEG') @@ -105,6 +106,7 @@ class Script(scripts.Script): return self.busy = True if self.model is not None and self.model == name: + self.busy = False # ensure busy is reset even if model is already loaded return from modules import modelloader, model_quant, ggml @@ -138,7 +140,7 @@ class Script(scripts.Script): self.model = None load_args = { 'pretrained_model_name_or_path': model_repo if not gguf_args else model_gguf } if model_subfolder: - load_args['subfolder'] = model_subfolder, + load_args['subfolder'] = model_subfolder # Comma was incorrect here self.llm = transformers.AutoModelForCausalLM.from_pretrained( **load_args, trust_remote_code=True, @@ -194,8 +196,8 @@ class Script(scripts.Script): # remove comments between brackets response = re.sub(r'<.*?>', '', response) - response = re.sub(r'\[.*?\]', '', response) - response = re.sub(r'\/.*?\/', '', response) + response = re.sub(r'\[.*?\]', '', response) # Fixed regex for brackets + response = re.sub(r'\/.*?\/', '', response) # Fixed regex for slashes # remove llm commentary removed = '' @@ -209,7 +211,7 @@ class Script(scripts.Script): debug_log(f'Prompt enhance: max={self.options.max_delim_index} removed="{removed}"') # remove bullets and lists - lines = [re.sub(r'^(\s*[-*]|\s*\d+)\s+', '', line).strip() for line in response.splitlines()] + lines = [re.sub(r'^(\s*[-*]|\s*\d+)\s+', '', line).strip() for line in response.splitlines()] # Fixed regex response = '\n'.join(lines) response = response.strip() @@ -235,7 +237,7 @@ class Script(scripts.Script): def enhance(self, model: str=None, prompt:str=None, system:str=None, prefix:str=None, suffix:str=None, sample:bool=None, tokens:int=None, temperature:float=None, penalty:float=None, thinking:bool=False, seed:int=-1, image=None, nsfw:bool=None): model = model or self.options.default - prompt = prompt or self.prompt.value + prompt = prompt or (self.prompt.value if self.prompt else "") # Check if self.prompt is None image = image or self.image prefix = prefix or '' suffix = suffix or '' @@ -244,6 +246,8 @@ class Script(scripts.Script): temperature = temperature or self.options.temperature thinking = thinking or self.options.thinking_mode sample = sample if sample is not None else self.options.do_sample + nsfw = nsfw if nsfw is not None else True # Default nsfw to True if not provided + while self.busy: time.sleep(0.1) self.load(model) @@ -252,22 +256,28 @@ class Script(scripts.Script): if self.llm is None: shared.log.error('Prompt enhance: model not loaded') return prompt - prompt, networks = self.extract(prompt) + prompt_text, networks = self.extract(prompt) # Use prompt_text after extraction debug_log(f'Prompt enhance: networks={networks}') + + current_image = None try: if image is not None and isinstance(image, gr.Image): - image = image.value - if image is not None and (image.width <= 64 or image.height <= 64): - image = None + current_image = image.value + elif image is not None and isinstance(image, Image.Image): # if image is already a PIL image + current_image = image + if current_image is not None and (current_image.width <= 64 or current_image.height <= 64): + current_image = None except Exception: - image = None + current_image = None + has_system = system is not None and len(system) > 4 mode = 'custom' if has_system else '' - if image is not None and isinstance(image, Image.Image): + + if current_image is not None and isinstance(current_image, Image.Image): if not self.tokenizer.is_processor: shared.log.error('Prompt enhance: image not supported by model') - return prompt - if prompt is not None and len(prompt) > 0: + return prompt_text # Return original text part if image cannot be processed + if prompt_text is not None and len(prompt_text) > 0: if not has_system: mode = 'i2i-prompt' system = self.options.i2i_prompt @@ -278,8 +288,8 @@ class Script(scripts.Script): {"type": "text", "text": system } ] }, { "role": "user", "content": [ - {"type": "text", "text": prompt}, - {"type": "image", "image": b64(image)} + {"type": "text", "text": prompt_text}, + {"type": "image", "image": b64(current_image)} ] }, ] else: @@ -293,7 +303,7 @@ class Script(scripts.Script): {"type": "text", "text": system } ] }, { "role": "user", "content": [ - {"type": "image", "image": b64(image)} + {"type": "image", "image": b64(current_image)} ] }, ] else: @@ -305,7 +315,7 @@ class Script(scripts.Script): mode = 't2i+tokenizer' chat_template = [ { "role": "system", "content": system }, - { "role": "user", "content": prompt }, + { "role": "user", "content": prompt_text }, ] else: mode = 't2i+processor' @@ -314,7 +324,7 @@ class Script(scripts.Script): {"type": "text", "text": system } ] }, { "role": "user", "content": [ - {"type": "text", "text": prompt}, + {"type": "text", "text": prompt_text}, ] }, ] @@ -334,7 +344,7 @@ class Script(scripts.Script): shared.log.error(f'Prompt enhance tokenize: {e}') errors.display(e, 'Prompt enhance') self.busy = False - return prompt + return prompt_text # Return original text part on error try: with devices.inference_context(): sd_models.move_model(self.llm, devices.device) @@ -370,18 +380,19 @@ class Script(scripts.Script): if not is_censored: response = self.clean(response) response = self.post(response, prefix, suffix, networks) - shared.log.info(f'Prompt enhance: model="{model}" mode="{mode}" nsfw={nsfw} time={t1-t0:.2f} inputs={input_len} outputs={outputs.shape[-1]} prompt={len(prompt)} response={len(response)}') + shared.log.info(f'Prompt enhance: model="{model}" mode="{mode}" nsfw={nsfw} time={t1-t0:.2f} inputs={input_len} outputs={outputs.shape[-1] if isinstance(outputs, torch.Tensor) else 0} prompt={len(prompt_text)} response={len(response)}') # Added check for outputs if debug_enabled: shared.log.trace(f'Prompt enhance: sample={sample} tokens={tokens} temperature={temperature} penalty={penalty} thinking={thinking}') - shared.log.trace(f'Prompt enhance: prompt="{prompt}"') + shared.log.trace(f'Prompt enhance: prompt="{prompt_text}"') shared.log.trace(f'Prompt enhance: response="{response}"') self.busy = False if is_censored: shared.log.warning(f'Prompt enhance: censored response="{response}"') - return prompt + return prompt # Return original full prompt on censorship return response - def apply(self, prompt, image, apply_prompt, llm_model, prompt_system, prompt_prefix, prompt_suffix, max_tokens, do_sample, temperature, repetition_penalty, thinking_mode): + # --- START OF CORRECTED METHOD --- + def apply(self, prompt, image, apply_prompt, llm_model, prompt_system, prompt_prefix, prompt_suffix, max_tokens, do_sample, temperature, repetition_penalty, thinking_mode, nsfw_mode): # Added nsfw_mode response = self.enhance( prompt=prompt, image=image, @@ -394,10 +405,12 @@ class Script(scripts.Script): temperature=temperature, penalty=repetition_penalty, thinking=thinking_mode, + nsfw=nsfw_mode # Pass nsfw_mode here ) if apply_prompt: return [response, response] return [response, gr.update()] + # --- END OF CORRECTED METHOD --- def get_custom(self, name): model_repo = self.options.models.get(name, {}).get('repo', None) or name @@ -433,7 +446,7 @@ class Script(scripts.Script): model_file = gr.Textbox(label='Model file', value=None, interactive=True, elem_id='prompt_enhance_model_file', placeholder='Optional GGUF model file inside GGUF model repo') with gr.Row(): custom_btn = gr.Button(value='Load custom model', elem_id='prompt_enhance_custom_load', variant='secondary') - custom_btn.click(fn=self.load, inputs=[model_file, model_repo, model_gguf, model_type, model_file], outputs=[]) + custom_btn.click(fn=self.load, inputs=[model_file, model_repo, model_gguf, model_type, model_file], outputs=[]) # Note: model_file passed twice, intentional? llm_model.change(fn=self.get_custom, inputs=[llm_model], outputs=[model_repo, model_gguf, model_type, model_file]) gr.HTML('
') with gr.Accordion('Options', open=False, elem_id='prompt_enhance_options'): @@ -447,14 +460,14 @@ class Script(scripts.Script): nsfw_mode = gr.Checkbox(label='NSFW allowed', value=True, interactive=True) thinking_mode = gr.Checkbox(label='Thinking mode', value=False, interactive=True) gr.HTML('
') - with gr.Accordion('Input', open=False, elem_id='prompt_enhance_system_prompt'): + with gr.Accordion('Input', open=False, elem_id='prompt_enhance_system_prompt'): # Corrected elem_id reference with gr.Row(): prompt_prefix = gr.Textbox(label='Prompt prefix', value='', placeholder='Optional prompt prefix', interactive=True, lines=2, elem_id='prompt_enhance_prefix') with gr.Row(): prompt_suffix = gr.Textbox(label='Prompt suffix', value='', placeholder='Optional prompt suffix', interactive=True, lines=2, elem_id='prompt_enhance_suffix') with gr.Row(): - prompt_system = gr.Textbox(label='System prompt', value='', interactive=True, lines=4, elem_id='prompt_enhance_system') - with gr.Accordion('Output', open=True, elem_id='prompt_enhance_system_prompt'): + prompt_system = gr.Textbox(label='System prompt', value='', interactive=True, lines=4, elem_id='prompt_enhance_system') # Default to empty as per diff + with gr.Accordion('Output', open=True, elem_id='prompt_enhance_output'): # Corrected elem_id reference with gr.Row(): prompt_output = gr.Textbox(label='Enhanced prompt', value='', interactive=True, lines=4) with gr.Row(): @@ -502,3 +515,4 @@ class Script(scripts.Script): ) p.extra_generation_params['LLM'] = llm_model shared.state.end() + From 7f2d77e9562f1de671bb912888c04b364e23ef41 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 18 May 2025 16:36:27 +0300 Subject: [PATCH 05/94] ROCm set MIOPEN_FIND_ENFORCE to SEARCH --- installer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/installer.py b/installer.py index fafcd70ce..c9a83f1cc 100644 --- a/installer.py +++ b/installer.py @@ -667,6 +667,8 @@ def install_rocm_zluda(): if os.environ.get("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", None) is None: os.environ.setdefault('TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL', '1') + if os.environ.get("MIOPEN_FIND_ENFORCE", None) is None: + os.environ.setdefault('MIOPEN_FIND_ENFORCE', 'SEARCH') if args.use_nightly: if rocm.version is None or float(rocm.version) >= 6.3: # assume the latest if version check fails From 2b824daf64431253a87e4750dd903cd9c372dbc0 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 18 May 2025 21:28:44 +0300 Subject: [PATCH 06/94] Revert MIOPEN_FIND_ENFORCE --- installer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/installer.py b/installer.py index c9a83f1cc..fafcd70ce 100644 --- a/installer.py +++ b/installer.py @@ -667,8 +667,6 @@ def install_rocm_zluda(): if os.environ.get("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", None) is None: os.environ.setdefault('TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL', '1') - if os.environ.get("MIOPEN_FIND_ENFORCE", None) is None: - os.environ.setdefault('MIOPEN_FIND_ENFORCE', 'SEARCH') if args.use_nightly: if rocm.version is None or float(rocm.version) >= 6.3: # assume the latest if version check fails From ba2eaaf2951b5244223fa00a487473b4d2dd419d Mon Sep 17 00:00:00 2001 From: hypercryptoman <110080854+hypercryptoman@users.noreply.github.com> Date: Mon, 19 May 2025 13:45:05 +1000 Subject: [PATCH 07/94] Fix: Correct model_file parameter usage for custom load button --- scripts/prompt_enhance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index fe2ea0851..45e2ab46f 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -446,7 +446,7 @@ class Script(scripts.Script): model_file = gr.Textbox(label='Model file', value=None, interactive=True, elem_id='prompt_enhance_model_file', placeholder='Optional GGUF model file inside GGUF model repo') with gr.Row(): custom_btn = gr.Button(value='Load custom model', elem_id='prompt_enhance_custom_load', variant='secondary') - custom_btn.click(fn=self.load, inputs=[model_file, model_repo, model_gguf, model_type, model_file], outputs=[]) # Note: model_file passed twice, intentional? + custom_btn.click(fn=self.load, inputs=[model_file, model_repo, model_gguf, model_type, None], outputs=[]) llm_model.change(fn=self.get_custom, inputs=[llm_model], outputs=[model_repo, model_gguf, model_type, model_file]) gr.HTML('
') with gr.Accordion('Options', open=False, elem_id='prompt_enhance_options'): From 05fced7395a16f9f27e7ba9f0faec1b0b6abd980 Mon Sep 17 00:00:00 2001 From: hypercryptoman <110080854+hypercryptoman@users.noreply.github.com> Date: Mon, 19 May 2025 13:53:24 +1000 Subject: [PATCH 08/94] Update prompt_enhance.py --- scripts/prompt_enhance.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index 45e2ab46f..118695feb 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -1,4 +1,3 @@ - from dataclasses import dataclass import io import os @@ -446,7 +445,8 @@ class Script(scripts.Script): model_file = gr.Textbox(label='Model file', value=None, interactive=True, elem_id='prompt_enhance_model_file', placeholder='Optional GGUF model file inside GGUF model repo') with gr.Row(): custom_btn = gr.Button(value='Load custom model', elem_id='prompt_enhance_custom_load', variant='secondary') - custom_btn.click(fn=self.load, inputs=[model_file, model_repo, model_gguf, model_type, None], outputs=[]) + # THIS IS THE CORRECTED LINE: + custom_btn.click(fn=self.load, inputs=[model_repo, model_repo, model_gguf, model_type, model_file], outputs=[]) llm_model.change(fn=self.get_custom, inputs=[llm_model], outputs=[model_repo, model_gguf, model_type, model_file]) gr.HTML('
') with gr.Accordion('Options', open=False, elem_id='prompt_enhance_options'): @@ -515,4 +515,4 @@ class Script(scripts.Script): ) p.extra_generation_params['LLM'] = llm_model shared.state.end() - + From ac05b96838fca2242acbbfd73198ac6ea3229c89 Mon Sep 17 00:00:00 2001 From: hypercryptoman <110080854+hypercryptoman@users.noreply.github.com> Date: Mon, 19 May 2025 13:55:16 +1000 Subject: [PATCH 09/94] Update prompt_enhance.py --- scripts/prompt_enhance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index 118695feb..fdfdfa86c 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -445,7 +445,6 @@ class Script(scripts.Script): model_file = gr.Textbox(label='Model file', value=None, interactive=True, elem_id='prompt_enhance_model_file', placeholder='Optional GGUF model file inside GGUF model repo') with gr.Row(): custom_btn = gr.Button(value='Load custom model', elem_id='prompt_enhance_custom_load', variant='secondary') - # THIS IS THE CORRECTED LINE: custom_btn.click(fn=self.load, inputs=[model_repo, model_repo, model_gguf, model_type, model_file], outputs=[]) llm_model.change(fn=self.get_custom, inputs=[llm_model], outputs=[model_repo, model_gguf, model_type, model_file]) gr.HTML('
') From 85f00f9edbc4433ce45f7394bf1d3585e4e06de7 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 23 May 2025 18:24:47 +0300 Subject: [PATCH 10/94] Enable dyn atten by default for ROCm --- modules/shared.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/shared.py b/modules/shared.py index 66a7ebfa7..cba39b41d 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -379,7 +379,9 @@ def get_default_modes(): default_cross_attention = "Scaled-Dot-Product" if devices.backend == "zluda": - default_sdp_options = ['Math attention', 'Flash attention', 'Dynamic attention'] + default_sdp_options = ['Flash attention', 'Math attention', 'Dynamic attention'] + elif devices.backend == "rocm": + default_sdp_options = ['Flash attention', 'Memory attention', 'Math attention', 'Dynamic attention'] else: default_sdp_options = ['Flash attention', 'Memory attention', 'Math attention'] From bfc5c7c457ba4a5ff243011f1fca65da8f89b48b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 24 May 2025 08:42:13 +0200 Subject: [PATCH 11/94] installer version check Signed-off-by: Vladimir Mandic --- installer.py | 48 ++++++++++++++++++++++++------------------------ launch.py | 5 ++++- wiki | 2 +- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/installer.py b/installer.py index fafcd70ce..714db5ff2 100644 --- a/installer.py +++ b/installer.py @@ -1392,6 +1392,7 @@ def check_version(offline=False, reset=True): # pylint: disable=unused-argument args.skip_git = True # pylint: disable=attribute-defined-outside-init ver = get_version() log.info(f'Version: {print_dict(ver)}') + branch_name = ver['branch'] if ver is not None and 'branch' in ver else 'master' if args.version or args.skip_git: return check_ui(ver) @@ -1406,30 +1407,29 @@ def check_version(offline=False, reset=True): # pylint: disable=unused-argument return commits = None try: - commits = requests.get('https://api.github.com/repos/vladmandic/sdnext/branches/master', timeout=10).json() - if commits['commit']['sha'] != commit: - if args.upgrade: - global quick_allowed # pylint: disable=global-statement - quick_allowed = False - log.info('Updating main repository') - try: - git('add .') - git('stash') - update('.', keep_branch=True) - # git('git stash pop') - ver = git('log -1 --pretty=format:"%h %ad"') - log.info(f'Repository upgraded: {ver}') - except Exception: - if not reset: - log.error('Repository error upgrading') - else: - log.warning('Repository: retrying upgrade...') - git_reset() - check_version(offline=offline, reset=False) - else: - dt = commits["commit"]["commit"]["author"]["date"] - commit = commits["commit"]["sha"][:8] - log.info(f'Version: check latest available hash={commit} updated={dt}') + commits = requests.get(f'https://api.github.com/repos/vladmandic/sdnext/branches/{branch_name}', timeout=10).json() + if commits['commit']['sha'] != commit and args.upgrade: + global quick_allowed # pylint: disable=global-statement + quick_allowed = False + log.info('Updating main repository') + try: + git('add .') + git('stash') + update('.', keep_branch=True) + # git('git stash pop') + ver = git('log -1 --pretty=format:"%h %ad"') + log.info(f'Repository upgraded: {ver}') + except Exception: + if not reset: + log.error('Repository error upgrading') + else: + log.warning('Repository: retrying upgrade...') + git_reset() + check_version(offline=offline, reset=False) + else: + dt = commits["commit"]["commit"]["author"]["date"] + commit = commits["commit"]["sha"][:8] + log.info(f'Version: app=sd.next latest={dt} hash={commit} branch={branch_name}') except Exception as e: log.error(f'Repository failed to check version: {e} {commits}') ts('latest', t_start) diff --git a/launch.py b/launch.py index c0c8e5b81..facf7d25a 100755 --- a/launch.py +++ b/launch.py @@ -264,8 +264,11 @@ def main(): installer.check_onnx() installer.check_diffusers() installer.check_modified_files() + if args.test: + installer.log.info('Startup: test mode') + installer.quick_allowed = False if args.reinstall: - installer.log.info('Forcing reinstall of all packages') + installer.log.info('Startup: force reinstall of all packages') installer.quick_allowed = False if args.skip_all: installer.log.info('Startup: skip all') diff --git a/wiki b/wiki index 25c8a39ad..ffd60bd24 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 25c8a39adecd2a9683aace950599330b72031032 +Subproject commit ffd60bd24aec14e8613202eec3f92a889c40216c From af3a44ccbeded07a315bb66be39efaf43c0f5651 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 24 May 2025 08:58:04 +0200 Subject: [PATCH 12/94] optional skimage Signed-off-by: Vladimir Mandic --- modules/consistory/utils/general_utils.py | 3 ++- modules/face/reswapper_utils.py | 3 ++- modules/processing_helpers.py | 6 +++++- scripts/outpainting_mk_2.py | 3 ++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/modules/consistory/utils/general_utils.py b/modules/consistory/utils/general_utils.py index 4493fa96e..bdf0686ca 100644 --- a/modules/consistory/utils/general_utils.py +++ b/modules/consistory/utils/general_utils.py @@ -6,15 +6,16 @@ import torch import torch.nn.functional as F import numpy as np -from skimage import filters ## Attention Utils def get_dynamic_threshold(tensor): + from skimage import filters return filters.threshold_otsu(tensor.float().cpu().numpy()) def attn_map_to_binary(attention_map, scaler=1.): + from skimage import filters attention_map_np = attention_map.float().cpu().numpy() threshold_value = filters.threshold_otsu(attention_map_np) * scaler binary_mask = (attention_map_np > threshold_value).astype(np.uint8) diff --git a/modules/face/reswapper_utils.py b/modules/face/reswapper_utils.py index f5dbf0c93..ae260054d 100644 --- a/modules/face/reswapper_utils.py +++ b/modules/face/reswapper_utils.py @@ -1,6 +1,5 @@ import cv2 import numpy as np -from skimage import transform as trans ### https://github.com/somanchiu/ReSwapper/blob/GAN/Image.py @@ -74,6 +73,7 @@ arcface_dst = np.array( def estimate_norm(lmk, image_size=112,mode='arcface'): # pylint: disable=unused-argument + from skimage import transform as trans if image_size%112==0: ratio = float(image_size)/112.0 diff_x = 0 @@ -126,6 +126,7 @@ def square_crop(im, S): def transform(data, center, output_size, scale, rotation): + from skimage import transform as trans scale_ratio = scale rot = float(rotation) * np.pi / 180.0 t1 = trans.SimilarityTransform(scale=scale_ratio) diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index 2535127ff..0f2d7bc6c 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -8,7 +8,6 @@ import torch import numpy as np import cv2 from PIL import Image -from skimage import exposure from blendmodes.blend import blendLayers, BlendType from modules import shared, devices, images, sd_models, sd_samplers, sd_hijack_hypertile, processing_vae, timer @@ -33,6 +32,7 @@ def setup_color_correction(image): def apply_color_correction(correction, original_image): + from skimage import exposure shared.log.debug(f"Applying color correction: correction={correction.shape} image={original_image}") np_image = np.asarray(original_image) np_recolor = cv2.cvtColor(np_image, cv2.COLOR_RGB2LAB) @@ -545,11 +545,15 @@ def apply_circular(enable: bool, model): if getattr(model, 'texture_tiling', False) == enable: return try: + i = 0 for layer in [layer for layer in model.unet.modules() if type(layer) is torch.nn.Conv2d]: + i += 1 layer.padding_mode = 'circular' if enable else 'zeros' for layer in [layer for layer in model.vae.modules() if type(layer) is torch.nn.Conv2d]: + i += 1 layer.padding_mode = 'circular' if enable else 'zeros' model.texture_tiling = enable + shared.log.debug(f'Apply texture tiling: enabled={enable} layers={i} cls={model.__class__.__name__} ') except Exception as e: debug(f"Diffusers tiling failed: {e}") diff --git a/scripts/outpainting_mk_2.py b/scripts/outpainting_mk_2.py index 383587cc6..7fd56e353 100644 --- a/scripts/outpainting_mk_2.py +++ b/scripts/outpainting_mk_2.py @@ -1,6 +1,5 @@ import math import numpy as np -import skimage import gradio as gr from PIL import Image, ImageDraw import modules.scripts as scripts @@ -60,6 +59,8 @@ def get_matched_noise(_np_src_image, np_mask_rgb, noise_q=1, color_variation=0.0 for c in range(3): np_mask_rgb[:, :, c] = hardened[:] return np_mask_rgb + + import skimage width = _np_src_image.shape[0] height = _np_src_image.shape[1] num_channels = _np_src_image.shape[2] From 2d79380bd71e376d9b1460248c5583ea504b5efc Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 01:12:28 +0300 Subject: [PATCH 13/94] NNCF implement better layer hijacks and remove all NNCF imports --- installer.py | 3 - modules/lora/lora_apply.py | 21 +++--- modules/model_quant.py | 40 ---------- modules/model_quant_nncf.py | 147 +++++++++++++++++++----------------- 4 files changed, 87 insertions(+), 124 deletions(-) diff --git a/installer.py b/installer.py index 714db5ff2..769256404 100644 --- a/installer.py +++ b/installer.py @@ -787,8 +787,6 @@ def install_torch_addons(): install('DeepCache') if opts.get('cuda_compile_backend', '') == 'olive-ai': install('olive-ai') - if opts.get('nncf_compress_weights', False) and not args.use_openvino: - install('nncf==2.16.0', 'nncf') if opts.get('optimum_quanto_weights', False): install('optimum-quanto==0.2.7', 'optimum-quanto') if opts.get('torchao_quantization', False): @@ -1175,7 +1173,6 @@ def install_optional(): install('albumentations==1.4.3', ignore=True) install('pydantic==1.10.21', ignore=True) reload('pydantic', '1.10.21') - install('nncf==2.16.0', ignore=True) install('gguf', ignore=True) install('av', ignore=True) try: diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index 68f723188..7b00ee389 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -45,8 +45,8 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n self.network_weights_backup = True else: self.network_weights_backup = weight.clone().to(devices.cpu) - if self.__class__.__name__.startswith('NNCF') and hasattr(self, "pre_ops") and len(self.pre_ops) == 1: - self.nncf_decompressor_backup = self.pre_ops["0"].to(devices.cpu) + if hasattr(self, "nncf_decompressor"): + self.nncf_decompressor_backup = self.nncf_decompressor.to(devices.cpu) if bias_backup is None: if getattr(self, 'bias', None) is not None: @@ -79,10 +79,10 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. continue try: t0 = time.time() - if self.__class__.__name__.startswith('NNCF') and hasattr(self, "pre_ops") and len(self.pre_ops) == 1: + if hasattr(self, "nncf_decompressor"): return_device = self.weight.data.device self.weight.data = self.weight.data.to(devices.device) - weight = self.pre_ops["0"].to(devices.device)(self, return_decompressed_only=True) + weight = self.nncf_decompressor.to(devices.device)(self, return_decompressed_only=True) self.weight.data = self.weight.data.to(return_device) else: weight = self.weight.to(devices.device) # must perform calc on gpu due to performance @@ -139,19 +139,18 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G # weight._quantize(devices.device) / weight.to(device=device) except Exception as e: shared.log.error(f'Network load: type=LoRA quant=bnb cls={self.__class__.__name__} type={self.quant_type} blocksize={self.blocksize} state={vars(self.quant_state)} weight={self.weight} bias={lora_weights} {e}') - elif not bias and self.__class__.__name__.startswith('NNCF') and hasattr(self, "pre_ops") and len(self.pre_ops) == 1: + elif not bias and hasattr(self, "nncf_decompressor"): num_bits = None is_asym_mode = None try: from modules.model_quant_nncf import nncf_compress_layer - num_bits = self.pre_ops["0"].num_bits - is_asym_mode = self.pre_ops["0"].quantization_mode == "asymmetric" + num_bits = self.nncf_decompressor.num_bits + is_asym_mode = self.nncf_decompressor.quantization_mode == "asymmetric" self.weight = torch.nn.Parameter(model_weights.to(devices.device), requires_grad=False) - dequant_weight = self.pre_ops["0"](self, return_decompressed_only=True) + dequant_weight = self.nncf_decompressor(self, return_decompressed_only=True) new_weight = dequant_weight.to(devices.device, dtype=torch.float32) + lora_weights.to(devices.device, dtype=torch.float32) self.weight = torch.nn.Parameter(new_weight, requires_grad=False) - self.pre_ops.pop("0") - self._custom_forward_fn = None # pylint: disable=protected-access + self.nncf_decompressor = None self = nncf_compress_layer(self, num_bits, is_asym_mode, torch_dtype=devices.dtype, quant_conv=shared.opts.nncf_quantize_conv_layers, group_size=shared.opts.nncf_compress_weights_group_size, use_int8_matmul=shared.opts.nncf_decompress_int8_matmul) self = self.to(device) del dequant_weight @@ -219,7 +218,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn else: self.weight = torch.nn.Parameter(weights_backup.to(device), requires_grad=False) if hasattr(self, "nncf_decompressor_backup"): - self.pre_ops["0"] = self.nncf_decompressor_backup.to(device) + self.nncf_decompressor = self.nncf_decompressor_backup.to(device) if bias_backup is not None: self.bias = None diff --git a/modules/model_quant.py b/modules/model_quant.py index a63123b34..3b453691d 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -9,7 +9,6 @@ from installer import installed, install, log, setup_logging ao = None bnb = None -intel_nncf = None optimum_quanto = None quant_last_model_name = None quant_last_model_device = None @@ -108,9 +107,6 @@ def create_nncf_config(kwargs = None, allow_nncf: bool = True, module: str = 'Mo from modules import shared if len(shared.opts.nncf_compress_weights) > 0 and (shared.opts.nncf_compress_mode == 'pre') and allow_nncf: if 'Model' in shared.opts.nncf_compress_weights or (module is not None and module in shared.opts.nncf_compress_weights) or module == 'any': - load_nncf(silent=True) - if intel_nncf is None: - return kwargs from modules.model_quant_nncf import NNCFQuantizer, NNCFConfig diffusers.quantizers.auto.AUTO_QUANTIZER_MAPPING["nncf"] = NNCFQuantizer transformers.quantizers.auto.AUTO_QUANTIZER_MAPPING["nncf"] = NNCFQuantizer @@ -259,35 +255,6 @@ def load_quanto(msg='', silent=False): return None -def load_nncf(msg='', silent=False): - global intel_nncf # pylint: disable=global-statement - if intel_nncf is not None: - return intel_nncf - if not installed('nncf'): - install('nncf==2.16.0', quiet=True) - log.warning('Quantization: nncf installed please restart') - install('jstyleson', quiet=True) - install('texttable', quiet=True) - install('tabulate', quiet=True) - try: - import nncf - intel_nncf = nncf - try: - nncf.common.logging.logger.warn_bkc_version_mismatch = lambda *args, **kwargs: None # silence the pytorch version warning - except Exception: - pass - fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access - log.debug(f'Quantization: type=nncf version={nncf.__version__} fn={fn}') # pylint: disable=protected-access - return intel_nncf - except Exception as e: - if len(msg) > 0: - log.error(f"{msg} failed to import nncf: {e}") - intel_nncf = None - if not silent: - raise - return None - - def apply_layerwise(sd_model, quiet:bool=False): import torch from diffusers.quantizers import quantization_config @@ -334,11 +301,7 @@ def apply_layerwise(sd_model, quiet:bool=False): def nncf_compress_model(model, op=None, sd_model=None, do_gc=True): global quant_last_model_name, quant_last_model_device # pylint: disable=global-statement from modules import devices, shared - from accelerate import init_empty_weights - - load_nncf('Quantize model: type=NNCF') from modules.model_quant_nncf import apply_nncf_to_module - from nncf.torch.nncf_module_replacement import replace_modules_by_nncf_modules # get around lazy import model.eval() @@ -355,9 +318,6 @@ def nncf_compress_model(model, op=None, sd_model=None, do_gc=True): if hasattr(model, "get_input_embeddings"): backup_embeddings = copy.deepcopy(model.get_input_embeddings()) - with init_empty_weights(): - model, _ = replace_modules_by_nncf_modules(model) - num_bits = 8 if shared.opts.nncf_compress_weights_mode in {"INT8", "INT8_SYM", "INT8_ASYM"} else 4 is_asym_mode = shared.opts.nncf_compress_weights_mode in {"INT8", "INT4", "INT8_ASYM", "INT4_ASYM"} model = apply_nncf_to_module(model, num_bits, is_asym_mode, quant_conv=shared.opts.nncf_quantize_conv_layers) diff --git a/modules/model_quant_nncf.py b/modules/model_quant_nncf.py index 3d8fb4624..2e07eced8 100644 --- a/modules/model_quant_nncf.py +++ b/modules/model_quant_nncf.py @@ -8,7 +8,6 @@ import torch from diffusers.quantizers.base import DiffusersQuantizer from diffusers.quantizers.quantization_config import QuantizationConfigMixin from diffusers.utils import get_module_from_name -from accelerate import init_empty_weights from accelerate.utils import CustomDtype from modules import devices, shared @@ -42,22 +41,29 @@ class QuantizationMethod(str, Enum): def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_conv=False, group_size=0, use_int8_matmul=False, param_name=None): # pylint: disable=unused-argument - if layer.__class__.__name__ in allowed_types: + layer_class_name = layer.__class__.__name__ + if layer_class_name in allowed_types: + is_conv_type = False + is_conv_transpose_type = False + is_linear_type = False + result_shape = None if torch_dtype is None: torch_dtype = devices.dtype - result_shape = None - if layer.__class__.__name__ in conv_types: - if is_asym_mode or not quant_conv: # don't quant convs with asym mode + if layer_class_name in conv_types: + if not quant_conv: return layer reduction_axes = [i for i in range(layer.weight.ndim) if i != 0] use_int8_matmul = False - if layer.__class__.__name__ in conv_transpose_types: - if is_asym_mode or not quant_conv: # don't quant convs with asym mode + is_conv_type = True + elif layer_class_name in conv_transpose_types: + if not quant_conv: return layer reduction_axes = [i for i in range(layer.weight.ndim) if i != 1] use_int8_matmul = False + is_conv_transpose_type = True else: + is_linear_type = True reduction_axes = -1 channel_size = layer.weight.shape[-1] use_int8_matmul = use_int8_matmul and not is_asym_mode and channel_size >= 32 and layer.weight.shape[0] >= 32 @@ -106,12 +112,9 @@ def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c zero_point = zero_point.to(torch_dtype) if use_int8_matmul: - layer._custom_forward_fn = linear_forward_int8_matmul # pylint: disable=protected-access scale = scale.squeeze(-1) if num_bits == 8: compressed_weight = compressed_weight.transpose(0,1) - else: - layer._custom_forward_fn = None # pylint: disable=protected-access if num_bits == 4: if is_asym_mode: @@ -128,7 +131,6 @@ def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c compressed_weight_shape=compressed_weight.shape, result_dtype=torch_dtype, result_shape=result_shape, - use_int8_matmul=use_int8_matmul, ) else: if is_asym_mode: @@ -143,14 +145,30 @@ def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c scale=scale.data, result_dtype=torch_dtype, result_shape=result_shape, - use_int8_matmul=use_int8_matmul, ) compressed_weight = decompressor.pack_weight(compressed_weight).to(return_device) decompressor = decompressor.to(return_device) - layer.register_pre_forward_operation(decompressor) + layer.weight.requires_grad = False layer.weight.data = compressed_weight + layer.nncf_decompressor = decompressor + + if is_linear_type: + if use_int8_matmul: + layer.forward = quantized_linear_forward_int8_matmul + else: + layer.forward = quantized_linear_forward + elif is_conv_type: + layer.forward = quantized_conv_forward + elif is_conv_transpose_type: + if layer_class_name.endswith("1d"): + layer.forward = quantized_conv_transpose_1d_forward + elif layer_class_name.endswith("2d"): + layer.forward = quantized_conv_transpose_2d_forward + elif layer_class_name.endswith("3d"): + layer.forward = quantized_conv_transpose_3d_forward + layer.forward = layer.forward.__get__(layer, layer.__class__) return layer @@ -159,7 +177,7 @@ def apply_nncf_to_module(model, num_bits, is_asym_mode, quant_conv=False): if not has_children: return model for param_name, module in model.named_children(): - if module.__class__.__name__.startswith("NNCF") and hasattr(module, "weight") and module.weight is not None: + if hasattr(module, "weight") and module.weight is not None: module = nncf_compress_layer( module, num_bits, @@ -205,8 +223,7 @@ class NNCFQuantizer(DiffusersQuantizer): state_dict: Dict[str, Any], **kwargs, ): - module, _ = get_module_from_name(model, param_name) - return module.__class__.__name__.startswith("NNCF") and param_name.endswith(".weight") + return param_name.endswith(".weight") def check_quantized_param(self, *args, **kwargs) -> bool: """ @@ -228,12 +245,6 @@ class NNCFQuantizer(DiffusersQuantizer): layer, tensor_name = get_module_from_name(model, param_name) layer._parameters[tensor_name] = torch.nn.Parameter(param_value).to(device=target_device) # pylint: disable=protected-access - # nncf_padding_value somehow ends up in the meta device with cogvideo even if we don't use init_empty_weights - # set it to the default value if it is in the meta device: - if layer.__class__.__name__ == "NNCFConv2d" and hasattr(layer, "get_padding_value_ref") and hasattr(layer, "_set_padding_value"): - if layer.get_padding_value_ref().device == torch.device("meta"): - layer._set_padding_value(torch.zeros([1])) - split_param_name = param_name.split(".") if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert): layer = nncf_compress_layer( @@ -266,18 +277,13 @@ class NNCFQuantizer(DiffusersQuantizer): keep_in_fp32_modules: List[str] = [], **kwargs, ): - from nncf.torch.nncf_module_replacement import replace_modules_by_nncf_modules - + model.config.quantization_config = self.quantization_config self.modules_to_not_convert = self.quantization_config.modules_to_not_convert if not isinstance(self.modules_to_not_convert, list): self.modules_to_not_convert = [self.modules_to_not_convert] if keep_in_fp32_modules is not None: self.modules_to_not_convert.extend(keep_in_fp32_modules) - model.config.quantization_config = self.quantization_config - with init_empty_weights(): - model, _ = replace_modules_by_nncf_modules(model) - def _process_model_after_weight_loading(self, model, **kwargs): return model @@ -470,28 +476,51 @@ def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTenso def int8_matmul( - input: torch.Tensor, + input: torch.FloatTensor, weight: torch.Tensor, - scale: torch.Tensor, + bias: torch.FloatTensor, + scale: torch.FloatTensor, compressed_weight_shape: torch.Size, -): +) -> torch.FloatTensor: if compressed_weight_shape is not None: weight = unpack_int4_compiled(weight, compressed_weight_shape, transpose=True) return_dtype = input.dtype output_shape = list(input.shape) output_shape[-1] = weight.shape[-1] input, scale = quantize_int8_matmul_input_compiled(input, scale) - return decompress_symmetric_compiled(torch._int_mm(input, weight), scale, return_dtype, output_shape) # pylint: disable=protected-access + result = decompress_symmetric_compiled(torch._int_mm(input, weight), scale, return_dtype, output_shape) # pylint: disable=protected-access + if bias is not None: + result.add_(bias) + return result -class linear_forward_int8_matmul(): - def __func__(self, input) -> torch.FloatTensor: - if self.pre_ops["0"].skip_int8_matmul: - return torch.nn.functional.linear(input, self.weight, self.bias) - result = int8_matmul(input, self.weight, self.pre_ops["0"].scale, getattr(self.pre_ops["0"], "compressed_weight_shape", None)) - if self.bias is not None: - result.add_(self.bias) - return result +def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: + if torch.numel(input[0]) / input[0].shape[-1] < 32: + return torch.nn.functional.linear(input, self.nncf_decompressor(self, return_decompressed_only=True, skip_int8_matmul=True), self.bias) + return int8_matmul(input, self.weight, self.bias, self.nncf_decompressor.scale, getattr(self.nncf_decompressor, "compressed_weight_shape", None)) + + +def quantized_linear_forward(self, input: torch.FloatTensor) -> torch.FloatTensor: + return torch.nn.functional.linear(input, self.nncf_decompressor(self, return_decompressed_only=True), self.bias) + + +def quantized_conv_forward(self, input) -> torch.FloatTensor: + return self._conv_forward(input, self.nncf_decompressor(self, return_decompressed_only=True), self.bias) + + +def quantized_conv_transpose_1d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: + output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 1, self.dilation) + return torch.nn.functional.conv_transpose1d(input, self.nncf_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + + +def quantized_conv_transpose_2d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: + output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 2, self.dilation) + return torch.nn.functional.conv_transpose2d(input, self.nncf_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + + +def quantized_conv_transpose_3d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: + output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 3, self.dilation) + return torch.nn.functional.conv_transpose3d(input, self.nncf_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) class INT8AsymmetricWeightsDecompressor(torch.nn.Module): @@ -530,7 +559,6 @@ class INT8SymmetricWeightsDecompressor(torch.nn.Module): scale: torch.Tensor, result_dtype: torch.dtype, result_shape: torch.Size, - use_int8_matmul: bool, ): super().__init__() self.num_bits = 8 @@ -538,9 +566,6 @@ class INT8SymmetricWeightsDecompressor(torch.nn.Module): self.scale = scale self.result_dtype = result_dtype self.result_shape = result_shape - self.use_int8_matmul = use_int8_matmul - self.skip_int8_matmul = False - self.input_scale = None def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: if debug: @@ -548,17 +573,10 @@ class INT8SymmetricWeightsDecompressor(torch.nn.Module): raise ValueError("Weight values are not in [-128, 127].") return weight.to(dtype=torch.int8) - def forward(self, x, input=None, *args, return_decompressed_only=False): # pylint: disable=unused-argument,keyword-arg-before-vararg - if self.use_int8_matmul: - if input is not None: - if torch.numel(input[0]) / input[0].shape[-1] < 32: - self.skip_int8_matmul = True - else: - self.skip_int8_matmul = False - return - result = decompress_symmetric_compiled(x.weight.transpose(0,1), self.scale.unsqueeze(-1), self.result_dtype, self.result_shape) - else: - result = decompress_symmetric_compiled(x.weight, self.scale, self.result_dtype, self.result_shape) + def forward(self, x, input=None, *args, return_decompressed_only=False, skip_int8_matmul=False): # pylint: disable=unused-argument,keyword-arg-before-vararg + if skip_int8_matmul: + return decompress_int4_symmetric_compiled(x.weight, self.scale.unsqueeze(-1), self.compressed_weight_shape, self.result_dtype, self.result_shape) + result = decompress_symmetric_compiled(x.weight, self.scale, self.result_dtype, self.result_shape) if return_decompressed_only: return result else: @@ -604,7 +622,6 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): compressed_weight_shape: torch.Size, result_dtype: torch.dtype, result_shape: torch.Size, - use_int8_matmul: bool, ): super().__init__() self.num_bits = 4 @@ -613,9 +630,6 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): self.compressed_weight_shape = compressed_weight_shape self.result_dtype = result_dtype self.result_shape = result_shape - self.use_int8_matmul = use_int8_matmul - self.skip_int8_matmul = False - self.input_scale = None def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: if debug: @@ -623,17 +637,10 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): raise ValueError("Tensor values are not in [-8, 7].") return pack_int4(weight.to(dtype=torch.int8)) - def forward(self, x, input=None, *arg, return_decompressed_only=False): # pylint: disable=keyword-arg-before-vararg,unused-argument - if self.use_int8_matmul: - if input is not None: - if torch.numel(input[0]) / input[0].shape[-1] < 32: - self.skip_int8_matmul = True - else: - self.skip_int8_matmul = False - return - result = decompress_int4_symmetric_compiled(x.weight, self.scale.unsqueeze(-1), self.compressed_weight_shape, self.result_dtype, self.result_shape) - else: - result = decompress_int4_symmetric_compiled(x.weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape) + def forward(self, x, input=None, *arg, return_decompressed_only=False, skip_int8_matmul=False): # pylint: disable=keyword-arg-before-vararg,unused-argument + if skip_int8_matmul: + return decompress_int4_symmetric_compiled(x.weight, self.scale.unsqueeze(-1), self.compressed_weight_shape, self.result_dtype, self.result_shape) + result = decompress_int4_symmetric_compiled(x.weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape) if return_decompressed_only: return result else: From cbc1bfe710e905b7d5fa1b246690c93ca53564db Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 01:24:06 +0300 Subject: [PATCH 14/94] Cleanup --- modules/model_quant_nncf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/model_quant_nncf.py b/modules/model_quant_nncf.py index 2e07eced8..2acdbcbbe 100644 --- a/modules/model_quant_nncf.py +++ b/modules/model_quant_nncf.py @@ -575,7 +575,7 @@ class INT8SymmetricWeightsDecompressor(torch.nn.Module): def forward(self, x, input=None, *args, return_decompressed_only=False, skip_int8_matmul=False): # pylint: disable=unused-argument,keyword-arg-before-vararg if skip_int8_matmul: - return decompress_int4_symmetric_compiled(x.weight, self.scale.unsqueeze(-1), self.compressed_weight_shape, self.result_dtype, self.result_shape) + return decompress_symmetric_compiled(x.weight.transpose(0,1), self.scale.unsqueeze(-1), self.result_dtype, self.result_shape) result = decompress_symmetric_compiled(x.weight, self.scale, self.result_dtype, self.result_shape) if return_decompressed_only: return result From 9c2e15433ef9048e288565ed084c7dda857eae0c Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 01:39:09 +0300 Subject: [PATCH 15/94] NNCF set required_packages to None --- modules/model_quant_nncf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/model_quant_nncf.py b/modules/model_quant_nncf.py index 2acdbcbbe..8efe64cbb 100644 --- a/modules/model_quant_nncf.py +++ b/modules/model_quant_nncf.py @@ -209,7 +209,7 @@ class NNCFQuantizer(DiffusersQuantizer): requires_parameters_quantization = True use_keep_in_fp32_modules = True requires_calibration = False - required_packages = ["nncf"] + required_packages = None torch_dtype = None def __init__(self, quantization_config, **kwargs): # pylint: disable=useless-parent-delegation From 4453efee761069525e8cd25a4ce71c87fe74a374 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 02:39:51 +0300 Subject: [PATCH 16/94] Rename NNCF to SDNQ and rename quant schemes --- modules/control/units/controlnet.py | 10 +-- modules/lora/lora_apply.py | 28 +++---- modules/model_quant.py | 70 ++++++++-------- ...odel_quant_nncf.py => model_quant_sdnq.py} | 84 ++++++++----------- modules/model_te.py | 9 +- modules/sd_models_utils.py | 4 +- modules/shared.py | 31 ++++--- 7 files changed, 114 insertions(+), 122 deletions(-) rename modules/{model_quant_nncf.py => model_quant_sdnq.py} (92%) diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index 7a971c340..d2e59348c 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -286,13 +286,13 @@ class ControlNet(): return if self.dtype is not None: self.model.to(self.dtype) - if "ControlNet" in opts.nncf_compress_weights: + if "ControlNet" in opts.sdnq_quantize_weights: try: - log.debug(f'Control {what} model NNCF Compress: id="{model_id}"') - from modules.model_quant import nncf_compress_model - self.model = nncf_compress_model(self.model) + log.debug(f'Control {what} model SDNQ Compress: id="{model_id}"') + from modules.model_quant import sdnq_quantize_model + self.model = sdnq_quantize_model(self.model) except Exception as e: - log.error(f'Control {what} model NNCF Compression failed: id="{model_id}" {e}') + log.error(f'Control {what} model SDNQ Compression failed: id="{model_id}" {e}') elif "ControlNet" in opts.optimum_quanto_weights: try: log.debug(f'Control {what} model Optimum Quanto: id="{model_id}"') diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index 7b00ee389..062413288 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -45,8 +45,8 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n self.network_weights_backup = True else: self.network_weights_backup = weight.clone().to(devices.cpu) - if hasattr(self, "nncf_decompressor"): - self.nncf_decompressor_backup = self.nncf_decompressor.to(devices.cpu) + if hasattr(self, "sdnq_decompressor"): + self.sdnq_decompressor_backup = self.sdnq_decompressor.to(devices.cpu) if bias_backup is None: if getattr(self, 'bias', None) is not None: @@ -79,10 +79,10 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. continue try: t0 = time.time() - if hasattr(self, "nncf_decompressor"): + if hasattr(self, "sdnq_decompressor"): return_device = self.weight.data.device self.weight.data = self.weight.data.to(devices.device) - weight = self.nncf_decompressor.to(devices.device)(self, return_decompressed_only=True) + weight = self.sdnq_decompressor.to(devices.device)(self, return_decompressed_only=True) self.weight.data = self.weight.data.to(return_device) else: weight = self.weight.to(devices.device) # must perform calc on gpu due to performance @@ -139,23 +139,23 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G # weight._quantize(devices.device) / weight.to(device=device) except Exception as e: shared.log.error(f'Network load: type=LoRA quant=bnb cls={self.__class__.__name__} type={self.quant_type} blocksize={self.blocksize} state={vars(self.quant_state)} weight={self.weight} bias={lora_weights} {e}') - elif not bias and hasattr(self, "nncf_decompressor"): + elif not bias and hasattr(self, "sdnq_decompressor"): num_bits = None is_asym_mode = None try: - from modules.model_quant_nncf import nncf_compress_layer - num_bits = self.nncf_decompressor.num_bits - is_asym_mode = self.nncf_decompressor.quantization_mode == "asymmetric" + from modules.model_quant_sdnq import sdnq_quantize_layer + num_bits = self.sdnq_decompressor.num_bits + is_asym_mode = self.sdnq_decompressor.quantization_mode == "asymmetric" self.weight = torch.nn.Parameter(model_weights.to(devices.device), requires_grad=False) - dequant_weight = self.nncf_decompressor(self, return_decompressed_only=True) + dequant_weight = self.sdnq_decompressor(self, return_decompressed_only=True) new_weight = dequant_weight.to(devices.device, dtype=torch.float32) + lora_weights.to(devices.device, dtype=torch.float32) self.weight = torch.nn.Parameter(new_weight, requires_grad=False) - self.nncf_decompressor = None - self = nncf_compress_layer(self, num_bits, is_asym_mode, torch_dtype=devices.dtype, quant_conv=shared.opts.nncf_quantize_conv_layers, group_size=shared.opts.nncf_compress_weights_group_size, use_int8_matmul=shared.opts.nncf_decompress_int8_matmul) + self.sdnq_decompressor = None + self = sdnq_quantize_layer(self, num_bits, is_asym_mode, torch_dtype=devices.dtype, quant_conv=shared.opts.sdnq_quantize_conv_layers, group_size=shared.opts.sdnq_quantize_weights_group_size, use_int8_matmul=shared.opts.sdnq_decompress_int8_matmul) self = self.to(device) del dequant_weight except Exception as e: - shared.log.error(f'Network load: type=LoRA quant=nncf cls={self.__class__.__name__} bits={num_bits} is_asym_mode={is_asym_mode} weight={self.weight} lora_weights={lora_weights} {e}') + shared.log.error(f'Network load: type=LoRA quant=sdnq cls={self.__class__.__name__} bits={num_bits} is_asym_mode={is_asym_mode} weight={self.weight} lora_weights={lora_weights} {e}') else: try: new_weight = model_weights.to(devices.device) + lora_weights.to(devices.device) @@ -217,8 +217,8 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn network_add_weights(self, model_weights=weights_backup, lora_weights=updown, deactivate=deactivate, device=device, bias=False) else: self.weight = torch.nn.Parameter(weights_backup.to(device), requires_grad=False) - if hasattr(self, "nncf_decompressor_backup"): - self.nncf_decompressor = self.nncf_decompressor_backup.to(device) + if hasattr(self, "sdnq_decompressor_backup"): + self.sdnq_decompressor = self.sdnq_decompressor_backup.to(device) if bias_backup is not None: self.bias = None diff --git a/modules/model_quant.py b/modules/model_quant.py index 3b453691d..c0cd6fb9f 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -103,35 +103,35 @@ def create_quanto_config(kwargs = None, allow_quanto: bool = True, module: str = return kwargs -def create_nncf_config(kwargs = None, allow_nncf: bool = True, module: str = 'Model'): +def create_sdnq_config(kwargs = None, allow_sdnq: bool = True, module: str = 'Model', weights_dtype: str = None): from modules import shared - if len(shared.opts.nncf_compress_weights) > 0 and (shared.opts.nncf_compress_mode == 'pre') and allow_nncf: - if 'Model' in shared.opts.nncf_compress_weights or (module is not None and module in shared.opts.nncf_compress_weights) or module == 'any': - from modules.model_quant_nncf import NNCFQuantizer, NNCFConfig - diffusers.quantizers.auto.AUTO_QUANTIZER_MAPPING["nncf"] = NNCFQuantizer - transformers.quantizers.auto.AUTO_QUANTIZER_MAPPING["nncf"] = NNCFQuantizer - diffusers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["nncf"] = NNCFConfig - transformers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["nncf"] = NNCFConfig + if len(shared.opts.sdnq_quantize_weights) > 0 and (shared.opts.sdnq_quantize_mode == 'pre') and allow_sdnq: + if 'Model' in shared.opts.sdnq_quantize_weights or (module is not None and module in shared.opts.sdnq_quantize_weights) or module == 'any': + from modules.model_quant_sdnq import SDNQQuantizer, SDNQConfig + diffusers.quantizers.auto.AUTO_QUANTIZER_MAPPING["sdnq"] = SDNQQuantizer + transformers.quantizers.auto.AUTO_QUANTIZER_MAPPING["sdnq"] = SDNQQuantizer + diffusers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["sdnq"] = SDNQConfig + transformers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["sdnq"] = SDNQConfig - nncf_config = NNCFConfig( - weights_dtype=shared.opts.nncf_compress_weights_mode.lower(), - group_size=shared.opts.nncf_compress_weights_group_size, - use_int8_matmul=shared.opts.nncf_decompress_int8_matmul, + sdnq_config = SDNQConfig( + weights_dtype=weights_dtype if weights_dtype is not None else shared.opts.sdnq_quantize_weights_mode, + group_size=shared.opts.sdnq_quantize_weights_group_size, + use_int8_matmul=shared.opts.sdnq_decompress_int8_matmul, ) - log.debug(f'Quantization: module="{module}" type=nncf dtype={shared.opts.nncf_compress_weights_mode}') + log.debug(f'Quantization: module="{module}" type=sdnq dtype={shared.opts.sdnq_quantize_weights_mode}') if kwargs is None: - return nncf_config + return sdnq_config else: - kwargs['quantization_config'] = nncf_config + kwargs['quantization_config'] = sdnq_config return kwargs return kwargs def check_quant(module: str = ''): from modules import shared - if 'Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization or 'Model' in shared.opts.quanto_quantization or 'Model' in shared.opts.nncf_compress_weights: + if 'Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization or 'Model' in shared.opts.quanto_quantization or 'Model' in shared.opts.sdnq_quantize_weights: return True - if module in shared.opts.bnb_quantization or module in shared.opts.torchao_quantization or module in shared.opts.quanto_quantization or module in shared.opts.nncf_compress_weights: + if module in shared.opts.bnb_quantization or module in shared.opts.torchao_quantization or module in shared.opts.quanto_quantization or module in shared.opts.sdnq_quantize_weights: return True return False @@ -165,10 +165,10 @@ def create_config(kwargs = None, allow: bool = True, module: str = 'Model'): if debug: log.trace(f'Quantization: type=quanto config={kwargs.get("quantization_config", None)}') return kwargs - kwargs = create_nncf_config(kwargs, allow_nncf=allow, module=module) + kwargs = create_sdnq_config(kwargs, allow_sdnq=allow, module=module) if kwargs is not None and 'quantization_config' in kwargs: if debug: - log.trace(f'Quantization: type=nncf config={kwargs.get("quantization_config", None)}') + log.trace(f'Quantization: type=sdnq config={kwargs.get("quantization_config", None)}') return kwargs return kwargs @@ -298,18 +298,18 @@ def apply_layerwise(sd_model, quiet:bool=False): log.error(f'Quantization: type=layerwise {e}') -def nncf_compress_model(model, op=None, sd_model=None, do_gc=True): +def sdnq_quantize_model(model, op=None, sd_model=None, do_gc=True): global quant_last_model_name, quant_last_model_device # pylint: disable=global-statement from modules import devices, shared - from modules.model_quant_nncf import apply_nncf_to_module + from modules.model_quant_sdnq import apply_sdnq_to_module model.eval() if model.__class__.__name__ in {"T5EncoderModel", "UMT5EncoderModel"}: import torch - from modules.model_quant_nncf import NNCF_T5DenseGatedActDense # T5DenseGatedActDense uses fp32 + from modules.model_quant_sdnq import SDNQ_T5DenseGatedActDense # T5DenseGatedActDense uses fp32 for i in range(len(model.encoder.block)): - model.encoder.block[i].layer[1].DenseReluDense = NNCF_T5DenseGatedActDense( + model.encoder.block[i].layer[1].DenseReluDense = SDNQ_T5DenseGatedActDense( model.encoder.block[i].layer[1].DenseReluDense, dtype=torch.float32 if devices.dtype != torch.bfloat16 else torch.bfloat16 ) @@ -318,15 +318,15 @@ def nncf_compress_model(model, op=None, sd_model=None, do_gc=True): if hasattr(model, "get_input_embeddings"): backup_embeddings = copy.deepcopy(model.get_input_embeddings()) - num_bits = 8 if shared.opts.nncf_compress_weights_mode in {"INT8", "INT8_SYM", "INT8_ASYM"} else 4 - is_asym_mode = shared.opts.nncf_compress_weights_mode in {"INT8", "INT4", "INT8_ASYM", "INT4_ASYM"} - model = apply_nncf_to_module(model, num_bits, is_asym_mode, quant_conv=shared.opts.nncf_quantize_conv_layers) - model.quantization_method = 'NNCF' + num_bits = 8 if shared.opts.sdnq_quantize_weights_mode in {"int8", "uint8"} else 4 + is_asym_mode = shared.opts.sdnq_quantize_weights_mode in {"uint8", "uint4"} + model = apply_sdnq_to_module(model, num_bits, is_asym_mode, quant_conv=shared.opts.sdnq_quantize_conv_layers) + model.quantization_method = 'SDNQ' if hasattr(model, "set_input_embeddings") and backup_embeddings is not None: model.set_input_embeddings(backup_embeddings) - if op is not None and shared.opts.nncf_quantize_shuffle_weights: + if op is not None and shared.opts.sdnq_quantize_shuffle_weights: if quant_last_model_name is not None: if "." in quant_last_model_name: last_model_names = quant_last_model_name.split(".") @@ -347,14 +347,14 @@ def nncf_compress_model(model, op=None, sd_model=None, do_gc=True): return model -def nncf_compress_weights(sd_model): +def sdnq_quantize_weights(sd_model): try: t0 = time.time() from modules import shared, devices, sd_models - log.info(f"Quantization: type=NNCF modules={shared.opts.nncf_compress_weights}") + log.info(f"Quantization: type=SDNQ modules={shared.opts.sdnq_quantize_weights}") global quant_last_model_name, quant_last_model_device # pylint: disable=global-statement - sd_model = sd_models.apply_function_to_model(sd_model, nncf_compress_model, shared.opts.nncf_compress_weights, op="nncf") + sd_model = sd_models.apply_function_to_model(sd_model, sdnq_quantize_model, shared.opts.sdnq_quantize_weights, op="sdnq") if quant_last_model_name is not None: if "." in quant_last_model_name: last_model_names = quant_last_model_name.split(".") @@ -366,9 +366,9 @@ def nncf_compress_weights(sd_model): quant_last_model_device = None t1 = time.time() - log.info(f"Quantization: type=NNCF time={t1-t0:.2f}") + log.info(f"Quantization: type=SDNQ time={t1-t0:.2f}") except Exception as e: - log.warning(f"Quantization: type=NNCF {e}") + log.warning(f"Quantization: type=SDNQ {e}") return sd_model @@ -529,8 +529,8 @@ def get_dit_args(load_config:dict={}, module:str=None, device_map:bool=False, al def do_post_load_quant(sd_model): from modules import shared - if shared.opts.nncf_compress_weights and shared.opts.nncf_compress_mode == 'post' and not (shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx"): - sd_model = nncf_compress_weights(sd_model) + if shared.opts.sdnq_quantize_weights and shared.opts.sdnq_quantize_mode == 'post': + sd_model = sdnq_quantize_weights(sd_model) if shared.opts.optimum_quanto_weights: sd_model = optimum_quanto_weights(sd_model) if shared.opts.torchao_quantization and shared.opts.torchao_quantization_mode == 'post': diff --git a/modules/model_quant_nncf.py b/modules/model_quant_sdnq.py similarity index 92% rename from modules/model_quant_nncf.py rename to modules/model_quant_sdnq.py index 8efe64cbb..9ed00916b 100644 --- a/modules/model_quant_nncf.py +++ b/modules/model_quant_sdnq.py @@ -19,17 +19,10 @@ torch_dtype_dict = { "int4": CustomDtype.INT4, "uint4": CustomDtype.INT4, } -weights_dtype_dict = { - "int8_asym": "uint8", - "int8_sym": "int8", - "int4_asym": "uint4", - "int4_sym": "int4", - "int8": "uint8", - "int4": "uint4", -} -linear_types = ["NNCFLinear", "Linear"] -conv_types = ["NNCFConv1d", "NNCFConv2d", "NNCFConv3d", "Conv1d", "Conv2d", "Conv3d"] -conv_transpose_types = ["NNCFConvTranspose1d", "NNCFConvTranspose2d", "NNCFConvTranspose3d", "ConvTranspose1d", "ConvTranspose2d", "ConvTranspose3d"] + +linear_types = ["Linear"] +conv_types = ["Conv1d", "Conv2d", "Conv3d"] +conv_transpose_types = ["ConvTranspose1d", "ConvTranspose2d", "ConvTranspose3d"] allowed_types = [] allowed_types.extend(linear_types) allowed_types.extend(conv_types) @@ -37,10 +30,10 @@ allowed_types.extend(conv_transpose_types) class QuantizationMethod(str, Enum): - NNCF = "nncf" + SDNQ = "sdnq" -def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_conv=False, group_size=0, use_int8_matmul=False, param_name=None): # pylint: disable=unused-argument +def sdnq_quantize_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_conv=False, group_size=0, use_int8_matmul=False, param_name=None): # pylint: disable=unused-argument layer_class_name = layer.__class__.__name__ if layer_class_name in allowed_types: is_conv_type = False @@ -106,7 +99,7 @@ def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c zero_point = None compressed_weight = quantize_int(layer.weight, scale, zero_point, is_asym_mode, num_bits) - if not shared.opts.nncf_decompress_fp32: + if not shared.opts.sdnq_decompress_fp32: scale = scale.to(torch_dtype) if zero_point is not None: zero_point = zero_point.to(torch_dtype) @@ -152,7 +145,7 @@ def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c layer.weight.requires_grad = False layer.weight.data = compressed_weight - layer.nncf_decompressor = decompressor + layer.sdnq_decompressor = decompressor if is_linear_type: if use_int8_matmul: @@ -172,38 +165,29 @@ def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c return layer -def apply_nncf_to_module(model, num_bits, is_asym_mode, quant_conv=False): +def apply_sdnq_to_module(model, num_bits, is_asym_mode, quant_conv=False): has_children = list(model.children()) if not has_children: return model for param_name, module in model.named_children(): if hasattr(module, "weight") and module.weight is not None: - module = nncf_compress_layer( + module = sdnq_quantize_layer( module, num_bits, is_asym_mode, torch_dtype=devices.dtype, quant_conv=quant_conv, - group_size=shared.opts.nncf_compress_weights_group_size, - use_int8_matmul=shared.opts.nncf_decompress_int8_matmul, + group_size=shared.opts.sdnq_quantize_weights_group_size, + use_int8_matmul=shared.opts.sdnq_decompress_int8_matmul, param_name=param_name, ) - module = apply_nncf_to_module(module, num_bits, is_asym_mode, quant_conv=quant_conv) + module = apply_sdnq_to_module(module, num_bits, is_asym_mode, quant_conv=quant_conv) return model -def nncf_send_to_device(model, device): - for child in model.children(): - if "WeightsDecompressor" in child.__class__.__name__: - child.scale = child.scale.to(device) - if hasattr(child, "zero_point"): - child.zero_point = child.zero_point.to(device) - nncf_send_to_device(child, device) - - -class NNCFQuantizer(DiffusersQuantizer): +class SDNQQuantizer(DiffusersQuantizer): r""" - Diffusers Quantizer for NNCF + Diffusers Quantizer for SDNQ """ requires_parameters_quantization = True @@ -247,7 +231,7 @@ class NNCFQuantizer(DiffusersQuantizer): split_param_name = param_name.split(".") if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert): - layer = nncf_compress_layer( + layer = sdnq_quantize_layer( layer, self.quantization_config.num_bits, self.quantization_config.is_asym_mode, @@ -321,14 +305,14 @@ class NNCFQuantizer(DiffusersQuantizer): @dataclass -class NNCFConfig(QuantizationConfigMixin): +class SDNQConfig(QuantizationConfigMixin): """ This is a wrapper class about all possible attributes and features that you can play with a model that has been - loaded using `nncf`. + loaded using `sdnq`. Args: weights_dtype (`str`, *optional*, defaults to `"int8"`): - The target dtype for the weights after quantization. Supported values are ("int8", "int8_sym", "int4", "int4_sym") + The target dtype for the weights after quantization. Supported values are ("int8", "uint8", "int4", "uint4") modules_to_not_convert (`list`, *optional*, default to `None`): The list of modules to not quantize, useful for quantizing models that explicitly require to have some modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers). @@ -336,23 +320,21 @@ class NNCFConfig(QuantizationConfigMixin): def __init__( # pylint: disable=super-init-not-called self, - weights_dtype: str = "int8_sym", + weights_dtype: str = "int8", group_size: int = 0, use_int8_matmul: bool = False, modules_to_not_convert: Optional[List[str]] = None, **kwargs, # pylint: disable=unused-argument ): - self.quant_method = QuantizationMethod.NNCF - self.weights_dtype = weights_dtype_dict[weights_dtype.lower()] + self.weights_dtype = weights_dtype + self.quant_method = QuantizationMethod.SDNQ self.group_size = group_size self.use_int8_matmul = use_int8_matmul self.modules_to_not_convert = modules_to_not_convert - - self.post_init() - self.num_bits = 8 if self.weights_dtype in {"int8", "uint8"} else 4 self.is_asym_mode = self.weights_dtype in {"uint8", "uint4"} self.is_integer = True + self.post_init() def post_init(self): r""" @@ -363,7 +345,7 @@ class NNCFConfig(QuantizationConfigMixin): raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights_dtype}") -class NNCF_T5DenseGatedActDense(torch.nn.Module): # forward can't find what self is without creating a class +class SDNQ_T5DenseGatedActDense(torch.nn.Module): # forward can't find what self is without creating a class def __init__(self, T5DenseGatedActDense, dtype): super().__init__() self.wi_0 = T5DenseGatedActDense.wi_0 @@ -496,31 +478,31 @@ def int8_matmul( def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: if torch.numel(input[0]) / input[0].shape[-1] < 32: - return torch.nn.functional.linear(input, self.nncf_decompressor(self, return_decompressed_only=True, skip_int8_matmul=True), self.bias) - return int8_matmul(input, self.weight, self.bias, self.nncf_decompressor.scale, getattr(self.nncf_decompressor, "compressed_weight_shape", None)) + return torch.nn.functional.linear(input, self.sdnq_decompressor(self, return_decompressed_only=True, skip_int8_matmul=True), self.bias) + return int8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale, getattr(self.sdnq_decompressor, "compressed_weight_shape", None)) def quantized_linear_forward(self, input: torch.FloatTensor) -> torch.FloatTensor: - return torch.nn.functional.linear(input, self.nncf_decompressor(self, return_decompressed_only=True), self.bias) + return torch.nn.functional.linear(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias) def quantized_conv_forward(self, input) -> torch.FloatTensor: - return self._conv_forward(input, self.nncf_decompressor(self, return_decompressed_only=True), self.bias) + return self._conv_forward(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias) def quantized_conv_transpose_1d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 1, self.dilation) - return torch.nn.functional.conv_transpose1d(input, self.nncf_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + return torch.nn.functional.conv_transpose1d(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) def quantized_conv_transpose_2d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 2, self.dilation) - return torch.nn.functional.conv_transpose2d(input, self.nncf_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + return torch.nn.functional.conv_transpose2d(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) def quantized_conv_transpose_3d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 3, self.dilation) - return torch.nn.functional.conv_transpose3d(input, self.nncf_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + return torch.nn.functional.conv_transpose3d(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) class INT8AsymmetricWeightsDecompressor(torch.nn.Module): @@ -647,7 +629,7 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): x.weight = result -if shared.opts.nncf_decompress_compile: +if shared.opts.sdnq_decompress_compile: try: torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit) # pylint: disable=protected-access decompress_asymmetric_compiled = torch.compile(decompress_asymmetric, fullgraph=True) @@ -662,7 +644,7 @@ if shared.opts.nncf_decompress_compile: quantize_int8_matmul_input_compiled = torch.compile(quantize_int8_matmul_input, fullgraph=True) unpack_int4_compiled = torch.compile(unpack_int4, fullgraph=True) except Exception as e: - shared.log.warning(f"Quantization: type=nncf Decompress using torch.compile is not available: {e}") + shared.log.warning(f"Quantization: type=sdnq Decompress using torch.compile is not available: {e}") decompress_asymmetric_compiled = decompress_asymmetric decompress_symmetric_compiled = decompress_symmetric decompress_int4_asymmetric_compiled = decompress_int4_asymmetric diff --git a/modules/model_te.py b/modules/model_te.py index d853eaf99..6472721ee 100644 --- a/modules/model_te.py +++ b/modules/model_te.py @@ -70,8 +70,13 @@ def load_t5(name=None, cache_dir=None): t5 = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder='text_encoder_3', quantization_config=quantization_config, cache_dir=cache_dir, torch_dtype=devices.dtype) elif 'int8' in name.lower(): - from modules.model_quant import create_nncf_config - quantization_config = create_nncf_config(kwargs=None, allow_nncf=True, module="any") + from modules.model_quant import create_sdnq_config + quantization_config = create_sdnq_config(kwargs=None, allow_sdnq=True, module='any', weights_dtype='int8') + t5 = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder='text_encoder_3', quantization_config=quantization_config, cache_dir=cache_dir, torch_dtype=devices.dtype) + + elif 'uint4' in name.lower(): + from modules.model_quant import create_sdnq_config + quantization_config = create_sdnq_config(kwargs=None, allow_sdnq=True, module='any', weights_dtype='uint4') t5 = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder='text_encoder_3', quantization_config=quantization_config, cache_dir=cache_dir, torch_dtype=devices.dtype) elif 'qint4' in name.lower(): diff --git a/modules/sd_models_utils.py b/modules/sd_models_utils.py index 2f10ad074..0e1b72f4c 100644 --- a/modules/sd_models_utils.py +++ b/modules/sd_models_utils.py @@ -161,10 +161,10 @@ def apply_function_to_model(sd_model, function, options, op=None): sd_model.decoder = None sd_model.decoder = sd_model.decoder_pipe.decoder = function(sd_model.decoder_pipe.decoder, op="decoder_pipe.decoder", sd_model=sd_model) if hasattr(sd_model, 'prior_pipe') and hasattr(sd_model.prior_pipe, 'prior'): - if op == "nncf" and "StableCascade" in sd_model.__class__.__name__: # fixes dtype errors + if op == "sdnq" and "StableCascade" in sd_model.__class__.__name__: # fixes dtype errors backup_clip_txt_pooled_mapper = copy.deepcopy(sd_model.prior_pipe.prior.clip_txt_pooled_mapper) sd_model.prior_pipe.prior = function(sd_model.prior_pipe.prior, op="prior_pipe.prior", sd_model=sd_model) - if op == "nncf" and "StableCascade" in sd_model.__class__.__name__: + if op == "sdnq" and "StableCascade" in sd_model.__class__.__name__: sd_model.prior_pipe.prior.clip_txt_pooled_mapper = backup_clip_txt_pooled_mapper if "TE" in options: if hasattr(sd_model, 'text_encoder') and hasattr(sd_model.text_encoder, 'config'): diff --git a/modules/shared.py b/modules/shared.py index cba39b41d..91def4d6e 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -527,19 +527,16 @@ options_templates.update(options_section(('quantization', "Quantization Settings "bnb_quantization_type": OptionInfo("nf4", "Quantization type", gr.Dropdown, {"choices": ['nf4', 'fp8', 'fp4'], "visible": native}), "bnb_quantization_storage": OptionInfo("uint8", "Backend storage", gr.Dropdown, {"choices": ["float16", "float32", "int8", "uint8", "float64", "bfloat16"], "visible": native}), - "nncf_compress_sep": OptionInfo("

NNCF: Neural Network Compression Framework

", "", gr.HTML), - "nncf_compress_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), - "nncf_compress_mode": OptionInfo("post", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native and not cmd_opts.use_openvino}), - "nncf_compress_weights_mode": OptionInfo("INT8_SYM", "Quantization type", gr.Dropdown, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'] if cmd_opts.use_openvino else ['INT8', 'INT8_SYM', 'INT4', 'INT4_SYM']}), - "nncf_compress_weights_raito": OptionInfo(0, "Compress ratio", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}), - "nncf_compress_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), - "nncf_quantize": OptionInfo([], "OpenVINO enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "TE"], "visible": cmd_opts.use_openvino}), - "nncf_quantize_mode": OptionInfo("INT8", "OpenVINO activations mode", gr.Dropdown, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}), - "nncf_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native and not cmd_opts.use_openvino}), - "nncf_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native and not cmd_opts.use_openvino}), - "nncf_decompress_compile": OptionInfo(devices.has_triton(), "Decompress using torch.compile", gr.Checkbox, {"visible": native and not cmd_opts.use_openvino}), - "nncf_decompress_int8_matmul": OptionInfo(False, "Use direct INT8 MatMul", gr.Checkbox, {"visible": native and not cmd_opts.use_openvino}), - "nncf_quantize_shuffle_weights": OptionInfo(False, "Shuffle weights in post mode", gr.Checkbox, {"visible": native and not cmd_opts.use_openvino}), + "sdnq_quantize_sep": OptionInfo("

SDNQ: SDNext Quantization

", "", gr.HTML), + "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), + "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native}), + "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ['int8', 'uint8', 'int4', 'uint4'], "visible": native}), + "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), + "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), + "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), + "sdnq_decompress_compile": OptionInfo(devices.has_triton(), "Decompress using torch.compile", gr.Checkbox, {"visible": native}), + "sdnq_decompress_int8_matmul": OptionInfo(False, "Use direct INT8 MatMul", gr.Checkbox, {"visible": native}), + "sdnq_quantize_shuffle_weights": OptionInfo(False, "Shuffle weights in post mode", gr.Checkbox, {"visible": native}), "quanto_quantization_sep": OptionInfo("

Optimum Quanto

", "", gr.HTML), "quanto_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), @@ -565,6 +562,14 @@ options_templates.update(options_section(('quantization', "Quantization Settings "nunchaku_quantization": OptionInfo([], "SVDQuant enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "nunchaku_attention": OptionInfo(False, "Nunchaku attention", gr.Checkbox, {"visible": native}), "nunchaku_offload": OptionInfo(False, "Nunchaku offloading", gr.Checkbox, {"visible": native}), + + "nncf_compress_sep": OptionInfo("

NNCF: Neural Network Compression Framework

", "", gr.HTML, {"visible": cmd_opts.use_openvino}), + "nncf_compress_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": cmd_opts.use_openvino}), + "nncf_compress_weights_mode": OptionInfo("INT8_SYM", "Quantization type", gr.Dropdown, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'], "visible": cmd_opts.use_openvino}), + "nncf_compress_weights_raito": OptionInfo(0, "Compress ratio", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}), + "nncf_compress_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": cmd_opts.use_openvino}), + "nncf_quantize": OptionInfo([], "Static Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "TE"], "visible": cmd_opts.use_openvino}), + "nncf_quantize_mode": OptionInfo("INT8", "OpenVINO activations mode", gr.Dropdown, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}), })) options_templates.update(options_section(('advanced', "Pipeline Modifiers"), { From 17df7ba83b3586d393fb32a6b807bb8168b899b1 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 02:41:29 +0300 Subject: [PATCH 17/94] Cleanup whitespace --- scripts/prompt_enhance.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index fdfdfa86c..e1b9c5502 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -246,7 +246,7 @@ class Script(scripts.Script): thinking = thinking or self.options.thinking_mode sample = sample if sample is not None else self.options.do_sample nsfw = nsfw if nsfw is not None else True # Default nsfw to True if not provided - + while self.busy: time.sleep(0.1) self.load(model) @@ -257,7 +257,7 @@ class Script(scripts.Script): return prompt prompt_text, networks = self.extract(prompt) # Use prompt_text after extraction debug_log(f'Prompt enhance: networks={networks}') - + current_image = None try: if image is not None and isinstance(image, gr.Image): @@ -268,10 +268,10 @@ class Script(scripts.Script): current_image = None except Exception: current_image = None - + has_system = system is not None and len(system) > 4 mode = 'custom' if has_system else '' - + if current_image is not None and isinstance(current_image, Image.Image): if not self.tokenizer.is_processor: shared.log.error('Prompt enhance: image not supported by model') @@ -514,4 +514,4 @@ class Script(scripts.Script): ) p.extra_generation_params['LLM'] = llm_model shared.state.end() - + From e314a7ca19ebe236614625d2ec0aab5af9cd9531 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 02:49:12 +0300 Subject: [PATCH 18/94] Update changelog --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ae633df4..d87182822 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Change Log for SD.Next +## Update for 2025-05-25 + +- **SDNQ Quantization** + - Renamed `NNCF` to `SDNQ` as NNCF implementation in SDNext re-implemented and optimized enough to not use any imports from NNCF anymore + - Renamed quantization scheme names to the underlying dtype names instead of NNCF names + - `INT8_SYM` -> `int8` + - `INT8` -> `uint8` + - `INT4_SYM` -> `int4` + - `INT4` -> `uint4` + - Use per token input quant with int8 matmul + - Implement better layer hijacks + - Fix Conv quant + ## Update for 2025-05-17 *Curious how your system is performing?* From 5fcd0be79c223f6c3aee9b69b1077e7f3650dd9d Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 02:50:05 +0300 Subject: [PATCH 19/94] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d87182822..01d878f29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - `INT8` -> `uint8` - `INT4_SYM` -> `int4` - `INT4` -> `uint4` + - Set the default quant mode to `post` - Use per token input quant with int8 matmul - Implement better layer hijacks - Fix Conv quant From 46e9a9a631d821e9a387b3da299895f8bdfc61b4 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 03:03:17 +0300 Subject: [PATCH 20/94] IPEX disable Dynamic Attention by default on PyTorch 2.7 --- CHANGELOG.md | 5 ++++- installer.py | 4 ---- modules/intel/ipex/__init__.py | 4 ++-- modules/intel/ipex/diffusers.py | 2 +- modules/intel/ipex/hijacks.py | 27 +++++++++++++++------------ 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01d878f29..346f081e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,13 @@ - `INT8` -> `uint8` - `INT4_SYM` -> `int4` - `INT4` -> `uint4` - - Set the default quant mode to `post` + - Set the default quant mode to `pre` - Use per token input quant with int8 matmul - Implement better layer hijacks - Fix Conv quant +- **IPEX** + - Disabe Dynamic Attention by default on PyTorch 2.7 + - Remove GradScaler hijack and use torch.amp.GradScaler instead ## Update for 2025-05-17 diff --git a/installer.py b/installer.py index 769256404..8ee43a780 100644 --- a/installer.py +++ b/installer.py @@ -727,10 +727,6 @@ def install_ipex(torch_command): if os.environ.get("PYTORCH_ENABLE_XPU_FALLBACK", None) is None: os.environ.setdefault('PYTORCH_ENABLE_XPU_FALLBACK', '1') # CPU fallback for unsupported ops - if os.environ.get('IPEX_FORCE_ATTENTION_SLICE', None) is None: - # XPU PyTorch doesn't support Flash Atten or Memory Atten yet so Battlemage goes OOM without this - os.environ.setdefault('IPEX_FORCE_ATTENTION_SLICE', '1') - # FP64 emulation causes random UR Errors #if os.environ.get("OverrideDefaultFP64Settings", None) is None: # os.environ.setdefault('OverrideDefaultFP64Settings', '1') diff --git a/modules/intel/ipex/__init__.py b/modules/intel/ipex/__init__.py index b0fcb655e..369367ef8 100644 --- a/modules/intel/ipex/__init__.py +++ b/modules/intel/ipex/__init__.py @@ -193,10 +193,10 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.ipc_collect = lambda *args, **kwargs: None torch.cuda.utilization = lambda *args, **kwargs: 0 - device_supports_fp64, can_allocate_plus_4gb = ipex_hijacks() + device_supports_fp64 = ipex_hijacks() try: from .diffusers import ipex_diffusers - ipex_diffusers(device_supports_fp64=device_supports_fp64, can_allocate_plus_4gb=can_allocate_plus_4gb) + ipex_diffusers(device_supports_fp64=device_supports_fp64) except Exception: # pylint: disable=broad-exception-caught pass torch.cuda.is_xpu_hijacked = True diff --git a/modules/intel/ipex/diffusers.py b/modules/intel/ipex/diffusers.py index 77a2fdac9..033b74cbe 100644 --- a/modules/intel/ipex/diffusers.py +++ b/modules/intel/ipex/diffusers.py @@ -81,7 +81,7 @@ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos, output_type="np"): return emb -def ipex_diffusers(device_supports_fp64=False, can_allocate_plus_4gb=False): +def ipex_diffusers(device_supports_fp64=False): diffusers.utils.torch_utils.fourier_filter = fourier_filter if not device_supports_fp64: # get around lazy imports diff --git a/modules/intel/ipex/hijacks.py b/modules/intel/ipex/hijacks.py index 8c308f921..0064e0695 100644 --- a/modules/intel/ipex/hijacks.py +++ b/modules/intel/ipex/hijacks.py @@ -9,16 +9,19 @@ from modules import devices, errors torch_version = float(torch.__version__[:3]) device_supports_fp64 = torch.xpu.has_fp64_dtype() if hasattr(torch.xpu, "has_fp64_dtype") else torch.xpu.get_device_properties(devices.device).has_fp64 -if os.environ.get('IPEX_FORCE_ATTENTION_SLICE', '0') == '0' and (torch.xpu.get_device_properties(devices.device).total_memory / 1024 / 1024 / 1024) > 4.1: - try: - x = torch.ones((33000,33000), dtype=torch.float32, device=devices.device) - del x - torch.xpu.empty_cache() - can_allocate_plus_4gb = True - except Exception: - can_allocate_plus_4gb = False +if os.environ.get('IPEX_FORCE_ATTENTION_SLICE', '0') == '0': + if torch_version >= 2.7: + use_dynamic_attention = False # torch 2.7 has flash atten support + elif (torch.xpu.get_device_properties(devices.device).total_memory / 1024 / 1024 / 1024) > 4.1: + try: + x = torch.ones((33000,33000), dtype=torch.float32, device=devices.device) + del x + torch.xpu.empty_cache() + use_dynamic_attention = False + except Exception: + use_dynamic_attention = True else: - can_allocate_plus_4gb = bool(os.environ.get('IPEX_FORCE_ATTENTION_SLICE', '0') == '-1') + use_dynamic_attention = bool(os.environ.get('IPEX_FORCE_ATTENTION_SLICE', '0') == '1') # pylint: disable=protected-access, missing-function-docstring, line-too-long, unnecessary-lambda, no-else-return @@ -122,7 +125,7 @@ def as_tensor(data, dtype=None, device=None): return original_as_tensor(data, dtype=dtype, device=device) -if can_allocate_plus_4gb: +if not use_dynamic_attention: original_scaled_dot_product_attention = torch.nn.functional.scaled_dot_product_attention else: # 32 bit attention workarounds for Alchemist: @@ -388,7 +391,7 @@ class torch_Generator(original_torch_Generator): # Hijack Functions: def ipex_hijacks(): - global device_supports_fp64, can_allocate_plus_4gb + global device_supports_fp64 if torch_version >= 2.4: torch.UntypedStorage.cuda = UntypedStorage_cuda torch.UntypedStorage.to = UntypedStorage_to @@ -456,4 +459,4 @@ def ipex_hijacks(): torch.cuda.amp.common = nullcontext() torch.cuda.amp.common.amp_definitely_not_available = lambda: False - return device_supports_fp64, can_allocate_plus_4gb + return device_supports_fp64 From 3f8ae754a03cc14aafd3c0857827f376ebcf9817 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 03:35:25 +0300 Subject: [PATCH 21/94] Update readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c93ec9fd8..524d32c0c 100644 --- a/README.md +++ b/README.md @@ -31,11 +31,11 @@ All individual features are not listed here, instead check [ChangeLog](CHANGELOG - Multiple [diffusion models](https://vladmandic.github.io/sdnext-docs/Model-Support/)! - Built-in Control for Text, Image, Batch and Video processing! - Multiplatform! - â–¹ **Windows | Linux | MacOS | nVidia CUDA | AMD ROCm | IntelArc/IPEX | DirectML | OpenVINO | ONNX+Olive | ZLUDA** + â–¹ **Windows | Linux | MacOS | nVidia CUDA | AMD ROCm | Intel Arc / IPEX XPU | DirectML | OpenVINO | ONNX+Olive | ZLUDA** - Platform specific autodetection and tuning performed on install -- Optimized processing with latest `torch` developments with built-in support for model compile, quantize and compress +- Optimized processing with latest `torch` developments with built-in support for model compile and quantize Compile backends: *Triton | StableFast | DeepCache | OneDiff | TeaCache | etc.* - Quantization and compression methods: *BitsAndBytes | TorchAO | Optimum-Quanto | NNCF* + Quantization methods: *BitsAndBytes | SDNQ | Optimum-Quanto | TorchAO* - **Interrogate/Captioning** with 150+ **OpenCLiP** models and 20+ built-in **VLMs** - Built-in queue management - Built in installer with automatic updates and dependency management From 4ad404182d42068b8b0213b395c231a16e03f322 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 04:17:22 +0300 Subject: [PATCH 22/94] cleanup --- modules/model_quant_sdnq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 9ed00916b..0cfa48dfa 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -477,7 +477,7 @@ def int8_matmul( def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: - if torch.numel(input[0]) / input[0].shape[-1] < 32: + if torch.numel(input) / input.shape[-1] < 32: return torch.nn.functional.linear(input, self.sdnq_decompressor(self, return_decompressed_only=True, skip_int8_matmul=True), self.bias) return int8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale, getattr(self.sdnq_decompressor, "compressed_weight_shape", None)) From d2159af10e27c3783676f1e2384ad00f4bf18e7e Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 04:24:28 +0300 Subject: [PATCH 23/94] cleanup --- modules/model_quant_sdnq.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 0cfa48dfa..c53a757ec 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -501,8 +501,8 @@ def quantized_conv_transpose_2d_forward(self, input: torch.FloatTensor, output_s def quantized_conv_transpose_3d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: - output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 3, self.dilation) - return torch.nn.functional.conv_transpose3d(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 3, self.dilation) + return torch.nn.functional.conv_transpose3d(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) class INT8AsymmetricWeightsDecompressor(torch.nn.Module): From 91bb07f6505bdf97503a259de18c36a98ac1a5b8 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 15:51:53 +0300 Subject: [PATCH 24/94] SDNQ remove unused args and simplify decompressors --- modules/lora/lora_apply.py | 10 ++--- modules/model_quant_sdnq.py | 74 +++++++++++++++---------------------- 2 files changed, 32 insertions(+), 52 deletions(-) diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index 062413288..ff9aa64c4 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -80,10 +80,7 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. try: t0 = time.time() if hasattr(self, "sdnq_decompressor"): - return_device = self.weight.data.device - self.weight.data = self.weight.data.to(devices.device) - weight = self.sdnq_decompressor.to(devices.device)(self, return_decompressed_only=True) - self.weight.data = self.weight.data.to(return_device) + weight = self.sdnq_decompressor.to(devices.device)(self.weight.to(devices.device)) else: weight = self.weight.to(devices.device) # must perform calc on gpu due to performance updown, ex_bias = module.calc_updown(weight) @@ -145,9 +142,8 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G try: from modules.model_quant_sdnq import sdnq_quantize_layer num_bits = self.sdnq_decompressor.num_bits - is_asym_mode = self.sdnq_decompressor.quantization_mode == "asymmetric" - self.weight = torch.nn.Parameter(model_weights.to(devices.device), requires_grad=False) - dequant_weight = self.sdnq_decompressor(self, return_decompressed_only=True) + is_asym_mode = self.sdnq_decompressor.is_asym_mode + dequant_weight = self.sdnq_decompressor.to(devices.device)(model_weights.to(devices.device)) new_weight = dequant_weight.to(devices.device, dtype=torch.float32) + lora_weights.to(devices.device, dtype=torch.float32) self.weight = torch.nn.Parameter(new_weight, requires_grad=False) self.sdnq_decompressor = None diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index c53a757ec..dd088bc33 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -404,15 +404,21 @@ def decompress_asymmetric(input: torch.Tensor, scale: torch.Tensor, zero_point: return result -def decompress_symmetric(input: torch.Tensor, scale: torch.Tensor, dtype: torch.dtype, result_shape: torch.Size) -> torch.Tensor: - result = input.to(dtype=scale.dtype).mul_(scale).to(dtype=dtype) +def decompress_symmetric(input: torch.Tensor, scale: torch.Tensor, dtype: torch.dtype, result_shape: torch.Size, skip_int8_matmul: bool = False) -> torch.Tensor: + if skip_int8_matmul: + result = input.transpose(0,1).to(dtype=scale.dtype).mul_(scale.unsqueeze(-1)).to(dtype=dtype) + else: + result = input.to(dtype=scale.dtype).mul_(scale).to(dtype=dtype) if result_shape is not None: result = result.reshape(result_shape) return result -def decompress_int4_asymmetric(input: torch.Tensor, scale: torch.Tensor, zero_point: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size) -> torch.Tensor: - return decompress_asymmetric(unpack_uint4(input, shape), scale, zero_point, dtype, result_shape) +def decompress_int4_asymmetric(input: torch.Tensor, scale: torch.Tensor, zero_point: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, skip_int8_matmul: bool = False) -> torch.Tensor: + if skip_int8_matmul: + return decompress_asymmetric(unpack_uint4(input, shape), scale.unsqueeze(-1), zero_point, dtype, result_shape) + else: + return decompress_asymmetric(unpack_uint4(input, shape), scale, zero_point, dtype, result_shape) def decompress_int4_symmetric(input: torch.Tensor, scale: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size) -> torch.Tensor: @@ -434,10 +440,8 @@ def pack_int4(tensor: torch.Tensor) -> torch.Tensor: return pack_uint4(tensor.to(dtype=torch.uint8)) -def unpack_uint4(packed_tensor: torch.Tensor, shape: torch.Size, transpose: Optional[bool] = False) -> torch.Tensor: +def unpack_uint4(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: result = torch.stack((torch.bitwise_and(packed_tensor, 15), torch.bitwise_right_shift(packed_tensor, 4)), dim=-1).reshape(shape) - if transpose: - result = result.transpose(0,1) return result @@ -478,31 +482,31 @@ def int8_matmul( def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: if torch.numel(input) / input.shape[-1] < 32: - return torch.nn.functional.linear(input, self.sdnq_decompressor(self, return_decompressed_only=True, skip_int8_matmul=True), self.bias) + return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_int8_matmul=True), self.bias) return int8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale, getattr(self.sdnq_decompressor, "compressed_weight_shape", None)) def quantized_linear_forward(self, input: torch.FloatTensor) -> torch.FloatTensor: - return torch.nn.functional.linear(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias) + return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight), self.bias) def quantized_conv_forward(self, input) -> torch.FloatTensor: - return self._conv_forward(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias) + return self._conv_forward(input, self.sdnq_decompressor(self.weight), self.bias) def quantized_conv_transpose_1d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 1, self.dilation) - return torch.nn.functional.conv_transpose1d(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + return torch.nn.functional.conv_transpose1d(input, self.sdnq_decompressor(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) def quantized_conv_transpose_2d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 2, self.dilation) - return torch.nn.functional.conv_transpose2d(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + return torch.nn.functional.conv_transpose2d(input, self.sdnq_decompressor(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) def quantized_conv_transpose_3d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 3, self.dilation) - return torch.nn.functional.conv_transpose3d(input, self.sdnq_decompressor(self, return_decompressed_only=True), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + return torch.nn.functional.conv_transpose3d(input, self.sdnq_decompressor(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) class INT8AsymmetricWeightsDecompressor(torch.nn.Module): @@ -515,7 +519,7 @@ class INT8AsymmetricWeightsDecompressor(torch.nn.Module): ): super().__init__() self.num_bits = 8 - self.quantization_mode = "asymmetric" + self.is_asym_mode = True self.scale = scale self.zero_point = zero_point self.result_dtype = result_dtype @@ -527,12 +531,8 @@ class INT8AsymmetricWeightsDecompressor(torch.nn.Module): raise ValueError("Weight values are not in [0, 255].") return weight.to(dtype=torch.uint8) - def forward(self, x, input=None, *args, return_decompressed_only=False): # pylint: disable=keyword-arg-before-vararg,unused-argument - result = decompress_asymmetric_compiled(x.weight, self.scale, self.zero_point, self.result_dtype, self.result_shape) - if return_decompressed_only: - return result - else: - x.weight = result + def forward(self, weight): + return decompress_asymmetric_compiled(weight, self.scale, self.zero_point, self.result_dtype, self.result_shape) class INT8SymmetricWeightsDecompressor(torch.nn.Module): @@ -544,7 +544,7 @@ class INT8SymmetricWeightsDecompressor(torch.nn.Module): ): super().__init__() self.num_bits = 8 - self.quantization_mode = "symmetric" + self.is_asym_mode = False self.scale = scale self.result_dtype = result_dtype self.result_shape = result_shape @@ -555,14 +555,8 @@ class INT8SymmetricWeightsDecompressor(torch.nn.Module): raise ValueError("Weight values are not in [-128, 127].") return weight.to(dtype=torch.int8) - def forward(self, x, input=None, *args, return_decompressed_only=False, skip_int8_matmul=False): # pylint: disable=unused-argument,keyword-arg-before-vararg - if skip_int8_matmul: - return decompress_symmetric_compiled(x.weight.transpose(0,1), self.scale.unsqueeze(-1), self.result_dtype, self.result_shape) - result = decompress_symmetric_compiled(x.weight, self.scale, self.result_dtype, self.result_shape) - if return_decompressed_only: - return result - else: - x.weight = result + def forward(self, weight, skip_int8_matmul=False): + return decompress_symmetric_compiled(weight, self.scale, self.result_dtype, self.result_shape, skip_int8_matmul=skip_int8_matmul) class INT4AsymmetricWeightsDecompressor(torch.nn.Module): @@ -576,7 +570,7 @@ class INT4AsymmetricWeightsDecompressor(torch.nn.Module): ): super().__init__() self.num_bits = 4 - self.quantization_mode = "asymmetric" + self.is_asym_mode = True self.scale = scale self.zero_point = zero_point self.compressed_weight_shape = compressed_weight_shape @@ -589,12 +583,8 @@ class INT4AsymmetricWeightsDecompressor(torch.nn.Module): raise ValueError("Weight values are not in [0, 15].") return pack_uint4(weight.to(dtype=torch.uint8)) - def forward(self, x, input=None, *args, return_decompressed_only=False): # pylint: disable=unused-argument,keyword-arg-before-vararg - result = decompress_int4_asymmetric_compiled(x.weight, self.scale, self.zero_point, self.compressed_weight_shape, self.result_dtype, self.result_shape) - if return_decompressed_only: - return result - else: - x.weight = result + def forward(self, weight): + return decompress_int4_asymmetric_compiled(weight, self.scale, self.zero_point, self.compressed_weight_shape, self.result_dtype, self.result_shape) class INT4SymmetricWeightsDecompressor(torch.nn.Module): @@ -607,7 +597,7 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): ): super().__init__() self.num_bits = 4 - self.quantization_mode = "symmetric" + self.is_asym_mode = False self.scale = scale self.compressed_weight_shape = compressed_weight_shape self.result_dtype = result_dtype @@ -619,14 +609,8 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): raise ValueError("Tensor values are not in [-8, 7].") return pack_int4(weight.to(dtype=torch.int8)) - def forward(self, x, input=None, *arg, return_decompressed_only=False, skip_int8_matmul=False): # pylint: disable=keyword-arg-before-vararg,unused-argument - if skip_int8_matmul: - return decompress_int4_symmetric_compiled(x.weight, self.scale.unsqueeze(-1), self.compressed_weight_shape, self.result_dtype, self.result_shape) - result = decompress_int4_symmetric_compiled(x.weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape) - if return_decompressed_only: - return result - else: - x.weight = result + def forward(self, weight, skip_int8_matmul=False): + return decompress_int4_symmetric_compiled(weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape, skip_int8_matmul=skip_int8_matmul) if shared.opts.sdnq_decompress_compile: From 02f15b28cc1dc02aa7c615a1ecdb0d96083f02fc Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 15:57:17 +0300 Subject: [PATCH 25/94] Cleanup --- modules/model_quant_sdnq.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index dd088bc33..57727ee14 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -414,15 +414,15 @@ def decompress_symmetric(input: torch.Tensor, scale: torch.Tensor, dtype: torch. return result -def decompress_int4_asymmetric(input: torch.Tensor, scale: torch.Tensor, zero_point: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, skip_int8_matmul: bool = False) -> torch.Tensor: +def decompress_int4_asymmetric(input: torch.Tensor, scale: torch.Tensor, zero_point: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size) -> torch.Tensor: + return decompress_asymmetric(unpack_uint4(input, shape), scale, zero_point, dtype, result_shape) + + +def decompress_int4_symmetric(input: torch.Tensor, scale: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, skip_int8_matmul: bool = False) -> torch.Tensor: if skip_int8_matmul: - return decompress_asymmetric(unpack_uint4(input, shape), scale.unsqueeze(-1), zero_point, dtype, result_shape) + return decompress_symmetric(unpack_int4(input, shape, dtype=scale.dtype), scale.unsqueeze(-1), dtype, result_shape) else: - return decompress_asymmetric(unpack_uint4(input, shape), scale, zero_point, dtype, result_shape) - - -def decompress_int4_symmetric(input: torch.Tensor, scale: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size) -> torch.Tensor: - return decompress_symmetric(unpack_int4(input, shape, dtype=scale.dtype), scale, dtype, result_shape) + return decompress_symmetric(unpack_int4(input, shape, dtype=scale.dtype), scale, dtype, result_shape) def pack_uint4(tensor: torch.Tensor) -> torch.Tensor: From ccf9deaf28e08dc25f3ac1206a7e3ecf44238d8d Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 18:30:50 +0300 Subject: [PATCH 26/94] Move SDNQ to the top of the settings list --- modules/shared.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index 91def4d6e..a79382ca2 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -522,11 +522,6 @@ options_templates.update(options_section(('backends', "Backend Settings"), { })) options_templates.update(options_section(('quantization', "Quantization Settings"), { - "bnb_quantization_sep": OptionInfo("

BitsAndBytes

", "", gr.HTML), - "bnb_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), - "bnb_quantization_type": OptionInfo("nf4", "Quantization type", gr.Dropdown, {"choices": ['nf4', 'fp8', 'fp4'], "visible": native}), - "bnb_quantization_storage": OptionInfo("uint8", "Backend storage", gr.Dropdown, {"choices": ["float16", "float32", "int8", "uint8", "float64", "bfloat16"], "visible": native}), - "sdnq_quantize_sep": OptionInfo("

SDNQ: SDNext Quantization

", "", gr.HTML), "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native}), @@ -538,6 +533,11 @@ options_templates.update(options_section(('quantization', "Quantization Settings "sdnq_decompress_int8_matmul": OptionInfo(False, "Use direct INT8 MatMul", gr.Checkbox, {"visible": native}), "sdnq_quantize_shuffle_weights": OptionInfo(False, "Shuffle weights in post mode", gr.Checkbox, {"visible": native}), + "bnb_quantization_sep": OptionInfo("

BitsAndBytes

", "", gr.HTML), + "bnb_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), + "bnb_quantization_type": OptionInfo("nf4", "Quantization type", gr.Dropdown, {"choices": ['nf4', 'fp8', 'fp4'], "visible": native}), + "bnb_quantization_storage": OptionInfo("uint8", "Backend storage", gr.Dropdown, {"choices": ["float16", "float32", "int8", "uint8", "float64", "bfloat16"], "visible": native}), + "quanto_quantization_sep": OptionInfo("

Optimum Quanto

", "", gr.HTML), "quanto_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "quanto_quantization_type": OptionInfo("int8", "Quantization weights type", gr.Dropdown, {"choices": ["float8", "int8", "int4", "int2"], "visible": native}), From 687c50dcc879650fa3a01237f9b2039440457659 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 19:48:45 +0300 Subject: [PATCH 27/94] SDNQ fix Lora --- modules/lora/lora_apply.py | 4 ++-- modules/model_quant_sdnq.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index ff9aa64c4..370beabea 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -80,7 +80,7 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. try: t0 = time.time() if hasattr(self, "sdnq_decompressor"): - weight = self.sdnq_decompressor.to(devices.device)(self.weight.to(devices.device)) + weight = self.sdnq_decompressor.to(devices.device)(self.weight.to(devices.device), skip_int8_matmul=True) else: weight = self.weight.to(devices.device) # must perform calc on gpu due to performance updown, ex_bias = module.calc_updown(weight) @@ -143,7 +143,7 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G from modules.model_quant_sdnq import sdnq_quantize_layer num_bits = self.sdnq_decompressor.num_bits is_asym_mode = self.sdnq_decompressor.is_asym_mode - dequant_weight = self.sdnq_decompressor.to(devices.device)(model_weights.to(devices.device)) + dequant_weight = self.sdnq_decompressor.to(devices.device)(model_weights.to(devices.device), skip_int8_matmul=True) new_weight = dequant_weight.to(devices.device, dtype=torch.float32) + lora_weights.to(devices.device, dtype=torch.float32) self.weight = torch.nn.Parameter(new_weight, requires_grad=False) self.sdnq_decompressor = None diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 57727ee14..5a3f1b81b 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -531,7 +531,7 @@ class INT8AsymmetricWeightsDecompressor(torch.nn.Module): raise ValueError("Weight values are not in [0, 255].") return weight.to(dtype=torch.uint8) - def forward(self, weight): + def forward(self, weight, **kwargs): return decompress_asymmetric_compiled(weight, self.scale, self.zero_point, self.result_dtype, self.result_shape) @@ -555,7 +555,7 @@ class INT8SymmetricWeightsDecompressor(torch.nn.Module): raise ValueError("Weight values are not in [-128, 127].") return weight.to(dtype=torch.int8) - def forward(self, weight, skip_int8_matmul=False): + def forward(self, weight, skip_int8_matmul=False, **kwargs): return decompress_symmetric_compiled(weight, self.scale, self.result_dtype, self.result_shape, skip_int8_matmul=skip_int8_matmul) @@ -577,7 +577,7 @@ class INT4AsymmetricWeightsDecompressor(torch.nn.Module): self.result_dtype = result_dtype self.result_shape = result_shape - def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: + def pack_weight(self, weight: torch.Tensor, **kwargs) -> torch.Tensor: if debug: if torch.any((weight < 0) | (weight > 15)): raise ValueError("Weight values are not in [0, 15].") @@ -609,7 +609,7 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): raise ValueError("Tensor values are not in [-8, 7].") return pack_int4(weight.to(dtype=torch.int8)) - def forward(self, weight, skip_int8_matmul=False): + def forward(self, weight, skip_int8_matmul=False, **kwargs): return decompress_int4_symmetric_compiled(weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape, skip_int8_matmul=skip_int8_matmul) From 742cd61d1f9420b31d6d881344ca456b333eab31 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 19:59:43 +0300 Subject: [PATCH 28/94] Add TeaCache for HiDream --- CHANGELOG.md | 2 + modules/model_hidream.py | 5 + modules/processing.py | 2 +- modules/teacache/__init__.py | 25 ++- modules/teacache/teacache_hidream.py | 267 +++++++++++++++++++++++++++ 5 files changed, 292 insertions(+), 9 deletions(-) create mode 100644 modules/teacache/teacache_hidream.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 346f081e1..81486e925 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ - **IPEX** - Disabe Dynamic Attention by default on PyTorch 2.7 - Remove GradScaler hijack and use torch.amp.GradScaler instead +- **Feature** + - TeaCache support for HiDream I1 ## Update for 2025-05-17 diff --git a/modules/model_hidream.py b/modules/model_hidream.py index bd0159130..f3c5c6483 100644 --- a/modules/model_hidream.py +++ b/modules/model_hidream.py @@ -94,6 +94,11 @@ def load_hidream(checkpoint_info, diffusers_load_config={}): load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, module='Model') shared.log.debug(f'Load model: type=HiDream model="{checkpoint_info.name}" repo="{repo_id}" offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}') + if shared.opts.teacache_enabled: + from modules import teacache + shared.log.debug(f'Transformers cache: type=teacache patch=forward cls={diffusers.HiDreamImageTransformer2DModel.__name__}') + diffusers.HiDreamImageTransformer2DModel.forward = teacache.teacache_hidream_forward # patch must be done before transformer is loaded + if 'I1' in repo_id: cls = diffusers.HiDreamImagePipeline elif 'E1' in repo_id: diff --git a/modules/processing.py b/modules/processing.py index 0472a058c..6fdb69d97 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -167,7 +167,7 @@ def process_images(p: StableDiffusionProcessing) -> Processed: shared.prompt_styles.apply_styles_to_extra(p) shared.prompt_styles.extract_comments(p) - if shared.opts.cuda_compile_backend == 'none': + if 'Model' not in shared.opts.cuda_compile: token_merge.apply_token_merging(p.sd_model) from modules import sd_hijack_freeu, para_attention, teacache sd_hijack_freeu.apply_freeu(p, not shared.native) diff --git a/modules/teacache/__init__.py b/modules/teacache/__init__.py index 89e3cc0c8..22abc7057 100644 --- a/modules/teacache/__init__.py +++ b/modules/teacache/__init__.py @@ -1,10 +1,11 @@ from .teacache_flux import teacache_flux_forward +from .teacache_hidream import teacache_hidream_forward from .teacache_ltx import teacache_ltx_forward from .teacache_mochi import teacache_mochi_forward from .teacache_cogvideox import teacache_cog_forward -supported_models = ['Flux', 'CogVideoX', 'Mochi', 'LTX'] +supported_models = ['Flux', 'CogVideoX', 'Mochi', 'LTX', 'HiDream'] def apply_teacache(p): @@ -15,11 +16,19 @@ def apply_teacache(p): return if not hasattr(shared.sd_model, 'transformer'): return - shared.sd_model.transformer.__class__.enable_teacache = shared.opts.teacache_thresh > 0 - shared.sd_model.transformer.__class__.cnt = 0 - shared.sd_model.transformer.__class__.num_steps = p.steps - shared.sd_model.transformer.__class__.rel_l1_thresh = shared.opts.teacache_thresh # 0.25 for 1.5x speedup, 0.4 for 1.8x speedup, 0.6 for 2.0x speedup, 0.8 for 2.25x speedup - shared.sd_model.transformer.__class__.accumulated_rel_l1_distance = 0 - shared.sd_model.transformer.__class__.previous_modulated_input = None - shared.sd_model.transformer.__class__.previous_residual = None + if shared.sd_model.__class__.__name__.startswith('HiDream'): + shared.sd_model.transformer.__class__.enable_teacache = shared.opts.teacache_thresh > 0 + shared.sd_model.transformer.__class__.cnt = 0 + shared.sd_model.transformer.__class__.num_steps = p.steps + shared.sd_model.transformer.__class__.ret_steps = p.steps * 0.1 + shared.sd_model.transformer.__class__.rel_l1_thresh = shared.opts.teacache_thresh # 0.17 for 1.5x speedup, 0.25 for 1.7x speedup, 0.3 for 2x speedup, 0.45 for 2.6x speedup + shared.sd_model.transformer.__class__.coefficients = [-3.13605009e+04, -7.12425503e+02, 4.91363285e+01, 8.26515490e+00, 1.08053901e-01] + else: + shared.sd_model.transformer.__class__.enable_teacache = shared.opts.teacache_thresh > 0 + shared.sd_model.transformer.__class__.cnt = 0 + shared.sd_model.transformer.__class__.num_steps = p.steps + shared.sd_model.transformer.__class__.rel_l1_thresh = shared.opts.teacache_thresh # 0.25 for 1.5x speedup, 0.4 for 1.8x speedup, 0.6 for 2.0x speedup, 0.8 for 2.25x speedup + shared.sd_model.transformer.__class__.accumulated_rel_l1_distance = 0 + shared.sd_model.transformer.__class__.previous_modulated_input = None + shared.sd_model.transformer.__class__.previous_residual = None shared.log.info(f'Transformers cache: type=teacache cls={shared.sd_model.__class__.__name__} thresh={shared.opts.teacache_thresh}') diff --git a/modules/teacache/teacache_hidream.py b/modules/teacache/teacache_hidream.py new file mode 100644 index 000000000..b3bd84bb8 --- /dev/null +++ b/modules/teacache/teacache_hidream.py @@ -0,0 +1,267 @@ +from typing import Any, Dict, List, Optional, Tuple +from transformers import PreTrainedTokenizerFast, LlamaForCausalLM +from diffusers import HiDreamImagePipeline +from diffusers.models import HiDreamImageTransformer2DModel +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.utils import logging, deprecate, USE_PEFT_BACKEND, logging, scale_lora_layers, unscale_lora_layers + +import torch +import numpy as np + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def teacache_hidream_forward( + self, + hidden_states: torch.Tensor, + timesteps: torch.LongTensor = None, + encoder_hidden_states_t5: torch.Tensor = None, + encoder_hidden_states_llama3: torch.Tensor = None, + pooled_embeds: torch.Tensor = None, + img_ids: Optional[torch.Tensor] = None, + img_sizes: Optional[List[Tuple[int, int]]] = None, + hidden_states_masks: Optional[torch.Tensor] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + return_dict: bool = True, + **kwargs, +): + encoder_hidden_states = kwargs.get("encoder_hidden_states", None) + + if encoder_hidden_states is not None: + deprecation_message = "The `encoder_hidden_states` argument is deprecated. Please use `encoder_hidden_states_t5` and `encoder_hidden_states_llama3` instead." + deprecate("encoder_hidden_states", "0.35.0", deprecation_message) + encoder_hidden_states_t5 = encoder_hidden_states[0] + encoder_hidden_states_llama3 = encoder_hidden_states[1] + + if img_ids is not None and img_sizes is not None and hidden_states_masks is None: + deprecation_message = ( + "Passing `img_ids` and `img_sizes` with unpachified `hidden_states` is deprecated and will be ignored." + ) + deprecate("img_ids", "0.35.0", deprecation_message) + + if hidden_states_masks is not None and (img_ids is None or img_sizes is None): + raise ValueError("if `hidden_states_masks` is passed, `img_ids` and `img_sizes` must also be passed.") + elif hidden_states_masks is not None and hidden_states.ndim != 3: + raise ValueError( + "if `hidden_states_masks` is passed, `hidden_states` must be a 3D tensors with shape (batch_size, patch_height * patch_width, patch_size * patch_size * channels)" + ) + + if attention_kwargs is not None: + attention_kwargs = attention_kwargs.copy() + lora_scale = attention_kwargs.pop("scale", 1.0) + else: + lora_scale = 1.0 + + if USE_PEFT_BACKEND: + # weight the lora layers by setting `lora_scale` for each PEFT layer + scale_lora_layers(self, lora_scale) + else: + if attention_kwargs is not None and attention_kwargs.get("scale", None) is not None: + logger.warning( + "Passing `scale` via `attention_kwargs` when not using the PEFT backend is ineffective." + ) + + # spatial forward + batch_size = hidden_states.shape[0] + hidden_states_type = hidden_states.dtype + + # Patchify the input + if hidden_states_masks is None: + hidden_states, hidden_states_masks, img_sizes, img_ids = self.patchify(hidden_states) + + # Embed the hidden states + hidden_states = self.x_embedder(hidden_states) + + # 0. time + timesteps = self.t_embedder(timesteps, hidden_states_type) + p_embedder = self.p_embedder(pooled_embeds) + temb = timesteps + p_embedder + + encoder_hidden_states = [encoder_hidden_states_llama3[k] for k in self.config.llama_layers] + + if self.caption_projection is not None: + new_encoder_hidden_states = [] + for i, enc_hidden_state in enumerate(encoder_hidden_states): + enc_hidden_state = self.caption_projection[i](enc_hidden_state) + enc_hidden_state = enc_hidden_state.view(batch_size, -1, hidden_states.shape[-1]) + new_encoder_hidden_states.append(enc_hidden_state) + encoder_hidden_states = new_encoder_hidden_states + encoder_hidden_states_t5 = self.caption_projection[-1](encoder_hidden_states_t5) + encoder_hidden_states_t5 = encoder_hidden_states_t5.view(batch_size, -1, hidden_states.shape[-1]) + encoder_hidden_states.append(encoder_hidden_states_t5) + + txt_ids = torch.zeros( + batch_size, + encoder_hidden_states[-1].shape[1] + + encoder_hidden_states[-2].shape[1] + + encoder_hidden_states[0].shape[1], + 3, + device=img_ids.device, + dtype=img_ids.dtype, + ) + ids = torch.cat((img_ids, txt_ids), dim=1) + image_rotary_emb = self.pe_embedder(ids) + + # 2. Blocks + block_id = 0 + initial_encoder_hidden_states = torch.cat([encoder_hidden_states[-1], encoder_hidden_states[-2]], dim=1) + initial_encoder_hidden_states_seq_len = initial_encoder_hidden_states.shape[1] + + if self.enable_teacache: + modulated_inp = timesteps.clone() + if self.cnt < self.ret_steps: + should_calc = True + self.accumulated_rel_l1_distance = 0 + else: + rescale_func = np.poly1d(self.coefficients) + self.accumulated_rel_l1_distance += rescale_func(((modulated_inp-self.previous_modulated_input).abs().mean() / self.previous_modulated_input.abs().mean()).cpu().item()) + if self.accumulated_rel_l1_distance < self.rel_l1_thresh: + should_calc = False + else: + should_calc = True + self.accumulated_rel_l1_distance = 0 + self.previous_modulated_input = modulated_inp + self.cnt += 1 + if self.cnt == self.num_steps: + self.cnt = 0 + + if self.enable_teacache: + if not should_calc: + hidden_states += self.previous_residual + else: + # 2. Blocks + ori_hidden_states = hidden_states.clone() + for bid, block in enumerate(self.double_stream_blocks): + cur_llama31_encoder_hidden_states = encoder_hidden_states[block_id] + cur_encoder_hidden_states = torch.cat( + [initial_encoder_hidden_states, cur_llama31_encoder_hidden_states], dim=1 + ) + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states, initial_encoder_hidden_states = self._gradient_checkpointing_func( + block, + hidden_states, + hidden_states_masks, + cur_encoder_hidden_states, + temb, + image_rotary_emb, + ) + else: + hidden_states, initial_encoder_hidden_states = block( + hidden_states=hidden_states, + hidden_states_masks=hidden_states_masks, + encoder_hidden_states=cur_encoder_hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + ) + initial_encoder_hidden_states = initial_encoder_hidden_states[:, :initial_encoder_hidden_states_seq_len] + block_id += 1 + + image_tokens_seq_len = hidden_states.shape[1] + hidden_states = torch.cat([hidden_states, initial_encoder_hidden_states], dim=1) + hidden_states_seq_len = hidden_states.shape[1] + if hidden_states_masks is not None: + encoder_attention_mask_ones = torch.ones( + (batch_size, initial_encoder_hidden_states.shape[1] + cur_llama31_encoder_hidden_states.shape[1]), + device=hidden_states_masks.device, + dtype=hidden_states_masks.dtype, + ) + hidden_states_masks = torch.cat([hidden_states_masks, encoder_attention_mask_ones], dim=1) + + for bid, block in enumerate(self.single_stream_blocks): + cur_llama31_encoder_hidden_states = encoder_hidden_states[block_id] + hidden_states = torch.cat([hidden_states, cur_llama31_encoder_hidden_states], dim=1) + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func( + block, + hidden_states, + hidden_states_masks, + None, + temb, + image_rotary_emb, + ) + else: + hidden_states = block( + hidden_states=hidden_states, + hidden_states_masks=hidden_states_masks, + encoder_hidden_states=None, + temb=temb, + image_rotary_emb=image_rotary_emb, + ) + hidden_states = hidden_states[:, :hidden_states_seq_len] + block_id += 1 + + hidden_states = hidden_states[:, :image_tokens_seq_len, ...] + self.previous_residual = hidden_states - ori_hidden_states + else: + for bid, block in enumerate(self.double_stream_blocks): + cur_llama31_encoder_hidden_states = encoder_hidden_states[block_id] + cur_encoder_hidden_states = torch.cat( + [initial_encoder_hidden_states, cur_llama31_encoder_hidden_states], dim=1 + ) + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states, initial_encoder_hidden_states = self._gradient_checkpointing_func( + block, + hidden_states, + hidden_states_masks, + cur_encoder_hidden_states, + temb, + image_rotary_emb, + ) + else: + hidden_states, initial_encoder_hidden_states = block( + hidden_states=hidden_states, + hidden_states_masks=hidden_states_masks, + encoder_hidden_states=cur_encoder_hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + ) + initial_encoder_hidden_states = initial_encoder_hidden_states[:, :initial_encoder_hidden_states_seq_len] + block_id += 1 + + image_tokens_seq_len = hidden_states.shape[1] + hidden_states = torch.cat([hidden_states, initial_encoder_hidden_states], dim=1) + hidden_states_seq_len = hidden_states.shape[1] + if hidden_states_masks is not None: + encoder_attention_mask_ones = torch.ones( + (batch_size, initial_encoder_hidden_states.shape[1] + cur_llama31_encoder_hidden_states.shape[1]), + device=hidden_states_masks.device, + dtype=hidden_states_masks.dtype, + ) + hidden_states_masks = torch.cat([hidden_states_masks, encoder_attention_mask_ones], dim=1) + + for bid, block in enumerate(self.single_stream_blocks): + cur_llama31_encoder_hidden_states = encoder_hidden_states[block_id] + hidden_states = torch.cat([hidden_states, cur_llama31_encoder_hidden_states], dim=1) + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func( + block, + hidden_states, + hidden_states_masks, + None, + temb, + image_rotary_emb, + ) + else: + hidden_states = block( + hidden_states=hidden_states, + hidden_states_masks=hidden_states_masks, + encoder_hidden_states=None, + temb=temb, + image_rotary_emb=image_rotary_emb, + ) + hidden_states = hidden_states[:, :hidden_states_seq_len] + block_id += 1 + + hidden_states = hidden_states[:, :image_tokens_seq_len, ...] + + output = self.final_layer(hidden_states, temb) + output = self.unpatchify(output, img_sizes, self.training) + + if USE_PEFT_BACKEND: + # remove `lora_scale` from each PEFT layer + unscale_lora_layers(self, lora_scale) + + if not return_dict: + return (output,) + return Transformer2DModelOutput(sample=output) \ No newline at end of file From 6dee9f5ac7260fa8bb0dd36325ba4458a30fee6e Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 21:21:01 +0300 Subject: [PATCH 29/94] Fix HiDream teacache not reseting --- modules/teacache/__init__.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/modules/teacache/__init__.py b/modules/teacache/__init__.py index 22abc7057..342a3130f 100644 --- a/modules/teacache/__init__.py +++ b/modules/teacache/__init__.py @@ -16,19 +16,14 @@ def apply_teacache(p): return if not hasattr(shared.sd_model, 'transformer'): return + shared.sd_model.transformer.__class__.enable_teacache = shared.opts.teacache_thresh > 0 + shared.sd_model.transformer.__class__.cnt = 0 + shared.sd_model.transformer.__class__.num_steps = p.steps + shared.sd_model.transformer.__class__.rel_l1_thresh = shared.opts.teacache_thresh # 0.25 for 1.5x speedup, 0.4 for 1.8x speedup, 0.6 for 2.0x speedup, 0.8 for 2.25x speedup + shared.sd_model.transformer.__class__.accumulated_rel_l1_distance = 0 + shared.sd_model.transformer.__class__.previous_modulated_input = None + shared.sd_model.transformer.__class__.previous_residual = None if shared.sd_model.__class__.__name__.startswith('HiDream'): - shared.sd_model.transformer.__class__.enable_teacache = shared.opts.teacache_thresh > 0 - shared.sd_model.transformer.__class__.cnt = 0 - shared.sd_model.transformer.__class__.num_steps = p.steps shared.sd_model.transformer.__class__.ret_steps = p.steps * 0.1 - shared.sd_model.transformer.__class__.rel_l1_thresh = shared.opts.teacache_thresh # 0.17 for 1.5x speedup, 0.25 for 1.7x speedup, 0.3 for 2x speedup, 0.45 for 2.6x speedup shared.sd_model.transformer.__class__.coefficients = [-3.13605009e+04, -7.12425503e+02, 4.91363285e+01, 8.26515490e+00, 1.08053901e-01] - else: - shared.sd_model.transformer.__class__.enable_teacache = shared.opts.teacache_thresh > 0 - shared.sd_model.transformer.__class__.cnt = 0 - shared.sd_model.transformer.__class__.num_steps = p.steps - shared.sd_model.transformer.__class__.rel_l1_thresh = shared.opts.teacache_thresh # 0.25 for 1.5x speedup, 0.4 for 1.8x speedup, 0.6 for 2.0x speedup, 0.8 for 2.25x speedup - shared.sd_model.transformer.__class__.accumulated_rel_l1_distance = 0 - shared.sd_model.transformer.__class__.previous_modulated_input = None - shared.sd_model.transformer.__class__.previous_residual = None shared.log.info(f'Transformers cache: type=teacache cls={shared.sd_model.__class__.__name__} thresh={shared.opts.teacache_thresh}') From 84ddfb286808cd16d0e96e87168542b13f4217ca Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 22:39:20 +0300 Subject: [PATCH 30/94] SDNQ fix lora apply --- modules/lora/lora_apply.py | 4 ++-- modules/model_quant_sdnq.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index 370beabea..67af8974a 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -80,7 +80,7 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. try: t0 = time.time() if hasattr(self, "sdnq_decompressor"): - weight = self.sdnq_decompressor.to(devices.device)(self.weight.to(devices.device), skip_int8_matmul=True) + weight = self.sdnq_decompressor.to(devices.device)(self.weight.to(devices.device), skip_int8_matmul=self.sdnq_decompressor.use_int8_matmul) else: weight = self.weight.to(devices.device) # must perform calc on gpu due to performance updown, ex_bias = module.calc_updown(weight) @@ -143,7 +143,7 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G from modules.model_quant_sdnq import sdnq_quantize_layer num_bits = self.sdnq_decompressor.num_bits is_asym_mode = self.sdnq_decompressor.is_asym_mode - dequant_weight = self.sdnq_decompressor.to(devices.device)(model_weights.to(devices.device), skip_int8_matmul=True) + dequant_weight = self.sdnq_decompressor.to(devices.device)(model_weights.to(devices.device), skip_int8_matmul=self.sdnq_decompressor.use_int8_matmul) new_weight = dequant_weight.to(devices.device, dtype=torch.float32) + lora_weights.to(devices.device, dtype=torch.float32) self.weight = torch.nn.Parameter(new_weight, requires_grad=False) self.sdnq_decompressor = None diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 5a3f1b81b..dda1d28d5 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -117,6 +117,7 @@ def sdnq_quantize_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c compressed_weight_shape=compressed_weight.shape, result_dtype=torch_dtype, result_shape=result_shape, + use_int8_matmul=use_int8_matmul, ) else: decompressor = INT4SymmetricWeightsDecompressor( @@ -124,6 +125,7 @@ def sdnq_quantize_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c compressed_weight_shape=compressed_weight.shape, result_dtype=torch_dtype, result_shape=result_shape, + use_int8_matmul=use_int8_matmul, ) else: if is_asym_mode: @@ -132,12 +134,14 @@ def sdnq_quantize_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c zero_point=zero_point.data, result_dtype=torch_dtype, result_shape=result_shape, + use_int8_matmul=use_int8_matmul, ) else: decompressor = INT8SymmetricWeightsDecompressor( scale=scale.data, result_dtype=torch_dtype, result_shape=result_shape, + use_int8_matmul=use_int8_matmul, ) compressed_weight = decompressor.pack_weight(compressed_weight).to(return_device) @@ -516,6 +520,7 @@ class INT8AsymmetricWeightsDecompressor(torch.nn.Module): zero_point: torch.Tensor, result_dtype: torch.dtype, result_shape: torch.Size, + use_int8_matmul: bool = False, ): super().__init__() self.num_bits = 8 @@ -524,6 +529,7 @@ class INT8AsymmetricWeightsDecompressor(torch.nn.Module): self.zero_point = zero_point self.result_dtype = result_dtype self.result_shape = result_shape + self.use_int8_matmul = use_int8_matmul def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: if debug: @@ -541,6 +547,7 @@ class INT8SymmetricWeightsDecompressor(torch.nn.Module): scale: torch.Tensor, result_dtype: torch.dtype, result_shape: torch.Size, + use_int8_matmul: bool = False, ): super().__init__() self.num_bits = 8 @@ -548,6 +555,7 @@ class INT8SymmetricWeightsDecompressor(torch.nn.Module): self.scale = scale self.result_dtype = result_dtype self.result_shape = result_shape + self.use_int8_matmul = use_int8_matmul def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: if debug: @@ -567,6 +575,7 @@ class INT4AsymmetricWeightsDecompressor(torch.nn.Module): compressed_weight_shape: torch.Size, result_dtype: torch.dtype, result_shape: torch.Size, + use_int8_matmul: bool = False, ): super().__init__() self.num_bits = 4 @@ -576,6 +585,7 @@ class INT4AsymmetricWeightsDecompressor(torch.nn.Module): self.compressed_weight_shape = compressed_weight_shape self.result_dtype = result_dtype self.result_shape = result_shape + self.use_int8_matmul = use_int8_matmul def pack_weight(self, weight: torch.Tensor, **kwargs) -> torch.Tensor: if debug: @@ -594,6 +604,7 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): compressed_weight_shape: torch.Size, result_dtype: torch.dtype, result_shape: torch.Size, + use_int8_matmul: bool = False, ): super().__init__() self.num_bits = 4 @@ -602,6 +613,7 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): self.compressed_weight_shape = compressed_weight_shape self.result_dtype = result_dtype self.result_shape = result_shape + self.use_int8_matmul = use_int8_matmul def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: if debug: From 4d9c2a860870e2ea4c918d48970eaf5c4bd72350 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 22:41:12 +0300 Subject: [PATCH 31/94] Cleanup --- modules/model_quant_sdnq.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index dda1d28d5..cc8b62433 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -587,13 +587,13 @@ class INT4AsymmetricWeightsDecompressor(torch.nn.Module): self.result_shape = result_shape self.use_int8_matmul = use_int8_matmul - def pack_weight(self, weight: torch.Tensor, **kwargs) -> torch.Tensor: + def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: if debug: if torch.any((weight < 0) | (weight > 15)): raise ValueError("Weight values are not in [0, 15].") return pack_uint4(weight.to(dtype=torch.uint8)) - def forward(self, weight): + def forward(self, weight, **kwargs): return decompress_int4_asymmetric_compiled(weight, self.scale, self.zero_point, self.compressed_weight_shape, self.result_dtype, self.result_shape) From 280be31883f2b55386752c26ae093b3d46720c79 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 27 May 2025 00:08:32 +0300 Subject: [PATCH 32/94] SDNQ fix Lora change --- modules/lora/lora_apply.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index 67af8974a..fca426c69 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -137,21 +137,22 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G except Exception as e: shared.log.error(f'Network load: type=LoRA quant=bnb cls={self.__class__.__name__} type={self.quant_type} blocksize={self.blocksize} state={vars(self.quant_state)} weight={self.weight} bias={lora_weights} {e}') elif not bias and hasattr(self, "sdnq_decompressor"): - num_bits = None - is_asym_mode = None try: from modules.model_quant_sdnq import sdnq_quantize_layer - num_bits = self.sdnq_decompressor.num_bits - is_asym_mode = self.sdnq_decompressor.is_asym_mode - dequant_weight = self.sdnq_decompressor.to(devices.device)(model_weights.to(devices.device), skip_int8_matmul=self.sdnq_decompressor.use_int8_matmul) + if hasattr(self, "sdnq_decompressor_backup"): + sdnq_decompressor = self.sdnq_decompressor_backup.to(devices.device) + else: + sdnq_decompressor = self.sdnq_decompressor.to(devices.device) + dequant_weight = sdnq_decompressor(model_weights.to(devices.device), skip_int8_matmul=sdnq_decompressor.use_int8_matmul) new_weight = dequant_weight.to(devices.device, dtype=torch.float32) + lora_weights.to(devices.device, dtype=torch.float32) self.weight = torch.nn.Parameter(new_weight, requires_grad=False) self.sdnq_decompressor = None - self = sdnq_quantize_layer(self, num_bits, is_asym_mode, torch_dtype=devices.dtype, quant_conv=shared.opts.sdnq_quantize_conv_layers, group_size=shared.opts.sdnq_quantize_weights_group_size, use_int8_matmul=shared.opts.sdnq_decompress_int8_matmul) + self = sdnq_quantize_layer(self, sdnq_decompressor.num_bits, sdnq_decompressor.is_asym_mode, torch_dtype=devices.dtype, quant_conv=shared.opts.sdnq_quantize_conv_layers, group_size=shared.opts.sdnq_quantize_weights_group_size, use_int8_matmul=shared.opts.sdnq_decompress_int8_matmul) self = self.to(device) + weight = None del dequant_weight except Exception as e: - shared.log.error(f'Network load: type=LoRA quant=sdnq cls={self.__class__.__name__} bits={num_bits} is_asym_mode={is_asym_mode} weight={self.weight} lora_weights={lora_weights} {e}') + shared.log.error(f'Network load: type=LoRA quant=sdnq cls={self.__class__.__name__} weight={self.weight} lora_weights={lora_weights} {e}') else: try: new_weight = model_weights.to(devices.device) + lora_weights.to(devices.device) From dec460e6655ab5df10de3fe37d858058c8167f36 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 27 May 2025 03:02:36 +0300 Subject: [PATCH 33/94] SDNQ use torch.bitwise ops instead of python --- modules/model_quant_sdnq.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index cc8b62433..0110b2db6 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -433,15 +433,14 @@ def pack_uint4(tensor: torch.Tensor) -> torch.Tensor: if tensor.dtype != torch.uint8: raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") packed_tensor = tensor.contiguous().reshape(-1, 2) - packed_tensor = torch.bitwise_and(packed_tensor[..., ::2], 15) | packed_tensor[..., 1::2] << 4 + packed_tensor = torch.bitwise_or(torch.bitwise_and(packed_tensor[:, 0], 15), torch.bitwise_left_shift(packed_tensor[:, 1], 4)) return packed_tensor def pack_int4(tensor: torch.Tensor) -> torch.Tensor: if tensor.dtype != torch.int8: raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.int8 type is supported.") - tensor = tensor + 8 - return pack_uint4(tensor.to(dtype=torch.uint8)) + return pack_uint4((tensor + 8).to(dtype=torch.uint8)) def unpack_uint4(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: From 79bb348927dda3424338f443f3e26407a7971863 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 27 May 2025 13:06:17 +0300 Subject: [PATCH 34/94] SDNQ sort quant schemes by recommended order --- modules/shared.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index a79382ca2..b9eb6518a 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -525,7 +525,7 @@ options_templates.update(options_section(('quantization', "Quantization Settings "sdnq_quantize_sep": OptionInfo("

SDNQ: SDNext Quantization

", "", gr.HTML), "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native}), - "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ['int8', 'uint8', 'int4', 'uint4'], "visible": native}), + "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ['int8', 'uint4', 'uint8', 'int4'], "visible": native}), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), @@ -565,7 +565,7 @@ options_templates.update(options_section(('quantization', "Quantization Settings "nncf_compress_sep": OptionInfo("

NNCF: Neural Network Compression Framework

", "", gr.HTML, {"visible": cmd_opts.use_openvino}), "nncf_compress_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": cmd_opts.use_openvino}), - "nncf_compress_weights_mode": OptionInfo("INT8_SYM", "Quantization type", gr.Dropdown, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'], "visible": cmd_opts.use_openvino}), + "nncf_compress_weights_mode": OptionInfo("INT8_SYM", "Quantization type", gr.Dropdown, {"choices": ['INT8', 'INT4_ASYM', 'INT8_SYM', 'INT4_SYM', 'NF4'], "visible": cmd_opts.use_openvino}), "nncf_compress_weights_raito": OptionInfo(0, "Compress ratio", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}), "nncf_compress_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": cmd_opts.use_openvino}), "nncf_quantize": OptionInfo([], "Static Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "TE"], "visible": cmd_opts.use_openvino}), From dece497f109699ba58fd849292b7ddaa60e9c68c Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 27 May 2025 15:49:21 +0300 Subject: [PATCH 35/94] Refactor SDNQ to use weights_dtype and rename decompress_int8_matmul to use_quantized_matmul --- modules/lora/lora_apply.py | 6 +- modules/model_quant.py | 15 +- modules/model_quant_sdnq.py | 541 ++++++++++++++++++------------------ modules/shared.py | 2 +- 4 files changed, 283 insertions(+), 281 deletions(-) diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index fca426c69..3ed2eb8c5 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -80,7 +80,7 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. try: t0 = time.time() if hasattr(self, "sdnq_decompressor"): - weight = self.sdnq_decompressor.to(devices.device)(self.weight.to(devices.device), skip_int8_matmul=self.sdnq_decompressor.use_int8_matmul) + weight = self.sdnq_decompressor.to(devices.device)(self.weight.to(devices.device), skip_int8_matmul=self.sdnq_decompressor.use_quantized_matmul) else: weight = self.weight.to(devices.device) # must perform calc on gpu due to performance updown, ex_bias = module.calc_updown(weight) @@ -143,11 +143,11 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G sdnq_decompressor = self.sdnq_decompressor_backup.to(devices.device) else: sdnq_decompressor = self.sdnq_decompressor.to(devices.device) - dequant_weight = sdnq_decompressor(model_weights.to(devices.device), skip_int8_matmul=sdnq_decompressor.use_int8_matmul) + dequant_weight = sdnq_decompressor(model_weights.to(devices.device), skip_int8_matmul=sdnq_decompressor.use_quantized_matmul) new_weight = dequant_weight.to(devices.device, dtype=torch.float32) + lora_weights.to(devices.device, dtype=torch.float32) self.weight = torch.nn.Parameter(new_weight, requires_grad=False) self.sdnq_decompressor = None - self = sdnq_quantize_layer(self, sdnq_decompressor.num_bits, sdnq_decompressor.is_asym_mode, torch_dtype=devices.dtype, quant_conv=shared.opts.sdnq_quantize_conv_layers, group_size=shared.opts.sdnq_quantize_weights_group_size, use_int8_matmul=shared.opts.sdnq_decompress_int8_matmul) + self = sdnq_quantize_layer(self, sdnq_decompressor.weights_dtype, torch_dtype=devices.dtype, group_size=shared.opts.sdnq_quantize_weights_group_size, quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, param_name=getattr(self, 'network_layer_name', None)) self = self.to(device) weight = None del dequant_weight diff --git a/modules/model_quant.py b/modules/model_quant.py index c0cd6fb9f..05c9585b3 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -116,7 +116,8 @@ def create_sdnq_config(kwargs = None, allow_sdnq: bool = True, module: str = 'Mo sdnq_config = SDNQConfig( weights_dtype=weights_dtype if weights_dtype is not None else shared.opts.sdnq_quantize_weights_mode, group_size=shared.opts.sdnq_quantize_weights_group_size, - use_int8_matmul=shared.opts.sdnq_decompress_int8_matmul, + quant_conv=shared.opts.sdnq_quantize_conv_layers, + use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, ) log.debug(f'Quantization: module="{module}" type=sdnq dtype={shared.opts.sdnq_quantize_weights_mode}') if kwargs is None: @@ -318,9 +319,15 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc=True): if hasattr(model, "get_input_embeddings"): backup_embeddings = copy.deepcopy(model.get_input_embeddings()) - num_bits = 8 if shared.opts.sdnq_quantize_weights_mode in {"int8", "uint8"} else 4 - is_asym_mode = shared.opts.sdnq_quantize_weights_mode in {"uint8", "uint4"} - model = apply_sdnq_to_module(model, num_bits, is_asym_mode, quant_conv=shared.opts.sdnq_quantize_conv_layers) + model = apply_sdnq_to_module( + model, + weights_dtype=shared.opts.sdnq_quantize_weights_mode, + torch_dtype=devices.dtype, + group_size=shared.opts.sdnq_quantize_weights_group_size, + quant_conv=shared.opts.sdnq_quantize_conv_layers, + use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, + param_name=op, + ) model.quantization_method = 'SDNQ' if hasattr(model, "set_input_embeddings") and backup_embeddings is not None: diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 0110b2db6..ad442a145 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -20,20 +20,32 @@ torch_dtype_dict = { "uint4": CustomDtype.INT4, } -linear_types = ["Linear"] -conv_types = ["Conv1d", "Conv2d", "Conv3d"] -conv_transpose_types = ["ConvTranspose1d", "ConvTranspose2d", "ConvTranspose3d"] -allowed_types = [] -allowed_types.extend(linear_types) -allowed_types.extend(conv_types) -allowed_types.extend(conv_transpose_types) +dtype_dict = { + "int8": {"min": -128, "max": 127, "num_bits": 8, "torch_dtype": torch.int8, "storage_dtype": torch.int8, "is_unsigned": False, "is_integer": True}, + "uint8": {"min": 0, "max": 255, "num_bits": 8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "int4": {"min": -8, "max": 7, "num_bits": 4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, + "uint4": {"min": 0, "max": 15, "num_bits": 4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "float8_e4m3fn": {"min": -448, "max": 448, "num_bits": 8, "torch_dtype": torch.float8_e4m3fn, "storage_dtype": torch.float8_e4m3fn, "is_unsigned": False, "is_integer": False}, + "float8_e4m3fnuz": {"min": -240, "max": 240, "num_bits": 8, "torch_dtype": torch.float8_e4m3fnuz, "storage_dtype": torch.float8_e4m3fnuz, "is_unsigned": False, "is_integer": False}, + "float8_e5m2": {"min": -57344, "max": 57344, "num_bits": 8, "torch_dtype": torch.float8_e5m2, "storage_dtype": torch.float8_e5m2, "is_unsigned": False, "is_integer": False}, + "float8_e5m2fnuz": {"min": -57344, "max": 57344, "num_bits": 8, "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False}, +} +if hasattr(torch, "float8_e8m0fnu"): + dtype_dict["float8_e8m0fnu"] = {"min": 5.87747e-39, "max": 1.70141e+38, "num_bits": 8, "torch_dtype": torch.float8_e8m0fnu, "storage_dtype": torch.float8_e8m0fnu, "is_unsigned": True, "is_integer": False}, + +quantized_matmul_dtypes = ("int8", "int4", "float8_e4m3fn") + +linear_types = ("Linear",) +conv_types = ("Conv1d", "Conv2d", "Conv3d") +conv_transpose_types = ("ConvTranspose1d", "ConvTranspose2d", "ConvTranspose3d") +allowed_types = linear_types + conv_types + conv_transpose_types class QuantizationMethod(str, Enum): SDNQ = "sdnq" -def sdnq_quantize_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_conv=False, group_size=0, use_int8_matmul=False, param_name=None): # pylint: disable=unused-argument +def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, param_name=None): # pylint: disable=unused-argument layer_class_name = layer.__class__.__name__ if layer_class_name in allowed_types: is_conv_type = False @@ -47,21 +59,21 @@ def sdnq_quantize_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c if not quant_conv: return layer reduction_axes = [i for i in range(layer.weight.ndim) if i != 0] - use_int8_matmul = False + use_quantized_matmul = False is_conv_type = True elif layer_class_name in conv_transpose_types: if not quant_conv: return layer reduction_axes = [i for i in range(layer.weight.ndim) if i != 1] - use_int8_matmul = False + use_quantized_matmul = False is_conv_transpose_type = True else: is_linear_type = True reduction_axes = -1 channel_size = layer.weight.shape[-1] - use_int8_matmul = use_int8_matmul and not is_asym_mode and channel_size >= 32 and layer.weight.shape[0] >= 32 + use_quantized_matmul = use_quantized_matmul and weights_dtype in quantized_matmul_dtypes and channel_size >= 32 and layer.weight.shape[0] >= 32 - if not use_int8_matmul and (group_size > 0 or (num_bits == 4 and group_size != -1)): + if not use_quantized_matmul and (group_size > 0 or (dtype_dict[weights_dtype]["num_bits"] == 4 and group_size != -1)): if group_size == 0: group_size = 64 num_of_groups = channel_size // group_size @@ -87,62 +99,36 @@ def sdnq_quantize_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c layer.weight.data = layer.weight.reshape(new_shape) if shared.opts.diffusers_offload_mode != "none": - return_device = layer.weight.data.device + return_device = layer.weight.device else: return_device = devices.device layer.weight.data = layer.weight.data.to(devices.device, dtype=torch.float32) - if is_asym_mode: - scale, zero_point = get_int_scale_asymmetric(layer.weight, reduction_axes, num_bits) + if dtype_dict[weights_dtype]["is_unsigned"]: + scale, zero_point = get_int_scale_asymmetric(layer.weight, reduction_axes, weights_dtype) else: - scale = get_int_scale_symmetric(layer.weight, reduction_axes, num_bits) + scale = get_int_scale_symmetric(layer.weight, reduction_axes, weights_dtype) zero_point = None - compressed_weight = quantize_int(layer.weight, scale, zero_point, is_asym_mode, num_bits) + compressed_weight = quantize_weight(layer.weight, scale, zero_point, weights_dtype) if not shared.opts.sdnq_decompress_fp32: scale = scale.to(torch_dtype) if zero_point is not None: zero_point = zero_point.to(torch_dtype) - if use_int8_matmul: + if use_quantized_matmul: scale = scale.squeeze(-1) - if num_bits == 8: + if dtype_dict[weights_dtype]["num_bits"] == 8: compressed_weight = compressed_weight.transpose(0,1) - if num_bits == 4: - if is_asym_mode: - decompressor = INT4AsymmetricWeightsDecompressor( - scale=scale.data, - zero_point=zero_point.data, - compressed_weight_shape=compressed_weight.shape, - result_dtype=torch_dtype, - result_shape=result_shape, - use_int8_matmul=use_int8_matmul, - ) - else: - decompressor = INT4SymmetricWeightsDecompressor( - scale=scale.data, - compressed_weight_shape=compressed_weight.shape, - result_dtype=torch_dtype, - result_shape=result_shape, - use_int8_matmul=use_int8_matmul, - ) - else: - if is_asym_mode: - decompressor = INT8AsymmetricWeightsDecompressor( - scale=scale.data, - zero_point=zero_point.data, - result_dtype=torch_dtype, - result_shape=result_shape, - use_int8_matmul=use_int8_matmul, - ) - else: - decompressor = INT8SymmetricWeightsDecompressor( - scale=scale.data, - result_dtype=torch_dtype, - result_shape=result_shape, - use_int8_matmul=use_int8_matmul, - ) + decompressor = decompressor_dict[weights_dtype]( + scale=scale, + zero_point=zero_point, + compressed_weight_shape=compressed_weight.shape, + result_dtype=torch_dtype, + result_shape=result_shape, + use_quantized_matmul=use_quantized_matmul, + ) compressed_weight = decompressor.pack_weight(compressed_weight).to(return_device) decompressor = decompressor.to(return_device) @@ -150,9 +136,10 @@ def sdnq_quantize_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c layer.weight.requires_grad = False layer.weight.data = compressed_weight layer.sdnq_decompressor = decompressor + del compressed_weight, scale, zero_point if is_linear_type: - if use_int8_matmul: + if use_quantized_matmul: layer.forward = quantized_linear_forward_int8_matmul else: layer.forward = quantized_linear_forward @@ -169,235 +156,58 @@ def sdnq_quantize_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c return layer -def apply_sdnq_to_module(model, num_bits, is_asym_mode, quant_conv=False): +def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, param_name=None): has_children = list(model.children()) if not has_children: return model - for param_name, module in model.named_children(): + for module_param_name, module in model.named_children(): if hasattr(module, "weight") and module.weight is not None: module = sdnq_quantize_layer( module, - num_bits, - is_asym_mode, - torch_dtype=devices.dtype, + weights_dtype=weights_dtype, + torch_dtype=torch_dtype, + group_size=group_size, quant_conv=quant_conv, - group_size=shared.opts.sdnq_quantize_weights_group_size, - use_int8_matmul=shared.opts.sdnq_decompress_int8_matmul, - param_name=param_name, + use_quantized_matmul=use_quantized_matmul, + param_name=module_param_name, + ) + module = apply_sdnq_to_module( + module, + weights_dtype=weights_dtype, + torch_dtype=torch_dtype, + group_size=group_size, + quant_conv=quant_conv, + use_quantized_matmul=use_quantized_matmul, + param_name=module_param_name, ) - module = apply_sdnq_to_module(module, num_bits, is_asym_mode, quant_conv=quant_conv) return model -class SDNQQuantizer(DiffusersQuantizer): - r""" - Diffusers Quantizer for SDNQ - """ - - requires_parameters_quantization = True - use_keep_in_fp32_modules = True - requires_calibration = False - required_packages = None - torch_dtype = None - - def __init__(self, quantization_config, **kwargs): # pylint: disable=useless-parent-delegation - super().__init__(quantization_config, **kwargs) - - def check_if_quantized_param( - self, - model, - param_value: "torch.Tensor", - param_name: str, - state_dict: Dict[str, Any], - **kwargs, - ): - return param_name.endswith(".weight") - - def check_quantized_param(self, *args, **kwargs) -> bool: - """ - needed for transformers compatibilty, returns self.check_if_quantized_param - """ - return self.check_if_quantized_param(*args, **kwargs) - - def create_quantized_param( # pylint: disable=arguments-differ - self, - model, - param_value: "torch.Tensor", - param_name: str, - target_device: "torch.device", - state_dict: Dict[str, Any], # pylint: disable=unused-argument - unexpected_keys: List[str], # pylint: disable=unused-argument - **kwargs, - ): - # load the model params to target_device first - layer, tensor_name = get_module_from_name(model, param_name) - layer._parameters[tensor_name] = torch.nn.Parameter(param_value).to(device=target_device) # pylint: disable=protected-access - - split_param_name = param_name.split(".") - if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert): - layer = sdnq_quantize_layer( - layer, - self.quantization_config.num_bits, - self.quantization_config.is_asym_mode, - torch_dtype=self.torch_dtype, - group_size=self.quantization_config.group_size, - use_int8_matmul=self.quantization_config.use_int8_matmul, - param_name=param_name, - ) - - def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: - max_memory = {key: val * 0.70 for key, val in max_memory.items()} - return max_memory - - def adjust_target_dtype(self, target_dtype: "torch.dtype") -> "torch.dtype": # pylint: disable=unused-argument,arguments-renamed - return torch_dtype_dict[self.quantization_config.weights_dtype] - - def update_torch_dtype(self, torch_dtype: "torch.dtype" = None) -> "torch.dtype": - if torch_dtype is None: - torch_dtype = devices.dtype - self.torch_dtype = torch_dtype - return torch_dtype - - def _process_model_before_weight_loading( # pylint: disable=arguments-differ - self, - model, - device_map, # pylint: disable=unused-argument - keep_in_fp32_modules: List[str] = [], - **kwargs, - ): - model.config.quantization_config = self.quantization_config - self.modules_to_not_convert = self.quantization_config.modules_to_not_convert - if not isinstance(self.modules_to_not_convert, list): - self.modules_to_not_convert = [self.modules_to_not_convert] - if keep_in_fp32_modules is not None: - self.modules_to_not_convert.extend(keep_in_fp32_modules) - - def _process_model_after_weight_loading(self, model, **kwargs): - return model - - def update_tp_plan(self, config): - """ - needed for transformers compatibilty, no-op function - """ - return config - - def update_unexpected_keys(self, model, unexpected_keys: List[str], prefix: str) -> List[str]: # pylint: disable=unused-argument - """ - needed for transformers compatibilty, no-op function - """ - return unexpected_keys - - def update_missing_keys_after_loading(self, model, missing_keys: List[str], prefix: str) -> List[str]: # pylint: disable=unused-argument - """ - needed for transformers compatibilty, no-op function - """ - return missing_keys - - def update_expected_keys(self, model, expected_keys: List[str], loaded_keys: List[str]) -> List[str]: # pylint: disable=unused-argument - """ - needed for transformers compatibilty, no-op function - """ - return expected_keys - - @property - def is_trainable(self): - return False - - @property - def is_serializable(self): - return False - - -@dataclass -class SDNQConfig(QuantizationConfigMixin): - """ - This is a wrapper class about all possible attributes and features that you can play with a model that has been - loaded using `sdnq`. - - Args: - weights_dtype (`str`, *optional*, defaults to `"int8"`): - The target dtype for the weights after quantization. Supported values are ("int8", "uint8", "int4", "uint4") - modules_to_not_convert (`list`, *optional*, default to `None`): - The list of modules to not quantize, useful for quantizing models that explicitly require to have some - modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers). - """ - - def __init__( # pylint: disable=super-init-not-called - self, - weights_dtype: str = "int8", - group_size: int = 0, - use_int8_matmul: bool = False, - modules_to_not_convert: Optional[List[str]] = None, - **kwargs, # pylint: disable=unused-argument - ): - self.weights_dtype = weights_dtype - self.quant_method = QuantizationMethod.SDNQ - self.group_size = group_size - self.use_int8_matmul = use_int8_matmul - self.modules_to_not_convert = modules_to_not_convert - self.num_bits = 8 if self.weights_dtype in {"int8", "uint8"} else 4 - self.is_asym_mode = self.weights_dtype in {"uint8", "uint4"} - self.is_integer = True - self.post_init() - - def post_init(self): - r""" - Safety checker that arguments are correct - """ - accepted_weights = ["int8", "uint8", "int4", "uint4"] - if self.weights_dtype not in accepted_weights: - raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights_dtype}") - - -class SDNQ_T5DenseGatedActDense(torch.nn.Module): # forward can't find what self is without creating a class - def __init__(self, T5DenseGatedActDense, dtype): - super().__init__() - self.wi_0 = T5DenseGatedActDense.wi_0 - self.wi_1 = T5DenseGatedActDense.wi_1 - self.wo = T5DenseGatedActDense.wo - self.dropout = T5DenseGatedActDense.dropout - self.act = T5DenseGatedActDense.act - self.torch_dtype = dtype - - def forward(self, hidden_states): - hidden_gelu = self.act(self.wi_0(hidden_states)) - hidden_linear = self.wi_1(hidden_states) - hidden_states = hidden_gelu * hidden_linear - hidden_states = self.dropout(hidden_states) - hidden_states = hidden_states.to(self.torch_dtype) # this line needs to be forced - hidden_states = self.wo(hidden_states) - return hidden_states - - -def get_int_scale_asymmetric(weight: torch.FloatTensor, reduction_axes: List[int], num_bits: int) -> Tuple[torch.FloatTensor, torch.FloatTensor]: +def get_int_scale_asymmetric(weight: torch.FloatTensor, reduction_axes: List[int], weights_dtype: str) -> Tuple[torch.FloatTensor, torch.FloatTensor]: zero_point = torch.amin(weight, dim=reduction_axes, keepdims=True) - max_values = torch.amax(weight, dim=reduction_axes, keepdims=True) - scale = (max_values - zero_point) / (2**num_bits - 1) + scale = torch.amax(weight, dim=reduction_axes, keepdims=True).sub_(zero_point).div_(dtype_dict[weights_dtype]["max"]) eps = torch.finfo(scale.dtype).eps # prevent divison by 0 scale = torch.where(torch.abs(scale) < eps, eps, scale) + if dtype_dict[weights_dtype]["min"] != 0: + zero_point.add_(dtype_dict[weights_dtype]["min"]) return scale, zero_point -def get_int_scale_symmetric(weight: torch.FloatTensor, reduction_axes: List[int], num_bits: int) -> torch.FloatTensor: - w_abs_min = torch.abs(torch.amin(weight, dim=reduction_axes, keepdims=True)) - w_max = torch.amax(weight, dim=reduction_axes, keepdims=True) - scale = torch.where(w_abs_min >= w_max, w_abs_min, -w_max) / (2 ** (num_bits - 1)) +def get_int_scale_symmetric(weight: torch.FloatTensor, reduction_axes: List[int], weights_dtype: str) -> torch.FloatTensor: + abs_min_values = torch.amin(weight, dim=reduction_axes, keepdims=True).abs_() + max_values = torch.amax(weight, dim=reduction_axes, keepdims=True) + scale = torch.where(abs_min_values >= max_values, abs_min_values, -max_values).div_(dtype_dict[weights_dtype]["max"]) eps = torch.finfo(scale.dtype).eps # prevent divison by 0 scale = torch.where(torch.abs(scale) < eps, eps, scale) return scale -def quantize_int(weight: torch.FloatTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, is_asym_mode: bool, num_bits: int, flatten: Optional[bool] = False) -> torch.ByteTensor: - dtype = torch.uint8 if is_asym_mode else torch.int8 - level_low = 0 if is_asym_mode else -(2 ** (num_bits - 1)) - level_high = 2**num_bits - 1 if is_asym_mode else 2 ** (num_bits - 1) - 1 +def quantize_weight(weight: torch.FloatTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, weights_dtype: str) -> torch.ByteTensor: if zero_point is not None: compressed_weight = torch.sub(weight, zero_point).div_(scale) else: compressed_weight = torch.div(weight, scale) - compressed_weight = compressed_weight.round_().clamp_(level_low, level_high).to(dtype) - if flatten: - compressed_weight = compressed_weight.flatten(0,-2) + compressed_weight = compressed_weight.round_().clamp_(dtype_dict[weights_dtype]["min"], dtype_dict[weights_dtype]["max"]).to(dtype_dict[weights_dtype]["torch_dtype"]) return compressed_weight @@ -519,16 +329,15 @@ class INT8AsymmetricWeightsDecompressor(torch.nn.Module): zero_point: torch.Tensor, result_dtype: torch.dtype, result_shape: torch.Size, - use_int8_matmul: bool = False, + **kwargs, ): super().__init__() - self.num_bits = 8 - self.is_asym_mode = True + self.weights_dtype = "uint8" + self.use_quantized_matmul = False self.scale = scale self.zero_point = zero_point self.result_dtype = result_dtype self.result_shape = result_shape - self.use_int8_matmul = use_int8_matmul def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: if debug: @@ -546,15 +355,15 @@ class INT8SymmetricWeightsDecompressor(torch.nn.Module): scale: torch.Tensor, result_dtype: torch.dtype, result_shape: torch.Size, - use_int8_matmul: bool = False, + use_quantized_matmul: bool = False, + **kwargs, ): super().__init__() - self.num_bits = 8 - self.is_asym_mode = False + self.weights_dtype = "int8" + self.use_quantized_matmul = use_quantized_matmul self.scale = scale self.result_dtype = result_dtype self.result_shape = result_shape - self.use_int8_matmul = use_int8_matmul def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: if debug: @@ -574,17 +383,16 @@ class INT4AsymmetricWeightsDecompressor(torch.nn.Module): compressed_weight_shape: torch.Size, result_dtype: torch.dtype, result_shape: torch.Size, - use_int8_matmul: bool = False, + **kwargs, ): super().__init__() - self.num_bits = 4 - self.is_asym_mode = True + self.weights_dtype = "uint4" + self.use_quantized_matmul = False self.scale = scale self.zero_point = zero_point self.compressed_weight_shape = compressed_weight_shape self.result_dtype = result_dtype self.result_shape = result_shape - self.use_int8_matmul = use_int8_matmul def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: if debug: @@ -603,16 +411,16 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): compressed_weight_shape: torch.Size, result_dtype: torch.dtype, result_shape: torch.Size, - use_int8_matmul: bool = False, + use_quantized_matmul: bool = False, + **kwargs, ): super().__init__() - self.num_bits = 4 - self.is_asym_mode = False + self.weights_dtype = "int4" + self.use_quantized_matmul = use_quantized_matmul self.scale = scale self.compressed_weight_shape = compressed_weight_shape self.result_dtype = result_dtype self.result_shape = result_shape - self.use_int8_matmul = use_int8_matmul def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: if debug: @@ -624,6 +432,193 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): return decompress_int4_symmetric_compiled(weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape, skip_int8_matmul=skip_int8_matmul) +decompressor_dict = { + "int8": INT8SymmetricWeightsDecompressor, + "uint8": INT8AsymmetricWeightsDecompressor, + "int4": INT4SymmetricWeightsDecompressor, + "uint4": INT4AsymmetricWeightsDecompressor, +} + + +class SDNQQuantizer(DiffusersQuantizer): + r""" + Diffusers Quantizer for SDNQ + """ + + requires_parameters_quantization = True + use_keep_in_fp32_modules = True + requires_calibration = False + required_packages = None + torch_dtype = None + + def __init__(self, quantization_config, **kwargs): # pylint: disable=useless-parent-delegation + super().__init__(quantization_config, **kwargs) + + def check_if_quantized_param( + self, + model, + param_value: "torch.Tensor", + param_name: str, + state_dict: Dict[str, Any], + **kwargs, + ): + return param_name.endswith(".weight") + + def check_quantized_param(self, *args, **kwargs) -> bool: + """ + needed for transformers compatibilty, returns self.check_if_quantized_param + """ + return self.check_if_quantized_param(*args, **kwargs) + + def create_quantized_param( # pylint: disable=arguments-differ + self, + model, + param_value: "torch.Tensor", + param_name: str, + target_device: "torch.device", + state_dict: Dict[str, Any], # pylint: disable=unused-argument + unexpected_keys: List[str], # pylint: disable=unused-argument + **kwargs, + ): + # load the model params to target_device first + layer, tensor_name = get_module_from_name(model, param_name) + layer._parameters[tensor_name] = torch.nn.Parameter(param_value).to(device=target_device) # pylint: disable=protected-access + + split_param_name = param_name.split(".") + if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert): + layer = sdnq_quantize_layer( + layer, + weights_dtype=self.quantization_config.weights_dtype, + torch_dtype=self.torch_dtype, + group_size=self.quantization_config.group_size, + quant_conv=self.quantization_config.quant_conv, + use_quantized_matmul=self.quantization_config.use_quantized_matmul, + param_name=param_name, + ) + + def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: + max_memory = {key: val * 0.70 for key, val in max_memory.items()} + return max_memory + + def adjust_target_dtype(self, target_dtype: "torch.dtype") -> "torch.dtype": # pylint: disable=unused-argument,arguments-renamed + return torch_dtype_dict[self.quantization_config.weights_dtype] + + def update_torch_dtype(self, torch_dtype: "torch.dtype" = None) -> "torch.dtype": + if torch_dtype is None: + torch_dtype = devices.dtype + self.torch_dtype = torch_dtype + return torch_dtype + + def _process_model_before_weight_loading( # pylint: disable=arguments-differ + self, + model, + device_map, # pylint: disable=unused-argument + keep_in_fp32_modules: List[str] = [], + **kwargs, + ): + model.config.quantization_config = self.quantization_config + self.modules_to_not_convert = self.quantization_config.modules_to_not_convert + if not isinstance(self.modules_to_not_convert, list): + self.modules_to_not_convert = [self.modules_to_not_convert] + if keep_in_fp32_modules is not None: + self.modules_to_not_convert.extend(keep_in_fp32_modules) + + def _process_model_after_weight_loading(self, model, **kwargs): + return model + + def update_tp_plan(self, config): + """ + needed for transformers compatibilty, no-op function + """ + return config + + def update_unexpected_keys(self, model, unexpected_keys: List[str], prefix: str) -> List[str]: # pylint: disable=unused-argument + """ + needed for transformers compatibilty, no-op function + """ + return unexpected_keys + + def update_missing_keys_after_loading(self, model, missing_keys: List[str], prefix: str) -> List[str]: # pylint: disable=unused-argument + """ + needed for transformers compatibilty, no-op function + """ + return missing_keys + + def update_expected_keys(self, model, expected_keys: List[str], loaded_keys: List[str]) -> List[str]: # pylint: disable=unused-argument + """ + needed for transformers compatibilty, no-op function + """ + return expected_keys + + @property + def is_trainable(self): + return False + + @property + def is_serializable(self): + return False + + +@dataclass +class SDNQConfig(QuantizationConfigMixin): + """ + This is a wrapper class about all possible attributes and features that you can play with a model that has been + loaded using `sdnq`. + + Args: + weights_dtype (`str`, *optional*, defaults to `"int8"`): + The target dtype for the weights after quantization. Supported values are ("int8", "uint8", "int4", "uint4") + modules_to_not_convert (`list`, *optional*, default to `None`): + The list of modules to not quantize, useful for quantizing models that explicitly require to have some + modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers). + """ + + def __init__( # pylint: disable=super-init-not-called + self, + weights_dtype: str = "int8", + group_size: int = 0, + quant_conv: bool = False, + use_quantized_matmul: bool = False, + modules_to_not_convert: Optional[List[str]] = None, + **kwargs, # pylint: disable=unused-argument + ): + self.weights_dtype = weights_dtype + self.quant_method = QuantizationMethod.SDNQ + self.group_size = group_size + self.quant_conv = quant_conv + self.use_quantized_matmul = use_quantized_matmul + self.modules_to_not_convert = modules_to_not_convert + self.post_init() + + def post_init(self): + r""" + Safety checker that arguments are correct + """ + accepted_weights = ["int8", "uint8", "int4", "uint4"] + if self.weights_dtype not in accepted_weights: + raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights_dtype}") + + +class SDNQ_T5DenseGatedActDense(torch.nn.Module): # forward can't find what self is without creating a class + def __init__(self, T5DenseGatedActDense, dtype): + super().__init__() + self.wi_0 = T5DenseGatedActDense.wi_0 + self.wi_1 = T5DenseGatedActDense.wi_1 + self.wo = T5DenseGatedActDense.wo + self.dropout = T5DenseGatedActDense.dropout + self.act = T5DenseGatedActDense.act + self.torch_dtype = dtype + + def forward(self, hidden_states): + hidden_gelu = self.act(self.wi_0(hidden_states)) + hidden_linear = self.wi_1(hidden_states) + hidden_states = hidden_gelu * hidden_linear + hidden_states = self.dropout(hidden_states) + hidden_states = hidden_states.to(self.torch_dtype) # this line needs to be forced + hidden_states = self.wo(hidden_states) + return hidden_states + + if shared.opts.sdnq_decompress_compile: try: torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit) # pylint: disable=protected-access diff --git a/modules/shared.py b/modules/shared.py index b9eb6518a..e4c3c8a6e 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -530,7 +530,7 @@ options_templates.update(options_section(('quantization', "Quantization Settings "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), "sdnq_decompress_compile": OptionInfo(devices.has_triton(), "Decompress using torch.compile", gr.Checkbox, {"visible": native}), - "sdnq_decompress_int8_matmul": OptionInfo(False, "Use direct INT8 MatMul", gr.Checkbox, {"visible": native}), + "sdnq_use_quantized_matmul": OptionInfo(False, "Use quantized MatMul", gr.Checkbox, {"visible": native}), "sdnq_quantize_shuffle_weights": OptionInfo(False, "Shuffle weights in post mode", gr.Checkbox, {"visible": native}), "bnb_quantization_sep": OptionInfo("

BitsAndBytes

", "", gr.HTML), From e94128a02e9cc0bfc843cb80304dc12e69813302 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 27 May 2025 16:11:04 +0300 Subject: [PATCH 36/94] SDNQ add force torch_gc to pre load mode --- modules/model_quant_sdnq.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index ad442a145..1fbb0f125 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -13,25 +13,19 @@ from modules import devices, shared debug = os.environ.get('SD_QUANT_DEBUG', None) is not None -torch_dtype_dict = { - "int8": torch.int8, - "uint8": torch.uint8, - "int4": CustomDtype.INT4, - "uint4": CustomDtype.INT4, -} dtype_dict = { - "int8": {"min": -128, "max": 127, "num_bits": 8, "torch_dtype": torch.int8, "storage_dtype": torch.int8, "is_unsigned": False, "is_integer": True}, - "uint8": {"min": 0, "max": 255, "num_bits": 8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, - "int4": {"min": -8, "max": 7, "num_bits": 4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, - "uint4": {"min": 0, "max": 15, "num_bits": 4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, - "float8_e4m3fn": {"min": -448, "max": 448, "num_bits": 8, "torch_dtype": torch.float8_e4m3fn, "storage_dtype": torch.float8_e4m3fn, "is_unsigned": False, "is_integer": False}, - "float8_e4m3fnuz": {"min": -240, "max": 240, "num_bits": 8, "torch_dtype": torch.float8_e4m3fnuz, "storage_dtype": torch.float8_e4m3fnuz, "is_unsigned": False, "is_integer": False}, - "float8_e5m2": {"min": -57344, "max": 57344, "num_bits": 8, "torch_dtype": torch.float8_e5m2, "storage_dtype": torch.float8_e5m2, "is_unsigned": False, "is_integer": False}, - "float8_e5m2fnuz": {"min": -57344, "max": 57344, "num_bits": 8, "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False}, + "int8": {"min": -128, "max": 127, "num_bits": 8, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.int8, "is_unsigned": False, "is_integer": True}, + "uint8": {"min": 0, "max": 255, "num_bits": 8, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "int4": {"min": -8, "max": 7, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, + "uint4": {"min": 0, "max": 15, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "float8_e4m3fn": {"min": -448, "max": 448, "num_bits": 8, "target_dtype": torch.float8_e4m3fn, "torch_dtype": torch.float8_e4m3fn, "storage_dtype": torch.float8_e4m3fn, "is_unsigned": False, "is_integer": False}, + "float8_e4m3fnuz": {"min": -240, "max": 240, "num_bits": 8, "target_dtype": torch.float8_e4m3fnuz, "torch_dtype": torch.float8_e4m3fnuz, "storage_dtype": torch.float8_e4m3fnuz, "is_unsigned": False, "is_integer": False}, + "float8_e5m2": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": torch.float8_e5m2, "torch_dtype": torch.float8_e5m2, "storage_dtype": torch.float8_e5m2, "is_unsigned": False, "is_integer": False}, + "float8_e5m2fnuz": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": torch.float8_e5m2fnuz, "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False}, } if hasattr(torch, "float8_e8m0fnu"): - dtype_dict["float8_e8m0fnu"] = {"min": 5.87747e-39, "max": 1.70141e+38, "num_bits": 8, "torch_dtype": torch.float8_e8m0fnu, "storage_dtype": torch.float8_e8m0fnu, "is_unsigned": True, "is_integer": False}, + dtype_dict["float8_e8m0fnu"] = {"min": 5.87747e-39, "max": 1.70141e+38, "num_bits": 8, "target_dtype": torch.float8_e8m0fnu, "torch_dtype": torch.float8_e8m0fnu, "storage_dtype": torch.float8_e8m0fnu, "is_unsigned": True, "is_integer": False}, quantized_matmul_dtypes = ("int8", "int4", "float8_e4m3fn") @@ -501,7 +495,7 @@ class SDNQQuantizer(DiffusersQuantizer): return max_memory def adjust_target_dtype(self, target_dtype: "torch.dtype") -> "torch.dtype": # pylint: disable=unused-argument,arguments-renamed - return torch_dtype_dict[self.quantization_config.weights_dtype] + return dtype_dict[self.quantization_config.weights_dtype]["target_dtype"] def update_torch_dtype(self, torch_dtype: "torch.dtype" = None) -> "torch.dtype": if torch_dtype is None: @@ -524,6 +518,7 @@ class SDNQQuantizer(DiffusersQuantizer): self.modules_to_not_convert.extend(keep_in_fp32_modules) def _process_model_after_weight_loading(self, model, **kwargs): + devices.torch_gc(force=True) return model def update_tp_plan(self, config): From 73999ac7104feae4749ad41febefc4eec66c80be Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 27 May 2025 16:24:04 +0300 Subject: [PATCH 37/94] Add soft gc to nncf quant layer --- modules/model_quant_sdnq.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 1fbb0f125..9a1be07ad 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -147,6 +147,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz elif layer_class_name.endswith("3d"): layer.forward = quantized_conv_transpose_3d_forward layer.forward = layer.forward.__get__(layer, layer.__class__) + devices.torch_gc(force=False, reason=f"SDNQ param_name: {param_name}") return layer From 3618e39cff8025020d8e727a8835e5de07a8a042 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 27 May 2025 19:46:30 +0300 Subject: [PATCH 38/94] SDNQ use device_map = gpu --- modules/model_quant.py | 26 ++++++++++------- modules/model_quant_sdnq.py | 58 ++++++++++++++++++------------------- 2 files changed, 43 insertions(+), 41 deletions(-) diff --git a/modules/model_quant.py b/modules/model_quant.py index 05c9585b3..5a5dff44a 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -151,25 +151,25 @@ def check_nunchaku(module: str = ''): def create_config(kwargs = None, allow: bool = True, module: str = 'Model'): if kwargs is None: kwargs = {} + kwargs = create_sdnq_config(kwargs, allow_sdnq=allow, module=module) + if kwargs is not None and 'quantization_config' in kwargs: + if debug: + log.trace(f'Quantization: type=sdnq config={kwargs.get("quantization_config", None)}') + return kwargs kwargs = create_bnb_config(kwargs, allow_bnb=allow, module=module) if kwargs is not None and 'quantization_config' in kwargs: if debug: log.trace(f'Quantization: type=bnb config={kwargs.get("quantization_config", None)}') return kwargs - kwargs = create_ao_config(kwargs, allow_ao=allow, module=module) - if kwargs is not None and 'quantization_config' in kwargs: - if debug: - log.trace(f'Quantization: type=torchao config={kwargs.get("quantization_config", None)}') - return kwargs kwargs = create_quanto_config(kwargs, allow_quanto=allow, module=module) if kwargs is not None and 'quantization_config' in kwargs: if debug: log.trace(f'Quantization: type=quanto config={kwargs.get("quantization_config", None)}') return kwargs - kwargs = create_sdnq_config(kwargs, allow_sdnq=allow, module=module) + kwargs = create_ao_config(kwargs, allow_ao=allow, module=module) if kwargs is not None and 'quantization_config' in kwargs: if debug: - log.trace(f'Quantization: type=sdnq config={kwargs.get("quantization_config", None)}') + log.trace(f'Quantization: type=torchao config={kwargs.get("quantization_config", None)}') return kwargs return kwargs @@ -349,6 +349,8 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc=True): quant_last_model_name = None quant_last_model_device = None model.to(devices.device) + elif shared.opts.diffusers_offload_mode == "model": + model = model.to(devices.cpu) if do_gc: devices.torch_gc(force=True) return model @@ -521,12 +523,14 @@ def get_dit_args(load_config:dict={}, module:str=None, device_map:bool=False, al # if 'variant' in config: # del config['variant'] if device_map: - if shared.opts.device_map == 'cpu': - config['device_map'] = 'cpu' - if shared.opts.device_map == 'gpu': - config['device_map'] = devices.device if devices.backend == "ipex" and os.environ.get('UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS', '0') != '1' and module in {'TE', 'LLM'}: config['device_map'] = 'cpu' # alchemist gpus hits the 4GB allocation limit with transformers, UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS emulates above 4GB allocations + elif shared.opts.device_map == 'cpu': + config['device_map'] = 'cpu' + elif shared.opts.device_map == 'gpu': + config['device_map'] = devices.device + elif 'Model' in shared.opts.sdnq_quantize_weights or (module is not None and module in shared.opts.sdnq_quantize_weights) or module == 'any': + config['device_map'] = devices.device if allow_quant: quant_args = create_config(module=module) else: diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 9a1be07ad..6d37774a0 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -92,10 +92,11 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz new_shape[last_dim_index - 1 : last_dim_index] = (int(num_of_groups), int(group_size)) layer.weight.data = layer.weight.reshape(new_shape) - if shared.opts.diffusers_offload_mode != "none": - return_device = layer.weight.device - else: + layer.weight.requires_grad = False + if shared.opts.diffusers_offload_mode in {"none", "model"}: return_device = devices.device + else: + return_device = devices.cpu layer.weight.data = layer.weight.data.to(devices.device, dtype=torch.float32) if dtype_dict[weights_dtype]["is_unsigned"]: @@ -103,7 +104,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz else: scale = get_int_scale_symmetric(layer.weight, reduction_axes, weights_dtype) zero_point = None - compressed_weight = quantize_weight(layer.weight, scale, zero_point, weights_dtype) + layer.weight.data = quantize_weight(layer.weight, scale, zero_point, weights_dtype) if not shared.opts.sdnq_decompress_fp32: scale = scale.to(torch_dtype) @@ -113,24 +114,17 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if use_quantized_matmul: scale = scale.squeeze(-1) if dtype_dict[weights_dtype]["num_bits"] == 8: - compressed_weight = compressed_weight.transpose(0,1) + layer.weight.data = layer.weight.data.transpose(0,1) - decompressor = decompressor_dict[weights_dtype]( + layer.sdnq_decompressor = decompressor_dict[weights_dtype]( scale=scale, zero_point=zero_point, - compressed_weight_shape=compressed_weight.shape, + compressed_weight_shape=layer.weight.data.shape, result_dtype=torch_dtype, result_shape=result_shape, use_quantized_matmul=use_quantized_matmul, - ) - - compressed_weight = decompressor.pack_weight(compressed_weight).to(return_device) - decompressor = decompressor.to(return_device) - - layer.weight.requires_grad = False - layer.weight.data = compressed_weight - layer.sdnq_decompressor = decompressor - del compressed_weight, scale, zero_point + ).to(return_device) + layer.weight.data = layer.sdnq_decompressor.pack_weight(layer.weight.data).to(return_device) if is_linear_type: if use_quantized_matmul: @@ -457,7 +451,12 @@ class SDNQQuantizer(DiffusersQuantizer): state_dict: Dict[str, Any], **kwargs, ): - return param_name.endswith(".weight") + if param_name.endswith(".weight"): + split_param_name = param_name.split(".") + if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert): + layer, _ = get_module_from_name(model, param_name) + return layer.__class__.__name__ in allowed_types + return False def check_quantized_param(self, *args, **kwargs) -> bool: """ @@ -477,19 +476,16 @@ class SDNQQuantizer(DiffusersQuantizer): ): # load the model params to target_device first layer, tensor_name = get_module_from_name(model, param_name) - layer._parameters[tensor_name] = torch.nn.Parameter(param_value).to(device=target_device) # pylint: disable=protected-access - - split_param_name = param_name.split(".") - if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert): - layer = sdnq_quantize_layer( - layer, - weights_dtype=self.quantization_config.weights_dtype, - torch_dtype=self.torch_dtype, - group_size=self.quantization_config.group_size, - quant_conv=self.quantization_config.quant_conv, - use_quantized_matmul=self.quantization_config.use_quantized_matmul, - param_name=param_name, - ) + layer.weight = torch.nn.Parameter(param_value.to(device=target_device), requires_grad=False) + layer = sdnq_quantize_layer( + layer, + weights_dtype=self.quantization_config.weights_dtype, + torch_dtype=self.torch_dtype, + group_size=self.quantization_config.group_size, + quant_conv=self.quantization_config.quant_conv, + use_quantized_matmul=self.quantization_config.use_quantized_matmul, + param_name=param_name, + ) def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: max_memory = {key: val * 0.70 for key, val in max_memory.items()} @@ -519,6 +515,8 @@ class SDNQQuantizer(DiffusersQuantizer): self.modules_to_not_convert.extend(keep_in_fp32_modules) def _process_model_after_weight_loading(self, model, **kwargs): + if shared.opts.diffusers_offload_mode == "model": + model = model.to(devices.cpu) devices.torch_gc(force=True) return model From 5d3c1832b2202333d9ee053dc4d17d470fcce67f Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 27 May 2025 20:29:15 +0300 Subject: [PATCH 39/94] SDNQ add FP8 quants --- modules/model_quant_sdnq.py | 62 ++++++++++++++++++------------------- modules/shared.py | 2 +- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 6d37774a0..0e4d1bff5 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -20,14 +20,14 @@ dtype_dict = { "int4": {"min": -8, "max": 7, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, "uint4": {"min": 0, "max": 15, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "float8_e4m3fn": {"min": -448, "max": 448, "num_bits": 8, "target_dtype": torch.float8_e4m3fn, "torch_dtype": torch.float8_e4m3fn, "storage_dtype": torch.float8_e4m3fn, "is_unsigned": False, "is_integer": False}, - "float8_e4m3fnuz": {"min": -240, "max": 240, "num_bits": 8, "target_dtype": torch.float8_e4m3fnuz, "torch_dtype": torch.float8_e4m3fnuz, "storage_dtype": torch.float8_e4m3fnuz, "is_unsigned": False, "is_integer": False}, "float8_e5m2": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": torch.float8_e5m2, "torch_dtype": torch.float8_e5m2, "storage_dtype": torch.float8_e5m2, "is_unsigned": False, "is_integer": False}, - "float8_e5m2fnuz": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": torch.float8_e5m2fnuz, "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False}, + "float8_e4m3fnuz": {"min": -240, "max": 240, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e4m3fnuz, "storage_dtype": torch.float8_e4m3fnuz, "is_unsigned": False, "is_integer": False}, + "float8_e5m2fnuz": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False}, } if hasattr(torch, "float8_e8m0fnu"): - dtype_dict["float8_e8m0fnu"] = {"min": 5.87747e-39, "max": 1.70141e+38, "num_bits": 8, "target_dtype": torch.float8_e8m0fnu, "torch_dtype": torch.float8_e8m0fnu, "storage_dtype": torch.float8_e8m0fnu, "is_unsigned": True, "is_integer": False}, + dtype_dict["float8_e8m0fnu"] = {"min": 5.87747e-39, "max": 1.70141e+38, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e8m0fnu, "storage_dtype": torch.float8_e8m0fnu, "is_unsigned": True, "is_integer": False} -quantized_matmul_dtypes = ("int8", "int4", "float8_e4m3fn") +quantized_matmul_dtypes = ("int8", "int4") # todo: float8_e4m3fn linear_types = ("Linear",) conv_types = ("Conv1d", "Conv2d", "Conv3d") @@ -100,9 +100,9 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz layer.weight.data = layer.weight.data.to(devices.device, dtype=torch.float32) if dtype_dict[weights_dtype]["is_unsigned"]: - scale, zero_point = get_int_scale_asymmetric(layer.weight, reduction_axes, weights_dtype) + scale, zero_point = get_scale_asymmetric(layer.weight, reduction_axes, weights_dtype) else: - scale = get_int_scale_symmetric(layer.weight, reduction_axes, weights_dtype) + scale = get_scale_symmetric(layer.weight, reduction_axes, weights_dtype) zero_point = None layer.weight.data = quantize_weight(layer.weight, scale, zero_point, weights_dtype) @@ -123,6 +123,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz result_dtype=torch_dtype, result_shape=result_shape, use_quantized_matmul=use_quantized_matmul, + weights_dtype=weights_dtype, ).to(return_device) layer.weight.data = layer.sdnq_decompressor.pack_weight(layer.weight.data).to(return_device) @@ -172,7 +173,7 @@ def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_si return model -def get_int_scale_asymmetric(weight: torch.FloatTensor, reduction_axes: List[int], weights_dtype: str) -> Tuple[torch.FloatTensor, torch.FloatTensor]: +def get_scale_asymmetric(weight: torch.FloatTensor, reduction_axes: List[int], weights_dtype: str) -> Tuple[torch.FloatTensor, torch.FloatTensor]: zero_point = torch.amin(weight, dim=reduction_axes, keepdims=True) scale = torch.amax(weight, dim=reduction_axes, keepdims=True).sub_(zero_point).div_(dtype_dict[weights_dtype]["max"]) eps = torch.finfo(scale.dtype).eps # prevent divison by 0 @@ -182,7 +183,7 @@ def get_int_scale_asymmetric(weight: torch.FloatTensor, reduction_axes: List[int return scale, zero_point -def get_int_scale_symmetric(weight: torch.FloatTensor, reduction_axes: List[int], weights_dtype: str) -> torch.FloatTensor: +def get_scale_symmetric(weight: torch.FloatTensor, reduction_axes: List[int], weights_dtype: str) -> torch.FloatTensor: abs_min_values = torch.amin(weight, dim=reduction_axes, keepdims=True).abs_() max_values = torch.amax(weight, dim=reduction_axes, keepdims=True) scale = torch.where(abs_min_values >= max_values, abs_min_values, -max_values).div_(dtype_dict[weights_dtype]["max"]) @@ -196,7 +197,9 @@ def quantize_weight(weight: torch.FloatTensor, scale: torch.FloatTensor, zero_po compressed_weight = torch.sub(weight, zero_point).div_(scale) else: compressed_weight = torch.div(weight, scale) - compressed_weight = compressed_weight.round_().clamp_(dtype_dict[weights_dtype]["min"], dtype_dict[weights_dtype]["max"]).to(dtype_dict[weights_dtype]["torch_dtype"]) + if dtype_dict[weights_dtype]["is_integer"]: + compressed_weight.round_() + compressed_weight = compressed_weight.clamp_(dtype_dict[weights_dtype]["min"], dtype_dict[weights_dtype]["max"]).to(dtype_dict[weights_dtype]["torch_dtype"]) return compressed_weight @@ -311,17 +314,18 @@ def quantized_conv_transpose_3d_forward(self, input: torch.FloatTensor, output_s return torch.nn.functional.conv_transpose3d(input, self.sdnq_decompressor(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) -class INT8AsymmetricWeightsDecompressor(torch.nn.Module): +class AsymmetricWeightsDecompressor(torch.nn.Module): def __init__( self, scale: torch.Tensor, zero_point: torch.Tensor, result_dtype: torch.dtype, result_shape: torch.Size, + weights_dtype: torch.dtype, **kwargs, ): super().__init__() - self.weights_dtype = "uint8" + self.weights_dtype = weights_dtype self.use_quantized_matmul = False self.scale = scale self.zero_point = zero_point @@ -329,36 +333,31 @@ class INT8AsymmetricWeightsDecompressor(torch.nn.Module): self.result_shape = result_shape def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: - if debug: - if torch.any((weight < 0) | (weight > 255)): - raise ValueError("Weight values are not in [0, 255].") - return weight.to(dtype=torch.uint8) + return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"]) def forward(self, weight, **kwargs): return decompress_asymmetric_compiled(weight, self.scale, self.zero_point, self.result_dtype, self.result_shape) -class INT8SymmetricWeightsDecompressor(torch.nn.Module): +class SymmetricWeightsDecompressor(torch.nn.Module): def __init__( self, scale: torch.Tensor, result_dtype: torch.dtype, result_shape: torch.Size, + weights_dtype: torch.dtype, use_quantized_matmul: bool = False, **kwargs, ): super().__init__() - self.weights_dtype = "int8" + self.weights_dtype = weights_dtype self.use_quantized_matmul = use_quantized_matmul self.scale = scale self.result_dtype = result_dtype self.result_shape = result_shape def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: - if debug: - if torch.any((weight < -128) | (weight > 127)): - raise ValueError("Weight values are not in [-128, 127].") - return weight.to(dtype=torch.int8) + return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"]) def forward(self, weight, skip_int8_matmul=False, **kwargs): return decompress_symmetric_compiled(weight, self.scale, self.result_dtype, self.result_shape, skip_int8_matmul=skip_int8_matmul) @@ -384,9 +383,6 @@ class INT4AsymmetricWeightsDecompressor(torch.nn.Module): self.result_shape = result_shape def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: - if debug: - if torch.any((weight < 0) | (weight > 15)): - raise ValueError("Weight values are not in [0, 15].") return pack_uint4(weight.to(dtype=torch.uint8)) def forward(self, weight, **kwargs): @@ -412,9 +408,6 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): self.result_shape = result_shape def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: - if debug: - if torch.any((weight < -8) | (weight > 7)): - raise ValueError("Tensor values are not in [-8, 7].") return pack_int4(weight.to(dtype=torch.int8)) def forward(self, weight, skip_int8_matmul=False, **kwargs): @@ -422,10 +415,15 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): decompressor_dict = { - "int8": INT8SymmetricWeightsDecompressor, - "uint8": INT8AsymmetricWeightsDecompressor, + "int8": SymmetricWeightsDecompressor, + "uint8": AsymmetricWeightsDecompressor, "int4": INT4SymmetricWeightsDecompressor, "uint4": INT4AsymmetricWeightsDecompressor, + "float8_e4m3fn": SymmetricWeightsDecompressor, + "float8_e4m3fnuz": SymmetricWeightsDecompressor, + "float8_e5m2": SymmetricWeightsDecompressor, + "float8_e5m2fnuz": SymmetricWeightsDecompressor, + "float8_e8m0fnu": AsymmetricWeightsDecompressor, } @@ -561,7 +559,8 @@ class SDNQConfig(QuantizationConfigMixin): Args: weights_dtype (`str`, *optional*, defaults to `"int8"`): - The target dtype for the weights after quantization. Supported values are ("int8", "uint8", "int4", "uint4") + The target dtype for the weights after quantization. Supported values are: + ("int8", "uint8", "int4", "uint4", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz", "float8_e8m0fnu") modules_to_not_convert (`list`, *optional*, default to `None`): The list of modules to not quantize, useful for quantizing models that explicitly require to have some modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers). @@ -583,12 +582,13 @@ class SDNQConfig(QuantizationConfigMixin): self.use_quantized_matmul = use_quantized_matmul self.modules_to_not_convert = modules_to_not_convert self.post_init() + self.is_integer = dtype_dict[self.weights_dtype]["is_integer"] def post_init(self): r""" Safety checker that arguments are correct """ - accepted_weights = ["int8", "uint8", "int4", "uint4"] + accepted_weights = ["int8", "uint8", "int4", "uint4", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz", "float8_e8m0fnu"] if self.weights_dtype not in accepted_weights: raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights_dtype}") diff --git a/modules/shared.py b/modules/shared.py index e4c3c8a6e..8fed9c635 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -525,7 +525,7 @@ options_templates.update(options_section(('quantization', "Quantization Settings "sdnq_quantize_sep": OptionInfo("

SDNQ: SDNext Quantization

", "", gr.HTML), "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native}), - "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ['int8', 'uint4', 'uint8', 'int4'], "visible": native}), + "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ['int8', 'uint4', "float8_e4m3fn", "float8_e5m2", 'uint8', 'int4'], "visible": native}), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), From b724cd7c5739399dedd5a8aee94165eab7af0dc0 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 27 May 2025 21:21:42 +0300 Subject: [PATCH 40/94] Update changelog --- CHANGELOG.md | 3 +++ modules/shared.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81486e925..07f916d52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,13 @@ - `INT8` -> `uint8` - `INT4_SYM` -> `int4` - `INT4` -> `uint4` + - Add `float8_e4m3fn` and `float8_e5m2` support - Set the default quant mode to `pre` - Use per token input quant with int8 matmul - Implement better layer hijacks - Fix Conv quant + - Fix lora weight change + - Use device_map = gpu - **IPEX** - Disabe Dynamic Attention by default on PyTorch 2.7 - Remove GradScaler hijack and use torch.amp.GradScaler instead diff --git a/modules/shared.py b/modules/shared.py index 8fed9c635..cdb71534d 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -525,7 +525,7 @@ options_templates.update(options_section(('quantization', "Quantization Settings "sdnq_quantize_sep": OptionInfo("

SDNQ: SDNext Quantization

", "", gr.HTML), "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native}), - "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ['int8', 'uint4', "float8_e4m3fn", "float8_e5m2", 'uint8', 'int4'], "visible": native}), + "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ['int8', 'uint4', "float8_e4m3fn", 'uint8', 'int4', "float8_e5m2"], "visible": native}), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), From b1b29e9001ac795035c8f6870f7a121b195746d5 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 27 May 2025 21:32:32 +0300 Subject: [PATCH 41/94] SDNQ disable device_map = gpu with TE and LLM --- modules/model_quant.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/model_quant.py b/modules/model_quant.py index 5a5dff44a..6dd7b4cff 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -529,8 +529,9 @@ def get_dit_args(load_config:dict={}, module:str=None, device_map:bool=False, al config['device_map'] = 'cpu' elif shared.opts.device_map == 'gpu': config['device_map'] = devices.device - elif 'Model' in shared.opts.sdnq_quantize_weights or (module is not None and module in shared.opts.sdnq_quantize_weights) or module == 'any': - config['device_map'] = devices.device + elif module not in {'TE', 'LLM'}: # Transformers loads the entire model into GPU at once and OOMs + if 'Model' in shared.opts.sdnq_quantize_weights or (module is not None and module in shared.opts.sdnq_quantize_weights) or module == 'any': + config['device_map'] = devices.device if allow_quant: quant_args = create_config(module=module) else: From d3e3fb98b0c352c1dcbc4f5cfdd225b8540da8f3 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 27 May 2025 21:45:52 +0300 Subject: [PATCH 42/94] Don't override user set device_map --- modules/model_quant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/model_quant.py b/modules/model_quant.py index 6dd7b4cff..9f255335e 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -529,7 +529,7 @@ def get_dit_args(load_config:dict={}, module:str=None, device_map:bool=False, al config['device_map'] = 'cpu' elif shared.opts.device_map == 'gpu': config['device_map'] = devices.device - elif module not in {'TE', 'LLM'}: # Transformers loads the entire model into GPU at once and OOMs + elif shared.opts.device_map == 'default' and module not in {'TE', 'LLM'}: # Transformers loads the entire model into GPU at once and OOMs if 'Model' in shared.opts.sdnq_quantize_weights or (module is not None and module in shared.opts.sdnq_quantize_weights) or module == 'any': config['device_map'] = devices.device if allow_quant: From 4ed15f5cce43a99987582c32cc18b6fc84ee9f22 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 27 May 2025 23:32:58 +0300 Subject: [PATCH 43/94] SDNQ revert device_map = gpu --- CHANGELOG.md | 1 - modules/model_quant.py | 3 --- 2 files changed, 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07f916d52..132f6f67c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,6 @@ - Implement better layer hijacks - Fix Conv quant - Fix lora weight change - - Use device_map = gpu - **IPEX** - Disabe Dynamic Attention by default on PyTorch 2.7 - Remove GradScaler hijack and use torch.amp.GradScaler instead diff --git a/modules/model_quant.py b/modules/model_quant.py index 9f255335e..b38451414 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -529,9 +529,6 @@ def get_dit_args(load_config:dict={}, module:str=None, device_map:bool=False, al config['device_map'] = 'cpu' elif shared.opts.device_map == 'gpu': config['device_map'] = devices.device - elif shared.opts.device_map == 'default' and module not in {'TE', 'LLM'}: # Transformers loads the entire model into GPU at once and OOMs - if 'Model' in shared.opts.sdnq_quantize_weights or (module is not None and module in shared.opts.sdnq_quantize_weights) or module == 'any': - config['device_map'] = devices.device if allow_quant: quant_args = create_config(module=module) else: From 1433dfe3de23e88348a98840bb442b62f2b10709 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 28 May 2025 03:16:29 +0300 Subject: [PATCH 44/94] SDNQ fix high RAM usage with pre mode --- CHANGELOG.md | 1 + modules/model_quant_sdnq.py | 33 +++++++++++++++++++++------------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 132f6f67c..1d4ae0edf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Implement better layer hijacks - Fix Conv quant - Fix lora weight change + * Fix high RAM usage with pre mode - **IPEX** - Disabe Dynamic Attention by default on PyTorch 2.7 - Remove GradScaler hijack and use torch.amp.GradScaler instead diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 0e4d1bff5..89ac6f60c 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -39,7 +39,7 @@ class QuantizationMethod(str, Enum): SDNQ = "sdnq" -def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, param_name=None): # pylint: disable=unused-argument +def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, param_name=None, pre_mode=False): # pylint: disable=unused-argument layer_class_name = layer.__class__.__name__ if layer_class_name in allowed_types: is_conv_type = False @@ -95,8 +95,10 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz layer.weight.requires_grad = False if shared.opts.diffusers_offload_mode in {"none", "model"}: return_device = devices.device - else: + elif pre_mode: return_device = devices.cpu + else: + return_device = layer.weight.device layer.weight.data = layer.weight.data.to(devices.device, dtype=torch.float32) if dtype_dict[weights_dtype]["is_unsigned"]: @@ -124,8 +126,9 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz result_shape=result_shape, use_quantized_matmul=use_quantized_matmul, weights_dtype=weights_dtype, - ).to(return_device) + ) layer.weight.data = layer.sdnq_decompressor.pack_weight(layer.weight.data).to(return_device) + layer.sdnq_decompressor = layer.sdnq_decompressor.to(return_device) if is_linear_type: if use_quantized_matmul: @@ -449,11 +452,16 @@ class SDNQQuantizer(DiffusersQuantizer): state_dict: Dict[str, Any], **kwargs, ): + param_value.data = param_value.data.clone() # safetensors is unable to release the cpu memory without this if param_name.endswith(".weight"): split_param_name = param_name.split(".") if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert): - layer, _ = get_module_from_name(model, param_name) - return layer.__class__.__name__ in allowed_types + layer_class_name = get_module_from_name(model, param_name)[0].__class__.__name__ + if layer_class_name in allowed_types: + if layer_class_name in conv_types or layer_class_name in conv_transpose_types: + return self.quantization_config.quant_conv + else: + return True return False def check_quantized_param(self, *args, **kwargs) -> bool: @@ -465,16 +473,16 @@ class SDNQQuantizer(DiffusersQuantizer): def create_quantized_param( # pylint: disable=arguments-differ self, model, - param_value: "torch.Tensor", + param_value: torch.FloatTensor, param_name: str, - target_device: "torch.device", + target_device: torch.device, # pylint: disable=unused-argument state_dict: Dict[str, Any], # pylint: disable=unused-argument unexpected_keys: List[str], # pylint: disable=unused-argument **kwargs, ): # load the model params to target_device first - layer, tensor_name = get_module_from_name(model, param_name) - layer.weight = torch.nn.Parameter(param_value.to(device=target_device), requires_grad=False) + layer, _ = get_module_from_name(model, param_name) + layer.weight = torch.nn.Parameter(param_value.to(device=devices.device, dtype=torch.float32), requires_grad=False) layer = sdnq_quantize_layer( layer, weights_dtype=self.quantization_config.weights_dtype, @@ -483,16 +491,17 @@ class SDNQQuantizer(DiffusersQuantizer): quant_conv=self.quantization_config.quant_conv, use_quantized_matmul=self.quantization_config.use_quantized_matmul, param_name=param_name, + pre_mode=True, ) def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: - max_memory = {key: val * 0.70 for key, val in max_memory.items()} + max_memory = {key: val * 0.80 for key, val in max_memory.items()} return max_memory - def adjust_target_dtype(self, target_dtype: "torch.dtype") -> "torch.dtype": # pylint: disable=unused-argument,arguments-renamed + def adjust_target_dtype(self, target_dtype: torch.dtype) -> torch.dtype: # pylint: disable=unused-argument,arguments-renamed return dtype_dict[self.quantization_config.weights_dtype]["target_dtype"] - def update_torch_dtype(self, torch_dtype: "torch.dtype" = None) -> "torch.dtype": + def update_torch_dtype(self, torch_dtype: torch.dtype = None) -> torch.dtype: if torch_dtype is None: torch_dtype = devices.dtype self.torch_dtype = torch_dtype From 0b564e23737da5e1785c3a254f0849554fd094ea Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 28 May 2025 04:07:45 +0300 Subject: [PATCH 45/94] Cleanup --- modules/model_quant_sdnq.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 89ac6f60c..26abcd9eb 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -93,13 +93,17 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz layer.weight.data = layer.weight.reshape(new_shape) layer.weight.requires_grad = False - if shared.opts.diffusers_offload_mode in {"none", "model"}: + if pre_mode: + if shared.opts.device_map != "gpu": + return_device = devices.cpu + else: + return_device = devices.device + elif shared.opts.diffusers_offload_mode in {"none", "model"}: return_device = devices.device - elif pre_mode: - return_device = devices.cpu else: return_device = layer.weight.device - layer.weight.data = layer.weight.data.to(devices.device, dtype=torch.float32) + if not pre_mode: + layer.weight.data = layer.weight.to(devices.device).to(dtype=torch.float32) if dtype_dict[weights_dtype]["is_unsigned"]: scale, zero_point = get_scale_asymmetric(layer.weight, reduction_axes, weights_dtype) @@ -116,18 +120,18 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if use_quantized_matmul: scale = scale.squeeze(-1) if dtype_dict[weights_dtype]["num_bits"] == 8: - layer.weight.data = layer.weight.data.transpose(0,1) + layer.weight.data = layer.weight.transpose(0,1) layer.sdnq_decompressor = decompressor_dict[weights_dtype]( scale=scale, zero_point=zero_point, - compressed_weight_shape=layer.weight.data.shape, + compressed_weight_shape=layer.weight.shape, result_dtype=torch_dtype, result_shape=result_shape, use_quantized_matmul=use_quantized_matmul, weights_dtype=weights_dtype, ) - layer.weight.data = layer.sdnq_decompressor.pack_weight(layer.weight.data).to(return_device) + layer.weight.data = layer.sdnq_decompressor.pack_weight(layer.weight).to(return_device) layer.sdnq_decompressor = layer.sdnq_decompressor.to(return_device) if is_linear_type: @@ -452,7 +456,8 @@ class SDNQQuantizer(DiffusersQuantizer): state_dict: Dict[str, Any], **kwargs, ): - param_value.data = param_value.data.clone() # safetensors is unable to release the cpu memory without this + if shared.opts.device_map != "gpu": + param_value.data = param_value.clone() # safetensors is unable to release the cpu memory without this if param_name.endswith(".weight"): split_param_name = param_name.split(".") if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert): @@ -475,14 +480,14 @@ class SDNQQuantizer(DiffusersQuantizer): model, param_value: torch.FloatTensor, param_name: str, - target_device: torch.device, # pylint: disable=unused-argument + target_device: torch.device, state_dict: Dict[str, Any], # pylint: disable=unused-argument unexpected_keys: List[str], # pylint: disable=unused-argument **kwargs, ): # load the model params to target_device first layer, _ = get_module_from_name(model, param_name) - layer.weight = torch.nn.Parameter(param_value.to(device=devices.device, dtype=torch.float32), requires_grad=False) + layer.weight = torch.nn.Parameter(param_value.to(devices.device).to(dtype=torch.float32), requires_grad=False) layer = sdnq_quantize_layer( layer, weights_dtype=self.quantization_config.weights_dtype, @@ -522,7 +527,7 @@ class SDNQQuantizer(DiffusersQuantizer): self.modules_to_not_convert.extend(keep_in_fp32_modules) def _process_model_after_weight_loading(self, model, **kwargs): - if shared.opts.diffusers_offload_mode == "model": + if shared.opts.diffusers_offload_mode != "none": model = model.to(devices.cpu) devices.torch_gc(force=True) return model From 569e9099d78c9e288fe6e536258ecd31f215b1d4 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 28 May 2025 12:44:07 +0300 Subject: [PATCH 46/94] Use torch.amax instead of torch.max --- modules/model_quant_sdnq.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 26abcd9eb..ac7ce71f6 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -93,13 +93,13 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz layer.weight.data = layer.weight.reshape(new_shape) layer.weight.requires_grad = False - if pre_mode: + if shared.opts.diffusers_offload_mode in {"none", "model"}: + return_device = devices.device + elif pre_mode: if shared.opts.device_map != "gpu": return_device = devices.cpu else: return_device = devices.device - elif shared.opts.diffusers_offload_mode in {"none", "model"}: - return_device = devices.device else: return_device = layer.weight.device if not pre_mode: @@ -264,8 +264,14 @@ def unpack_int4(packed_tensor: torch.Tensor, shape: torch.Size, dtype: Optional[ return result +def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.FloatTensor, torch.FloatTensor]: + input_scale = torch.div(input.abs().amax(dim=-1), 448).unsqueeze(-1) + input = torch.div(input, input_scale).clamp_(-448, 448).to(torch.float8_e4m3fn) + return input, input_scale + + def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.ByteTensor, torch.FloatTensor]: - input_scale = torch.div(input.abs().max(dim=-1).values, 127).unsqueeze(-1) + input_scale = torch.div(input.abs().amax(dim=-1), 127).unsqueeze(-1) input = torch.div(input, input_scale).round_().clamp_(-128, 127).to(torch.int8).flatten(0,-2).contiguous() scale = torch.mul(input_scale, scale).flatten(0,-2).contiguous() if scale.dtype == torch.float16: # fp16 will overflow From 1961e88c136d64fcc3790591a5a46e293e5a7bcc Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 28 May 2025 13:42:29 +0300 Subject: [PATCH 47/94] Set SDPA as the default on all backends and enable Dyn SDPA on ROCm, DML, CPU and MPS --- CHANGELOG.md | 5 ++++- modules/shared.py | 11 ++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d4ae0edf..f9d3fa8ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,10 @@ - Disabe Dynamic Attention by default on PyTorch 2.7 - Remove GradScaler hijack and use torch.amp.GradScaler instead - **Feature** - - TeaCache support for HiDream I1 + - TeaCache support for HiDream I1 +- **Changes** + - Set the default attention optimizer to Scaled-Dot-Product on all backends + - Enable Dynamic attention for Scaled-Dot-Product with ROCm, DirectML, MPS and CPU backends ## Update for 2025-05-17 diff --git a/modules/shared.py b/modules/shared.py index cdb71534d..c329ef1fb 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -369,18 +369,11 @@ def get_default_modes(): default_offload_mode = "sequential" default_diffusers_offload_min_gpu_memory = 0 - if devices.backend == "directml": # Force BMM for DirectML instead of SDP - default_cross_attention = "Dynamic Attention BMM" if native else "Sub-quadratic" - elif devices.backend == "cpu": - default_cross_attention = "Scaled-Dot-Product" if native else "Doggettx's" - elif devices.backend == "mps": - default_cross_attention = "Scaled-Dot-Product" if native else "Doggettx's" - else: # cuda, rocm, zluda, ipex, openvino - default_cross_attention = "Scaled-Dot-Product" + default_cross_attention = "Scaled-Dot-Product" if devices.backend == "zluda": default_sdp_options = ['Flash attention', 'Math attention', 'Dynamic attention'] - elif devices.backend == "rocm": + elif devices.backend in {"rocm", "directml", "cpu", "mps"}: default_sdp_options = ['Flash attention', 'Memory attention', 'Math attention', 'Dynamic attention'] else: default_sdp_options = ['Flash attention', 'Memory attention', 'Math attention'] From d8e8f47ce5c9aa4fe9fc338a25c9a011b7112769 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 28 May 2025 15:18:39 +0300 Subject: [PATCH 48/94] SDNQ add an option to toggle quantize with GPU --- CHANGELOG.md | 5 +++-- modules/model_quant.py | 2 +- modules/model_quant_sdnq.py | 17 +++++++++++++---- modules/shared.py | 1 + 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9d3fa8ce..ea5672a57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,14 +13,15 @@ - Set the default quant mode to `pre` - Use per token input quant with int8 matmul - Implement better layer hijacks + - Add an option to toggle quantize with GPU - Fix Conv quant - Fix lora weight change - * Fix high RAM usage with pre mode + - Fix high RAM usage with pre mode - **IPEX** - Disabe Dynamic Attention by default on PyTorch 2.7 - Remove GradScaler hijack and use torch.amp.GradScaler instead - **Feature** - - TeaCache support for HiDream I1 + - TeaCache support for HiDream I1 - **Changes** - Set the default attention optimizer to Scaled-Dot-Product on all backends - Enable Dynamic attention for Scaled-Dot-Product with ROCm, DirectML, MPS and CPU backends diff --git a/modules/model_quant.py b/modules/model_quant.py index b38451414..96579c4eb 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -349,7 +349,7 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc=True): quant_last_model_name = None quant_last_model_device = None model.to(devices.device) - elif shared.opts.diffusers_offload_mode == "model": + elif shared.opts.diffusers_offload_mode != "none": model = model.to(devices.cpu) if do_gc: devices.torch_gc(force=True) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index ac7ce71f6..771029069 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -96,14 +96,19 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if shared.opts.diffusers_offload_mode in {"none", "model"}: return_device = devices.device elif pre_mode: - if shared.opts.device_map != "gpu": + if shared.opts.sdnq_quantize_with_gpu: return_device = devices.cpu - else: + elif shared.opts.device_map == "gpu": return_device = devices.device + else: + return_device = layer.weight.device else: return_device = layer.weight.device if not pre_mode: - layer.weight.data = layer.weight.to(devices.device).to(dtype=torch.float32) + if shared.opts.sdnq_quantize_with_gpu: + layer.weight.data = layer.weight.to(devices.device).to(dtype=torch.float32) + else: + layer.weight.data = layer.weight.to(dtype=torch.float32) if dtype_dict[weights_dtype]["is_unsigned"]: scale, zero_point = get_scale_asymmetric(layer.weight, reduction_axes, weights_dtype) @@ -493,7 +498,11 @@ class SDNQQuantizer(DiffusersQuantizer): ): # load the model params to target_device first layer, _ = get_module_from_name(model, param_name) - layer.weight = torch.nn.Parameter(param_value.to(devices.device).to(dtype=torch.float32), requires_grad=False) + if shared.opts.sdnq_quantize_with_gpu: + param_value = param_value.to(devices.device).to(dtype=torch.float32) + else: + param_value = param_value.to(target_device).to(dtype=torch.float32) + layer.weight = torch.nn.Parameter(param_value, requires_grad=False) layer = sdnq_quantize_layer( layer, weights_dtype=self.quantization_config.weights_dtype, diff --git a/modules/shared.py b/modules/shared.py index c329ef1fb..82e290902 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -524,6 +524,7 @@ options_templates.update(options_section(('quantization', "Quantization Settings "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), "sdnq_decompress_compile": OptionInfo(devices.has_triton(), "Decompress using torch.compile", gr.Checkbox, {"visible": native}), "sdnq_use_quantized_matmul": OptionInfo(False, "Use quantized MatMul", gr.Checkbox, {"visible": native}), + "sdnq_quantize_with_gpu": OptionInfo(False, "Quantize with the GPU", gr.Checkbox, {"visible": native}), "sdnq_quantize_shuffle_weights": OptionInfo(False, "Shuffle weights in post mode", gr.Checkbox, {"visible": native}), "bnb_quantization_sep": OptionInfo("

BitsAndBytes

", "", gr.HTML), From e06cbea7aa33e8be0f45aba23db1c0160b299243 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 28 May 2025 15:55:08 +0300 Subject: [PATCH 49/94] Cleanup --- modules/model_quant_sdnq.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 771029069..022b132e4 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -96,10 +96,10 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if shared.opts.diffusers_offload_mode in {"none", "model"}: return_device = devices.device elif pre_mode: - if shared.opts.sdnq_quantize_with_gpu: - return_device = devices.cpu - elif shared.opts.device_map == "gpu": + if shared.opts.device_map == "gpu": return_device = devices.device + elif shared.opts.sdnq_quantize_with_gpu: + return_device = devices.cpu else: return_device = layer.weight.device else: From dd0dbc476fd68426cc2e8da43c3cfd692cf34604 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 28 May 2025 17:25:38 +0300 Subject: [PATCH 50/94] SDNQ fix asym quant formula for dtypes with non zero minimums --- modules/model_quant_sdnq.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 022b132e4..a2490b7b2 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -187,11 +187,11 @@ def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_si def get_scale_asymmetric(weight: torch.FloatTensor, reduction_axes: List[int], weights_dtype: str) -> Tuple[torch.FloatTensor, torch.FloatTensor]: zero_point = torch.amin(weight, dim=reduction_axes, keepdims=True) - scale = torch.amax(weight, dim=reduction_axes, keepdims=True).sub_(zero_point).div_(dtype_dict[weights_dtype]["max"]) + scale = torch.amax(weight, dim=reduction_axes, keepdims=True).sub_(zero_point).div_(dtype_dict[weights_dtype]["max"] - dtype_dict[weights_dtype]["min"]) eps = torch.finfo(scale.dtype).eps # prevent divison by 0 scale = torch.where(torch.abs(scale) < eps, eps, scale) if dtype_dict[weights_dtype]["min"] != 0: - zero_point.add_(dtype_dict[weights_dtype]["min"]) + zero_point.sub_(torch.mul(scale, dtype_dict[weights_dtype]["min"])) return scale, zero_point From dd33c4d5839d59f6761029bc8d282b80a11dea79 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 28 May 2025 17:46:06 +0300 Subject: [PATCH 51/94] Fix scale and zero_point not being moved by tensor.to --- modules/model_quant_sdnq.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index a2490b7b2..b6d338b1b 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -345,10 +345,10 @@ class AsymmetricWeightsDecompressor(torch.nn.Module): super().__init__() self.weights_dtype = weights_dtype self.use_quantized_matmul = False - self.scale = scale - self.zero_point = zero_point self.result_dtype = result_dtype self.result_shape = result_shape + self.register_buffer("scale", scale) + self.register_buffer("zero_point", zero_point) def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"]) @@ -370,9 +370,9 @@ class SymmetricWeightsDecompressor(torch.nn.Module): super().__init__() self.weights_dtype = weights_dtype self.use_quantized_matmul = use_quantized_matmul - self.scale = scale self.result_dtype = result_dtype self.result_shape = result_shape + self.register_buffer("scale", scale) def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"]) @@ -394,11 +394,11 @@ class INT4AsymmetricWeightsDecompressor(torch.nn.Module): super().__init__() self.weights_dtype = "uint4" self.use_quantized_matmul = False - self.scale = scale - self.zero_point = zero_point self.compressed_weight_shape = compressed_weight_shape self.result_dtype = result_dtype self.result_shape = result_shape + self.register_buffer("scale", scale) + self.register_buffer("zero_point", zero_point) def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: return pack_uint4(weight.to(dtype=torch.uint8)) @@ -420,10 +420,10 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): super().__init__() self.weights_dtype = "int4" self.use_quantized_matmul = use_quantized_matmul - self.scale = scale self.compressed_weight_shape = compressed_weight_shape self.result_dtype = result_dtype self.result_shape = result_shape + self.register_buffer("scale", scale) def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: return pack_int4(weight.to(dtype=torch.int8)) From 3698f8bb8459a51416dbdb1c26cd3bc2ded56b0e Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 29 May 2025 03:11:59 +0300 Subject: [PATCH 52/94] SDNQ add experimental FP8 matmul --- modules/model_quant_sdnq.py | 61 +++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index b6d338b1b..256299f4e 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -27,7 +27,7 @@ dtype_dict = { if hasattr(torch, "float8_e8m0fnu"): dtype_dict["float8_e8m0fnu"] = {"min": 5.87747e-39, "max": 1.70141e+38, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e8m0fnu, "storage_dtype": torch.float8_e8m0fnu, "is_unsigned": True, "is_integer": False} -quantized_matmul_dtypes = ("int8", "int4") # todo: float8_e4m3fn +quantized_matmul_dtypes = ("int8", "int4", "float8_e4m3fn") linear_types = ("Linear",) conv_types = ("Conv1d", "Conv2d", "Conv3d") @@ -123,9 +123,14 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz zero_point = zero_point.to(torch_dtype) if use_quantized_matmul: - scale = scale.squeeze(-1) + scale = scale.transpose(0,1) if dtype_dict[weights_dtype]["num_bits"] == 8: - layer.weight.data = layer.weight.transpose(0,1) + layer.weight.data = layer.weight.transpose(0,1).contiguous() + if not dtype_dict[weights_dtype]["is_integer"]: + stride = layer.weight.stride() + if stride[0] > stride[1] and stride[1] == 1: + layer.weight.data = layer.weight.t().contiguous().t() + scale = scale.to(torch.float32) layer.sdnq_decompressor = decompressor_dict[weights_dtype]( scale=scale, @@ -141,7 +146,10 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if is_linear_type: if use_quantized_matmul: - layer.forward = quantized_linear_forward_int8_matmul + if dtype_dict[weights_dtype]["is_integer"]: + layer.forward = quantized_linear_forward_int8_matmul + else: + layer.forward = quantized_linear_forward_fp8_matmul else: layer.forward = quantized_linear_forward elif is_conv_type: @@ -222,9 +230,9 @@ def decompress_asymmetric(input: torch.Tensor, scale: torch.Tensor, zero_point: return result -def decompress_symmetric(input: torch.Tensor, scale: torch.Tensor, dtype: torch.dtype, result_shape: torch.Size, skip_int8_matmul: bool = False) -> torch.Tensor: - if skip_int8_matmul: - result = input.transpose(0,1).to(dtype=scale.dtype).mul_(scale.unsqueeze(-1)).to(dtype=dtype) +def decompress_symmetric(input: torch.Tensor, scale: torch.Tensor, dtype: torch.dtype, result_shape: torch.Size, skip_quantized_matmul: bool = False) -> torch.Tensor: + if skip_quantized_matmul: + result = input.transpose(0,1).to(dtype=scale.dtype).mul_(scale.transpose(0,1)).to(dtype=dtype) else: result = input.to(dtype=scale.dtype).mul_(scale).to(dtype=dtype) if result_shape is not None: @@ -236,9 +244,9 @@ def decompress_int4_asymmetric(input: torch.Tensor, scale: torch.Tensor, zero_po return decompress_asymmetric(unpack_uint4(input, shape), scale, zero_point, dtype, result_shape) -def decompress_int4_symmetric(input: torch.Tensor, scale: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, skip_int8_matmul: bool = False) -> torch.Tensor: - if skip_int8_matmul: - return decompress_symmetric(unpack_int4(input, shape, dtype=scale.dtype), scale.unsqueeze(-1), dtype, result_shape) +def decompress_int4_symmetric(input: torch.Tensor, scale: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, skip_quantized_matmul: bool = False) -> torch.Tensor: + if skip_quantized_matmul: + return decompress_symmetric(unpack_int4(input, shape, dtype=scale.dtype), scale.transpose(0,1), dtype, result_shape) else: return decompress_symmetric(unpack_int4(input, shape, dtype=scale.dtype), scale, dtype, result_shape) @@ -270,7 +278,8 @@ def unpack_int4(packed_tensor: torch.Tensor, shape: torch.Size, dtype: Optional[ def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.FloatTensor, torch.FloatTensor]: - input_scale = torch.div(input.abs().amax(dim=-1), 448).unsqueeze(-1) + input = input.flatten(0,-2).contiguous() + input_scale = torch.div(input.abs().amax(dim=-1), 448).unsqueeze(-1).to(torch.float32) input = torch.div(input, input_scale).clamp_(-448, 448).to(torch.float8_e4m3fn) return input, input_scale @@ -284,6 +293,19 @@ def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTenso return input, scale +def fp8_matmul( + input: torch.FloatTensor, + weight: torch.Tensor, + bias: torch.FloatTensor, + scale: torch.FloatTensor, +) -> torch.FloatTensor: + return_dtype = input.dtype + output_shape = list(input.shape) + output_shape[-1] = weight.shape[-1] + input, input_scale = quantize_fp8_matmul_input(input) + return torch._scaled_mm(input, weight, input_scale, scale, bias=bias, out_dtype=return_dtype).reshape(output_shape) + + def int8_matmul( input: torch.FloatTensor, weight: torch.Tensor, @@ -303,9 +325,15 @@ def int8_matmul( return result +def quantized_linear_forward_fp8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: + if input.shape[-1] % 16 != 0 or self.weight.shape[-1] % 16 != 0 or self.weight.shape[-1] % 16 != 0: + return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) + return fp8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale) + + def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: if torch.numel(input) / input.shape[-1] < 32: - return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_int8_matmul=True), self.bias) + return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) return int8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale, getattr(self.sdnq_decompressor, "compressed_weight_shape", None)) @@ -377,8 +405,8 @@ class SymmetricWeightsDecompressor(torch.nn.Module): def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"]) - def forward(self, weight, skip_int8_matmul=False, **kwargs): - return decompress_symmetric_compiled(weight, self.scale, self.result_dtype, self.result_shape, skip_int8_matmul=skip_int8_matmul) + def forward(self, weight, skip_quantized_matmul=False, **kwargs): + return decompress_symmetric_compiled(weight, self.scale, self.result_dtype, self.result_shape, skip_quantized_matmul=skip_quantized_matmul) class INT4AsymmetricWeightsDecompressor(torch.nn.Module): @@ -428,8 +456,8 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: return pack_int4(weight.to(dtype=torch.int8)) - def forward(self, weight, skip_int8_matmul=False, **kwargs): - return decompress_int4_symmetric_compiled(weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape, skip_int8_matmul=skip_int8_matmul) + def forward(self, weight, skip_quantized_matmul=False, **kwargs): + return decompress_int4_symmetric_compiled(weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape, skip_quantized_matmul=skip_quantized_matmul) decompressor_dict = { @@ -649,6 +677,7 @@ if shared.opts.sdnq_decompress_compile: decompress_symmetric_compiled = torch.compile(decompress_symmetric, fullgraph=True) decompress_int4_asymmetric_compiled = torch.compile(decompress_int4_asymmetric, fullgraph=True) decompress_int4_symmetric_compiled = torch.compile(decompress_int4_symmetric, fullgraph=True) + fp8_matmul = torch.compile(fp8_matmul, fullgraph=True) if devices.backend != "ipex": # pytorch uses the cpu device in torch._int_mm op with ipex + torch.compile quantize_int8_matmul_input_compiled = quantize_int8_matmul_input unpack_int4_compiled = unpack_int4 From 67e0f4d833eb4bcb07f4e64c77be57f7fe4265af Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 29 May 2025 03:22:40 +0300 Subject: [PATCH 53/94] Cleanup --- modules/model_quant_sdnq.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 256299f4e..eb73c0973 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -279,8 +279,9 @@ def unpack_int4(packed_tensor: torch.Tensor, shape: torch.Size, dtype: Optional[ def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.FloatTensor, torch.FloatTensor]: input = input.flatten(0,-2).contiguous() - input_scale = torch.div(input.abs().amax(dim=-1), 448).unsqueeze(-1).to(torch.float32) + input_scale = torch.div(input.abs().amax(dim=-1), 448).unsqueeze(-1) input = torch.div(input, input_scale).clamp_(-448, 448).to(torch.float8_e4m3fn) + input_scale = input_scale.to(torch.float32) return input, input_scale From 2cc5a58b0f8aa9098e5748dfb91b59e323861037 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 29 May 2025 03:26:47 +0300 Subject: [PATCH 54/94] Update changelog --- CHANGELOG.md | 2 ++ modules/model_quant_sdnq.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea5672a57..87cd8316e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - `INT4_SYM` -> `int4` - `INT4` -> `uint4` - Add `float8_e4m3fn` and `float8_e5m2` support + - Add quantized matmul support for `float8_e4m3fn` - Set the default quant mode to `pre` - Use per token input quant with int8 matmul - Implement better layer hijacks @@ -17,6 +18,7 @@ - Fix Conv quant - Fix lora weight change - Fix high RAM usage with pre mode + - Fix scale and zero_point not being offloaded - **IPEX** - Disabe Dynamic Attention by default on PyTorch 2.7 - Remove GradScaler hijack and use torch.amp.GradScaler instead diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index eb73c0973..c6803777e 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -327,7 +327,7 @@ def int8_matmul( def quantized_linear_forward_fp8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: - if input.shape[-1] % 16 != 0 or self.weight.shape[-1] % 16 != 0 or self.weight.shape[-1] % 16 != 0: + if input.shape[-1] % 16 != 0 or self.weight.shape[0] % 16 != 0 or self.weight.shape[1] % 16 != 0: return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) return fp8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale) From cf2d1e56a6c1b983bdde790ba7b51ea59b6e8957 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 29 May 2025 03:32:11 +0300 Subject: [PATCH 55/94] Update changelog --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87cd8316e..aa601b1dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,9 @@ # Change Log for SD.Next -## Update for 2025-05-25 +## Update for 2025-05-29 - **SDNQ Quantization** - - Renamed `NNCF` to `SDNQ` as NNCF implementation in SDNext re-implemented and optimized enough to not use any imports from NNCF anymore + - Renamed `NNCF` to `SDNQ` as NNCF implementation in SDNext re-implemented, optimized and evolved enough to become its own quantization method - Renamed quantization scheme names to the underlying dtype names instead of NNCF names - `INT8_SYM` -> `int8` - `INT8` -> `uint8` @@ -12,10 +12,10 @@ - Add `float8_e4m3fn` and `float8_e5m2` support - Add quantized matmul support for `float8_e4m3fn` - Set the default quant mode to `pre` - - Use per token input quant with int8 matmul + - Use per token input quant with int8 and fp8 quantized matmul - Implement better layer hijacks - Add an option to toggle quantize with GPU - - Fix Conv quant + - Fix conv quant and add support for conv quant with asym modes - Fix lora weight change - Fix high RAM usage with pre mode - Fix scale and zero_point not being offloaded From 14893b761798a4ca39542156bcbf9a5015f54210 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 29 May 2025 03:43:57 +0300 Subject: [PATCH 56/94] Don't make the weights contiguous with int8 matmul --- modules/model_quant_sdnq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index c6803777e..894d7e18b 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -125,7 +125,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if use_quantized_matmul: scale = scale.transpose(0,1) if dtype_dict[weights_dtype]["num_bits"] == 8: - layer.weight.data = layer.weight.transpose(0,1).contiguous() + layer.weight.data = layer.weight.transpose(0,1) if not dtype_dict[weights_dtype]["is_integer"]: stride = layer.weight.stride() if stride[0] > stride[1] and stride[1] == 1: From 2351efb8f70e5ff99d30307c850d56b1505b41b0 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 29 May 2025 14:58:00 +0300 Subject: [PATCH 57/94] Remove redundant shape check --- modules/model_quant_sdnq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 894d7e18b..a8839d7ab 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -327,7 +327,7 @@ def int8_matmul( def quantized_linear_forward_fp8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: - if input.shape[-1] % 16 != 0 or self.weight.shape[0] % 16 != 0 or self.weight.shape[1] % 16 != 0: + if self.weight.shape[0] % 16 != 0 or self.weight.shape[1] % 16 != 0: return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) return fp8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale) From df8b31fcfce5c302421f77c2077f5b56ab32e3d0 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 29 May 2025 16:35:40 +0300 Subject: [PATCH 58/94] Don't downcast scale with fp8 matmul --- modules/model_quant_sdnq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index a8839d7ab..2dd10b850 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -117,7 +117,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz zero_point = None layer.weight.data = quantize_weight(layer.weight, scale, zero_point, weights_dtype) - if not shared.opts.sdnq_decompress_fp32: + if not shared.opts.sdnq_decompress_fp32 and not (use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]): scale = scale.to(torch_dtype) if zero_point is not None: zero_point = zero_point.to(torch_dtype) From 90324f9c8c32b27e74c93fbfebad15e61df990a1 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 29 May 2025 18:25:12 +0300 Subject: [PATCH 59/94] SDNQ fix lora with quant matmul --- modules/lora/lora_apply.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index 3ed2eb8c5..7edc9f461 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -80,7 +80,7 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. try: t0 = time.time() if hasattr(self, "sdnq_decompressor"): - weight = self.sdnq_decompressor.to(devices.device)(self.weight.to(devices.device), skip_int8_matmul=self.sdnq_decompressor.use_quantized_matmul) + weight = self.sdnq_decompressor.to(devices.device)(self.weight.to(devices.device), skip_quantized_matmul=self.sdnq_decompressor.use_quantized_matmul) else: weight = self.weight.to(devices.device) # must perform calc on gpu due to performance updown, ex_bias = module.calc_updown(weight) @@ -143,7 +143,7 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G sdnq_decompressor = self.sdnq_decompressor_backup.to(devices.device) else: sdnq_decompressor = self.sdnq_decompressor.to(devices.device) - dequant_weight = sdnq_decompressor(model_weights.to(devices.device), skip_int8_matmul=sdnq_decompressor.use_quantized_matmul) + dequant_weight = sdnq_decompressor(model_weights.to(devices.device), skip_quantized_matmul=sdnq_decompressor.use_quantized_matmul) new_weight = dequant_weight.to(devices.device, dtype=torch.float32) + lora_weights.to(devices.device, dtype=torch.float32) self.weight = torch.nn.Parameter(new_weight, requires_grad=False) self.sdnq_decompressor = None From 54154cf69880a201009fbb94f532cf7dc4764e0c Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 29 May 2025 20:22:49 +0300 Subject: [PATCH 60/94] Cleanup --- modules/intel/ipex/hijacks.py | 2 ++ modules/model_quant_sdnq.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/intel/ipex/hijacks.py b/modules/intel/ipex/hijacks.py index 0064e0695..0ce8abdc5 100644 --- a/modules/intel/ipex/hijacks.py +++ b/modules/intel/ipex/hijacks.py @@ -20,6 +20,8 @@ if os.environ.get('IPEX_FORCE_ATTENTION_SLICE', '0') == '0': use_dynamic_attention = False except Exception: use_dynamic_attention = True + else: + use_dynamic_attention = True else: use_dynamic_attention = bool(os.environ.get('IPEX_FORCE_ATTENTION_SLICE', '0') == '1') diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 2dd10b850..cf2805894 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -1,4 +1,4 @@ -# pylint: disable=redefined-builtin,no-member +# pylint: disable=redefined-builtin,no-member,protected-access from typing import Any, Dict, List, Tuple, Optional, Union from dataclasses import dataclass @@ -320,7 +320,7 @@ def int8_matmul( output_shape = list(input.shape) output_shape[-1] = weight.shape[-1] input, scale = quantize_int8_matmul_input_compiled(input, scale) - result = decompress_symmetric_compiled(torch._int_mm(input, weight), scale, return_dtype, output_shape) # pylint: disable=protected-access + result = decompress_symmetric_compiled(torch._int_mm(input, weight), scale, return_dtype, output_shape) if bias is not None: result.add_(bias) return result @@ -673,7 +673,7 @@ class SDNQ_T5DenseGatedActDense(torch.nn.Module): # forward can't find what self if shared.opts.sdnq_decompress_compile: try: - torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit) # pylint: disable=protected-access + torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit) decompress_asymmetric_compiled = torch.compile(decompress_asymmetric, fullgraph=True) decompress_symmetric_compiled = torch.compile(decompress_symmetric, fullgraph=True) decompress_int4_asymmetric_compiled = torch.compile(decompress_int4_asymmetric, fullgraph=True) From b4e615e76007449ddd16e36fee333b5132227c08 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 30 May 2025 00:16:54 +0300 Subject: [PATCH 61/94] SDNQ add FP8 row wise scaling workaround for SM89 on Windows --- modules/model_quant_sdnq.py | 40 ++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index cf2805894..17e9fd435 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -4,6 +4,7 @@ from typing import Any, Dict, List, Tuple, Optional, Union from dataclasses import dataclass from enum import Enum import os +import sys import torch from diffusers.quantizers.base import DiffusersQuantizer from diffusers.quantizers.quantization_config import QuantizationConfigMixin @@ -149,7 +150,10 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if dtype_dict[weights_dtype]["is_integer"]: layer.forward = quantized_linear_forward_int8_matmul else: - layer.forward = quantized_linear_forward_fp8_matmul + if devices.backend == "cuda" and sys.platform == "win32" and float(torch.__version__[:3]) <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9): + layer.forward = quantized_linear_forward_fp8_matmul_sm89 + else: + layer.forward = quantized_linear_forward_fp8_matmul else: layer.forward = quantized_linear_forward elif is_conv_type: @@ -285,6 +289,15 @@ def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.FloatTens return input, input_scale +def quantize_fp8_matmul_input_sm89(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.ByteTensor, torch.FloatTensor]: + input_scale = torch.div(input.abs().amax(dim=-1), 448).unsqueeze(-1) + input = torch.div(input, input_scale).clamp_(-448, 448).to(torch.float8_e4m3fn).flatten(0,-2).contiguous() + scale = torch.mul(input_scale, scale).flatten(0,-2).contiguous() + if scale.dtype == torch.float16: # fp16 will overflow + scale = scale.to(dtype=torch.float32) + return input, scale + + def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.ByteTensor, torch.FloatTensor]: input_scale = torch.div(input.abs().amax(dim=-1), 127).unsqueeze(-1) input = torch.div(input, input_scale).round_().clamp_(-128, 127).to(torch.int8).flatten(0,-2).contiguous() @@ -307,6 +320,24 @@ def fp8_matmul( return torch._scaled_mm(input, weight, input_scale, scale, bias=bias, out_dtype=return_dtype).reshape(output_shape) +# sm89 doesn't support row wise scale in Windows +def fp8_matmul_sm89( + input: torch.FloatTensor, + weight: torch.Tensor, + bias: torch.FloatTensor, + scale: torch.FloatTensor, +) -> torch.FloatTensor: + return_dtype = input.dtype + output_shape = list(input.shape) + output_shape[-1] = weight.shape[-1] + dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) + input, scale = quantize_fp8_matmul_input_sm89(input, scale) + result = decompress_symmetric_compiled(torch._scaled_mm(input, weight, dummy_input_scale, dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, output_shape) + if bias is not None: + result.add_(bias) + return result + + def int8_matmul( input: torch.FloatTensor, weight: torch.Tensor, @@ -332,6 +363,12 @@ def quantized_linear_forward_fp8_matmul(self, input: torch.FloatTensor) -> torch return fp8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale) +def quantized_linear_forward_fp8_matmul_sm89(self, input: torch.FloatTensor) -> torch.FloatTensor: + if self.weight.shape[0] % 16 != 0 or self.weight.shape[1] % 16 != 0: + return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) + return fp8_matmul_sm89(input, self.weight, self.bias, self.sdnq_decompressor.scale) + + def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: if torch.numel(input) / input.shape[-1] < 32: return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) @@ -679,6 +716,7 @@ if shared.opts.sdnq_decompress_compile: decompress_int4_asymmetric_compiled = torch.compile(decompress_int4_asymmetric, fullgraph=True) decompress_int4_symmetric_compiled = torch.compile(decompress_int4_symmetric, fullgraph=True) fp8_matmul = torch.compile(fp8_matmul, fullgraph=True) + fp8_matmul_sm89 = torch.compile(fp8_matmul_sm89, fullgraph=True) if devices.backend != "ipex": # pytorch uses the cpu device in torch._int_mm op with ipex + torch.compile quantize_int8_matmul_input_compiled = quantize_int8_matmul_input unpack_int4_compiled = unpack_int4 From d8dea9031f23c7caa2ff50b58905e9ba2745f4db Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 30 May 2025 01:13:37 +0300 Subject: [PATCH 62/94] SDNQ do FP8 matmul shape check only once --- modules/model_quant_sdnq.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 17e9fd435..ab66b0cdb 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -65,8 +65,11 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz else: is_linear_type = True reduction_axes = -1 - channel_size = layer.weight.shape[-1] - use_quantized_matmul = use_quantized_matmul and weights_dtype in quantized_matmul_dtypes and channel_size >= 32 and layer.weight.shape[0] >= 32 + output_channel_size, channel_size = layer.weight.shape + if use_quantized_matmul: + use_quantized_matmul = weights_dtype in quantized_matmul_dtypes and channel_size >= 32 and output_channel_size >= 32 + if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: + use_quantized_matmul = output_channel_size % 16 == 0 and channel_size % 16 == 0 if not use_quantized_matmul and (group_size > 0 or (dtype_dict[weights_dtype]["num_bits"] == 4 and group_size != -1)): if group_size == 0: @@ -358,14 +361,10 @@ def int8_matmul( def quantized_linear_forward_fp8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: - if self.weight.shape[0] % 16 != 0 or self.weight.shape[1] % 16 != 0: - return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) return fp8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale) def quantized_linear_forward_fp8_matmul_sm89(self, input: torch.FloatTensor) -> torch.FloatTensor: - if self.weight.shape[0] % 16 != 0 or self.weight.shape[1] % 16 != 0: - return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) return fp8_matmul_sm89(input, self.weight, self.bias, self.sdnq_decompressor.scale) From 599224d392394c10533079f348ab5b64f1657354 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 30 May 2025 01:31:41 +0300 Subject: [PATCH 63/94] SDNQ reduce 5 reshape ops to 2 with quantized input --- modules/model_quant_sdnq.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index ab66b0cdb..be1aa7efe 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -286,25 +286,27 @@ def unpack_int4(packed_tensor: torch.Tensor, shape: torch.Size, dtype: Optional[ def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.FloatTensor, torch.FloatTensor]: input = input.flatten(0,-2).contiguous() - input_scale = torch.div(input.abs().amax(dim=-1), 448).unsqueeze(-1) + input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 448) input = torch.div(input, input_scale).clamp_(-448, 448).to(torch.float8_e4m3fn) input_scale = input_scale.to(torch.float32) return input, input_scale def quantize_fp8_matmul_input_sm89(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.ByteTensor, torch.FloatTensor]: - input_scale = torch.div(input.abs().amax(dim=-1), 448).unsqueeze(-1) - input = torch.div(input, input_scale).clamp_(-448, 448).to(torch.float8_e4m3fn).flatten(0,-2).contiguous() - scale = torch.mul(input_scale, scale).flatten(0,-2).contiguous() + input = input.flatten(0,-2).contiguous() + input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 448) + input = torch.div(input, input_scale).clamp_(-448, 448).to(torch.float8_e4m3fn) + scale = torch.mul(input_scale, scale) if scale.dtype == torch.float16: # fp16 will overflow scale = scale.to(dtype=torch.float32) return input, scale def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.ByteTensor, torch.FloatTensor]: - input_scale = torch.div(input.abs().amax(dim=-1), 127).unsqueeze(-1) - input = torch.div(input, input_scale).round_().clamp_(-128, 127).to(torch.int8).flatten(0,-2).contiguous() - scale = torch.mul(input_scale, scale).flatten(0,-2).contiguous() + input = input.flatten(0,-2).contiguous() + input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 127) + input = torch.div(input, input_scale).round_().clamp_(-128, 127).to(torch.int8) + scale = torch.mul(input_scale, scale) if scale.dtype == torch.float16: # fp16 will overflow scale = scale.to(dtype=torch.float32) return input, scale From 3c8be0f55f2c82771d976ea5f7a74f387261a797 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 30 May 2025 04:47:29 +0300 Subject: [PATCH 64/94] SDNQ add uint2 --- CHANGELOG.md | 2 +- modules/model_quant_sdnq.py | 72 +++++++++++++++++++++++++++++++++++-- modules/shared.py | 2 +- 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa601b1dc..95be9e83a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ - `INT8` -> `uint8` - `INT4_SYM` -> `int4` - `INT4` -> `uint4` - - Add `float8_e4m3fn` and `float8_e5m2` support + - Add `float8_e4m3fn`, `float8_e5m2` and `uint2` support - Add quantized matmul support for `float8_e4m3fn` - Set the default quant mode to `pre` - Use per token input quant with int8 and fp8 quantized matmul diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index be1aa7efe..87da6b592 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -20,6 +20,7 @@ dtype_dict = { "uint8": {"min": 0, "max": 255, "num_bits": 8, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "int4": {"min": -8, "max": 7, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, "uint4": {"min": 0, "max": 15, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint2": {"min": 0, "max": 3, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "float8_e4m3fn": {"min": -448, "max": 448, "num_bits": 8, "target_dtype": torch.float8_e4m3fn, "torch_dtype": torch.float8_e4m3fn, "storage_dtype": torch.float8_e4m3fn, "is_unsigned": False, "is_integer": False}, "float8_e5m2": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": torch.float8_e5m2, "torch_dtype": torch.float8_e5m2, "storage_dtype": torch.float8_e5m2, "is_unsigned": False, "is_integer": False}, "float8_e4m3fnuz": {"min": -240, "max": 240, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e4m3fnuz, "storage_dtype": torch.float8_e4m3fnuz, "is_unsigned": False, "is_integer": False}, @@ -71,9 +72,12 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: use_quantized_matmul = output_channel_size % 16 == 0 and channel_size % 16 == 0 - if not use_quantized_matmul and (group_size > 0 or (dtype_dict[weights_dtype]["num_bits"] == 4 and group_size != -1)): + if not use_quantized_matmul and (group_size > 0 or (dtype_dict[weights_dtype]["num_bits"] < 8 and group_size != -1)): if group_size == 0: - group_size = 64 + if dtype_dict[weights_dtype]["num_bits"] < 4: + group_size = 32 + else: + group_size = 64 num_of_groups = channel_size // group_size if group_size >= channel_size: @@ -258,6 +262,10 @@ def decompress_int4_symmetric(input: torch.Tensor, scale: torch.Tensor, shape: t return decompress_symmetric(unpack_int4(input, shape, dtype=scale.dtype), scale, dtype, result_shape) +def decompress_int2_asymmetric(input: torch.Tensor, scale: torch.Tensor, zero_point: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size) -> torch.Tensor: + return decompress_asymmetric(unpack_uint2(input, shape), scale, zero_point, dtype, result_shape) + + def pack_uint4(tensor: torch.Tensor) -> torch.Tensor: if tensor.dtype != torch.uint8: raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") @@ -272,6 +280,23 @@ def pack_int4(tensor: torch.Tensor) -> torch.Tensor: return pack_uint4((tensor + 8).to(dtype=torch.uint8)) +def pack_uint2(tensor: torch.Tensor) -> torch.Tensor: + if tensor.dtype != torch.uint8: + raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") + packed_tensor = tensor.contiguous().reshape(-1, 4) + packed_tensor = torch.bitwise_or( + torch.bitwise_or( + torch.bitwise_and(packed_tensor[:, 0], 3), + torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 1], 3), 2) + ), + torch.bitwise_or( + torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 2], 3), 4), + torch.bitwise_left_shift(packed_tensor[:, 3], 6) + ), + ) + return packed_tensor + + def unpack_uint4(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: result = torch.stack((torch.bitwise_and(packed_tensor, 15), torch.bitwise_right_shift(packed_tensor, 4)), dim=-1).reshape(shape) return result @@ -284,6 +309,19 @@ def unpack_int4(packed_tensor: torch.Tensor, shape: torch.Size, dtype: Optional[ return result +def unpack_uint2(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: + result = torch.stack( + ( + torch.bitwise_and(packed_tensor, 3), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 2), 3), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 4), 3), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 6), 3), + ), + dim=-1 + ).reshape(shape) + return result + + def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.FloatTensor, torch.FloatTensor]: input = input.flatten(0,-2).contiguous() input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 448) @@ -499,11 +537,38 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): return decompress_int4_symmetric_compiled(weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape, skip_quantized_matmul=skip_quantized_matmul) +class INT2AsymmetricWeightsDecompressor(torch.nn.Module): + def __init__( + self, + scale: torch.Tensor, + zero_point: torch.Tensor, + compressed_weight_shape: torch.Size, + result_dtype: torch.dtype, + result_shape: torch.Size, + **kwargs, + ): + super().__init__() + self.weights_dtype = "uint2" + self.use_quantized_matmul = False + self.compressed_weight_shape = compressed_weight_shape + self.result_dtype = result_dtype + self.result_shape = result_shape + self.register_buffer("scale", scale) + self.register_buffer("zero_point", zero_point) + + def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: + return pack_uint2(weight.to(dtype=torch.uint8)) + + def forward(self, weight, **kwargs): + return decompress_int2_asymmetric_compiled(weight, self.scale, self.zero_point, self.compressed_weight_shape, self.result_dtype, self.result_shape) + + decompressor_dict = { "int8": SymmetricWeightsDecompressor, "uint8": AsymmetricWeightsDecompressor, "int4": INT4SymmetricWeightsDecompressor, "uint4": INT4AsymmetricWeightsDecompressor, + "uint2": INT2AsymmetricWeightsDecompressor, "float8_e4m3fn": SymmetricWeightsDecompressor, "float8_e4m3fnuz": SymmetricWeightsDecompressor, "float8_e5m2": SymmetricWeightsDecompressor, @@ -716,6 +781,7 @@ if shared.opts.sdnq_decompress_compile: decompress_symmetric_compiled = torch.compile(decompress_symmetric, fullgraph=True) decompress_int4_asymmetric_compiled = torch.compile(decompress_int4_asymmetric, fullgraph=True) decompress_int4_symmetric_compiled = torch.compile(decompress_int4_symmetric, fullgraph=True) + decompress_int2_asymmetric_compiled = torch.compile(decompress_int2_asymmetric, fullgraph=True) fp8_matmul = torch.compile(fp8_matmul, fullgraph=True) fp8_matmul_sm89 = torch.compile(fp8_matmul_sm89, fullgraph=True) if devices.backend != "ipex": # pytorch uses the cpu device in torch._int_mm op with ipex + torch.compile @@ -731,6 +797,7 @@ if shared.opts.sdnq_decompress_compile: decompress_symmetric_compiled = decompress_symmetric decompress_int4_asymmetric_compiled = decompress_int4_asymmetric decompress_int4_symmetric_compiled = decompress_int4_symmetric + decompress_int2_asymmetric_compiled = decompress_int2_asymmetric quantize_int8_matmul_input_compiled = quantize_int8_matmul_input unpack_int4_compiled = unpack_int4 else: @@ -738,5 +805,6 @@ else: decompress_symmetric_compiled = decompress_symmetric decompress_int4_asymmetric_compiled = decompress_int4_asymmetric decompress_int4_symmetric_compiled = decompress_int4_symmetric + decompress_int2_asymmetric_compiled = decompress_int2_asymmetric quantize_int8_matmul_input_compiled = quantize_int8_matmul_input unpack_int4_compiled = unpack_int4 diff --git a/modules/shared.py b/modules/shared.py index 82e290902..aafc6af11 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -518,7 +518,7 @@ options_templates.update(options_section(('quantization', "Quantization Settings "sdnq_quantize_sep": OptionInfo("

SDNQ: SDNext Quantization

", "", gr.HTML), "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native}), - "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ['int8', 'uint4', "float8_e4m3fn", 'uint8', 'int4', "float8_e5m2"], "visible": native}), + "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ['int8', 'uint4', "float8_e4m3fn", 'uint8', 'int4', "float8_e5m2", "uint2"], "visible": native}), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), From d1491962d97b871de6802276c7f82f5af7ed1f5a Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 30 May 2025 05:41:02 +0300 Subject: [PATCH 65/94] One bit --- CHANGELOG.md | 2 +- modules/model_quant_sdnq.py | 118 ++++++++++++++++++++++-------------- modules/shared.py | 2 +- 3 files changed, 73 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95be9e83a..1fafb1add 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ - `INT8` -> `uint8` - `INT4_SYM` -> `int4` - `INT4` -> `uint4` - - Add `float8_e4m3fn`, `float8_e5m2` and `uint2` support + - Add `float8_e4m3fn`, `float8_e5m2`, `uint2` and `uint1` support - Add quantized matmul support for `float8_e4m3fn` - Set the default quant mode to `pre` - Use per token input quant with int8 and fp8 quantized matmul diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 87da6b592..50ffc49f8 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -21,6 +21,7 @@ dtype_dict = { "int4": {"min": -8, "max": 7, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, "uint4": {"min": 0, "max": 15, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "uint2": {"min": 0, "max": 3, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint1": {"min": 0, "max": 1, "num_bits": 1, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "float8_e4m3fn": {"min": -448, "max": 448, "num_bits": 8, "target_dtype": torch.float8_e4m3fn, "torch_dtype": torch.float8_e4m3fn, "storage_dtype": torch.float8_e4m3fn, "is_unsigned": False, "is_integer": False}, "float8_e5m2": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": torch.float8_e5m2, "torch_dtype": torch.float8_e5m2, "storage_dtype": torch.float8_e5m2, "is_unsigned": False, "is_integer": False}, "float8_e4m3fnuz": {"min": -240, "max": 240, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e4m3fnuz, "storage_dtype": torch.float8_e4m3fnuz, "is_unsigned": False, "is_integer": False}, @@ -146,8 +147,8 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz compressed_weight_shape=layer.weight.shape, result_dtype=torch_dtype, result_shape=result_shape, - use_quantized_matmul=use_quantized_matmul, weights_dtype=weights_dtype, + use_quantized_matmul=use_quantized_matmul, ) layer.weight.data = layer.sdnq_decompressor.pack_weight(layer.weight).to(return_device) layer.sdnq_decompressor = layer.sdnq_decompressor.to(return_device) @@ -251,8 +252,8 @@ def decompress_symmetric(input: torch.Tensor, scale: torch.Tensor, dtype: torch. return result -def decompress_int4_asymmetric(input: torch.Tensor, scale: torch.Tensor, zero_point: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size) -> torch.Tensor: - return decompress_asymmetric(unpack_uint4(input, shape), scale, zero_point, dtype, result_shape) +def decompress_packed_int_asymmetric(input: torch.Tensor, scale: torch.Tensor, zero_point: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str) -> torch.Tensor: + return decompress_asymmetric(packed_int_function_dict[weights_dtype]["unpack"](input, shape), scale, zero_point, dtype, result_shape) def decompress_int4_symmetric(input: torch.Tensor, scale: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, skip_quantized_matmul: bool = False) -> torch.Tensor: @@ -262,10 +263,6 @@ def decompress_int4_symmetric(input: torch.Tensor, scale: torch.Tensor, shape: t return decompress_symmetric(unpack_int4(input, shape, dtype=scale.dtype), scale, dtype, result_shape) -def decompress_int2_asymmetric(input: torch.Tensor, scale: torch.Tensor, zero_point: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size) -> torch.Tensor: - return decompress_asymmetric(unpack_uint2(input, shape), scale, zero_point, dtype, result_shape) - - def pack_uint4(tensor: torch.Tensor) -> torch.Tensor: if tensor.dtype != torch.uint8: raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") @@ -297,6 +294,35 @@ def pack_uint2(tensor: torch.Tensor) -> torch.Tensor: return packed_tensor +def pack_uint1(tensor: torch.Tensor) -> torch.Tensor: + if tensor.dtype != torch.uint8: + raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") + packed_tensor = tensor.contiguous().reshape(-1, 8) + packed_tensor = torch.bitwise_or( + torch.bitwise_or( + torch.bitwise_or( + torch.bitwise_and(packed_tensor[:, 0], 1), + torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 1], 1), 1), + ), + torch.bitwise_or( + torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 2], 1), 2), + torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 3], 1), 3) + ) + ), + torch.bitwise_or( + torch.bitwise_or( + torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 4], 1), 4), + torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 5], 1), 5) + ), + torch.bitwise_or( + torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 6], 1), 6), + torch.bitwise_left_shift(packed_tensor[:, 7], 7) + ) + ), + ) + return packed_tensor + + def unpack_uint4(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: result = torch.stack((torch.bitwise_and(packed_tensor, 15), torch.bitwise_right_shift(packed_tensor, 4)), dim=-1).reshape(shape) return result @@ -322,6 +348,23 @@ def unpack_uint2(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor return result +def unpack_uint1(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: + result = torch.stack( + ( + torch.bitwise_and(packed_tensor, 1), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 1), 1), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 2), 1), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 3), 1), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 4), 1), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 5), 1), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 6), 1), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 7), 1), + ), + dim=-1 + ).reshape(shape) + return result + + def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.FloatTensor, torch.FloatTensor]: input = input.flatten(0,-2).contiguous() input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 448) @@ -444,7 +487,7 @@ class AsymmetricWeightsDecompressor(torch.nn.Module): zero_point: torch.Tensor, result_dtype: torch.dtype, result_shape: torch.Size, - weights_dtype: torch.dtype, + weights_dtype: str, **kwargs, ): super().__init__() @@ -468,7 +511,7 @@ class SymmetricWeightsDecompressor(torch.nn.Module): scale: torch.Tensor, result_dtype: torch.dtype, result_shape: torch.Size, - weights_dtype: torch.dtype, + weights_dtype: str, use_quantized_matmul: bool = False, **kwargs, ): @@ -486,7 +529,7 @@ class SymmetricWeightsDecompressor(torch.nn.Module): return decompress_symmetric_compiled(weight, self.scale, self.result_dtype, self.result_shape, skip_quantized_matmul=skip_quantized_matmul) -class INT4AsymmetricWeightsDecompressor(torch.nn.Module): +class PackedINTAsymmetricWeightsDecompressor(torch.nn.Module): def __init__( self, scale: torch.Tensor, @@ -494,10 +537,11 @@ class INT4AsymmetricWeightsDecompressor(torch.nn.Module): compressed_weight_shape: torch.Size, result_dtype: torch.dtype, result_shape: torch.Size, + weights_dtype: str, **kwargs, ): super().__init__() - self.weights_dtype = "uint4" + self.weights_dtype = weights_dtype self.use_quantized_matmul = False self.compressed_weight_shape = compressed_weight_shape self.result_dtype = result_dtype @@ -506,10 +550,10 @@ class INT4AsymmetricWeightsDecompressor(torch.nn.Module): self.register_buffer("zero_point", zero_point) def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: - return pack_uint4(weight.to(dtype=torch.uint8)) + return packed_int_function_dict[self.weights_dtype]["pack"](weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"])) def forward(self, weight, **kwargs): - return decompress_int4_asymmetric_compiled(weight, self.scale, self.zero_point, self.compressed_weight_shape, self.result_dtype, self.result_shape) + return decompress_packed_int_asymmetric_compiled(weight, self.scale, self.zero_point, self.compressed_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype) class INT4SymmetricWeightsDecompressor(torch.nn.Module): @@ -537,38 +581,13 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): return decompress_int4_symmetric_compiled(weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape, skip_quantized_matmul=skip_quantized_matmul) -class INT2AsymmetricWeightsDecompressor(torch.nn.Module): - def __init__( - self, - scale: torch.Tensor, - zero_point: torch.Tensor, - compressed_weight_shape: torch.Size, - result_dtype: torch.dtype, - result_shape: torch.Size, - **kwargs, - ): - super().__init__() - self.weights_dtype = "uint2" - self.use_quantized_matmul = False - self.compressed_weight_shape = compressed_weight_shape - self.result_dtype = result_dtype - self.result_shape = result_shape - self.register_buffer("scale", scale) - self.register_buffer("zero_point", zero_point) - - def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: - return pack_uint2(weight.to(dtype=torch.uint8)) - - def forward(self, weight, **kwargs): - return decompress_int2_asymmetric_compiled(weight, self.scale, self.zero_point, self.compressed_weight_shape, self.result_dtype, self.result_shape) - - decompressor_dict = { "int8": SymmetricWeightsDecompressor, "uint8": AsymmetricWeightsDecompressor, "int4": INT4SymmetricWeightsDecompressor, - "uint4": INT4AsymmetricWeightsDecompressor, - "uint2": INT2AsymmetricWeightsDecompressor, + "uint4": PackedINTAsymmetricWeightsDecompressor, + "uint2": PackedINTAsymmetricWeightsDecompressor, + "uint1": PackedINTAsymmetricWeightsDecompressor, "float8_e4m3fn": SymmetricWeightsDecompressor, "float8_e4m3fnuz": SymmetricWeightsDecompressor, "float8_e5m2": SymmetricWeightsDecompressor, @@ -577,6 +596,14 @@ decompressor_dict = { } +packed_int_function_dict = { + "int4": {"pack": pack_int4, "unpack": unpack_int4}, + "uint4": {"pack": pack_uint4, "unpack": unpack_uint4}, + "uint2": {"pack": pack_uint2, "unpack": unpack_uint2}, + "uint1": {"pack": pack_uint1, "unpack": unpack_uint1}, +} + + class SDNQQuantizer(DiffusersQuantizer): r""" Diffusers Quantizer for SDNQ @@ -779,9 +806,8 @@ if shared.opts.sdnq_decompress_compile: torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit) decompress_asymmetric_compiled = torch.compile(decompress_asymmetric, fullgraph=True) decompress_symmetric_compiled = torch.compile(decompress_symmetric, fullgraph=True) - decompress_int4_asymmetric_compiled = torch.compile(decompress_int4_asymmetric, fullgraph=True) + decompress_packed_int_asymmetric_compiled = torch.compile(decompress_packed_int_asymmetric, fullgraph=True) decompress_int4_symmetric_compiled = torch.compile(decompress_int4_symmetric, fullgraph=True) - decompress_int2_asymmetric_compiled = torch.compile(decompress_int2_asymmetric, fullgraph=True) fp8_matmul = torch.compile(fp8_matmul, fullgraph=True) fp8_matmul_sm89 = torch.compile(fp8_matmul_sm89, fullgraph=True) if devices.backend != "ipex": # pytorch uses the cpu device in torch._int_mm op with ipex + torch.compile @@ -795,16 +821,14 @@ if shared.opts.sdnq_decompress_compile: shared.log.warning(f"Quantization: type=sdnq Decompress using torch.compile is not available: {e}") decompress_asymmetric_compiled = decompress_asymmetric decompress_symmetric_compiled = decompress_symmetric - decompress_int4_asymmetric_compiled = decompress_int4_asymmetric + decompress_packed_int_asymmetric_compiled = decompress_packed_int_asymmetric decompress_int4_symmetric_compiled = decompress_int4_symmetric - decompress_int2_asymmetric_compiled = decompress_int2_asymmetric quantize_int8_matmul_input_compiled = quantize_int8_matmul_input unpack_int4_compiled = unpack_int4 else: decompress_asymmetric_compiled = decompress_asymmetric decompress_symmetric_compiled = decompress_symmetric - decompress_int4_asymmetric_compiled = decompress_int4_asymmetric + decompress_packed_int_asymmetric_compiled = decompress_packed_int_asymmetric decompress_int4_symmetric_compiled = decompress_int4_symmetric - decompress_int2_asymmetric_compiled = decompress_int2_asymmetric quantize_int8_matmul_input_compiled = quantize_int8_matmul_input unpack_int4_compiled = unpack_int4 diff --git a/modules/shared.py b/modules/shared.py index aafc6af11..8cb4bcb9c 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -518,7 +518,7 @@ options_templates.update(options_section(('quantization', "Quantization Settings "sdnq_quantize_sep": OptionInfo("

SDNQ: SDNext Quantization

", "", gr.HTML), "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native}), - "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ['int8', 'uint4', "float8_e4m3fn", 'uint8', 'int4', "float8_e5m2", "uint2"], "visible": native}), + "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ['int8', 'uint4', "float8_e4m3fn", 'uint8', 'int4', "float8_e5m2", "uint2", "uint1"], "visible": native}), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), From 4e184f41af8a3b7b64c837fdb129b034ab7ad704 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 30 May 2025 08:44:30 +0200 Subject: [PATCH 66/94] update changelog Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 17 +++++++++++++++-- extensions-builtin/stable-diffusion-webui-rembg | 2 +- wiki | 2 +- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fafb1add..de1824aa5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,22 @@ # Change Log for SD.Next -## Update for 2025-05-29 +## Update for 2025-05-30 + +### Highlights for 2025-05-30 + +This release is all about quantization: with new SD.Next own quantization method: **SDNQ** +SDNQ is based on NNCF, but has been re-implemented, optimized and evolved enough to become its own quantization method! +It's fully cross-platform, supports all GPUs and includes 8-bit, 4-bit, 2-bit and 1-bit quantization methods. +Also unlike most traditional methods, its also applicable to nearly all model types. + +*Hint*: Even if you may not need quantization for your current model, it may be worth trying it out as it can significantly improve performance or capabilities of your existing workflow! For example, you may not have issues with SD15 or SDXL, but you may have been limited running at high resolutions or with multiple ControlNet due to VRAM requirements - this will significantly reduce memory requirements. + +[ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) + +### Details for 2025-05-30 - **SDNQ Quantization** - - Renamed `NNCF` to `SDNQ` as NNCF implementation in SDNext re-implemented, optimized and evolved enough to become its own quantization method + - Renamed `NNCF` to `SDNQ` - Renamed quantization scheme names to the underlying dtype names instead of NNCF names - `INT8_SYM` -> `int8` - `INT8` -> `uint8` diff --git a/extensions-builtin/stable-diffusion-webui-rembg b/extensions-builtin/stable-diffusion-webui-rembg index 4dca8aea6..fbf4215c3 160000 --- a/extensions-builtin/stable-diffusion-webui-rembg +++ b/extensions-builtin/stable-diffusion-webui-rembg @@ -1 +1 @@ -Subproject commit 4dca8aea66f7c5be262d7dc3a8dd9509036dab78 +Subproject commit fbf4215c3af8924bb830e4c58887af7acdefccbf diff --git a/wiki b/wiki index ffd60bd24..4e2b6a082 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit ffd60bd24aec14e8613202eec3f92a889c40216c +Subproject commit 4e2b6a082bae84ce5a4adccecf54521d134dc069 From 9168a66fd2bcb8ec3b995e059eb363b5f3e8f193 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 30 May 2025 08:53:39 +0200 Subject: [PATCH 67/94] update requirements Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 12 ++++++------ installer.py | 2 +- package.json | 1 + requirements.txt | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de1824aa5..d2ac998f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -204,7 +204,7 @@ What else? implemented as an extension for **SD.Next** (for the moment while dev is ongoing) generate high-quality videos with pretty much unlimited duration and with limited VRAM! install as any other extension and for details see extension [README](https://github.com/vladmandic/sd-extension-framepack/blob/main/README.md) - - I2V & FLF2V support with explicit strength controls + - I2V & FLF2V support with explicit strength controls - complex actions: modify prompts for each section of the video - LoRA support: use normal **HunyuanVideo** LoRAs - decode: use local, tiny or remote VAE @@ -316,7 +316,7 @@ There are quite a few other performance and quality-of-life improvements in this - **Models** - [HiDream-I1](https://huggingface.co/HiDream-ai/HiDream-I1-Full) in fast, dev and full variants! - new absolutely massive image generative foundation model with **17B** parameters and 4 text-encoders with additional **8.3B** parameters + new absolutely massive image generative foundation model with **17B** parameters and 4 text-encoders with additional **8.3B** parameters simply select from *networks -> models -> reference* due to size (over 25B params in 58GB), offloading and on-the-fly quantization are pretty much a necessity see [HiDream Wiki page](https://github.com/vladmandic/sdnext/wiki/HiDream) for details @@ -375,7 +375,7 @@ Time for another major release with ~120 commits and [ChangeLog](https://github. *Highlights?* Video...Brand new Video processing module with support for all latest models: **WAN21, Hunyuan, LTX, Cog, Allegro, Mochi1, Latte1** in both *T2V* and *I2V* workflows And combined with *on-the-fly quantization*, support for *Local/Tiny/Remote* VAE, acceleration modules such as *FasterCache or PAB*, and more! -Models...And support for new models: **CogView-4**, **SANA 1.5**, +Models...And support for new models: **CogView-4**, **SANA 1.5**, *Plus...* - New **Prompt Enhance** using LLM, @@ -674,7 +674,7 @@ We're back with another update with nearly 100 commits! - updated **CUDA** receipe to `torch==2.6.0` with `cuda==12.6` and add prebuilt image - added **ROCm** receipe and prebuilt image - added **IPEX** receipe and add prebuilt image - - added **OpenVINO** receipe and prebuilt image + - added **OpenVINO** receipe and prebuilt image - **System** - improve **python==3.12** compatibility - **Torch** @@ -747,7 +747,7 @@ Just one week after latest release and what a week it was with over 50 commits! - **GitHub** - rename core repo from to - old repo url should automatically redirect to new one for seamless transition and in-place upgrades + old repo url should automatically redirect to new one for seamless transition and in-place upgrades all internal links have been updated wiki content and docs site have been updated - **Docs**: @@ -1096,7 +1096,7 @@ We've also added support for several new models such as highly anticipated [NVLa And several new SOTA video models: [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video), [Hunyuan Video](https://huggingface.co/tencent/HunyuanVideo) and [Genmo Mochi.1 Preview](https://huggingface.co/genmo/mochi-1-preview) And a lot of **Control** and **IPAdapter** goodies -- for **SDXL** there is new [ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0), improved *Union* and *Tiling* models +- for **SDXL** there is new [ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0), improved *Union* and *Tiling* models - for **FLUX.1** there are [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) as well as official *Canny* and *Depth* models, a cool [Redux](https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev) model as well as [XLabs](https://huggingface.co/XLabs-AI/flux-ip-adapter-v2) IP-adapter - for **SD3.5** there are official *Canny*, *Blur* and *Depth* models in addition to existing 3rd party models diff --git a/installer.py b/installer.py index 8ee43a780..cecf6d0c6 100644 --- a/installer.py +++ b/installer.py @@ -546,7 +546,7 @@ def check_diffusers(): t_start = time.time() if args.skip_all or args.skip_git or args.experimental: return - sha = '20379d9d1395b8e95977faf80facff43065ba75f' # diffusers commit hash + sha = '6508da6f06a0da1054ae6a808d0025c04b70f0e8' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/package.json b/package.json index 15ccbaff1..8e6126230 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "eslint-config-airbnb-base": "^15.0.0", "eslint-plugin-css": "^0.9.2", "eslint-plugin-html": "^8.1.1", + "eslint-plugin-import": "^2.31.0", "eslint-plugin-json": "^3.1.0", "eslint-plugin-markdown": "^4.0.1", "eslint-plugin-node": "^11.1.0" diff --git a/requirements.txt b/requirements.txt index 3c267794f..5c464a7a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,7 @@ torchsde==0.2.6 antlr4-python3-runtime==4.9.3 requests==2.32.3 tqdm==4.67.1 -accelerate==1.6.0 +accelerate==1.7.0 opencv-contrib-python-headless==4.9.0.80 einops==0.4.1 gradio==3.43.2 @@ -52,7 +52,7 @@ numba==0.61.2 protobuf==4.25.3 pytorch_lightning==1.9.4 tokenizers==0.21.1 -transformers==4.51.3 +transformers==4.52.3 urllib3==1.26.19 Pillow==10.4.0 timm==0.9.16 From 98a11fc86cde814add3a66c8fac14bdecef6e4e2 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 30 May 2025 11:04:56 +0200 Subject: [PATCH 68/94] fix gallery duplicate entries Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 8 +++--- javascript/extraNetworks.js | 10 +++---- javascript/gallery.js | 54 ++++++++++++++++++++++++++++++------- modules/ui_loadsave.py | 2 +- 4 files changed, 55 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2ac998f8..4c989db14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ Also unlike most traditional methods, its also applicable to nearly all model ty - `INT8` -> `uint8` - `INT4_SYM` -> `int4` - `INT4` -> `uint4` - - Add `float8_e4m3fn`, `float8_e5m2`, `uint2` and `uint1` support + - Add `float8_e4m3fn`, `float8_e5m2`, `uint2` and `uint1` support - Add quantized matmul support for `float8_e4m3fn` - Set the default quant mode to `pre` - Use per token input quant with int8 and fp8 quantized matmul @@ -34,12 +34,14 @@ Also unlike most traditional methods, its also applicable to nearly all model ty - Fix scale and zero_point not being offloaded - **IPEX** - Disabe Dynamic Attention by default on PyTorch 2.7 - - Remove GradScaler hijack and use torch.amp.GradScaler instead + - Remove GradScaler hijack and use `torch.amp.GradScaler` instead - **Feature** - TeaCache support for HiDream I1 -- **Changes** +- **Changes** - Set the default attention optimizer to Scaled-Dot-Product on all backends - Enable Dynamic attention for Scaled-Dot-Product with ROCm, DirectML, MPS and CPU backends +- **Fixes** + - Gallery duplicate entries ## Update for 2025-05-17 diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index 83f97ba57..aa51e20c1 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -12,11 +12,11 @@ const getENActiveTab = () => { if (gradioApp().getElementById('video_prompt')?.checkVisibility()) return 'video'; if (gradioApp().getElementById('framepack_prompt_row')?.checkVisibility()) return 'framepack'; // legacy method - if (gradioApp().getElementById('tab_txt2img').style.display === 'block') tabName = 'txt2img'; - else if (gradioApp().getElementById('tab_img2img').style.display === 'block') tabName = 'img2img'; - else if (gradioApp().getElementById('tab_control').style.display === 'block') tabName = 'control'; - else if (gradioApp().getElementById('tab_video').style.display === 'block') tabName = 'video'; - else if (gradioApp().getElementById('tab_framepack_tab').style.display === 'block') tabName = 'framepack'; + if (gradioApp().getElementById('tab_txt2img')?.style.display === 'block') tabName = 'txt2img'; + else if (gradioApp().getElementById('tab_img2img')?.style.display === 'block') tabName = 'img2img'; + else if (gradioApp().getElementById('tab_control')?.style.display === 'block') tabName = 'control'; + else if (gradioApp().getElementById('tab_video')?.style.display === 'block') tabName = 'video'; + else if (gradioApp().getElementById('tab_framepack_tab')?.style.display === 'block') tabName = 'framepack'; // log('getENActiveTab', tabName); return tabName; }; diff --git a/javascript/gallery.js b/javascript/gallery.js index cea21fa12..32ae0bde4 100644 --- a/javascript/gallery.js +++ b/javascript/gallery.js @@ -5,6 +5,8 @@ let url; let currentImage; let pruneImagesTimer; let outstanding = 0; +let lastSort = 0; +let lastSortName = 'none'; const el = { folders: undefined, files: undefined, @@ -124,9 +126,14 @@ class GalleryFile extends HTMLElement { } async connectedCallback() { - if (this.shadow.children.length > 0) return; + if (this.shadow.children.length > 0) { + return; + } const ext = this.name.split('.').pop().toLowerCase(); - if (!['jpg', 'jpeg', 'png', 'gif', 'webp', 'jxl', 'svg', 'mp4'].includes(ext)) return; + if (!['jpg', 'jpeg', 'png', 'gif', 'webp', 'jxl', 'svg', 'mp4'].includes(ext)) { + console.error(`gallery: type=${ext} file=${this.name} unsupported`); + return; + } this.hash = await getHash(`${this.folder}/${this.name}/${this.size}/${this.mtime}`); // eslint-disable-line no-use-before-define const style = document.createElement('style'); const width = opts.browser_fixed_width ? `${opts.extra_networks_card_size}px` : 'unset'; @@ -143,7 +150,6 @@ class GalleryFile extends HTMLElement { `; const cache = (this.hash && opts.browser_cache) ? await idbGet(this.hash) : undefined; - this.shadow.appendChild(style); const img = document.createElement('img'); img.className = 'gallery-file'; img.loading = 'lazy'; @@ -196,14 +202,20 @@ class GalleryFile extends HTMLElement { img.src = `file=${this.src}`; } } - if (!ok) return; + if (!ok) { + return; + } img.onclick = () => { currentImage = this.src; el.btnSend.click(); }; img.title = `Folder: ${this.folder}\nFile: ${this.name}\nSize: ${this.size.toLocaleString()} bytes\nModified: ${this.mtime.toLocaleString()}`; + if (this.shadow.children.length > 0) { + return; // avoid double-adding + } this.title = img.title; this.style.display = this.title.toLowerCase().includes(el.search.value.toLowerCase()) ? 'unset' : 'none'; + this.shadow.appendChild(style); this.shadow.appendChild(img); } } @@ -272,49 +284,68 @@ async function gallerySearch(evt) { }, 250); } +const findDuplicates = (arr, key) => { + const map = new Map(); + return arr.filter(item => { + const value = item[key]; + if (map.has(value)) return true; + map.set(value, true); + return false; + }); +}; + async function gallerySort(btn) { const t0 = performance.now(); const arr = Array.from(el.files.children).filter((node) => node.name); // filter out separators + if (arr.length === 0) return; // no files to sort + if (btn) lastSort = btn.charCodeAt(0); + lastSortName = 'none'; const fragment = document.createDocumentFragment(); - el.files.innerHTML = ''; - log('gallerySort', btn.charCodeAt(0)); - switch (btn.charCodeAt(0)) { + switch (lastSort) { case 61789: // name asc + lastSortName = 'name asc'; arr .sort((a, b) => a.name.localeCompare(b.name)) .forEach((node) => fragment.appendChild(node)); break; case 61790: // name dsc + lastSortName = 'name dsc'; arr .sort((b, a) => a.name.localeCompare(b.name)) .forEach((node) => fragment.appendChild(node)); break; case 61792: // size asc + lastSortName = 'size asc'; arr .sort((a, b) => a.size - b.size) .forEach((node) => fragment.appendChild(node)); break; case 61793: // size dsc + lastSortName = 'size dsc'; arr .sort((b, a) => a.size - b.size) .forEach((node) => fragment.appendChild(node)); break; case 61794: // resolution asc + lastSortName = 'resolution asc'; arr .sort((a, b) => a.width * a.height - b.width * b.height) .forEach((node) => fragment.appendChild(node)); break; case 61795: // resolution dsc + lastSortName = 'resolution dsc'; arr .sort((b, a) => a.width * a.height - b.width * b.height) .forEach((node) => fragment.appendChild(node)); break; case 61662: + lastSortName = 'modified asc'; arr .sort((a, b) => a.mtime - b.mtime) .forEach((node) => fragment.appendChild(node)); break; case 61661: + lastSortName = 'modified dsc'; arr .sort((b, a) => a.mtime - b.mtime) .forEach((node) => fragment.appendChild(node)); @@ -322,14 +353,16 @@ async function gallerySort(btn) { default: break; } + if (fragment.children.length === 0) return; + el.files.innerHTML = ''; el.files.appendChild(fragment); addSeparators(); const t1 = performance.now(); - el.status.innerText = `Sort | ${arr.length.toLocaleString()} images | ${Math.floor(t1 - t0).toLocaleString()}ms`; + log(`gallerySort: char=${lastSort} len=${arr.length} time=${Math.floor(t1 - t0)} sort=${lastSortName}`); + el.status.innerText = `Sort | ${lastSortName} | ${arr.length.toLocaleString()} images | ${Math.floor(t1 - t0).toLocaleString()}ms`; } async function fetchFilesHT(evt) { - el.status.innerText = `Folder | ${evt.target.name}`; const t0 = performance.now(); const fragment = document.createDocumentFragment(); el.status.innerText = `Folder | ${evt.target.name} | in-progress`; @@ -389,7 +422,7 @@ async function fetchFilesWS(evt) { // fetch file-by-file list over websockets const file = new GalleryFile(data[0], data[1]); fragment.appendChild(file); if (numFiles % 100 === 0) { - el.status.innerText = `Folder | ${evt.target.name} | ${numFiles.toLocaleString()} images | ${Math.floor(t1 - t0).toLocaleString()}ms`; + el.status.innerText = `Folder | ${evt.target.name} | ${numFiles.toLocaleString()} images | in-progress | ${Math.floor(t1 - t0).toLocaleString()}ms`; el.files.appendChild(fragment); fragment = document.createDocumentFragment(); } @@ -397,6 +430,7 @@ async function fetchFilesWS(evt) { // fetch file-by-file list over websockets }; ws.onclose = (event) => { el.files.appendChild(fragment); + // gallerySort(); log(`gallery: folder=${evt.target.name} num=${numFiles} time=${Math.floor(t1 - t0)}ms`); el.status.innerText = `Folder | ${evt.target.name} | ${numFiles.toLocaleString()} images | ${Math.floor(t1 - t0).toLocaleString()}ms`; addSeparators(); diff --git a/modules/ui_loadsave.py b/modules/ui_loadsave.py index 79efbff62..a8e3dce9b 100644 --- a/modules/ui_loadsave.py +++ b/modules/ui_loadsave.py @@ -48,7 +48,7 @@ class UiLoadsave: if debug_ui and key in self.component_mapping and not key.startswith('customscript'): errors.log.warning(f'UI duplicate: key="{key}" id={getattr(obj, "elem_id", None)} class={getattr(obj, "elem_classes", None)}') if hasattr(obj, 'skip'): - print('HERE', key) + pass if (field == 'value') and (key not in self.component_mapping): self.component_mapping[key] = x if field == 'open' and key not in self.component_mapping: From 29bd2af779a170ffee21b0cd49ff5b81444e69ca Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 30 May 2025 12:20:13 +0300 Subject: [PATCH 69/94] SDNQ add 6-bit support --- CHANGELOG.md | 4 +- modules/model_quant_sdnq.py | 99 +++++++++++++++++++++++++++---------- modules/shared.py | 24 ++++----- 3 files changed, 86 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c989db14..968a613a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ This release is all about quantization: with new SD.Next own quantization method: **SDNQ** SDNQ is based on NNCF, but has been re-implemented, optimized and evolved enough to become its own quantization method! -It's fully cross-platform, supports all GPUs and includes 8-bit, 4-bit, 2-bit and 1-bit quantization methods. +It's fully cross-platform, supports all GPUs and includes 8-bit, 6-bit, 4-bit, 2-bit and 1-bit quantization methods. Also unlike most traditional methods, its also applicable to nearly all model types. *Hint*: Even if you may not need quantization for your current model, it may be worth trying it out as it can significantly improve performance or capabilities of your existing workflow! For example, you may not have issues with SD15 or SDXL, but you may have been limited running at high resolutions or with multiple ControlNet due to VRAM requirements - this will significantly reduce memory requirements. @@ -22,7 +22,7 @@ Also unlike most traditional methods, its also applicable to nearly all model ty - `INT8` -> `uint8` - `INT4_SYM` -> `int4` - `INT4` -> `uint4` - - Add `float8_e4m3fn`, `float8_e5m2`, `uint2` and `uint1` support + - Add `float8_e4m3fn`, `float8_e5m2`, `int6`, `uint6`, `uint2` and `uint1` support - Add quantized matmul support for `float8_e4m3fn` - Set the default quant mode to `pre` - Use per token input quant with int8 and fp8 quantized matmul diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 50ffc49f8..fdfa617c4 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -18,6 +18,8 @@ debug = os.environ.get('SD_QUANT_DEBUG', None) is not None dtype_dict = { "int8": {"min": -128, "max": 127, "num_bits": 8, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.int8, "is_unsigned": False, "is_integer": True}, "uint8": {"min": 0, "max": 255, "num_bits": 8, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "int6": {"min": -32, "max": 31, "num_bits": 6, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, + "uint6": {"min": 0, "max": 63, "num_bits": 6, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "int4": {"min": -8, "max": 7, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, "uint4": {"min": 0, "max": 15, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "uint2": {"min": 0, "max": 3, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, @@ -30,7 +32,7 @@ dtype_dict = { if hasattr(torch, "float8_e8m0fnu"): dtype_dict["float8_e8m0fnu"] = {"min": 5.87747e-39, "max": 1.70141e+38, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e8m0fnu, "storage_dtype": torch.float8_e8m0fnu, "is_unsigned": True, "is_integer": False} -quantized_matmul_dtypes = ("int8", "int4", "float8_e4m3fn") +quantized_matmul_dtypes = ("int8", "int6", "int4", "float8_e4m3fn") linear_types = ("Linear",) conv_types = ("Conv1d", "Conv2d", "Conv3d") @@ -73,7 +75,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: use_quantized_matmul = output_channel_size % 16 == 0 and channel_size % 16 == 0 - if not use_quantized_matmul and (group_size > 0 or (dtype_dict[weights_dtype]["num_bits"] < 8 and group_size != -1)): + if not use_quantized_matmul and (group_size > 0 or (dtype_dict[weights_dtype]["num_bits"] < 6 and group_size != -1)): if group_size == 0: if dtype_dict[weights_dtype]["num_bits"] < 4: group_size = 32 @@ -256,11 +258,32 @@ def decompress_packed_int_asymmetric(input: torch.Tensor, scale: torch.Tensor, z return decompress_asymmetric(packed_int_function_dict[weights_dtype]["unpack"](input, shape), scale, zero_point, dtype, result_shape) -def decompress_int4_symmetric(input: torch.Tensor, scale: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, skip_quantized_matmul: bool = False) -> torch.Tensor: +def decompress_packed_int_symmetric(input: torch.Tensor, scale: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str, skip_quantized_matmul: bool = False) -> torch.Tensor: if skip_quantized_matmul: - return decompress_symmetric(unpack_int4(input, shape, dtype=scale.dtype), scale.transpose(0,1), dtype, result_shape) + return decompress_symmetric(packed_int_function_dict[weights_dtype]["unpack"](input, shape, dtype=scale.dtype), scale.transpose(0,1), dtype, result_shape) else: - return decompress_symmetric(unpack_int4(input, shape, dtype=scale.dtype), scale, dtype, result_shape) + return decompress_symmetric(packed_int_function_dict[weights_dtype]["unpack"](input, shape, dtype=scale.dtype), scale, dtype, result_shape) + + +def pack_uint6(tensor: torch.Tensor) -> torch.Tensor: + if tensor.dtype != torch.uint8: + raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") + packed_tensor = tensor.contiguous().reshape(-1, 4) + packed_tensor = torch.stack( + ( + torch.bitwise_or(torch.bitwise_and(packed_tensor[:, 0], 63), torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 3], 2), 192)), + torch.bitwise_or(torch.bitwise_and(packed_tensor[:, 1], 63), torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 3], 4), 192)), + torch.bitwise_or(torch.bitwise_and(packed_tensor[:, 2], 63), torch.bitwise_left_shift(packed_tensor[:, 3], 6)), + ), + dim=-1 + ) + return packed_tensor + + +def pack_int6(tensor: torch.Tensor) -> torch.Tensor: + if tensor.dtype != torch.int8: + raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.int8 type is supported.") + return pack_uint6((tensor + 32).to(dtype=torch.uint8)) def pack_uint4(tensor: torch.Tensor) -> torch.Tensor: @@ -323,6 +346,32 @@ def pack_uint1(tensor: torch.Tensor) -> torch.Tensor: return packed_tensor +def unpack_uint6(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: + result = torch.stack( + ( + torch.bitwise_and(packed_tensor[:, 0], 63), + torch.bitwise_and(packed_tensor[:, 1], 63), + torch.bitwise_and(packed_tensor[:, 2], 63), + torch.bitwise_or( + torch.bitwise_or( + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 0], 2), 48), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 1], 4), 12), + ), + torch.bitwise_right_shift(packed_tensor[:, 2], 6) + ) + ), + dim=-1 + ).reshape(shape) + return result + + +def unpack_int6(packed_tensor: torch.Tensor, shape: torch.Size, dtype: Optional[torch.dtype] = torch.int8, transpose: Optional[bool] = False) -> torch.Tensor: + result = unpack_uint6(packed_tensor, shape).to(dtype=dtype).sub_(32) + if transpose: + result = result.transpose(0,1) + return result + + def unpack_uint4(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: result = torch.stack((torch.bitwise_and(packed_tensor, 15), torch.bitwise_right_shift(packed_tensor, 4)), dim=-1).reshape(shape) return result @@ -430,13 +479,14 @@ def int8_matmul( bias: torch.FloatTensor, scale: torch.FloatTensor, compressed_weight_shape: torch.Size, + weights_dtype: str, ) -> torch.FloatTensor: if compressed_weight_shape is not None: - weight = unpack_int4_compiled(weight, compressed_weight_shape, transpose=True) + weight = packed_int_function_dict[weights_dtype]["unpack"](weight, compressed_weight_shape, transpose=True) return_dtype = input.dtype output_shape = list(input.shape) output_shape[-1] = weight.shape[-1] - input, scale = quantize_int8_matmul_input_compiled(input, scale) + input, scale = quantize_int8_matmul_input(input, scale) result = decompress_symmetric_compiled(torch._int_mm(input, weight), scale, return_dtype, output_shape) if bias is not None: result.add_(bias) @@ -454,7 +504,7 @@ def quantized_linear_forward_fp8_matmul_sm89(self, input: torch.FloatTensor) -> def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: if torch.numel(input) / input.shape[-1] < 32: return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) - return int8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale, getattr(self.sdnq_decompressor, "compressed_weight_shape", None)) + return int8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale, getattr(self.sdnq_decompressor, "compressed_weight_shape", None), self.sdnq_decompressor.weights_dtype) def quantized_linear_forward(self, input: torch.FloatTensor) -> torch.FloatTensor: @@ -556,18 +606,19 @@ class PackedINTAsymmetricWeightsDecompressor(torch.nn.Module): return decompress_packed_int_asymmetric_compiled(weight, self.scale, self.zero_point, self.compressed_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype) -class INT4SymmetricWeightsDecompressor(torch.nn.Module): +class PackedINTSymmetricWeightsDecompressor(torch.nn.Module): def __init__( self, scale: torch.Tensor, compressed_weight_shape: torch.Size, result_dtype: torch.dtype, result_shape: torch.Size, + weights_dtype: str, use_quantized_matmul: bool = False, **kwargs, ): super().__init__() - self.weights_dtype = "int4" + self.weights_dtype = weights_dtype self.use_quantized_matmul = use_quantized_matmul self.compressed_weight_shape = compressed_weight_shape self.result_dtype = result_dtype @@ -575,16 +626,18 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): self.register_buffer("scale", scale) def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: - return pack_int4(weight.to(dtype=torch.int8)) + return packed_int_function_dict[self.weights_dtype]["pack"](weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"])) def forward(self, weight, skip_quantized_matmul=False, **kwargs): - return decompress_int4_symmetric_compiled(weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape, skip_quantized_matmul=skip_quantized_matmul) + return decompress_packed_int_symmetric_compiled(weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype, skip_quantized_matmul=skip_quantized_matmul) decompressor_dict = { "int8": SymmetricWeightsDecompressor, "uint8": AsymmetricWeightsDecompressor, - "int4": INT4SymmetricWeightsDecompressor, + "int6": PackedINTSymmetricWeightsDecompressor, + "uint6": PackedINTAsymmetricWeightsDecompressor, + "int4": PackedINTSymmetricWeightsDecompressor, "uint4": PackedINTAsymmetricWeightsDecompressor, "uint2": PackedINTAsymmetricWeightsDecompressor, "uint1": PackedINTAsymmetricWeightsDecompressor, @@ -597,6 +650,8 @@ decompressor_dict = { packed_int_function_dict = { + "int6": {"pack": pack_int6, "unpack": unpack_int6}, + "uint6": {"pack": pack_uint6, "unpack": unpack_uint6}, "int4": {"pack": pack_int4, "unpack": unpack_int4}, "uint4": {"pack": pack_uint4, "unpack": unpack_uint4}, "uint2": {"pack": pack_uint2, "unpack": unpack_uint2}, @@ -807,28 +862,18 @@ if shared.opts.sdnq_decompress_compile: decompress_asymmetric_compiled = torch.compile(decompress_asymmetric, fullgraph=True) decompress_symmetric_compiled = torch.compile(decompress_symmetric, fullgraph=True) decompress_packed_int_asymmetric_compiled = torch.compile(decompress_packed_int_asymmetric, fullgraph=True) - decompress_int4_symmetric_compiled = torch.compile(decompress_int4_symmetric, fullgraph=True) + decompress_packed_int_symmetric_compiled = torch.compile(decompress_packed_int_symmetric, fullgraph=True) fp8_matmul = torch.compile(fp8_matmul, fullgraph=True) fp8_matmul_sm89 = torch.compile(fp8_matmul_sm89, fullgraph=True) - if devices.backend != "ipex": # pytorch uses the cpu device in torch._int_mm op with ipex + torch.compile - quantize_int8_matmul_input_compiled = quantize_int8_matmul_input - unpack_int4_compiled = unpack_int4 - int8_matmul = torch.compile(int8_matmul, fullgraph=True) - else: - quantize_int8_matmul_input_compiled = torch.compile(quantize_int8_matmul_input, fullgraph=True) - unpack_int4_compiled = torch.compile(unpack_int4, fullgraph=True) + int8_matmul = torch.compile(int8_matmul, fullgraph=True) except Exception as e: shared.log.warning(f"Quantization: type=sdnq Decompress using torch.compile is not available: {e}") decompress_asymmetric_compiled = decompress_asymmetric decompress_symmetric_compiled = decompress_symmetric decompress_packed_int_asymmetric_compiled = decompress_packed_int_asymmetric - decompress_int4_symmetric_compiled = decompress_int4_symmetric - quantize_int8_matmul_input_compiled = quantize_int8_matmul_input - unpack_int4_compiled = unpack_int4 + decompress_packed_int_symmetric_compiled = decompress_packed_int_symmetric else: decompress_asymmetric_compiled = decompress_asymmetric decompress_symmetric_compiled = decompress_symmetric decompress_packed_int_asymmetric_compiled = decompress_packed_int_asymmetric - decompress_int4_symmetric_compiled = decompress_int4_symmetric - quantize_int8_matmul_input_compiled = quantize_int8_matmul_input - unpack_int4_compiled = unpack_int4 + decompress_packed_int_symmetric_compiled = decompress_packed_int_symmetric diff --git a/modules/shared.py b/modules/shared.py index 8cb4bcb9c..e73ec0441 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -505,20 +505,20 @@ options_templates.update(options_section(('backends', "Backend Settings"), { "openvino_sep": OptionInfo("

OpenVINO

", "", gr.HTML, {"visible": cmd_opts.use_openvino}), "openvino_devices": OptionInfo([], "OpenVINO devices to use", gr.CheckboxGroup, {"choices": get_openvino_device_list() if cmd_opts.use_openvino else [], "visible": cmd_opts.use_openvino}), # pylint: disable=E0606 - "openvino_accuracy": OptionInfo("performance", "OpenVINO accuracy mode", gr.Radio, {"choices": ['performance', 'accuracy'], "visible": cmd_opts.use_openvino}), + "openvino_accuracy": OptionInfo("performance", "OpenVINO accuracy mode", gr.Radio, {"choices": ["performance", "accuracy"], "visible": cmd_opts.use_openvino}), "openvino_disable_model_caching": OptionInfo(True, "OpenVINO disable model caching", gr.Checkbox, {"visible": cmd_opts.use_openvino}), "openvino_disable_memory_cleanup": OptionInfo(True, "OpenVINO disable memory cleanup after compile", gr.Checkbox, {"visible": cmd_opts.use_openvino}), "directml_sep": OptionInfo("

DirectML

", "", gr.HTML, {"visible": devices.backend == "directml"}), - "directml_memory_provider": OptionInfo(default_memory_provider, 'DirectML memory stats provider', gr.Radio, {"choices": memory_providers, "visible": devices.backend == "directml"}), + "directml_memory_provider": OptionInfo(default_memory_provider, "DirectML memory stats provider", gr.Radio, {"choices": memory_providers, "visible": devices.backend == "directml"}), "directml_catch_nan": OptionInfo(False, "DirectML retry ops for NaN", gr.Checkbox, {"visible": devices.backend == "directml"}), })) -options_templates.update(options_section(('quantization', "Quantization Settings"), { +options_templates.update(options_section(("quantization", "Quantization Settings"), { "sdnq_quantize_sep": OptionInfo("

SDNQ: SDNext Quantization

", "", gr.HTML), "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), - "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native}), - "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ['int8', 'uint4', "float8_e4m3fn", 'uint8', 'int4', "float8_e5m2", "uint2", "uint1"], "visible": native}), + "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ["pre", "post"], "visible": native}), + "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ["int8", "int6", "uint4", "float8_e4m3fn", "uint8", "uint6", "int4", "float8_e5m2", "uint2", "uint1"], "visible": native}), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), @@ -529,7 +529,7 @@ options_templates.update(options_section(('quantization', "Quantization Settings "bnb_quantization_sep": OptionInfo("

BitsAndBytes

", "", gr.HTML), "bnb_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), - "bnb_quantization_type": OptionInfo("nf4", "Quantization type", gr.Dropdown, {"choices": ['nf4', 'fp8', 'fp4'], "visible": native}), + "bnb_quantization_type": OptionInfo("nf4", "Quantization type", gr.Dropdown, {"choices": ["nf4", "fp8", "fp4"], "visible": native}), "bnb_quantization_storage": OptionInfo("uint8", "Backend storage", gr.Dropdown, {"choices": ["float16", "float32", "int8", "uint8", "float64", "bfloat16"], "visible": native}), "quanto_quantization_sep": OptionInfo("

Optimum Quanto

", "", gr.HTML), @@ -538,14 +538,14 @@ options_templates.update(options_section(('quantization', "Quantization Settings "optimum_quanto_sep": OptionInfo("

Optimum Quanto: post-load

", "", gr.HTML), "optimum_quanto_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), - "optimum_quanto_weights_type": OptionInfo("qint8", "Quantization weights type", gr.Dropdown, {"choices": ['qint8', 'qfloat8_e4m3fn', 'qfloat8_e5m2', 'qint4', 'qint2'], "visible": native}), - "optimum_quanto_activations_type": OptionInfo("none", "Quantization activations type ", gr.Dropdown, {"choices": ['none', 'qint8', 'qfloat8_e4m3fn', 'qfloat8_e5m2'], "visible": native}), + "optimum_quanto_weights_type": OptionInfo("qint8", "Quantization weights type", gr.Dropdown, {"choices": ["qint8", "qfloat8_e4m3fn", "qfloat8_e5m2", "qint4", "qint2"], "visible": native}), + "optimum_quanto_activations_type": OptionInfo("none", "Quantization activations type ", gr.Dropdown, {"choices": ["none", "qint8", "qfloat8_e4m3fn", "qfloat8_e5m2"], "visible": native}), "optimum_quanto_shuffle_weights": OptionInfo(False, "Shuffle weights in post mode", gr.Checkbox, {"visible": native}), "torchao_sep": OptionInfo("

TorchAO

", "", gr.HTML), "torchao_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM"], "visible": native}), - "torchao_quantization_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native}), - "torchao_quantization_type": OptionInfo("int8_weight_only", "Quantization type", gr.Dropdown, {"choices": ['int4_weight_only', 'int8_dynamic_activation_int4_weight', 'int8_weight_only', 'int8_dynamic_activation_int8_weight', 'float8_weight_only', 'float8_dynamic_activation_float8_weight', 'float8_static_activation_float8_weight'], "visible": native}), + "torchao_quantization_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ["pre", "post"], "visible": native}), + "torchao_quantization_type": OptionInfo("int8_weight_only", "Quantization type", gr.Dropdown, {"choices": ["int4_weight_only", "int8_dynamic_activation_int4_weight", "int8_weight_only", "int8_dynamic_activation_int8_weight", "float8_weight_only", "float8_dynamic_activation_float8_weight", "float8_static_activation_float8_weight"], "visible": native}), "layerwise_quantization_sep": OptionInfo("

Layerwise Casting

", "", gr.HTML), "layerwise_quantization": OptionInfo([], "Layerwise casting enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "TE"], "visible": native}), @@ -559,11 +559,11 @@ options_templates.update(options_section(('quantization', "Quantization Settings "nncf_compress_sep": OptionInfo("

NNCF: Neural Network Compression Framework

", "", gr.HTML, {"visible": cmd_opts.use_openvino}), "nncf_compress_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": cmd_opts.use_openvino}), - "nncf_compress_weights_mode": OptionInfo("INT8_SYM", "Quantization type", gr.Dropdown, {"choices": ['INT8', 'INT4_ASYM', 'INT8_SYM', 'INT4_SYM', 'NF4'], "visible": cmd_opts.use_openvino}), + "nncf_compress_weights_mode": OptionInfo("INT8_SYM", "Quantization type", gr.Dropdown, {"choices": ["INT8", "INT4_ASYM", "INT8_SYM", "INT4_SYM", "NF4"], "visible": cmd_opts.use_openvino}), "nncf_compress_weights_raito": OptionInfo(0, "Compress ratio", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}), "nncf_compress_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": cmd_opts.use_openvino}), "nncf_quantize": OptionInfo([], "Static Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "TE"], "visible": cmd_opts.use_openvino}), - "nncf_quantize_mode": OptionInfo("INT8", "OpenVINO activations mode", gr.Dropdown, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}), + "nncf_quantize_mode": OptionInfo("INT8", "OpenVINO activations mode", gr.Dropdown, {"choices": ["INT8", "FP8_E4M3", "FP8_E5M2"], "visible": cmd_opts.use_openvino}), })) options_templates.update(options_section(('advanced', "Pipeline Modifiers"), { From 36febda6e6f906b544a4285e254be9e232279c35 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 30 May 2025 13:07:23 +0300 Subject: [PATCH 70/94] SDNQ update supported dtypes --- modules/model_quant_sdnq.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index fdfa617c4..342086fe5 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -803,7 +803,7 @@ class SDNQConfig(QuantizationConfigMixin): Args: weights_dtype (`str`, *optional*, defaults to `"int8"`): The target dtype for the weights after quantization. Supported values are: - ("int8", "uint8", "int4", "uint4", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz", "float8_e8m0fnu") + ("int8", "uint8", "int6", "uint6", "int4", "uint4", "uint2", "uint1", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz", "float8_e8m0fnu") modules_to_not_convert (`list`, *optional*, default to `None`): The list of modules to not quantize, useful for quantizing models that explicitly require to have some modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers). @@ -831,7 +831,7 @@ class SDNQConfig(QuantizationConfigMixin): r""" Safety checker that arguments are correct """ - accepted_weights = ["int8", "uint8", "int4", "uint4", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz", "float8_e8m0fnu"] + accepted_weights = ["int8", "uint8", "int6", "uint6", "int4", "uint4", "uint2", "uint1", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz", "float8_e8m0fnu"] if self.weights_dtype not in accepted_weights: raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights_dtype}") From f81cb22c004bd4e6c08e11be9e2987e6e7a70599 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 30 May 2025 13:32:03 +0300 Subject: [PATCH 71/94] SDNQ fix new transformers --- modules/model_quant_sdnq.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 342086fe5..5b728ead8 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -761,6 +761,9 @@ class SDNQQuantizer(DiffusersQuantizer): devices.torch_gc(force=True) return model + def get_cuda_warm_up_factor(self): + return 32 // dtype_dict[self.quantization_config.weights_dtype]["num_bits"] + def update_tp_plan(self, config): """ needed for transformers compatibilty, no-op function From 87a801e24d381d7ea3cbdfeb5957ab1a769db19a Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 30 May 2025 13:54:49 +0300 Subject: [PATCH 72/94] SDNQ remove memory fix hijack --- modules/model_quant_sdnq.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 5b728ead8..a40faee58 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -681,8 +681,6 @@ class SDNQQuantizer(DiffusersQuantizer): state_dict: Dict[str, Any], **kwargs, ): - if shared.opts.device_map != "gpu": - param_value.data = param_value.clone() # safetensors is unable to release the cpu memory without this if param_name.endswith(".weight"): split_param_name = param_name.split(".") if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert): From 4654acde3c73cfe1fcbce484c4e3e10815d11a46 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 30 May 2025 14:59:45 +0300 Subject: [PATCH 73/94] SDNQ re-enable memory fix for diffusers --- modules/model_quant_sdnq.py | 3 +++ modules/teacache/teacache_hidream.py | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index a40faee58..f7c867e90 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -6,6 +6,7 @@ from enum import Enum import os import sys import torch +import diffusers from diffusers.quantizers.base import DiffusersQuantizer from diffusers.quantizers.quantization_config import QuantizationConfigMixin from diffusers.utils import get_module_from_name @@ -681,6 +682,8 @@ class SDNQQuantizer(DiffusersQuantizer): state_dict: Dict[str, Any], **kwargs, ): + if hasattr(diffusers, model.__class__.__name__): + param_value.data = param_value.clone() # safetensors is unable to release the cpu memory without this if param_name.endswith(".weight"): split_param_name = param_name.split(".") if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert): diff --git a/modules/teacache/teacache_hidream.py b/modules/teacache/teacache_hidream.py index b3bd84bb8..8f7f4b859 100644 --- a/modules/teacache/teacache_hidream.py +++ b/modules/teacache/teacache_hidream.py @@ -1,7 +1,4 @@ from typing import Any, Dict, List, Optional, Tuple -from transformers import PreTrainedTokenizerFast, LlamaForCausalLM -from diffusers import HiDreamImagePipeline -from diffusers.models import HiDreamImageTransformer2DModel from diffusers.models.modeling_outputs import Transformer2DModelOutput from diffusers.utils import logging, deprecate, USE_PEFT_BACKEND, logging, scale_lora_layers, unscale_lora_layers From c85cc6b397fb3abbc79a188e9ea3424cc5d34420 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 30 May 2025 15:21:29 +0300 Subject: [PATCH 74/94] SDNQ enable quant with GPU by default and don't do unnecessary clones --- modules/model_quant_sdnq.py | 9 +++++---- modules/shared.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index f7c867e90..20d329e8c 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -6,7 +6,6 @@ from enum import Enum import os import sys import torch -import diffusers from diffusers.quantizers.base import DiffusersQuantizer from diffusers.quantizers.quantization_config import QuantizationConfigMixin from diffusers.utils import get_module_from_name @@ -682,17 +681,17 @@ class SDNQQuantizer(DiffusersQuantizer): state_dict: Dict[str, Any], **kwargs, ): - if hasattr(diffusers, model.__class__.__name__): - param_value.data = param_value.clone() # safetensors is unable to release the cpu memory without this if param_name.endswith(".weight"): split_param_name = param_name.split(".") if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert): layer_class_name = get_module_from_name(model, param_name)[0].__class__.__name__ if layer_class_name in allowed_types: if layer_class_name in conv_types or layer_class_name in conv_transpose_types: - return self.quantization_config.quant_conv + if self.quantization_config.quant_conv: + return True else: return True + param_value.data = param_value.clone() # safetensors is unable to release the cpu memory without this return False def check_quantized_param(self, *args, **kwargs) -> bool: @@ -716,6 +715,8 @@ class SDNQQuantizer(DiffusersQuantizer): if shared.opts.sdnq_quantize_with_gpu: param_value = param_value.to(devices.device).to(dtype=torch.float32) else: + if param_value.dtype != torch.float32 and param_value.device == torch.device(target_device): + param_value = param_value.clone() param_value = param_value.to(target_device).to(dtype=torch.float32) layer.weight = torch.nn.Parameter(param_value, requires_grad=False) layer = sdnq_quantize_layer( diff --git a/modules/shared.py b/modules/shared.py index e73ec0441..f67a2f0b4 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -524,7 +524,7 @@ options_templates.update(options_section(("quantization", "Quantization Settings "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), "sdnq_decompress_compile": OptionInfo(devices.has_triton(), "Decompress using torch.compile", gr.Checkbox, {"visible": native}), "sdnq_use_quantized_matmul": OptionInfo(False, "Use quantized MatMul", gr.Checkbox, {"visible": native}), - "sdnq_quantize_with_gpu": OptionInfo(False, "Quantize with the GPU", gr.Checkbox, {"visible": native}), + "sdnq_quantize_with_gpu": OptionInfo(True, "Quantize with the GPU", gr.Checkbox, {"visible": native}), "sdnq_quantize_shuffle_weights": OptionInfo(False, "Shuffle weights in post mode", gr.Checkbox, {"visible": native}), "bnb_quantization_sep": OptionInfo("

BitsAndBytes

", "", gr.HTML), From db816d7088ec10b96ad248045c6613701288b2e6 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 30 May 2025 16:02:26 +0300 Subject: [PATCH 75/94] Cleanup --- modules/model_quant_sdnq.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 20d329e8c..35ef15744 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -467,7 +467,7 @@ def fp8_matmul_sm89( output_shape[-1] = weight.shape[-1] dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) input, scale = quantize_fp8_matmul_input_sm89(input, scale) - result = decompress_symmetric_compiled(torch._scaled_mm(input, weight, dummy_input_scale, dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, output_shape) + result = decompress_symmetric(torch._scaled_mm(input, weight, dummy_input_scale, dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, output_shape) if bias is not None: result.add_(bias) return result @@ -487,7 +487,7 @@ def int8_matmul( output_shape = list(input.shape) output_shape[-1] = weight.shape[-1] input, scale = quantize_int8_matmul_input(input, scale) - result = decompress_symmetric_compiled(torch._int_mm(input, weight), scale, return_dtype, output_shape) + result = decompress_symmetric(torch._int_mm(input, weight), scale, return_dtype, output_shape) if bias is not None: result.add_(bias) return result @@ -713,11 +713,15 @@ class SDNQQuantizer(DiffusersQuantizer): # load the model params to target_device first layer, _ = get_module_from_name(model, param_name) if shared.opts.sdnq_quantize_with_gpu: - param_value = param_value.to(devices.device).to(dtype=torch.float32) - else: - if param_value.dtype != torch.float32 and param_value.device == torch.device(target_device): + if param_value.dtype == torch.float32 and devices.same_device(param_value.device, devices.device): param_value = param_value.clone() - param_value = param_value.to(target_device).to(dtype=torch.float32) + else: + param_value = param_value.to(devices.device).to(dtype=torch.float32) + else: + if param_value.dtype == torch.float32 and devices.same_device(param_value.device, target_device): + param_value = param_value.clone() + else: + param_value = param_value.to(target_device).to(dtype=torch.float32) layer.weight = torch.nn.Parameter(param_value, requires_grad=False) layer = sdnq_quantize_layer( layer, From b5d588fa459565ddee2f4ad5331189b4a33b2f64 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 30 May 2025 16:29:59 +0300 Subject: [PATCH 76/94] SDNQ remove unnecessary bitwise ands --- modules/model_quant_sdnq.py | 38 ++++++++++--------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 35ef15744..69254f692 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -271,9 +271,9 @@ def pack_uint6(tensor: torch.Tensor) -> torch.Tensor: packed_tensor = tensor.contiguous().reshape(-1, 4) packed_tensor = torch.stack( ( - torch.bitwise_or(torch.bitwise_and(packed_tensor[:, 0], 63), torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 3], 2), 192)), - torch.bitwise_or(torch.bitwise_and(packed_tensor[:, 1], 63), torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 3], 4), 192)), - torch.bitwise_or(torch.bitwise_and(packed_tensor[:, 2], 63), torch.bitwise_left_shift(packed_tensor[:, 3], 6)), + torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 3], 2), 192)), + torch.bitwise_or(packed_tensor[:, 1], torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 3], 4), 192)), + torch.bitwise_or(packed_tensor[:, 2], torch.bitwise_left_shift(packed_tensor[:, 3], 6)), ), dim=-1 ) @@ -290,7 +290,7 @@ def pack_uint4(tensor: torch.Tensor) -> torch.Tensor: if tensor.dtype != torch.uint8: raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") packed_tensor = tensor.contiguous().reshape(-1, 2) - packed_tensor = torch.bitwise_or(torch.bitwise_and(packed_tensor[:, 0], 15), torch.bitwise_left_shift(packed_tensor[:, 1], 4)) + packed_tensor = torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_left_shift(packed_tensor[:, 1], 4)) return packed_tensor @@ -305,14 +305,8 @@ def pack_uint2(tensor: torch.Tensor) -> torch.Tensor: raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") packed_tensor = tensor.contiguous().reshape(-1, 4) packed_tensor = torch.bitwise_or( - torch.bitwise_or( - torch.bitwise_and(packed_tensor[:, 0], 3), - torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 1], 3), 2) - ), - torch.bitwise_or( - torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 2], 3), 4), - torch.bitwise_left_shift(packed_tensor[:, 3], 6) - ), + torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_left_shift(packed_tensor[:, 1], 2)), + torch.bitwise_or(torch.bitwise_left_shift(packed_tensor[:, 2], 4), torch.bitwise_left_shift(packed_tensor[:, 3], 6)), ) return packed_tensor @@ -323,24 +317,12 @@ def pack_uint1(tensor: torch.Tensor) -> torch.Tensor: packed_tensor = tensor.contiguous().reshape(-1, 8) packed_tensor = torch.bitwise_or( torch.bitwise_or( - torch.bitwise_or( - torch.bitwise_and(packed_tensor[:, 0], 1), - torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 1], 1), 1), - ), - torch.bitwise_or( - torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 2], 1), 2), - torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 3], 1), 3) - ) + torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_left_shift(packed_tensor[:, 1], 1)), + torch.bitwise_or(torch.bitwise_left_shift(packed_tensor[:, 2], 2), torch.bitwise_left_shift(packed_tensor[:, 3], 3)) ), torch.bitwise_or( - torch.bitwise_or( - torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 4], 1), 4), - torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 5], 1), 5) - ), - torch.bitwise_or( - torch.bitwise_left_shift(torch.bitwise_and(packed_tensor[:, 6], 1), 6), - torch.bitwise_left_shift(packed_tensor[:, 7], 7) - ) + torch.bitwise_or(torch.bitwise_left_shift(packed_tensor[:, 4], 4), torch.bitwise_left_shift(packed_tensor[:, 5], 5)), + torch.bitwise_or(torch.bitwise_left_shift(packed_tensor[:, 6], 6), torch.bitwise_left_shift(packed_tensor[:, 7], 7)) ), ) return packed_tensor From 959b759721752b668cb37e06e7db64f2ad9730fc Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 30 May 2025 16:45:59 +0300 Subject: [PATCH 77/94] Cleanup --- modules/model_quant_sdnq.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 69254f692..4d6f62129 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -3,7 +3,6 @@ from typing import Any, Dict, List, Tuple, Optional, Union from dataclasses import dataclass from enum import Enum -import os import sys import torch from diffusers.quantizers.base import DiffusersQuantizer @@ -13,8 +12,6 @@ from accelerate.utils import CustomDtype from modules import devices, shared -debug = os.environ.get('SD_QUANT_DEBUG', None) is not None - dtype_dict = { "int8": {"min": -128, "max": 127, "num_bits": 8, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.int8, "is_unsigned": False, "is_integer": True}, "uint8": {"min": 0, "max": 255, "num_bits": 8, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, @@ -372,7 +369,7 @@ def unpack_uint2(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor torch.bitwise_and(packed_tensor, 3), torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 2), 3), torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 4), 3), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 6), 3), + torch.bitwise_right_shift(packed_tensor, 6), ), dim=-1 ).reshape(shape) @@ -389,7 +386,7 @@ def unpack_uint1(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 4), 1), torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 5), 1), torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 6), 1), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 7), 1), + torch.bitwise_right_shift(packed_tensor, 7), ), dim=-1 ).reshape(shape) From 109c0d7e49f1446f82bc3cec383b5cdf6e1d3c9f Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 30 May 2025 21:09:53 +0300 Subject: [PATCH 78/94] SDNQ use tensorwise FP8 matmul on CPU --- modules/model_quant_sdnq.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 4d6f62129..46413748b 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -157,8 +157,8 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if dtype_dict[weights_dtype]["is_integer"]: layer.forward = quantized_linear_forward_int8_matmul else: - if devices.backend == "cuda" and sys.platform == "win32" and float(torch.__version__[:3]) <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9): - layer.forward = quantized_linear_forward_fp8_matmul_sm89 + if devices.backend == "cpu" or (devices.backend == "cuda" and sys.platform == "win32" and float(torch.__version__[:3]) <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)): + layer.forward = quantized_linear_forward_fp8_matmul_tensorwise else: layer.forward = quantized_linear_forward_fp8_matmul else: @@ -401,7 +401,7 @@ def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.FloatTens return input, input_scale -def quantize_fp8_matmul_input_sm89(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.ByteTensor, torch.FloatTensor]: +def quantize_fp8_matmul_input_tensorwise(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.ByteTensor, torch.FloatTensor]: input = input.flatten(0,-2).contiguous() input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 448) input = torch.div(input, input_scale).clamp_(-448, 448).to(torch.float8_e4m3fn) @@ -435,7 +435,7 @@ def fp8_matmul( # sm89 doesn't support row wise scale in Windows -def fp8_matmul_sm89( +def fp8_matmul_tensorwise( input: torch.FloatTensor, weight: torch.Tensor, bias: torch.FloatTensor, @@ -445,7 +445,7 @@ def fp8_matmul_sm89( output_shape = list(input.shape) output_shape[-1] = weight.shape[-1] dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) - input, scale = quantize_fp8_matmul_input_sm89(input, scale) + input, scale = quantize_fp8_matmul_input_tensorwise(input, scale) result = decompress_symmetric(torch._scaled_mm(input, weight, dummy_input_scale, dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, output_shape) if bias is not None: result.add_(bias) @@ -476,8 +476,8 @@ def quantized_linear_forward_fp8_matmul(self, input: torch.FloatTensor) -> torch return fp8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale) -def quantized_linear_forward_fp8_matmul_sm89(self, input: torch.FloatTensor) -> torch.FloatTensor: - return fp8_matmul_sm89(input, self.weight, self.bias, self.sdnq_decompressor.scale) +def quantized_linear_forward_fp8_matmul_tensorwise(self, input: torch.FloatTensor) -> torch.FloatTensor: + return fp8_matmul_tensorwise(input, self.weight, self.bias, self.sdnq_decompressor.scale) def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: @@ -852,7 +852,7 @@ if shared.opts.sdnq_decompress_compile: decompress_packed_int_asymmetric_compiled = torch.compile(decompress_packed_int_asymmetric, fullgraph=True) decompress_packed_int_symmetric_compiled = torch.compile(decompress_packed_int_symmetric, fullgraph=True) fp8_matmul = torch.compile(fp8_matmul, fullgraph=True) - fp8_matmul_sm89 = torch.compile(fp8_matmul_sm89, fullgraph=True) + fp8_matmul_tensorwise = torch.compile(fp8_matmul_tensorwise, fullgraph=True) int8_matmul = torch.compile(int8_matmul, fullgraph=True) except Exception as e: shared.log.warning(f"Quantization: type=sdnq Decompress using torch.compile is not available: {e}") From 046840c8be23370001da165076a81465260c62cb Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 31 May 2025 00:52:56 +0300 Subject: [PATCH 79/94] Fix HiDream sampling --- modules/sd_samplers_common.py | 2 +- modules/sd_samplers_diffusers.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 5eb4a5f94..52fd6313e 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -9,7 +9,7 @@ from modules import shared, devices, processing, images, sd_vae_approx, sd_vae_t SamplerData = namedtuple('SamplerData', ['name', 'constructor', 'aliases', 'options']) approximation_indexes = { "Simple": 0, "Approximate": 1, "TAESD": 2, "Full VAE": 3 } -flow_models = ['f1', 'sd3', 'lumina', 'auraflow', 'sana', 'lumina2', 'cogview4'] +flow_models = ['f1', 'sd3', 'lumina', 'auraflow', 'sana', 'lumina2', 'cogview4', 'h1'] warned = False queue_lock = threading.Lock() diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index ca5fa21fd..f151ab151 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -66,7 +66,7 @@ config = { # prediction_type is ideally set in model as well, but it maybe needed that we do auto-detect of model type in the future 'All': { 'num_train_timesteps': 1000, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'linear', 'prediction_type': 'epsilon' }, - 'UniPC': { 'predict_x0': True, 'sample_max_value': 1.0, 'solver_order': 2, 'solver_type': 'bh2', 'thresholding': False, 'use_beta_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_karras_sigmas': False, 'lower_order_final': True, 'timestep_spacing': 'linspace', 'final_sigmas_type': 'zero', 'rescale_betas_zero_snr': False }, + 'UniPC': { 'flow_shift': 1, 'predict_x0': True, 'sample_max_value': 1.0, 'solver_order': 2, 'solver_type': 'bh2', 'thresholding': False, 'use_beta_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_karras_sigmas': False, 'lower_order_final': True, 'timestep_spacing': 'linspace', 'final_sigmas_type': 'zero', 'rescale_betas_zero_snr': False }, 'DDIM': { 'clip_sample': False, 'set_alpha_to_one': True, 'steps_offset': 0, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'leading', 'rescale_betas_zero_snr': False, 'thresholding': False }, 'Euler': { 'steps_offset': 0, 'interpolation_type': "linear", 'rescale_betas_zero_snr': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'use_beta_sigmas': False, 'use_exponential_sigmas': False, 'use_karras_sigmas': False }, @@ -269,6 +269,8 @@ class DiffusionSampler: self.config['beta_end'] = shared.opts.schedulers_beta_end if 'shift' in self.config: self.config['shift'] = shared.opts.schedulers_shift if shared.opts.schedulers_shift > 0 else 3 + if 'flow_shift' in self.config: + self.config['flow_shift'] = shared.opts.schedulers_shift if shared.opts.schedulers_shift > 0 else 3 if 'use_dynamic_shifting' in self.config: self.config['use_dynamic_shifting'] = True if shared.opts.schedulers_shift == 0 else shared.opts.schedulers_dynamic_shift if 'use_beta_sigmas' in self.config and 'sigma_schedule' in self.config: From cefe46005263fceef8da5432d4d4ccd466db9fa8 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 31 May 2025 01:27:59 +0300 Subject: [PATCH 80/94] SDNQ skip FP8 matmul for input len < 32 --- modules/model_quant_sdnq.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 46413748b..d57504046 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -15,8 +15,8 @@ from modules import devices, shared dtype_dict = { "int8": {"min": -128, "max": 127, "num_bits": 8, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.int8, "is_unsigned": False, "is_integer": True}, "uint8": {"min": 0, "max": 255, "num_bits": 8, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, - "int6": {"min": -32, "max": 31, "num_bits": 6, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, - "uint6": {"min": 0, "max": 63, "num_bits": 6, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "int6": {"min": -32, "max": 31, "num_bits": 6, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, + "uint6": {"min": 0, "max": 63, "num_bits": 6, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "int4": {"min": -8, "max": 7, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, "uint4": {"min": 0, "max": 15, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "uint2": {"min": 0, "max": 3, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, @@ -473,10 +473,14 @@ def int8_matmul( def quantized_linear_forward_fp8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: + if torch.numel(input) / input.shape[-1] < 32: + return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) return fp8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale) def quantized_linear_forward_fp8_matmul_tensorwise(self, input: torch.FloatTensor) -> torch.FloatTensor: + if torch.numel(input) / input.shape[-1] < 32: + return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) return fp8_matmul_tensorwise(input, self.weight, self.bias, self.sdnq_decompressor.scale) From 539fae32345080d97affa5c0ee6503fe1c1964c9 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 1 Jun 2025 21:01:56 +0300 Subject: [PATCH 81/94] Update naming --- README.md | 2 +- modules/shared.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 524d32c0c..831cf2b20 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ All individual features are not listed here, instead check [ChangeLog](CHANGELOG - Platform specific autodetection and tuning performed on install - Optimized processing with latest `torch` developments with built-in support for model compile and quantize Compile backends: *Triton | StableFast | DeepCache | OneDiff | TeaCache | etc.* - Quantization methods: *BitsAndBytes | SDNQ | Optimum-Quanto | TorchAO* + Quantization methods: *SDNQ | BitsAndBytes | Optimum-Quanto | TorchAO* - **Interrogate/Captioning** with 150+ **OpenCLiP** models and 20+ built-in **VLMs** - Built-in queue management - Built in installer with automatic updates and dependency management diff --git a/modules/shared.py b/modules/shared.py index f67a2f0b4..a84708042 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -515,7 +515,7 @@ options_templates.update(options_section(('backends', "Backend Settings"), { })) options_templates.update(options_section(("quantization", "Quantization Settings"), { - "sdnq_quantize_sep": OptionInfo("

SDNQ: SDNext Quantization

", "", gr.HTML), + "sdnq_quantize_sep": OptionInfo("

SDNQ: SD.Next Quantization

", "", gr.HTML), "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ["pre", "post"], "visible": native}), "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ["int8", "int6", "uint4", "float8_e4m3fn", "uint8", "uint6", "int4", "float8_e5m2", "uint2", "uint1"], "visible": native}), @@ -523,7 +523,7 @@ options_templates.update(options_section(("quantization", "Quantization Settings "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), "sdnq_decompress_compile": OptionInfo(devices.has_triton(), "Decompress using torch.compile", gr.Checkbox, {"visible": native}), - "sdnq_use_quantized_matmul": OptionInfo(False, "Use quantized MatMul", gr.Checkbox, {"visible": native}), + "sdnq_use_quantized_matmul": OptionInfo(False, "Use Quantized MatMul", gr.Checkbox, {"visible": native}), "sdnq_quantize_with_gpu": OptionInfo(True, "Quantize with the GPU", gr.Checkbox, {"visible": native}), "sdnq_quantize_shuffle_weights": OptionInfo(False, "Shuffle weights in post mode", gr.Checkbox, {"visible": native}), From c77162fb82910fe58bee51e5e362f571fb76e9ad Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 1 Jun 2025 21:31:43 +0200 Subject: [PATCH 82/94] update wiki and changelog Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 17 ++++++++++------- wiki | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 968a613a9..4c5068b9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,22 @@ # Change Log for SD.Next -## Update for 2025-05-30 +## Update for 2025-06-01 -### Highlights for 2025-05-30 +### Highlights for 2025-06-01 -This release is all about quantization: with new SD.Next own quantization method: **SDNQ** -SDNQ is based on NNCF, but has been re-implemented, optimized and evolved enough to become its own quantization method! -It's fully cross-platform, supports all GPUs and includes 8-bit, 6-bit, 4-bit, 2-bit and 1-bit quantization methods. -Also unlike most traditional methods, its also applicable to nearly all model types. +This release is all about quantization: with new SD.Next own quantization method: **SDNQ** +**SDNQ** is based on **NNCF**, but has been re-implemented, optimized and evolved enough to become its own quantization method! +It's fully cross-platform, supports all GPUs and includes tons of quantization methods: +- *8-bit, 6-bit, 4-bit, 2-bit and 1-bit int and uint* +- *8-bit e5 and e4 float* + +Also unlike most traditional methods, its also applicable to nearly all model types *Hint*: Even if you may not need quantization for your current model, it may be worth trying it out as it can significantly improve performance or capabilities of your existing workflow! For example, you may not have issues with SD15 or SDXL, but you may have been limited running at high resolutions or with multiple ControlNet due to VRAM requirements - this will significantly reduce memory requirements. [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) -### Details for 2025-05-30 +### Details for 2025-06-01 - **SDNQ Quantization** - Renamed `NNCF` to `SDNQ` diff --git a/wiki b/wiki index 4e2b6a082..68b0dc008 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 4e2b6a082bae84ce5a4adccecf54521d134dc069 +Subproject commit 68b0dc008e38f95e39d47f1d15f1118b8a7986be From 839295f79a1df4794fa01f4291678662aaf4111f Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 1 Jun 2025 23:10:08 +0300 Subject: [PATCH 83/94] Add fp8 fnuz to sdnq options --- modules/model_quant_sdnq.py | 7 ++----- modules/shared.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index d57504046..c220901c9 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -26,8 +26,6 @@ dtype_dict = { "float8_e4m3fnuz": {"min": -240, "max": 240, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e4m3fnuz, "storage_dtype": torch.float8_e4m3fnuz, "is_unsigned": False, "is_integer": False}, "float8_e5m2fnuz": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False}, } -if hasattr(torch, "float8_e8m0fnu"): - dtype_dict["float8_e8m0fnu"] = {"min": 5.87747e-39, "max": 1.70141e+38, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e8m0fnu, "storage_dtype": torch.float8_e8m0fnu, "is_unsigned": True, "is_integer": False} quantized_matmul_dtypes = ("int8", "int6", "int4", "float8_e4m3fn") @@ -628,7 +626,6 @@ decompressor_dict = { "float8_e4m3fnuz": SymmetricWeightsDecompressor, "float8_e5m2": SymmetricWeightsDecompressor, "float8_e5m2fnuz": SymmetricWeightsDecompressor, - "float8_e8m0fnu": AsymmetricWeightsDecompressor, } @@ -795,7 +792,7 @@ class SDNQConfig(QuantizationConfigMixin): Args: weights_dtype (`str`, *optional*, defaults to `"int8"`): The target dtype for the weights after quantization. Supported values are: - ("int8", "uint8", "int6", "uint6", "int4", "uint4", "uint2", "uint1", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz", "float8_e8m0fnu") + ("int8", "uint8", "int6", "uint6", "int4", "uint4", "uint2", "uint1", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz") modules_to_not_convert (`list`, *optional*, default to `None`): The list of modules to not quantize, useful for quantizing models that explicitly require to have some modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers). @@ -823,7 +820,7 @@ class SDNQConfig(QuantizationConfigMixin): r""" Safety checker that arguments are correct """ - accepted_weights = ["int8", "uint8", "int6", "uint6", "int4", "uint4", "uint2", "uint1", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz", "float8_e8m0fnu"] + accepted_weights = ["int8", "uint8", "int6", "uint6", "int4", "uint4", "uint2", "uint1", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz"] if self.weights_dtype not in accepted_weights: raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights_dtype}") diff --git a/modules/shared.py b/modules/shared.py index a84708042..d9288d511 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -518,7 +518,7 @@ options_templates.update(options_section(("quantization", "Quantization Settings "sdnq_quantize_sep": OptionInfo("

SDNQ: SD.Next Quantization

", "", gr.HTML), "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ["pre", "post"], "visible": native}), - "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ["int8", "int6", "uint4", "float8_e4m3fn", "uint8", "uint6", "int4", "float8_e5m2", "uint2", "uint1"], "visible": native}), + "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ["int8", "int6", "uint4", "float8_e4m3fn", "uint8", "uint6", "int4", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "uint2", "uint1"], "visible": native}), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), From acefa588349d2a1198734b2103442fdc7acffb06 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 1 Jun 2025 23:16:00 +0300 Subject: [PATCH 84/94] SDNQ don't force fp32 with fp8 tensorwise matmul --- modules/model_quant_sdnq.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index c220901c9..6adfe7b73 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -45,6 +45,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz is_conv_type = False is_conv_transpose_type = False is_linear_type = False + use_tensorwise_fp8_matmul = False result_shape = None if torch_dtype is None: torch_dtype = devices.dtype @@ -69,6 +70,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz use_quantized_matmul = weights_dtype in quantized_matmul_dtypes and channel_size >= 32 and output_channel_size >= 32 if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: use_quantized_matmul = output_channel_size % 16 == 0 and channel_size % 16 == 0 + use_tensorwise_fp8_matmul = devices.backend == "cpu" or (devices.backend == "cuda" and sys.platform == "win32" and float(torch.__version__[:3]) <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) if not use_quantized_matmul and (group_size > 0 or (dtype_dict[weights_dtype]["num_bits"] < 6 and group_size != -1)): if group_size == 0: @@ -136,7 +138,8 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz stride = layer.weight.stride() if stride[0] > stride[1] and stride[1] == 1: layer.weight.data = layer.weight.t().contiguous().t() - scale = scale.to(torch.float32) + if not use_tensorwise_fp8_matmul: + scale = scale.to(torch.float32) layer.sdnq_decompressor = decompressor_dict[weights_dtype]( scale=scale, @@ -155,7 +158,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if dtype_dict[weights_dtype]["is_integer"]: layer.forward = quantized_linear_forward_int8_matmul else: - if devices.backend == "cpu" or (devices.backend == "cuda" and sys.platform == "win32" and float(torch.__version__[:3]) <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)): + if use_tensorwise_fp8_matmul: layer.forward = quantized_linear_forward_fp8_matmul_tensorwise else: layer.forward = quantized_linear_forward_fp8_matmul From 9669b3601034709608510e627c40160b69210550 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 1 Jun 2025 23:29:16 +0300 Subject: [PATCH 85/94] SDNQ fix older PyTorch with FP8 matmul --- modules/model_quant_sdnq.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 6adfe7b73..5b5bc1c60 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -432,7 +432,7 @@ def fp8_matmul( output_shape = list(input.shape) output_shape[-1] = weight.shape[-1] input, input_scale = quantize_fp8_matmul_input(input) - return torch._scaled_mm(input, weight, input_scale, scale, bias=bias, out_dtype=return_dtype).reshape(output_shape) + return torch._scaled_mm(input, weight, scale_a=input_scale, scale_b=scale, bias=bias, out_dtype=return_dtype).reshape(output_shape) # sm89 doesn't support row wise scale in Windows @@ -447,7 +447,7 @@ def fp8_matmul_tensorwise( output_shape[-1] = weight.shape[-1] dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) input, scale = quantize_fp8_matmul_input_tensorwise(input, scale) - result = decompress_symmetric(torch._scaled_mm(input, weight, dummy_input_scale, dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, output_shape) + result = decompress_symmetric(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, output_shape) if bias is not None: result.add_(bias) return result From 766aec32d5c6fcf59f81f08a4cefb5c605cc5a1a Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 1 Jun 2025 23:35:00 +0300 Subject: [PATCH 86/94] Update changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c5068b9d..91546354c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ This release is all about quantization: with new SD.Next own quantization method **SDNQ** is based on **NNCF**, but has been re-implemented, optimized and evolved enough to become its own quantization method! It's fully cross-platform, supports all GPUs and includes tons of quantization methods: - *8-bit, 6-bit, 4-bit, 2-bit and 1-bit int and uint* -- *8-bit e5 and e4 float* +- *8-bit e5, e4 and fnuz float* Also unlike most traditional methods, its also applicable to nearly all model types @@ -25,7 +25,7 @@ Also unlike most traditional methods, its also applicable to nearly all model ty - `INT8` -> `uint8` - `INT4_SYM` -> `int4` - `INT4` -> `uint4` - - Add `float8_e4m3fn`, `float8_e5m2`, `int6`, `uint6`, `uint2` and `uint1` support + - Add `float8_e4m3fn`, `float8_e5m2`, `float8_e4m3fnuz`, `float8_e5m2fnuz`, `int6`, `uint6`, `uint2` and `uint1` support - Add quantized matmul support for `float8_e4m3fn` - Set the default quant mode to `pre` - Use per token input quant with int8 and fp8 quantized matmul From b146025a5ed963241a19656393a5f7fb3a0fe9b5 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 2 Jun 2025 00:17:39 +0300 Subject: [PATCH 87/94] SDNQ add int2 --- CHANGELOG.md | 2 +- modules/model_quant_sdnq.py | 52 +++++++++++++++---------------------- modules/shared.py | 2 +- 3 files changed, 23 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91546354c..46cf6b8dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ Also unlike most traditional methods, its also applicable to nearly all model ty - `INT8` -> `uint8` - `INT4_SYM` -> `int4` - `INT4` -> `uint4` - - Add `float8_e4m3fn`, `float8_e5m2`, `float8_e4m3fnuz`, `float8_e5m2fnuz`, `int6`, `uint6`, `uint2` and `uint1` support + - Add `float8_e4m3fn`, `float8_e5m2`, `float8_e4m3fnuz`, `float8_e5m2fnuz`, `int6`, `uint6`, `int2`, `uint2` and `uint1` support - Add quantized matmul support for `float8_e4m3fn` - Set the default quant mode to `pre` - Use per token input quant with int8 and fp8 quantized matmul diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 5b5bc1c60..bded102a3 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -19,6 +19,7 @@ dtype_dict = { "uint6": {"min": 0, "max": 63, "num_bits": 6, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "int4": {"min": -8, "max": 7, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, "uint4": {"min": 0, "max": 15, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "int2": {"min": -2, "max": 1, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, "uint2": {"min": 0, "max": 3, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "uint1": {"min": 0, "max": 1, "num_bits": 1, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "float8_e4m3fn": {"min": -448, "max": 448, "num_bits": 8, "target_dtype": torch.float8_e4m3fn, "torch_dtype": torch.float8_e4m3fn, "storage_dtype": torch.float8_e4m3fn, "is_unsigned": False, "is_integer": False}, @@ -258,9 +259,22 @@ def decompress_packed_int_asymmetric(input: torch.Tensor, scale: torch.Tensor, z def decompress_packed_int_symmetric(input: torch.Tensor, scale: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str, skip_quantized_matmul: bool = False) -> torch.Tensor: if skip_quantized_matmul: - return decompress_symmetric(packed_int_function_dict[weights_dtype]["unpack"](input, shape, dtype=scale.dtype), scale.transpose(0,1), dtype, result_shape) + return decompress_symmetric(unpack_int_symetric(input, shape, weights_dtype, dtype=scale.dtype), scale.transpose(0,1), dtype, result_shape) else: - return decompress_symmetric(packed_int_function_dict[weights_dtype]["unpack"](input, shape, dtype=scale.dtype), scale, dtype, result_shape) + return decompress_symmetric(unpack_int_symetric(input, shape, weights_dtype, dtype=scale.dtype), scale, dtype, result_shape) + + +def pack_int_symetric(tensor: torch.Tensor, weights_dtype: str) -> torch.Tensor: + return packed_int_function_dict[weights_dtype]["pack"](tensor.to(dtype=dtype_dict[weights_dtype]["torch_dtype"]).sub_(dtype_dict[weights_dtype]["min"]).to(dtype=dtype_dict[weights_dtype]["storage_dtype"])) + + +def unpack_int_symetric(packed_tensor: torch.Tensor, shape: torch.Size, weights_dtype: str, dtype: Optional[torch.dtype] = None, transpose: Optional[bool] = False) -> torch.Tensor: + if dtype is None: + dtype = dtype_dict[weights_dtype]["torch_dtype"] + result = packed_int_function_dict[weights_dtype]["unpack"](packed_tensor, shape).to(dtype=dtype).add_(dtype_dict[weights_dtype]["min"]) + if transpose: + result = result.transpose(0,1) + return result def pack_uint6(tensor: torch.Tensor) -> torch.Tensor: @@ -278,12 +292,6 @@ def pack_uint6(tensor: torch.Tensor) -> torch.Tensor: return packed_tensor -def pack_int6(tensor: torch.Tensor) -> torch.Tensor: - if tensor.dtype != torch.int8: - raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.int8 type is supported.") - return pack_uint6((tensor + 32).to(dtype=torch.uint8)) - - def pack_uint4(tensor: torch.Tensor) -> torch.Tensor: if tensor.dtype != torch.uint8: raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") @@ -292,12 +300,6 @@ def pack_uint4(tensor: torch.Tensor) -> torch.Tensor: return packed_tensor -def pack_int4(tensor: torch.Tensor) -> torch.Tensor: - if tensor.dtype != torch.int8: - raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.int8 type is supported.") - return pack_uint4((tensor + 8).to(dtype=torch.uint8)) - - def pack_uint2(tensor: torch.Tensor) -> torch.Tensor: if tensor.dtype != torch.uint8: raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") @@ -345,25 +347,11 @@ def unpack_uint6(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor return result -def unpack_int6(packed_tensor: torch.Tensor, shape: torch.Size, dtype: Optional[torch.dtype] = torch.int8, transpose: Optional[bool] = False) -> torch.Tensor: - result = unpack_uint6(packed_tensor, shape).to(dtype=dtype).sub_(32) - if transpose: - result = result.transpose(0,1) - return result - - def unpack_uint4(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: result = torch.stack((torch.bitwise_and(packed_tensor, 15), torch.bitwise_right_shift(packed_tensor, 4)), dim=-1).reshape(shape) return result -def unpack_int4(packed_tensor: torch.Tensor, shape: torch.Size, dtype: Optional[torch.dtype] = torch.int8, transpose: Optional[bool] = False) -> torch.Tensor: - result = unpack_uint4(packed_tensor, shape).to(dtype=dtype).sub_(8) - if transpose: - result = result.transpose(0,1) - return result - - def unpack_uint2(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: result = torch.stack( ( @@ -610,7 +598,7 @@ class PackedINTSymmetricWeightsDecompressor(torch.nn.Module): self.register_buffer("scale", scale) def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: - return packed_int_function_dict[self.weights_dtype]["pack"](weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"])) + return pack_int_symetric(weight, self.weights_dtype) def forward(self, weight, skip_quantized_matmul=False, **kwargs): return decompress_packed_int_symmetric_compiled(weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype, skip_quantized_matmul=skip_quantized_matmul) @@ -623,6 +611,7 @@ decompressor_dict = { "uint6": PackedINTAsymmetricWeightsDecompressor, "int4": PackedINTSymmetricWeightsDecompressor, "uint4": PackedINTAsymmetricWeightsDecompressor, + "int2": PackedINTSymmetricWeightsDecompressor, "uint2": PackedINTAsymmetricWeightsDecompressor, "uint1": PackedINTAsymmetricWeightsDecompressor, "float8_e4m3fn": SymmetricWeightsDecompressor, @@ -633,10 +622,11 @@ decompressor_dict = { packed_int_function_dict = { - "int6": {"pack": pack_int6, "unpack": unpack_int6}, + "int6": {"pack": pack_uint6, "unpack": unpack_uint6}, "uint6": {"pack": pack_uint6, "unpack": unpack_uint6}, - "int4": {"pack": pack_int4, "unpack": unpack_int4}, + "int4": {"pack": pack_uint4, "unpack": unpack_uint4}, "uint4": {"pack": pack_uint4, "unpack": unpack_uint4}, + "int2": {"pack": pack_uint2, "unpack": unpack_uint2}, "uint2": {"pack": pack_uint2, "unpack": unpack_uint2}, "uint1": {"pack": pack_uint1, "unpack": unpack_uint1}, } diff --git a/modules/shared.py b/modules/shared.py index d9288d511..a1c089c48 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -518,7 +518,7 @@ options_templates.update(options_section(("quantization", "Quantization Settings "sdnq_quantize_sep": OptionInfo("

SDNQ: SD.Next Quantization

", "", gr.HTML), "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ["pre", "post"], "visible": native}), - "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ["int8", "int6", "uint4", "float8_e4m3fn", "uint8", "uint6", "int4", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "uint2", "uint1"], "visible": native}), + "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ["int8", "int6", "uint4", "float8_e4m3fn", "uint8", "uint6", "int4", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "int2", "uint2", "uint1"], "visible": native}), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), From 8f1a1d7311e4a0252cb2dd18d90c5c149a4000f9 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 2 Jun 2025 00:28:29 +0300 Subject: [PATCH 88/94] SDNQ expand quantized_matmul_dtypes for CPU --- modules/model_quant_sdnq.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index bded102a3..8dc4e0ed8 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -28,7 +28,9 @@ dtype_dict = { "float8_e5m2fnuz": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False}, } -quantized_matmul_dtypes = ("int8", "int6", "int4", "float8_e4m3fn") +quantized_matmul_dtypes = ("int8", "int6", "int4", "int2", "float8_e4m3fn") +if devices.backend in {"cpu", "openvino"}: + quantized_matmul_dtypes += ("float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz") linear_types = ("Linear",) conv_types = ("Conv1d", "Conv2d", "Conv3d") @@ -71,7 +73,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz use_quantized_matmul = weights_dtype in quantized_matmul_dtypes and channel_size >= 32 and output_channel_size >= 32 if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: use_quantized_matmul = output_channel_size % 16 == 0 and channel_size % 16 == 0 - use_tensorwise_fp8_matmul = devices.backend == "cpu" or (devices.backend == "cuda" and sys.platform == "win32" and float(torch.__version__[:3]) <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) + use_tensorwise_fp8_matmul = devices.backend in {"cpu", "openvino"} or (devices.backend == "cuda" and sys.platform == "win32" and float(torch.__version__[:3]) <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) if not use_quantized_matmul and (group_size > 0 or (dtype_dict[weights_dtype]["num_bits"] < 6 and group_size != -1)): if group_size == 0: From e8588c91eabe4cc7d01e582033048ed5c27bc3d1 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 2 Jun 2025 00:53:10 +0300 Subject: [PATCH 89/94] SDNQ enable matmul support for float8_e5m2 --- CHANGELOG.md | 2 +- modules/model_quant_sdnq.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46cf6b8dc..dfae0d239 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ Also unlike most traditional methods, its also applicable to nearly all model ty - `INT4_SYM` -> `int4` - `INT4` -> `uint4` - Add `float8_e4m3fn`, `float8_e5m2`, `float8_e4m3fnuz`, `float8_e5m2fnuz`, `int6`, `uint6`, `int2`, `uint2` and `uint1` support - - Add quantized matmul support for `float8_e4m3fn` + - Add quantized matmul support for `float8_e4m3fn` and `float8_e5m2` - Set the default quant mode to `pre` - Use per token input quant with int8 and fp8 quantized matmul - Implement better layer hijacks diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 8dc4e0ed8..4b5288958 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -28,9 +28,9 @@ dtype_dict = { "float8_e5m2fnuz": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False}, } -quantized_matmul_dtypes = ("int8", "int6", "int4", "int2", "float8_e4m3fn") +quantized_matmul_dtypes = ("int8", "int6", "int4", "int2", "float8_e4m3fn", "float8_e5m2") if devices.backend in {"cpu", "openvino"}: - quantized_matmul_dtypes += ("float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz") + quantized_matmul_dtypes += ("float8_e4m3fnuz", "float8_e5m2fnuz") linear_types = ("Linear",) conv_types = ("Conv1d", "Conv2d", "Conv3d") From ea7fe3bb73294de294b15b82e83c8f012b6f9a3d Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 2 Jun 2025 01:11:41 +0300 Subject: [PATCH 90/94] OpenVINO update dtype_mapping --- modules/intel/openvino/__init__.py | 40 ++++++++++++++---------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py index e822c9274..bb5ee9c64 100644 --- a/modules/intel/openvino/__init__.py +++ b/modules/intel/openvino/__init__.py @@ -52,6 +52,24 @@ DEFAULT_OPENVINO_PYTHON_CONFIG = MappingProxyType( }, ) +dtype_mapping = { + torch.float32: Type.f32, + torch.float64: Type.f64, + torch.float16: Type.f16, + torch.bfloat16: Type.bf16, + torch.float8_e4m3fn: Type.f8e4m3, + torch.float8_e5m2: Type.f8e5m2, + torch.int64: Type.i64, + torch.uint64: Type.u64, + torch.int32: Type.i32, + torch.uint32: Type.u32, + torch.int8: Type.i8, + torch.uint8: Type.u8, + torch.bool: Type.boolean + } +if hasattr(torch, "float8_e8m0fnu"): + dtype_mapping[torch.float8_e8m0fnu] = Type.f8e8m0 + class OpenVINOGraphModule(torch.nn.Module): def __init__(self, gm, partition_id, use_python_fusion_cache, model_hash_str: str = None, file_name="", int_inputs=[]): @@ -219,17 +237,6 @@ def openvino_compile(gm: GraphModule, *example_inputs, model_hash_str: str = Non f.write("\n") f.close() - dtype_mapping = { - torch.float32: Type.f32, - torch.float64: Type.f64, - torch.float16: Type.f16, - torch.int64: Type.i64, - torch.int32: Type.i32, - torch.uint8: Type.u8, - torch.int8: Type.i8, - torch.bool: Type.boolean - } - idx_minus = 0 for idx, input_data in enumerate(example_inputs): if isinstance(input_data, int): @@ -283,17 +290,6 @@ def openvino_compile_cached_model(cached_model_path, *example_inputs): global dont_use_nncf global dont_use_quant - dtype_mapping = { - torch.float32: Type.f32, - torch.float64: Type.f64, - torch.float16: Type.f16, - torch.int64: Type.i64, - torch.int32: Type.i32, - torch.uint8: Type.u8, - torch.int8: Type.i8, - torch.bool: Type.boolean - } - for idx, input_data in enumerate(example_inputs): om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype]) om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape))) From 82f5634d533e111fe3c04a6f22fd51adfb40b92e Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 2 Jun 2025 01:39:51 +0300 Subject: [PATCH 91/94] SDNQ use torch.bool for uint1 --- modules/model_quant_sdnq.py | 39 ++----------------------------------- 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 4b5288958..a4416860f 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -21,7 +21,7 @@ dtype_dict = { "uint4": {"min": 0, "max": 15, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "int2": {"min": -2, "max": 1, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, "uint2": {"min": 0, "max": 3, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, - "uint1": {"min": 0, "max": 1, "num_bits": 1, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint1": {"min": 0, "max": 1, "num_bits": 1, "target_dtype": torch.bool, "torch_dtype": torch.bool, "storage_dtype": torch.bool, "is_unsigned": True, "is_integer": True}, "float8_e4m3fn": {"min": -448, "max": 448, "num_bits": 8, "target_dtype": torch.float8_e4m3fn, "torch_dtype": torch.float8_e4m3fn, "storage_dtype": torch.float8_e4m3fn, "is_unsigned": False, "is_integer": False}, "float8_e5m2": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": torch.float8_e5m2, "torch_dtype": torch.float8_e5m2, "storage_dtype": torch.float8_e5m2, "is_unsigned": False, "is_integer": False}, "float8_e4m3fnuz": {"min": -240, "max": 240, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e4m3fnuz, "storage_dtype": torch.float8_e4m3fnuz, "is_unsigned": False, "is_integer": False}, @@ -313,23 +313,6 @@ def pack_uint2(tensor: torch.Tensor) -> torch.Tensor: return packed_tensor -def pack_uint1(tensor: torch.Tensor) -> torch.Tensor: - if tensor.dtype != torch.uint8: - raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") - packed_tensor = tensor.contiguous().reshape(-1, 8) - packed_tensor = torch.bitwise_or( - torch.bitwise_or( - torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_left_shift(packed_tensor[:, 1], 1)), - torch.bitwise_or(torch.bitwise_left_shift(packed_tensor[:, 2], 2), torch.bitwise_left_shift(packed_tensor[:, 3], 3)) - ), - torch.bitwise_or( - torch.bitwise_or(torch.bitwise_left_shift(packed_tensor[:, 4], 4), torch.bitwise_left_shift(packed_tensor[:, 5], 5)), - torch.bitwise_or(torch.bitwise_left_shift(packed_tensor[:, 6], 6), torch.bitwise_left_shift(packed_tensor[:, 7], 7)) - ), - ) - return packed_tensor - - def unpack_uint6(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: result = torch.stack( ( @@ -367,23 +350,6 @@ def unpack_uint2(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor return result -def unpack_uint1(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: - result = torch.stack( - ( - torch.bitwise_and(packed_tensor, 1), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 1), 1), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 2), 1), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 3), 1), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 4), 1), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 5), 1), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 6), 1), - torch.bitwise_right_shift(packed_tensor, 7), - ), - dim=-1 - ).reshape(shape) - return result - - def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.FloatTensor, torch.FloatTensor]: input = input.flatten(0,-2).contiguous() input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 448) @@ -615,7 +581,7 @@ decompressor_dict = { "uint4": PackedINTAsymmetricWeightsDecompressor, "int2": PackedINTSymmetricWeightsDecompressor, "uint2": PackedINTAsymmetricWeightsDecompressor, - "uint1": PackedINTAsymmetricWeightsDecompressor, + "uint1": AsymmetricWeightsDecompressor, "float8_e4m3fn": SymmetricWeightsDecompressor, "float8_e4m3fnuz": SymmetricWeightsDecompressor, "float8_e5m2": SymmetricWeightsDecompressor, @@ -630,7 +596,6 @@ packed_int_function_dict = { "uint4": {"pack": pack_uint4, "unpack": unpack_uint4}, "int2": {"pack": pack_uint2, "unpack": unpack_uint2}, "uint2": {"pack": pack_uint2, "unpack": unpack_uint2}, - "uint1": {"pack": pack_uint1, "unpack": unpack_uint1}, } From 4e3795a0a59225bac7ff13b98a3a1a9dd6094d0a Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 2 Jun 2025 03:31:51 +0300 Subject: [PATCH 92/94] SDNQ fix packed int8 matmul --- modules/model_quant_sdnq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index a4416860f..8231a3828 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -418,7 +418,7 @@ def int8_matmul( weights_dtype: str, ) -> torch.FloatTensor: if compressed_weight_shape is not None: - weight = packed_int_function_dict[weights_dtype]["unpack"](weight, compressed_weight_shape, transpose=True) + weight = unpack_int_symetric(weight, compressed_weight_shape, weights_dtype, dtype=torch.int8, transpose=True) return_dtype = input.dtype output_shape = list(input.shape) output_shape[-1] = weight.shape[-1] From b1d6897621c38e19a53535118999f5e4e306bce8 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 2 Jun 2025 15:44:52 +0200 Subject: [PATCH 93/94] update changelog Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 12 ++++++++---- wiki | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfae0d239..6ec735e0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Change Log for SD.Next -## Update for 2025-06-01 +## Update for 2025-06-02 -### Highlights for 2025-06-01 +### Highlights for 2025-06-02 This release is all about quantization: with new SD.Next own quantization method: **SDNQ** **SDNQ** is based on **NNCF**, but has been re-implemented, optimized and evolved enough to become its own quantization method! @@ -12,11 +12,15 @@ It's fully cross-platform, supports all GPUs and includes tons of quantization m Also unlike most traditional methods, its also applicable to nearly all model types -*Hint*: Even if you may not need quantization for your current model, it may be worth trying it out as it can significantly improve performance or capabilities of your existing workflow! For example, you may not have issues with SD15 or SDXL, but you may have been limited running at high resolutions or with multiple ControlNet due to VRAM requirements - this will significantly reduce memory requirements. +*Hint*: Even if you may not need quantization for your current model, it may be worth trying it out as it can significantly improve performance or capabilities of your existing workflow! For example, you may not have issues with SD15 or SDXL, but you may have been limited running at high resolutions or with multiple ControlNet due to VRAM requirements - this will significantly reduce memory requirements. And on-the-fly quantization takes just few seconds during model load, there is no need to have multiple quant models permanently saved. + +On a different topic, **SD.Next Wiki & Docs** and its **UI Hints** and **UI Localization** system are community efforts and any contributions are welcome! +You dont need any coding experience, but if you learned something and you find documentation either wrong or insufficient, please do suggest edits! +Take a look at [Docs](https://github.com/vladmandic/sdnext/wiki/Docs), [Hints](https://github.com/vladmandic/sdnext/wiki/Hints) and [Localization](https://github.com/vladmandic/sdnext/wiki/Locale) contribution guides [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) -### Details for 2025-06-01 +### Details for 2025-06-02 - **SDNQ Quantization** - Renamed `NNCF` to `SDNQ` diff --git a/wiki b/wiki index 68b0dc008..693b0dafa 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 68b0dc008e38f95e39d47f1d15f1118b8a7986be +Subproject commit 693b0dafa0dea17fe8057201f397e80a9b46b0c0 From 7d05bed459972190cfdbdc2d3e510d9e4c51cbfb Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 2 Jun 2025 15:50:42 +0200 Subject: [PATCH 94/94] update changelog Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ec735e0b..5816d5f9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ Take a look at [Docs](https://github.com/vladmandic/sdnext/wiki/Docs), [Hints](h - Enable Dynamic attention for Scaled-Dot-Product with ROCm, DirectML, MPS and CPU backends - **Fixes** - Gallery duplicate entries + - Prompt enhancement args mismatch ## Update for 2025-05-17