From 7a4848dada887cf059e91a50dc583baed32e0ff1 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 15 Apr 2025 02:16:18 +0300 Subject: [PATCH] Fix NNCF not applying for TE only quant --- modules/sd_models.py | 2 +- modules/sd_models_utils.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index e06e56272..25e0c7005 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -600,7 +600,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No prompt_parser_diffusers.cache.clear() set_diffuser_options(sd_model, vae, op, offload=False) - if 'Model' in shared.opts.nncf_compress_weights and not ('Model' in shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx"): + if shared.opts.nncf_compress_weights and not (shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx"): sd_model = model_quant.nncf_compress_weights(sd_model) # run this before move model so it can be compressed in CPU if shared.opts.optimum_quanto_weights: sd_model = model_quant.optimum_quanto_weights(sd_model) # run this before move model so it can be compressed in CPU diff --git a/modules/sd_models_utils.py b/modules/sd_models_utils.py index ae50e5748..273d1b39d 100644 --- a/modules/sd_models_utils.py +++ b/modules/sd_models_utils.py @@ -197,6 +197,15 @@ def apply_function_to_model(sd_model, function, options, op=None): dtype=torch.float32 if devices.dtype != torch.bfloat16 else torch.bfloat16 ) sd_model.text_encoder_3 = function(sd_model.text_encoder_3, op="text_encoder_3", sd_model=sd_model) + if hasattr(sd_model, 'text_encoder_4') and hasattr(sd_model.text_encoder_4, 'config'): + if op == "nncf" and sd_model.text_encoder_4.__class__.__name__ in {"T5EncoderModel", "UMT5EncoderModel"}: + from modules.sd_hijack import NNCF_T5DenseGatedActDense # T5DenseGatedActDense uses fp32 + for i in range(len(sd_model.text_encoder_4.encoder.block)): + sd_model.text_encoder_4.encoder.block[i].layer[1].DenseReluDense = NNCF_T5DenseGatedActDense( + sd_model.text_encoder_4.encoder.block[i].layer[1].DenseReluDense, + dtype=torch.float32 if devices.dtype != torch.bfloat16 else torch.bfloat16 + ) + sd_model.text_encoder_4 = function(sd_model.text_encoder_4, op="text_encoder_4", sd_model=sd_model) if hasattr(sd_model, 'prior_pipe') and hasattr(sd_model.prior_pipe, 'text_encoder') and hasattr(sd_model.prior_pipe.text_encoder, 'config'): sd_model.prior_pipe.text_encoder = function(sd_model.prior_pipe.text_encoder, op="prior_pipe.text_encoder", sd_model=sd_model) if "VAE" in options: