diff --git a/TODO.md b/TODO.md index 595f8938d..73b11c743 100644 --- a/TODO.md +++ b/TODO.md @@ -19,7 +19,7 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - Video: add generate context menu - Video: API support - Video: STG: -- Video SmoothCache: https://github.com/huggingface/diffusers/issues/11135 +- Video: SmoothCache: https://github.com/huggingface/diffusers/issues/11135 - SoftFill: https://github.com/zacheryvaughn/softfill-pipelines ## Code TODO diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py index 2d2dc0b79..a5584575f 100644 --- a/modules/intel/openvino/__init__.py +++ b/modules/intel/openvino/__init__.py @@ -479,8 +479,8 @@ def openvino_fx(subgraph, example_inputs, options=None): subgraph_type[3] is torch.nn.modules.linear.Linear): dont_use_faketensors = True - dont_use_nncf = bool("Text Encoder" not in shared.opts.nncf_compress_weights) - dont_use_quant = bool("Text Encoder" not in shared.opts.nncf_quantize) + dont_use_nncf = bool("TE" not in shared.opts.nncf_compress_weights) + dont_use_quant = bool("TE" not in shared.opts.nncf_quantize) # Create a hash to be used for caching shared.compiled_model_state.model_hash_str = "" diff --git a/modules/model_cogview.py b/modules/model_cogview.py index a0594cc16..a37dd88ca 100644 --- a/modules/model_cogview.py +++ b/modules/model_cogview.py @@ -38,7 +38,7 @@ def load_cogview3(checkpoint_info, diffusers_load_config={}): **quant_args, ) - diffusers_load_config, quant_args = load_common(diffusers_load_config, module='Text Encoder') + diffusers_load_config, quant_args = load_common(diffusers_load_config, module='TE') text_encoder = transformers.T5EncoderModel.from_pretrained( repo_id, subfolder="text_encoder", @@ -71,7 +71,7 @@ def load_cogview4(checkpoint_info, diffusers_load_config={}): **quant_args, ) - diffusers_load_config, quant_args = load_common(diffusers_load_config, module='Text Encoder') + diffusers_load_config, quant_args = load_common(diffusers_load_config, module='TE') text_encoder = transformers.AutoModelForCausalLM.from_pretrained( repo_id, subfolder="text_encoder", diff --git a/modules/model_flux.py b/modules/model_flux.py index 97b39aedc..74ba5c8dd 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -112,12 +112,12 @@ def load_quants(kwargs, repo_id, cache_dir, allow_quant): quant_args = model_quant.create_config(allow=allow_quant) if not quant_args: return kwargs - if 'transformer' not in kwargs and ('Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization or 'Model' in shared.opts.quanto_quantization): + if 'transformer' not in kwargs and (('Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization or 'Model' in shared.opts.quanto_quantization) or ('Transformer' in shared.opts.bnb_quantization or 'Transformer' in shared.opts.torchao_quantization or 'Transformer' in shared.opts.quanto_quantization)): kwargs['transformer'] = diffusers.FluxTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) - quant_args = model_quant.create_config(allow=allow_quant, module='Text Encoder') + quant_args = model_quant.create_config(allow=allow_quant, module='TE') if not quant_args: return kwargs - if 'text_encoder_2' not in kwargs and ('Text Encoder' in shared.opts.bnb_quantization or 'Text Encoder' in shared.opts.torchao_quantization or 'Text Encoder' in shared.opts.quanto_quantization): + if 'text_encoder_2' not in kwargs and ('TE' in shared.opts.bnb_quantization or 'TE' in shared.opts.torchao_quantization or 'TE' in shared.opts.quanto_quantization): kwargs['text_encoder_2'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder_2", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) except Exception as e: shared.log.error(f'Quantization: {e}') @@ -192,7 +192,7 @@ def load_transformer(file_path): # triggered by opts.sd_unet change transformer, _text_encoder_2 = load_flux_nf4(file_path, prequantized=False) if transformer is not None: return transformer - quant_args = model_quant.create_config() + quant_args = model_quant.create_config(module='Transformer') if quant_args: shared.log.info(f'Load module: type=UNet/Transformer file="{file_path}" offload={shared.opts.diffusers_offload_mode} quant=torchao dtype={devices.dtype}') transformer = diffusers.FluxTransformer2DModel.from_single_file(file_path, **diffusers_load_config, **quant_args) diff --git a/modules/model_lumina.py b/modules/model_lumina.py index 9e9ca8eff..e6bef6773 100644 --- a/modules/model_lumina.py +++ b/modules/model_lumina.py @@ -35,9 +35,9 @@ def load_lumina2(checkpoint_info, diffusers_load_config={}): quant_args = model_quant.create_config() kwargs = {} repo_id = sd_models.path_to_repo(checkpoint_info.name) - if ('Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization): + if (('Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization or 'Model' in shared.opts.quanto_quantization) or ('Transformer' in shared.opts.bnb_quantization or 'Transformer' in shared.opts.torchao_quantization or 'Transformer' in shared.opts.quanto_quantization)): kwargs['transformer'] = diffusers.Lumina2Transformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=shared.opts.diffusers_dir, torch_dtype=devices.dtype, **quant_args) - if ('Text Encoder' in shared.opts.bnb_quantization or 'Text Encoder' in shared.opts.torchao_quantization): + if ('TE' in shared.opts.bnb_quantization or 'TE' in shared.opts.torchao_quantization): kwargs['text_encoder'] = transformers.AutoModel.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=shared.opts.diffusers_dir, torch_dtype=devices.dtype, **quant_args) sd_model = diffusers.Lumina2Text2ImgPipeline.from_pretrained(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **diffusers_load_config, **quant_args, **kwargs) return sd_model diff --git a/modules/model_quant.py b/modules/model_quant.py index e0fad40f9..4078dd01a 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -238,7 +238,7 @@ def apply_layerwise(sd_model, quiet:bool=False): m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=non_blocking) m.quantization_method = 'LayerWise' log.quiet(quiet, f'Quantization: type=layerwise module={module} cls={cls} storage={storage_dtype} compute={devices.dtype} blocking={not non_blocking}') - if module.startswith('text_encoder') and ('Model' in shared.opts.layerwise_quantization or 'Text Encoder' in shared.opts.layerwise_quantization) and ('clip' not in cls.lower()): + if module.startswith('text_encoder') and ('Model' in shared.opts.layerwise_quantization or 'TE' in shared.opts.layerwise_quantization) and ('clip' not in cls.lower()): m = getattr(sd_model, module) if hasattr(m, 'enable_layerwise_casting'): m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=non_blocking) diff --git a/modules/model_sana.py b/modules/model_sana.py index 509db4c88..8fb07a6da 100644 --- a/modules/model_sana.py +++ b/modules/model_sana.py @@ -1,4 +1,3 @@ -import os import time import torch import diffusers @@ -15,7 +14,7 @@ def load_quants(kwargs, repo_id, cache_dir): if 'transformer' not in kwargs and ('Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization): kwargs['transformer'] = diffusers.models.SanaTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, **load_args, **quant_args) shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') - if 'text_encoder' not in kwargs and ('Text Encoder' in shared.opts.bnb_quantization or 'Text Encoder' in shared.opts.torchao_quantization): + if 'text_encoder' not in kwargs and ('TE' in shared.opts.bnb_quantization or 'TE' in shared.opts.torchao_quantization or 'TE' in shared.opts.quanto_quantization): kwargs['text_encoder'] = transformers.AutoModelForCausalLM.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=cache_dir, **load_args, **quant_args) shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') return kwargs diff --git a/modules/model_sd3.py b/modules/model_sd3.py index 59789155e..eaae0684f 100644 --- a/modules/model_sd3.py +++ b/modules/model_sd3.py @@ -55,10 +55,10 @@ def load_quants(kwargs, repo_id, cache_dir): quant_args = model_quant.create_config() if not quant_args: return kwargs - if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: + if 'transformer' not in kwargs and (('Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization or 'Model' in shared.opts.quanto_quantization) or ('Transformer' in shared.opts.bnb_quantization or 'Transformer' in shared.opts.torchao_quantization or 'Transformer' in shared.opts.quanto_quantization)): kwargs['transformer'] = diffusers.SD3Transformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') - if 'text_encoder_3' not in kwargs and ('Text Encoder' in shared.opts.bnb_quantization or 'Text Encoder' in shared.opts.torchao_quantization): + if 'text_encoder_3' not in kwargs and ('TE' in shared.opts.bnb_quantization or 'TE' in shared.opts.torchao_quantization or 'TE' in shared.opts.quanto_quantization): kwargs['text_encoder_3'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder_3", variant='fp16', cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') return kwargs diff --git a/modules/onnx_impl/pipelines/__init__.py b/modules/onnx_impl/pipelines/__init__.py index a11b07fc7..62db80cd5 100644 --- a/modules/onnx_impl/pipelines/__init__.py +++ b/modules/onnx_impl/pipelines/__init__.py @@ -368,7 +368,7 @@ class OnnxRawPipeline(PipelineBase): if shared.opts.cuda_compile_backend == "olive-ai": submodels_for_olive = [] - if "Text Encoder" in shared.opts.cuda_compile: + if "TE" in shared.opts.cuda_compile: if not self.is_refiner: submodels_for_olive.append("text_encoder") if self._is_sdxl: diff --git a/modules/sd_models.py b/modules/sd_models.py index 43e926924..a7307561d 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -130,7 +130,7 @@ def set_diffuser_options(sd_model, vae=None, op:str='model', offload:bool=True, model.requires_grad_(False) model.eval() return model - sd_model = apply_function_to_model(sd_model, eval_model, ["Model", "VAE", "Text Encoder"], op="eval") + sd_model = apply_function_to_model(sd_model, eval_model, ["Model", "VAE", "TE"], op="eval") if len(shared.opts.torchao_quantization) > 0 and shared.opts.torchao_quantization_mode == 'post': sd_model = model_quant.torchao_quantization(sd_model) diff --git a/modules/sd_models_utils.py b/modules/sd_models_utils.py index cfbf38136..0b2794bac 100644 --- a/modules/sd_models_utils.py +++ b/modules/sd_models_utils.py @@ -164,7 +164,7 @@ def apply_function_to_model(sd_model, function, options, op=None): sd_model.prior_pipe.prior = function(sd_model.prior_pipe.prior, op="prior_pipe.prior", sd_model=sd_model) if op == "nncf" and "StableCascade" in sd_model.__class__.__name__: sd_model.prior_pipe.prior.clip_txt_pooled_mapper = backup_clip_txt_pooled_mapper - if "Text Encoder" in options: + if "TE" in options: if hasattr(sd_model, 'text_encoder') and hasattr(sd_model.text_encoder, 'config'): if hasattr(sd_model, 'decoder_pipe') and hasattr(sd_model.decoder_pipe, 'text_encoder') and hasattr(sd_model.decoder_pipe.text_encoder, 'config'): sd_model.decoder_pipe.text_encoder = function(sd_model.decoder_pipe.text_encoder, op="decoder_pipe.text_encoder", sd_model=sd_model) diff --git a/modules/shared.py b/modules/shared.py index f4967cf93..5d785f1a5 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -494,7 +494,7 @@ options_templates.update(options_section(('backends', "Backend Settings"), { "olive_cache_optimized": OptionInfo(True, 'Olive cache optimized models'), "ipex_sep": OptionInfo("

IPEX

", "", gr.HTML, {"visible": devices.backend == "ipex"}), - "ipex_optimize": OptionInfo([], "IPEX Optimize", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "Upscaler"], "visible": devices.backend == "ipex"}), + "ipex_optimize": OptionInfo([], "IPEX Optimize", gr.CheckboxGroup, {"choices": ["Model", "VAE", "TE", "Upscaler"], "visible": devices.backend == "ipex"}), "openvino_sep": OptionInfo("

OpenVINO

", "", gr.HTML, {"visible": cmd_opts.use_openvino}), "openvino_devices": OptionInfo([], "OpenVINO devices to use", gr.CheckboxGroup, {"choices": get_openvino_device_list() if cmd_opts.use_openvino else [], "visible": cmd_opts.use_openvino}), # pylint: disable=E0606 @@ -509,36 +509,36 @@ options_templates.update(options_section(('backends', "Backend Settings"), { options_templates.update(options_section(('quantization', "Quantization Settings"), { "bnb_quantization_sep": OptionInfo("

BitsAndBytes

", "", gr.HTML), - "bnb_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "LLM"], "visible": native}), + "bnb_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM"], "visible": native}), "bnb_quantization_type": OptionInfo("nf4", "Quantization type", gr.Dropdown, {"choices": ['nf4', 'fp8', 'fp4'], "visible": native}), "bnb_quantization_storage": OptionInfo("uint8", "Backend storage", gr.Dropdown, {"choices": ["float16", "float32", "int8", "uint8", "float64", "bfloat16"], "visible": native}), "quanto_quantization_sep": OptionInfo("

Optimum Quanto

", "", gr.HTML), - "quanto_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": native}), + "quanto_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM"], "visible": native}), "quanto_quantization_type": OptionInfo("int8", "Quantization weights type", gr.Dropdown, {"choices": ["float8", "int8", "int4", "int2"], "visible": native}), "optimum_quanto_sep": OptionInfo("

Optimum Quanto: post-load

", "", gr.HTML), - "optimum_quanto_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "ControlNet"], "visible": native}), + "optimum_quanto_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "optimum_quanto_weights_type": OptionInfo("qint8", "Quantization weights type", gr.Dropdown, {"choices": ['qint8', 'qfloat8_e4m3fn', 'qfloat8_e5m2', 'qint4', 'qint2'], "visible": native}), "optimum_quanto_activations_type": OptionInfo("none", "Quantization activations type ", gr.Dropdown, {"choices": ['none', 'qint8', 'qfloat8_e4m3fn', 'qfloat8_e5m2'], "visible": native}), "optimum_quanto_shuffle_weights": OptionInfo(False, "Shuffle weights", gr.Checkbox, {"visible": native}), "torchao_sep": OptionInfo("

TorchAO

", "", gr.HTML), - "torchao_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": native}), + "torchao_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM"], "visible": native}), "torchao_quantization_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native}), "torchao_quantization_type": OptionInfo("int8_weight_only", "Quantization type", gr.Dropdown, {"choices": ['int4_weight_only', 'int8_dynamic_activation_int4_weight', 'int8_weight_only', 'int8_dynamic_activation_int8_weight', 'float8_weight_only', 'float8_dynamic_activation_float8_weight', 'float8_static_activation_float8_weight'], "visible": native}), "nncf_compress_sep": OptionInfo("

NNCF: Neural Network Compression Framework

", "", gr.HTML), - "nncf_compress_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "ControlNet"], "visible": native}), + "nncf_compress_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "nncf_compress_weights_mode": OptionInfo("INT8", "Quantization type", gr.Dropdown, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'] if cmd_opts.use_openvino else ['INT8']}), "nncf_compress_weights_raito": OptionInfo(0, "Compress ratio", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}), "nncf_compress_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 512, "step": 1, "visible": cmd_opts.use_openvino}), - "nncf_quantize": OptionInfo([], "OpenVINO enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": cmd_opts.use_openvino}), + "nncf_quantize": OptionInfo([], "OpenVINO enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "TE"], "visible": cmd_opts.use_openvino}), "nncf_quantize_mode": OptionInfo("INT8", "OpenVINO mode", gr.Dropdown, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}), "nncf_quantize_shuffle_weights": OptionInfo(False, "Shuffle weights", gr.Checkbox, {"visible": native}), "layerwise_quantization_sep": OptionInfo("

Layerwise Casting

", "", gr.HTML), - "layerwise_quantization": OptionInfo([], "Layerwise casting enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "Text Encoder"], "visible": native}), + "layerwise_quantization": OptionInfo([], "Layerwise casting enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "TE"], "visible": native}), "layerwise_quantization_storage": OptionInfo("float8_e4m3fn", "Layerwise casting storage", gr.Dropdown, {"choices": ["float8_e4m3fn", "float8_e5m2"], "visible": native}), "layerwise_quantization_nonblocking": OptionInfo(False, "Layerwise non-blocking operations", gr.Checkbox, {"visible": native}), })) @@ -612,7 +612,7 @@ options_templates.update(options_section(('advanced', "Pipeline Modifiers"), { options_templates.update(options_section(('compile', "Model Compile"), { "cuda_compile_sep": OptionInfo("

Model Compile

", "", gr.HTML), - "cuda_compile": OptionInfo([] if not cmd_opts.use_openvino else ["Model", "VAE", "Upscaler"], "Compile Model", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "Upscaler"]}), + "cuda_compile": OptionInfo([] if not cmd_opts.use_openvino else ["Model", "VAE", "Upscaler"], "Compile Model", gr.CheckboxGroup, {"choices": ["Model", "VAE", "TE", "Upscaler"]}), "cuda_compile_backend": OptionInfo("none" if not cmd_opts.use_openvino else "openvino_fx", "Model compile backend", gr.Radio, {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'migraphx', 'ipex', 'onediff', 'stable-fast', 'deep-cache', 'olive-ai', 'openvino_fx']}), "cuda_compile_mode": OptionInfo("default", "Model compile mode", gr.Radio, {"choices": ['default', 'reduce-overhead', 'max-autotune', 'max-autotune-no-cudagraphs']}), "cuda_compile_fullgraph": OptionInfo(True if not cmd_opts.use_openvino else False, "Model compile fullgraph"), diff --git a/modules/video_models/video_load.py b/modules/video_models/video_load.py index 3bfeb2786..13d0dc935 100644 --- a/modules/video_models/video_load.py +++ b/modules/video_models/video_load.py @@ -19,7 +19,7 @@ def load_model(selected: models_def.Model): # text encoder try: - quant_args = model_quant.create_config(module='Text Encoder') + quant_args = model_quant.create_config(module='TE') debug(f'Video load: module=te repo="{selected.te or selected.repo}" folder="{selected.te_folder}" cls={selected.te_cls.__name__} quant={video_utils.get_quant(quant_args)}') text_encoder = selected.te_cls.from_pretrained( pretrained_model_name_or_path=selected.te or selected.repo, @@ -35,7 +35,7 @@ def load_model(selected: models_def.Model): # transformer try: - quant_args = model_quant.create_config(module='Model') + quant_args = model_quant.create_config(module='Video') debug(f'Video load: module=transformer repo="{selected.dit or selected.repo}" folder="{selected.dit_folder}" cls={selected.dit_cls.__name__} quant={video_utils.get_quant(quant_args)}') transformer = selected.dit_cls.from_pretrained( pretrained_model_name_or_path=selected.dit or selected.repo, diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py index a1933673e..09f30360e 100644 --- a/scripts/ltxvideo.py +++ b/scripts/ltxvideo.py @@ -24,7 +24,7 @@ def load_quants(kwargs, repo_id): if 'transformer' not in kwargs and ('Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization): kwargs['transformer'] = diffusers.LTXVideoTransformer3DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args) shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') - if 'text_encoder' not in kwargs and ('Text Encoder' in shared.opts.bnb_quantization or 'Text Encoder' in shared.opts.torchao_quantization): + if 'text_encoder' not in kwargs and ('TE' in shared.opts.bnb_quantization or 'TE' in shared.opts.torchao_quantization): kwargs['text_encoder'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args) shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') return kwargs