diff --git a/CHANGELOG.md b/CHANGELOG.md index 20b92b865..a8d77aa81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,12 +13,15 @@ - enable taesd preview and non-full quality mode - enable base LoRA support +- add support for 4bit quantized t5 text encoder + simply select in *settings -> model -> text encoder* - simplified loading of model in single-file safetensors format loading sd3 can now be performed fully offline - add support for nncf compressed weights, thanks @Disty0! - add support for sampler shift for Euler FlowMatch see *settings -> samplers*, also available as param in xyz grid higher shift means model will spend more time on structure and less on details +- add support for selecting text encoder in xyz grid ### Improvements: General diff --git a/modules/model_sd3.py b/modules/model_sd3.py index 312976183..4d8aaf250 100644 --- a/modules/model_sd3.py +++ b/modules/model_sd3.py @@ -86,6 +86,19 @@ def load_t5(pipe, module, te3=None, cache_dir=None): torch_dtype=pipe.text_encoder.dtype, ) setattr(pipe, module, t5) + elif 'fp4' in te3.lower(): + modelloader.hf_login() + from installer import install + install('bitsandbytes', quiet=True) + quantization_config = transformers.BitsAndBytesConfig(load_in_4bit=True) + t5 = transformers.T5EncoderModel.from_pretrained( + repo_id, + subfolder='text_encoder_3', + quantization_config=quantization_config, + cache_dir=cache_dir, + torch_dtype=pipe.text_encoder.dtype, + ) + setattr(pipe, module, t5) elif 'fp8' in te3.lower(): modelloader.hf_login() from installer import install diff --git a/modules/sd_models.py b/modules/sd_models.py index 81214bae5..b06afefe8 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1528,18 +1528,18 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None, def reload_text_encoder(initial=False): - if initial and (shared.opts.sd_te3 is None or shared.opts.sd_te3 == 'None'): + if initial and (shared.opts.sd_text_encoder is None or shared.opts.sd_text_encoder == 'None'): return # dont unload signature = inspect.signature(shared.sd_model.__class__.__init__, follow_wrapped=True, eval_str=True).parameters t5 = [k for k, v in signature.items() if 'T5EncoderModel' in str(v)] if len(t5) > 0: from modules.model_sd3 import load_t5 - shared.log.debug(f'Load: t5={shared.opts.sd_te3} module="{t5[0]}"') - load_t5(pipe=shared.sd_model, module=t5[0], te3=shared.opts.sd_te3, cache_dir=shared.opts.diffusers_dir) + shared.log.debug(f'Load: t5={shared.opts.sd_text_encoder} module="{t5[0]}"') + load_t5(pipe=shared.sd_model, module=t5[0], te3=shared.opts.sd_text_encoder, cache_dir=shared.opts.diffusers_dir) elif hasattr(shared.sd_model, 'text_encoder_3'): from modules.model_sd3 import load_t5 - shared.log.debug(f'Load: t5={shared.opts.sd_te3} module="text_encoder_3"') - load_t5(pipe=shared.sd_model, module='text_encoder_3', te3=shared.opts.sd_te3, cache_dir=shared.opts.diffusers_dir) + shared.log.debug(f'Load: t5={shared.opts.sd_text_encoder} module="text_encoder_3"') + load_t5(pipe=shared.sd_model, module='text_encoder_3', te3=shared.opts.sd_text_encoder, cache_dir=shared.opts.diffusers_dir) def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model', force=False): diff --git a/modules/shared.py b/modules/shared.py index 4c5782c19..bd5fd4369 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -391,7 +391,7 @@ options_templates.update(options_section(('sd', "Execution & Models"), { "sd_model_refiner": OptionInfo('None', "Refiner model", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints), "sd_vae": OptionInfo("Automatic", "VAE model", gr.Dropdown, lambda: {"choices": shared_items.sd_vae_items()}, refresh=shared_items.refresh_vae_list), "sd_unet": OptionInfo("None", "UNET model", gr.Dropdown, lambda: {"choices": shared_items.sd_unet_items()}, refresh=shared_items.refresh_unet_list), - "sd_te3": OptionInfo('None', "Text encoder model", gr.Dropdown, lambda: {"choices": ['None', 'T5 FP8', 'T5 FP16']}), + "sd_text_encoder": OptionInfo('None', "Text encoder model", gr.Dropdown, lambda: {"choices": ['None', 'T5 FP4', 'T5 FP8', 'T5 FP16']}), "sd_checkpoint_autoload": OptionInfo(True, "Model autoload on start"), "sd_model_dict": OptionInfo('None', "Use separate base dict", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_tiles()}, refresh=refresh_checkpoints), "stream_load": OptionInfo(False, "Load models using stream loading method", gr.Checkbox, {"visible": not native }), diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index 6ba75c775..c4b44282d 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -138,6 +138,11 @@ def apply_vae(p, x, xs): sd_vae.reload_vae_weights(shared.sd_model, vae_file=find_vae(x)) +def apply_te(p, x, xs): + shared.opts.data["sd_text_encoder"] = x + sd_models.reload_text_encoder() + + def apply_styles(p: processing.StableDiffusionProcessingTxt2Img, x: str, _): p.styles.extend(x.split(',')) @@ -230,6 +235,7 @@ axis_options = [ AxisOption("Prompt S/R", str, apply_prompt, fmt=format_value), AxisOption("Model", str, apply_checkpoint, fmt=format_value, cost=1.0, choices=lambda: sorted(sd_models.checkpoints_list)), AxisOption("VAE", str, apply_vae, cost=0.7, choices=lambda: ['None'] + list(sd_vae.vae_dict)), + AxisOption("Text encoder", str, apply_te, cost=0.7, choices=lambda: ['None', 'T5 FP4', 'T5 FP8', 'T5 FP16']), AxisOption("Styles", str, apply_styles, choices=lambda: [s.name for s in shared.prompt_styles.styles.values()]), AxisOption("Seed", int, apply_field("seed")), AxisOption("Steps", int, apply_field("steps")), diff --git a/webui.py b/webui.py index b305767a8..c9af90e75 100644 --- a/webui.py +++ b/webui.py @@ -168,7 +168,7 @@ def load_model(): thread_refiner.join() shared.opts.onchange("sd_model_checkpoint", wrap_queued_call(lambda: modules.sd_models.reload_model_weights(op='model')), call=False) shared.opts.onchange("sd_model_refiner", wrap_queued_call(lambda: modules.sd_models.reload_model_weights(op='refiner')), call=False) - shared.opts.onchange("sd_te3", wrap_queued_call(lambda: modules.sd_models.reload_text_encoder()), call=False) + shared.opts.onchange("sd_text_encoder", wrap_queued_call(lambda: modules.sd_models.reload_text_encoder()), call=False) shared.opts.onchange("sd_model_dict", wrap_queued_call(lambda: modules.sd_models.reload_model_weights(op='dict')), call=False) shared.opts.onchange("sd_vae", wrap_queued_call(lambda: modules.sd_vae.reload_vae_weights()), call=False) shared.opts.onchange("sd_backend", wrap_queued_call(lambda: modules.sd_models.change_backend()), call=False)