diff --git a/CHANGELOG.md b/CHANGELOG.md index a4307eb66..2affef04b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,19 +3,27 @@ ## Pending - Diffusers==0.30.0 -- https://github.com/huggingface/diffusers/issues/8579 - https://github.com/huggingface/diffusers/issues/8546 - https://github.com/huggingface/diffusers/pull/8566 -- https://github.com/huggingface/diffusers/pull/8506 +- https://github.com/huggingface/diffusers/pull/8584 -## Update for 2024-06-15 +## Update for 2024-06-16 + +### Improvements + +- support for T5 text-encoder loader in **all** models that use T5 + *example*: load FP8 quantized T5 text-encoder into PixArt Sigma +- support for `torch-directml` **0.2.2**, thanks @lshqqytiger! + new directml is finally based on modern `torch` 2.3.1! +- **sd3** enable taesd preview and non-full quality mode +- **sd3** enable base LoRA support +- **sd3** simplified loading of model in single-file safetensors format + loading sd3 can now be performed fully offline + +### Fixes -- support for `torch-directml` **0.2.2**, thanks @lshqqytiger! -- **sd3** enable taesd preview and non-full quality mode -- **sd3** enable base LoRA support -- **sd3** simplified loading of model in single-file safetensors format - fix unsaturated outputs, force apply vae config on model load -- fix control second pass resize +- fix control second pass resize - fix api face-hires - fix **hunyuandit** set attention processor - fix civitai download without name diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index 4e45a86bd..574b223bf 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -162,7 +162,6 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No list_available_networks() networks_on_disk = [available_network_aliases.get(name, None) for name in names] failed_to_load_networks = [] - recompile_model = False if shared.compiled_model_state is not None and shared.compiled_model_state.is_compiled: if len(names) == len(shared.compiled_model_state.lora_model): diff --git a/modules/control/units/xs_pipe.py b/modules/control/units/xs_pipe.py index 14581c0f1..7e717b542 100644 --- a/modules/control/units/xs_pipe.py +++ b/modules/control/units/xs_pipe.py @@ -1048,7 +1048,7 @@ class StableDiffusionXLControlNetXSPipeline( self.upcast_vae() latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) - if not output_type == "latent": + if output_type != "latent": # make sure the VAE is in float32 mode, as it overflows in float16 needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast @@ -1064,7 +1064,7 @@ class StableDiffusionXLControlNetXSPipeline( else: image = latents - if not output_type == "latent": + if output_type != "latent": # apply watermark if available if self.watermark is not None: image = self.watermark.apply_watermark(image) @@ -1907,7 +1907,7 @@ class StableDiffusionControlNetXSPipeline( self.controlnet.to("cpu") torch.cuda.empty_cache() - if not output_type == "latent": + if output_type != "latent": image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[ 0 ] diff --git a/modules/model_sd3.py b/modules/model_sd3.py index e2004c1f4..312976183 100644 --- a/modules/model_sd3.py +++ b/modules/model_sd3.py @@ -13,7 +13,6 @@ loggedin = False def load_sd3(fn=None, cache_dir=None, config=None): from modules import devices, modelloader - modelloader.hf_login() repo_id = 'stabilityai/stable-diffusion-3-medium-diffusers' model_id = 'stabilityai/stable-diffusion-3-medium-diffusers' dtype = torch.float16 @@ -56,6 +55,7 @@ def load_sd3(fn=None, cache_dir=None, config=None): else: kwargs = {} else: + modelloader.hf_login() model_id = repo_id loader = diffusers.StableDiffusion3Pipeline.from_pretrained pipe = loader( @@ -71,34 +71,45 @@ def load_sd3(fn=None, cache_dir=None, config=None): return pipe -def load_te3(pipe, te3=None, cache_dir=None): +def load_t5(pipe, module, te3=None, cache_dir=None): from modules import devices, modelloader - modelloader.hf_login() repo_id = 'stabilityai/stable-diffusion-3-medium-diffusers' - if pipe is None or not hasattr(pipe, 'text_encoder_3'): + if pipe is None or not hasattr(pipe, module): return pipe if 'fp16' in te3.lower(): - pipe.text_encoder_3 = transformers.T5EncoderModel.from_pretrained( + modelloader.hf_login() + t5 = transformers.T5EncoderModel.from_pretrained( repo_id, subfolder='text_encoder_3', # torch_dtype=dtype, cache_dir=cache_dir, torch_dtype=pipe.text_encoder.dtype, ) + setattr(pipe, module, t5) elif 'fp8' in te3.lower(): + modelloader.hf_login() from installer import install install('bitsandbytes', quiet=True) quantization_config = transformers.BitsAndBytesConfig(load_in_8bit=True) - pipe.text_encoder_3 = transformers.T5EncoderModel.from_pretrained( + t5 = transformers.T5EncoderModel.from_pretrained( repo_id, subfolder='text_encoder_3', quantization_config=quantization_config, cache_dir=cache_dir, torch_dtype=pipe.text_encoder.dtype, ) + setattr(pipe, module, t5) + """ + if hasattr(pipe, 'remove_all_hooks'): + pipe.remove_all_hooks() + nn = getattr(pipe, module) + import accelerate + accelerate.hooks.remove_hook_from_module(nn, recurse=True) + nn.to(device=devices.device) + """ else: - pipe.text_encoder_3 = None - if getattr(pipe, 'text_encoder_3', None) is not None and getattr(pipe, 'tokenizer_3', None) is None: + setattr(pipe, module, None) + if getattr(pipe, 'text_encoder_3', None) is not None and getattr(pipe, 'tokenizer_3', None) is None: # not needed anymore pipe.tokenizer_3 = transformers.T5TokenizerFast.from_pretrained( repo_id, subfolder='tokenizer_3', diff --git a/modules/sd_models.py b/modules/sd_models.py index 5ceed8d5d..81214bae5 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -997,7 +997,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No elif model_type in ['PixArt-Sigma']: # forced pipeline try: # shared.opts.data['cuda_dtype'] = 'FP32' # override - shared.opts.data['diffusers_model_cpu_offload'] = True # override + # shared.opts.data['diffusers_model_cpu_offload'] = True # override devices.set_cuda_params() sd_model = diffusers.PixArtSigmaPipeline.from_pretrained( checkpoint_info.path, @@ -1168,7 +1168,7 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No move_model(sd_model, devices.device) timer.record("move") - reload_text_encoder() + reload_text_encoder(initial=True) if shared.opts.ipex_optimize: sd_model = sd_models_compile.ipex_optimize(sd_model) @@ -1527,11 +1527,19 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None, shared.log.info(f'Model load finished: {memory_stats()} cached={len(checkpoints_loaded.keys())}') -def reload_text_encoder(): - if hasattr(shared.sd_model, 'text_encoder_3'): - from modules.model_sd3 import load_te3 - shared.log.debug(f'Load: TE3={shared.opts.sd_te3}') - load_te3(shared.sd_model, shared.opts.sd_te3, cache_dir=shared.opts.diffusers_dir) +def reload_text_encoder(initial=False): + if initial and (shared.opts.sd_te3 is None or shared.opts.sd_te3 == 'None'): + return # dont unload + signature = inspect.signature(shared.sd_model.__class__.__init__, follow_wrapped=True, eval_str=True).parameters + t5 = [k for k, v in signature.items() if 'T5EncoderModel' in str(v)] + if len(t5) > 0: + from modules.model_sd3 import load_t5 + shared.log.debug(f'Load: t5={shared.opts.sd_te3} module="{t5[0]}"') + load_t5(pipe=shared.sd_model, module=t5[0], te3=shared.opts.sd_te3, cache_dir=shared.opts.diffusers_dir) + elif hasattr(shared.sd_model, 'text_encoder_3'): + from modules.model_sd3 import load_t5 + shared.log.debug(f'Load: t5={shared.opts.sd_te3} module="text_encoder_3"') + load_t5(pipe=shared.sd_model, module='text_encoder_3', te3=shared.opts.sd_te3, cache_dir=shared.opts.diffusers_dir) def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model', force=False):