From 9e6928410f44a686bcfef0fd8f3f2fadd47eaf73 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 25 Aug 2025 10:16:38 -0400 Subject: [PATCH] offloading improvements Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 3 ++- modules/sd_hijack_te.py | 3 +++ modules/sd_offload.py | 2 +- modules/shared_defaults.py | 2 ++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80acae580..ec96f4fcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2025-08-24 +## Update for 2025-08-25 - **Models** - **Chroma** final versions: [Chroma1-HD](https://huggingface.co/lodestones/Chroma1-HD), [Chroma1-Base](https://huggingface.co/lodestones/Chroma1-Base) and [Chroma1-Flash](https://huggingface.co/lodestones/Chroma1-Flash) @@ -8,6 +8,7 @@ - updated [SD.Next Model Samples Gallery](https://vladmandic.github.io/sd-samples/compare.html) - **Core** - enable offload during pre-forward by default + - improve offloading of very large models - update `requirements` - **UI** - improved image scaling in img2img and control interfaces diff --git a/modules/sd_hijack_te.py b/modules/sd_hijack_te.py index 960d87430..2ee73f903 100644 --- a/modules/sd_hijack_te.py +++ b/modules/sd_hijack_te.py @@ -11,6 +11,9 @@ def hijack_encode_prompt(*args, **kwargs): # if hasattr(shared.sd_model, 'text_encoder') and shared.sd_model.text_encoder is not None: # sd_models.move_model(shared.sd_model.text_encoder, devices.device) try: + prompt = kwargs.get('prompt', None) or (args[0] if len(args) > 0 else None) + if prompt is not None: + shared.log.debug(f'Encode: prompt="{prompt}" hijack=True') res = shared.sd_model.orig_encode_prompt(*args, **kwargs) except Exception as e: shared.log.error(f'Encode prompt: {e}') diff --git a/modules/sd_offload.py b/modules/sd_offload.py index 4a253a428..8678d8002 100644 --- a/modules/sd_offload.py +++ b/modules/sd_offload.py @@ -211,7 +211,7 @@ class OffloadHook(accelerate.hooks.ModelHook): for module_name in get_module_names(pipe): module_instance = getattr(pipe, module_name, None) module_cls = module_instance.__class__.__name__ - if (module_cls != module.__class__.__name__) and (module_cls not in self.offload_never) and (not devices.same_device(module_instance.device, devices.cpu)): + if (id(module) != id(module_instance)) and (module_cls not in self.offload_never) and (not devices.same_device(module_instance.device, devices.cpu)): apply_balanced_offload_to_module(module_instance, op='pre') if not devices.same_device(module.device, devices.device): diff --git a/modules/shared_defaults.py b/modules/shared_defaults.py index 8ada27a0c..a2d23b2f0 100644 --- a/modules/shared_defaults.py +++ b/modules/shared_defaults.py @@ -20,10 +20,12 @@ def get_default_modes(cmd_opts, mem_stat): cmd_opts.medvram = True # VAE Tiling and other stuff default_offload_mode = "balanced" default_diffusers_offload_min_gpu_memory = 0 + default_diffusers_offload_always = ', '.join(['T5EncoderModel', 'UMT5EncoderModel']) log.info(f"Device detect: memory={gpu_memory:.1f} default=balanced optimization=medvram") elif gpu_memory >= 24: default_offload_mode = "balanced" default_diffusers_offload_max_gpu_memory = 0.8 + default_diffusers_offload_always = ', '.join(['T5EncoderModel', 'UMT5EncoderModel']) default_diffusers_offload_never = ', '.join(['CLIPTextModel', 'CLIPTextModelWithProjection', 'AutoencoderKL']) log.info(f"Device detect: memory={gpu_memory:.1f} default=balanced optimization=highvram") else: