mirror of
https://github.com/vladmandic/automatic
synced 2026-09-18 16:54:33 +02:00
add diffusers_offload_nonblocking setting
Signed-off-by: Vladimir Mandic <mandic00@live.com>
This commit is contained in:
Vladimir Mandic
@@ -16,6 +16,7 @@
|
||||
**Wiki** search: uses github api to search online wiki pages
|
||||
- quicksettings reset button to restore all quicksettings to default values
|
||||
because things do sometimes get wrong...
|
||||
- updated real-time hints, thanks @CalamitousFelicitousness
|
||||
- modernui checkbox/radio styling
|
||||
- **Offloading**
|
||||
- changed **default** values for offloading based on detected gpu memory
|
||||
@@ -41,6 +42,7 @@
|
||||
- fix api progress reporting endpoint
|
||||
- fix openvino backend failing to compile
|
||||
- fix nunchaku fallback on unsupported model
|
||||
- reapply offloading on ipadapter load
|
||||
- api set default script-name
|
||||
- avoid forced gc and rely on thresholds
|
||||
- add missing interrogate in output panel
|
||||
|
||||
Submodule extensions-builtin/sdnext-modernui updated: 7a872dfee6...43ed2ea510
@@ -11,7 +11,7 @@ import json
|
||||
from PIL import Image
|
||||
import diffusers
|
||||
import transformers
|
||||
from modules import processing, shared, devices, sd_models, errors
|
||||
from modules import processing, shared, devices, sd_models, errors, model_quant
|
||||
|
||||
|
||||
clip_loaded = None
|
||||
@@ -198,6 +198,8 @@ def load_image_encoder(pipe: diffusers.DiffusionPipeline, adapter_names: list[st
|
||||
else:
|
||||
image_encoder = transformers.CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=clip_subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, use_safetensors=True)
|
||||
shared.log.debug(f'IP adapter load: encoder="{clip_repo}/{clip_subfolder}" cls={pipe.image_encoder.__class__.__name__}')
|
||||
sd_models.clear_caches()
|
||||
image_encoder = model_quant.do_post_load_quant(image_encoder, allow=True)
|
||||
if hasattr(pipe, 'register_modules'):
|
||||
pipe.register_modules(image_encoder=image_encoder)
|
||||
else:
|
||||
@@ -223,6 +225,7 @@ def load_feature_extractor(pipe):
|
||||
pipe.register_modules(feature_extractor=feature_extractor)
|
||||
else:
|
||||
pipe.feature_extractor = feature_extractor
|
||||
sd_models.apply_balanced_offload(pipe.feature_extractor)
|
||||
shared.log.debug(f'IP adapter load: extractor={pipe.feature_extractor.__class__.__name__}')
|
||||
except Exception as e:
|
||||
shared.log.error(f'IP adapter load: extractor {e}')
|
||||
@@ -256,14 +259,14 @@ def parse_params(p: processing.StableDiffusionProcessing, adapters: list, adapte
|
||||
adapter_masks = mask_processor.preprocess(adapter_masks, height=p.height, width=p.width)
|
||||
if adapter_images is None:
|
||||
shared.log.error('IP adapter: no image provided')
|
||||
return False
|
||||
return [], [], [], [], [], []
|
||||
if len(adapters) < len(adapter_images):
|
||||
adapter_images = adapter_images[:len(adapters)]
|
||||
if len(adapters) < len(adapter_masks):
|
||||
adapter_masks = adapter_masks[:len(adapters)]
|
||||
if len(adapter_masks) > 0 and len(adapter_masks) != len(adapter_images):
|
||||
shared.log.error('IP adapter: image and mask count mismatch')
|
||||
return False
|
||||
return [], [], [], [], [], []
|
||||
adapter_scales = get_scales(adapter_scales, adapter_images)
|
||||
p.ip_adapter_scales = adapter_scales.copy()
|
||||
adapter_crops = get_crops(adapter_crops, adapter_images)
|
||||
|
||||
+9
-10
@@ -302,7 +302,7 @@ def load_fp8_model_layerwise(checkpoint_info, load_model_func, diffusers_load_co
|
||||
repo_path = checkpoint_info.path
|
||||
try:
|
||||
import torch
|
||||
from modules import devices
|
||||
from modules import devices, shared
|
||||
from diffusers.quantizers import quantization_config
|
||||
if not hasattr(quantization_config.QuantizationMethod, 'LAYERWISE'):
|
||||
setattr(quantization_config.QuantizationMethod, 'LAYERWISE', 'layerwise') # noqa: B010
|
||||
@@ -315,7 +315,7 @@ def load_fp8_model_layerwise(checkpoint_info, load_model_func, diffusers_load_co
|
||||
model = load_model_func(repo_path, **load_args)
|
||||
model = upcast_non_layerwise_modules(model, devices.dtype)
|
||||
model._skip_layerwise_casting_patterns = None # pylint: disable=protected-access
|
||||
model.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=False, skip_modules_pattern=[])
|
||||
model.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=shared.opts.diffusers_offload_nonblocking, skip_modules_pattern=[])
|
||||
model.layerwise_storage_dtype = storage_dtype
|
||||
model.quantization_method = 'LayerWise'
|
||||
except Exception as e:
|
||||
@@ -336,7 +336,6 @@ def apply_layerwise(sd_model, quiet:bool=False):
|
||||
storage_dtype = None
|
||||
log.warning(f'Quantization: type=layerwise storage={shared.opts.layerwise_quantization_storage} not supported')
|
||||
return
|
||||
non_blocking = False
|
||||
if not hasattr(quantization_config.QuantizationMethod, 'LAYERWISE'):
|
||||
setattr(quantization_config.QuantizationMethod, 'LAYERWISE', 'layerwise') # noqa: B010
|
||||
for module in sd_models.get_signature(sd_model).keys():
|
||||
@@ -347,25 +346,25 @@ def apply_layerwise(sd_model, quiet:bool=False):
|
||||
m = getattr(sd_model, module)
|
||||
if getattr(m, "quantization_method", None) in {'LayerWise', quantization_config.QuantizationMethod.LAYERWISE}: # pylint: disable=no-member
|
||||
storage_dtype = getattr(m, "layerwise_storage_dtype", storage_dtype)
|
||||
m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=non_blocking)
|
||||
m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=shared.opts.diffusers_offload_nonblocking)
|
||||
elif module.startswith('unet') and ('Model' in shared.opts.layerwise_quantization):
|
||||
if hasattr(m, 'enable_layerwise_casting'):
|
||||
m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=non_blocking)
|
||||
m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=shared.opts.diffusers_offload_nonblocking)
|
||||
m.layerwise_storage_dtype = storage_dtype
|
||||
m.quantization_method = 'LayerWise'
|
||||
log.quiet(quiet, f'Quantization: type=layerwise module={module} cls={cls} storage={storage_dtype} compute={devices.dtype} blocking={not non_blocking}')
|
||||
log.quiet(quiet, f'Quantization: type=layerwise module={module} cls={cls} storage={storage_dtype} compute={devices.dtype} blocking={not shared.opts.diffusers_offload_nonblocking}')
|
||||
elif module.startswith('transformer') and ('Model' in shared.opts.layerwise_quantization):
|
||||
if hasattr(m, 'enable_layerwise_casting'):
|
||||
m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=non_blocking)
|
||||
m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=shared.opts.diffusers_offload_nonblocking)
|
||||
m.layerwise_storage_dtype = storage_dtype
|
||||
m.quantization_method = 'LayerWise'
|
||||
log.quiet(quiet, f'Quantization: type=layerwise module={module} cls={cls} storage={storage_dtype} compute={devices.dtype} blocking={not non_blocking}')
|
||||
log.quiet(quiet, f'Quantization: type=layerwise module={module} cls={cls} storage={storage_dtype} compute={devices.dtype} blocking={not shared.opts.diffusers_offload_nonblocking}')
|
||||
elif module.startswith('text_encoder') and ('TE' in shared.opts.layerwise_quantization) and ('clip' not in cls.lower()):
|
||||
if hasattr(m, 'enable_layerwise_casting'):
|
||||
m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=non_blocking)
|
||||
m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=shared.opts.diffusers_offload_nonblocking)
|
||||
m.layerwise_storage_dtype = storage_dtype
|
||||
m.quantization_method = quantization_config.QuantizationMethod.LAYERWISE # pylint: disable=no-member
|
||||
log.quiet(quiet, f'Quantization: type=layerwise module={module} cls={cls} storage={storage_dtype} compute={devices.dtype} blocking={not non_blocking}')
|
||||
log.quiet(quiet, f'Quantization: type=layerwise module={module} cls={cls} storage={storage_dtype} compute={devices.dtype} blocking={not shared.opts.diffusers_offload_nonblocking}')
|
||||
except Exception as e:
|
||||
if 'Hook with name' not in str(e):
|
||||
log.error(f'Quantization: type=layerwise {e}')
|
||||
|
||||
@@ -63,10 +63,10 @@ def hijack_set_module_tensor_simple(
|
||||
old_value = getattr(module, tensor_name)
|
||||
with devices.inference_context():
|
||||
if tensor_name in module._buffers: # pylint: disable=protected-access
|
||||
module._buffers[tensor_name] = value.to(device) # pylint: disable=protected-access
|
||||
module._buffers[tensor_name] = value.to(device, non_blocking=shared.opts.diffusers_offload_nonblocking) # pylint: disable=protected-access
|
||||
elif value is not None or not devices.same_device(device, module._parameters[tensor_name].device): # pylint: disable=protected-access
|
||||
param_cls = type(module._parameters[tensor_name]) # pylint: disable=protected-access
|
||||
module._parameters[tensor_name] = param_cls(value, requires_grad=old_value.requires_grad).to(device) # pylint: disable=protected-access
|
||||
module._parameters[tensor_name] = param_cls(value, requires_grad=old_value.requires_grad).to(device, non_blocking=shared.opts.diffusers_offload_nonblocking) # pylint: disable=protected-access
|
||||
t1 = time.time()
|
||||
tensor_to_timer += (t1 - t0)
|
||||
|
||||
|
||||
@@ -74,7 +74,7 @@ def apply_group_offload(sd_model, op:str='model'):
|
||||
'offload_device': devices.cpu,
|
||||
'offload_type': 'block_level', # 'leaf_level',
|
||||
'num_blocks_per_group': 1,
|
||||
'non_blocking': False,
|
||||
'non_blocking': shared.opts.diffusers_offload_nonblocking,
|
||||
'use_stream': False,
|
||||
'record_stream': False,
|
||||
'low_cpu_mem_usage': False,
|
||||
|
||||
@@ -165,6 +165,7 @@ options_templates.update(options_section(('offload', "Model Offloading"), {
|
||||
"offload_sep": OptionInfo("<h2>Model Offloading</h2>", "", gr.HTML),
|
||||
"diffusers_offload_mode": OptionInfo(startup_offload_mode, "Model offload mode", gr.Radio, {"choices": ['none', 'balanced', 'group', 'model', 'sequential']}),
|
||||
"diffusers_offload_pre": OptionInfo(False, "Offload during pre-forward"),
|
||||
"diffusers_offload_nonblocking": OptionInfo(False, "Non-blocking move operations"),
|
||||
"diffusers_offload_min_gpu_memory": OptionInfo(startup_offload_min_gpu, "Balanced offload GPU low watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }),
|
||||
"diffusers_offload_max_gpu_memory": OptionInfo(startup_offload_max_gpu, "Balanced offload GPU high watermark", gr.Slider, {"minimum": 0.1, "maximum": 1, "step": 0.01 }),
|
||||
"diffusers_offload_max_cpu_memory": OptionInfo(0.90, "Balanced offload CPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": False }),
|
||||
|
||||
Reference in New Issue
Block a user