diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e6dde723..d9af68d6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ **Wiki** search: uses github api to search online wiki pages - quicksettings reset button to restore all quicksettings to default values because things do sometimes get wrong... + - updated real-time hints, thanks @CalamitousFelicitousness - modernui checkbox/radio styling - **Offloading** - changed **default** values for offloading based on detected gpu memory @@ -41,6 +42,7 @@ - fix api progress reporting endpoint - fix openvino backend failing to compile - fix nunchaku fallback on unsupported model + - reapply offloading on ipadapter load - api set default script-name - avoid forced gc and rely on thresholds - add missing interrogate in output panel diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 7a872dfee..43ed2ea51 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 7a872dfee69895128657ee135fd3c0c5e0d3ee62 +Subproject commit 43ed2ea51049a5926fe553aed3e9a717657352a6 diff --git a/modules/ipadapter.py b/modules/ipadapter.py index 4852e9582..ad2caec2e 100644 --- a/modules/ipadapter.py +++ b/modules/ipadapter.py @@ -11,7 +11,7 @@ import json from PIL import Image import diffusers import transformers -from modules import processing, shared, devices, sd_models, errors +from modules import processing, shared, devices, sd_models, errors, model_quant clip_loaded = None @@ -198,6 +198,8 @@ def load_image_encoder(pipe: diffusers.DiffusionPipeline, adapter_names: list[st else: image_encoder = transformers.CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=clip_subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, use_safetensors=True) shared.log.debug(f'IP adapter load: encoder="{clip_repo}/{clip_subfolder}" cls={pipe.image_encoder.__class__.__name__}') + sd_models.clear_caches() + image_encoder = model_quant.do_post_load_quant(image_encoder, allow=True) if hasattr(pipe, 'register_modules'): pipe.register_modules(image_encoder=image_encoder) else: @@ -223,6 +225,7 @@ def load_feature_extractor(pipe): pipe.register_modules(feature_extractor=feature_extractor) else: pipe.feature_extractor = feature_extractor + sd_models.apply_balanced_offload(pipe.feature_extractor) shared.log.debug(f'IP adapter load: extractor={pipe.feature_extractor.__class__.__name__}') except Exception as e: shared.log.error(f'IP adapter load: extractor {e}') @@ -256,14 +259,14 @@ def parse_params(p: processing.StableDiffusionProcessing, adapters: list, adapte adapter_masks = mask_processor.preprocess(adapter_masks, height=p.height, width=p.width) if adapter_images is None: shared.log.error('IP adapter: no image provided') - return False + return [], [], [], [], [], [] if len(adapters) < len(adapter_images): adapter_images = adapter_images[:len(adapters)] if len(adapters) < len(adapter_masks): adapter_masks = adapter_masks[:len(adapters)] if len(adapter_masks) > 0 and len(adapter_masks) != len(adapter_images): shared.log.error('IP adapter: image and mask count mismatch') - return False + return [], [], [], [], [], [] adapter_scales = get_scales(adapter_scales, adapter_images) p.ip_adapter_scales = adapter_scales.copy() adapter_crops = get_crops(adapter_crops, adapter_images) diff --git a/modules/model_quant.py b/modules/model_quant.py index d150cab2e..66c538f90 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -302,7 +302,7 @@ def load_fp8_model_layerwise(checkpoint_info, load_model_func, diffusers_load_co repo_path = checkpoint_info.path try: import torch - from modules import devices + from modules import devices, shared from diffusers.quantizers import quantization_config if not hasattr(quantization_config.QuantizationMethod, 'LAYERWISE'): setattr(quantization_config.QuantizationMethod, 'LAYERWISE', 'layerwise') # noqa: B010 @@ -315,7 +315,7 @@ def load_fp8_model_layerwise(checkpoint_info, load_model_func, diffusers_load_co model = load_model_func(repo_path, **load_args) model = upcast_non_layerwise_modules(model, devices.dtype) model._skip_layerwise_casting_patterns = None # pylint: disable=protected-access - model.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=False, skip_modules_pattern=[]) + model.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=shared.opts.diffusers_offload_nonblocking, skip_modules_pattern=[]) model.layerwise_storage_dtype = storage_dtype model.quantization_method = 'LayerWise' except Exception as e: @@ -336,7 +336,6 @@ def apply_layerwise(sd_model, quiet:bool=False): storage_dtype = None log.warning(f'Quantization: type=layerwise storage={shared.opts.layerwise_quantization_storage} not supported') return - non_blocking = False if not hasattr(quantization_config.QuantizationMethod, 'LAYERWISE'): setattr(quantization_config.QuantizationMethod, 'LAYERWISE', 'layerwise') # noqa: B010 for module in sd_models.get_signature(sd_model).keys(): @@ -347,25 +346,25 @@ def apply_layerwise(sd_model, quiet:bool=False): m = getattr(sd_model, module) if getattr(m, "quantization_method", None) in {'LayerWise', quantization_config.QuantizationMethod.LAYERWISE}: # pylint: disable=no-member storage_dtype = getattr(m, "layerwise_storage_dtype", storage_dtype) - m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=non_blocking) + m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=shared.opts.diffusers_offload_nonblocking) elif module.startswith('unet') and ('Model' in shared.opts.layerwise_quantization): if hasattr(m, 'enable_layerwise_casting'): - m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=non_blocking) + m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=shared.opts.diffusers_offload_nonblocking) m.layerwise_storage_dtype = storage_dtype m.quantization_method = 'LayerWise' - log.quiet(quiet, f'Quantization: type=layerwise module={module} cls={cls} storage={storage_dtype} compute={devices.dtype} blocking={not non_blocking}') + log.quiet(quiet, f'Quantization: type=layerwise module={module} cls={cls} storage={storage_dtype} compute={devices.dtype} blocking={not shared.opts.diffusers_offload_nonblocking}') elif module.startswith('transformer') and ('Model' in shared.opts.layerwise_quantization): if hasattr(m, 'enable_layerwise_casting'): - m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=non_blocking) + m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=shared.opts.diffusers_offload_nonblocking) m.layerwise_storage_dtype = storage_dtype m.quantization_method = 'LayerWise' - log.quiet(quiet, f'Quantization: type=layerwise module={module} cls={cls} storage={storage_dtype} compute={devices.dtype} blocking={not non_blocking}') + log.quiet(quiet, f'Quantization: type=layerwise module={module} cls={cls} storage={storage_dtype} compute={devices.dtype} blocking={not shared.opts.diffusers_offload_nonblocking}') elif module.startswith('text_encoder') and ('TE' in shared.opts.layerwise_quantization) and ('clip' not in cls.lower()): if hasattr(m, 'enable_layerwise_casting'): - m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=non_blocking) + m.enable_layerwise_casting(compute_dtype=devices.dtype, storage_dtype=storage_dtype, non_blocking=shared.opts.diffusers_offload_nonblocking) m.layerwise_storage_dtype = storage_dtype m.quantization_method = quantization_config.QuantizationMethod.LAYERWISE # pylint: disable=no-member - log.quiet(quiet, f'Quantization: type=layerwise module={module} cls={cls} storage={storage_dtype} compute={devices.dtype} blocking={not non_blocking}') + log.quiet(quiet, f'Quantization: type=layerwise module={module} cls={cls} storage={storage_dtype} compute={devices.dtype} blocking={not shared.opts.diffusers_offload_nonblocking}') except Exception as e: if 'Hook with name' not in str(e): log.error(f'Quantization: type=layerwise {e}') diff --git a/modules/sd_hijack_accelerate.py b/modules/sd_hijack_accelerate.py index 7f312a029..2ff355769 100644 --- a/modules/sd_hijack_accelerate.py +++ b/modules/sd_hijack_accelerate.py @@ -63,10 +63,10 @@ def hijack_set_module_tensor_simple( old_value = getattr(module, tensor_name) with devices.inference_context(): if tensor_name in module._buffers: # pylint: disable=protected-access - module._buffers[tensor_name] = value.to(device) # pylint: disable=protected-access + module._buffers[tensor_name] = value.to(device, non_blocking=shared.opts.diffusers_offload_nonblocking) # pylint: disable=protected-access elif value is not None or not devices.same_device(device, module._parameters[tensor_name].device): # pylint: disable=protected-access param_cls = type(module._parameters[tensor_name]) # pylint: disable=protected-access - module._parameters[tensor_name] = param_cls(value, requires_grad=old_value.requires_grad).to(device) # pylint: disable=protected-access + module._parameters[tensor_name] = param_cls(value, requires_grad=old_value.requires_grad).to(device, non_blocking=shared.opts.diffusers_offload_nonblocking) # pylint: disable=protected-access t1 = time.time() tensor_to_timer += (t1 - t0) diff --git a/modules/sd_offload.py b/modules/sd_offload.py index 6fe484f87..57fc93734 100644 --- a/modules/sd_offload.py +++ b/modules/sd_offload.py @@ -74,7 +74,7 @@ def apply_group_offload(sd_model, op:str='model'): 'offload_device': devices.cpu, 'offload_type': 'block_level', # 'leaf_level', 'num_blocks_per_group': 1, - 'non_blocking': False, + 'non_blocking': shared.opts.diffusers_offload_nonblocking, 'use_stream': False, 'record_stream': False, 'low_cpu_mem_usage': False, diff --git a/modules/shared.py b/modules/shared.py index 2036b2ffd..6fed4d343 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -165,6 +165,7 @@ options_templates.update(options_section(('offload', "Model Offloading"), { "offload_sep": OptionInfo("